Allow delta enum callers to specify $select properties (#4460)

<!-- PR description-->

This fixes a perf regression in #4456 

Context: 
URL cache only needs a subset of drive item properties while doing delta queries. See https://github.com/alcionai/corso/pull/4074 for details.  Changes in #4456 were applying the default item property set for all delta enumerator consumers including URL cache. This PR fixes the memory regression. 

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2023-10-11 01:37:42 +05:30 committed by GitHub
parent 7196b5d278
commit 3656e04676
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 30 additions and 10 deletions

View File

@ -298,7 +298,8 @@ func (c *Collections) Get(
items, du, err := c.handler.EnumerateDriveItemsDelta( items, du, err := c.handler.EnumerateDriveItemsDelta(
ictx, ictx,
driveID, driveID,
prevDeltaLink) prevDeltaLink,
api.DefaultDriveItemProps())
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }

View File

@ -86,6 +86,7 @@ type EnumerateDriveItemsDeltaer interface {
EnumerateDriveItemsDelta( EnumerateDriveItemsDelta(
ctx context.Context, ctx context.Context,
driveID, prevDeltaLink string, driveID, prevDeltaLink string,
selectProps []string,
) ( ) (
[]models.DriveItemable, []models.DriveItemable,
api.DeltaUpdate, api.DeltaUpdate,

View File

@ -137,8 +137,9 @@ func (h itemBackupHandler) IncludesDir(dir string) bool {
func (h itemBackupHandler) EnumerateDriveItemsDelta( func (h itemBackupHandler) EnumerateDriveItemsDelta(
ctx context.Context, ctx context.Context,
driveID, prevDeltaLink string, driveID, prevDeltaLink string,
selectProps []string,
) ([]models.DriveItemable, api.DeltaUpdate, error) { ) ([]models.DriveItemable, api.DeltaUpdate, error) {
return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink) return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink, selectProps)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -140,8 +140,9 @@ func (h libraryBackupHandler) IncludesDir(dir string) bool {
func (h libraryBackupHandler) EnumerateDriveItemsDelta( func (h libraryBackupHandler) EnumerateDriveItemsDelta(
ctx context.Context, ctx context.Context,
driveID, prevDeltaLink string, driveID, prevDeltaLink string,
selectProps []string,
) ([]models.DriveItemable, api.DeltaUpdate, error) { ) ([]models.DriveItemable, api.DeltaUpdate, error) {
return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink) return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink, selectProps)
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/services/m365/api"
) )
const ( const (
@ -156,7 +157,11 @@ func (uc *urlCache) refreshCache(
// Issue a delta query to graph // Issue a delta query to graph
logger.Ctx(ctx).Info("refreshing url cache") logger.Ctx(ctx).Info("refreshing url cache")
items, du, err := uc.edid.EnumerateDriveItemsDelta(ctx, uc.driveID, uc.prevDelta) items, du, err := uc.edid.EnumerateDriveItemsDelta(
ctx,
uc.driveID,
uc.prevDelta,
api.URLCacheDriveItemProps())
if err != nil { if err != nil {
uc.idToProps = make(map[string]itemProps) uc.idToProps = make(map[string]itemProps)
return clues.Stack(err) return clues.Stack(err)

View File

@ -97,7 +97,11 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
nfid := ptr.Val(newFolder.GetId()) nfid := ptr.Val(newFolder.GetId())
// Get the previous delta to feed into url cache // Get the previous delta to feed into url cache
_, du, err := ac.EnumerateDriveItemsDelta(ctx, suite.driveID, "") _, du, err := ac.EnumerateDriveItemsDelta(
ctx,
suite.driveID,
"",
api.URLCacheDriveItemProps())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, du.URL) require.NotEmpty(t, du.URL)

View File

@ -163,8 +163,13 @@ func (h *BackupHandler) Get(context.Context, string, map[string]string) (*http.R
func (h BackupHandler) EnumerateDriveItemsDelta( func (h BackupHandler) EnumerateDriveItemsDelta(
ctx context.Context, ctx context.Context,
driveID, prevDeltaLink string, driveID, prevDeltaLink string,
selectProps []string,
) ([]models.DriveItemable, api.DeltaUpdate, error) { ) ([]models.DriveItemable, api.DeltaUpdate, error) {
return h.DriveItemEnumeration.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink) return h.DriveItemEnumeration.EnumerateDriveItemsDelta(
ctx,
driveID,
prevDeltaLink,
selectProps)
} }
func (h BackupHandler) GetItem(ctx context.Context, _, _ string) (models.DriveItemable, error) { func (h BackupHandler) GetItem(ctx context.Context, _, _ string) (models.DriveItemable, error) {
@ -282,6 +287,7 @@ type EnumeratesDriveItemsDelta struct {
func (edi EnumeratesDriveItemsDelta) EnumerateDriveItemsDelta( func (edi EnumeratesDriveItemsDelta) EnumerateDriveItemsDelta(
_ context.Context, _ context.Context,
driveID, _ string, driveID, _ string,
_ []string,
) ( ) (
[]models.DriveItemable, []models.DriveItemable,
api.DeltaUpdate, api.DeltaUpdate,

View File

@ -120,8 +120,8 @@ func DefaultDriveItemProps() []string {
"shared") "shared")
} }
// URL cache only needs a subset of item properties // URL cache only needs to fetch a small subset of item properties
func DriveItemSelectURLCache() []string { func URLCacheDriveItemProps() []string {
return idAnd( return idAnd(
"content.downloadUrl", "content.downloadUrl",
"deleted", "deleted",

View File

@ -203,12 +203,13 @@ func (c Drives) EnumerateDriveItemsDelta(
ctx context.Context, ctx context.Context,
driveID string, driveID string,
prevDeltaLink string, prevDeltaLink string,
selectProps []string,
) ( ) (
[]models.DriveItemable, []models.DriveItemable,
DeltaUpdate, DeltaUpdate,
error, error,
) { ) {
pager := c.newDriveItemDeltaPager(driveID, prevDeltaLink, DefaultDriveItemProps()...) pager := c.newDriveItemDeltaPager(driveID, prevDeltaLink, selectProps...)
items, du, err := deltaEnumerateItems[models.DriveItemable](ctx, pager, prevDeltaLink) items, du, err := deltaEnumerateItems[models.DriveItemable](ctx, pager, prevDeltaLink)
if err != nil { if err != nil {

View File

@ -188,7 +188,7 @@ func (suite *DrivePagerIntgSuite) TestEnumerateDriveItems() {
items, du, err := suite.its. items, du, err := suite.its.
ac. ac.
Drives(). Drives().
EnumerateDriveItemsDelta(ctx, suite.its.user.driveID, "") EnumerateDriveItemsDelta(ctx, suite.its.user.driveID, "", api.DefaultDriveItemProps())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, items, "no items found in user's drive") require.NotEmpty(t, items, "no items found in user's drive")
assert.NotEmpty(t, du.URL, "should have a delta link") assert.NotEmpty(t, du.URL, "should have a delta link")