From 14639af017a49c0dcacc4c7b369c83b15b0d4847 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 28 Jun 2023 09:05:58 -0600 Subject: [PATCH] add the non-delta item pager to drives (#3639) introduces a non-delta item pager to drives_pager, and a func for building an item collision detection cache. Implementation is coming in the next PR. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../services/m365/api/contacts_pager_test.go | 2 +- src/pkg/services/m365/api/drive.go | 4 +- src/pkg/services/m365/api/drive_pager.go | 75 ++++++++++++++++ src/pkg/services/m365/api/drive_pager_test.go | 89 +++++++++++++++++++ .../services/m365/api/events_pager_test.go | 2 +- src/pkg/services/m365/api/helper_test.go | 9 ++ src/pkg/services/m365/api/mail_pager_test.go | 2 +- 7 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 src/pkg/services/m365/api/drive_pager_test.go diff --git a/src/pkg/services/m365/api/contacts_pager_test.go b/src/pkg/services/m365/api/contacts_pager_test.go index a61209636..71d28b219 100644 --- a/src/pkg/services/m365/api/contacts_pager_test.go +++ b/src/pkg/services/m365/api/contacts_pager_test.go @@ -30,7 +30,7 @@ func (suite *ContactsPagerIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *ContactsPagerIntgSuite) TestGetItemsInContainerByCollisionKey() { +func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsInContainerByCollisionKey() { t := suite.T() ac := suite.its.ac.Contacts() diff --git a/src/pkg/services/m365/api/drive.go b/src/pkg/services/m365/api/drive.go index c5fbfbe63..79dfd90a1 100644 --- a/src/pkg/services/m365/api/drive.go +++ b/src/pkg/services/m365/api/drive.go @@ -37,14 +37,14 @@ var ErrFolderNotFound = clues.New("folder not found") // GetFolderByName will lookup the specified folder by name within the parentFolderID folder. func (c Drives) GetFolderByName( ctx context.Context, - driveID, parentFolderID, folderID string, + driveID, parentFolderID, folderName string, ) (models.DriveItemable, error) { // The `Children().Get()` API doesn't yet support $filter, so using that to find a folder // will be sub-optimal. // Instead, we leverage OneDrive path-based addressing - // https://learn.microsoft.com/en-us/graph/onedrive-addressing-driveitems#path-based-addressing // - which allows us to lookup an item by its path relative to the parent ID - rawURL := fmt.Sprintf(itemByPathRawURLFmt, driveID, parentFolderID, folderID) + rawURL := fmt.Sprintf(itemByPathRawURLFmt, driveID, parentFolderID, folderName) builder := drives.NewItemItemsDriveItemItemRequestBuilder(rawURL, c.Stable.Adapter()) foundItem, err := builder.Get(ctx, nil) diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go index 697dbe8d2..df82feb44 100644 --- a/src/pkg/services/m365/api/drive_pager.go +++ b/src/pkg/services/m365/api/drive_pager.go @@ -17,6 +17,81 @@ import ( "github.com/alcionai/corso/src/pkg/logger" ) +// --------------------------------------------------------------------------- +// non-delta item pager +// --------------------------------------------------------------------------- + +var _ itemPager[models.DriveItemable] = &driveItemPageCtrl{} + +type driveItemPageCtrl struct { + gs graph.Servicer + builder *drives.ItemItemsItemChildrenRequestBuilder + options *drives.ItemItemsItemChildrenRequestBuilderGetRequestConfiguration +} + +func (c Drives) NewDriveItemPager( + driveID, containerID string, + selectProps ...string, +) itemPager[models.DriveItemable] { + options := &drives.ItemItemsItemChildrenRequestBuilderGetRequestConfiguration{ + QueryParameters: &drives.ItemItemsItemChildrenRequestBuilderGetQueryParameters{ + Top: ptr.To(maxNonDeltaPageSize), + }, + } + + if len(selectProps) > 0 { + options.QueryParameters.Select = selectProps + } + + builder := c.Stable. + Client(). + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(containerID). + Children() + + return &driveItemPageCtrl{c.Stable, builder, options} +} + +//lint:ignore U1000 False Positive +func (p *driveItemPageCtrl) getPage(ctx context.Context) (PageLinkValuer[models.DriveItemable], error) { + page, err := p.builder.Get(ctx, p.options) + if err != nil { + return nil, graph.Stack(ctx, err) + } + + return EmptyDeltaLinker[models.DriveItemable]{PageLinkValuer: page}, nil +} + +//lint:ignore U1000 False Positive +func (p *driveItemPageCtrl) setNext(nextLink string) { + p.builder = drives.NewItemItemsItemChildrenRequestBuilder(nextLink, p.gs.Adapter()) +} + +func (c Drives) GetItemsInContainerByCollisionKey( + ctx context.Context, + driveID, containerID string, +) (map[string]string, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...) + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating drive items") + } + + m := map[string]string{} + + for _, item := range items { + m[DriveItemCollisionKey(item)] = ptr.Val(item.GetId()) + } + + return m, nil +} + +// --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- // delta item pager // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/drive_pager_test.go b/src/pkg/services/m365/api/drive_pager_test.go new file mode 100644 index 000000000..e44968ab9 --- /dev/null +++ b/src/pkg/services/m365/api/drive_pager_test.go @@ -0,0 +1,89 @@ +package api_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type DrivePagerIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestDrivePagerIntgSuite(t *testing.T) { + suite.Run(t, &DrivePagerIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tester.M365AcctCredEnvs}), + }) +} + +func (suite *DrivePagerIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey() { + table := []struct { + name string + driveID string + rootFolderID string + }{ + { + name: "user drive", + driveID: suite.its.userDriveID, + rootFolderID: suite.its.userDriveRootFolderID, + }, + { + name: "site drive", + driveID: suite.its.siteDriveID, + rootFolderID: suite.its.siteDriveRootFolderID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + items, err := suite.its.ac.Stable. + Client(). + Drives(). + ByDriveId(test.driveID). + Items(). + ByDriveItemId(test.rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(err)) + + ims := items.GetValue() + expect := make([]string, 0, len(ims)) + + assert.NotEmptyf( + t, + ims, + "need at least one item to compare in user %s drive %s folder %s", + suite.its.userID, test.driveID, test.rootFolderID) + + results, err := suite.its.ac.Drives().GetItemsInContainerByCollisionKey(ctx, test.driveID, test.rootFolderID) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, results) + + for k, v := range results { + assert.NotEmpty(t, k, "all keys should be populated") + assert.NotEmpty(t, v, "all values should be populated") + } + + for _, e := range expect { + _, ok := results[e] + assert.Truef(t, ok, "expected results to contain collision key: %s", e) + } + }) + } +} diff --git a/src/pkg/services/m365/api/events_pager_test.go b/src/pkg/services/m365/api/events_pager_test.go index 477e91620..9ff7438e8 100644 --- a/src/pkg/services/m365/api/events_pager_test.go +++ b/src/pkg/services/m365/api/events_pager_test.go @@ -30,7 +30,7 @@ func (suite *EventsPagerIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *EventsPagerIntgSuite) TestGetItemsInContainerByCollisionKey() { +func (suite *EventsPagerIntgSuite) TestEvents_GetItemsInContainerByCollisionKey() { t := suite.T() ac := suite.its.ac.Events() diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 87db90526..8db0a9fd6 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -9,10 +9,12 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type intgTesterSetup struct { ac api.Client + gockAC api.Client userID string userDriveID string userDriveRootFolderID string @@ -34,6 +36,11 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.ac, err = api.NewClient(creds) require.NoError(t, err, clues.ToCore(err)) + its.gockAC, err = mock.NewClient(creds) + require.NoError(t, err, clues.ToCore(err)) + + // user drive + its.userID = tester.M365UserID(t) userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) @@ -48,6 +55,8 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.siteID = tester.M365SiteID(t) + // site + siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/pkg/services/m365/api/mail_pager_test.go b/src/pkg/services/m365/api/mail_pager_test.go index 11028cfff..d86c98123 100644 --- a/src/pkg/services/m365/api/mail_pager_test.go +++ b/src/pkg/services/m365/api/mail_pager_test.go @@ -30,7 +30,7 @@ func (suite *MailPagerIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *MailPagerIntgSuite) TestGetItemsInContainerByCollisionKey() { +func (suite *MailPagerIntgSuite) TestMail_GetItemsInContainerByCollisionKey() { t := suite.T() ac := suite.its.ac.Mail()