diff --git a/src/internal/m365/collection/drive/collections_tree_test.go b/src/internal/m365/collection/drive/collections_tree_test.go index 06cbb7de9..d2f07aeab 100644 --- a/src/internal/m365/collection/drive/collections_tree_test.go +++ b/src/internal/m365/collection/drive/collections_tree_test.go @@ -571,7 +571,7 @@ type populateTreeExpected struct { type populateTreeTest struct { name string - enumerator mock.EnumerateDriveItemsDelta + enumerator mock.EnumerateItemsDeltaByDrive tree *folderyMcFolderFace limiter *pagerLimiter expect populateTreeExpected @@ -749,47 +749,48 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( }, }, }, - { - name: "many folders with files across multiple deltas", - tree: newFolderyMcFolderFace(nil, rootID), - enumerator: mock.DriveEnumerator( - mock.Drive(id(drive)).With( - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - fileAt(folder))), - mock.Delta(id(delta), nil).With(aPage( - folderxAtRoot("sib"), - filexAt("fsib", "sib"))), - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - folderxAt("chld", folder), - filexAt("fchld", "chld"))), - )), - limiter: newPagerLimiter(control.DefaultOptions()), - expect: populateTreeExpected{ - counts: countTD.Expected{ - count.TotalFoldersProcessed: 7, - count.TotalFilesProcessed: 3, - count.TotalPagesEnumerated: 4, - }, - err: require.NoError, - numLiveFiles: 3, - numLiveFolders: 4, - sizeBytes: 3 * 42, - treeContainsFolderIDs: []string{ - rootID, - id(folder), - idx(folder, "sib"), - idx(folder, "chld"), - }, - treeContainsTombstoneIDs: []string{}, - treeContainsFileIDsWithParent: map[string]string{ - id(file): id(folder), - idx(file, "fsib"): idx(folder, "sib"), - idx(file, "fchld"): idx(folder, "chld"), - }, - }, - }, + // TODO: restore after mock.DriveEnumerator support lands. + // { + // name: "many folders with files across multiple deltas", + // tree: newFolderyMcFolderFace(nil, rootID), + // enumerator: mock.DriveEnumerator( + // mock.Drive(id(drive)).With( + // mock.Delta(id(delta), nil).With(aPage( + // folderAtRoot(), + // fileAt(folder))), + // mock.Delta(id(delta), nil).With(aPage( + // folderxAtRoot("sib"), + // filexAt("fsib", "sib"))), + // mock.Delta(id(delta), nil).With(aPage( + // folderAtRoot(), + // folderxAt("chld", folder), + // filexAt("fchld", "chld"))), + // )), + // limiter: newPagerLimiter(control.DefaultOptions()), + // expect: populateTreeExpected{ + // counts: countTD.Expected{ + // count.TotalFoldersProcessed: 7, + // count.TotalFilesProcessed: 3, + // count.TotalPagesEnumerated: 4, + // }, + // err: require.NoError, + // numLiveFiles: 3, + // numLiveFolders: 4, + // sizeBytes: 3 * 42, + // treeContainsFolderIDs: []string{ + // rootID, + // id(folder), + // idx(folder, "sib"), + // idx(folder, "chld"), + // }, + // treeContainsTombstoneIDs: []string{}, + // treeContainsFileIDsWithParent: map[string]string{ + // id(file): id(folder), + // idx(file, "fsib"): idx(folder, "sib"), + // idx(file, "fchld"): idx(folder, "chld"), + // }, + // }, + // }, { // technically you won't see this behavior from graph deltas, since deletes always // precede creates/updates. But it's worth checking that we can handle it anyways. @@ -960,6 +961,15 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( } } +// TODO: remove when unifying test tree structs +type populateTreeTestMulti struct { + name string + enumerator mock.EnumerateDriveItemsDelta + tree *folderyMcFolderFace + limiter *pagerLimiter + expect populateTreeExpected +} + // this test focuses on quirks that can only arise from cases that occur across // multiple delta enumerations. // It is not concerned with unifying previous paths or post-processing collections. @@ -968,22 +978,22 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() drv.SetId(ptr.To(id(drive))) drv.SetName(ptr.To(name(drive))) - table := []populateTreeTest{ + table := []populateTreeTestMulti{ { name: "sanity case: normal enumeration split across multiple deltas", tree: newFolderyMcFolderFace(nil, rootID), enumerator: mock.DriveEnumerator( mock.Drive(id(drive)).With( - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - fileAt(folder))), - mock.Delta(id(delta), nil).With(aPage( - folderxAtRoot("sib"), - filexAt("fsib", "sib"))), - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - folderxAt("chld", folder), - filexAt("fchld", "chld"))), + mock.Delta(id(delta), nil).With(pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...), + mock.Delta(id(delta), nil).With(pagesOf(pageItems( + driveItem(idx(folder, "sib"), namex(folder, "sib"), parentDir(), rootID, isFolder), + driveItem(idx(file, "fsib"), namex(file, "fsib"), parentDir(namex(folder, "sib")), idx(folder, "sib"), isFolder)))...), + mock.Delta(id(delta), nil).With(pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(idx(folder, "chld"), namex(folder, "chld"), parentDir(name(folder)), id(folder), isFolder), + driveItem(idx(file, "fchld"), namex(file, "fchld"), parentDir(name(folder), namex(folder, "chld")), idx(folder, "chld"), isFolder)))...), )), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1018,18 +1028,19 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() tree: newFolderyMcFolderFace(nil, rootID), enumerator: mock.DriveEnumerator( mock.Drive(id(drive)).With( - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - fileAt(folder))), + mock.Delta(id(delta), nil). + With(pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...), // a (delete,create) pair in the same delta can occur when // a user deletes and restores an item in-between deltas. - mock.Delta(id(delta), nil).With( - aPage( - delItem(id(folder), rootID, isFolder), - delItem(id(file), id(folder), isFile)), - aPage( - folderAtRoot(), - fileAt(folder))), + mock.Delta(id(delta), nil). + With(pagesOf(pageItems( + delItem(id(folder), parentDir(), rootID, isFolder), + delItem(id(file), parentDir(), id(folder), isFile)))...). + With(pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...), )), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1058,12 +1069,14 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() tree: newFolderyMcFolderFace(nil, rootID), enumerator: mock.DriveEnumerator( mock.Drive(id(drive)).With( - mock.Delta(id(delta), nil).With(aPage( - folderAtRoot(), - fileAt(folder))), - mock.Delta(id(delta), nil).With(aPage( - driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder), - driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile))), + mock.Delta(id(delta), nil). + With(pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...), + mock.Delta(id(delta), nil). + With(pagesOf(pageItems( + driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder), + driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile)))...), )), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1096,22 +1109,23 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() mock.Drive(id(drive)).With( mock.Delta(id(delta), nil).With( // first page: create /root/folder and /root/folder/file - aPage( - folderAtRoot(), - fileAt(folder)), + pagesOf(pageItems( + driveItem(id(folder), name(folder), parentDir(), rootID, isFolder), + driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...). // assume the user makes changes at this point: // 1. delete /root/folder // 2. create a new /root/folder // 3. move /root/folder/file from old to new folder (same file ID) // in drive deltas, this will show up as another folder creation sharing // the same dirname, but we won't see the delete until... - aPage( + With(pagesOf(pageItems( driveItem(idx(folder, 2), name(folder), parentDir(), rootID, isFolder), - driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile))), + driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile)))...), // the next delta, containing the delete marker for the original /root/folder - mock.Delta(id(delta), nil).With(aPage( - delItem(id(folder), rootID, isFolder), - )), + mock.Delta(id(delta), nil). + With(pagesOf(pageItems( + delItem(id(folder), parentDir(), rootID, isFolder), + ))...), )), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1140,7 +1154,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() } for _, test := range table { suite.Run(test.name, func() { - runPopulateTreeTest(suite.T(), drv, test) + runPopulateTreeTestMulti(suite.T(), drv, test) }) } } @@ -1201,6 +1215,62 @@ func runPopulateTreeTest( } } +func runPopulateTreeTestMulti( + t *testing.T, + drv models.Driveable, + test populateTreeTestMulti, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + mbh := mock.DefaultDriveBHWithMulti(user, pagerForDrives(drv), test.enumerator) + c := collWithMBH(mbh) + counter := count.New() + + _, err := c.populateTree( + ctx, + test.tree, + drv, + id(delta), + test.limiter, + counter, + fault.New(true)) + + test.expect.err(t, err, clues.ToCore(err)) + + assert.Equal( + t, + test.expect.numLiveFolders, + test.tree.countLiveFolders(), + "count live folders in tree") + + cAndS := test.tree.countLiveFilesAndSizes() + assert.Equal( + t, + test.expect.numLiveFiles, + cAndS.numFiles, + "count live files in tree") + assert.Equal( + t, + test.expect.sizeBytes, + cAndS.totalBytes, + "count total bytes in tree") + test.expect.counts.Compare(t, counter) + + for _, id := range test.expect.treeContainsFolderIDs { + assert.NotNil(t, test.tree.folderIDToNode[id], "node exists") + } + + for _, id := range test.expect.treeContainsTombstoneIDs { + assert.NotNil(t, test.tree.tombstones[id], "tombstone exists") + } + + for iID, pID := range test.expect.treeContainsFileIDsWithParent { + assert.Contains(t, test.tree.fileIDToParentID, iID, "file should exist in tree") + assert.Equal(t, pID, test.tree.fileIDToParentID[iID], "file should reference correct parent") + } +} + // --------------------------------------------------------------------------- // folder tests // --------------------------------------------------------------------------- diff --git a/src/internal/m365/service/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go index 157da4271..62b740c82 100644 --- a/src/internal/m365/service/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -2,6 +2,7 @@ package mock import ( "context" + "fmt" "net/http" "github.com/alcionai/clues" @@ -31,7 +32,8 @@ type BackupHandler[T any] struct { // and plug in the selector scope there. Sel selectors.Selector - DriveItemEnumeration EnumerateItemsDeltaByDrive + DriveItemEnumeration EnumerateItemsDeltaByDrive + DriveItemEnumerationMulti EnumerateDriveItemsDelta GI GetsItem GIP GetsItemPermission @@ -135,6 +137,18 @@ func DefaultDriveBHWith( return mbh } +func DefaultDriveBHWithMulti( + resource string, + drivePager *apiMock.Pager[models.Driveable], + enumerator EnumerateDriveItemsDelta, +) *BackupHandler[models.DriveItemable] { + mbh := DefaultOneDriveBH(resource) + mbh.DrivePagerV = drivePager + mbh.DriveItemEnumerationMulti = enumerator + + return mbh +} + func (h BackupHandler[T]) PathPrefix(tID, driveID string) (path.Path, error) { pp, err := h.PathPrefixFn(tID, h.ProtectedResource.ID(), driveID) if err != nil { @@ -205,6 +219,14 @@ func (h BackupHandler[T]) EnumerateDriveItemsDelta( driveID, prevDeltaLink string, cc api.CallConfig, ) pagers.NextPageResulter[models.DriveItemable] { + if h.DriveItemEnumerationMulti.DrivePagers != nil { + return h.DriveItemEnumerationMulti.EnumerateDriveItemsDelta( + ctx, + driveID, + prevDeltaLink, + cc) + } + return h.DriveItemEnumeration.EnumerateDriveItemsDelta( ctx, driveID, @@ -323,7 +345,124 @@ func (m GetsItem) GetItem( } // --------------------------------------------------------------------------- -// Enumerates Drive Items +// Drive Items Enumerator +// --------------------------------------------------------------------------- + +type EnumerateDriveItemsDelta struct { + DrivePagers map[string]*DriveDeltaEnumerator +} + +func DriveEnumerator( + ds ...*DriveDeltaEnumerator, +) EnumerateDriveItemsDelta { + enumerator := EnumerateDriveItemsDelta{ + DrivePagers: map[string]*DriveDeltaEnumerator{}, + } + + for _, drive := range ds { + enumerator.DrivePagers[drive.DriveID] = drive + } + + return enumerator +} + +func (en EnumerateDriveItemsDelta) EnumerateDriveItemsDelta( + _ context.Context, + driveID, _ string, + _ api.CallConfig, +) pagers.NextPageResulter[models.DriveItemable] { + iterator := en.DrivePagers[driveID] + return iterator.nextDelta() +} + +type DriveDeltaEnumerator struct { + DriveID string + idx int + DeltaQueries []*DeltaQuery +} + +func Drive(driveID string) *DriveDeltaEnumerator { + return &DriveDeltaEnumerator{DriveID: driveID} +} + +func (dde *DriveDeltaEnumerator) With(ds ...*DeltaQuery) *DriveDeltaEnumerator { + dde.DeltaQueries = ds + return dde +} + +func (dde *DriveDeltaEnumerator) nextDelta() *DeltaQuery { + if dde.idx == len(dde.DeltaQueries) { + // at the end of the enumeration, return an empty page with no items, + // not even the root. This is what graph api would do to signify an absence + // of changes in the delta. + lastDU := dde.DeltaQueries[dde.idx-1].DeltaUpdate + + return &DeltaQuery{ + DeltaUpdate: lastDU, + Pages: []NextPage{{ + Items: []models.DriveItemable{}, + }}, + } + } + + if dde.idx > len(dde.DeltaQueries) { + // a panic isn't optimal here, but since this mechanism is internal to testing, + // it's an acceptable way to have the tests ensure we don't over-enumerate deltas. + panic(fmt.Sprintf("delta index %d larger than count of delta iterations in mock", dde.idx)) + } + + pages := dde.DeltaQueries[dde.idx] + + dde.idx++ + + return pages +} + +var _ pagers.NextPageResulter[models.DriveItemable] = &DeltaQuery{} + +type DeltaQuery struct { + idx int + Pages []NextPage + DeltaUpdate pagers.DeltaUpdate + Err error +} + +func Delta( + resultDeltaID string, + err error, +) *DeltaQuery { + return &DeltaQuery{ + DeltaUpdate: pagers.DeltaUpdate{URL: resultDeltaID}, + Err: err, + } +} + +func (dq *DeltaQuery) NextPage() ([]models.DriveItemable, bool, bool) { + if dq.idx >= len(dq.Pages) { + return nil, false, true + } + + np := dq.Pages[dq.idx] + dq.idx = dq.idx + 1 + + return np.Items, np.Reset, false +} + +func (dq *DeltaQuery) With( + pages ...NextPage, +) *DeltaQuery { + dq.Pages = append(dq.Pages, pages...) + return dq +} + +func (dq *DeltaQuery) Cancel() {} + +func (dq *DeltaQuery) Results() (pagers.DeltaUpdate, error) { + return dq.DeltaUpdate, dq.Err +} + +// --------------------------------------------------------------------------- +// old version - Enumerates Drive Items // --------------------------------------------------------------------------- type NextPage struct {