get things working with new base

src/internal/m365/collection/drive/collections_tree_test.go

@@ -571,7 +571,7 @@ type populateTreeExpected struct {
 
 type populateTreeTest struct {
 	name       string
-	enumerator mock.EnumerateDriveItemsDelta
+	enumerator mock.EnumerateItemsDeltaByDrive
 	tree       *folderyMcFolderFace
 	limiter    *pagerLimiter
 	expect     populateTreeExpected
@@ -749,47 +749,48 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta(
 				},
 			},
 		},
-		{
+		// TODO: restore after mock.DriveEnumerator support lands.
-			name: "many folders with files across multiple deltas",
+		// {
-			tree: newFolderyMcFolderFace(nil, rootID),
+		// 	name: "many folders with files across multiple deltas",
-			enumerator: mock.DriveEnumerator(
+		// 	tree: newFolderyMcFolderFace(nil, rootID),
-				mock.Drive(id(drive)).With(
+		// 	enumerator: mock.DriveEnumerator(
-					mock.Delta(id(delta), nil).With(aPage(
+		// 		mock.Drive(id(drive)).With(
-						folderAtRoot(),
+		// 			mock.Delta(id(delta), nil).With(aPage(
-						fileAt(folder))),
+		// 				folderAtRoot(),
-					mock.Delta(id(delta), nil).With(aPage(
+		// 				fileAt(folder))),
-						folderxAtRoot("sib"),
+		// 			mock.Delta(id(delta), nil).With(aPage(
-						filexAt("fsib", "sib"))),
+		// 				folderxAtRoot("sib"),
-					mock.Delta(id(delta), nil).With(aPage(
+		// 				filexAt("fsib", "sib"))),
-						folderAtRoot(),
+		// 			mock.Delta(id(delta), nil).With(aPage(
-						folderxAt("chld", folder),
+		// 				folderAtRoot(),
-						filexAt("fchld", "chld"))),
+		// 				folderxAt("chld", folder),
-				)),
+		// 				filexAt("fchld", "chld"))),
-			limiter: newPagerLimiter(control.DefaultOptions()),
+		// 		)),
-			expect: populateTreeExpected{
+		// 	limiter: newPagerLimiter(control.DefaultOptions()),
-				counts: countTD.Expected{
+		// 	expect: populateTreeExpected{
-					count.TotalFoldersProcessed: 7,
+		// 		counts: countTD.Expected{
-					count.TotalFilesProcessed:   3,
+		// 			count.TotalFoldersProcessed: 7,
-					count.TotalPagesEnumerated:  4,
+		// 			count.TotalFilesProcessed:   3,
-				},
+		// 			count.TotalPagesEnumerated:  4,
-				err:            require.NoError,
+		// 		},
-				numLiveFiles:   3,
+		// 		err:            require.NoError,
-				numLiveFolders: 4,
+		// 		numLiveFiles:   3,
-				sizeBytes:      3 * 42,
+		// 		numLiveFolders: 4,
-				treeContainsFolderIDs: []string{
+		// 		sizeBytes:      3 * 42,
-					rootID,
+		// 		treeContainsFolderIDs: []string{
-					id(folder),
+		// 			rootID,
-					idx(folder, "sib"),
+		// 			id(folder),
-					idx(folder, "chld"),
+		// 			idx(folder, "sib"),
-				},
+		// 			idx(folder, "chld"),
-				treeContainsTombstoneIDs: []string{},
+		// 		},
-				treeContainsFileIDsWithParent: map[string]string{
+		// 		treeContainsTombstoneIDs: []string{},
-					id(file):           id(folder),
+		// 		treeContainsFileIDsWithParent: map[string]string{
-					idx(file, "fsib"):  idx(folder, "sib"),
+		// 			id(file):           id(folder),
-					idx(file, "fchld"): idx(folder, "chld"),
+		// 			idx(file, "fsib"):  idx(folder, "sib"),
-				},
+		// 			idx(file, "fchld"): idx(folder, "chld"),
-			},
+		// 		},
-		},
+		// 	},
+		// },
 		{
 			// technically you won't see this behavior from graph deltas, since deletes always
 			// precede creates/updates.  But it's worth checking that we can handle it anyways.
@@ -960,6 +961,15 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta(
 	}
 }
 
+// TODO: remove when unifying test tree structs
+type populateTreeTestMulti struct {
+	name       string
+	enumerator mock.EnumerateDriveItemsDelta
+	tree       *folderyMcFolderFace
+	limiter    *pagerLimiter
+	expect     populateTreeExpected
+}
+
 // this test focuses on quirks that can only arise from cases that occur across
 // multiple delta enumerations.
 // It is not concerned with unifying previous paths or post-processing collections.
@@ -968,22 +978,22 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
 	drv.SetId(ptr.To(id(drive)))
 	drv.SetName(ptr.To(name(drive)))
 
-	table := []populateTreeTest{
+	table := []populateTreeTestMulti{
 		{
 			name: "sanity case: normal enumeration split across multiple deltas",
 			tree: newFolderyMcFolderFace(nil, rootID),
 			enumerator: mock.DriveEnumerator(
 				mock.Drive(id(drive)).With(
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).With(pagesOf(pageItems(
-						folderAtRoot(),
+						driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
-						fileAt(folder))),
+						driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).With(pagesOf(pageItems(
-						folderxAtRoot("sib"),
+						driveItem(idx(folder, "sib"), namex(folder, "sib"), parentDir(), rootID, isFolder),
-						filexAt("fsib", "sib"))),
+						driveItem(idx(file, "fsib"), namex(file, "fsib"), parentDir(namex(folder, "sib")), idx(folder, "sib"), isFolder)))...),
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).With(pagesOf(pageItems(
-						folderAtRoot(),
+						driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
-						folderxAt("chld", folder),
+						driveItem(idx(folder, "chld"), namex(folder, "chld"), parentDir(name(folder)), id(folder), isFolder),
-						filexAt("fchld", "chld"))),
+						driveItem(idx(file, "fchld"), namex(file, "fchld"), parentDir(name(folder), namex(folder, "chld")), idx(folder, "chld"), isFolder)))...),
 				)),
 			limiter: newPagerLimiter(control.DefaultOptions()),
 			expect: populateTreeExpected{
@@ -1018,18 +1028,19 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
 			tree: newFolderyMcFolderFace(nil, rootID),
 			enumerator: mock.DriveEnumerator(
 				mock.Drive(id(drive)).With(
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).
-						folderAtRoot(),
+						With(pagesOf(pageItems(
-						fileAt(folder))),
+							driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
+							driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
 					// a (delete,create) pair in the same delta can occur when
 					// a user deletes and restores an item in-between deltas.
-					mock.Delta(id(delta), nil).With(
+					mock.Delta(id(delta), nil).
-						aPage(
+						With(pagesOf(pageItems(
-							delItem(id(folder), rootID, isFolder),
+							delItem(id(folder), parentDir(), rootID, isFolder),
-							delItem(id(file), id(folder), isFile)),
+							delItem(id(file), parentDir(), id(folder), isFile)))...).
-						aPage(
+						With(pagesOf(pageItems(
-							folderAtRoot(),
+							driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
-							fileAt(folder))),
+							driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
 				)),
 			limiter: newPagerLimiter(control.DefaultOptions()),
 			expect: populateTreeExpected{
@@ -1058,12 +1069,14 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
 			tree: newFolderyMcFolderFace(nil, rootID),
 			enumerator: mock.DriveEnumerator(
 				mock.Drive(id(drive)).With(
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).
-						folderAtRoot(),
+						With(pagesOf(pageItems(
-						fileAt(folder))),
+							driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
-					mock.Delta(id(delta), nil).With(aPage(
+							driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
-						driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder),
+					mock.Delta(id(delta), nil).
-						driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile))),
+						With(pagesOf(pageItems(
+							driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder),
+							driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile)))...),
 				)),
 			limiter: newPagerLimiter(control.DefaultOptions()),
 			expect: populateTreeExpected{
@@ -1096,22 +1109,23 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
 				mock.Drive(id(drive)).With(
 					mock.Delta(id(delta), nil).With(
 						// first page: create /root/folder and /root/folder/file
-						aPage(
+						pagesOf(pageItems(
-							folderAtRoot(),
+							driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
-							fileAt(folder)),
+							driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...).
 						// assume the user makes changes at this point:
 						// 1. delete /root/folder
 						// 2. create a new /root/folder
 						// 3. move /root/folder/file from old to new folder (same file ID)
 						// in drive deltas, this will show up as another folder creation sharing
 						// the same dirname, but we won't see the delete until...
-						aPage(
+						With(pagesOf(pageItems(
 							driveItem(idx(folder, 2), name(folder), parentDir(), rootID, isFolder),
-							driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile))),
+							driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile)))...),
 					// the next delta, containing the delete marker for the original /root/folder
-					mock.Delta(id(delta), nil).With(aPage(
+					mock.Delta(id(delta), nil).
-						delItem(id(folder), rootID, isFolder),
+						With(pagesOf(pageItems(
-					)),
+							delItem(id(folder), parentDir(), rootID, isFolder),
+						))...),
 				)),
 			limiter: newPagerLimiter(control.DefaultOptions()),
 			expect: populateTreeExpected{
@@ -1140,7 +1154,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
 	}
 	for _, test := range table {
 		suite.Run(test.name, func() {
-			runPopulateTreeTest(suite.T(), drv, test)
+			runPopulateTreeTestMulti(suite.T(), drv, test)
 		})
 	}
 }
@@ -1201,6 +1215,62 @@ func runPopulateTreeTest(
 	}
 }
 
+func runPopulateTreeTestMulti(
+	t *testing.T,
+	drv models.Driveable,
+	test populateTreeTestMulti,
+) {
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	mbh := mock.DefaultDriveBHWithMulti(user, pagerForDrives(drv), test.enumerator)
+	c := collWithMBH(mbh)
+	counter := count.New()
+
+	_, err := c.populateTree(
+		ctx,
+		test.tree,
+		drv,
+		id(delta),
+		test.limiter,
+		counter,
+		fault.New(true))
+
+	test.expect.err(t, err, clues.ToCore(err))
+
+	assert.Equal(
+		t,
+		test.expect.numLiveFolders,
+		test.tree.countLiveFolders(),
+		"count live folders in tree")
+
+	cAndS := test.tree.countLiveFilesAndSizes()
+	assert.Equal(
+		t,
+		test.expect.numLiveFiles,
+		cAndS.numFiles,
+		"count live files in tree")
+	assert.Equal(
+		t,
+		test.expect.sizeBytes,
+		cAndS.totalBytes,
+		"count total bytes in tree")
+	test.expect.counts.Compare(t, counter)
+
+	for _, id := range test.expect.treeContainsFolderIDs {
+		assert.NotNil(t, test.tree.folderIDToNode[id], "node exists")
+	}
+
+	for _, id := range test.expect.treeContainsTombstoneIDs {
+		assert.NotNil(t, test.tree.tombstones[id], "tombstone exists")
+	}
+
+	for iID, pID := range test.expect.treeContainsFileIDsWithParent {
+		assert.Contains(t, test.tree.fileIDToParentID, iID, "file should exist in tree")
+		assert.Equal(t, pID, test.tree.fileIDToParentID[iID], "file should reference correct parent")
+	}
+}
+
 // ---------------------------------------------------------------------------
 // folder tests
 // ---------------------------------------------------------------------------

src/internal/m365/service/onedrive/mock/handlers.go

@@ -2,6 +2,7 @@ package mock
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
 	"github.com/alcionai/clues"
@@ -31,7 +32,8 @@ type BackupHandler[T any] struct {
 	// and plug in the selector scope there.
 	Sel selectors.Selector
 
-	DriveItemEnumeration EnumerateItemsDeltaByDrive
+	DriveItemEnumeration      EnumerateItemsDeltaByDrive
+	DriveItemEnumerationMulti EnumerateDriveItemsDelta
 
 	GI  GetsItem
 	GIP GetsItemPermission
@@ -135,6 +137,18 @@ func DefaultDriveBHWith(
 	return mbh
 }
 
+func DefaultDriveBHWithMulti(
+	resource string,
+	drivePager *apiMock.Pager[models.Driveable],
+	enumerator EnumerateDriveItemsDelta,
+) *BackupHandler[models.DriveItemable] {
+	mbh := DefaultOneDriveBH(resource)
+	mbh.DrivePagerV = drivePager
+	mbh.DriveItemEnumerationMulti = enumerator
+
+	return mbh
+}
+
 func (h BackupHandler[T]) PathPrefix(tID, driveID string) (path.Path, error) {
 	pp, err := h.PathPrefixFn(tID, h.ProtectedResource.ID(), driveID)
 	if err != nil {
@@ -205,6 +219,14 @@ func (h BackupHandler[T]) EnumerateDriveItemsDelta(
 	driveID, prevDeltaLink string,
 	cc api.CallConfig,
 ) pagers.NextPageResulter[models.DriveItemable] {
+	if h.DriveItemEnumerationMulti.DrivePagers != nil {
+		return h.DriveItemEnumerationMulti.EnumerateDriveItemsDelta(
+			ctx,
+			driveID,
+			prevDeltaLink,
+			cc)
+	}
+
 	return h.DriveItemEnumeration.EnumerateDriveItemsDelta(
 		ctx,
 		driveID,
@@ -323,7 +345,124 @@ func (m GetsItem) GetItem(
 }
 
 // ---------------------------------------------------------------------------
-// Enumerates Drive Items
+// Drive Items Enumerator
+// ---------------------------------------------------------------------------
+
+type EnumerateDriveItemsDelta struct {
+	DrivePagers map[string]*DriveDeltaEnumerator
+}
+
+func DriveEnumerator(
+	ds ...*DriveDeltaEnumerator,
+) EnumerateDriveItemsDelta {
+	enumerator := EnumerateDriveItemsDelta{
+		DrivePagers: map[string]*DriveDeltaEnumerator{},
+	}
+
+	for _, drive := range ds {
+		enumerator.DrivePagers[drive.DriveID] = drive
+	}
+
+	return enumerator
+}
+
+func (en EnumerateDriveItemsDelta) EnumerateDriveItemsDelta(
+	_ context.Context,
+	driveID, _ string,
+	_ api.CallConfig,
+) pagers.NextPageResulter[models.DriveItemable] {
+	iterator := en.DrivePagers[driveID]
+	return iterator.nextDelta()
+}
+
+type DriveDeltaEnumerator struct {
+	DriveID      string
+	idx          int
+	DeltaQueries []*DeltaQuery
+}
+
+func Drive(driveID string) *DriveDeltaEnumerator {
+	return &DriveDeltaEnumerator{DriveID: driveID}
+}
+
+func (dde *DriveDeltaEnumerator) With(ds ...*DeltaQuery) *DriveDeltaEnumerator {
+	dde.DeltaQueries = ds
+	return dde
+}
+
+func (dde *DriveDeltaEnumerator) nextDelta() *DeltaQuery {
+	if dde.idx == len(dde.DeltaQueries) {
+		// at the end of the enumeration, return an empty page with no items,
+		// not even the root.  This is what graph api would do to signify an absence
+		// of changes in the delta.
+		lastDU := dde.DeltaQueries[dde.idx-1].DeltaUpdate
+
+		return &DeltaQuery{
+			DeltaUpdate: lastDU,
+			Pages: []NextPage{{
+				Items: []models.DriveItemable{},
+			}},
+		}
+	}
+
+	if dde.idx > len(dde.DeltaQueries) {
+		// a panic isn't optimal here, but since this mechanism is internal to testing,
+		// it's an acceptable way to have the tests ensure we don't over-enumerate deltas.
+		panic(fmt.Sprintf("delta index %d larger than count of delta iterations in mock", dde.idx))
+	}
+
+	pages := dde.DeltaQueries[dde.idx]
+
+	dde.idx++
+
+	return pages
+}
+
+var _ pagers.NextPageResulter[models.DriveItemable] = &DeltaQuery{}
+
+type DeltaQuery struct {
+	idx         int
+	Pages       []NextPage
+	DeltaUpdate pagers.DeltaUpdate
+	Err         error
+}
+
+func Delta(
+	resultDeltaID string,
+	err error,
+) *DeltaQuery {
+	return &DeltaQuery{
+		DeltaUpdate: pagers.DeltaUpdate{URL: resultDeltaID},
+		Err:         err,
+	}
+}
+
+func (dq *DeltaQuery) NextPage() ([]models.DriveItemable, bool, bool) {
+	if dq.idx >= len(dq.Pages) {
+		return nil, false, true
+	}
+
+	np := dq.Pages[dq.idx]
+	dq.idx = dq.idx + 1
+
+	return np.Items, np.Reset, false
+}
+
+func (dq *DeltaQuery) With(
+	pages ...NextPage,
+) *DeltaQuery {
+	dq.Pages = append(dq.Pages, pages...)
+	return dq
+}
+
+func (dq *DeltaQuery) Cancel() {}
+
+func (dq *DeltaQuery) Results() (pagers.DeltaUpdate, error) {
+	return dq.DeltaUpdate, dq.Err
+}
+
+// ---------------------------------------------------------------------------
+// old version - Enumerates Drive Items
 // ---------------------------------------------------------------------------
 
 type NextPage struct {