diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index 4b0d20084..44bcf9627 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strings" + "sync" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -272,13 +273,6 @@ func (c *Collections) Get( excludedItemIDs = map[string]struct{}{} oldPrevPaths = oldPrevPathsByDriveID[driveID] prevDeltaLink = prevDriveIDToDelta[driveID] - - // itemCollection is used to identify which collection a - // file belongs to. This is useful to delete a file from the - // collection it was previously in, in case it was moved to a - // different collection within the same delta query - // item ID -> item ID - itemCollection = map[string]string{} ) delete(driveTombstones, driveID) @@ -295,13 +289,16 @@ func (c *Collections) Get( "previous metadata for drive", "num_paths_entries", len(oldPrevPaths)) - items, du, err := c.handler.EnumerateDriveItemsDelta( - ictx, + du, newPrevPaths, err := c.PopulateDriveCollections( + ctx, driveID, + driveName, + oldPrevPaths, + excludedItemIDs, prevDeltaLink, - api.DefaultDriveItemProps()) + errs) if err != nil { - return nil, false, err + return nil, false, clues.Stack(err) } // It's alright to have an empty folders map (i.e. no folders found) but not @@ -313,20 +310,6 @@ func (c *Collections) Get( driveIDToDeltaLink[driveID] = du.URL } - newPrevPaths, err := c.UpdateCollections( - ctx, - driveID, - driveName, - items, - oldPrevPaths, - itemCollection, - excludedItemIDs, - du.Reset, - errs) - if err != nil { - return nil, false, clues.Stack(err) - } - // Avoid the edge case where there's no paths but we do have a valid delta // token. We can accomplish this by adding an empty paths map for this // drive. If we don't have this then the next backup won't use the delta @@ -688,224 +671,290 @@ func (c *Collections) getCollectionPath( return collectionPath, nil } -// UpdateCollections initializes and adds the provided drive items to Collections -// A new collection is created for every drive folder (or package). +// PopulateDriveCollections initializes and adds the provided drive items to Collections +// A new collection is created for every drive folder. // oldPrevPaths is the unchanged data that was loaded from the metadata file. // This map is not modified during the call. // currPrevPaths starts as a copy of oldPaths and is updated as changes are found in // the returned results. Items are added to this collection throughout the call. // newPrevPaths, ie: the items added during this call, get returned as a map. -func (c *Collections) UpdateCollections( +func (c *Collections) PopulateDriveCollections( ctx context.Context, driveID, driveName string, - items []models.DriveItemable, oldPrevPaths map[string]string, - currPrevPaths map[string]string, - excluded map[string]struct{}, - invalidPrevDelta bool, + excludedItemIDs map[string]struct{}, + prevDeltaLink string, errs *fault.Bus, -) (map[string]string, error) { +) (api.DeltaUpdate, map[string]string, error) { var ( - el = errs.Local() - newPrevPaths = map[string]string{} + el = errs.Local() + newPrevPaths = map[string]string{} + invalidPrevDelta = len(prevDeltaLink) == 0 + ch = make(chan api.NextPage[models.DriveItemable], 1) + wg = sync.WaitGroup{} + + // currPrevPaths is used to identify which collection a + // file belongs to. This is useful to delete a file from the + // collection it was previously in, in case it was moved to a + // different collection within the same delta query + // item ID -> item ID + currPrevPaths = map[string]string{} ) if !invalidPrevDelta { maps.Copy(newPrevPaths, oldPrevPaths) } - for _, item := range items { - if el.Failure() != nil { - break - } + go func() { + defer wg.Done() - var ( - itemID = ptr.Val(item.GetId()) - itemName = ptr.Val(item.GetName()) - isFolder = item.GetFolder() != nil || item.GetPackageEscaped() != nil - ictx = clues.Add( - ctx, - "item_id", itemID, - "item_name", clues.Hide(itemName), - "item_is_folder", isFolder) - ) - - if item.GetMalware() != nil { - addtl := graph.ItemInfo(item) - skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) - - if isFolder { - skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) + for pg := range ch { + if el.Failure() != nil { + // exhaust the channel to ensure it closes + continue } - errs.AddSkip(ctx, skip) - logger.Ctx(ctx).Infow("malware detected", "item_details", addtl) - - continue - } - - // Deleted file or folder. - if item.GetDeleted() != nil { - if err := c.handleDelete( - itemID, - driveID, - oldPrevPaths, - currPrevPaths, - newPrevPaths, - isFolder, - excluded, - invalidPrevDelta); err != nil { - return nil, clues.Stack(err).WithClues(ictx) + if pg.Reset { + newPrevPaths = map[string]string{} + currPrevPaths = map[string]string{} + c.CollectionMap[driveID] = map[string]*Collection{} + invalidPrevDelta = true } - continue - } + for _, item := range pg.Items { + if el.Failure() != nil { + continue + } - collectionPath, err := c.getCollectionPath(driveID, item) - if err != nil { - el.AddRecoverable(ctx, clues.Stack(err). - WithClues(ictx). - Label(fault.LabelForceNoBackupCreation)) - - continue - } - - // Skip items that don't match the folder selectors we were given. - if shouldSkip(ctx, collectionPath, c.handler, driveName) { - logger.Ctx(ictx).Debugw("path not selected", "skipped_path", collectionPath.String()) - continue - } - - switch { - case isFolder: - // Deletions are handled above so this is just moves/renames. - var prevPath path.Path - - prevPathStr, ok := oldPrevPaths[itemID] - if ok { - prevPath, err = path.FromDataLayerPath(prevPathStr, false) + err := c.processItem( + ctx, + item, + driveID, + driveName, + oldPrevPaths, + currPrevPaths, + newPrevPaths, + excludedItemIDs, + invalidPrevDelta, + el) if err != nil { - el.AddRecoverable(ctx, clues.Wrap(err, "invalid previous path"). - WithClues(ictx). - With("prev_path_string", path.LoggableDir(prevPathStr))) - } - } else if item.GetRoot() != nil { - // Root doesn't move or get renamed. - prevPath = collectionPath - } - - // Moved folders don't cause delta results for any subfolders nested in - // them. We need to go through and update paths to handle that. We only - // update newPaths so we don't accidentally clobber previous deletes. - updatePath(newPrevPaths, itemID, collectionPath.String()) - - found, err := updateCollectionPaths(driveID, itemID, c.CollectionMap, collectionPath) - if err != nil { - return nil, clues.Stack(err).WithClues(ictx) - } - - if found { - continue - } - - colScope := CollectionScopeFolder - if item.GetPackageEscaped() != nil { - colScope = CollectionScopePackage - } - - col, err := NewCollection( - c.handler, - c.protectedResource, - collectionPath, - prevPath, - driveID, - c.statusUpdater, - c.ctrl, - colScope, - invalidPrevDelta, - nil) - if err != nil { - return nil, clues.Stack(err).WithClues(ictx) - } - - col.driveName = driveName - - c.CollectionMap[driveID][itemID] = col - c.NumContainers++ - - if item.GetRoot() != nil { - continue - } - - // Add an entry to fetch permissions into this collection. This assumes - // that OneDrive always returns all folders on the path of an item - // before the item. This seems to hold true for now at least. - if col.Add(item) { - c.NumItems++ - } - - case item.GetFile() != nil: - // Deletions are handled above so this is just moves/renames. - if len(ptr.Val(item.GetParentReference().GetId())) == 0 { - return nil, clues.New("file without parent ID").WithClues(ictx) - } - - // Get the collection for this item. - parentID := ptr.Val(item.GetParentReference().GetId()) - ictx = clues.Add(ictx, "parent_id", parentID) - - collection, ok := c.CollectionMap[driveID][parentID] - if !ok { - return nil, clues.New("item seen before parent folder").WithClues(ictx) - } - - // This will only kick in if the file was moved multiple times - // within a single delta query. We delete the file from the previous - // collection so that it doesn't appear in two places. - prevParentContainerID, ok := currPrevPaths[itemID] - if ok { - prevColl, found := c.CollectionMap[driveID][prevParentContainerID] - if !found { - return nil, clues.New("previous collection not found"). - With("prev_parent_container_id", prevParentContainerID). - WithClues(ictx) - } - - if ok := prevColl.Remove(itemID); !ok { - return nil, clues.New("removing item from prev collection"). - With("prev_parent_container_id", prevParentContainerID). - WithClues(ictx) + el.AddRecoverable(ctx, clues.Stack(err)) } } - - currPrevPaths[itemID] = parentID - - if collection.Add(item) { - c.NumItems++ - c.NumFiles++ - } - - // Do this after adding the file to the collection so if we fail to add - // the item to the collection for some reason and we're using best effort - // we don't just end up deleting the item in the resulting backup. The - // resulting backup will be slightly incorrect, but it will have the most - // data that we were able to preserve. - if !invalidPrevDelta { - // Always add a file to the excluded list. The file may have been - // renamed/moved/modified, so we still have to drop the - // original one and download a fresh copy. - excluded[itemID+metadata.DataFileSuffix] = struct{}{} - excluded[itemID+metadata.MetaFileSuffix] = struct{}{} - } - - default: - el.AddRecoverable(ictx, clues.New("item is neither folder nor file"). - WithClues(ictx). - Label(fault.LabelForceNoBackupCreation)) } + }() + + wg.Add(1) + + du, err := c.handler.EnumerateDriveItemsDelta( + ctx, + ch, + driveID, + prevDeltaLink) + if err != nil { + return du, nil, clues.Stack(err) } - return newPrevPaths, el.Failure() + wg.Wait() + + return du, newPrevPaths, el.Failure() +} + +func (c *Collections) processItem( + ctx context.Context, + item models.DriveItemable, + driveID, driveName string, + oldPrevPaths, currPrevPaths, newPrevPaths map[string]string, + excluded map[string]struct{}, + invalidPrevDelta bool, + skipper fault.AddSkipper, +) error { + var ( + itemID = ptr.Val(item.GetId()) + itemName = ptr.Val(item.GetName()) + isFolder = item.GetFolder() != nil || item.GetPackageEscaped() != nil + ictx = clues.Add( + ctx, + "item_id", itemID, + "item_name", clues.Hide(itemName), + "item_is_folder", isFolder) + ) + + if item.GetMalware() != nil { + addtl := graph.ItemInfo(item) + skip := fault.FileSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) + + if isFolder { + skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl) + } + + skipper.AddSkip(ctx, skip) + logger.Ctx(ctx).Infow("malware detected", "item_details", addtl) + + return nil + } + + // Deleted file or folder. + if item.GetDeleted() != nil { + err := c.handleDelete( + itemID, + driveID, + oldPrevPaths, + currPrevPaths, + newPrevPaths, + isFolder, + excluded, + invalidPrevDelta) + + return clues.Stack(err).WithClues(ictx).OrNil() + } + + collectionPath, err := c.getCollectionPath(driveID, item) + if err != nil { + return clues.Stack(err). + WithClues(ictx). + Label(fault.LabelForceNoBackupCreation) + } + + // Skip items that don't match the folder selectors we were given. + if shouldSkip(ctx, collectionPath, c.handler, driveName) { + logger.Ctx(ictx).Debugw("path not selected", "skipped_path", collectionPath.String()) + return nil + } + + switch { + case isFolder: + // Deletions are handled above so this is just moves/renames. + var prevPath path.Path + + prevPathStr, ok := oldPrevPaths[itemID] + if ok { + prevPath, err = path.FromDataLayerPath(prevPathStr, false) + if err != nil { + return clues.Wrap(err, "invalid previous path"). + WithClues(ictx). + With("prev_path_string", path.LoggableDir(prevPathStr)) + } + } else if item.GetRoot() != nil { + // Root doesn't move or get renamed. + prevPath = collectionPath + } + + // Moved folders don't cause delta results for any subfolders nested in + // them. We need to go through and update paths to handle that. We only + // update newPaths so we don't accidentally clobber previous deletes. + updatePath(newPrevPaths, itemID, collectionPath.String()) + + found, err := updateCollectionPaths( + driveID, + itemID, + c.CollectionMap, + collectionPath) + if err != nil { + return clues.Stack(err).WithClues(ictx) + } + + if found { + return nil + } + + colScope := CollectionScopeFolder + if item.GetPackageEscaped() != nil { + colScope = CollectionScopePackage + } + + col, err := NewCollection( + c.handler, + c.protectedResource, + collectionPath, + prevPath, + driveID, + c.statusUpdater, + c.ctrl, + colScope, + invalidPrevDelta, + nil) + if err != nil { + return clues.Stack(err).WithClues(ictx) + } + + col.driveName = driveName + + c.CollectionMap[driveID][itemID] = col + c.NumContainers++ + + if item.GetRoot() != nil { + return nil + } + + // Add an entry to fetch permissions into this collection. This assumes + // that OneDrive always returns all folders on the path of an item + // before the item. This seems to hold true for now at least. + if col.Add(item) { + c.NumItems++ + } + + case item.GetFile() != nil: + // Deletions are handled above so this is just moves/renames. + if len(ptr.Val(item.GetParentReference().GetId())) == 0 { + return clues.New("file without parent ID").WithClues(ictx) + } + + // Get the collection for this item. + parentID := ptr.Val(item.GetParentReference().GetId()) + ictx = clues.Add(ictx, "parent_id", parentID) + + collection, ok := c.CollectionMap[driveID][parentID] + if !ok { + return clues.New("item seen before parent folder").WithClues(ictx) + } + + // This will only kick in if the file was moved multiple times + // within a single delta query. We delete the file from the previous + // collection so that it doesn't appear in two places. + prevParentContainerID, ok := currPrevPaths[itemID] + if ok { + prevColl, found := c.CollectionMap[driveID][prevParentContainerID] + if !found { + return clues.New("previous collection not found"). + With("prev_parent_container_id", prevParentContainerID). + WithClues(ictx) + } + + if ok := prevColl.Remove(itemID); !ok { + return clues.New("removing item from prev collection"). + With("prev_parent_container_id", prevParentContainerID). + WithClues(ictx) + } + } + + currPrevPaths[itemID] = parentID + + if collection.Add(item) { + c.NumItems++ + c.NumFiles++ + } + + // Do this after adding the file to the collection so if we fail to add + // the item to the collection for some reason and we're using best effort + // we don't just end up deleting the item in the resulting backup. The + // resulting backup will be slightly incorrect, but it will have the most + // data that we were able to preserve. + if !invalidPrevDelta { + // Always add a file to the excluded list. The file may have been + // renamed/moved/modified, so we still have to drop the + // original one and download a fresh copy. + excluded[itemID+metadata.DataFileSuffix] = struct{}{} + excluded[itemID+metadata.MetaFileSuffix] = struct{}{} + } + + default: + return clues.New("item is neither folder nor file"). + WithClues(ictx). + Label(fault.LabelForceNoBackupCreation) + } + + return nil } type dirScopeChecker interface { @@ -913,7 +962,12 @@ type dirScopeChecker interface { IncludesDir(dir string) bool } -func shouldSkip(ctx context.Context, drivePath path.Path, dsc dirScopeChecker, driveName string) bool { +func shouldSkip( + ctx context.Context, + drivePath path.Path, + dsc dirScopeChecker, + driveName string, +) bool { return !includePath(ctx, dsc, drivePath) || (drivePath.Category() == path.LibrariesCategory && restrictedDirectory == driveName) } diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index bae8019a8..d2b061f65 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -119,7 +119,7 @@ func getDelList(files ...string) map[string]struct{} { return delList } -func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() { +func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any())[0] const ( @@ -690,8 +690,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() { expectedItemCount: 0, expectedFileCount: 0, expectedContainerCount: 1, - expectedPrevPaths: nil, - expectedExcludes: map[string]struct{}{}, + expectedPrevPaths: map[string]string{ + "root": expectedPath(""), + }, + expectedExcludes: map[string]struct{}{}, }, { name: "1 root file, 1 folder, 1 package, 1 good file, 1 malware", @@ -732,15 +734,31 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() { defer flush() var ( - excludes = map[string]struct{}{} - currPrevPaths = map[string]string{} - errs = fault.New(true) + mbh = mock.DefaultOneDriveBH(user) + du = api.DeltaUpdate{ + URL: "notempty", + Reset: false, + } + excludes = map[string]struct{}{} + errs = fault.New(true) ) - maps.Copy(currPrevPaths, test.inputFolderMap) + mbh.DriveItemEnumeration = mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID: { + {Items: test.items}, + }, + }, + DeltaUpdate: map[string]api.DeltaUpdate{driveID: du}, + } + + sel := selectors.NewOneDriveBackup([]string{user}) + sel.Include([]selectors.OneDriveScope{test.scope}) + + mbh.Sel = sel.Selector c := NewCollections( - &itemBackupHandler{api.Drives{}, user, test.scope}, + mbh, tenant, idname.NewProvider(user, user), nil, @@ -748,18 +766,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() { c.CollectionMap[driveID] = map[string]*Collection{} - newPrevPaths, err := c.UpdateCollections( + _, newPrevPaths, err := c.PopulateDriveCollections( ctx, driveID, "General", - test.items, test.inputFolderMap, - currPrevPaths, excludes, - false, + "smarf", errs) test.expect(t, err, clues.ToCore(err)) - assert.Equal(t, len(test.expectedCollectionIDs), len(c.CollectionMap[driveID]), "total collections") + assert.ElementsMatch( + t, + maps.Keys(test.expectedCollectionIDs), + maps.Keys(c.CollectionMap[driveID])) assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count") assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count") @@ -1166,7 +1185,6 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { tenant = "a-tenant" user = "a-user" empty = "" - next = "next" delta = "delta1" delta2 = "delta2" ) @@ -1208,7 +1226,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { table := []struct { name string drives []models.Driveable - items map[string][]apiMock.PagerResult[models.DriveItemable] + enumerator mock.EnumeratesDriveItemsDelta[models.DriveItemable] canUsePreviousBackup bool errCheck assert.ErrorAssertionFunc prevFolderPaths map[string]map[string]string @@ -1227,16 +1245,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_DelFileOnly_NoFolders_NoErrors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), // will be present, not needed delItem("file", driveBasePath1, "root", true, false, false), - }, - DeltaLink: &delta, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1259,16 +1280,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_NoFolderDeltas_NoErrors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("file", "file", driveBasePath1, "root", true, false, false), - }, - DeltaLink: &delta, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1291,18 +1315,20 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_NoErrors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1329,19 +1355,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_NoErrors_FileRenamedMultiple", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), driveItem("file", "file2", driveBasePath1+"/folder", "folder", true, false, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1368,18 +1396,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_NoErrors_FileMovedMultiple", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), driveItem("file", "file2", driveBasePath1, "root", true, false, false), - }, - DeltaLink: &delta, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1408,18 +1439,20 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_EmptyDelta_NoErrors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - }, - DeltaLink: &empty, // probably will never happen with graph - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: empty, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1446,27 +1479,150 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_TwoItemPages_NoErrors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - NextLink: &next, - ResetDelta: true, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, + }, + canUsePreviousBackup: true, + errCheck: assert.NoError, + prevFolderPaths: map[string]map[string]string{ + driveID1: {}, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + rootFolderPath1: {data.NewState: {}}, + folderPath1: {data.NewState: {"folder", "file", "file2"}}, + }, + expectedDeltaURLs: map[string]string{ + driveID1: delta, + }, + expectedFolderPaths: map[string]map[string]string{ + driveID1: { + "root": rootFolderPath1, + "folder": folderPath1, + }, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, + }, + { + name: "OneDrive_TwoItemPages_WithReset", + drives: []models.Driveable{drive1}, + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, + }, + { + Items: []models.DriveItemable{}, + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), + }, + }, + }, + }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, + }, + canUsePreviousBackup: true, + errCheck: assert.NoError, + prevFolderPaths: map[string]map[string]string{ + driveID1: {}, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + rootFolderPath1: {data.NewState: {}}, + folderPath1: {data.NewState: {"folder", "file", "file2"}}, + }, + expectedDeltaURLs: map[string]string{ + driveID1: delta, + }, + expectedFolderPaths: map[string]map[string]string{ + driveID1: { + "root": rootFolderPath1, + "folder": folderPath1, + }, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, + }, + { + name: "OneDrive_TwoItemPages_WithResetCombinedWithItems", + drives: []models.Driveable{drive1}, + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), + }, + }, + }, + }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1498,29 +1654,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { drive1, drive2, }, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, - }, - driveID2: { - { - Values: []models.DriveItemable{ + driveID2: { + {Items: []models.DriveItemable{ driveRootItem("root2"), driveItem("folder2", "folder", driveBasePath2, "root2", false, true, false), driveItem("file2", "file", driveBasePath2+"/folder", "folder2", true, false, false), - }, - DeltaLink: &delta2, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + driveID2: {URL: delta2, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1562,29 +1717,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { drive1, drive2, }, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, - }, - driveID2: { - { - Values: []models.DriveItemable{ + driveID2: { + {Items: []models.DriveItemable{ driveRootItem("root"), driveItem("folder", "folder", driveBasePath2, "root", false, true, false), driveItem("file2", "file", driveBasePath2+"/folder", "folder", true, false, false), - }, - DeltaLink: &delta2, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + driveID2: {URL: delta2, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1623,12 +1777,16 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_Errors", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Err: assert.AnError, + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{}}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {}, + }, + Err: map[string]error{driveID1: assert.AnError}, }, canUsePreviousBackup: false, errCheck: assert.Error, @@ -1641,67 +1799,81 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedDelList: nil, }, { - name: "OneDrive_TwoItemPage_NoDeltaError", + name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("file", "file", driveBasePath1, "root", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{}, + Reset: true, }, - NextLink: &next, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file2", "file", driveBasePath1+"/folder", "folder", true, false, false), + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false), + }, }, - DeltaLink: &delta, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, prevFolderPaths: map[string]map[string]string{ driveID1: { - "root": rootFolderPath1, + "root": rootFolderPath1, + "folder": folderPath1, }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - rootFolderPath1: {data.NotMovedState: {"file"}}, - expectedPath1("/folder"): {data.NewState: {"folder", "file2"}}, + rootFolderPath1: {data.NewState: {}}, + expectedPath1("/folder"): {data.DeletedState: {}}, + expectedPath1("/folder2"): {data.NewState: {"folder2", "file"}}, }, expectedDeltaURLs: map[string]string{ driveID1: delta, }, expectedFolderPaths: map[string]map[string]string{ driveID1: { - "root": rootFolderPath1, - "folder": folderPath1, + "root": rootFolderPath1, + "folder2": expectedPath1("/folder2"), }, }, - expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - rootFolderPath1: getDelList("file", "file2"), - }), - doNotMergeItems: map[string]bool{}, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + expectedPath1("/folder2"): true, + }, }, { - name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder", + name: "OneDrive_OneItemPage_InvalidPrevDeltaCombinedWithItems_DeleteNonExistentFolder", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{}, + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false), + }, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1735,18 +1907,37 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder2", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + // on the first page, if this is the total data, we'd expect both folder and folder2 + // since new previousPaths merge with the old previousPaths. + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder2", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false), + }, + }, + { + Items: []models.DriveItemable{}, + Reset: true, + }, + { + // but after a delta reset, we treat this as the total end set of folders, which means + // we don't expect folder to exist any longer. + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder2", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false), + }, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1783,28 +1974,31 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "OneDrive Two Item Pages with Malware", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - malwareItem("malware", "malware", driveBasePath1+"/folder", "folder", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + malwareItem("malware", "malware", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - NextLink: &next, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), - malwareItem("malware2", "malware2", driveBasePath1+"/folder", "folder", true, false, false), + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false), + malwareItem("malware2", "malware2", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1832,30 +2026,38 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedSkippedCount: 2, }, { - name: "One Drive Deleted Folder In New Results", + name: "One Drive Deleted Folder In New Results With Invalid Delta", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), - driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false), - driveItem("file2", "file2", driveBasePath1+"/folder2", "folder2", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false), + driveItem("file2", "file2", driveBasePath1+"/folder2", "folder2", true, false, false), + }, }, - NextLink: &next, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - delItem("folder2", driveBasePath1, "root", false, true, false), - delItem("file2", driveBasePath1, "root", true, false, false), + { + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + delItem("folder2", driveBasePath1, "root", false, true, false), + delItem("file2", driveBasePath1, "root", true, false, false), + }, }, - DeltaLink: &delta2, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta2, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1888,19 +2090,24 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, { - name: "One Drive Random Folder Delete", + name: "One Drive Folder Delete After Invalid Delta", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - delItem("folder", driveBasePath1, "root", false, true, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + delItem("folder", driveBasePath1, "root", false, true, false), + }, + Reset: true, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1929,19 +2136,24 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, { - name: "One Drive Random Item Delete", + name: "One Drive Item Delete After Invalid Delta", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - delItem("file", driveBasePath1, "root", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + delItem("file", driveBasePath1, "root", true, false, false), + }, + Reset: true, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -1969,26 +2181,29 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "One Drive Folder Made And Deleted", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - NextLink: &next, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - delItem("folder", driveBasePath1, "root", false, true, false), - delItem("file", driveBasePath1, "root", true, false, false), + { + Items: []models.DriveItemable{ + driveRootItem("root"), + delItem("folder", driveBasePath1, "root", false, true, false), + delItem("file", driveBasePath1, "root", true, false, false), + }, }, - DeltaLink: &delta2, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta2, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -2014,25 +2229,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "One Drive Item Made And Deleted", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ - driveRootItem("root"), - driveItem("folder", "folder", driveBasePath1, "root", false, true, false), - driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + { + Items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("folder", "folder", driveBasePath1, "root", false, true, false), + driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), + }, }, - NextLink: &next, - }, - { - Values: []models.DriveItemable{ - driveRootItem("root"), - delItem("file", driveBasePath1, "root", true, false, false), + { + Items: []models.DriveItemable{ + driveRootItem("root"), + delItem("file", driveBasePath1, "root", true, false, false), + }, }, - DeltaLink: &delta, - ResetDelta: true, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -2061,17 +2279,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "One Drive Random Folder Delete", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), delItem("folder", driveBasePath1, "root", false, true, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -2097,17 +2317,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "One Drive Random Item Delete", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), delItem("file", driveBasePath1, "root", true, false, false), - }, - DeltaLink: &delta, - ResetDelta: true, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta, Reset: true}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -2133,15 +2355,18 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { { name: "TwoPriorDrives_OneTombstoned", drives: []models.Driveable{drive1}, - items: map[string][]apiMock.PagerResult[models.DriveItemable]{ - driveID1: { - { - Values: []models.DriveItemable{ + enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{ + Pages: map[string][]api.NextPage[models.DriveItemable]{ + driveID1: { + {Items: []models.DriveItemable{ driveRootItem("root"), // will be present - }, - DeltaLink: &delta, + }}, }, }, + DeltaUpdate: map[string]api.DeltaUpdate{ + driveID1: {URL: delta}, + }, + Err: map[string]error{driveID1: nil}, }, canUsePreviousBackup: true, errCheck: assert.NoError, @@ -2176,18 +2401,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, } - itemPagers := map[string]api.DeltaPager[models.DriveItemable]{} - - for driveID := range test.items { - itemPagers[driveID] = &apiMock.DeltaPager[models.DriveItemable]{ - ToReturn: test.items[driveID], - } - } - mbh := mock.DefaultOneDriveBH("a-user") mbh.DrivePagerV = mockDrivePager - mbh.ItemPagerV = itemPagers - mbh.DriveItemEnumeration = mock.PagerResultToEDID(test.items) + mbh.DriveItemEnumeration = test.enumerator c := NewCollections( mbh, @@ -2262,10 +2478,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { collectionCount++ - // TODO(ashmrtn): We should really be getting items in the collection - // via the Items() channel, but we don't have a way to mock out the - // actual item fetch yet (mostly wiring issues). The lack of that makes - // this check a bit more bittle since internal details can change. + // TODO: We should really be getting items in the collection + // via the Items() channel. The lack of that makes this check a bit more + // bittle since internal details can change. The wiring to support + // mocked GetItems is available. We just haven't plugged it in yet. col, ok := baseCol.(*Collection) require.True(t, ok, "getting onedrive.Collection handle") diff --git a/src/internal/m365/service/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go index 219fb6392..e122df1c8 100644 --- a/src/internal/m365/service/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -292,8 +292,8 @@ func (m GetsItem) GetItem( // Enumerates Drive Items // --------------------------------------------------------------------------- -type NextPage[T any] struct { - Items []T +type NextPage struct { + Items []models.DriveItemable Reset bool } @@ -305,7 +305,7 @@ var _ api.NextPageResulter[models.DriveItemable] = &DriveItemsDeltaPager{} type DriveItemsDeltaPager struct { Idx int - Pages []NextPage[models.DriveItemable] + Pages []NextPage DeltaUpdate api.DeltaUpdate Err error } diff --git a/src/internal/m365/service/sharepoint/backup_test.go b/src/internal/m365/service/sharepoint/backup_test.go index 6edcfd067..8a6dc6466 100644 --- a/src/internal/m365/service/sharepoint/backup_test.go +++ b/src/internal/m365/service/sharepoint/backup_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/m365/collection/drive" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/service/onedrive/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" @@ -90,16 +91,29 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { defer flush() var ( - paths = map[string]string{} - currPaths = map[string]string{} - excluded = map[string]struct{}{} - collMap = map[string]map[string]*drive.Collection{ + mbh = mock.DefaultSharePointBH(siteID) + du = api.DeltaUpdate{ + URL: "notempty", + Reset: false, + } + paths = map[string]string{} + excluded = map[string]struct{}{} + collMap = map[string]map[string]*drive.Collection{ driveID: {}, } ) + mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{ + DrivePagers: map[string]mock.DriveItemsDeltaPager{ + driveID: mock.DriveItemsDeltaPager{ + Pages: []mock.NextPage{{Items: test.items}}, + DeltaUpdate: du, + }, + }, + } + c := drive.NewCollections( - drive.NewLibraryBackupHandler(api.Drives{}, siteID, test.scope, path.SharePointService), + mbh, tenantID, idname.NewProvider(siteID, siteID), nil, @@ -107,15 +121,13 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { c.CollectionMap = collMap - _, err := c.UpdateCollections( + _, _, err := c.PopulateDriveCollections( ctx, driveID, "General", - test.items, paths, - currPaths, excluded, - true, + "", fault.New(true)) test.expect(t, err, clues.ToCore(err))