From 80feeace3b53c0fa4b149df4bf10bd2208f6d602 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 10 Nov 2023 14:45:27 -0700 Subject: [PATCH] ensure invalid deltas result in new state collections (#4623) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../m365/collection/drive/collections.go | 9 +- .../m365/collection/drive/collections_test.go | 114 +++++++++++++++++- 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index 7a1fa0327..8520e42b5 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -335,7 +335,7 @@ func (c *Collections) Get( logger.Ctx(ictx).Infow( "previous metadata for drive", - "num_paths_entries", len(oldPrevPaths)) + "count_old_prev_paths", len(oldPrevPaths)) du, newPrevPaths, err := c.PopulateDriveCollections( ctx, @@ -368,7 +368,7 @@ func (c *Collections) Get( logger.Ctx(ictx).Infow( "persisted metadata for drive", - "num_new_paths_entries", len(newPrevPaths), + "count_new_prev_paths", len(newPrevPaths), "delta_reset", du.Reset) numDriveItems := c.NumItems - numPrevItems @@ -762,6 +762,7 @@ func (c *Collections) PopulateDriveCollections( ctx = clues.Add(ctx, "delta_reset_occurred", true) newPrevPaths = map[string]string{} currPrevPaths = map[string]string{} + seenFolders = map[string]string{} c.CollectionMap[driveID] = map[string]*Collection{} invalidPrevDelta = true } @@ -929,6 +930,10 @@ func (c *Collections) processItem( delete(newPrevPaths, alreadyHandledFolderID) } + if invalidPrevDelta { + prevPath = nil + } + seenFolders[collectionPath.String()] = itemID col, err := NewCollection( diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index 0f0619089..f2e866c65 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -2403,6 +2403,118 @@ func (suite *CollectionsUnitSuite) TestGet() { fullPath(1, name(folder)): true, }, }, + { + name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtExistingLocation", + drives: []models.Driveable{drive1}, + enumerator: mock.EnumerateItemsDeltaByDrive{ + DrivePagers: map[string]*mock.DriveItemsDeltaPager{ + idx(drive, 1): { + Pages: []mock.NextPage{ + { + Items: []models.DriveItemable{ + driveRootItem(rootID), + driveItem(id(folder), name(folder), parent(1), rootID, isFolder), + driveItem(id(file), name(file), parent(1, name(folder)), id(folder), isFile), + }, + }, + { + Items: []models.DriveItemable{}, + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem(rootID), + driveItem(id(folder), name(folder), parent(1), rootID, isFolder), + driveItem(id(file), name(file), parent(1, name(folder)), id(folder), isFile), + }, + }, + }, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, + }, + }, + }, + canUsePreviousBackup: true, + errCheck: assert.NoError, + previousPaths: map[string]map[string]string{ + idx(drive, 1): { + rootID: fullPath(1), + id(folder): fullPath(1, name(folder)), + }, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + fullPath(1): {data.NewState: {}}, + fullPath(1, name(folder)): { + data.NewState: {id(folder), id(file)}, + }, + }, + expectedDeltaURLs: map[string]string{ + idx(drive, 1): delta, + }, + expectedPreviousPaths: map[string]map[string]string{ + idx(drive, 1): { + rootID: fullPath(1), + id(folder): fullPath(1, name(folder)), + }, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + fullPath(1): true, + fullPath(1, name(folder)): true, + }, + }, + { + name: "OneDrive_OneItemPage_ImmediateInvalidPrevDelta_MoveFolderToPreviouslyExistingPath", + drives: []models.Driveable{drive1}, + enumerator: mock.EnumerateItemsDeltaByDrive{ + DrivePagers: map[string]*mock.DriveItemsDeltaPager{ + idx(drive, 1): { + Pages: []mock.NextPage{ + { + Items: []models.DriveItemable{}, + Reset: true, + }, + { + Items: []models.DriveItemable{ + driveRootItem(rootID), + driveItem(idx(folder, 2), name(folder), parent(1), rootID, isFolder), + driveItem(idx(file, 2), name(file), parent(1, name(folder)), idx(folder, 2), isFile), + }, + }, + }, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, + }, + }, + }, + canUsePreviousBackup: true, + errCheck: assert.NoError, + previousPaths: map[string]map[string]string{ + idx(drive, 1): { + rootID: fullPath(1), + id(folder): fullPath(1, name(folder)), + }, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + fullPath(1): {data.NewState: {}}, + fullPath(1, name(folder)): { + data.DeletedState: {}, + data.NewState: {idx(folder, 2), idx(file, 2)}, + }, + }, + expectedDeltaURLs: map[string]string{ + idx(drive, 1): delta, + }, + expectedPreviousPaths: map[string]map[string]string{ + idx(drive, 1): { + rootID: fullPath(1), + idx(folder, 2): fullPath(1, name(folder)), + }, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + fullPath(1): true, + fullPath(1, name(folder)): true, + }, + }, { name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation", drives: []models.Driveable{drive1}, @@ -2555,7 +2667,7 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedCollections: map[string]map[data.CollectionState][]string{ fullPath(1): {data.NewState: {}}, - fullPath(1, name(folder)): {data.NotMovedState: {id(folder), id(file)}}, + fullPath(1, name(folder)): {data.NewState: {id(folder), id(file)}}, fullPath(1, namex(folder, 2)): {data.DeletedState: {}}, }, expectedDeltaURLs: map[string]string{