diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index 379c012c8..5d999878a 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -62,7 +62,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "Invalid item", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(id(item), name(item), d.dir(), rootID, -1), }, previousPaths: map[string]string{}, @@ -70,11 +70,11 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.Error, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedExcludes: map[string]struct{}{}, expectedTopLevelPackages: map[string]struct{}{}, @@ -82,7 +82,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "Single File", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFile(d.dir(), rootID), }, previousPaths: map[string]string{}, @@ -90,14 +90,14 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedItemCount: 1, expectedFileCount: 1, expectedContainerCount: 1, // Root folder is skipped since it's always present. expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedExcludes: makeExcludeMap(fileID()), expectedTopLevelPackages: map[string]struct{}{}, @@ -105,7 +105,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "Single Folder", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), }, previousPaths: map[string]string{}, @@ -113,12 +113,12 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNew(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNew(t, d.strPath(t, folderName())), }, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, expectedItemCount: 1, expectedContainerCount: 2, @@ -128,7 +128,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "Single Folder created twice", // deleted a created with same name in between a backup items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), driveItem(folderID(2), folderName(), d.dir(), rootID, isFolder), }, @@ -137,12 +137,12 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(2): asNew(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(2): asNew(t, d.strPath(t, folderName())), }, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), }, expectedItemCount: 1, expectedContainerCount: 2, @@ -152,7 +152,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "Single Package", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(id(pkg), name(pkg), d.dir(), rootID, isPackage), }, previousPaths: map[string]string{}, @@ -160,25 +160,25 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - id(pkg): asNew(t, d.strPath(name(pkg))), + rootID: asNotMoved(t, d.strPath(t)), + id(pkg): asNew(t, d.strPath(t, name(pkg))), }, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - id(pkg): d.strPath(name(pkg)), + rootID: d.strPath(t), + id(pkg): d.strPath(t, name(pkg)), }, expectedItemCount: 1, expectedContainerCount: 2, expectedExcludes: map[string]struct{}{}, expectedTopLevelPackages: map[string]struct{}{ - d.strPath(name(pkg)): {}, + d.strPath(t, name(pkg)): {}, }, expectedCountPackages: 1, }, { name: "Single Package with subfolder", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(id(pkg), name(pkg), d.dir(), rootID, isPackage), driveItem(folderID(), folderName(), d.dir(name(pkg)), id(pkg), isFolder), driveItem(id(subfolder), name(subfolder), d.dir(name(pkg)), id(pkg), isFolder), @@ -188,29 +188,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - id(pkg): asNew(t, d.strPath(name(pkg))), - folderID(): asNew(t, d.strPath(name(pkg), folderName())), - id(subfolder): asNew(t, d.strPath(name(pkg), name(subfolder))), + rootID: asNotMoved(t, d.strPath(t)), + id(pkg): asNew(t, d.strPath(t, name(pkg))), + folderID(): asNew(t, d.strPath(t, name(pkg), folderName())), + id(subfolder): asNew(t, d.strPath(t, name(pkg), name(subfolder))), }, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - id(pkg): d.strPath(name(pkg)), - folderID(): d.strPath(name(pkg), folderName()), - id(subfolder): d.strPath(name(pkg), name(subfolder)), + rootID: d.strPath(t), + id(pkg): d.strPath(t, name(pkg)), + folderID(): d.strPath(t, name(pkg), folderName()), + id(subfolder): d.strPath(t, name(pkg), name(subfolder)), }, expectedItemCount: 3, expectedContainerCount: 4, expectedExcludes: map[string]struct{}{}, expectedTopLevelPackages: map[string]struct{}{ - d.strPath(name(pkg)): {}, + d.strPath(t, name(pkg)): {}, }, expectedCountPackages: 3, }, { name: "1 root file, 1 folder, 1 package, 2 files, 3 collections", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFile(d.dir(), rootID, "inRoot"), driveFolder(d.dir(), rootID), driveItem(id(pkg), name(pkg), d.dir(), rootID, isPackage), @@ -222,20 +222,20 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNew(t, d.strPath(folderName())), - id(pkg): asNew(t, d.strPath(name(pkg))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNew(t, d.strPath(t, folderName())), + id(pkg): asNew(t, d.strPath(t, name(pkg))), }, expectedItemCount: 5, expectedFileCount: 3, expectedContainerCount: 3, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(pkg): d.strPath(name(pkg)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(pkg): d.strPath(t, name(pkg)), }, expectedTopLevelPackages: map[string]struct{}{ - d.strPath(name(pkg)): {}, + d.strPath(t, name(pkg)): {}, }, expectedCountPackages: 1, expectedExcludes: makeExcludeMap(fileID("inRoot"), fileID("inFolder"), fileID("inPackage")), @@ -243,7 +243,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "contains folder selector", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFile(d.dir(), rootID, "inRoot"), driveFolder(d.dir(), rootID), driveItem(id(subfolder), name(subfolder), d.dir(folderName()), folderID(), isFolder), @@ -258,9 +258,9 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - folderID(): asNew(t, d.strPath(folderName())), - id(subfolder): asNew(t, d.strPath(folderName(), name(subfolder))), - folderID(2): asNew(t, d.strPath(folderName(), name(subfolder), folderName())), + folderID(): asNew(t, d.strPath(t, folderName())), + id(subfolder): asNew(t, d.strPath(t, folderName(), name(subfolder))), + folderID(2): asNew(t, d.strPath(t, folderName(), name(subfolder), folderName())), }, expectedItemCount: 5, expectedFileCount: 2, @@ -268,9 +268,9 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { // just "folder" isn't added here because the include check is done on the // parent path since we only check later if something is a folder or not. expectedPrevPaths: map[string]string{ - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), - folderID(2): d.strPath(folderName(), name(subfolder), folderName()), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), + folderID(2): d.strPath(t, folderName(), name(subfolder), folderName()), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID("inFolder"), fileID("inFolder2")), @@ -278,7 +278,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "prefix subfolder selector", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFile(d.dir(), rootID, "inRoot"), driveFolder(d.dir(), rootID), driveItem(id(subfolder), name(subfolder), d.dir(folderName()), folderID(), isFolder), @@ -295,15 +295,15 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - id(subfolder): asNew(t, d.strPath(folderName(), name(subfolder))), - folderID(2): asNew(t, d.strPath(folderName(), name(subfolder), folderName())), + id(subfolder): asNew(t, d.strPath(t, folderName(), name(subfolder))), + folderID(2): asNew(t, d.strPath(t, folderName(), name(subfolder), folderName())), }, expectedItemCount: 3, expectedFileCount: 1, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - id(subfolder): d.strPath(folderName(), name(subfolder)), - folderID(2): d.strPath(folderName(), name(subfolder), folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), + folderID(2): d.strPath(t, folderName(), name(subfolder), folderName()), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID("inFolder2")), @@ -311,7 +311,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "match subfolder selector", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFile(d.dir(), rootID), driveFolder(d.dir(), rootID), driveItem(id(subfolder), name(subfolder), d.dir(folderName()), folderID(), isFolder), @@ -325,14 +325,14 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - id(subfolder): asNew(t, d.strPath(folderName(), name(subfolder))), + id(subfolder): asNew(t, d.strPath(t, folderName(), name(subfolder))), }, expectedItemCount: 2, expectedFileCount: 1, expectedContainerCount: 1, // No child folders for subfolder so nothing here. expectedPrevPaths: map[string]string{ - id(subfolder): d.strPath(folderName(), name(subfolder)), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID("inSubfolder")), @@ -340,27 +340,27 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "not moved folder tree", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNotMoved(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNotMoved(t, d.strPath(t, folderName())), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -368,27 +368,27 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asMoved(t, d.strPath(folderName("a")), d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asMoved(t, d.strPath(t, folderName("a")), d.strPath(t, folderName())), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -396,28 +396,28 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree twice within backup", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(folderID(1), folderName(), d.dir(), rootID, isFolder), driveItem(folderID(2), folderName(), d.dir(), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(1): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(1): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(2): asNew(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(2): asNew(t, d.strPath(t, folderName())), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -425,28 +425,28 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "deleted folder tree twice within backup", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), delItem(folderID(), rootID, isFolder), driveItem(folderID(), name(drivePfx), d.dir(), rootID, isFolder), delItem(folderID(), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(): d.strPath(), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asDeleted(t, d.strPath("")), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asDeleted(t, d.strPath(t, "")), }, expectedItemCount: 0, expectedFileCount: 0, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + rootID: d.strPath(t), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -454,29 +454,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree twice within backup including delete", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), delItem(folderID(), rootID, isFolder), driveItem(folderID(2), folderName(), d.dir(), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(2): asNew(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(2): asNew(t, d.strPath(t, folderName())), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -484,28 +484,28 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "deleted folder tree twice within backup with addition", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(folderID(1), folderName(), d.dir(), rootID, isFolder), delItem(folderID(1), rootID, isFolder), driveItem(folderID(2), folderName(), d.dir(), rootID, isFolder), delItem(folderID(2), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(1): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(1): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -513,7 +513,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree with file no previous", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), driveItem(fileID(), fileName(), d.dir(folderName()), folderID(), isFile), driveItem(folderID(), folderName(2), d.dir(), rootID, isFolder), @@ -523,15 +523,15 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNew(t, d.strPath(folderName(2))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNew(t, d.strPath(t, folderName(2))), }, expectedItemCount: 2, expectedFileCount: 1, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName(2)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName(2)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID()), @@ -539,7 +539,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree with file no previous 1", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), driveItem(fileID(), fileName(), d.dir(folderName()), folderID(), isFile), }, @@ -548,15 +548,15 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNew(t, d.strPath(folderName())), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNew(t, d.strPath(t, folderName())), }, expectedItemCount: 2, expectedFileCount: 1, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID()), @@ -564,29 +564,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree and subfolder 1", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), driveItem(id(subfolder), name(subfolder), d.dir(), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asMoved(t, d.strPath(folderName("a")), d.strPath(folderName())), - id(subfolder): asMoved(t, d.strPath(folderName("a"), name(subfolder)), d.strPath(name(subfolder))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asMoved(t, d.strPath(t, folderName("a")), d.strPath(t, folderName())), + id(subfolder): asMoved(t, d.strPath(t, folderName("a"), name(subfolder)), d.strPath(t, name(subfolder))), }, expectedItemCount: 2, expectedFileCount: 0, expectedContainerCount: 3, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -594,29 +594,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree and subfolder 2", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(id(subfolder), name(subfolder), d.dir(), rootID, isFolder), driveFolder(d.dir(), rootID), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asMoved(t, d.strPath(folderName("a")), d.strPath(folderName())), - id(subfolder): asMoved(t, d.strPath(folderName("a"), name(subfolder)), d.strPath(name(subfolder))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asMoved(t, d.strPath(t, folderName("a")), d.strPath(t, folderName())), + id(subfolder): asMoved(t, d.strPath(t, folderName("a"), name(subfolder)), d.strPath(t, name(subfolder))), }, expectedItemCount: 2, expectedFileCount: 0, expectedContainerCount: 3, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -624,7 +624,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "move subfolder when moving parent", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(folderID(2), folderName(2), d.dir(), rootID, isFolder), driveItem(id(item), name(item), d.dir(folderName(2)), folderID(2), isFile), // Need to see the parent folder first (expected since that's what Graph @@ -635,26 +635,26 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { driveFolder(d.dir(), rootID), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(2): asNew(t, d.strPath(folderName(2))), - folderID(): asMoved(t, d.strPath(folderName("a")), d.strPath(folderName())), - id(subfolder): asMoved(t, d.strPath(folderName("a"), name(subfolder)), d.strPath(folderName(), name(subfolder))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(2): asNew(t, d.strPath(t, folderName(2))), + folderID(): asMoved(t, d.strPath(t, folderName("a")), d.strPath(t, folderName())), + id(subfolder): asMoved(t, d.strPath(t, folderName("a"), name(subfolder)), d.strPath(t, folderName(), name(subfolder))), }, expectedItemCount: 5, expectedFileCount: 2, expectedContainerCount: 4, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - folderID(2): d.strPath(folderName(2)), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID(2): d.strPath(t, folderName(2)), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(id(item), id(item, 2)), @@ -662,29 +662,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "moved folder tree multiple times", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveFolder(d.dir(), rootID), driveItem(fileID(), fileName(), d.dir(folderName()), folderID(), isFile), driveItem(folderID(), folderName(2), d.dir(), rootID, isFolder), }, previousPaths: map[string]string{ - folderID(): d.strPath(folderName("a")), - id(subfolder): d.strPath(folderName("a"), name(subfolder)), + folderID(): d.strPath(t, folderName("a")), + id(subfolder): d.strPath(t, folderName("a"), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asMoved(t, d.strPath(folderName("a")), d.strPath(folderName(2))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asMoved(t, d.strPath(t, folderName("a")), d.strPath(t, folderName(2))), }, expectedItemCount: 2, expectedFileCount: 1, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName(2)), - id(subfolder): d.strPath(folderName(2), name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName(2)), + id(subfolder): d.strPath(t, folderName(2), name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(fileID()), @@ -692,28 +692,28 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "deleted folder and package", items: []models.DriveItemable{ - driveRootFolder(), // root is always present, but not necessary here + rootFolder(), // root is always present, but not necessary here delItem(folderID(), rootID, isFolder), delItem(id(pkg), rootID, isPackage), }, previousPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(pkg): d.strPath(name(pkg)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(pkg): d.strPath(t, name(pkg)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asDeleted(t, d.strPath(folderName())), - id(pkg): asDeleted(t, d.strPath(name(pkg))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asDeleted(t, d.strPath(t, folderName())), + id(pkg): asDeleted(t, d.strPath(t, name(pkg))), }, expectedItemCount: 0, expectedFileCount: 0, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -721,23 +721,23 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "delete folder without previous", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), delItem(folderID(), rootID, isFolder), }, previousPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedItemCount: 0, expectedFileCount: 0, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -745,29 +745,29 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "delete folder tree move subfolder", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), delItem(folderID(), rootID, isFolder), driveItem(id(subfolder), name(subfolder), d.dir(), rootID, isFolder), }, previousPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(subfolder): d.strPath(folderName(), name(subfolder)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(subfolder): d.strPath(t, folderName(), name(subfolder)), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asDeleted(t, d.strPath(folderName())), - id(subfolder): asMoved(t, d.strPath(folderName(), name(subfolder)), d.strPath(name(subfolder))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asDeleted(t, d.strPath(t, folderName())), + id(subfolder): asMoved(t, d.strPath(t, folderName(), name(subfolder)), d.strPath(t, name(subfolder))), }, expectedItemCount: 1, expectedFileCount: 0, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - id(subfolder): d.strPath(name(subfolder)), + rootID: d.strPath(t), + id(subfolder): d.strPath(t, name(subfolder)), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -775,23 +775,23 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "delete file", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), delItem(id(item), rootID, isFile), }, previousPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, scope: anyFolderScope, topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedItemCount: 1, expectedFileCount: 1, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: makeExcludeMap(id(item)), @@ -799,7 +799,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "item before parent errors", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(fileID(), fileName(), d.dir(folderName()), folderID(), isFile), driveFolder(d.dir(), rootID), }, @@ -808,13 +808,13 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.Error, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), + rootID: asNotMoved(t, d.strPath(t)), }, expectedItemCount: 0, expectedFileCount: 0, expectedContainerCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), + rootID: d.strPath(t), }, expectedTopLevelPackages: map[string]struct{}{}, expectedExcludes: map[string]struct{}{}, @@ -822,7 +822,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { { name: "1 root file, 1 folder, 1 package, 1 good file, 1 malware", items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem(fileID(), fileID(), d.dir(), rootID, isFile), driveFolder(d.dir(), rootID), driveItem(id(pkg), name(pkg), d.dir(), rootID, isPackage), @@ -834,21 +834,21 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { topLevelPackages: map[string]struct{}{}, expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - rootID: asNotMoved(t, d.strPath()), - folderID(): asNew(t, d.strPath(folderName())), - id(pkg): asNew(t, d.strPath(name(pkg))), + rootID: asNotMoved(t, d.strPath(t)), + folderID(): asNew(t, d.strPath(t, folderName())), + id(pkg): asNew(t, d.strPath(t, name(pkg))), }, expectedItemCount: 4, expectedFileCount: 2, expectedContainerCount: 3, expectedSkippedCount: 1, expectedPrevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - id(pkg): d.strPath(name(pkg)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + id(pkg): d.strPath(t, name(pkg)), }, expectedTopLevelPackages: map[string]struct{}{ - d.strPath(name(pkg)): {}, + d.strPath(t, name(pkg)): {}, }, expectedCountPackages: 1, expectedExcludes: makeExcludeMap(fileID(), fileID("good")), @@ -941,6 +941,7 @@ func (suite *CollectionsUnitSuite) TestPopulateDriveCollections() { } func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { + t := suite.T() d := drive() d2 := drive(2) @@ -966,7 +967,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -977,7 +978,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { }, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }, canUsePreviousBackup: true, @@ -1008,7 +1009,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1017,7 +1018,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { expectedDeltas: map[string]string{}, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }, canUsePreviousBackup: true, @@ -1064,7 +1065,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1073,7 +1074,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { expectedDeltas: map[string]string{d.id: ""}, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }, canUsePreviousBackup: true, @@ -1091,7 +1092,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1105,7 +1106,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d2.id: { - folderID(2): d2.strPath(), + folderID(2): d2.strPath(t), }, }), } @@ -1117,10 +1118,10 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { }, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, d2.id: { - folderID(2): d2.strPath(), + folderID(2): d2.strPath(t), }, }, canUsePreviousBackup: true, @@ -1158,7 +1159,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), graph.NewMetadataEntry( @@ -1172,7 +1173,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { }, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }, canUsePreviousBackup: true, @@ -1190,7 +1191,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1201,7 +1202,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(2): d2.strPath(), + folderID(2): d2.strPath(t), }, }), } @@ -1224,7 +1225,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1254,8 +1255,8 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), - folderID(2): d.strPath(), + folderID(1): d.strPath(t), + folderID(2): d.strPath(t), }, }), } @@ -1266,8 +1267,8 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { }, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), - folderID(2): d.strPath(), + folderID(1): d.strPath(t), + folderID(2): d.strPath(t), }, }, expectedAlerts: []string{fault.AlertPreviousPathCollision}, @@ -1288,8 +1289,8 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), - folderID(2): d.strPath(), + folderID(1): d.strPath(t), + folderID(2): d.strPath(t), }, }), } @@ -1303,7 +1304,7 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { bupMD.PreviousPathFileName, map[string]map[string]string{ d2.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }), } @@ -1315,11 +1316,11 @@ func (suite *CollectionsUnitSuite) TestDeserializeMetadata() { }, expectedPaths: map[string]map[string]string{ d.id: { - folderID(1): d.strPath(), - folderID(2): d.strPath(), + folderID(1): d.strPath(t), + folderID(2): d.strPath(t), }, d2.id: { - folderID(1): d.strPath(), + folderID(1): d.strPath(t), }, }, expectedAlerts: []string{fault.AlertPreviousPathCollision}, @@ -1429,6 +1430,7 @@ func (suite *CollectionsUnitSuite) TestGet() { false) require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err)) + t := suite.T() d := drive(1) d2 := drive(2) @@ -1460,19 +1462,19 @@ func (suite *CollectionsUnitSuite) TestGet() { canUsePreviousBackup: true, errCheck: assert.NoError, previousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NotMovedState: {}}, + d.strPath(t): {data.NotMovedState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID()), + d.strPath(t): makeExcludeMap(fileID()), }), }, { @@ -1485,19 +1487,19 @@ func (suite *CollectionsUnitSuite) TestGet() { canUsePreviousBackup: true, errCheck: assert.NoError, previousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NotMovedState: {fileID()}}, + d.strPath(t): {data.NotMovedState: {fileID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID()), + d.strPath(t): makeExcludeMap(fileID()), }), }, { @@ -1512,22 +1514,22 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{}, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID()}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1543,22 +1545,22 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{}, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID()}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1574,24 +1576,24 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NotMovedState: {fileID()}}, - d.strPath(folderName()): {data.NewState: {folderID()}}, + d.strPath(t): {data.NotMovedState: {fileID()}}, + d.strPath(t, folderName()): {data.NewState: {folderID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID()), + d.strPath(t): makeExcludeMap(fileID()), }), }, { @@ -1611,22 +1613,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1651,22 +1653,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1689,22 +1691,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1726,10 +1728,10 @@ func (suite *CollectionsUnitSuite) TestGet() { d2.id: {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID()}}, - d2.strPath(): {data.NewState: {}}, - d2.strPath(folderName()): {data.NewState: {folderID(2), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID()}}, + d2.strPath(t): {data.NewState: {}}, + d2.strPath(t, folderName()): {data.NewState: {folderID(2), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), @@ -1737,20 +1739,20 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, d2.id: { - rootID: d2.strPath(), - folderID(2): d2.strPath(folderName()), + rootID: d2.strPath(t), + folderID(2): d2.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, - d2.strPath(): true, - d2.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, + d2.strPath(t): true, + d2.strPath(t, folderName()): true, }, }, { @@ -1773,10 +1775,10 @@ func (suite *CollectionsUnitSuite) TestGet() { d2.id: {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID()}}, - d2.strPath(): {data.NewState: {}}, - d2.strPath(folderName()): {data.NewState: {folderID(), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID()}}, + d2.strPath(t): {data.NewState: {}}, + d2.strPath(t, folderName()): {data.NewState: {folderID(), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), @@ -1784,20 +1786,20 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, d2.id: { - rootID: d2.strPath(), - folderID(): d2.strPath(folderName()), + rootID: d2.strPath(t), + folderID(): d2.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, - d2.strPath(): true, - d2.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, + d2.strPath(t): true, + d2.strPath(t, folderName()): true, }, }, { @@ -1828,29 +1830,29 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.DeletedState: {}}, - d.strPath(folderName(2)): {data.NewState: {folderID(2), fileID()}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.DeletedState: {}}, + d.strPath(t, folderName(2)): {data.NewState: {folderID(2), fileID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(2): d.strPath(folderName(2)), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName(2)), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, - d.strPath(folderName(2)): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, + d.strPath(t, folderName(2)): true, }, }, { @@ -1866,29 +1868,29 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.DeletedState: {}}, - d.strPath(folderName(2)): {data.NewState: {folderID(2), fileID()}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.DeletedState: {}}, + d.strPath(t, folderName(2)): {data.NewState: {folderID(2), fileID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(2): d.strPath(folderName(2)), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName(2)), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, - d.strPath(folderName(2)): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, + d.strPath(t, folderName(2)): true, }, }, { @@ -1907,13 +1909,13 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): { + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): { // Old folder path should be marked as deleted since it should compare // by ID. data.DeletedState: {}, @@ -1925,14 +1927,14 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1951,13 +1953,13 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): { + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): { data.NewState: {folderID(), fileID()}, }, }, @@ -1966,14 +1968,14 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -1989,13 +1991,13 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): { + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): { data.DeletedState: {}, data.NewState: {folderID(2), fileID(2)}, }, @@ -2005,14 +2007,14 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -2028,13 +2030,13 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): { + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): { // Old folder path should be marked as deleted since it should compare // by ID. data.DeletedState: {}, @@ -2046,14 +2048,14 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(2): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(2): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -2075,22 +2077,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID(), fileID(2)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, expectedSkippedCount: 2, }, @@ -2114,30 +2116,30 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - folderID(2): d.strPath(folderName(2)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID(2): d.strPath(t, folderName(2)), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(), fileID()}}, - d.strPath(folderName(2)): {data.DeletedState: {}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(), fileID()}}, + d.strPath(t, folderName(2)): {data.DeletedState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL, 2), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, - d.strPath(folderName(2)): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, + d.strPath(t, folderName(2)): true, }, }, { @@ -2151,26 +2153,26 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.DeletedState: {}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.DeletedState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -2184,23 +2186,23 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, + d.strPath(t): {data.NewState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, + d.strPath(t): true, }, }, { @@ -2220,19 +2222,19 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, + d.strPath(t): {data.NewState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL, 2), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, + d.strPath(t): true, }, }, { @@ -2255,22 +2257,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID(1), fileID(1)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID(1), fileID(1)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL, 2), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(1): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(1): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -2291,20 +2293,20 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NotMovedState: {}}, - d.strPath(folderName()): {data.DeletedState: {}}, + d.strPath(t): {data.NotMovedState: {}}, + d.strPath(t, folderName()): {data.DeletedState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL, 2), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), @@ -2328,27 +2330,27 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.DeletedState: {}, data.NewState: {folderID(1), fileID(1)}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.DeletedState: {}, data.NewState: {folderID(1), fileID(1)}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL, 2), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(1): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(1): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): false, - d.strPath(folderName()): true, + d.strPath(t): false, + d.strPath(t, folderName()): true, }, }, { @@ -2366,22 +2368,22 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, - d.strPath(folderName()): {data.NewState: {folderID()}}, + d.strPath(t): {data.NewState: {}}, + d.strPath(t, folderName()): {data.NewState: {folderID()}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, - d.strPath(folderName()): true, + d.strPath(t): true, + d.strPath(t, folderName()): true, }, }, { @@ -2397,19 +2399,19 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, + d.strPath(t): {data.NewState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, + d.strPath(t): true, }, }, { @@ -2425,19 +2427,19 @@ func (suite *CollectionsUnitSuite) TestGet() { id(drivePfx, 1): {}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NewState: {}}, + d.strPath(t): {data.NewState: {}}, }, expectedDeltaURLs: map[string]string{ id(drivePfx, 1): id(deltaURL), }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), + rootID: d.strPath(t), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d.strPath(): true, + d.strPath(t): true, }, }, { @@ -2448,20 +2450,20 @@ func (suite *CollectionsUnitSuite) TestGet() { canUsePreviousBackup: true, errCheck: assert.NoError, previousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, - d2.id: {rootID: d2.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, + d2.id: {rootID: d2.strPath(t)}, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): {data.NotMovedState: {}}, - d2.strPath(): {data.DeletedState: {}}, + d.strPath(t): {data.NotMovedState: {}}, + d2.strPath(t): {data.DeletedState: {}}, }, expectedDeltaURLs: map[string]string{id(drivePfx, 1): id(deltaURL)}, expectedPreviousPaths: map[string]map[string]string{ - id(drivePfx, 1): {rootID: d.strPath()}, + id(drivePfx, 1): {rootID: d.strPath(t)}, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), doNotMergeItems: map[string]bool{ - d2.strPath(): true, + d2.strPath(t): true, }, }, { @@ -2485,34 +2487,34 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName()), - folderID(2): d.strPath(folderName()), - folderID(3): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID(2): d.strPath(t, folderName()), + folderID(3): d.strPath(t, folderName()), }, d2.id: { - rootID: d2.strPath(), - folderID(): d2.strPath(folderName()), - folderID(2): d2.strPath(folderName(2)), + rootID: d2.strPath(t), + folderID(): d2.strPath(t, folderName()), + folderID(2): d2.strPath(t, folderName(2)), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): { + d.strPath(t): { data.NewState: {folderID(), folderID(2)}, }, - d.strPath(folderName()): { + d.strPath(t, folderName()): { data.NotMovedState: {folderID(), fileID()}, }, - d.strPath(folderName(2)): { + d.strPath(t, folderName(2)): { data.MovedState: {folderID(2), fileID(2)}, }, - d2.strPath(): { + d2.strPath(t): { data.NewState: {folderID(), folderID(2)}, }, - d2.strPath(folderName()): { + d2.strPath(t, folderName()): { data.NotMovedState: {folderID(), fileID()}, }, - d2.strPath(folderName(2)): { + d2.strPath(t, folderName(2)): { data.NotMovedState: {folderID(2), fileID(2)}, }, }, @@ -2522,20 +2524,20 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(): d.strPath(folderName(2)), // note: this is a bug, but is currently expected - folderID(2): d.strPath(folderName(2)), - folderID(3): d.strPath(folderName(2)), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName(2)), // note: this is a bug, but is currently expected + folderID(2): d.strPath(t, folderName(2)), + folderID(3): d.strPath(t, folderName(2)), }, d2.id: { - rootID: d2.strPath(), - folderID(): d2.strPath(folderName()), - folderID(2): d2.strPath(folderName(2)), + rootID: d2.strPath(t), + folderID(): d2.strPath(t, folderName()), + folderID(2): d2.strPath(t, folderName(2)), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID(), fileID(2)), - d2.strPath(): makeExcludeMap(fileID(), fileID(2)), + d.strPath(t): makeExcludeMap(fileID(), fileID(2)), + d2.strPath(t): makeExcludeMap(fileID(), fileID(2)), }), doNotMergeItems: map[string]bool{}, }, @@ -2553,18 +2555,18 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(nav): d.strPath(folderName(fanny)), + rootID: d.strPath(t), + folderID(nav): d.strPath(t, folderName(fanny)), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): { + d.strPath(t): { data.NewState: {folderID(fanny, 2)}, }, - d.strPath(folderName(nav)): { + d.strPath(t, folderName(nav)): { data.MovedState: {folderID(nav), fileID()}, }, - d.strPath(folderName(fanny)): { + d.strPath(t, folderName(fanny)): { data.NewState: {folderID(fanny, 2), fileID(2)}, }, }, @@ -2573,13 +2575,13 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(nav): d.strPath(folderName(nav)), - folderID(fanny, 2): d.strPath(folderName(nav)), // note: this is a bug, but currently expected + rootID: d.strPath(t), + folderID(nav): d.strPath(t, folderName(nav)), + folderID(fanny, 2): d.strPath(t, folderName(nav)), // note: this is a bug, but currently expected }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID(), fileID(2)), + d.strPath(t): makeExcludeMap(fileID(), fileID(2)), }), doNotMergeItems: map[string]bool{}, }, @@ -2598,27 +2600,27 @@ func (suite *CollectionsUnitSuite) TestGet() { errCheck: assert.NoError, previousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(nav): d.strPath(folderName(nav)), - folderID(fanny): d.strPath(folderName(fanny)), - folderID(foo): d.strPath(folderName(nav), folderName(foo)), - folderID(bar): d.strPath(folderName(fanny), folderName(foo)), + rootID: d.strPath(t), + folderID(nav): d.strPath(t, folderName(nav)), + folderID(fanny): d.strPath(t, folderName(fanny)), + folderID(foo): d.strPath(t, folderName(nav), folderName(foo)), + folderID(bar): d.strPath(t, folderName(fanny), folderName(foo)), }, }, expectedCollections: map[string]map[data.CollectionState][]string{ - d.strPath(): { + d.strPath(t): { data.NotMovedState: {fileID(1)}, }, - d.strPath(folderName(nav)): { + d.strPath(t, folderName(nav)): { data.NotMovedState: {folderID(nav)}, }, - d.strPath(folderName(nav), folderName(foo)): { + d.strPath(t, folderName(nav), folderName(foo)): { data.MovedState: {folderID(bar)}, }, - d.strPath(folderName(fanny)): { + d.strPath(t, folderName(fanny)): { data.NotMovedState: {folderID(fanny)}, }, - d.strPath(folderName(fanny), folderName(foo)): { + d.strPath(t, folderName(fanny), folderName(foo)): { data.MovedState: {folderID(foo)}, }, }, @@ -2627,15 +2629,15 @@ func (suite *CollectionsUnitSuite) TestGet() { }, expectedPreviousPaths: map[string]map[string]string{ id(drivePfx, 1): { - rootID: d.strPath(), - folderID(nav): d.strPath(folderName(nav)), - folderID(fanny): d.strPath(folderName(fanny)), - folderID(foo): d.strPath(folderName(nav), folderName(foo)), // note: this is a bug, but currently expected - folderID(bar): d.strPath(folderName(nav), folderName(foo)), + rootID: d.strPath(t), + folderID(nav): d.strPath(t, folderName(nav)), + folderID(fanny): d.strPath(t, folderName(fanny)), + folderID(foo): d.strPath(t, folderName(nav), folderName(foo)), // note: this is a bug, but currently expected + folderID(bar): d.strPath(t, folderName(nav), folderName(foo)), }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - d.strPath(): makeExcludeMap(fileID(1)), + d.strPath(t): makeExcludeMap(fileID(1)), }), doNotMergeItems: map[string]bool{}, }, diff --git a/src/internal/m365/collection/drive/collections_tree.go b/src/internal/m365/collection/drive/collections_tree.go index b1e061f21..1bd1adefe 100644 --- a/src/internal/m365/collection/drive/collections_tree.go +++ b/src/internal/m365/collection/drive/collections_tree.go @@ -221,6 +221,7 @@ func (c *Collections) makeDriveCollections( ctx, tree, drv, + prevPaths, prevDeltaLink, countPagesInDelta, errs) @@ -457,6 +458,14 @@ func (c *Collections) enumeratePageOfItems( return err } + // special case: we only want to add a limited number of files + // to each collection. But if one collection fills up, we don't + // want to break out of the whole backup. That allows us to preview + // many folders with a small selection of files in each. + if errors.Is(err, errHitCollectionLimit) { + continue + } + el.AddRecoverable(ictx, clues.Wrap(err, "adding folder")) } } @@ -479,8 +488,6 @@ func (c *Collections) addFolderToTree( isDeleted = folder.GetDeleted() != nil isMalware = folder.GetMalware() != nil isPkg = folder.GetPackageEscaped() != nil - parent = folder.GetParentReference() - parentID string notSelected bool ) @@ -489,10 +496,6 @@ func (c *Collections) addFolderToTree( return nil, errHitLimit } - if parent != nil { - parentID = ptr.Val(parent.GetId()) - } - defer func() { switch { case notSelected: @@ -525,7 +528,7 @@ func (c *Collections) addFolderToTree( } if isDeleted { - err := tree.setTombstone(ctx, folderID) + err := tree.setTombstone(ctx, folder) return nil, clues.Stack(err).OrNil() } @@ -541,7 +544,7 @@ func (c *Collections) addFolderToTree( return nil, nil } - err = tree.setFolder(ctx, parentID, folderID, folderName, isPkg) + err = tree.setFolder(ctx, folder) return nil, clues.Stack(err).OrNil() } @@ -635,22 +638,32 @@ func (c *Collections) addFileToTree( if parentNotNil && !alreadySeen { countSize := tree.countLiveFilesAndSizes() - // Don't add new items if the new collection has already reached it's limit. - // item moves and updates are generally allowed through. - if limiter.atContainerItemsLimit(len(parentNode.files)) || limiter.hitItemLimit(countSize.numFiles) { + // Tell the enumerator to exit if we've already hit the total + // limit of bytes or items in this backup. + if limiter.alreadyHitTotalBytesLimit(countSize.totalBytes) || + limiter.hitItemLimit(countSize.numFiles) { return nil, errHitLimit } - // Skip large files that don't fit within the size limit. - // unlike the other checks, which see if we're already at the limit, this check - // needs to be forward-facing to ensure we don't go far over the limit. + // Don't add new items if the new collection has already reached it's limit. + // item moves and updates are generally allowed through. + if limiter.atContainerItemsLimit(len(parentNode.files)) { + return nil, errHitCollectionLimit + } + + // Don't include large files that don't fit within the size limit. + // Unlike the other checks, which see if we're already at the limit, + // this check needs to be forward-facing to ensure we don't go far + // over the limit // Example case: a 1gb limit and a 25gb file. - if limiter.hitTotalBytesLimit(fileSize + countSize.totalBytes) { - return nil, errHitLimit + if limiter.willStepOverBytesLimit(countSize.totalBytes, fileSize) { + // don't return errHitLimit here; we only want to skip the + // current file. We may not want to skip files after it. + return nil, nil } } - err := tree.addFile(parentID, fileID, file) + err := tree.addFile(file) if err != nil { return nil, clues.StackWC(ctx, err) } @@ -776,6 +789,7 @@ func (c *Collections) turnTreeIntoCollections( ctx context.Context, tree *folderyMcFolderFace, drv models.Driveable, + prevPaths map[string]string, prevDeltaLink string, countPagesInDelta int, errs *fault.Bus, @@ -792,12 +806,11 @@ func (c *Collections) turnTreeIntoCollections( } var ( - collections = []data.BackupCollection{} - newPrevPaths = map[string]string{} - uc *urlCache - el = errs.Local() - driveID = ptr.Val(drv.GetId()) - driveName = ptr.Val(drv.GetName()) + collections = []data.BackupCollection{} + uc *urlCache + el = errs.Local() + driveID = ptr.Val(drv.GetId()) + driveName = ptr.Val(drv.GetName()) ) // Attach an url cache to the drive if the number of discovered items is @@ -825,15 +838,11 @@ func (c *Collections) turnTreeIntoCollections( } } - for id, cbl := range collectables { + for _, cbl := range collectables { if el.Failure() != nil { break } - if cbl.currPath != nil { - newPrevPaths[id] = cbl.currPath.String() - } - coll, err := NewCollection( c.handler, c.protectedResource, @@ -856,5 +865,16 @@ func (c *Collections) turnTreeIntoCollections( collections = append(collections, coll) } - return collections, newPrevPaths, tree.generateExcludeItemIDs(), el.Failure() + if el.Failure() != nil { + return nil, nil, nil, el.Failure() + } + + // use the collectables and old previous paths + // to generate new previous paths + newPrevPaths, err := tree.generateNewPreviousPaths(collectables, prevPaths) + if err != nil { + return nil, nil, nil, clues.WrapWC(ctx, err, "generating new previous paths") + } + + return collections, newPrevPaths, tree.generateExcludeItemIDs(), nil } diff --git a/src/internal/m365/collection/drive/collections_tree_test.go b/src/internal/m365/collection/drive/collections_tree_test.go index 3773e8abd..ea57ef787 100644 --- a/src/internal/m365/collection/drive/collections_tree_test.go +++ b/src/internal/m365/collection/drive/collections_tree_test.go @@ -237,6 +237,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_GetTree() { // to ensure we stitch the parts together correctly. func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { d := drive() + t := suite.T() table := []struct { name string @@ -265,7 +266,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { delta(id(deltaURL), nil).with( aPage()))), prevPaths: map[string]string{ - folderID(): d.strPath(folderName()), + folderID(): d.strPath(t, folderName()), }, expectCounts: countTD.Expected{ count.PrevPaths: 1, @@ -277,7 +278,9 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { enumerator: driveEnumerator( d.newEnumer().with( delta(id(deltaURL), nil).with( - aPage(d.folderAtRoot(), d.fileAt(folder))))), + aPage( + d.folderAt(root), + d.fileAt(folder))))), prevPaths: map[string]string{}, expectCounts: countTD.Expected{ count.PrevPaths: 0, @@ -289,9 +292,11 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { enumerator: driveEnumerator( d.newEnumer().with( delta(id(deltaURL), nil).with( - aPage(d.folderAtRoot(), d.fileAt(folder))))), + aPage( + d.folderAt(root), + d.fileAt(folder))))), prevPaths: map[string]string{ - folderID(): d.strPath(folderName()), + folderID(): d.strPath(t, folderName()), }, expectCounts: countTD.Expected{ count.PrevPaths: 1, @@ -319,7 +324,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { aReset(), aPage()))), prevPaths: map[string]string{ - folderID(): d.strPath(folderName()), + folderID(): d.strPath(t, folderName()), }, expectCounts: countTD.Expected{ count.PrevPaths: 1, @@ -332,7 +337,9 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { d.newEnumer().with( deltaWReset(id(deltaURL), nil).with( aReset(), - aPage(d.folderAtRoot(), d.fileAt(folder))))), + aPage( + d.folderAt(root), + d.fileAt(folder))))), prevPaths: map[string]string{}, expectCounts: countTD.Expected{ count.PrevPaths: 0, @@ -345,9 +352,11 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { d.newEnumer().with( deltaWReset(id(deltaURL), nil).with( aReset(), - aPage(d.folderAtRoot(), d.fileAt(folder))))), + aPage( + d.folderAt(root), + d.fileAt(folder))))), prevPaths: map[string]string{ - folderID(): d.strPath(folderName()), + folderID(): d.strPath(t, folderName()), }, expectCounts: countTD.Expected{ count.PrevPaths: 1, @@ -384,6 +393,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeDriveCollections() { func (suite *CollectionsTreeUnitSuite) TestCollections_AddPrevPathsToTree_errors() { d := drive() + t := suite.T() table := []struct { name string @@ -395,8 +405,8 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddPrevPathsToTree_errors name: "no error - normal usage", tree: treeWithFolders, prevPaths: map[string]string{ - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), + folderID("parent"): d.strPath(t, folderName("parent")), + folderID(): d.strPath(t, folderName("parent"), folderName()), }, expectErr: require.NoError, }, @@ -410,7 +420,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddPrevPathsToTree_errors name: "no error - folder not visited in this delta", tree: treeWithFolders, prevPaths: map[string]string{ - id("santa"): d.strPath(name("santa")), + id("santa"): d.strPath(t, name("santa")), }, expectErr: require.NoError, }, @@ -418,7 +428,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddPrevPathsToTree_errors name: "empty key in previous paths", tree: treeWithFolders, prevPaths: map[string]string{ - "": d.strPath(folderName("parent")), + "": d.strPath(t, folderName("parent")), }, expectErr: require.Error, }, @@ -460,6 +470,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddPrevPathsToTree_errors func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() { d := drive() + t := suite.T() type expected struct { prevPaths map[string]string @@ -481,9 +492,9 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() enableURLCache: true, expect: expected{ prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent")), + folderID(): d.strPath(t, folderName("parent"), folderName()), }, collections: func(t *testing.T, d *deltaDrive) expectedCollections { return expectCollections( @@ -500,13 +511,13 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() aColl( d.fullPath(t, folderName("parent"), folderName()), nil, - fileID())) + fileID("f"))) }, globalExcludedFileIDs: makeExcludeMap( fileID("r"), fileID("p"), fileID("d"), - fileID()), + fileID("f")), }, }, { @@ -514,16 +525,20 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() tree: fullTree, enableURLCache: true, prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent-prev")), - folderID(): d.strPath(folderName("parent-prev"), folderName()), - folderID("tombstone"): d.strPath(folderName("tombstone-prev")), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent-prev")), + folderID(): d.strPath(t, folderName("parent-prev"), folderName()), + folderID("prev"): d.strPath(t, folderName("parent-prev"), folderName("prev")), + folderID("prev-chld"): d.strPath(t, folderName("parent-prev"), folderName("prev"), folderName("prev-chld")), + folderID("tombstone"): d.strPath(t, folderName("tombstone-prev")), }, expect: expected{ prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent")), + folderID(): d.strPath(t, folderName("parent"), folderName()), + folderID("prev"): d.strPath(t, folderName("parent"), folderName("prev")), + folderID("prev-chld"): d.strPath(t, folderName("parent"), folderName("prev"), folderName("prev-chld")), }, collections: func(t *testing.T, d *deltaDrive) expectedCollections { return expectCollections( @@ -540,31 +555,35 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() aColl( d.fullPath(t, folderName("parent"), folderName()), d.fullPath(t, folderName("parent-prev"), folderName()), - fileID()), + fileID("f")), aColl(nil, d.fullPath(t, folderName("tombstone-prev")))) }, globalExcludedFileIDs: makeExcludeMap( fileID("r"), fileID("p"), fileID("d"), - fileID()), + fileID("f")), }, }, { - name: "all folders moved - todo: path separator string check", - tree: fullTreeWithNames("parent", "tombstone"), + name: "all folders moved - path separator string check", + tree: fullTreeWithNames("pa/rent", "to/mbstone"), enableURLCache: true, prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent-prev")), - folderID(): d.strPath(folderName("parent-prev"), folderName()), - folderID("tombstone"): d.strPath(folderName("tombstone-prev")), + rootID: d.strPath(t), + folderID("pa/rent"): d.strPath(t, folderName("parent/prev")), + folderID(): d.strPath(t, folderName("parent/prev"), folderName()), + folderID("pr/ev"): d.strPath(t, folderName("parent/prev"), folderName("pr/ev")), + folderID("prev/chld"): d.strPath(t, folderName("parent/prev"), folderName("pr/ev"), folderName("prev/chld")), + folderID("to/mbstone"): d.strPath(t, folderName("tombstone/prev")), }, expect: expected{ prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), + rootID: d.strPath(t), + folderID("pa/rent"): d.strPath(t, folderName("pa/rent")), + folderID(): d.strPath(t, folderName("pa/rent"), folderName()), + folderID("pr/ev"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev")), + folderID("prev/chld"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev"), folderName("prev/chld")), }, collections: func(t *testing.T, d *deltaDrive) expectedCollections { return expectCollections( @@ -575,37 +594,45 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() d.fullPath(t), fileID("r")), aColl( - d.fullPath(t, folderName("parent")), - d.fullPath(t, folderName("parent-prev")), + d.fullPath(t, folderName("pa/rent")), + d.fullPath(t, folderName("parent/prev")), fileID("p")), aColl( - d.fullPath(t, folderName("parent"), folderName()), - d.fullPath(t, folderName("parent-prev"), folderName()), - fileID()), - aColl(nil, d.fullPath(t, folderName("tombstone-prev")))) + d.fullPath(t, folderName("pa/rent"), folderName()), + d.fullPath(t, folderName("parent/prev"), folderName()), + fileID("f")), + aColl(nil, d.fullPath(t, folderName("tombstone/prev")))) }, globalExcludedFileIDs: makeExcludeMap( fileID("r"), fileID("p"), fileID("d"), - fileID()), + fileID("f")), }, }, { - name: "no folders moved", + name: "nothing in the tree was moved " + + "but there were some folders in the previous paths that " + + "didn't appear in the delta so those have to appear in the " + + "new previous paths but those weren't moved either so " + + "everything should have the same path at the end", tree: fullTree, enableURLCache: true, prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), - folderID("tombstone"): d.strPath(folderName("tombstone")), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent")), + folderID(): d.strPath(t, folderName("parent"), folderName()), + folderID("tombstone"): d.strPath(t, folderName("tombstone")), + folderID("prev"): d.strPath(t, folderName("prev")), + folderID("prev-chld"): d.strPath(t, folderName("prev"), folderName("prev-chld")), }, expect: expected{ prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent")), - folderID(): d.strPath(folderName("parent"), folderName()), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent")), + folderID(): d.strPath(t, folderName("parent"), folderName()), + folderID("prev"): d.strPath(t, folderName("prev")), + folderID("prev-chld"): d.strPath(t, folderName("prev"), folderName("prev-chld")), }, collections: func(t *testing.T, d *deltaDrive) expectedCollections { return expectCollections( @@ -622,14 +649,64 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() aColl( d.fullPath(t, folderName("parent"), folderName()), d.fullPath(t, folderName("parent"), folderName()), - fileID()), + fileID("f")), aColl(nil, d.fullPath(t, folderName("tombstone")))) }, globalExcludedFileIDs: makeExcludeMap( fileID("r"), fileID("p"), fileID("d"), - fileID()), + fileID("f")), + }, + }, + { + name: "nothing in the tree was moved " + + "but there were some folders in the previous paths that " + + "didn't appear in the delta so those have to appear in the " + + "new previous paths but those weren't moved either so " + + "everything should have the same path at the end " + + "- the version with path separators chars in the directory names", + tree: fullTreeWithNames("pa/rent", "to/mbstone"), + enableURLCache: true, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID("pa/rent"): d.strPath(t, folderName("pa/rent")), + folderID(): d.strPath(t, folderName("pa/rent"), folderName()), + folderID("pr/ev"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev")), + folderID("prev/chld"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev"), folderName("prev/chld")), + folderID("to/mbstone"): d.strPath(t, folderName("to/mbstone")), + }, + expect: expected{ + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID("pa/rent"): d.strPath(t, folderName("pa/rent")), + folderID(): d.strPath(t, folderName("pa/rent"), folderName()), + folderID("pr/ev"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev")), + folderID("prev/chld"): d.strPath(t, folderName("pa/rent"), folderName("pr/ev"), folderName("prev/chld")), + }, + collections: func(t *testing.T, d *deltaDrive) expectedCollections { + return expectCollections( + false, + true, + aColl( + d.fullPath(t), + d.fullPath(t), + fileID("r")), + aColl( + d.fullPath(t, folderName("pa/rent")), + d.fullPath(t, folderName("pa/rent")), + fileID("p")), + aColl( + d.fullPath(t, folderName("pa/rent"), folderName()), + d.fullPath(t, folderName("pa/rent"), folderName()), + fileID("f")), + aColl(nil, d.fullPath(t, folderName("to/mbstone")))) + }, + globalExcludedFileIDs: makeExcludeMap( + fileID("r"), + fileID("p"), + fileID("d"), + fileID("f")), }, }, } @@ -656,6 +733,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_TurnTreeIntoCollections() ctx, tree, d.able, + test.prevPaths, deltaURL, countPages, fault.New(true)) @@ -782,10 +860,10 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( enumerator: driveEnumerator( d.newEnumer().with( delta(id(deltaURL), nil).with( - aPage(d.folderAtRoot()), - aPage(d.folderAtRoot("sib")), + aPage(d.folderAt(root)), + aPage(d.folderAt(root, "sib")), aPage( - d.folderAtRoot(), + d.folderAt(root), d.folderAt(folder, "chld"))))), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -815,13 +893,13 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder)), aPage( - d.folderAtRoot("sib"), + d.folderAt(root, "sib"), d.fileAt("sib", "fsib")), aPage( - d.folderAtRoot(), + d.folderAt(root), d.folderAt(folder, "chld"), d.fileAt("chld", "fchld"))))), limiter: newPagerLimiter(control.DefaultOptions()), @@ -917,7 +995,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder)), aPage(delItem(folderID(), rootID, isFolder))))), limiter: newPagerLimiter(control.DefaultOptions()), @@ -950,7 +1028,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot("parent"), + d.folderAt(root, "parent"), driveItem(folderID(), folderName("moved"), d.dir(), folderID("parent"), isFolder), driveFile(d.dir(folderName("parent"), folderName()), folderID())), aPage(delItem(folderID(), folderID("parent"), isFolder))))), @@ -986,7 +1064,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( delta(id(deltaURL), nil).with( aPage(delItem(folderID(), rootID, isFolder)), aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))))), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1018,7 +1096,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( delta(id(deltaURL), nil).with( aPage(delItem(folderID(), rootID, isFolder)), aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))))), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1049,13 +1127,13 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder)), aPage( - d.folderAtRoot("sib"), + d.folderAt(root, "sib"), d.fileAt("sib", "fsib")), aPage( - d.folderAtRoot(), + d.folderAt(root), d.folderAt(folder, "chld"), d.fileAt("chld", "fchld"))))), limiter: newPagerLimiter(minimumLimitOpts()), @@ -1085,13 +1163,13 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta( d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder)), aPage( - d.folderAtRoot("sib"), + d.folderAt(root, "sib"), d.fileAt("sib", "fsib")), aPage( - d.folderAtRoot(), + d.folderAt(root), d.folderAt(folder, "chld"), d.fileAt("chld", "fchld"))))), limiter: newPagerLimiter(minimumLimitOpts()), @@ -1136,15 +1214,15 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() d.newEnumer().with( delta(id(deltaURL), nil). with(aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))), delta(id(deltaURL), nil). with(aPage( - d.folderAtRoot("sib"), + d.folderAt(root, "sib"), d.fileAt("sib", "fsib"))), delta(id(deltaURL), nil). with(aPage( - d.folderAtRoot(), + d.folderAt(root), d.folderAt(folder, "chld"), d.fileAt("chld", "fchld"))))), limiter: newPagerLimiter(control.DefaultOptions()), @@ -1182,7 +1260,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))), // a (delete,create) pair in the same delta can occur when // a user deletes and restores an item in-between deltas. @@ -1191,7 +1269,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() delItem(folderID(), rootID, isFolder), delItem(fileID(), folderID(), isFile)), aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))))), limiter: newPagerLimiter(control.DefaultOptions()), expect: populateTreeExpected{ @@ -1222,7 +1300,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() d.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder))), delta(id(deltaURL), nil).with( aPage( @@ -1260,7 +1338,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta() delta(id(deltaURL), nil).with( // first page: create /root/folder and /root/folder/file aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder)), // assume the user makes changes at this point: // * create a new /root/folder @@ -1442,9 +1520,9 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold name: "many folders in a hierarchy", tree: treeWithRoot, page: aPage( - d.folderAtRoot(), - d.folderAtRoot("sib"), - d.folderAt(folder, "chld")), + d.folderAt(root), + d.folderAt(folder, "chld"), + d.folderAt(root, "sib")), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ counts: countTD.Expected{ @@ -1465,7 +1543,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold name: "create->delete", tree: treeWithRoot, page: aPage( - d.folderAtRoot(), + d.folderAt(root), delItem(folderID(), rootID, isFolder)), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ @@ -1485,7 +1563,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold name: "move->delete", tree: treeWithFolders, page: aPage( - d.folderAtRoot("parent"), + d.folderAt(root, "parent"), driveItem(folderID(), folderName("moved"), d.dir(folderName("parent")), folderID("parent"), isFolder), delItem(folderID(), folderID("parent"), isFolder)), limiter: newPagerLimiter(control.DefaultOptions()), @@ -1510,7 +1588,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold tree: treeWithRoot, page: aPage( delItem(folderID(), rootID, isFolder), - d.folderAtRoot()), + d.folderAt(root)), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ counts: countTD.Expected{ @@ -1531,7 +1609,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold tree: treeWithRoot, page: aPage( delItem(folderID(), rootID, isFolder), - d.folderAtRoot()), + d.folderAt(root)), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ counts: countTD.Expected{ @@ -1596,7 +1674,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_fold func (suite *CollectionsTreeUnitSuite) TestCollections_AddFolderToTree() { var ( d = drive() - fld = custom.ToCustomDriveItem(d.folderAtRoot()) + fld = custom.ToCustomDriveItem(d.folderAt(root)) subFld = custom.ToCustomDriveItem(driveFolder(d.dir(folderName("parent")), folderID("parent"))) pack = custom.ToCustomDriveItem(driveItem(id(pkg), name(pkg), d.dir(), rootID, isPackage)) del = custom.ToCustomDriveItem(delItem(folderID(), rootID, isFolder)) @@ -1871,13 +1949,13 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_MakeFolderCollectionPath( }{ { name: "root", - folder: driveRootFolder(), + folder: rootFolder(), expect: basePath.String(), expectErr: require.NoError, }, { name: "folder", - folder: d.folderAtRoot(), + folder: d.folderAt(root), expect: folderPath.String(), expectErr: require.NoError, }, @@ -1935,7 +2013,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file { name: "one file at root", tree: treeWithRoot, - page: aPage(d.fileAtRoot()), + page: aPage(d.fileAt(root)), expect: expected{ counts: countTD.Expected{ count.TotalDeleteFilesProcessed: 0, @@ -1954,8 +2032,8 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file name: "many files in a hierarchy", tree: treeWithRoot, page: aPage( - d.fileAtRoot(), - d.folderAtRoot(), + d.fileAt(root), + d.folderAt(root), d.fileAt(folder, "fchld")), expect: expected{ counts: countTD.Expected{ @@ -1976,7 +2054,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file name: "many updates to the same file", tree: treeWithRoot, page: aPage( - d.fileAtRoot(), + d.fileAt(root), driveItem(fileID(), fileName(1), d.dir(), rootID, isFile), driveItem(fileID(), fileName(2), d.dir(), rootID, isFile)), expect: expected{ @@ -2031,7 +2109,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file name: "create->delete", tree: treeWithRoot, page: aPage( - d.fileAtRoot(), + d.fileAt(root), delItem(fileID(), rootID, isFile)), expect: expected{ counts: countTD.Expected{ @@ -2049,7 +2127,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file name: "move->delete", tree: treeWithFileAtRoot, page: aPage( - d.folderAtRoot(), + d.folderAt(root), d.fileAt(folder), delItem(fileID(), folderID(), isFile)), expect: expected{ @@ -2069,7 +2147,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file tree: treeWithFileAtRoot, page: aPage( delItem(fileID(), rootID, isFile), - d.fileAtRoot()), + d.fileAt(root)), expect: expected{ counts: countTD.Expected{ count.TotalDeleteFilesProcessed: 1, @@ -2089,7 +2167,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file tree: treeWithRoot, page: aPage( delItem(fileID(), rootID, isFile), - d.fileAtRoot()), + d.fileAt(root)), expect: expected{ counts: countTD.Expected{ count.TotalDeleteFilesProcessed: 1, @@ -2140,10 +2218,18 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_EnumeratePageOfItems_file func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { d := drive() + unlimitedItemsPerContainer := newPagerLimiter(minimumLimitOpts()) + unlimitedItemsPerContainer.limits.MaxItemsPerContainer = 9001 + + unlimitedTotalBytesAndFiles := newPagerLimiter(minimumLimitOpts()) + unlimitedTotalBytesAndFiles.limits.MaxBytes = 9001 + unlimitedTotalBytesAndFiles.limits.MaxItems = 9001 + type expected struct { counts countTD.Expected err require.ErrorAssertionFunc shouldHitLimit bool + shouldHitCollLimit bool skipped assert.ValueAssertionFunc treeContainsFileIDsWithParent map[string]string countLiveFiles int @@ -2160,7 +2246,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { { name: "add new file", tree: treeWithRoot, - file: d.fileAtRoot(), + file: d.fileAt(root), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ counts: countTD.Expected{ @@ -2178,7 +2264,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { { name: "duplicate file", tree: treeWithFileAtRoot, - file: d.fileAtRoot(), + file: d.fileAt(root), limiter: newPagerLimiter(control.DefaultOptions()), expect: expected{ counts: countTD.Expected{ @@ -2260,8 +2346,45 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { { name: "already at container file limit", tree: treeWithFileAtRoot, - file: d.fileAtRoot(2), - limiter: newPagerLimiter(minimumLimitOpts()), + file: d.fileAt(root, 2), + limiter: unlimitedTotalBytesAndFiles, + expect: expected{ + counts: countTD.Expected{ + count.TotalFilesProcessed: 1, + }, + err: require.Error, + shouldHitCollLimit: true, + skipped: assert.Nil, + treeContainsFileIDsWithParent: map[string]string{ + fileID(): rootID, + }, + countLiveFiles: 1, + countTotalBytes: defaultFileSize, + }, + }, + { + name: "goes over total byte limit", + tree: treeWithRoot, + file: d.fileAt(root), + limiter: unlimitedItemsPerContainer, + expect: expected{ + counts: countTD.Expected{ + count.TotalFilesProcessed: 1, + }, + // no error here, since byte limit shouldn't + // make the func return an error. + err: require.NoError, + skipped: assert.Nil, + treeContainsFileIDsWithParent: map[string]string{}, + countLiveFiles: 0, + countTotalBytes: 0, + }, + }, + { + name: "already over total byte limit", + tree: treeWithFileAtRoot, + file: d.fileAt(root, 2), + limiter: unlimitedItemsPerContainer, expect: expected{ counts: countTD.Expected{ count.TotalFilesProcessed: 1, @@ -2276,23 +2399,6 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { countTotalBytes: defaultFileSize, }, }, - { - name: "goes over total byte limit", - tree: treeWithRoot, - file: d.fileAtRoot(), - limiter: newPagerLimiter(minimumLimitOpts()), - expect: expected{ - counts: countTD.Expected{ - count.TotalFilesProcessed: 1, - }, - err: require.Error, - shouldHitLimit: true, - skipped: assert.Nil, - treeContainsFileIDsWithParent: map[string]string{}, - countLiveFiles: 0, - countTotalBytes: 0, - }, - }, } for _, test := range table { suite.Run(test.name, func() { @@ -2318,6 +2424,10 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_AddFileToTree() { test.expect.err(t, err, clues.ToCore(err)) test.expect.skipped(t, skipped) + if test.expect.shouldHitCollLimit { + require.ErrorIs(t, err, errHitCollectionLimit, clues.ToCore(err)) + } + if test.expect.shouldHitLimit { require.ErrorIs(t, err, errHitLimit, clues.ToCore(err)) } diff --git a/src/internal/m365/collection/drive/delta_tree.go b/src/internal/m365/collection/drive/delta_tree.go index 9b8ea128e..03b359af6 100644 --- a/src/internal/m365/collection/drive/delta_tree.go +++ b/src/internal/m365/collection/drive/delta_tree.go @@ -2,8 +2,11 @@ package drive import ( "context" + "sort" + "strings" "github.com/alcionai/clues" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" @@ -78,33 +81,29 @@ type nodeyMcNodeFace struct { // required for mid-enumeration folder moves, else we have to walk // the tree completely to remove the node from its old parent. parent *nodeyMcNodeFace - // the microsoft item ID. Mostly because we might as well - // attach that to the node if we're also attaching the dir. - id string - // single directory name, not a path - name string + // folder is the actual drive item for this directory. + // we save this so that, during post-processing, it can + // get moved into the collection files, which will cause + // the collection processor to generate a permissions + // metadata file for the folder. + folder *custom.DriveItem // contains the complete previous path prev path.Path // folderID -> node children map[string]*nodeyMcNodeFace // file item ID -> file metadata files map[string]*custom.DriveItem - // for special handling protocols around packages - isPackage bool } func newNodeyMcNodeFace( parent *nodeyMcNodeFace, - id, name string, - isPackage bool, + folder *custom.DriveItem, ) *nodeyMcNodeFace { return &nodeyMcNodeFace{ - parent: parent, - id: id, - name: name, - children: map[string]*nodeyMcNodeFace{}, - files: map[string]*custom.DriveItem{}, - isPackage: isPackage, + parent: parent, + folder: folder, + children: map[string]*nodeyMcNodeFace{}, + files: map[string]*custom.DriveItem{}, } } @@ -134,9 +133,14 @@ func (face *folderyMcFolderFace) getNode(id string) *nodeyMcNodeFace { // values are updated to match (isPackage is assumed not to change). func (face *folderyMcFolderFace) setFolder( ctx context.Context, - parentID, id, name string, - isPackage bool, + folder *custom.DriveItem, ) error { + var ( + id = ptr.Val(folder.GetId()) + name = ptr.Val(folder.GetName()) + parentFolder = folder.GetParentReference() + ) + // need to ensure we have the minimum requirements met for adding a node. if len(id) == 0 { return clues.NewWC(ctx, "missing folder ID") @@ -146,16 +150,20 @@ func (face *folderyMcFolderFace) setFolder( return clues.NewWC(ctx, "missing folder name") } - if len(parentID) == 0 && id != face.rootID { + if (parentFolder == nil || len(ptr.Val(parentFolder.GetId())) == 0) && + id != face.rootID { return clues.NewWC(ctx, "non-root folder missing parent id") } // only set the root node once. if id == face.rootID { if face.root == nil { - root := newNodeyMcNodeFace(nil, id, name, isPackage) + root := newNodeyMcNodeFace(nil, folder) face.root = root face.folderIDToNode[id] = root + } else { + // but update the folder each time, to stay in sync with changes + face.root.folder = folder } return nil @@ -167,7 +175,7 @@ func (face *folderyMcFolderFace) setFolder( // 3. existing folder migrated to new location. // 4. tombstoned folder restored. - parent, ok := face.folderIDToNode[parentID] + parentNode, ok := face.folderIDToNode[ptr.Val(parentFolder.GetId())] if !ok { return clues.NewWC(ctx, "folder added before parent") } @@ -184,9 +192,9 @@ func (face *folderyMcFolderFace) setFolder( if zombey, tombstoned := face.tombstones[id]; tombstoned { delete(face.tombstones, id) - zombey.parent = parent - zombey.name = name - parent.children[id] = zombey + zombey.parent = parentNode + zombey.folder = folder + parentNode.children[id] = zombey face.folderIDToNode[id] = zombey return nil @@ -204,21 +212,21 @@ func (face *folderyMcFolderFace) setFolder( // technically shouldn't be possible but better to keep the problem tracked // just in case. logger.Ctx(ctx).Info("non-root folder already exists with no parent ref") - } else if nodey.parent != parent { + } else if nodey.parent != parentNode { // change type 3. we need to ensure the old parent stops pointing to this node. delete(nodey.parent.children, id) } - nodey.name = name - nodey.parent = parent + nodey.parent = parentNode + nodey.folder = folder } else { // change type 1: new addition - nodey = newNodeyMcNodeFace(parent, id, name, isPackage) + nodey = newNodeyMcNodeFace(parentNode, folder) } // ensure the parent points to this node, and that the node is registered // in the map of all nodes in the tree. - parent.children[id] = nodey + parentNode.children[id] = nodey face.folderIDToNode[id] = nodey return nil @@ -226,8 +234,10 @@ func (face *folderyMcFolderFace) setFolder( func (face *folderyMcFolderFace) setTombstone( ctx context.Context, - id string, + folder *custom.DriveItem, ) error { + id := ptr.Val(folder.GetId()) + if len(id) == 0 { return clues.NewWC(ctx, "missing tombstone folder ID") } @@ -254,7 +264,7 @@ func (face *folderyMcFolderFace) setTombstone( } if _, alreadyBuried := face.tombstones[id]; !alreadyBuried { - face.tombstones[id] = newNodeyMcNodeFace(nil, id, "", false) + face.tombstones[id] = newNodeyMcNodeFace(nil, folder) } return nil @@ -298,7 +308,7 @@ func (face *folderyMcFolderFace) setPreviousPath( return nil } - zombey := newNodeyMcNodeFace(nil, folderID, "", false) + zombey := newNodeyMcNodeFace(nil, custom.NewDriveItem(folderID, "")) zombey.prev = prev face.tombstones[folderID] = zombey @@ -318,13 +328,20 @@ func (face *folderyMcFolderFace) hasFile(id string) bool { // file was already added to the tree and is getting relocated, // this func will update and/or clean up all the old references. func (face *folderyMcFolderFace) addFile( - parentID, id string, file *custom.DriveItem, ) error { - if len(parentID) == 0 { + var ( + parentFolder = file.GetParentReference() + id = ptr.Val(file.GetId()) + parentID string + ) + + if parentFolder == nil || len(ptr.Val(parentFolder.GetId())) == 0 { return clues.New("item added without parent folder ID") } + parentID = ptr.Val(parentFolder.GetId()) + if len(id) == 0 { return clues.New("item added without ID") } @@ -419,17 +436,22 @@ func (face *folderyMcFolderFace) walkTreeAndBuildCollections( return nil } - isRoot := node == face.root + var ( + id = ptr.Val(node.folder.GetId()) + name = ptr.Val(node.folder.GetName()) + isPackage = node.folder.GetPackageEscaped() != nil + isRoot = node == face.root + ) if !isRoot { - location = location.Append(node.name) + location = location.Append(name) } for _, child := range node.children { err := face.walkTreeAndBuildCollections( child, location, - node.isPackage || isChildOfPackage, + isPackage || isChildOfPackage, result) if err != nil { return err @@ -444,19 +466,134 @@ func (face *folderyMcFolderFace) walkTreeAndBuildCollections( "path_suffix", location.Elements()) } + files := node.files + + if !isRoot { + // add the folder itself to the list of files inside the folder. + // that will cause the collection processor to generate a metadata + // file to hold the folder's permissions. + files = maps.Clone(node.files) + files[id] = node.folder + } + cbl := collectable{ currPath: collectionPath, - files: node.files, - folderID: node.id, - isPackageOrChildOfPackage: node.isPackage || isChildOfPackage, + files: files, + folderID: id, + isPackageOrChildOfPackage: isPackage || isChildOfPackage, prevPath: node.prev, } - result[node.id] = cbl + result[id] = cbl return nil } +type idPrevPathTup struct { + id string + prevPath string +} + +// fuses the collectables and old prevPaths into a +// new prevPaths map. +func (face *folderyMcFolderFace) generateNewPreviousPaths( + collectables map[string]collectable, + prevPaths map[string]string, +) (map[string]string, error) { + var ( + // id -> currentPath + results = map[string]string{} + // prevPath -> currentPath + movedPaths = map[string]string{} + // prevPath -> {} + tombstoned = map[string]struct{}{} + ) + + // first, move all collectables into the new maps + + for id, cbl := range collectables { + if cbl.currPath == nil { + tombstoned[cbl.prevPath.String()] = struct{}{} + continue + } + + cp := cbl.currPath.String() + results[id] = cp + + if cbl.prevPath != nil && cbl.prevPath.String() != cp { + movedPaths[cbl.prevPath.String()] = cp + } + } + + // next, create a slice of tuples representing any + // old prevPath entry whose ID isn't already bound to + // a collectable. + + unseenPrevPaths := []idPrevPathTup{} + + for id, p := range prevPaths { + // if the current folder was tombstoned, skip it + if _, ok := tombstoned[p]; ok { + continue + } + + if _, ok := results[id]; !ok { + unseenPrevPaths = append(unseenPrevPaths, idPrevPathTup{id, p}) + } + } + + // sort the slice by path, ascending. + // This ensures we work from root to leaf when replacing prefixes, + // and thus we won't need to walk every unseen path from leaf to + // root looking for a matching prefix. + + sortByLeastPath := func(i, j int) bool { + return unseenPrevPaths[i].prevPath < unseenPrevPaths[j].prevPath + } + + sort.Slice(unseenPrevPaths, sortByLeastPath) + + for _, un := range unseenPrevPaths { + elems := path.NewElements(un.prevPath) + + pb, err := path.Builder{}.UnescapeAndAppend(elems...) + if err != nil { + return nil, err + } + + parent := pb.Dir().String() + + // if the parent was tombstoned, add this prevPath entry to the + // tombstoned map; that'll allow the tombstone identification to + // cascade to children, and it won't get added to the results. + if _, ok := tombstoned[parent]; ok { + tombstoned[un.prevPath] = struct{}{} + continue + } + + // if the parent wasn't moved, add the same path to the result set + parentCurrentPath, ok := movedPaths[parent] + if !ok { + results[un.id] = un.prevPath + continue + } + + // if the parent was moved, replace the prefix and + // add it to the result set + // TODO: should probably use path.UpdateParent for this. + // but I want the quality-of-life of feeding it strings + // instead of parsing strings to paths here first. + newPath := strings.Replace(un.prevPath, parent, parentCurrentPath, 1) + + results[un.id] = newPath + + // add the current string to the moved list, that'll allow it to cascade to all children. + movedPaths[un.prevPath] = newPath + } + + return results, nil +} + func (face *folderyMcFolderFace) generateExcludeItemIDs() map[string]struct{} { result := map[string]struct{}{} diff --git a/src/internal/m365/collection/drive/delta_tree_test.go b/src/internal/m365/collection/drive/delta_tree_test.go index 5177a57f1..ac69d27db 100644 --- a/src/internal/m365/collection/drive/delta_tree_test.go +++ b/src/internal/m365/collection/drive/delta_tree_test.go @@ -4,11 +4,13 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/custom" @@ -46,14 +48,16 @@ func (suite *DeltaTreeUnitSuite) TestNewNodeyMcNodeFace() { var ( t = suite.T() parent = &nodeyMcNodeFace{} + d = drive() + fld = custom.ToCustomDriveItem(d.folderAt(root)) ) - nodeFace := newNodeyMcNodeFace(parent, "id", "name", true) + nodeFace := newNodeyMcNodeFace(parent, fld) assert.Equal(t, parent, nodeFace.parent) - assert.Equal(t, "id", nodeFace.id) - assert.Equal(t, "name", nodeFace.name) + assert.Equal(t, folderID(), ptr.Val(nodeFace.folder.GetId())) + assert.Equal(t, folderName(), ptr.Val(nodeFace.folder.GetName())) assert.Nil(t, nodeFace.prev) - assert.True(t, nodeFace.isPackage) + assert.NotNil(t, nodeFace.folder.GetParentReference()) assert.NotNil(t, nodeFace.children) assert.NotNil(t, nodeFace.files) } @@ -65,98 +69,101 @@ func (suite *DeltaTreeUnitSuite) TestNewNodeyMcNodeFace() { // note that this test is focused on the SetFolder function, // and intentionally does not verify the resulting node tree func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder() { + d := drive() + table := []struct { tname string tree func(t *testing.T, d *deltaDrive) *folderyMcFolderFace - parentID string - id string - name string - isPackage bool + folder func() *custom.DriveItem expectErr assert.ErrorAssertionFunc }{ { - tname: "add root", - tree: newTree, - id: rootID, - name: rootName, - isPackage: true, + tname: "add root", + tree: newTree, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(rootFolder()) + }, expectErr: assert.NoError, }, { - tname: "root already exists", - tree: treeWithRoot, - id: rootID, - name: rootName, + tname: "root already exists", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(rootFolder()) + }, expectErr: assert.NoError, }, { - tname: "add folder", - tree: treeWithRoot, - parentID: rootID, - id: folderID(), - name: folderName(), + tname: "add folder", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(d.folderAt(root)) + }, expectErr: assert.NoError, }, { - tname: "add package", - tree: treeWithRoot, - parentID: rootID, - id: folderID(), - name: folderName(), - isPackage: true, + tname: "add package", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(d.packageAtRoot()) + }, expectErr: assert.NoError, }, { - tname: "missing ID", - tree: treeWithRoot, - parentID: rootID, - name: folderName(), - isPackage: true, + tname: "missing ID", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + far := d.folderAt(root) + far.SetId(nil) + + return custom.ToCustomDriveItem(far) + }, expectErr: assert.Error, }, { - tname: "missing name", - tree: treeWithRoot, - parentID: rootID, - id: folderID(), - isPackage: true, + tname: "missing name", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + far := d.folderAt(root) + far.SetName(nil) + + return custom.ToCustomDriveItem(far) + }, expectErr: assert.Error, }, { - tname: "missing parentID", - tree: treeWithRoot, - id: folderID(), - name: folderName(), - isPackage: true, + tname: "missing parent", + tree: treeWithRoot, + folder: func() *custom.DriveItem { + far := d.folderAt(root) + far.SetParentReference(nil) + + return custom.ToCustomDriveItem(far) + }, expectErr: assert.Error, }, { - tname: "already tombstoned", - tree: treeWithTombstone, - parentID: rootID, - id: folderID(), - name: folderName(), + tname: "already tombstoned", + tree: treeWithTombstone, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(d.folderAt(root)) + }, expectErr: assert.NoError, }, { tname: "add folder before parent", - tree: func(t *testing.T, d *deltaDrive) *folderyMcFolderFace { - return &folderyMcFolderFace{ - folderIDToNode: map[string]*nodeyMcNodeFace{}, - } + tree: newTree, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(d.folderAt(root)) }, - parentID: rootID, - id: folderID(), - name: folderName(), - isPackage: true, expectErr: assert.Error, }, { - tname: "folder already exists", - tree: treeWithFolders, - parentID: folderID("parent"), - id: folderID(), - name: folderName(), + tname: "folder already exists", + tree: treeWithFolders, + folder: func() *custom.DriveItem { + return custom.ToCustomDriveItem(d.folderAt(root)) + }, expectErr: assert.NoError, }, } @@ -168,30 +175,37 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder() { defer flush() tree := test.tree(t, drive()) + folder := test.folder() - err := tree.setFolder( - ctx, - test.parentID, - test.id, - test.name, - test.isPackage) + err := tree.setFolder(ctx, folder) test.expectErr(t, err, clues.ToCore(err)) if err != nil { return } - result := tree.folderIDToNode[test.id] + result := tree.folderIDToNode[ptr.Val(folder.GetId())] require.NotNil(t, result) - assert.Equal(t, test.id, result.id) - assert.Equal(t, test.name, result.name) - assert.Equal(t, test.isPackage, result.isPackage) - _, ded := tree.tombstones[test.id] + var ( + expectID = ptr.Val(folder.GetId()) + expectName = ptr.Val(folder.GetName()) + expectIsPackage = folder.GetPackageEscaped() == nil + resultID = ptr.Val(result.folder.GetId()) + resultName = ptr.Val(result.folder.GetName()) + resultIsPackage = result.folder.GetPackageEscaped() == nil + ) + + assert.Equal(t, expectID, resultID) + assert.Equal(t, expectName, resultName) + assert.Equal(t, expectIsPackage, resultIsPackage) + + _, ded := tree.tombstones[expectID] assert.False(t, ded) - if len(test.parentID) > 0 { - parent := tree.folderIDToNode[test.parentID] + if folder.GetParentReference() != nil { + expectParentID := ptr.Val(folder.GetParentReference().GetId()) + parent := tree.folderIDToNode[expectParentID] assert.Equal(t, parent, result.parent) } }) @@ -236,9 +250,11 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddTombstone() { ctx, flush := tester.NewContext(t) defer flush() - tree := test.tree(t, drive()) + d := drive() + tree := test.tree(t, d) + tomb := delItem(test.id, rootID, isFolder) - err := tree.setTombstone(ctx, test.id) + err := tree.setTombstone(ctx, custom.ToCustomDriveItem(tomb)) test.expectErr(t, err, clues.ToCore(err)) if err != nil { @@ -381,7 +397,7 @@ func (an assertNode) compare( ) { var nodeCount int - t.Run("assert_tree_shape/root", func(_t *testing.T) { + t.Run("assert_tree_shape-root", func(_t *testing.T) { nodeCount = compareNodes(_t, tree.root, an) }) @@ -413,13 +429,15 @@ func compareNodes( var nodeCount int for _, expectChild := range expect.children { - t.Run(expectChild.self.id, func(_t *testing.T) { - nodeChild := node.children[expectChild.self.id] + expectID := ptr.Val(expectChild.self.folder.GetId()) + + t.Run(expectID, func(_t *testing.T) { + nodeChild := node.children[expectID] require.NotNilf( _t, nodeChild, "child must exist with id %q", - expectChild.self.id) + expectID) // ensure each child points to the current node as its parent assert.Equal( @@ -453,11 +471,15 @@ func (ts tombs) compare( require.Len(t, tombstones, len(ts), "count of tombstoned nodes") for _, entombed := range ts { - zombey := tombstones[entombed.self.id] + expectID := ptr.Val(entombed.self.folder.GetId()) + + zombey := tombstones[expectID] require.NotNil(t, zombey, "tombstone must exist") assert.Nil(t, zombey.parent, "tombstoned node should not have a parent reference") - t.Run("assert_tombstones/"+zombey.id, func(_t *testing.T) { + resultID := ptr.Val(zombey.folder.GetId()) + + t.Run("assert_tombstones-"+resultID, func(_t *testing.T) { compareNodes(_t, zombey, entombed) }) } @@ -471,37 +493,54 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTree() ctx, flush := tester.NewContext(t) defer flush() - tree := treeWithRoot(t, drive()) + d := drive() + tree := treeWithRoot(t, d) - set := func( - parentID, fid, fname string, - isPackage bool, - ) { - err := tree.setFolder(ctx, parentID, fid, fname, isPackage) + set := func(folder *custom.DriveItem) { + err := tree.setFolder(ctx, folder) require.NoError(t, err, clues.ToCore(err)) } + idOf := func(node *nodeyMcNodeFace) string { + return ptr.Val(node.folder.GetId()) + } + + customFolder := func(parent, self string) *custom.DriveItem { + var di models.DriveItemable + + if parent == rootID { + di = d.folderAt(root, self) + } else { + di = driveFolder( + d.dir("doesn't matter for this test"), + folderID(parent), + self) + } + + return custom.ToCustomDriveItem(di) + } + // assert the root exists assert.NotNil(t, tree.root) - assert.Equal(t, rootID, tree.root.id) - assert.Equal(t, rootID, tree.folderIDToNode[rootID].id) + assert.Equal(t, rootID, idOf(tree.root)) + assert.Equal(t, rootID, idOf(tree.folderIDToNode[rootID])) an(tree.root).compare(t, tree, true) // add a child at the root - set(rootID, id("lefty"), name("l"), false) + set(customFolder(rootID, "lefty")) - lefty := tree.folderIDToNode[id("lefty")] + lefty := tree.folderIDToNode[folderID("lefty")] an( tree.root, an(lefty)). compare(t, tree, true) // add another child at the root - set(rootID, id("righty"), name("r"), false) + set(customFolder(rootID, "righty")) - righty := tree.folderIDToNode[id("righty")] + righty := tree.folderIDToNode[folderID("righty")] an( tree.root, an(lefty), @@ -509,9 +548,9 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTree() compare(t, tree, true) // add a child to lefty - set(lefty.id, id("bloaty"), name("bl"), false) + set(customFolder("lefty", "bloaty")) - bloaty := tree.folderIDToNode[id("bloaty")] + bloaty := tree.folderIDToNode[folderID("bloaty")] an( tree.root, an(lefty, an(bloaty)), @@ -519,9 +558,9 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTree() compare(t, tree, true) // add another child to lefty - set(lefty.id, id("brightly"), name("br"), false) + set(customFolder("lefty", "brightly")) - brightly := tree.folderIDToNode[id("brightly")] + brightly := tree.folderIDToNode[folderID("brightly")] an( tree.root, an(lefty, an(bloaty), an(brightly)), @@ -529,7 +568,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTree() compare(t, tree, true) // relocate brightly underneath righty - set(righty.id, brightly.id, brightly.name, false) + set(customFolder("righty", "brightly")) an( tree.root, @@ -538,7 +577,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTree() compare(t, tree, true) // relocate righty and subtree beneath lefty - set(lefty.id, righty.id, righty.name, false) + set(customFolder("lefty", "righty")) an( tree.root, @@ -557,34 +596,45 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst ctx, flush := tester.NewContext(t) defer flush() - tree := treeWithRoot(t, drive()) + d := drive() + tree := treeWithRoot(t, d) - set := func( - parentID, fid, fname string, - isPackage bool, - ) { - err := tree.setFolder(ctx, parentID, fid, fname, isPackage) + set := func(folder *custom.DriveItem) { + err := tree.setFolder(ctx, folder) require.NoError(t, err, clues.ToCore(err)) } - tomb := func( - tid string, - loc path.Elements, - ) { - err := tree.setTombstone(ctx, tid) + customFolder := func(parent, self string) *custom.DriveItem { + var di models.DriveItemable + + if parent == rootID { + di = d.folderAt(root, self) + } else { + di = driveFolder( + d.dir("doesn't matter for this test"), + folderID(parent), + self) + } + + return custom.ToCustomDriveItem(di) + } + + tomb := func(folder *custom.DriveItem) { + err := tree.setTombstone(ctx, folder) require.NoError(t, err, clues.ToCore(err)) } // create a simple tree // root > branchy > [leafy, bob] - set(tree.root.id, id("branchy"), name("br"), false) - branchy := tree.folderIDToNode[id("branchy")] + set(customFolder(rootID, "branchy")) - set(branchy.id, id("leafy"), name("l"), false) - set(branchy.id, id("bob"), name("bobbers"), false) + branchy := tree.folderIDToNode[folderID("branchy")] - leafy := tree.folderIDToNode[id("leafy")] - bob := tree.folderIDToNode[id("bob")] + set(customFolder("branchy", "leafy")) + set(customFolder("branchy", "bob")) + + leafy := tree.folderIDToNode[folderID("leafy")] + bob := tree.folderIDToNode[folderID("bob")] an( tree.root, @@ -596,14 +646,8 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb().compare(t, tree.tombstones) - var ( - branchyLoc = path.NewElements("root/branchy") - leafyLoc = path.NewElements("root/branchy/leafy") - bobLoc = path.NewElements("root/branchy/bob") - ) - // tombstone bob - tomb(bob.id, bobLoc) + tomb(customFolder("branchy", "bob")) an( tree.root, an(branchy, an(leafy))). @@ -612,7 +656,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb(an(bob)).compare(t, tree.tombstones) // tombstone leafy - tomb(leafy.id, leafyLoc) + tomb(customFolder("branchy", "leafy")) an( tree.root, an(branchy)). @@ -621,7 +665,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb(an(bob), an(leafy)).compare(t, tree.tombstones) // resurrect leafy - set(branchy.id, leafy.id, leafy.name, false) + set(customFolder("branchy", "leafy")) an( tree.root, @@ -631,7 +675,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb(an(bob)).compare(t, tree.tombstones) // resurrect bob - set(branchy.id, bob.id, bob.name, false) + set(customFolder("branchy", "bob")) an( tree.root, @@ -644,7 +688,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb().compare(t, tree.tombstones) // tombstone branchy - tomb(branchy.id, branchyLoc) + tomb(customFolder(rootID, "branchy")) an(tree.root).compare(t, tree, false) // note: the folder count here *will be wrong*. @@ -667,7 +711,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst compare(t, tree.tombstones) // resurrect branchy - set(tree.root.id, branchy.id, branchy.name, false) + set(customFolder(rootID, "branchy")) an( tree.root, @@ -680,7 +724,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb().compare(t, tree.tombstones) // tombstone branchy - tomb(branchy.id, branchyLoc) + tomb(customFolder(rootID, "branchy")) an(tree.root).compare(t, tree, false) @@ -692,7 +736,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst compare(t, tree.tombstones) // tombstone bob - tomb(bob.id, bobLoc) + tomb(customFolder("branchy", "bob")) an(tree.root).compare(t, tree, false) @@ -702,7 +746,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst compare(t, tree.tombstones) // resurrect branchy - set(tree.root.id, branchy.id, branchy.name, false) + set(customFolder(rootID, "branchy")) an( tree.root, @@ -712,7 +756,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_SetFolder_correctTombst entomb(an(bob)).compare(t, tree.tombstones) // resurrect bob - set(branchy.id, bob.id, bob.name, false) + set(customFolder("branchy", "bob")) an( tree.root, @@ -735,7 +779,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree func(t *testing.T, d *deltaDrive) *folderyMcFolderFace id string oldParentID string - parentID string + parent any contentSize int64 expectErr assert.ErrorAssertionFunc expectFiles map[string]string @@ -745,7 +789,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithRoot, id: fileID(), oldParentID: "", - parentID: rootID, + parent: root, contentSize: defaultFileSize, expectErr: assert.NoError, expectFiles: map[string]string{fileID(): rootID}, @@ -755,7 +799,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithFolders, id: fileID(), oldParentID: "", - parentID: folderID(), + parent: folder, contentSize: 24, expectErr: assert.NoError, expectFiles: map[string]string{fileID(): folderID()}, @@ -765,7 +809,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithFileAtRoot, id: fileID(), oldParentID: rootID, - parentID: rootID, + parent: root, contentSize: 84, expectErr: assert.NoError, expectFiles: map[string]string{fileID(): rootID}, @@ -775,7 +819,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithFileInFolder, id: fileID(), oldParentID: folderID(), - parentID: rootID, + parent: root, contentSize: 48, expectErr: assert.NoError, expectFiles: map[string]string{fileID(): rootID}, @@ -785,7 +829,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithFileInTombstone, id: fileID(), oldParentID: folderID(), - parentID: rootID, + parent: root, contentSize: 2, expectErr: assert.NoError, expectFiles: map[string]string{fileID(): rootID}, @@ -795,7 +839,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithTombstone, id: "", oldParentID: "", - parentID: folderID(), + parent: folder, contentSize: 4, expectErr: assert.Error, expectFiles: map[string]string{}, @@ -805,7 +849,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithTombstone, id: fileID(), oldParentID: "", - parentID: folderID(), + parent: folder, contentSize: 8, expectErr: assert.Error, expectFiles: map[string]string{}, @@ -815,7 +859,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithTombstone, id: fileID(), oldParentID: "", - parentID: folderID("not-in-tree"), + parent: "not-in-tree", contentSize: 16, expectErr: assert.Error, expectFiles: map[string]string{}, @@ -825,7 +869,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { tree: treeWithTombstone, id: fileID(), oldParentID: "", - parentID: "", + parent: nil, contentSize: 16, expectErr: assert.Error, expectFiles: map[string]string{}, @@ -833,14 +877,14 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { } for _, test := range table { suite.Run(test.tname, func() { - t := suite.T() - d := drive() - tree := test.tree(t, d) + var ( + t = suite.T() + d = drive() + tree = test.tree(t, d) + df = custom.ToCustomDriveItem(d.fileWSizeAt(test.contentSize, test.parent)) + ) - err := tree.addFile( - test.parentID, - test.id, - custom.ToCustomDriveItem(d.fileWSizeAt(test.contentSize, test.parentID))) + err := tree.addFile(df) test.expectErr(t, err, clues.ToCore(err)) assert.Equal(t, test.expectFiles, tree.fileIDToParentID) @@ -848,7 +892,12 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { return } - parent := tree.getNode(test.parentID) + parentID := folderID(test.parent) + if test.parent == root { + parentID = rootID + } + + parent := tree.getNode(parentID) require.NotNil(t, parent) assert.Contains(t, parent.files, fileID()) @@ -857,7 +906,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_AddFile() { assert.Equal(t, 1, countSize.numFiles, "should have one file in the tree") assert.Equal(t, test.contentSize, countSize.totalBytes, "tree should be sized to test file contents") - if len(test.oldParentID) > 0 && test.oldParentID != test.parentID { + if len(test.oldParentID) > 0 && test.oldParentID != parentID { old := tree.getNode(test.oldParentID) require.NotNil(t, old) @@ -912,10 +961,12 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_DeleteFile() { } func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_addAndDeleteFile() { - t := suite.T() - d := drive() - tree := treeWithRoot(t, d) - fID := id(file) + var ( + t = suite.T() + d = drive() + tree = treeWithRoot(t, d) + fID = fileID() + ) require.Len(t, tree.fileIDToParentID, 0) require.Len(t, tree.deletedFileIDs, 0) @@ -927,7 +978,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_addAndDeleteFile() { assert.Len(t, tree.deletedFileIDs, 1) assert.Contains(t, tree.deletedFileIDs, fID) - err := tree.addFile(rootID, fID, custom.ToCustomDriveItem(d.fileAtRoot())) + err := tree.addFile(custom.ToCustomDriveItem(d.fileAt(root))) require.NoError(t, err, clues.ToCore(err)) assert.Len(t, tree.fileIDToParentID, 1) @@ -973,9 +1024,9 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateExcludeItemIDs( name: "files in folders and tombstones", tree: fullTree, expect: makeExcludeMap( - fileID(), fileID("r"), fileID("p"), + fileID("f"), fileID("d")), }, } @@ -1032,7 +1083,7 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() rootID: { currPath: d.fullPath(t), files: map[string]*custom.DriveItem{ - fileID(): custom.ToCustomDriveItem(d.fileAtRoot()), + fileID(): custom.ToCustomDriveItem(d.fileAt(root)), }, folderID: rootID, isPackageOrChildOfPackage: false, @@ -1051,15 +1102,18 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() isPackageOrChildOfPackage: false, }, folderID("parent"): { - currPath: d.fullPath(t, folderName("parent")), - files: map[string]*custom.DriveItem{}, + currPath: d.fullPath(t, folderName("parent")), + files: map[string]*custom.DriveItem{ + folderID("parent"): custom.ToCustomDriveItem(d.folderAt(root)), + }, folderID: folderID("parent"), isPackageOrChildOfPackage: false, }, folderID(): { currPath: d.fullPath(t, folderName("parent"), folderName()), files: map[string]*custom.DriveItem{ - fileID(): custom.ToCustomDriveItem(d.fileAt("parent")), + folderID(): custom.ToCustomDriveItem(d.folderAt("parent")), + fileID(): custom.ToCustomDriveItem(d.fileAt(folder)), }, folderID: folderID(), isPackageOrChildOfPackage: false, @@ -1073,10 +1127,10 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() defer flush() tree := treeWithRoot(t, d) - err := tree.setFolder(ctx, rootID, id(pkg), name(pkg), true) + err := tree.setFolder(ctx, custom.ToCustomDriveItem(d.packageAtRoot())) require.NoError(t, err, clues.ToCore(err)) - err = tree.setFolder(ctx, id(pkg), folderID(), folderName(), false) + err = tree.setFolder(ctx, custom.ToCustomDriveItem(d.folderAt(pkg))) require.NoError(t, err, clues.ToCore(err)) return tree @@ -1089,15 +1143,19 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() folderID: rootID, isPackageOrChildOfPackage: false, }, - id(pkg): { - currPath: d.fullPath(t, name(pkg)), - files: map[string]*custom.DriveItem{}, - folderID: id(pkg), + folderID(pkg): { + currPath: d.fullPath(t, folderName(pkg)), + files: map[string]*custom.DriveItem{ + folderID(pkg): custom.ToCustomDriveItem(d.packageAtRoot()), + }, + folderID: folderID(pkg), isPackageOrChildOfPackage: true, }, folderID(): { - currPath: d.fullPath(t, name(pkg), folderName()), - files: map[string]*custom.DriveItem{}, + currPath: d.fullPath(t, folderName(pkg), folderName()), + files: map[string]*custom.DriveItem{ + folderID(): custom.ToCustomDriveItem(d.folderAt("parent")), + }, folderID: folderID(), isPackageOrChildOfPackage: true, }, @@ -1108,9 +1166,9 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() tree: treeWithFileInFolder, expectErr: require.NoError, prevPaths: map[string]string{ - rootID: d.strPath(), - folderID("parent"): d.strPath(folderName("parent-prev")), - folderID(): d.strPath(folderName("parent-prev"), folderName()), + rootID: d.strPath(t), + folderID("parent"): d.strPath(t, folderName("parent-prev")), + folderID(): d.strPath(t, folderName("parent-prev"), folderName()), }, expect: map[string]collectable{ rootID: { @@ -1121,8 +1179,10 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() prevPath: d.fullPath(t), }, folderID("parent"): { - currPath: d.fullPath(t, folderName("parent")), - files: map[string]*custom.DriveItem{}, + currPath: d.fullPath(t, folderName("parent")), + files: map[string]*custom.DriveItem{ + folderID("parent"): custom.ToCustomDriveItem(d.folderAt(root)), + }, folderID: folderID("parent"), isPackageOrChildOfPackage: false, prevPath: d.fullPath(t, folderName("parent-prev")), @@ -1132,7 +1192,8 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() folderID: folderID(), isPackageOrChildOfPackage: false, files: map[string]*custom.DriveItem{ - fileID(): custom.ToCustomDriveItem(d.fileAt("parent")), + folderID(): custom.ToCustomDriveItem(d.folderAt("parent")), + fileID(): custom.ToCustomDriveItem(d.fileAt(folder)), }, prevPath: d.fullPath(t, folderName("parent-prev"), folderName()), }, @@ -1142,8 +1203,8 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() name: "root and tombstones", tree: treeWithFileInTombstone, prevPaths: map[string]string{ - rootID: d.strPath(), - folderID(): d.strPath(folderName()), + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), }, expectErr: require.NoError, expect: map[string]collectable{ @@ -1205,3 +1266,231 @@ func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateCollectables() }) } } + +func (suite *DeltaTreeUnitSuite) TestFolderyMcFolderFace_GenerateNewPreviousPaths() { + t := suite.T() + d := drive() + + table := []struct { + name string + collectables map[string]collectable + prevPaths map[string]string + expect map[string]string + }{ + { + name: "empty collectables, empty prev paths", + collectables: map[string]collectable{}, + prevPaths: map[string]string{}, + expect: map[string]string{}, + }, + { + name: "empty collectables", + collectables: map[string]collectable{}, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + }, + }, + { + name: "empty prev paths", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): {currPath: d.fullPath(t, folderName())}, + }, + prevPaths: map[string]string{}, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + }, + }, + { + name: "collectables replace old prev as new location", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): { + prevPath: d.fullPath(t, folderName("old")), + currPath: d.fullPath(t, folderName()), + }, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("old")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + }, + }, + { + name: "children of parents not moved maintain location", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): { + prevPath: d.fullPath(t, folderName()), + currPath: d.fullPath(t, folderName()), + }, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID("c1"): d.strPath(t, folderName(), folderName("c1")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID("c1"): d.strPath(t, folderName(), folderName("c1")), + }, + }, + { + name: "updates cascade to unseen children", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): { + prevPath: d.fullPath(t, folderName("old")), + currPath: d.fullPath(t, folderName()), + }, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("old")), + folderID("c1"): d.strPath(t, folderName("old"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("old"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("old"), folderName("c2"), folderName("c3")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID("c1"): d.strPath(t, folderName(), folderName("c1")), + folderID("c2"): d.strPath(t, folderName(), folderName("c2")), + folderID("c3"): d.strPath(t, folderName(), folderName("c2"), folderName("c3")), + }, + }, + { + name: "updates cascade to unseen children - escaped path separator", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): { + prevPath: d.fullPath(t, folderName("o/ld")), + currPath: d.fullPath(t, folderName("n/ew")), + }, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("o/ld")), + folderID("c1"): d.strPath(t, folderName("o/ld"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("o/ld"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("o/ld"), folderName("c2"), folderName("c3")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("n/ew")), + folderID("c1"): d.strPath(t, folderName("n/ew"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("n/ew"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("n/ew"), folderName("c2"), folderName("c3")), + }, + }, + { + name: "tombstoned directories get removed", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): {prevPath: d.fullPath(t, folderName("old"))}, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("old")), + folderID("c1"): d.strPath(t, folderName("old"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("old"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("old"), folderName("c2"), folderName("c3")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + }, + }, + { + name: "mix of moved and tombstoned", + collectables: map[string]collectable{ + rootID: {currPath: d.fullPath(t)}, + folderID(): { + prevPath: d.fullPath(t, folderName("old")), + currPath: d.fullPath(t, folderName()), + }, + folderID("c3"): {prevPath: d.fullPath(t, folderName("old"), folderName("c2"), folderName("c3"))}, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("old")), + folderID("c1"): d.strPath(t, folderName("old"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("old"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("old"), folderName("c2"), folderName("c3")), + folderID("c4"): d.strPath(t, folderName("old"), folderName("c2"), folderName("c3"), folderName("c4")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName()), + folderID("c1"): d.strPath(t, folderName(), folderName("c1")), + folderID("c2"): d.strPath(t, folderName(), folderName("c2")), + }, + }, + { + // tests the equivalent of: + // mv root:/foo -> root:/bar + // mkdir root:/foo + // mkdir root:/foo/c1 + // mv root:/bar/c1/c2 -> root:/foo/c1/c2 + name: "moved and replaced with same name", + collectables: map[string]collectable{ + rootID: { + prevPath: d.fullPath(t), + currPath: d.fullPath(t), + }, + folderID(): { + prevPath: d.fullPath(t, folderName("foo")), + currPath: d.fullPath(t, folderName("bar")), + }, + folderID(2): { + currPath: d.fullPath(t, folderName("foo")), + }, + folderID("2c1"): { + currPath: d.fullPath(t, folderName("foo"), folderName("c1")), + }, + folderID("c2"): { + prevPath: d.fullPath(t, folderName("foo"), folderName("c1"), folderName("c2")), + currPath: d.fullPath(t, folderName("foo"), folderName("c1"), folderName("c2")), + }, + }, + prevPaths: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("foo")), + folderID("c1"): d.strPath(t, folderName("foo"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("foo"), folderName("c1"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("foo"), folderName("c1"), folderName("c2"), folderName("c3")), + }, + expect: map[string]string{ + rootID: d.strPath(t), + folderID(): d.strPath(t, folderName("bar")), + folderID("c1"): d.strPath(t, folderName("bar"), folderName("c1")), + folderID(2): d.strPath(t, folderName("foo")), + folderID("2c1"): d.strPath(t, folderName("foo"), folderName("c1")), + folderID("c2"): d.strPath(t, folderName("foo"), folderName("c1"), folderName("c2")), + folderID("c3"): d.strPath(t, folderName("foo"), folderName("c1"), folderName("c2"), folderName("c3")), + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + tree := newTree(t, d) + + results, err := tree.generateNewPreviousPaths( + test.collectables, + test.prevPaths) + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, test.expect, results) + }) + } +} diff --git a/src/internal/m365/collection/drive/helper_test.go b/src/internal/m365/collection/drive/helper_test.go index dc04127b5..1782916cf 100644 --- a/src/internal/m365/collection/drive/helper_test.go +++ b/src/internal/m365/collection/drive/helper_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strings" "testing" "time" @@ -214,13 +215,13 @@ func collWithMBHAndOpts( func aPage(items ...models.DriveItemable) nextPage { return nextPage{ - Items: append([]models.DriveItemable{driveRootFolder()}, items...), + Items: append([]models.DriveItemable{rootFolder()}, items...), } } func aPageWReset(items ...models.DriveItemable) nextPage { return nextPage{ - Items: append([]models.DriveItemable{driveRootFolder()}, items...), + Items: append([]models.DriveItemable{rootFolder()}, items...), Reset: true, } } @@ -310,9 +311,15 @@ func aColl( ) *collectionAssertion { ids := make([]string, 0, 2*len(fileIDs)) - for _, fUD := range fileIDs { - ids = append(ids, fUD+metadata.DataFileSuffix) - ids = append(ids, fUD+metadata.MetaFileSuffix) + for _, fID := range fileIDs { + ids = append(ids, fID+metadata.DataFileSuffix) + ids = append(ids, fID+metadata.MetaFileSuffix) + } + + // should expect all non-root, non-tombstone collections to contain + // a dir meta file for storing permissions. + if curr != nil && !strings.HasSuffix(curr.Folder(false), root) { + ids = append(ids, metadata.DirMetaFileSuffix) } return &collectionAssertion{ @@ -453,9 +460,10 @@ func newTree(t *testing.T, d *deltaDrive) *folderyMcFolderFace { func treeWithRoot(t *testing.T, d *deltaDrive) *folderyMcFolderFace { tree := newFolderyMcFolderFace(defaultTreePfx(t, d), rootID) + root := custom.ToCustomDriveItem(rootFolder()) //nolint:forbidigo - err := tree.setFolder(context.Background(), "", rootID, rootName, false) + err := tree.setFolder(context.Background(), root) require.NoError(t, err, clues.ToCore(err)) return tree @@ -477,9 +485,10 @@ func treeWithFoldersAfterReset(t *testing.T, d *deltaDrive) *folderyMcFolderFace func treeWithTombstone(t *testing.T, d *deltaDrive) *folderyMcFolderFace { tree := treeWithRoot(t, d) + folder := custom.ToCustomDriveItem(d.folderAt(root)) //nolint:forbidigo - err := tree.setTombstone(context.Background(), folderID()) + err := tree.setTombstone(context.Background(), folder) require.NoError(t, err, clues.ToCore(err)) return tree @@ -487,13 +496,15 @@ func treeWithTombstone(t *testing.T, d *deltaDrive) *folderyMcFolderFace { func treeWithFolders(t *testing.T, d *deltaDrive) *folderyMcFolderFace { tree := treeWithRoot(t, d) + parent := custom.ToCustomDriveItem(d.folderAt(root, "parent")) + folder := custom.ToCustomDriveItem(d.folderAt("parent")) //nolint:forbidigo - err := tree.setFolder(context.Background(), rootID, folderID("parent"), folderName("parent"), true) + err := tree.setFolder(context.Background(), parent) require.NoError(t, err, clues.ToCore(err)) //nolint:forbidigo - err = tree.setFolder(context.Background(), folderID("parent"), folderID(), folderName(), false) + err = tree.setFolder(context.Background(), folder) require.NoError(t, err, clues.ToCore(err)) return tree @@ -502,7 +513,8 @@ func treeWithFolders(t *testing.T, d *deltaDrive) *folderyMcFolderFace { func treeWithFileAtRoot(t *testing.T, d *deltaDrive) *folderyMcFolderFace { tree := treeWithRoot(t, d) - err := tree.addFile(rootID, fileID(), custom.ToCustomDriveItem(d.fileAtRoot())) + f := custom.ToCustomDriveItem(d.fileAt(root)) + err := tree.addFile(f) require.NoError(t, err, clues.ToCore(err)) return tree @@ -518,7 +530,8 @@ func treeWithDeletedFile(t *testing.T, d *deltaDrive) *folderyMcFolderFace { func treeWithFileInFolder(t *testing.T, d *deltaDrive) *folderyMcFolderFace { tree := treeWithFolders(t, d) - err := tree.addFile(folderID(), fileID(), custom.ToCustomDriveItem(d.fileAt(folder))) + f := custom.ToCustomDriveItem(d.fileAt(folder)) + err := tree.addFile(f) require.NoError(t, err, clues.ToCore(err)) return tree @@ -545,7 +558,7 @@ func fullTree(t *testing.T, d *deltaDrive) *folderyMcFolderFace { } func fullTreeWithNames( - parentFolderX, tombstoneX any, + parentFolderSuffix, tombstoneSuffix any, ) func(t *testing.T, d *deltaDrive) *folderyMcFolderFace { return func(t *testing.T, d *deltaDrive) *folderyMcFolderFace { ctx, flush := tester.NewContext(t) @@ -553,56 +566,47 @@ func fullTreeWithNames( tree := treeWithRoot(t, d) - // file in root - df := driveFile(d.dir(), rootID, "r") - err := tree.addFile( - rootID, - fileID("r"), - custom.ToCustomDriveItem(df)) + // file "r" in root + df := custom.ToCustomDriveItem(d.fileAt(root, "r")) + err := tree.addFile(df) require.NoError(t, err, clues.ToCore(err)) // root -> folderID(parentX) - err = tree.setFolder(ctx, rootID, folderID(parentFolderX), folderName(parentFolderX), false) + parent := custom.ToCustomDriveItem(d.folderAt(root, parentFolderSuffix)) + err = tree.setFolder(ctx, parent) require.NoError(t, err, clues.ToCore(err)) - // file in folderID(parentX) - df = driveFile(d.dir(folderName(parentFolderX)), folderID(parentFolderX), "p") - err = tree.addFile( - folderID(parentFolderX), - fileID("p"), - custom.ToCustomDriveItem(df)) + // file "p" in folderID(parentX) + df = custom.ToCustomDriveItem(d.fileAt(parentFolderSuffix, "p")) + err = tree.addFile(df) require.NoError(t, err, clues.ToCore(err)) // folderID(parentX) -> folderID() - err = tree.setFolder(ctx, folderID(parentFolderX), folderID(), folderName(), false) + fld := custom.ToCustomDriveItem(d.folderAt(parentFolderSuffix)) + err = tree.setFolder(ctx, fld) require.NoError(t, err, clues.ToCore(err)) - // file in folderID() - df = driveFile(d.dir(folderName()), folderID()) - err = tree.addFile( - folderID(), - fileID(), - custom.ToCustomDriveItem(df)) + // file "f" in folderID() + df = custom.ToCustomDriveItem(d.fileAt(folder, "f")) + err = tree.addFile(df) require.NoError(t, err, clues.ToCore(err)) // tombstone - have to set a non-tombstone folder first, // then add the item, // then tombstone the folder - err = tree.setFolder(ctx, rootID, folderID(tombstoneX), folderName(tombstoneX), false) + tomb := custom.ToCustomDriveItem(d.folderAt(root, tombstoneSuffix)) + err = tree.setFolder(ctx, tomb) require.NoError(t, err, clues.ToCore(err)) - // file in tombstone - df = driveFile(d.dir(folderName(tombstoneX)), folderID(tombstoneX), "t") - err = tree.addFile( - folderID(tombstoneX), - fileID("t"), - custom.ToCustomDriveItem(df)) + // file "t" in tombstone + df = custom.ToCustomDriveItem(d.fileAt(tombstoneSuffix, "t")) + err = tree.addFile(df) require.NoError(t, err, clues.ToCore(err)) - err = tree.setTombstone(ctx, folderID(tombstoneX)) + err = tree.setTombstone(ctx, tomb) require.NoError(t, err, clues.ToCore(err)) - // deleted file + // deleted file "d" tree.deleteFile(fileID("d")) return tree @@ -1355,22 +1359,24 @@ func (dd *deltaDrive) fileAt( parentSuffix any, fileSuffixes ...any, ) models.DriveItemable { - return driveItem( - fileID(fileSuffixes...), - fileName(fileSuffixes...), - dd.dir(folderName(parentSuffix)), - folderID(parentSuffix), - isFile) -} + if parentSuffix == root { + return driveItem( + fileID(fileSuffixes...), + fileName(fileSuffixes...), + dd.dir(), + rootID, + isFile) + } -func (dd *deltaDrive) fileAtRoot( - fileSuffixes ...any, -) models.DriveItemable { return driveItem( fileID(fileSuffixes...), fileName(fileSuffixes...), + // the file's parent directory isn't used; + // this parameter is an artifact of the driveItem + // api and doesn't need to be populated for test + // success. dd.dir(), - rootID, + folderID(parentSuffix), isFile) } @@ -1391,28 +1397,25 @@ func (dd *deltaDrive) fileWURLAtRoot( return di } -func (dd *deltaDrive) fileWSizeAtRoot( - size int64, - fileSuffixes ...any, -) models.DriveItemable { - return driveItemWSize( - fileID(fileSuffixes...), - fileName(fileSuffixes...), - dd.dir(), - rootID, - size, - isFile) -} - func (dd *deltaDrive) fileWSizeAt( size int64, parentSuffix any, fileSuffixes ...any, ) models.DriveItemable { + if parentSuffix == root { + return driveItemWSize( + fileID(fileSuffixes...), + fileName(fileSuffixes...), + dd.dir(), + rootID, + size, + isFile) + } + return driveItemWSize( fileID(fileSuffixes...), fileName(fileSuffixes...), - dd.dir(folderName(parentSuffix)), + dd.dir(), folderID(parentSuffix), size, isFile) @@ -1442,9 +1445,9 @@ func driveFolder( isFolder) } -func driveRootFolder() models.DriveItemable { +func rootFolder() models.DriveItemable { rootFolder := models.NewDriveItem() - rootFolder.SetName(ptr.To(rootName)) + rootFolder.SetName(ptr.To(root)) rootFolder.SetId(ptr.To(rootID)) rootFolder.SetRoot(models.NewRoot()) rootFolder.SetFolder(models.NewFolder()) @@ -1452,29 +1455,40 @@ func driveRootFolder() models.DriveItemable { return rootFolder } -func (dd *deltaDrive) folderAtRoot( - folderSuffixes ...any, -) models.DriveItemable { - return driveItem( - folderID(folderSuffixes...), - folderName(folderSuffixes...), - dd.dir(), - rootID, - isFolder) -} - func (dd *deltaDrive) folderAt( parentSuffix any, folderSuffixes ...any, ) models.DriveItemable { + if parentSuffix == root { + return driveItem( + folderID(folderSuffixes...), + folderName(folderSuffixes...), + dd.dir(), + rootID, + isFolder) + } + return driveItem( folderID(folderSuffixes...), folderName(folderSuffixes...), + // we should be putting in the full location here, not just the + // parent suffix. But that full location would be unused because + // our unit tests don't utilize folder subselection (which is the + // only reason we need to provide the dir). dd.dir(folderName(parentSuffix)), folderID(parentSuffix), isFolder) } +func (dd *deltaDrive) packageAtRoot() models.DriveItemable { + return driveItem( + folderID(pkg), + folderName(pkg), + dd.dir(), + rootID, + isPackage) +} + // --------------------------------------------------------------------------- // id, name, path factories // --------------------------------------------------------------------------- @@ -1482,6 +1496,16 @@ func (dd *deltaDrive) folderAt( // assumption is only one suffix per id. Mostly using // the variadic as an "optional" extension. func id(v string, suffixes ...any) string { + if len(suffixes) > 1 { + // this should fail any tests. we could pass in a + // testing.T instead and fail the call here, but that + // produces a whole lot of chaff where this check should + // still get us the expected failure + return fmt.Sprintf( + "too many suffixes in the ID; should only be 0 or 1, got %d", + len(suffixes)) + } + id := fmt.Sprintf("id_%s", v) // a bit weird, but acts as a quality of life @@ -1505,6 +1529,16 @@ func id(v string, suffixes ...any) string { // assumption is only one suffix per name. Mostly using // the variadic as an "optional" extension. func name(v string, suffixes ...any) string { + if len(suffixes) > 1 { + // this should fail any tests. we could pass in a + // testing.T instead and fail the call here, but that + // produces a whole lot of chaff where this check should + // still get us the expected failure + return fmt.Sprintf( + "too many suffixes in the Name; should only be 0 or 1, got %d", + len(suffixes)) + } + name := fmt.Sprintf("n_%s", v) // a bit weird, but acts as a quality of life @@ -1542,20 +1576,19 @@ func toPath(elems ...string) string { } // produces the full path for the provided drive -func (dd *deltaDrive) strPath(elems ...string) string { - return toPath(append( - []string{ - tenant, - path.OneDriveService.String(), - user, - path.FilesCategory.String(), - odConsts.DriveFolderPrefixBuilder(dd.id).String(), - }, - elems...)...) +func (dd *deltaDrive) strPath(t *testing.T, elems ...string) string { + return dd.fullPath(t, elems...).String() } func (dd *deltaDrive) fullPath(t *testing.T, elems ...string) path.Path { - p, err := path.FromDataLayerPath(dd.strPath(elems...), false) + p, err := odConsts.DriveFolderPrefixBuilder(dd.id). + Append(elems...). + ToDataLayerPath( + tenant, + user, + path.OneDriveService, + path.FilesCategory, + false) require.NoError(t, err, clues.ToCore(err)) return p @@ -1564,9 +1597,9 @@ func (dd *deltaDrive) fullPath(t *testing.T, elems ...string) path.Path { // produces a complete path prefix up to the drive root folder with any // elements passed in appended to the generated prefix. func (dd *deltaDrive) dir(elems ...string) string { - return toPath(append( - []string{odConsts.DriveFolderPrefixBuilder(dd.id).String()}, - elems...)...) + return odConsts.DriveFolderPrefixBuilder(dd.id). + Append(elems...). + String() } // common item names @@ -1583,7 +1616,7 @@ const ( nav = "nav" pkg = "package" rootID = odConsts.RootID - rootName = odConsts.RootPathDir + root = odConsts.RootPathDir subfolder = "subfolder" tenant = "t" user = "u" diff --git a/src/internal/m365/collection/drive/limiter.go b/src/internal/m365/collection/drive/limiter.go index 8c508be90..fcaa3dbd9 100644 --- a/src/internal/m365/collection/drive/limiter.go +++ b/src/internal/m365/collection/drive/limiter.go @@ -6,7 +6,10 @@ import ( "github.com/alcionai/corso/src/pkg/control" ) -var errHitLimit = clues.New("hit limiter limits") +var ( + errHitLimit = clues.New("hit limiter limits") + errHitCollectionLimit = clues.New("hit item limits within the current collection") +) type driveEnumerationStats struct { numPages int @@ -111,9 +114,22 @@ func (l pagerLimiter) hitItemLimit(itemCount int) bool { return l.enabled() && itemCount >= l.limits.MaxItems } -// hitTotalBytesLimit returns true if the limiter is enabled and has reached the limit +// alreadyHitTotalBytesLimit returns true if the limiter is enabled and has reached the limit // for the accumulated byte size of all items (the file contents, not the item metadata) // added to collections for this backup. -func (l pagerLimiter) hitTotalBytesLimit(i int64) bool { - return l.enabled() && i >= l.limits.MaxBytes +func (l pagerLimiter) alreadyHitTotalBytesLimit(i int64) bool { + return l.enabled() && i > l.limits.MaxBytes +} + +// willStepOverBytesLimit returns true if the limiter is enabled and the provided addition +// of bytes is greater than the limit plus some padding (to ensure we can always hit +// the limit). +func (l pagerLimiter) willStepOverBytesLimit(current, addition int64) bool { + if !l.enabled() { + return false + } + + limitPlusPadding := int64(float64(l.limits.MaxBytes) * 1.03) + + return (current + addition) > limitPlusPadding } diff --git a/src/internal/m365/collection/drive/limiter_test.go b/src/internal/m365/collection/drive/limiter_test.go index 87c4a62e1..c13e4b372 100644 --- a/src/internal/m365/collection/drive/limiter_test.go +++ b/src/internal/m365/collection/drive/limiter_test.go @@ -34,9 +34,14 @@ type backupLimitTest struct { // Collection name -> set of item IDs. We can't check item data because // that's not mocked out. Metadata is checked separately. expectedItemIDsInCollection map[string][]string + // Collection name -> set of item IDs. We can't check item data because + // that's not mocked out. Metadata is checked separately. + // the tree version has some different (more accurate) expectations + // for success + expectedItemIDsInCollectionTree map[string][]string } -func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { +func backupLimitTable(t *testing.T, d1, d2 *deltaDrive) []backupLimitTest { return []backupLimitTest{ { name: "OneDrive SinglePage ExcludeItemsOverMaxSize", @@ -50,12 +55,13 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { }, enumerator: driveEnumerator( d1.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d1.fileWSizeAtRoot(7, "f1"), - d1.fileWSizeAtRoot(1, "f2"), - d1.fileWSizeAtRoot(1, "f3"))))), + delta(id(deltaURL), nil).with( + aPage( + d1.fileWSizeAt(7, root, "f1"), + d1.fileWSizeAt(1, root, "f2"), + d1.fileWSizeAt(1, root, "f3"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f2"), fileID("f3")}, + d1.strPath(t): {fileID("f2"), fileID("f3")}, }, }, { @@ -70,12 +76,13 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { }, enumerator: driveEnumerator( d1.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d1.fileWSizeAtRoot(1, "f1"), - d1.fileWSizeAtRoot(2, "f2"), - d1.fileWSizeAtRoot(1, "f3"))))), + delta(id(deltaURL), nil).with( + aPage( + d1.fileWSizeAt(1, root, "f1"), + d1.fileWSizeAt(2, root, "f2"), + d1.fileWSizeAt(1, root, "f3"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2")}, + d1.strPath(t): {fileID("f1"), fileID("f2")}, }, }, { @@ -90,14 +97,15 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { }, enumerator: driveEnumerator( d1.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d1.fileWSizeAtRoot(1, "f1"), - d1.folderAtRoot(), - d1.fileWSizeAt(2, folder, "f2"), - d1.fileWSizeAt(1, folder, "f3"))))), + delta(id(deltaURL), nil).with( + aPage( + d1.fileWSizeAt(1, root, "f1"), + d1.folderAt(root), + d1.fileWSizeAt(2, folder, "f2"), + d1.fileWSizeAt(1, folder, "f3"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1")}, - d1.strPath(folderName()): {folderID(), fileID("f2")}, + d1.strPath(t): {fileID("f1")}, + d1.strPath(t, folderName()): {folderID(), fileID("f2")}, }, }, { @@ -112,15 +120,16 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { }, enumerator: driveEnumerator( d1.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3"), - d1.fileAtRoot("f4"), - d1.fileAtRoot("f5"), - d1.fileAtRoot("f6"))))), + delta(id(deltaURL), nil).with( + aPage( + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3"), + d1.fileAt(root, "f4"), + d1.fileAt(root, "f5"), + d1.fileAt(root, "f6"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, }, }, { @@ -137,19 +146,19 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2")), aPage( // Repeated items shouldn't count against the limit. - d1.fileAtRoot("f1"), - d1.folderAtRoot(), + d1.fileAt(root, "f1"), + d1.folderAt(root), d1.fileAt(folder, "f3"), d1.fileAt(folder, "f4"), d1.fileAt(folder, "f5"), d1.fileAt(folder, "f6"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2")}, - d1.strPath(folderName()): {folderID(), fileID("f3")}, + d1.strPath(t): {fileID("f1"), fileID("f2")}, + d1.strPath(t, folderName()): {folderID(), fileID("f3")}, }, }, { @@ -166,16 +175,16 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f3"), d1.fileAt(folder, "f4"), d1.fileAt(folder, "f5"), d1.fileAt(folder, "f6"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2")}, + d1.strPath(t): {fileID("f1"), fileID("f2")}, }, }, { @@ -192,18 +201,22 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f4"), d1.fileAt(folder, "f5"))))), expectedItemIDsInCollection: map[string][]string{ - // Root has an additional item. It's hard to fix that in the code - // though. - d1.strPath(): {fileID("f1"), fileID("f2")}, - d1.strPath(folderName()): {folderID(), fileID("f4")}, + // Root has an additional item. It's hard to fix that in the code though. + d1.strPath(t): {fileID("f1"), fileID("f2")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4")}, + }, + expectedItemIDsInCollectionTree: map[string][]string{ + // the tree version doesn't have this problem. + d1.strPath(t): {fileID("f1")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4")}, }, }, { @@ -220,18 +233,18 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f1"), d1.fileAt(folder, "f2")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), // Updated item that shouldn't count against the limit a second time. d1.fileAt(folder, "f2"), d1.fileAt(folder, "f3"), d1.fileAt(folder, "f4"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {}, - d1.strPath(folderName()): {folderID(), fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t): {}, + d1.strPath(t, folderName()): {folderID(), fileID("f1"), fileID("f2"), fileID("f3")}, }, }, { @@ -248,19 +261,26 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - // Put folder 0 at limit. - d1.folderAtRoot(), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + // Put root/folder at limit. + d1.folderAt(root), d1.fileAt(folder, "f3"), d1.fileAt(folder, "f4")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), // Try to move item from root to folder 0 which is already at the limit. d1.fileAt(folder, "f1"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2")}, - d1.strPath(folderName()): {folderID(), fileID("f3"), fileID("f4")}, + d1.strPath(t): {fileID("f1"), fileID("f2")}, + d1.strPath(t, folderName()): {folderID(), fileID("f3"), fileID("f4")}, + }, + expectedItemIDsInCollectionTree: map[string][]string{ + d1.strPath(t): {fileID("f2")}, + // note that the tree version allows f1 to get moved. + // we've already committed to backing up the file as part of the preview, + // it doesn't seem rational to prevent its movement + d1.strPath(t, folderName()): {folderID(), fileID("f3"), fileID("f4"), fileID("f1")}, }, }, { @@ -277,18 +297,18 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f4")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f5"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, - d1.strPath(folderName()): {folderID(), fileID("f4"), fileID("f5")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4"), fileID("f5")}, }, }, { @@ -305,21 +325,21 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f4"), d1.fileAt(folder, "f5"), // This container shouldn't be returned. - d1.folderAtRoot(2), + d1.folderAt(root, 2), d1.fileAt(2, "f7"), d1.fileAt(2, "f8"), d1.fileAt(2, "f9"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, - d1.strPath(folderName()): {folderID(), fileID("f4"), fileID("f5")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4"), fileID("f5")}, }, }, { @@ -336,22 +356,22 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f4"), d1.fileAt(folder, "f5")), aPage( // This container shouldn't be returned. - d1.folderAtRoot(2), + d1.folderAt(root, 2), d1.fileAt(2, "f7"), d1.fileAt(2, "f8"), d1.fileAt(2, "f9"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, - d1.strPath(folderName()): {folderID(), fileID("f4"), fileID("f5")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4"), fileID("f5")}, }, }, { @@ -366,22 +386,24 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { }, enumerator: driveEnumerator( d1.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3"), - d1.fileAtRoot("f4"), - d1.fileAtRoot("f5")))), + delta(id(deltaURL), nil).with( + aPage( + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3"), + d1.fileAt(root, "f4"), + d1.fileAt(root, "f5")))), d2.newEnumer().with( - delta(id(deltaURL), nil).with(aPage( - d2.fileAtRoot("f1"), - d2.fileAtRoot("f2"), - d2.fileAtRoot("f3"), - d2.fileAtRoot("f4"), - d2.fileAtRoot("f5"))))), + delta(id(deltaURL), nil).with( + aPage( + d2.fileAt(root, "f1"), + d2.fileAt(root, "f2"), + d2.fileAt(root, "f3"), + d2.fileAt(root, "f4"), + d2.fileAt(root, "f5"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, - d2.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, + d2.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, }, }, { @@ -397,18 +419,18 @@ func backupLimitTable(d1, d2 *deltaDrive) []backupLimitTest { d1.newEnumer().with( delta(id(deltaURL), nil).with( aPage( - d1.fileAtRoot("f1"), - d1.fileAtRoot("f2"), - d1.fileAtRoot("f3")), + d1.fileAt(root, "f1"), + d1.fileAt(root, "f2"), + d1.fileAt(root, "f3")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f4")), aPage( - d1.folderAtRoot(), + d1.folderAt(root), d1.fileAt(folder, "f5"))))), expectedItemIDsInCollection: map[string][]string{ - d1.strPath(): {fileID("f1"), fileID("f2"), fileID("f3")}, - d1.strPath(folderName()): {folderID(), fileID("f4"), fileID("f5")}, + d1.strPath(t): {fileID("f1"), fileID("f2"), fileID("f3")}, + d1.strPath(t, folderName()): {folderID(), fileID("f4"), fileID("f5")}, }, }, } @@ -427,8 +449,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_noTree() { // checks that don't examine metadata, collection states, etc. They really just // check the expected items appear. func (suite *LimiterUnitSuite) TestGet_PreviewLimits_tree() { - suite.T().Skip("TODO: unskip when tree produces collections") - opts := control.DefaultOptions() opts.ToggleFeatures.UseDeltaTree = true @@ -441,7 +461,7 @@ func iterGetPreviewLimitsTests( ) { d1, d2 := drive(), drive(2) - for _, test := range backupLimitTable(d1, d2) { + for _, test := range backupLimitTable(suite.T(), d1, d2) { suite.Run(test.name, func() { runGetPreviewLimits( suite.T(), @@ -521,9 +541,15 @@ func runGetPreviewLimits( itemIDs = append(itemIDs, id) } + expectItemIDs := test.expectedItemIDsInCollection[folderPath] + + if opts.ToggleFeatures.UseDeltaTree && test.expectedItemIDsInCollectionTree != nil { + expectItemIDs = test.expectedItemIDsInCollectionTree[folderPath] + } + assert.ElementsMatchf( t, - test.expectedItemIDsInCollection[folderPath], + expectItemIDs, itemIDs, "item IDs in collection with path:\n\t%q", folderPath) @@ -542,6 +568,9 @@ type defaultLimitTestExpects struct { numItems int numContainers int numItemsPerContainer int + // the tree handling behavior may deviate under certain conditions + // since it allows one file to slightly step over the byte limit + numItemsTreePadding int } type defaultLimitTest struct { @@ -641,6 +670,7 @@ func defaultLimitsTable() []defaultLimitTest { numItems: int(defaultPreviewMaxBytes) / 1024 / 1024, numContainers: 1, numItemsPerContainer: int(defaultPreviewMaxBytes) / 1024 / 1024, + numItemsTreePadding: 1, }, }, } @@ -666,8 +696,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsNoTree() { // These tests run a reduced set of checks that really just look for item counts // and such. Other tests are expected to provide more comprehensive checks. func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsWithTree() { - suite.T().Skip("TODO: unskip when tree produces collections") - opts := control.DefaultOptions() opts.ToggleFeatures.UseDeltaTree = true @@ -714,7 +742,7 @@ func runGetPreviewLimitsDefaults( for containerIdx := 0; containerIdx < test.numContainers; containerIdx++ { page := nextPage{ Items: []models.DriveItemable{ - driveRootFolder(), + rootFolder(), driveItem( folderID(containerIdx), folderName(containerIdx), @@ -798,11 +826,16 @@ func runGetPreviewLimitsDefaults( numItems += len(col.driveItems) // Add one to account for the folder permissions item. + expected := test.expect.numItemsPerContainer + 1 + if opts.ToggleFeatures.UseDeltaTree { + expected += test.expect.numItemsTreePadding + } + assert.Len( t, col.driveItems, - test.expect.numItemsPerContainer+1, - "items in container %v", + expected, + "number of items in collection at:\n\t%+v", col.FullPath()) } @@ -810,12 +843,18 @@ func runGetPreviewLimitsDefaults( t, test.expect.numContainers, numContainers, - "total containers") + "total count of collections") + + // Add one to account for the folder permissions item. + expected := test.expect.numItems + test.expect.numContainers + if opts.ToggleFeatures.UseDeltaTree { + expected += test.expect.numItemsTreePadding + } // Each container also gets an item so account for that here. assert.Equal( t, - test.expect.numItems+test.expect.numContainers, + expected, numItems, - "total items across all containers") + "total sum of item counts in all collections") } diff --git a/src/internal/m365/collection/drive/url_cache_test.go b/src/internal/m365/collection/drive/url_cache_test.go index 14494610c..46cee7cd1 100644 --- a/src/internal/m365/collection/drive/url_cache_test.go +++ b/src/internal/m365/collection/drive/url_cache_test.go @@ -509,7 +509,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() { pages: []nextPage{ aPage( d.fileWURLAtRoot(aURL(1), false, 1), - d.folderAtRoot(2)), + d.folderAt(root, 2)), }, expectedItemProps: map[string]itemProps{ fileID(2): {}, diff --git a/src/pkg/services/m365/custom/drive_item.go b/src/pkg/services/m365/custom/drive_item.go index bafa141df..47c25b286 100644 --- a/src/pkg/services/m365/custom/drive_item.go +++ b/src/pkg/services/m365/custom/drive_item.go @@ -33,6 +33,15 @@ type DriveItem struct { additionalData map[string]any } +func NewDriveItem( + id, name string, +) *DriveItem { + return &DriveItem{ + id: ptr.To(id), + name: ptr.To(name), + } +} + // Disable revive linter since we want to follow naming scheme used by graph SDK here. // nolint: revive func (c *DriveItem) GetId() *string {