diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index 8e48587c8..ff4bdc85d 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -23,6 +23,7 @@ import ( bupMD "github.com/alcionai/corso/src/pkg/backup/metadata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -206,8 +207,7 @@ func DeserializeMap[T any](reader io.ReadCloser, alreadyFound map[string]T) erro return nil } -// Retrieves drive data as set of `data.Collections` and a set of item names to -// be excluded from the upcoming backup. +// Retrieves drive data as set of `data.Collections`. func (c *Collections) Get( ctx context.Context, prevMetadata []data.RestoreCollection, @@ -258,6 +258,13 @@ func (c *Collections) Get( excludedItemIDs = map[string]struct{}{} oldPrevPaths = oldPrevPathsByDriveID[driveID] prevDeltaLink = prevDriveIDToDelta[driveID] + + // packagePaths is keyed by folder paths to a parent directory + // which is marked as a package by its driveItem GetPackage + // property. Packages are only marked at the top level folder, + // so we need this map to identify and mark all subdirs as also + // being package cased. + packagePaths = map[string]struct{}{} ) delete(driveTombstones, driveID) @@ -280,6 +287,7 @@ func (c *Collections) Get( driveName, oldPrevPaths, excludedItemIDs, + packagePaths, prevDeltaLink, errs) if err != nil { @@ -665,6 +673,7 @@ func (c *Collections) PopulateDriveCollections( driveID, driveName string, oldPrevPaths map[string]string, excludedItemIDs map[string]struct{}, + topLevelPackages map[string]struct{}, prevDeltaLink string, errs *fault.Bus, ) (api.DeltaUpdate, map[string]string, error) { @@ -719,6 +728,7 @@ func (c *Collections) PopulateDriveCollections( currPrevPaths, newPrevPaths, excludedItemIDs, + topLevelPackages, invalidPrevDelta, el) if err != nil { @@ -740,7 +750,8 @@ func (c *Collections) processItem( item models.DriveItemable, driveID, driveName string, oldPrevPaths, currPrevPaths, newPrevPaths map[string]string, - excluded map[string]struct{}, + excludedItemIDs map[string]struct{}, + topLevelPackages map[string]struct{}, invalidPrevDelta bool, skipper fault.AddSkipper, ) error { @@ -778,7 +789,7 @@ func (c *Collections) processItem( currPrevPaths, newPrevPaths, isFolder, - excluded, + excludedItemIDs, invalidPrevDelta) return clues.Stack(err).WithClues(ictx).OrNil() @@ -833,6 +844,17 @@ func (c *Collections) processItem( return nil } + isPackage := item.GetPackageEscaped() != nil + if isPackage { + // mark this path as a package type for all other collections. + // any subfolder should get marked as a childOfPackage below. + topLevelPackages[collectionPath.String()] = struct{}{} + } + + childOfPackage := filters. + PathPrefix(maps.Keys(topLevelPackages)). + Compare(collectionPath.String()) + col, err := NewCollection( c.handler, c.protectedResource, @@ -841,7 +863,7 @@ func (c *Collections) processItem( driveID, c.statusUpdater, c.ctrl, - item.GetPackageEscaped() != nil, + isPackage || childOfPackage, invalidPrevDelta, nil) if err != nil { @@ -914,8 +936,8 @@ func (c *Collections) processItem( // Always add a file to the excluded list. The file may have been // renamed/moved/modified, so we still have to drop the // original one and download a fresh copy. - excluded[itemID+metadata.DataFileSuffix] = struct{}{} - excluded[itemID+metadata.MetaFileSuffix] = struct{}{} + excludedItemIDs[itemID+metadata.DataFileSuffix] = struct{}{} + excludedItemIDs[itemID+metadata.MetaFileSuffix] = struct{}{} } default: diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index 073adbe6d..fa8023224 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -109,7 +109,7 @@ func TestOneDriveCollectionsUnitSuite(t *testing.T) { suite.Run(t, &OneDriveCollectionsUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func getDelList(files ...string) map[string]struct{} { +func makeExcludeMap(files ...string) map[string]struct{} { delList := map[string]struct{}{} for _, file := range files { delList[file+metadata.DataFileSuffix] = struct{}{} @@ -127,7 +127,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { tenant = "tenant" user = "user" folder = "/folder" - folderSub = "/folder/subfolder" + subFolder = "/subfolder" pkg = "/package" ) @@ -137,18 +137,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedStatePath := getExpectedStatePathGenerator(suite.T(), bh, tenant, testBaseDrivePath) tests := []struct { - name string - items []models.DriveItemable - inputFolderMap map[string]string - scope selectors.OneDriveScope - expect assert.ErrorAssertionFunc - expectedCollectionIDs map[string]statePath - expectedItemCount int - expectedContainerCount int - expectedFileCount int - expectedSkippedCount int - expectedPrevPaths map[string]string - expectedExcludes map[string]struct{} + name string + items []models.DriveItemable + inputFolderMap map[string]string + topLevelPackages map[string]struct{} + scope selectors.OneDriveScope + expect assert.ErrorAssertionFunc + expectedCollectionIDs map[string]statePath + expectedItemCount int + expectedContainerCount int + expectedFileCount int + expectedSkippedCount int + expectedPrevPaths map[string]string + expectedExcludes map[string]struct{} + expectedTopLevelPackages map[string]struct{} + expectedCountPackages int }{ { name: "Invalid item", @@ -156,9 +159,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveRootItem("root"), driveItem("item", "item", testBaseDrivePath, "root", false, false, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.Error, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.Error, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), }, @@ -166,7 +170,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, }, { name: "Single File", @@ -174,9 +179,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveRootItem("root"), driveItem("file", "file", testBaseDrivePath, "root", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), }, @@ -187,7 +193,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: getDelList("file"), + expectedExcludes: makeExcludeMap("file"), + expectedTopLevelPackages: map[string]struct{}{}, }, { name: "Single Folder", @@ -195,9 +202,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveRootItem("root"), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NewState, folder), @@ -206,9 +214,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "root": expectedPath(""), "folder": expectedPath("/folder"), }, - expectedItemCount: 1, - expectedContainerCount: 2, - expectedExcludes: map[string]struct{}{}, + expectedItemCount: 1, + expectedContainerCount: 2, + expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, }, { name: "Single Package", @@ -216,9 +225,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveRootItem("root"), driveItem("package", "package", testBaseDrivePath, "root", false, false, true), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "package": expectedStatePath(data.NewState, pkg), @@ -230,6 +240,42 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedItemCount: 1, expectedContainerCount: 2, expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{ + expectedPath("/package"): {}, + }, + expectedCountPackages: 1, + }, + { + name: "Single Package with subfolder", + items: []models.DriveItemable{ + driveRootItem("root"), + driveItem("package", "package", testBaseDrivePath, "root", false, false, true), + driveItem("folder", "folder", testBaseDrivePath+pkg, "package", false, true, false), + driveItem("subfolder", "subfolder", testBaseDrivePath+pkg, "package", false, true, false), + }, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, + expectedCollectionIDs: map[string]statePath{ + "root": expectedStatePath(data.NotMovedState, ""), + "package": expectedStatePath(data.NewState, pkg), + "folder": expectedStatePath(data.NewState, pkg+folder), + "subfolder": expectedStatePath(data.NewState, pkg+subFolder), + }, + expectedPrevPaths: map[string]string{ + "root": expectedPath(""), + "package": expectedPath(pkg), + "folder": expectedPath(pkg + folder), + "subfolder": expectedPath(pkg + subFolder), + }, + expectedItemCount: 3, + expectedContainerCount: 4, + expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{ + expectedPath(pkg): {}, + }, + expectedCountPackages: 3, }, { name: "1 root file, 1 folder, 1 package, 2 files, 3 collections", @@ -241,9 +287,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NewState, folder), @@ -257,7 +304,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/folder"), "package": expectedPath("/package"), }, - expectedExcludes: getDelList("fileInRoot", "fileInFolder", "fileInPackage"), + expectedTopLevelPackages: map[string]struct{}{ + expectedPath("/package"): {}, + }, + expectedCountPackages: 1, + expectedExcludes: makeExcludeMap("fileInRoot", "fileInFolder", "fileInPackage"), }, { name: "contains folder selector", @@ -266,19 +317,27 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false), - driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false), + driveItem("folder2", "folder", testBaseDrivePath+folder+subFolder, "subfolder", false, true, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false), - driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false), + driveItem( + "fileInFolder2", + "fileInFolder2", + testBaseDrivePath+folder+subFolder+folder, + "folder2", + true, + false, + false), driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0], + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "folder": expectedStatePath(data.NewState, folder), - "subfolder": expectedStatePath(data.NewState, folderSub), - "folder2": expectedStatePath(data.NewState, folderSub+folder), + "subfolder": expectedStatePath(data.NewState, folder+subFolder), + "folder2": expectedStatePath(data.NewState, folder+subFolder+folder), }, expectedItemCount: 5, expectedFileCount: 2, @@ -287,10 +346,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { // parent path since we only check later if something is a folder or not. expectedPrevPaths: map[string]string{ "folder": expectedPath(folder), - "subfolder": expectedPath(folderSub), - "folder2": expectedPath(folderSub + folder), + "subfolder": expectedPath(folder + subFolder), + "folder2": expectedPath(folder + subFolder + folder), }, - expectedExcludes: getDelList("fileInFolder", "fileInFolder2"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("fileInFolder", "fileInFolder2"), }, { name: "prefix subfolder selector", @@ -299,28 +359,36 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false), driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false), - driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false), + driveItem("folder2", "folder", testBaseDrivePath+folder+subFolder, "subfolder", false, true, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false), - driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false), + driveItem( + "fileInFolder2", + "fileInFolder2", + testBaseDrivePath+folder+subFolder+folder, + "folder2", + true, + false, + false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: (&selectors.OneDriveBackup{}). - Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0], + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - "subfolder": expectedStatePath(data.NewState, folderSub), - "folder2": expectedStatePath(data.NewState, folderSub+folder), + "subfolder": expectedStatePath(data.NewState, folder+subFolder), + "folder2": expectedStatePath(data.NewState, folder+subFolder+folder), }, expectedItemCount: 3, expectedFileCount: 1, expectedContainerCount: 2, expectedPrevPaths: map[string]string{ - "subfolder": expectedPath(folderSub), - "folder2": expectedPath(folderSub + folder), + "subfolder": expectedPath(folder + subFolder), + "folder2": expectedPath(folder + subFolder + folder), }, - expectedExcludes: getDelList("fileInFolder2"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("fileInFolder2"), }, { name: "match subfolder selector", @@ -331,23 +399,32 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false), driveItem("package", "package", testBaseDrivePath, "root", false, false, true), driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false), - driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, "subfolder", true, false, false), + driveItem( + "fileInSubfolder", + "fileInSubfolder", + testBaseDrivePath+folder+subFolder, + "subfolder", + true, + false, + false), driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0], + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ - "subfolder": expectedStatePath(data.NewState, folderSub), + "subfolder": expectedStatePath(data.NewState, folder+subFolder), }, expectedItemCount: 2, expectedFileCount: 1, expectedContainerCount: 1, // No child folders for subfolder so nothing here. expectedPrevPaths: map[string]string{ - "subfolder": expectedPath(folderSub), + "subfolder": expectedPath(folder + subFolder), }, - expectedExcludes: getDelList("fileInSubfolder"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("fileInSubfolder"), }, { name: "not moved folder tree", @@ -357,10 +434,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { }, inputFolderMap: map[string]string{ "folder": expectedPath(folder), - "subfolder": expectedPath(folderSub), + "subfolder": expectedPath(folder + subFolder), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NotMovedState, folder), @@ -371,9 +449,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), "folder": expectedPath(folder), - "subfolder": expectedPath(folderSub), + "subfolder": expectedPath(folder + subFolder), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "moved folder tree", @@ -385,8 +464,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/a-folder"), "subfolder": expectedPath("/a-folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.MovedState, folder, "/a-folder"), @@ -397,9 +477,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), "folder": expectedPath(folder), - "subfolder": expectedPath(folderSub), + "subfolder": expectedPath(folder + subFolder), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "moved folder tree with file no previous", @@ -409,9 +490,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false), driveItem("folder", "folder2", testBaseDrivePath, "root", false, true, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NewState, "/folder2"), @@ -423,7 +505,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "root": expectedPath(""), "folder": expectedPath("/folder2"), }, - expectedExcludes: getDelList("file"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("file"), }, { name: "moved folder tree with file no previous 1", @@ -432,9 +515,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false), driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NewState, folder), @@ -446,7 +530,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "root": expectedPath(""), "folder": expectedPath(folder), }, - expectedExcludes: getDelList("file"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("file"), }, { name: "moved folder tree and subfolder 1", @@ -459,8 +544,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/a-folder"), "subfolder": expectedPath("/a-folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.MovedState, folder, "/a-folder"), @@ -474,7 +560,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath(folder), "subfolder": expectedPath("/subfolder"), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "moved folder tree and subfolder 2", @@ -487,8 +574,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/a-folder"), "subfolder": expectedPath("/a-folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.MovedState, folder, "/a-folder"), @@ -502,7 +590,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath(folder), "subfolder": expectedPath("/subfolder"), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "move subfolder when moving parent", @@ -528,13 +617,14 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/a-folder"), "subfolder": expectedPath("/a-folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.MovedState, folder, "/a-folder"), "folder2": expectedStatePath(data.NewState, "/folder2"), - "subfolder": expectedStatePath(data.MovedState, folderSub, "/a-folder/subfolder"), + "subfolder": expectedStatePath(data.MovedState, folder+subFolder, "/a-folder/subfolder"), }, expectedItemCount: 5, expectedFileCount: 2, @@ -545,7 +635,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder2": expectedPath("/folder2"), "subfolder": expectedPath("/folder/subfolder"), }, - expectedExcludes: getDelList("itemInSubfolder", "itemInFolder2"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("itemInSubfolder", "itemInFolder2"), }, { name: "moved folder tree multiple times", @@ -559,8 +650,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/a-folder"), "subfolder": expectedPath("/a-folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.MovedState, "/folder2", "/a-folder"), @@ -573,7 +665,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/folder2"), "subfolder": expectedPath("/folder2/subfolder"), }, - expectedExcludes: getDelList("file"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("file"), }, { name: "deleted folder and package", @@ -587,8 +680,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/folder"), "package": expectedPath("/package"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.DeletedState, folder), @@ -600,7 +694,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "delete folder without previous", @@ -611,8 +706,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { inputFolderMap: map[string]string{ "root": expectedPath(""), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), }, @@ -622,7 +718,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "delete folder tree move subfolder", @@ -636,12 +733,13 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/folder"), "subfolder": expectedPath("/folder/subfolder"), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.DeletedState, folder), - "subfolder": expectedStatePath(data.MovedState, "/subfolder", folderSub), + "subfolder": expectedStatePath(data.MovedState, "/subfolder", folder+subFolder), }, expectedItemCount: 1, expectedFileCount: 0, @@ -650,7 +748,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "root": expectedPath(""), "subfolder": expectedPath("/subfolder"), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "delete file", @@ -661,8 +760,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { inputFolderMap: map[string]string{ "root": expectedPath(""), }, - scope: anyFolder, - expect: assert.NoError, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), }, @@ -672,7 +772,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: getDelList("item"), + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: makeExcludeMap("item"), }, { name: "item before parent errors", @@ -681,9 +782,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false), driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.Error, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.Error, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), }, @@ -693,7 +795,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { expectedPrevPaths: map[string]string{ "root": expectedPath(""), }, - expectedExcludes: map[string]struct{}{}, + expectedTopLevelPackages: map[string]struct{}{}, + expectedExcludes: map[string]struct{}{}, }, { name: "1 root file, 1 folder, 1 package, 1 good file, 1 malware", @@ -705,9 +808,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { driveItem("goodFile", "goodFile", testBaseDrivePath+folder, "folder", true, false, false), malwareItem("malwareFile", "malwareFile", testBaseDrivePath+folder, "folder", true, false, false), }, - inputFolderMap: map[string]string{}, - scope: anyFolder, - expect: assert.NoError, + inputFolderMap: map[string]string{}, + scope: anyFolder, + topLevelPackages: map[string]struct{}{}, + expect: assert.NoError, expectedCollectionIDs: map[string]statePath{ "root": expectedStatePath(data.NotMovedState, ""), "folder": expectedStatePath(data.NewState, folder), @@ -722,7 +826,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "folder": expectedPath("/folder"), "package": expectedPath("/package"), }, - expectedExcludes: getDelList("fileInRoot", "goodFile"), + expectedTopLevelPackages: map[string]struct{}{ + expectedPath("/package"): {}, + }, + expectedCountPackages: 1, + expectedExcludes: makeExcludeMap("fileInRoot", "goodFile"), }, } @@ -772,7 +880,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { "General", test.inputFolderMap, excludes, - "smarf", + test.topLevelPackages, + "prevdelta", errs) test.expect(t, err, clues.ToCore(err)) assert.ElementsMatch( @@ -782,7 +891,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count") assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count") - assert.Equal(t, test.expectedSkippedCount, len(errs.Skipped()), "skipped items") + assert.Equal(t, test.expectedSkippedCount, len(errs.Skipped()), "skipped item count") for id, sp := range test.expectedCollectionIDs { if !assert.Containsf(t, c.CollectionMap[driveID], id, "missing collection with id %s", id) { @@ -795,8 +904,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { assert.Equalf(t, sp.prevPath, c.CollectionMap[driveID][id].PreviousPath(), "prev path for collection %s", id) } - assert.Equal(t, test.expectedPrevPaths, newPrevPaths, "metadata paths") - assert.Equal(t, test.expectedExcludes, excludes, "exclude list") + assert.Equal(t, test.expectedPrevPaths, newPrevPaths, "previous paths") + assert.Equal(t, test.expectedTopLevelPackages, test.topLevelPackages, "top level packages") + assert.Equal(t, test.expectedExcludes, excludes, "excluded item IDs map") + + var countPackages int + + for _, drives := range c.CollectionMap { + for _, col := range drives { + if col.isPackageOrChildOfPackage { + countPackages++ + } + } + } + + assert.Equal(t, test.expectedCountPackages, countPackages, "count of collections marked as packages") }) } } @@ -808,7 +930,6 @@ func (suite *OneDriveCollectionsUnitSuite) TestDeserializeMetadata() { driveID2 := "2" deltaURL1 := "url/1" deltaURL2 := "url/2" - folderID1 := "folder1" folderID2 := "folder2" path1 := "folder1/path" @@ -1273,7 +1394,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveID1: {"root": rootFolderPath1}, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - rootFolderPath1: getDelList("file"), + rootFolderPath1: makeExcludeMap("file"), }), }, { @@ -1307,7 +1428,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveID1: {"root": rootFolderPath1}, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - rootFolderPath1: getDelList("file"), + rootFolderPath1: makeExcludeMap("file"), }), }, { @@ -1426,7 +1547,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ - rootFolderPath1: getDelList("file"), + rootFolderPath1: makeExcludeMap("file"), }), }, { diff --git a/src/internal/m365/service/sharepoint/backup_test.go b/src/internal/m365/service/sharepoint/backup_test.go index e9ae4c10f..35e48edf6 100644 --- a/src/internal/m365/service/sharepoint/backup_test.go +++ b/src/internal/m365/service/sharepoint/backup_test.go @@ -101,6 +101,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { collMap = map[string]map[string]*drive.Collection{ driveID: {}, } + topLevelPackages = map[string]struct{}{} ) mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{ @@ -127,7 +128,8 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { "General", paths, excluded, - "", + topLevelPackages, + "notempty", fault.New(true)) test.expect(t, err, clues.ToCore(err)) @@ -135,6 +137,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count") assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count") + assert.Empty(t, topLevelPackages, "should not find package type folders") for _, collPath := range test.expectedCollectionIDs { assert.Contains(t, c.CollectionMap[driveID], collPath)