larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mark complete subtrees as packages (#4501)

Browse Source 
marks the full subtree of a drive package item
as being within the package.  This ensures
that all collections whose directory is a subdir of a package-type folder are themselves marked as package-typed.

---

#### Does this PR need a docs update or release note?

- [x] 🕐 Yes, but in a later PR

#### Type of change

- [x] 🐛 Bugfix

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-10-24 17:01:06 -06:00 committed by GitHub
parent 4d0be25020
commit 89f59eea91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 276 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/internal/m365/collection/drive/collections.go
View File

@ -23,6 +23,7 @@ import (
bupMD "github.com/alcionai/corso/src/pkg/backup/metadata"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
@ -206,8 +207,7 @@ func DeserializeMap[T any](reader io.ReadCloser, alreadyFound map[string]T) erro
return nil
}
// Retrieves drive data as set of `data.Collections` and a set of item names to
// be excluded from the upcoming backup.
// Retrieves drive data as set of `data.Collections`.
func (c *Collections) Get(
ctx context.Context,
prevMetadata []data.RestoreCollection,
@ -258,6 +258,13 @@ func (c *Collections) Get(
excludedItemIDs = map[string]struct{}{}
oldPrevPaths = oldPrevPathsByDriveID[driveID]
prevDeltaLink = prevDriveIDToDelta[driveID]
// packagePaths is keyed by folder paths to a parent directory
// which is marked as a package by its driveItem GetPackage
// property. Packages are only marked at the top level folder,
// so we need this map to identify and mark all subdirs as also
// being package cased.
packagePaths = map[string]struct{}{}
)
delete(driveTombstones, driveID)
@ -280,6 +287,7 @@ func (c *Collections) Get(
driveName,
oldPrevPaths,
excludedItemIDs,
packagePaths,
prevDeltaLink,
errs)
if err != nil {
@ -665,6 +673,7 @@ func (c *Collections) PopulateDriveCollections(
driveID, driveName string,
oldPrevPaths map[string]string,
excludedItemIDs map[string]struct{},
topLevelPackages map[string]struct{},
prevDeltaLink string,
errs *fault.Bus,
) (api.DeltaUpdate, map[string]string, error) {
@ -719,6 +728,7 @@ func (c *Collections) PopulateDriveCollections(
currPrevPaths,
newPrevPaths,
excludedItemIDs,
topLevelPackages,
invalidPrevDelta,
el)
if err != nil {
@ -740,7 +750,8 @@ func (c *Collections) processItem(
item models.DriveItemable,
driveID, driveName string,
oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
excluded map[string]struct{},
excludedItemIDs map[string]struct{},
topLevelPackages map[string]struct{},
invalidPrevDelta bool,
skipper fault.AddSkipper,
) error {
@ -778,7 +789,7 @@ func (c *Collections) processItem(
currPrevPaths,
newPrevPaths,
isFolder,
excluded,
excludedItemIDs,
invalidPrevDelta)
return clues.Stack(err).WithClues(ictx).OrNil()
@ -833,6 +844,17 @@ func (c *Collections) processItem(
return nil
}
isPackage := item.GetPackageEscaped() != nil
if isPackage {
// mark this path as a package type for all other collections.
// any subfolder should get marked as a childOfPackage below.
topLevelPackages[collectionPath.String()] = struct{}{}
}
childOfPackage := filters.
PathPrefix(maps.Keys(topLevelPackages)).
Compare(collectionPath.String())
col, err := NewCollection(
c.handler,
c.protectedResource,
@ -841,7 +863,7 @@ func (c *Collections) processItem(
driveID,
c.statusUpdater,
c.ctrl,
item.GetPackageEscaped() != nil,
isPackage || childOfPackage,
invalidPrevDelta,
nil)
if err != nil {
@ -914,8 +936,8 @@ func (c *Collections) processItem(
// Always add a file to the excluded list. The file may have been
// renamed/moved/modified, so we still have to drop the
// original one and download a fresh copy.
excluded[itemID+metadata.DataFileSuffix] = struct{}{}
excluded[itemID+metadata.MetaFileSuffix] = struct{}{}
excludedItemIDs[itemID+metadata.DataFileSuffix] = struct{}{}
excludedItemIDs[itemID+metadata.MetaFileSuffix] = struct{}{}
}
default:

365
src/internal/m365/collection/drive/collections_test.go
View File

@ -109,7 +109,7 @@ func TestOneDriveCollectionsUnitSuite(t *testing.T) {
suite.Run(t, &OneDriveCollectionsUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func getDelList(files ...string) map[string]struct{} {
func makeExcludeMap(files ...string) map[string]struct{} {
delList := map[string]struct{}{}
for _, file := range files {
delList[file+metadata.DataFileSuffix] = struct{}{}
@ -127,7 +127,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
tenant = "tenant"
user = "user"
folder = "/folder"
folderSub = "/folder/subfolder"
subFolder = "/subfolder"
pkg = "/package"
)
@ -137,18 +137,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedStatePath := getExpectedStatePathGenerator(suite.T(), bh, tenant, testBaseDrivePath)
tests := []struct {
name string
items []models.DriveItemable
inputFolderMap map[string]string
scope selectors.OneDriveScope
expect assert.ErrorAssertionFunc
expectedCollectionIDs map[string]statePath
expectedItemCount int
expectedContainerCount int
expectedFileCount int
expectedSkippedCount int
expectedPrevPaths map[string]string
expectedExcludes map[string]struct{}
name string
items []models.DriveItemable
inputFolderMap map[string]string
topLevelPackages map[string]struct{}
scope selectors.OneDriveScope
expect assert.ErrorAssertionFunc
expectedCollectionIDs map[string]statePath
expectedItemCount int
expectedContainerCount int
expectedFileCount int
expectedSkippedCount int
expectedPrevPaths map[string]string
expectedExcludes map[string]struct{}
expectedTopLevelPackages map[string]struct{}
expectedCountPackages int
}{
{
name: "Invalid item",
@ -156,9 +159,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveRootItem("root"),
driveItem("item", "item", testBaseDrivePath, "root", false, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.Error,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.Error,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -166,7 +170,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
},
{
name: "Single File",
@ -174,9 +179,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveRootItem("root"),
driveItem("file", "file", testBaseDrivePath, "root", true, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -187,7 +193,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: getDelList("file"),
expectedExcludes: makeExcludeMap("file"),
expectedTopLevelPackages: map[string]struct{}{},
},
{
name: "Single Folder",
@ -195,9 +202,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveRootItem("root"),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, folder),
@ -206,9 +214,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"root": expectedPath(""),
"folder": expectedPath("/folder"),
},
expectedItemCount: 1,
expectedContainerCount: 2,
expectedExcludes: map[string]struct{}{},
expectedItemCount: 1,
expectedContainerCount: 2,
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
},
{
name: "Single Package",
@ -216,9 +225,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveRootItem("root"),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"package": expectedStatePath(data.NewState, pkg),
@ -230,6 +240,42 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedItemCount: 1,
expectedContainerCount: 2,
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{
expectedPath("/package"): {},
},
expectedCountPackages: 1,
},
{
name: "Single Package with subfolder",
items: []models.DriveItemable{
driveRootItem("root"),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("folder", "folder", testBaseDrivePath+pkg, "package", false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+pkg, "package", false, true, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"package": expectedStatePath(data.NewState, pkg),
"folder": expectedStatePath(data.NewState, pkg+folder),
"subfolder": expectedStatePath(data.NewState, pkg+subFolder),
},
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
"package": expectedPath(pkg),
"folder": expectedPath(pkg + folder),
"subfolder": expectedPath(pkg + subFolder),
},
expectedItemCount: 3,
expectedContainerCount: 4,
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{
expectedPath(pkg): {},
},
expectedCountPackages: 3,
},
{
name: "1 root file, 1 folder, 1 package, 2 files, 3 collections",
@ -241,9 +287,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, folder),
@ -257,7 +304,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/folder"),
"package": expectedPath("/package"),
},
expectedExcludes: getDelList("fileInRoot", "fileInFolder", "fileInPackage"),
expectedTopLevelPackages: map[string]struct{}{
expectedPath("/package"): {},
},
expectedCountPackages: 1,
expectedExcludes: makeExcludeMap("fileInRoot", "fileInFolder", "fileInPackage"),
},
{
name: "contains folder selector",
@ -266,19 +317,27 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folder+subFolder, "subfolder", false, true, false),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false),
driveItem(
"fileInFolder2",
"fileInFolder2",
testBaseDrivePath+folder+subFolder+folder,
"folder2",
true,
false,
false),
driveItem("fileInFolderPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"folder": expectedStatePath(data.NewState, folder),
"subfolder": expectedStatePath(data.NewState, folderSub),
"folder2": expectedStatePath(data.NewState, folderSub+folder),
"subfolder": expectedStatePath(data.NewState, folder+subFolder),
"folder2": expectedStatePath(data.NewState, folder+subFolder+folder),
},
expectedItemCount: 5,
expectedFileCount: 2,
@ -287,10 +346,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
// parent path since we only check later if something is a folder or not.
expectedPrevPaths: map[string]string{
"folder": expectedPath(folder),
"subfolder": expectedPath(folderSub),
"folder2": expectedPath(folderSub + folder),
"subfolder": expectedPath(folder + subFolder),
"folder2": expectedPath(folder + subFolder + folder),
},
expectedExcludes: getDelList("fileInFolder", "fileInFolder2"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("fileInFolder", "fileInFolder2"),
},
{
name: "prefix subfolder selector",
@ -299,28 +359,36 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("fileInRoot", "fileInRoot", testBaseDrivePath, "root", true, false, false),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folderSub, "subfolder", false, true, false),
driveItem("folder2", "folder", testBaseDrivePath+folder+subFolder, "subfolder", false, true, false),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInFolder2", "fileInFolder2", testBaseDrivePath+folderSub+folder, "folder2", true, false, false),
driveItem(
"fileInFolder2",
"fileInFolder2",
testBaseDrivePath+folder+subFolder+folder,
"folder2",
true,
false,
false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).
Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"subfolder": expectedStatePath(data.NewState, folderSub),
"folder2": expectedStatePath(data.NewState, folderSub+folder),
"subfolder": expectedStatePath(data.NewState, folder+subFolder),
"folder2": expectedStatePath(data.NewState, folder+subFolder+folder),
},
expectedItemCount: 3,
expectedFileCount: 1,
expectedContainerCount: 2,
expectedPrevPaths: map[string]string{
"subfolder": expectedPath(folderSub),
"folder2": expectedPath(folderSub + folder),
"subfolder": expectedPath(folder + subFolder),
"folder2": expectedPath(folder + subFolder + folder),
},
expectedExcludes: getDelList("fileInFolder2"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("fileInFolder2"),
},
{
name: "match subfolder selector",
@ -331,23 +399,32 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("subfolder", "subfolder", testBaseDrivePath+folder, "folder", false, true, false),
driveItem("package", "package", testBaseDrivePath, "root", false, false, true),
driveItem("fileInFolder", "fileInFolder", testBaseDrivePath+folder, "folder", true, false, false),
driveItem("fileInSubfolder", "fileInSubfolder", testBaseDrivePath+folderSub, "subfolder", true, false, false),
driveItem(
"fileInSubfolder",
"fileInSubfolder",
testBaseDrivePath+folder+subFolder,
"subfolder",
true,
false,
false),
driveItem("fileInPackage", "fileInPackage", testBaseDrivePath+pkg, "package", true, false, false),
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"subfolder": expectedStatePath(data.NewState, folderSub),
"subfolder": expectedStatePath(data.NewState, folder+subFolder),
},
expectedItemCount: 2,
expectedFileCount: 1,
expectedContainerCount: 1,
// No child folders for subfolder so nothing here.
expectedPrevPaths: map[string]string{
"subfolder": expectedPath(folderSub),
"subfolder": expectedPath(folder + subFolder),
},
expectedExcludes: getDelList("fileInSubfolder"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("fileInSubfolder"),
},
{
name: "not moved folder tree",
@ -357,10 +434,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
},
inputFolderMap: map[string]string{
"folder": expectedPath(folder),
"subfolder": expectedPath(folderSub),
"subfolder": expectedPath(folder + subFolder),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NotMovedState, folder),
@ -371,9 +449,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
"folder": expectedPath(folder),
"subfolder": expectedPath(folderSub),
"subfolder": expectedPath(folder + subFolder),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "moved folder tree",
@ -385,8 +464,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/a-folder"),
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, folder, "/a-folder"),
@ -397,9 +477,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
"folder": expectedPath(folder),
"subfolder": expectedPath(folderSub),
"subfolder": expectedPath(folder + subFolder),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "moved folder tree with file no previous",
@ -409,9 +490,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false),
driveItem("folder", "folder2", testBaseDrivePath, "root", false, true, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, "/folder2"),
@ -423,7 +505,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"root": expectedPath(""),
"folder": expectedPath("/folder2"),
},
expectedExcludes: getDelList("file"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("file"),
},
{
name: "moved folder tree with file no previous 1",
@ -432,9 +515,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, folder),
@ -446,7 +530,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"root": expectedPath(""),
"folder": expectedPath(folder),
},
expectedExcludes: getDelList("file"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("file"),
},
{
name: "moved folder tree and subfolder 1",
@ -459,8 +544,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/a-folder"),
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, folder, "/a-folder"),
@ -474,7 +560,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath(folder),
"subfolder": expectedPath("/subfolder"),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "moved folder tree and subfolder 2",
@ -487,8 +574,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/a-folder"),
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, folder, "/a-folder"),
@ -502,7 +590,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath(folder),
"subfolder": expectedPath("/subfolder"),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "move subfolder when moving parent",
@ -528,13 +617,14 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/a-folder"),
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, folder, "/a-folder"),
"folder2": expectedStatePath(data.NewState, "/folder2"),
"subfolder": expectedStatePath(data.MovedState, folderSub, "/a-folder/subfolder"),
"subfolder": expectedStatePath(data.MovedState, folder+subFolder, "/a-folder/subfolder"),
},
expectedItemCount: 5,
expectedFileCount: 2,
@ -545,7 +635,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder2": expectedPath("/folder2"),
"subfolder": expectedPath("/folder/subfolder"),
},
expectedExcludes: getDelList("itemInSubfolder", "itemInFolder2"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("itemInSubfolder", "itemInFolder2"),
},
{
name: "moved folder tree multiple times",
@ -559,8 +650,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/a-folder"),
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, "/folder2", "/a-folder"),
@ -573,7 +665,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/folder2"),
"subfolder": expectedPath("/folder2/subfolder"),
},
expectedExcludes: getDelList("file"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("file"),
},
{
name: "deleted folder and package",
@ -587,8 +680,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/folder"),
"package": expectedPath("/package"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.DeletedState, folder),
@ -600,7 +694,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "delete folder without previous",
@ -611,8 +706,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
inputFolderMap: map[string]string{
"root": expectedPath(""),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -622,7 +718,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "delete folder tree move subfolder",
@ -636,12 +733,13 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/folder"),
"subfolder": expectedPath("/folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.DeletedState, folder),
"subfolder": expectedStatePath(data.MovedState, "/subfolder", folderSub),
"subfolder": expectedStatePath(data.MovedState, "/subfolder", folder+subFolder),
},
expectedItemCount: 1,
expectedFileCount: 0,
@ -650,7 +748,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"root": expectedPath(""),
"subfolder": expectedPath("/subfolder"),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "delete file",
@ -661,8 +760,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
inputFolderMap: map[string]string{
"root": expectedPath(""),
},
scope: anyFolder,
expect: assert.NoError,
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -672,7 +772,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: getDelList("item"),
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: makeExcludeMap("item"),
},
{
name: "item before parent errors",
@ -681,9 +782,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("file", "file", testBaseDrivePath+"/folder", "folder", true, false, false),
driveItem("folder", "folder", testBaseDrivePath, "root", false, true, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.Error,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.Error,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -693,7 +795,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
expectedPrevPaths: map[string]string{
"root": expectedPath(""),
},
expectedExcludes: map[string]struct{}{},
expectedTopLevelPackages: map[string]struct{}{},
expectedExcludes: map[string]struct{}{},
},
{
name: "1 root file, 1 folder, 1 package, 1 good file, 1 malware",
@ -705,9 +808,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
driveItem("goodFile", "goodFile", testBaseDrivePath+folder, "folder", true, false, false),
malwareItem("malwareFile", "malwareFile", testBaseDrivePath+folder, "folder", true, false, false),
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
inputFolderMap: map[string]string{},
scope: anyFolder,
topLevelPackages: map[string]struct{}{},
expect: assert.NoError,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, folder),
@ -722,7 +826,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"folder": expectedPath("/folder"),
"package": expectedPath("/package"),
},
expectedExcludes: getDelList("fileInRoot", "goodFile"),
expectedTopLevelPackages: map[string]struct{}{
expectedPath("/package"): {},
},
expectedCountPackages: 1,
expectedExcludes: makeExcludeMap("fileInRoot", "goodFile"),
},
}
@ -772,7 +880,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
"General",
test.inputFolderMap,
excludes,
"smarf",
test.topLevelPackages,
"prevdelta",
errs)
test.expect(t, err, clues.ToCore(err))
assert.ElementsMatch(
@ -782,7 +891,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
assert.Equal(t, test.expectedSkippedCount, len(errs.Skipped()), "skipped items")
assert.Equal(t, test.expectedSkippedCount, len(errs.Skipped()), "skipped item count")
for id, sp := range test.expectedCollectionIDs {
if !assert.Containsf(t, c.CollectionMap[driveID], id, "missing collection with id %s", id) {
@ -795,8 +904,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
assert.Equalf(t, sp.prevPath, c.CollectionMap[driveID][id].PreviousPath(), "prev path for collection %s", id)
}
assert.Equal(t, test.expectedPrevPaths, newPrevPaths, "metadata paths")
assert.Equal(t, test.expectedExcludes, excludes, "exclude list")
assert.Equal(t, test.expectedPrevPaths, newPrevPaths, "previous paths")
assert.Equal(t, test.expectedTopLevelPackages, test.topLevelPackages, "top level packages")
assert.Equal(t, test.expectedExcludes, excludes, "excluded item IDs map")
var countPackages int
for _, drives := range c.CollectionMap {
for _, col := range drives {
if col.isPackageOrChildOfPackage {
countPackages++
}
}
}
assert.Equal(t, test.expectedCountPackages, countPackages, "count of collections marked as packages")
})
}
}
@ -808,7 +930,6 @@ func (suite *OneDriveCollectionsUnitSuite) TestDeserializeMetadata() {
driveID2 := "2"
deltaURL1 := "url/1"
deltaURL2 := "url/2"
folderID1 := "folder1"
folderID2 := "folder2"
path1 := "folder1/path"
@ -1273,7 +1394,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
driveID1: {"root": rootFolderPath1},
},
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{
rootFolderPath1: getDelList("file"),
rootFolderPath1: makeExcludeMap("file"),
}),
},
{
@ -1307,7 +1428,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
driveID1: {"root": rootFolderPath1},
},
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{
rootFolderPath1: getDelList("file"),
rootFolderPath1: makeExcludeMap("file"),
}),
},
{
@ -1426,7 +1547,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
},
},
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{
rootFolderPath1: getDelList("file"),
rootFolderPath1: makeExcludeMap("file"),
}),
},
{

5
src/internal/m365/service/sharepoint/backup_test.go
View File

@ -101,6 +101,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
collMap = map[string]map[string]*drive.Collection{
driveID: {},
}
topLevelPackages = map[string]struct{}{}
)
mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{
@ -127,7 +128,8 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
"General",
paths,
excluded,
"",
topLevelPackages,
"notempty",
fault.New(true))
test.expect(t, err, clues.ToCore(err))
@ -135,6 +137,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
assert.Empty(t, topLevelPackages, "should not find package type folders")
for _, collPath := range test.expectedCollectionIDs {
assert.Contains(t, c.CollectionMap[driveID], collPath)