apply channels and streaming to drive collections
Updates drive collection processing with the new pattern of streaming pages from the api using channels. This is the last in a multipart update that has been separated for ease of review. Everything should now pass.
This commit is contained in:
ryanfkeepers
parent
45aac829dc
commit
c805c8f8e5
@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -272,13 +273,6 @@ func (c *Collections) Get(
|
||||
excludedItemIDs = map[string]struct{}{}
|
||||
oldPrevPaths = oldPrevPathsByDriveID[driveID]
|
||||
prevDeltaLink = prevDriveIDToDelta[driveID]
|
||||
|
||||
// itemCollection is used to identify which collection a
|
||||
// file belongs to. This is useful to delete a file from the
|
||||
// collection it was previously in, in case it was moved to a
|
||||
// different collection within the same delta query
|
||||
// item ID -> item ID
|
||||
itemCollection = map[string]string{}
|
||||
)
|
||||
|
||||
delete(driveTombstones, driveID)
|
||||
@ -295,13 +289,16 @@ func (c *Collections) Get(
|
||||
"previous metadata for drive",
|
||||
"num_paths_entries", len(oldPrevPaths))
|
||||
|
||||
items, du, err := c.handler.EnumerateDriveItemsDelta(
|
||||
ictx,
|
||||
du, newPrevPaths, err := c.PopulateDriveCollections(
|
||||
ctx,
|
||||
driveID,
|
||||
driveName,
|
||||
oldPrevPaths,
|
||||
excludedItemIDs,
|
||||
prevDeltaLink,
|
||||
api.DefaultDriveItemProps())
|
||||
errs)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
return nil, false, clues.Stack(err)
|
||||
}
|
||||
|
||||
// It's alright to have an empty folders map (i.e. no folders found) but not
|
||||
@ -313,20 +310,6 @@ func (c *Collections) Get(
|
||||
driveIDToDeltaLink[driveID] = du.URL
|
||||
}
|
||||
|
||||
newPrevPaths, err := c.UpdateCollections(
|
||||
ctx,
|
||||
driveID,
|
||||
driveName,
|
||||
items,
|
||||
oldPrevPaths,
|
||||
itemCollection,
|
||||
excludedItemIDs,
|
||||
du.Reset,
|
||||
errs)
|
||||
if err != nil {
|
||||
return nil, false, clues.Stack(err)
|
||||
}
|
||||
|
||||
// Avoid the edge case where there's no paths but we do have a valid delta
|
||||
// token. We can accomplish this by adding an empty paths map for this
|
||||
// drive. If we don't have this then the next backup won't use the delta
|
||||
@ -688,37 +671,104 @@ func (c *Collections) getCollectionPath(
|
||||
return collectionPath, nil
|
||||
}
|
||||
|
||||
// UpdateCollections initializes and adds the provided drive items to Collections
|
||||
// A new collection is created for every drive folder (or package).
|
||||
// PopulateDriveCollections initializes and adds the provided drive items to Collections
|
||||
// A new collection is created for every drive folder.
|
||||
// oldPrevPaths is the unchanged data that was loaded from the metadata file.
|
||||
// This map is not modified during the call.
|
||||
// currPrevPaths starts as a copy of oldPaths and is updated as changes are found in
|
||||
// the returned results. Items are added to this collection throughout the call.
|
||||
// newPrevPaths, ie: the items added during this call, get returned as a map.
|
||||
func (c *Collections) UpdateCollections(
|
||||
func (c *Collections) PopulateDriveCollections(
|
||||
ctx context.Context,
|
||||
driveID, driveName string,
|
||||
items []models.DriveItemable,
|
||||
oldPrevPaths map[string]string,
|
||||
currPrevPaths map[string]string,
|
||||
excluded map[string]struct{},
|
||||
invalidPrevDelta bool,
|
||||
excludedItemIDs map[string]struct{},
|
||||
prevDeltaLink string,
|
||||
errs *fault.Bus,
|
||||
) (map[string]string, error) {
|
||||
) (api.DeltaUpdate, map[string]string, error) {
|
||||
var (
|
||||
el = errs.Local()
|
||||
newPrevPaths = map[string]string{}
|
||||
invalidPrevDelta = len(prevDeltaLink) == 0
|
||||
ch = make(chan api.NextPage[models.DriveItemable], 1)
|
||||
wg = sync.WaitGroup{}
|
||||
|
||||
// currPrevPaths is used to identify which collection a
|
||||
// file belongs to. This is useful to delete a file from the
|
||||
// collection it was previously in, in case it was moved to a
|
||||
// different collection within the same delta query
|
||||
// item ID -> item ID
|
||||
currPrevPaths = map[string]string{}
|
||||
)
|
||||
|
||||
if !invalidPrevDelta {
|
||||
maps.Copy(newPrevPaths, oldPrevPaths)
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
for pg := range ch {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
// exhaust the channel to ensure it closes
|
||||
continue
|
||||
}
|
||||
|
||||
if pg.Reset {
|
||||
newPrevPaths = map[string]string{}
|
||||
currPrevPaths = map[string]string{}
|
||||
c.CollectionMap[driveID] = map[string]*Collection{}
|
||||
invalidPrevDelta = true
|
||||
}
|
||||
|
||||
for _, item := range pg.Items {
|
||||
if el.Failure() != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := c.processItem(
|
||||
ctx,
|
||||
item,
|
||||
driveID,
|
||||
driveName,
|
||||
oldPrevPaths,
|
||||
currPrevPaths,
|
||||
newPrevPaths,
|
||||
excludedItemIDs,
|
||||
invalidPrevDelta,
|
||||
el)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.Stack(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
du, err := c.handler.EnumerateDriveItemsDelta(
|
||||
ctx,
|
||||
ch,
|
||||
driveID,
|
||||
prevDeltaLink)
|
||||
if err != nil {
|
||||
return du, nil, clues.Stack(err)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return du, newPrevPaths, el.Failure()
|
||||
}
|
||||
|
||||
func (c *Collections) processItem(
|
||||
ctx context.Context,
|
||||
item models.DriveItemable,
|
||||
driveID, driveName string,
|
||||
oldPrevPaths, currPrevPaths, newPrevPaths map[string]string,
|
||||
excluded map[string]struct{},
|
||||
invalidPrevDelta bool,
|
||||
skipper fault.AddSkipper,
|
||||
) error {
|
||||
var (
|
||||
itemID = ptr.Val(item.GetId())
|
||||
itemName = ptr.Val(item.GetName())
|
||||
@ -738,15 +788,15 @@ func (c *Collections) UpdateCollections(
|
||||
skip = fault.ContainerSkip(fault.SkipMalware, driveID, itemID, itemName, addtl)
|
||||
}
|
||||
|
||||
errs.AddSkip(ctx, skip)
|
||||
skipper.AddSkip(ctx, skip)
|
||||
logger.Ctx(ctx).Infow("malware detected", "item_details", addtl)
|
||||
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deleted file or folder.
|
||||
if item.GetDeleted() != nil {
|
||||
if err := c.handleDelete(
|
||||
err := c.handleDelete(
|
||||
itemID,
|
||||
driveID,
|
||||
oldPrevPaths,
|
||||
@ -754,26 +804,22 @@ func (c *Collections) UpdateCollections(
|
||||
newPrevPaths,
|
||||
isFolder,
|
||||
excluded,
|
||||
invalidPrevDelta); err != nil {
|
||||
return nil, clues.Stack(err).WithClues(ictx)
|
||||
}
|
||||
invalidPrevDelta)
|
||||
|
||||
continue
|
||||
return clues.Stack(err).WithClues(ictx).OrNil()
|
||||
}
|
||||
|
||||
collectionPath, err := c.getCollectionPath(driveID, item)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.Stack(err).
|
||||
return clues.Stack(err).
|
||||
WithClues(ictx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
|
||||
continue
|
||||
Label(fault.LabelForceNoBackupCreation)
|
||||
}
|
||||
|
||||
// Skip items that don't match the folder selectors we were given.
|
||||
if shouldSkip(ctx, collectionPath, c.handler, driveName) {
|
||||
logger.Ctx(ictx).Debugw("path not selected", "skipped_path", collectionPath.String())
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
|
||||
switch {
|
||||
@ -785,9 +831,9 @@ func (c *Collections) UpdateCollections(
|
||||
if ok {
|
||||
prevPath, err = path.FromDataLayerPath(prevPathStr, false)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.Wrap(err, "invalid previous path").
|
||||
return clues.Wrap(err, "invalid previous path").
|
||||
WithClues(ictx).
|
||||
With("prev_path_string", path.LoggableDir(prevPathStr)))
|
||||
With("prev_path_string", path.LoggableDir(prevPathStr))
|
||||
}
|
||||
} else if item.GetRoot() != nil {
|
||||
// Root doesn't move or get renamed.
|
||||
@ -799,13 +845,17 @@ func (c *Collections) UpdateCollections(
|
||||
// update newPaths so we don't accidentally clobber previous deletes.
|
||||
updatePath(newPrevPaths, itemID, collectionPath.String())
|
||||
|
||||
found, err := updateCollectionPaths(driveID, itemID, c.CollectionMap, collectionPath)
|
||||
found, err := updateCollectionPaths(
|
||||
driveID,
|
||||
itemID,
|
||||
c.CollectionMap,
|
||||
collectionPath)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err).WithClues(ictx)
|
||||
return clues.Stack(err).WithClues(ictx)
|
||||
}
|
||||
|
||||
if found {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
|
||||
colScope := CollectionScopeFolder
|
||||
@ -825,7 +875,7 @@ func (c *Collections) UpdateCollections(
|
||||
invalidPrevDelta,
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err).WithClues(ictx)
|
||||
return clues.Stack(err).WithClues(ictx)
|
||||
}
|
||||
|
||||
col.driveName = driveName
|
||||
@ -834,7 +884,7 @@ func (c *Collections) UpdateCollections(
|
||||
c.NumContainers++
|
||||
|
||||
if item.GetRoot() != nil {
|
||||
continue
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add an entry to fetch permissions into this collection. This assumes
|
||||
@ -847,7 +897,7 @@ func (c *Collections) UpdateCollections(
|
||||
case item.GetFile() != nil:
|
||||
// Deletions are handled above so this is just moves/renames.
|
||||
if len(ptr.Val(item.GetParentReference().GetId())) == 0 {
|
||||
return nil, clues.New("file without parent ID").WithClues(ictx)
|
||||
return clues.New("file without parent ID").WithClues(ictx)
|
||||
}
|
||||
|
||||
// Get the collection for this item.
|
||||
@ -856,7 +906,7 @@ func (c *Collections) UpdateCollections(
|
||||
|
||||
collection, ok := c.CollectionMap[driveID][parentID]
|
||||
if !ok {
|
||||
return nil, clues.New("item seen before parent folder").WithClues(ictx)
|
||||
return clues.New("item seen before parent folder").WithClues(ictx)
|
||||
}
|
||||
|
||||
// This will only kick in if the file was moved multiple times
|
||||
@ -866,13 +916,13 @@ func (c *Collections) UpdateCollections(
|
||||
if ok {
|
||||
prevColl, found := c.CollectionMap[driveID][prevParentContainerID]
|
||||
if !found {
|
||||
return nil, clues.New("previous collection not found").
|
||||
return clues.New("previous collection not found").
|
||||
With("prev_parent_container_id", prevParentContainerID).
|
||||
WithClues(ictx)
|
||||
}
|
||||
|
||||
if ok := prevColl.Remove(itemID); !ok {
|
||||
return nil, clues.New("removing item from prev collection").
|
||||
return clues.New("removing item from prev collection").
|
||||
With("prev_parent_container_id", prevParentContainerID).
|
||||
WithClues(ictx)
|
||||
}
|
||||
@ -899,13 +949,12 @@ func (c *Collections) UpdateCollections(
|
||||
}
|
||||
|
||||
default:
|
||||
el.AddRecoverable(ictx, clues.New("item is neither folder nor file").
|
||||
return clues.New("item is neither folder nor file").
|
||||
WithClues(ictx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
}
|
||||
Label(fault.LabelForceNoBackupCreation)
|
||||
}
|
||||
|
||||
return newPrevPaths, el.Failure()
|
||||
return nil
|
||||
}
|
||||
|
||||
type dirScopeChecker interface {
|
||||
@ -913,7 +962,12 @@ type dirScopeChecker interface {
|
||||
IncludesDir(dir string) bool
|
||||
}
|
||||
|
||||
func shouldSkip(ctx context.Context, drivePath path.Path, dsc dirScopeChecker, driveName string) bool {
|
||||
func shouldSkip(
|
||||
ctx context.Context,
|
||||
drivePath path.Path,
|
||||
dsc dirScopeChecker,
|
||||
driveName string,
|
||||
) bool {
|
||||
return !includePath(ctx, dsc, drivePath) ||
|
||||
(drivePath.Category() == path.LibrariesCategory && restrictedDirectory == driveName)
|
||||
}
|
||||
|
||||
@ -119,7 +119,7 @@ func getDelList(files ...string) map[string]struct{} {
|
||||
return delList
|
||||
}
|
||||
|
||||
func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() {
|
||||
func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
|
||||
anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any())[0]
|
||||
|
||||
const (
|
||||
@ -690,7 +690,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() {
|
||||
expectedItemCount: 0,
|
||||
expectedFileCount: 0,
|
||||
expectedContainerCount: 1,
|
||||
expectedPrevPaths: nil,
|
||||
expectedPrevPaths: map[string]string{
|
||||
"root": expectedPath(""),
|
||||
},
|
||||
expectedExcludes: map[string]struct{}{},
|
||||
},
|
||||
{
|
||||
@ -732,15 +734,31 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() {
|
||||
defer flush()
|
||||
|
||||
var (
|
||||
mbh = mock.DefaultOneDriveBH(user)
|
||||
du = api.DeltaUpdate{
|
||||
URL: "notempty",
|
||||
Reset: false,
|
||||
}
|
||||
excludes = map[string]struct{}{}
|
||||
currPrevPaths = map[string]string{}
|
||||
errs = fault.New(true)
|
||||
)
|
||||
|
||||
maps.Copy(currPrevPaths, test.inputFolderMap)
|
||||
mbh.DriveItemEnumeration = mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID: {
|
||||
{Items: test.items},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{driveID: du},
|
||||
}
|
||||
|
||||
sel := selectors.NewOneDriveBackup([]string{user})
|
||||
sel.Include([]selectors.OneDriveScope{test.scope})
|
||||
|
||||
mbh.Sel = sel.Selector
|
||||
|
||||
c := NewCollections(
|
||||
&itemBackupHandler{api.Drives{}, user, test.scope},
|
||||
mbh,
|
||||
tenant,
|
||||
idname.NewProvider(user, user),
|
||||
nil,
|
||||
@ -748,18 +766,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestUpdateCollections() {
|
||||
|
||||
c.CollectionMap[driveID] = map[string]*Collection{}
|
||||
|
||||
newPrevPaths, err := c.UpdateCollections(
|
||||
_, newPrevPaths, err := c.PopulateDriveCollections(
|
||||
ctx,
|
||||
driveID,
|
||||
"General",
|
||||
test.items,
|
||||
test.inputFolderMap,
|
||||
currPrevPaths,
|
||||
excludes,
|
||||
false,
|
||||
"smarf",
|
||||
errs)
|
||||
test.expect(t, err, clues.ToCore(err))
|
||||
assert.Equal(t, len(test.expectedCollectionIDs), len(c.CollectionMap[driveID]), "total collections")
|
||||
assert.ElementsMatch(
|
||||
t,
|
||||
maps.Keys(test.expectedCollectionIDs),
|
||||
maps.Keys(c.CollectionMap[driveID]))
|
||||
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
|
||||
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
|
||||
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
|
||||
@ -1166,7 +1185,6 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
tenant = "a-tenant"
|
||||
user = "a-user"
|
||||
empty = ""
|
||||
next = "next"
|
||||
delta = "delta1"
|
||||
delta2 = "delta2"
|
||||
)
|
||||
@ -1208,7 +1226,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
table := []struct {
|
||||
name string
|
||||
drives []models.Driveable
|
||||
items map[string][]apiMock.PagerResult[models.DriveItemable]
|
||||
enumerator mock.EnumeratesDriveItemsDelta[models.DriveItemable]
|
||||
canUsePreviousBackup bool
|
||||
errCheck assert.ErrorAssertionFunc
|
||||
prevFolderPaths map[string]map[string]string
|
||||
@ -1227,16 +1245,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_DelFileOnly_NoFolders_NoErrors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"), // will be present, not needed
|
||||
delItem("file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1259,16 +1280,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_NoFolderDeltas_NoErrors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("file", "file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1291,18 +1315,20 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_NoErrors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1329,19 +1355,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_NoErrors_FileRenamedMultiple",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
driveItem("file", "file2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1368,18 +1396,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_NoErrors_FileMovedMultiple",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
driveItem("file", "file2", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1408,18 +1439,20 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_EmptyDelta_NoErrors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &empty, // probably will never happen with graph
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: empty, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1446,28 +1479,151 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_TwoItemPages_NoErrors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
ResetDelta: true,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
driveID1: {},
|
||||
},
|
||||
expectedCollections: map[string]map[data.CollectionState][]string{
|
||||
rootFolderPath1: {data.NewState: {}},
|
||||
folderPath1: {data.NewState: {"folder", "file", "file2"}},
|
||||
},
|
||||
expectedDeltaURLs: map[string]string{
|
||||
driveID1: delta,
|
||||
},
|
||||
expectedFolderPaths: map[string]map[string]string{
|
||||
driveID1: {
|
||||
"root": rootFolderPath1,
|
||||
"folder": folderPath1,
|
||||
},
|
||||
},
|
||||
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}),
|
||||
doNotMergeItems: map[string]bool{
|
||||
rootFolderPath1: true,
|
||||
folderPath1: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OneDrive_TwoItemPages_WithReset",
|
||||
drives: []models.Driveable{drive1},
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{},
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
driveID1: {},
|
||||
},
|
||||
expectedCollections: map[string]map[data.CollectionState][]string{
|
||||
rootFolderPath1: {data.NewState: {}},
|
||||
folderPath1: {data.NewState: {"folder", "file", "file2"}},
|
||||
},
|
||||
expectedDeltaURLs: map[string]string{
|
||||
driveID1: delta,
|
||||
},
|
||||
expectedFolderPaths: map[string]map[string]string{
|
||||
driveID1: {
|
||||
"root": rootFolderPath1,
|
||||
"folder": folderPath1,
|
||||
},
|
||||
},
|
||||
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}),
|
||||
doNotMergeItems: map[string]bool{
|
||||
rootFolderPath1: true,
|
||||
folderPath1: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OneDrive_TwoItemPages_WithResetCombinedWithItems",
|
||||
drives: []models.Driveable{drive1},
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1498,29 +1654,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
drive1,
|
||||
drive2,
|
||||
},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
}},
|
||||
},
|
||||
driveID2: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root2"),
|
||||
driveItem("folder2", "folder", driveBasePath2, "root2", false, true, false),
|
||||
driveItem("file2", "file", driveBasePath2+"/folder", "folder2", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta2,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
driveID2: {URL: delta2, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1562,29 +1717,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
drive1,
|
||||
drive2,
|
||||
},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
}},
|
||||
},
|
||||
driveID2: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath2, "root", false, true, false),
|
||||
driveItem("file2", "file", driveBasePath2+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta2,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
driveID2: {URL: delta2, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -1623,12 +1777,16 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_Errors",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Err: assert.AnError,
|
||||
{Items: []models.DriveItemable{}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {},
|
||||
},
|
||||
Err: map[string]error{driveID1: assert.AnError},
|
||||
},
|
||||
canUsePreviousBackup: false,
|
||||
errCheck: assert.Error,
|
||||
@ -1641,37 +1799,41 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
expectedDelList: nil,
|
||||
},
|
||||
{
|
||||
name: "OneDrive_TwoItemPage_NoDeltaError",
|
||||
name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("file", "file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
Items: []models.DriveItemable{},
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
driveID1: {
|
||||
"root": rootFolderPath1,
|
||||
"folder": folderPath1,
|
||||
},
|
||||
},
|
||||
expectedCollections: map[string]map[data.CollectionState][]string{
|
||||
rootFolderPath1: {data.NotMovedState: {"file"}},
|
||||
expectedPath1("/folder"): {data.NewState: {"folder", "file2"}},
|
||||
rootFolderPath1: {data.NewState: {}},
|
||||
expectedPath1("/folder"): {data.DeletedState: {}},
|
||||
expectedPath1("/folder2"): {data.NewState: {"folder2", "file"}},
|
||||
},
|
||||
expectedDeltaURLs: map[string]string{
|
||||
driveID1: delta,
|
||||
@ -1679,30 +1841,40 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
expectedFolderPaths: map[string]map[string]string{
|
||||
driveID1: {
|
||||
"root": rootFolderPath1,
|
||||
"folder": folderPath1,
|
||||
"folder2": expectedPath1("/folder2"),
|
||||
},
|
||||
},
|
||||
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{
|
||||
rootFolderPath1: getDelList("file", "file2"),
|
||||
}),
|
||||
doNotMergeItems: map[string]bool{},
|
||||
expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}),
|
||||
doNotMergeItems: map[string]bool{
|
||||
rootFolderPath1: true,
|
||||
folderPath1: true,
|
||||
expectedPath1("/folder2"): true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder",
|
||||
name: "OneDrive_OneItemPage_InvalidPrevDeltaCombinedWithItems_DeleteNonExistentFolder",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{},
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1735,19 +1907,38 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
// on the first page, if this is the total data, we'd expect both folder and folder2
|
||||
// since new previousPaths merge with the old previousPaths.
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{},
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
// but after a delta reset, we treat this as the total end set of folders, which means
|
||||
// we don't expect folder to exist any longer.
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1783,29 +1974,32 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "OneDrive Two Item Pages with Malware",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
malwareItem("malware", "malware", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
malwareItem("malware2", "malware2", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1832,31 +2026,39 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
expectedSkippedCount: 2,
|
||||
},
|
||||
{
|
||||
name: "One Drive Deleted Folder In New Results",
|
||||
name: "One Drive Deleted Folder In New Results With Invalid Delta",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file2", "file2", driveBasePath1+"/folder2", "folder2", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Reset: true,
|
||||
},
|
||||
{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
delItem("folder2", driveBasePath1, "root", false, true, false),
|
||||
delItem("file2", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta2,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta2, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1888,20 +2090,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "One Drive Random Folder Delete",
|
||||
name: "One Drive Folder Delete After Invalid Delta",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("folder", driveBasePath1, "root", false, true, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
Reset: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1929,20 +2136,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "One Drive Random Item Delete",
|
||||
name: "One Drive Item Delete After Invalid Delta",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
Reset: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -1969,27 +2181,30 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "One Drive Folder Made And Deleted",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("folder", driveBasePath1, "root", false, true, false),
|
||||
delItem("file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta2,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta2, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -2014,26 +2229,29 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "One Drive Item Made And Deleted",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
|
||||
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
|
||||
},
|
||||
NextLink: &next,
|
||||
},
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
prevFolderPaths: map[string]map[string]string{
|
||||
@ -2061,17 +2279,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "One Drive Random Folder Delete",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("folder", driveBasePath1, "root", false, true, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -2097,17 +2317,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "One Drive Random Item Delete",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"),
|
||||
delItem("file", driveBasePath1, "root", true, false, false),
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
ResetDelta: true,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta, Reset: true},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -2133,15 +2355,18 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
{
|
||||
name: "TwoPriorDrives_OneTombstoned",
|
||||
drives: []models.Driveable{drive1},
|
||||
items: map[string][]apiMock.PagerResult[models.DriveItemable]{
|
||||
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
|
||||
Pages: map[string][]api.NextPage[models.DriveItemable]{
|
||||
driveID1: {
|
||||
{
|
||||
Values: []models.DriveItemable{
|
||||
{Items: []models.DriveItemable{
|
||||
driveRootItem("root"), // will be present
|
||||
},
|
||||
DeltaLink: &delta,
|
||||
}},
|
||||
},
|
||||
},
|
||||
DeltaUpdate: map[string]api.DeltaUpdate{
|
||||
driveID1: {URL: delta},
|
||||
},
|
||||
Err: map[string]error{driveID1: nil},
|
||||
},
|
||||
canUsePreviousBackup: true,
|
||||
errCheck: assert.NoError,
|
||||
@ -2176,18 +2401,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
},
|
||||
}
|
||||
|
||||
itemPagers := map[string]api.DeltaPager[models.DriveItemable]{}
|
||||
|
||||
for driveID := range test.items {
|
||||
itemPagers[driveID] = &apiMock.DeltaPager[models.DriveItemable]{
|
||||
ToReturn: test.items[driveID],
|
||||
}
|
||||
}
|
||||
|
||||
mbh := mock.DefaultOneDriveBH("a-user")
|
||||
mbh.DrivePagerV = mockDrivePager
|
||||
mbh.ItemPagerV = itemPagers
|
||||
mbh.DriveItemEnumeration = mock.PagerResultToEDID(test.items)
|
||||
mbh.DriveItemEnumeration = test.enumerator
|
||||
|
||||
c := NewCollections(
|
||||
mbh,
|
||||
@ -2262,10 +2478,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
|
||||
|
||||
collectionCount++
|
||||
|
||||
// TODO(ashmrtn): We should really be getting items in the collection
|
||||
// via the Items() channel, but we don't have a way to mock out the
|
||||
// actual item fetch yet (mostly wiring issues). The lack of that makes
|
||||
// this check a bit more bittle since internal details can change.
|
||||
// TODO: We should really be getting items in the collection
|
||||
// via the Items() channel. The lack of that makes this check a bit more
|
||||
// bittle since internal details can change. The wiring to support
|
||||
// mocked GetItems is available. We just haven't plugged it in yet.
|
||||
col, ok := baseCol.(*Collection)
|
||||
require.True(t, ok, "getting onedrive.Collection handle")
|
||||
|
||||
|
||||
@ -292,8 +292,8 @@ func (m GetsItem) GetItem(
|
||||
// Enumerates Drive Items
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type NextPage[T any] struct {
|
||||
Items []T
|
||||
type NextPage struct {
|
||||
Items []models.DriveItemable
|
||||
Reset bool
|
||||
}
|
||||
|
||||
@ -305,7 +305,7 @@ var _ api.NextPageResulter[models.DriveItemable] = &DriveItemsDeltaPager{}
|
||||
|
||||
type DriveItemsDeltaPager struct {
|
||||
Idx int
|
||||
Pages []NextPage[models.DriveItemable]
|
||||
Pages []NextPage
|
||||
DeltaUpdate api.DeltaUpdate
|
||||
Err error
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
||||
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
|
||||
"github.com/alcionai/corso/src/internal/m365/service/onedrive/mock"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -90,16 +91,29 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
|
||||
defer flush()
|
||||
|
||||
var (
|
||||
mbh = mock.DefaultSharePointBH(siteID)
|
||||
du = api.DeltaUpdate{
|
||||
URL: "notempty",
|
||||
Reset: false,
|
||||
}
|
||||
paths = map[string]string{}
|
||||
currPaths = map[string]string{}
|
||||
excluded = map[string]struct{}{}
|
||||
collMap = map[string]map[string]*drive.Collection{
|
||||
driveID: {},
|
||||
}
|
||||
)
|
||||
|
||||
mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{
|
||||
DrivePagers: map[string]mock.DriveItemsDeltaPager{
|
||||
driveID: mock.DriveItemsDeltaPager{
|
||||
Pages: []mock.NextPage{{Items: test.items}},
|
||||
DeltaUpdate: du,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
c := drive.NewCollections(
|
||||
drive.NewLibraryBackupHandler(api.Drives{}, siteID, test.scope, path.SharePointService),
|
||||
mbh,
|
||||
tenantID,
|
||||
idname.NewProvider(siteID, siteID),
|
||||
nil,
|
||||
@ -107,15 +121,13 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
|
||||
|
||||
c.CollectionMap = collMap
|
||||
|
||||
_, err := c.UpdateCollections(
|
||||
_, _, err := c.PopulateDriveCollections(
|
||||
ctx,
|
||||
driveID,
|
||||
"General",
|
||||
test.items,
|
||||
paths,
|
||||
currPaths,
|
||||
excluded,
|
||||
true,
|
||||
"",
|
||||
fault.New(true))
|
||||
|
||||
test.expect(t, err, clues.ToCore(err))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user