larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix up tests, apply latest design

Browse Source
This commit is contained in:
ryanfkeepers 2023-10-11 10:55:39 -06:00
parent c805c8f8e5
commit 96b9f752ba
9 changed files with 374 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/internal/m365/collection/drive/collections.go
View File

@ -6,7 +6,6 @@ import (
"fmt"
"io"
"strings"
"sync"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -690,8 +689,6 @@ func (c *Collections) PopulateDriveCollections(
el = errs.Local()
newPrevPaths = map[string]string{}
invalidPrevDelta = len(prevDeltaLink) == 0
ch = make(chan api.NextPage[models.DriveItemable], 1)
wg = sync.WaitGroup{}
// currPrevPaths is used to identify which collection a
// file belongs to. This is useful to delete a file from the
@ -705,58 +702,54 @@ func (c *Collections) PopulateDriveCollections(
maps.Copy(newPrevPaths, oldPrevPaths)
}
go func() {
defer wg.Done()
pager := c.handler.EnumerateDriveItemsDelta(
ctx,
driveID,
prevDeltaLink,
api.CallConfig{
Select: api.DefaultDriveItemProps(),
})
for pg := range ch {
page, reset, done := pager.NextPage()
for ; !done; page, reset, done = pager.NextPage() {
if el.Failure() != nil {
break
}
if reset {
newPrevPaths = map[string]string{}
currPrevPaths = map[string]string{}
c.CollectionMap[driveID] = map[string]*Collection{}
invalidPrevDelta = true
}
for _, item := range page {
if el.Failure() != nil {
// exhaust the channel to ensure it closes
continue
break
}
if pg.Reset {
newPrevPaths = map[string]string{}
currPrevPaths = map[string]string{}
c.CollectionMap[driveID] = map[string]*Collection{}
invalidPrevDelta = true
}
for _, item := range pg.Items {
if el.Failure() != nil {
continue
}
err := c.processItem(
ctx,
item,
driveID,
driveName,
oldPrevPaths,
currPrevPaths,
newPrevPaths,
excludedItemIDs,
invalidPrevDelta,
el)
if err != nil {
el.AddRecoverable(ctx, clues.Stack(err))
}
err := c.processItem(
ctx,
item,
driveID,
driveName,
oldPrevPaths,
currPrevPaths,
newPrevPaths,
excludedItemIDs,
invalidPrevDelta,
el)
if err != nil {
el.AddRecoverable(ctx, clues.Stack(err))
}
}
}()
}
wg.Add(1)
du, err := c.handler.EnumerateDriveItemsDelta(
ctx,
ch,
driveID,
prevDeltaLink)
du, err := pager.Results()
if err != nil {
return du, nil, clues.Stack(err)
}
wg.Wait()
return du, newPrevPaths, el.Failure()
}

642
src/internal/m365/collection/drive/collections_test.go
View File

@ -743,13 +743,13 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() {
errs = fault.New(true)
)
mbh.DriveItemEnumeration = mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID: {
{Items: test.items},
Pages: []mock.NextPage{{Items: test.items}},
DeltaUpdate: du,
},
},
DeltaUpdate: map[string]api.DeltaUpdate{driveID: du},
}
sel := selectors.NewOneDriveBackup([]string{user})
@ -1226,7 +1226,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
table := []struct {
name string
drives []models.Driveable
enumerator mock.EnumeratesDriveItemsDelta[models.DriveItemable]
enumerator mock.EnumerateItemsDeltaByDrive
canUsePreviousBackup bool
errCheck assert.ErrorAssertionFunc
prevFolderPaths map[string]map[string]string
@ -1245,19 +1245,18 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_DelFileOnly_NoFolders_NoErrors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
driveRootItem("root"), // will be present, not needed
delItem("file", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{{
Items: []models.DriveItemable{
driveRootItem("root"), // will be present, not needed
delItem("file", driveBasePath1, "root", true, false, false),
},
}},
DeltaUpdate: api.DeltaUpdate{URL: delta},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1280,19 +1279,18 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_NoFolderDeltas_NoErrors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("file", "file", driveBasePath1, "root", true, false, false),
},
}},
DeltaUpdate: api.DeltaUpdate{URL: delta},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1315,20 +1313,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_NoErrors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1355,21 +1352,20 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_NoErrors_FileRenamedMultiple",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
driveItem("file", "file2", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
driveItem("file", "file2", driveBasePath1+"/folder", "folder", true, false, false),
},
}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1396,21 +1392,18 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_NoErrors_FileMovedMultiple",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
driveItem("file", "file2", driveBasePath1, "root", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1439,20 +1432,17 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_EmptyDelta_NoErrors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: empty, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: empty, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1479,29 +1469,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_TwoItemPages_NoErrors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1530,40 +1519,40 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_TwoItemPages_WithReset",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1592,37 +1581,36 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_TwoItemPages_WithResetCombinedWithItems",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1654,28 +1642,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
drive1,
drive2,
},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
driveID2: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root2"),
driveItem("folder2", "folder", driveBasePath2, "root2", false, true, false),
driveItem("file2", "file", driveBasePath2+"/folder", "folder2", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
driveID2: {URL: delta2, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1717,28 +1702,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
drive1,
drive2,
},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
driveID2: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath2, "root", false, true, false),
driveItem("file2", "file", driveBasePath2+"/folder", "folder", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
driveID2: {URL: delta2, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1777,16 +1759,14 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_Errors",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{}},
Pages: []mock.NextPage{{Items: []models.DriveItemable{}}},
DeltaUpdate: api.DeltaUpdate{},
Err: assert.AnError,
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {},
},
Err: map[string]error{driveID1: assert.AnError},
},
canUsePreviousBackup: false,
errCheck: assert.Error,
@ -1801,26 +1781,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1854,26 +1833,25 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_InvalidPrevDeltaCombinedWithItems_DeleteNonExistentFolder",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{},
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder2", "folder2", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1907,37 +1885,36 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
// on the first page, if this is the total data, we'd expect both folder and folder2
// since new previousPaths merge with the old previousPaths.
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
},
},
{
Items: []models.DriveItemable{},
Reset: true,
},
{
// but after a delta reset, we treat this as the total end set of folders, which means
// we don't expect folder to exist any longer.
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
Pages: []mock.NextPage{
{
// on the first page, if this is the total data, we'd expect both folder and folder2
// since new previousPaths merge with the old previousPaths.
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
},
},
{
Items: []models.DriveItemable{},
Reset: true,
},
{
// but after a delta reset, we treat this as the total end set of folders, which means
// we don't expect folder to exist any longer.
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder2", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder2", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -1974,31 +1951,30 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "OneDrive Two Item Pages with Malware",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
malwareItem("malware", "malware", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
malwareItem("malware2", "malware2", driveBasePath1+"/folder", "folder", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
malwareItem("malware", "malware", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder", "folder", true, false, false),
malwareItem("malware2", "malware2", driveBasePath1+"/folder", "folder", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2028,36 +2004,35 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Deleted Folder In New Results With Invalid Delta",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder2", "folder2", true, false, false),
},
},
{
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
delItem("folder2", driveBasePath1, "root", false, true, false),
delItem("file2", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
driveItem("folder2", "folder2", driveBasePath1, "root", false, true, false),
driveItem("file2", "file2", driveBasePath1+"/folder2", "folder2", true, false, false),
},
},
{
Reset: true,
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
delItem("folder2", driveBasePath1, "root", false, true, false),
delItem("file2", driveBasePath1, "root", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta2, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2092,22 +2067,19 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Folder Delete After Invalid Delta",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Pages: []mock.NextPage{{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("folder", driveBasePath1, "root", false, true, false),
},
Reset: true,
},
}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2138,22 +2110,21 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Item Delete After Invalid Delta",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("file", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("file", driveBasePath1, "root", true, false, false),
},
Reset: true,
},
Reset: true,
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2181,29 +2152,28 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Folder Made And Deleted",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("folder", driveBasePath1, "root", false, true, false),
delItem("file", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("folder", driveBasePath1, "root", false, true, false),
delItem("file", driveBasePath1, "root", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta2, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2229,28 +2199,27 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Item Made And Deleted",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("file", driveBasePath1, "root", true, false, false),
Pages: []mock.NextPage{
{
Items: []models.DriveItemable{
driveRootItem("root"),
driveItem("folder", "folder", driveBasePath1, "root", false, true, false),
driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false),
},
},
{
Items: []models.DriveItemable{
driveRootItem("root"),
delItem("file", driveBasePath1, "root", true, false, false),
},
},
},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2279,19 +2248,16 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Random Folder Delete",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
delItem("folder", driveBasePath1, "root", false, true, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2317,19 +2283,16 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "One Drive Random Item Delete",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"),
delItem("file", driveBasePath1, "root", true, false, false),
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta, Reset: true},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,
@ -2355,18 +2318,15 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
{
name: "TwoPriorDrives_OneTombstoned",
drives: []models.Driveable{drive1},
enumerator: mock.EnumeratesDriveItemsDelta[models.DriveItemable]{
Pages: map[string][]api.NextPage[models.DriveItemable]{
enumerator: mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID1: {
{Items: []models.DriveItemable{
Pages: []mock.NextPage{{Items: []models.DriveItemable{
driveRootItem("root"), // will be present
}},
}}},
DeltaUpdate: api.DeltaUpdate{URL: delta},
},
},
DeltaUpdate: map[string]api.DeltaUpdate{
driveID1: {URL: delta},
},
Err: map[string]error{driveID1: nil},
},
canUsePreviousBackup: true,
errCheck: assert.NoError,

3
src/internal/m365/collection/drive/url_cache.go
View File

@ -165,7 +165,8 @@ func (uc *urlCache) refreshCache(
Select: api.URLCacheDriveItemProps(),
})
for page, reset, done := pager.NextPage(); !done; {
page, reset, done := pager.NextPage()
for ; !done; page, reset, done = pager.NextPage() {
err := uc.updateCache(
ctx,
page,

26
src/internal/m365/collection/drive/url_cache_test.go
View File

@ -204,7 +204,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
table := []struct {
name string
pages []mock.NextPage[models.DriveItemable]
pages []mock.NextPage
pagerErr error
expectedItemProps map[string]itemProps
expectErr assert.ErrorAssertionFunc
@ -212,7 +212,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
}{
{
name: "single item in cache",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
}},
@ -232,7 +232,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "multiple items in cache",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
fileItem("2", "file2", "root", "root", "https://dummy2.com", false),
@ -272,7 +272,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "multiple pages",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
fileItem("2", "file2", "root", "root", "https://dummy2.com", false),
@ -314,7 +314,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "multiple pages with resets",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{
Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
@ -372,7 +372,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "multiple pages with resets and combo reset+items in page",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{
Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
@ -426,7 +426,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "duplicate items with potentially new urls",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
fileItem("2", "file2", "root", "root", "https://dummy2.com", false),
@ -458,7 +458,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "deleted items",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
fileItem("2", "file2", "root", "root", "https://dummy2.com", false),
@ -484,7 +484,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "item not found in cache",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
}},
@ -501,7 +501,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
},
{
name: "delta query error",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{}},
},
pagerErr: errors.New("delta query error"),
@ -519,7 +519,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
{
name: "folder item",
pages: []mock.NextPage[models.DriveItemable]{
pages: []mock.NextPage{
{Items: []models.DriveItemable{
fileItem("1", "file1", "root", "root", "https://dummy1.com", false),
driveItem("2", "folder2", "root", "root", false, true, false),
@ -548,8 +548,8 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() {
defer flush()
medi := mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]mock.DriveItemsDeltaPager{
driveID: mock.DriveItemsDeltaPager{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID: {
Pages: test.pages,
Err: test.pagerErr,
DeltaUpdate: api.DeltaUpdate{URL: deltaString},

6
src/internal/m365/service/onedrive/mock/handlers.go
View File

@ -298,7 +298,7 @@ type NextPage struct {
}
type EnumerateItemsDeltaByDrive struct {
DrivePagers map[string]DriveItemsDeltaPager
DrivePagers map[string]*DriveItemsDeltaPager
}
var _ api.NextPageResulter[models.DriveItemable] = &DriveItemsDeltaPager{}
@ -316,7 +316,7 @@ func (edibd EnumerateItemsDeltaByDrive) EnumerateDriveItemsDelta(
_ api.CallConfig,
) api.NextPageResulter[models.DriveItemable] {
didp := edibd.DrivePagers[driveID]
return &didp
return didp
}
func (edi *DriveItemsDeltaPager) NextPage() ([]models.DriveItemable, bool, bool) {
@ -325,7 +325,7 @@ func (edi *DriveItemsDeltaPager) NextPage() ([]models.DriveItemable, bool, bool)
}
np := edi.Pages[edi.Idx]
edi.Idx++
edi.Idx = edi.Idx + 1
return np.Items, np.Reset, false
}

4
src/internal/m365/service/sharepoint/backup_test.go
View File

@ -104,8 +104,8 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
)
mbh.DriveItemEnumeration = mock.EnumerateItemsDeltaByDrive{
DrivePagers: map[string]mock.DriveItemsDeltaPager{
driveID: mock.DriveItemsDeltaPager{
DrivePagers: map[string]*mock.DriveItemsDeltaPager{
driveID: {
Pages: []mock.NextPage{{Items: test.items}},
DeltaUpdate: du,
},

8
src/pkg/fault/skipped.go
View File

@ -1,9 +1,17 @@
package fault
import (
"context"
"github.com/alcionai/corso/src/cli/print"
)
// AddSkipper presents an interface that allows callers to
// write additional skipped items to the complying struct.
type AddSkipper interface {
AddSkip(ctx context.Context, s *Skipped)
}
// skipCause identifies the well-known conditions to Skip an item. It is
// important that skip cause enumerations do not overlap with general error
// handling. Skips must be well known, well documented, and consistent.

3
src/pkg/services/m365/api/drive_pager_test.go
View File

@ -199,7 +199,8 @@ func (suite *DrivePagerIntgSuite) TestEnumerateDriveItems() {
Select: api.DefaultDriveItemProps(),
})
for page, reset, done := pager.NextPage(); !done; {
page, reset, done := pager.NextPage()
for ; !done; page, reset, done = pager.NextPage() {
items = append(items, page...)
assert.False(t, reset, "should not reset")

10
src/pkg/services/m365/api/item_pager.go
View File

@ -223,8 +223,9 @@ func batchEnumerateItems[T any](
go enumerateItems[T](ctx, pager, &npr)
for is, _, done := npr.NextPage(); !done; {
items = append(items, is...)
page, _, done := npr.NextPage()
for ; !done; page, _, done = npr.NextPage() {
items = append(items, page...)
}
_, err := npr.Results()
@ -346,12 +347,13 @@ func batchDeltaEnumerateItems[T any](
go deltaEnumerateItems[T](ctx, pager, &npr, prevDeltaLink)
for is, reset, done := npr.NextPage(); !done; {
page, reset, done := npr.NextPage()
for ; !done; page, reset, done = npr.NextPage() {
if reset {
results = []T{}
}
results = append(results, is...)
results = append(results, page...)
}
du, err := npr.Results()