larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

get things working with new base

Browse Source
This commit is contained in:
ryanfkeepers 2023-12-06 14:13:53 -07:00
parent 4edd1c5165
commit 7ea8c41736
2 changed files with 289 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
src/internal/m365/collection/drive/collections_tree_test.go
View File

@ -571,7 +571,7 @@ type populateTreeExpected struct {
type populateTreeTest struct {
name string
enumerator mock.EnumerateDriveItemsDelta
enumerator mock.EnumerateItemsDeltaByDrive
tree *folderyMcFolderFace
limiter *pagerLimiter
expect populateTreeExpected
@ -749,47 +749,48 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta(
},
},
},
{
name: "many folders with files across multiple deltas",
tree: newFolderyMcFolderFace(nil, rootID),
enumerator: mock.DriveEnumerator(
mock.Drive(id(drive)).With(
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
fileAt(folder))),
mock.Delta(id(delta), nil).With(aPage(
folderxAtRoot("sib"),
filexAt("fsib", "sib"))),
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
folderxAt("chld", folder),
filexAt("fchld", "chld"))),
)),
limiter: newPagerLimiter(control.DefaultOptions()),
expect: populateTreeExpected{
counts: countTD.Expected{
count.TotalFoldersProcessed: 7,
count.TotalFilesProcessed: 3,
count.TotalPagesEnumerated: 4,
},
err: require.NoError,
numLiveFiles: 3,
numLiveFolders: 4,
sizeBytes: 3 * 42,
treeContainsFolderIDs: []string{
rootID,
id(folder),
idx(folder, "sib"),
idx(folder, "chld"),
},
treeContainsTombstoneIDs: []string{},
treeContainsFileIDsWithParent: map[string]string{
id(file): id(folder),
idx(file, "fsib"): idx(folder, "sib"),
idx(file, "fchld"): idx(folder, "chld"),
},
},
},
// TODO: restore after mock.DriveEnumerator support lands.
// {
// name: "many folders with files across multiple deltas",
// tree: newFolderyMcFolderFace(nil, rootID),
// enumerator: mock.DriveEnumerator(
// mock.Drive(id(drive)).With(
// mock.Delta(id(delta), nil).With(aPage(
// folderAtRoot(),
// fileAt(folder))),
// mock.Delta(id(delta), nil).With(aPage(
// folderxAtRoot("sib"),
// filexAt("fsib", "sib"))),
// mock.Delta(id(delta), nil).With(aPage(
// folderAtRoot(),
// folderxAt("chld", folder),
// filexAt("fchld", "chld"))),
// )),
// limiter: newPagerLimiter(control.DefaultOptions()),
// expect: populateTreeExpected{
// counts: countTD.Expected{
// count.TotalFoldersProcessed: 7,
// count.TotalFilesProcessed: 3,
// count.TotalPagesEnumerated: 4,
// },
// err: require.NoError,
// numLiveFiles: 3,
// numLiveFolders: 4,
// sizeBytes: 3 * 42,
// treeContainsFolderIDs: []string{
// rootID,
// id(folder),
// idx(folder, "sib"),
// idx(folder, "chld"),
// },
// treeContainsTombstoneIDs: []string{},
// treeContainsFileIDsWithParent: map[string]string{
// id(file): id(folder),
// idx(file, "fsib"): idx(folder, "sib"),
// idx(file, "fchld"): idx(folder, "chld"),
// },
// },
// },
{
// technically you won't see this behavior from graph deltas, since deletes always
// precede creates/updates. But it's worth checking that we can handle it anyways.
@ -960,6 +961,15 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_singleDelta(
}
}
// TODO: remove when unifying test tree structs
type populateTreeTestMulti struct {
name string
enumerator mock.EnumerateDriveItemsDelta
tree *folderyMcFolderFace
limiter *pagerLimiter
expect populateTreeExpected
}
// this test focuses on quirks that can only arise from cases that occur across
// multiple delta enumerations.
// It is not concerned with unifying previous paths or post-processing collections.
@ -968,22 +978,22 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
drv.SetId(ptr.To(id(drive)))
drv.SetName(ptr.To(name(drive)))
table := []populateTreeTest{
table := []populateTreeTestMulti{
{
name: "sanity case: normal enumeration split across multiple deltas",
tree: newFolderyMcFolderFace(nil, rootID),
enumerator: mock.DriveEnumerator(
mock.Drive(id(drive)).With(
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
fileAt(folder))),
mock.Delta(id(delta), nil).With(aPage(
folderxAtRoot("sib"),
filexAt("fsib", "sib"))),
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
folderxAt("chld", folder),
filexAt("fchld", "chld"))),
mock.Delta(id(delta), nil).With(pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
mock.Delta(id(delta), nil).With(pagesOf(pageItems(
driveItem(idx(folder, "sib"), namex(folder, "sib"), parentDir(), rootID, isFolder),
driveItem(idx(file, "fsib"), namex(file, "fsib"), parentDir(namex(folder, "sib")), idx(folder, "sib"), isFolder)))...),
mock.Delta(id(delta), nil).With(pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(idx(folder, "chld"), namex(folder, "chld"), parentDir(name(folder)), id(folder), isFolder),
driveItem(idx(file, "fchld"), namex(file, "fchld"), parentDir(name(folder), namex(folder, "chld")), idx(folder, "chld"), isFolder)))...),
)),
limiter: newPagerLimiter(control.DefaultOptions()),
expect: populateTreeExpected{
@ -1018,18 +1028,19 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
tree: newFolderyMcFolderFace(nil, rootID),
enumerator: mock.DriveEnumerator(
mock.Drive(id(drive)).With(
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
fileAt(folder))),
mock.Delta(id(delta), nil).
With(pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
// a (delete,create) pair in the same delta can occur when
// a user deletes and restores an item in-between deltas.
mock.Delta(id(delta), nil).With(
aPage(
delItem(id(folder), rootID, isFolder),
delItem(id(file), id(folder), isFile)),
aPage(
folderAtRoot(),
fileAt(folder))),
mock.Delta(id(delta), nil).
With(pagesOf(pageItems(
delItem(id(folder), parentDir(), rootID, isFolder),
delItem(id(file), parentDir(), id(folder), isFile)))...).
With(pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
)),
limiter: newPagerLimiter(control.DefaultOptions()),
expect: populateTreeExpected{
@ -1058,12 +1069,14 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
tree: newFolderyMcFolderFace(nil, rootID),
enumerator: mock.DriveEnumerator(
mock.Drive(id(drive)).With(
mock.Delta(id(delta), nil).With(aPage(
folderAtRoot(),
fileAt(folder))),
mock.Delta(id(delta), nil).With(aPage(
driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder),
driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile))),
mock.Delta(id(delta), nil).
With(pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...),
mock.Delta(id(delta), nil).
With(pagesOf(pageItems(
driveItem(id(folder), namex(folder, "rename"), parentDir(), rootID, isFolder),
driveItem(id(file), namex(file, "rename"), parentDir(namex(folder, "rename")), id(folder), isFile)))...),
)),
limiter: newPagerLimiter(control.DefaultOptions()),
expect: populateTreeExpected{
@ -1096,22 +1109,23 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
mock.Drive(id(drive)).With(
mock.Delta(id(delta), nil).With(
// first page: create /root/folder and /root/folder/file
aPage(
folderAtRoot(),
fileAt(folder)),
pagesOf(pageItems(
driveItem(id(folder), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), id(folder), isFile)))...).
// assume the user makes changes at this point:
// 1. delete /root/folder
// 2. create a new /root/folder
// 3. move /root/folder/file from old to new folder (same file ID)
// in drive deltas, this will show up as another folder creation sharing
// the same dirname, but we won't see the delete until...
aPage(
With(pagesOf(pageItems(
driveItem(idx(folder, 2), name(folder), parentDir(), rootID, isFolder),
driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile))),
driveItem(id(file), name(file), parentDir(name(folder)), idx(folder, 2), isFile)))...),
// the next delta, containing the delete marker for the original /root/folder
mock.Delta(id(delta), nil).With(aPage(
delItem(id(folder), rootID, isFolder),
)),
mock.Delta(id(delta), nil).
With(pagesOf(pageItems(
delItem(id(folder), parentDir(), rootID, isFolder),
))...),
)),
limiter: newPagerLimiter(control.DefaultOptions()),
expect: populateTreeExpected{
@ -1140,7 +1154,7 @@ func (suite *CollectionsTreeUnitSuite) TestCollections_PopulateTree_multiDelta()
}
for _, test := range table {
suite.Run(test.name, func() {
runPopulateTreeTest(suite.T(), drv, test)
runPopulateTreeTestMulti(suite.T(), drv, test)
})
}
}
@ -1201,6 +1215,62 @@ func runPopulateTreeTest(
}
}
func runPopulateTreeTestMulti(
t *testing.T,
drv models.Driveable,
test populateTreeTestMulti,
) {
ctx, flush := tester.NewContext(t)
defer flush()
mbh := mock.DefaultDriveBHWithMulti(user, pagerForDrives(drv), test.enumerator)
c := collWithMBH(mbh)
counter := count.New()
_, err := c.populateTree(
ctx,
test.tree,
drv,
id(delta),
test.limiter,
counter,
fault.New(true))
test.expect.err(t, err, clues.ToCore(err))
assert.Equal(
t,
test.expect.numLiveFolders,
test.tree.countLiveFolders(),
"count live folders in tree")
cAndS := test.tree.countLiveFilesAndSizes()
assert.Equal(
t,
test.expect.numLiveFiles,
cAndS.numFiles,
"count live files in tree")
assert.Equal(
t,
test.expect.sizeBytes,
cAndS.totalBytes,
"count total bytes in tree")
test.expect.counts.Compare(t, counter)
for _, id := range test.expect.treeContainsFolderIDs {
assert.NotNil(t, test.tree.folderIDToNode[id], "node exists")
}
for _, id := range test.expect.treeContainsTombstoneIDs {
assert.NotNil(t, test.tree.tombstones[id], "tombstone exists")
}
for iID, pID := range test.expect.treeContainsFileIDsWithParent {
assert.Contains(t, test.tree.fileIDToParentID, iID, "file should exist in tree")
assert.Equal(t, pID, test.tree.fileIDToParentID[iID], "file should reference correct parent")
}
}
// ---------------------------------------------------------------------------
// folder tests
// ---------------------------------------------------------------------------

143
src/internal/m365/service/onedrive/mock/handlers.go
View File

@ -2,6 +2,7 @@ package mock
import (
"context"
"fmt"
"net/http"
"github.com/alcionai/clues"
@ -31,7 +32,8 @@ type BackupHandler[T any] struct {
// and plug in the selector scope there.
Sel selectors.Selector
DriveItemEnumeration EnumerateItemsDeltaByDrive
DriveItemEnumeration EnumerateItemsDeltaByDrive
DriveItemEnumerationMulti EnumerateDriveItemsDelta
GI GetsItem
GIP GetsItemPermission
@ -135,6 +137,18 @@ func DefaultDriveBHWith(
return mbh
}
func DefaultDriveBHWithMulti(
resource string,
drivePager *apiMock.Pager[models.Driveable],
enumerator EnumerateDriveItemsDelta,
) *BackupHandler[models.DriveItemable] {
mbh := DefaultOneDriveBH(resource)
mbh.DrivePagerV = drivePager
mbh.DriveItemEnumerationMulti = enumerator
return mbh
}
func (h BackupHandler[T]) PathPrefix(tID, driveID string) (path.Path, error) {
pp, err := h.PathPrefixFn(tID, h.ProtectedResource.ID(), driveID)
if err != nil {
@ -205,6 +219,14 @@ func (h BackupHandler[T]) EnumerateDriveItemsDelta(
driveID, prevDeltaLink string,
cc api.CallConfig,
) pagers.NextPageResulter[models.DriveItemable] {
if h.DriveItemEnumerationMulti.DrivePagers != nil {
return h.DriveItemEnumerationMulti.EnumerateDriveItemsDelta(
ctx,
driveID,
prevDeltaLink,
cc)
}
return h.DriveItemEnumeration.EnumerateDriveItemsDelta(
ctx,
driveID,
@ -323,7 +345,124 @@ func (m GetsItem) GetItem(
}
// ---------------------------------------------------------------------------
// Enumerates Drive Items
// Drive Items Enumerator
// ---------------------------------------------------------------------------
type EnumerateDriveItemsDelta struct {
DrivePagers map[string]*DriveDeltaEnumerator
}
func DriveEnumerator(
ds ...*DriveDeltaEnumerator,
) EnumerateDriveItemsDelta {
enumerator := EnumerateDriveItemsDelta{
DrivePagers: map[string]*DriveDeltaEnumerator{},
}
for _, drive := range ds {
enumerator.DrivePagers[drive.DriveID] = drive
}
return enumerator
}
func (en EnumerateDriveItemsDelta) EnumerateDriveItemsDelta(
_ context.Context,
driveID, _ string,
_ api.CallConfig,
) pagers.NextPageResulter[models.DriveItemable] {
iterator := en.DrivePagers[driveID]
return iterator.nextDelta()
}
type DriveDeltaEnumerator struct {
DriveID string
idx int
DeltaQueries []*DeltaQuery
}
func Drive(driveID string) *DriveDeltaEnumerator {
return &DriveDeltaEnumerator{DriveID: driveID}
}
func (dde *DriveDeltaEnumerator) With(ds ...*DeltaQuery) *DriveDeltaEnumerator {
dde.DeltaQueries = ds
return dde
}
func (dde *DriveDeltaEnumerator) nextDelta() *DeltaQuery {
if dde.idx == len(dde.DeltaQueries) {
// at the end of the enumeration, return an empty page with no items,
// not even the root. This is what graph api would do to signify an absence
// of changes in the delta.
lastDU := dde.DeltaQueries[dde.idx-1].DeltaUpdate
return &DeltaQuery{
DeltaUpdate: lastDU,
Pages: []NextPage{{
Items: []models.DriveItemable{},
}},
}
}
if dde.idx > len(dde.DeltaQueries) {
// a panic isn't optimal here, but since this mechanism is internal to testing,
// it's an acceptable way to have the tests ensure we don't over-enumerate deltas.
panic(fmt.Sprintf("delta index %d larger than count of delta iterations in mock", dde.idx))
}
pages := dde.DeltaQueries[dde.idx]
dde.idx++
return pages
}
var _ pagers.NextPageResulter[models.DriveItemable] = &DeltaQuery{}
type DeltaQuery struct {
idx int
Pages []NextPage
DeltaUpdate pagers.DeltaUpdate
Err error
}
func Delta(
resultDeltaID string,
err error,
) *DeltaQuery {
return &DeltaQuery{
DeltaUpdate: pagers.DeltaUpdate{URL: resultDeltaID},
Err: err,
}
}
func (dq *DeltaQuery) NextPage() ([]models.DriveItemable, bool, bool) {
if dq.idx >= len(dq.Pages) {
return nil, false, true
}
np := dq.Pages[dq.idx]
dq.idx = dq.idx + 1
return np.Items, np.Reset, false
}
func (dq *DeltaQuery) With(
pages ...NextPage,
) *DeltaQuery {
dq.Pages = append(dq.Pages, pages...)
return dq
}
func (dq *DeltaQuery) Cancel() {}
func (dq *DeltaQuery) Results() (pagers.DeltaUpdate, error) {
return dq.DeltaUpdate, dq.Err
}
// ---------------------------------------------------------------------------
// old version - Enumerates Drive Items
// ---------------------------------------------------------------------------
type NextPage struct {