larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use the tree-based drive backup by standard (#4980)

Browse Source 
reverses the drive backup process toggle by making the tree-based backup standard, and the old, linear enumeration the toggleable fallback.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #4690

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2024-01-23 11:57:11 -07:00 committed by GitHub
parent 5b4389d4e1
commit d501a30761
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 89 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/cli/backup/onedrive.go
View File

@ -67,11 +67,11 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
flags.AddUserFlag(c)
flags.AddGenericBackupFlags(c)
fs.BoolVar(
&flags.UseDeltaTreeFV,
flags.UseDeltaTreeFN,
&flags.UseOldDeltaProcessFV,
flags.UseOldDeltaProcessFN,
false,
"process backups using the delta tree instead of standard enumeration")
cobra.CheckErr(fs.MarkHidden(flags.UseDeltaTreeFN))
"process backups using the old delta processor instead of tree-based enumeration")
cobra.CheckErr(fs.MarkHidden(flags.UseOldDeltaProcessFN))
case listCommand:
c, _ = utils.AddCommand(cmd, oneDriveListCmd())

4
src/cli/flags/onedrive.go
View File

@ -13,7 +13,7 @@ const (
FileModifiedAfterFN = "file-modified-after"
FileModifiedBeforeFN = "file-modified-before"
UseDeltaTreeFN = "use-delta-tree"
UseOldDeltaProcessFN = "use-old-delta-process"
)
var (
@ -25,7 +25,7 @@ var (
FileModifiedAfterFV string
FileModifiedBeforeFV string
UseDeltaTreeFV bool
UseOldDeltaProcessFV bool
)
// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the

4
src/cli/utils/options.go
View File

@ -28,7 +28,7 @@ func Control() control.Options {
opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
opt.ToggleFeatures.DisableLazyItemReader = flags.DisableLazyItemReaderFV
opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV
opt.ToggleFeatures.UseDeltaTree = flags.UseDeltaTreeFV
opt.ToggleFeatures.UseOldDeltaProcess = flags.UseOldDeltaProcessFV
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
return opt
@ -58,7 +58,7 @@ func ParseBackupOptions() control.BackupConfig {
opt.M365.DeltaPageSize = dps
opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
opt.M365.UseDriveDeltaTree = flags.UseDeltaTreeFV
opt.M365.UseOldDriveDeltaProcess = flags.UseOldDeltaProcessFV
opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV

46
src/internal/m365/backup_test.go
View File

@ -390,6 +390,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
sel := selectors.NewSharePointBackup(siteIDs)
sel.Include(sel.LibraryFolders([]string{"foo"}, selectors.PrefixMatch()))
sel.Include(sel.Library("Documents"))
sel.SetDiscreteOwnerIDName(site.ID(), site.Name())
@ -407,21 +408,40 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup")
require.Len(t, cols, 2) // 1 collection, 1 path prefix directory to ensure the root path exists.
var (
hasDocumentsColl bool
hasMetadataColl bool
)
documentsColl, err := path.BuildPrefix(
suite.connector.tenant,
siteID,
path.SharePointService,
path.LibrariesCategory)
require.NoError(t, err, clues.ToCore(err))
metadataColl, err := path.BuildMetadata(
suite.connector.tenant,
siteID,
path.SharePointService,
path.LibrariesCategory,
false)
require.NoError(t, err, clues.ToCore(err))
for i, col := range cols {
fp := col.FullPath()
t.Logf("Collection %d: %s", i, fp)
hasDocumentsColl = hasDocumentsColl || fp.Equal(documentsColl)
hasMetadataColl = hasMetadataColl || fp.Equal(metadataColl)
}
require.Truef(t, hasDocumentsColl, "found documents collection %s", documentsColl)
require.Truef(t, hasMetadataColl, "found metadata collection %s", metadataColl)
// No excludes yet as this isn't an incremental backup.
assert.True(t, excludes.Empty())
t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String())
assert.Equal(
t,
path.SharePointMetadataService.String(),
cols[0].FullPath().Service().String())
t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String())
assert.Equal(
t,
path.SharePointService.String(),
cols[1].FullPath().Service().String())
}
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {

2
src/internal/m365/collection/drive/collections.go
View File

@ -285,7 +285,7 @@ func (c *Collections) Get(
globalExcludeItemIDs *prefixmatcher.StringSetMatchBuilder,
errs *fault.Bus,
) ([]data.BackupCollection, bool, error) {
if c.ctrl.ToggleFeatures.UseDeltaTree {
if !c.ctrl.ToggleFeatures.UseOldDeltaProcess {
colls, canUsePrevBackup, err := c.getTree(ctx, prevMetadata, globalExcludeItemIDs, errs)
return colls,

4
src/internal/m365/collection/drive/collections_test.go
View File

@ -2634,7 +2634,9 @@ func (suite *CollectionsUnitSuite) TestGet() {
tenant,
idname.NewProvider(user, user),
func(*support.ControllerOperationStatus) {},
control.Options{ToggleFeatures: control.Toggles{}},
control.Options{ToggleFeatures: control.Toggles{
UseOldDeltaProcess: true,
}},
count.New())
prevDelta := "prev-delta"

4
src/internal/m365/collection/drive/helper_test.go
View File

@ -207,9 +207,7 @@ func collWithMBH(mbh BackupHandler) *Collections {
tenant,
idname.NewProvider(user, user),
func(*support.ControllerOperationStatus) {},
control.Options{ToggleFeatures: control.Toggles{
UseDeltaTree: true,
}},
control.Options{ToggleFeatures: control.Toggles{}},
count.New())
}

9
src/internal/m365/collection/drive/limiter_test.go
View File

@ -450,8 +450,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_noTree() {
// check the expected items appear.
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_tree() {
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
iterGetPreviewLimitsTests(suite, opts)
}
@ -537,7 +535,7 @@ func runGetPreviewLimits(
expectItemIDs := test.expectedItemIDsInCollection[folderPath]
if opts.ToggleFeatures.UseDeltaTree && test.expectedItemIDsInCollectionTree != nil {
if !opts.ToggleFeatures.UseOldDeltaProcess && test.expectedItemIDsInCollectionTree != nil {
expectItemIDs = test.expectedItemIDsInCollectionTree[folderPath]
}
@ -691,7 +689,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsNoTree() {
// and such. Other tests are expected to provide more comprehensive checks.
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsWithTree() {
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
for _, test := range defaultLimitsTable() {
suite.Run(test.name, func() {
@ -815,7 +812,7 @@ func runGetPreviewLimitsDefaults(
// Add one to account for the folder permissions item.
expected := test.expect.numItemsPerContainer + 1
if opts.ToggleFeatures.UseDeltaTree {
if !opts.ToggleFeatures.UseOldDeltaProcess {
expected += test.expect.numItemsTreePadding
}
@ -835,7 +832,7 @@ func runGetPreviewLimitsDefaults(
// Add one to account for the folder permissions item.
expected := test.expect.numItems + test.expect.numContainers
if opts.ToggleFeatures.UseDeltaTree {
if !opts.ToggleFeatures.UseOldDeltaProcess {
expected += test.expect.numItemsTreePadding
}

56
src/internal/m365/controller_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/idname"
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
@ -1297,7 +1298,7 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
runRestoreBackupTest(suite.T(), test, cfg)
}
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
func (suite *ControllerIntegrationSuite) TestProduceBackupCollections_createsPrefixCollections() {
table := []struct {
name string
resourceCat resource.Category
@ -1311,9 +1312,9 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
selectorFunc: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup([]string{suite.user})
sel.Include(
sel.ContactFolders([]string{selectors.NoneTgt}),
sel.EventCalendars([]string{selectors.NoneTgt}),
sel.MailFolders([]string{selectors.NoneTgt}))
sel.ContactFolders(selectors.None()),
sel.EventCalendars(selectors.None()),
sel.MailFolders(selectors.None()))
return sel.Selector
},
@ -1329,7 +1330,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
resourceCat: resource.Users,
selectorFunc: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{suite.user})
sel.Include(sel.Folders([]string{selectors.NoneTgt}))
sel.Include(sel.Folders(selectors.None()))
return sel.Selector
},
@ -1344,10 +1345,11 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
selectorFunc: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{tconfig.M365SiteID(t)})
sel.Include(
sel.LibraryFolders([]string{selectors.NoneTgt}),
sel.Library(selectors.NoneTgt),
sel.LibraryFolders(selectors.None()),
// not yet in use
// sel.Pages([]string{selectors.NoneTgt}),
// sel.Lists([]string{selectors.NoneTgt}),
// sel.Pages(selectors.None()),
// sel.Lists(selectors.None()),
)
return sel.Selector
@ -1366,10 +1368,11 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
selectorFunc: func(t *testing.T) selectors.Selector {
sel := selectors.NewGroupsBackup([]string{tconfig.M365TeamID(t)})
sel.Include(
sel.LibraryFolders([]string{selectors.NoneTgt}),
sel.Library(selectors.NoneTgt),
sel.LibraryFolders(selectors.None()),
// not yet in use
// sel.Pages([]string{selectors.NoneTgt}),
// sel.Lists([]string{selectors.NoneTgt}),
// sel.Pages(selectors.None()),
// sel.Lists(selectors.None()),
)
return sel.Selector
@ -1423,8 +1426,14 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
t.Logf("Backup enumeration complete in %v\n", time.Since(start))
// Use a map to find duplicates.
foundCategories := []string{}
for _, col := range dcs {
foundCategories := map[string]struct{}{}
var foundPrefixCollection bool
for i, col := range dcs {
fp := col.FullPath()
t.Logf("collection %d: %s", i, fp)
// TODO(ashmrtn): We should be able to remove the below if we change how
// status updates are done. Ideally we shouldn't have to fetch items in
// these collections to avoid deadlocking.
@ -1437,23 +1446,26 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
}
// Ignore metadata collections.
fullPath := col.FullPath()
if fullPath.Service() != test.service {
if fp.Service() != test.service {
continue
}
assert.Empty(t, fullPath.Folders(), "non-prefix collection")
assert.NotEqual(t, col.State(), data.NewState, "prefix collection marked as new")
foundCategories = append(foundCategories, fullPath.Category().String())
assert.Zero(t, found, "non-empty collection")
if len(fp.Folders()) == 0 {
t.Log("prefix collection tests")
assert.NotEqual(t, col.State(), data.NewState, "prefix collection must have New status")
assert.Zero(t, found, "prefix collection must not contain items")
foundPrefixCollection = true
}
assert.ElementsMatch(t, test.categories, foundCategories)
foundCategories[fp.Category().String()] = struct{}{}
}
assert.ElementsMatch(t, test.categories, maps.Keys(foundCategories))
backupCtrl.Wait()
assert.NoError(t, errs.Failure())
assert.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
assert.True(t, foundPrefixCollection, "backup must generate a prefix collection")
})
}
}

6
src/internal/m365/onedrive_test.go
View File

@ -510,7 +510,6 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
restoreCfg.IncludePermissions = true
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),
@ -763,7 +762,6 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
restoreCfg.IncludePermissions = true
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),
@ -857,7 +855,6 @@ func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) {
restoreCfg.IncludePermissions = false
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),
@ -1066,7 +1063,6 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
restoreCfg.IncludePermissions = true
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),
@ -1265,7 +1261,6 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion
restoreCfg.IncludePermissions = true
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),
@ -1385,7 +1380,6 @@ func testRestoreFolderNamedFolderRegression(
restoreCfg.IncludePermissions = true
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
cfg := m365Stub.ConfigInfo{
Tenant: suite.Tenant(),

1
src/internal/operations/backup_test.go
View File

@ -409,6 +409,7 @@ func (suite *BackupOpUnitSuite) TestNewBackupOperation_configuredOptionsMatchInp
RunMigrations: true,
DisableSlidingWindowLimiter: true,
UseDeltaTree: true,
UseOldDeltaProcess: true,
DisableLazyItemReader: true,
},
PreviewLimits: control.PreviewItemLimits{

2
src/internal/operations/test/m365/backup_helper.go
View File

@ -270,8 +270,6 @@ func RunMergeBaseGroupsUpdate(
whatSet = deeTD.CategoryFromRepoRef
)
opts.ToggleFeatures.UseDeltaTree = true
// Need outside the inner test case so bod lasts for the entire test.
bo, bod := PrepNewTestBackupOp(
t,

1
src/internal/operations/test/m365/driveish.go
View File

@ -870,7 +870,6 @@ func RunDriveAssistBaseGroupsUpdate(
opts = control.DefaultOptions()
)
opts.ToggleFeatures.UseDeltaTree = true
opts.ItemExtensionFactory = []extensions.CreateItemExtensioner{
NewCreateSingleFileFailExtension(),
}

6
src/internal/operations/test/m365/groups/groups_test.go
View File

@ -108,8 +108,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeGroups() {
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunBasicDriveishBackupTests(
suite,
path.GroupsService,
@ -119,8 +117,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeGroups() {
func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeIncrementalGroups() {
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
runGroupsIncrementalBackupTests(suite, suite.its, opts)
}
@ -133,8 +129,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeExtensionsGroups() {
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunDriveishBackupWithExtensionsTests(
suite,
path.GroupsService,

6
src/internal/operations/test/m365/onedrive/onedrive_test.go
View File

@ -116,8 +116,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeOneDrive() {
sel.Include(selTD.OneDriveBackupFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunBasicDriveishBackupTests(
suite,
path.OneDriveService,
@ -127,8 +125,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeOneDrive() {
func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeIncrementalOneDrive() {
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
runOneDriveIncrementalBackupTests(suite, suite.its, opts)
}
@ -141,8 +137,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeExtensionsOneDrive(
sel.Include(selTD.OneDriveBackupFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunDriveishBackupWithExtensionsTests(
suite,
path.OneDriveService,

6
src/internal/operations/test/m365/sharepoint/sharepoint_test.go
View File

@ -172,8 +172,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeSharePoint() {
sel.Include(selTD.SharePointBackupFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunBasicDriveishBackupTests(
suite,
path.SharePointService,
@ -183,8 +181,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeSharePoint() {
func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeIncrementalSharePoint() {
opts := control.DefaultOptions()
opts.ToggleFeatures.UseDeltaTree = true
runSharePointIncrementalBackupTests(suite, suite.its, opts)
}
@ -197,8 +193,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeExtensionsSharePo
sel.Include(selTD.SharePointBackupFolderScope(sel))
opts.ToggleFeatures.UseDeltaTree = true
RunDriveishBackupWithExtensionsTests(
suite,
path.SharePointService,

6
src/internal/tester/tconfig/config.go
View File

@ -162,7 +162,7 @@ func ReadTestConfig() (map[string]string, error) {
TestCfgSiteID,
os.Getenv(EnvCorsoM365TestSiteID),
vpr.GetString(TestCfgSiteID),
"4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
"10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
fallbackTo(
testEnv,
TestCfgTeamID,
@ -174,7 +174,7 @@ func ReadTestConfig() (map[string]string, error) {
TestCfgTeamSiteID,
os.Getenv(EnvCorsoM365TestTeamSiteID),
vpr.GetString(TestCfgTeamSiteID),
"8df7082a-62eb-4165-97f9-ead8cf04c0d8,11810f05-e8bd-45e2-9728-92e434260750")
"10rqc2.sharepoint.com,8df7082a-62eb-4165-97f9-ead8cf04c0d8,11810f05-e8bd-45e2-9728-92e434260750")
fallbackTo(
testEnv,
TestCfgSecondaryTeamID,
@ -204,7 +204,7 @@ func ReadTestConfig() (map[string]string, error) {
TestCfgSecondarySiteID,
os.Getenv(EnvCorsoSecondaryM365TestSiteID),
vpr.GetString(TestCfgSecondarySiteID),
"053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c")
"10rqc2.sharepoint.com,053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c")
fallbackTo(
testEnv,
TestCfgUnlicensedUserID,

2
src/pkg/control/backup.go
View File

@ -64,7 +64,7 @@ type BackupM365Config struct {
ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"`
// see: https://github.com/alcionai/corso/issues/4688
UseDriveDeltaTree bool `json:"useDriveDeltaTree"`
UseOldDriveDeltaProcess bool `json:"useOldDriveDeltaProcess"`
}
type Parallelism struct {

1
src/pkg/control/options.go
View File

@ -100,6 +100,7 @@ type Toggles struct {
// see: https://github.com/alcionai/corso/issues/4688
UseDeltaTree bool `json:"useDeltaTree"`
UseOldDeltaProcess bool `json:"useOldDeltaProcess"`
// AddDisableLazyItemReader disables lazy item reader, such that we fall
// back to prefetch reader. This flag is currently only meant for groups