use the tree-based drive backup by standard (#4980)
reverses the drive backup process toggle by making the tree-based backup standard, and the old, linear enumeration the toggleable fallback. --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🌻 Feature #### Issue(s) * #4690 #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
5b4389d4e1
commit
d501a30761
@ -67,11 +67,11 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
|||||||
flags.AddUserFlag(c)
|
flags.AddUserFlag(c)
|
||||||
flags.AddGenericBackupFlags(c)
|
flags.AddGenericBackupFlags(c)
|
||||||
fs.BoolVar(
|
fs.BoolVar(
|
||||||
&flags.UseDeltaTreeFV,
|
&flags.UseOldDeltaProcessFV,
|
||||||
flags.UseDeltaTreeFN,
|
flags.UseOldDeltaProcessFN,
|
||||||
false,
|
false,
|
||||||
"process backups using the delta tree instead of standard enumeration")
|
"process backups using the old delta processor instead of tree-based enumeration")
|
||||||
cobra.CheckErr(fs.MarkHidden(flags.UseDeltaTreeFN))
|
cobra.CheckErr(fs.MarkHidden(flags.UseOldDeltaProcessFN))
|
||||||
|
|
||||||
case listCommand:
|
case listCommand:
|
||||||
c, _ = utils.AddCommand(cmd, oneDriveListCmd())
|
c, _ = utils.AddCommand(cmd, oneDriveListCmd())
|
||||||
|
|||||||
@ -13,7 +13,7 @@ const (
|
|||||||
FileModifiedAfterFN = "file-modified-after"
|
FileModifiedAfterFN = "file-modified-after"
|
||||||
FileModifiedBeforeFN = "file-modified-before"
|
FileModifiedBeforeFN = "file-modified-before"
|
||||||
|
|
||||||
UseDeltaTreeFN = "use-delta-tree"
|
UseOldDeltaProcessFN = "use-old-delta-process"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -25,7 +25,7 @@ var (
|
|||||||
FileModifiedAfterFV string
|
FileModifiedAfterFV string
|
||||||
FileModifiedBeforeFV string
|
FileModifiedBeforeFV string
|
||||||
|
|
||||||
UseDeltaTreeFV bool
|
UseOldDeltaProcessFV bool
|
||||||
)
|
)
|
||||||
|
|
||||||
// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the
|
// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the
|
||||||
|
|||||||
@ -28,7 +28,7 @@ func Control() control.Options {
|
|||||||
opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
||||||
opt.ToggleFeatures.DisableLazyItemReader = flags.DisableLazyItemReaderFV
|
opt.ToggleFeatures.DisableLazyItemReader = flags.DisableLazyItemReaderFV
|
||||||
opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
||||||
opt.ToggleFeatures.UseDeltaTree = flags.UseDeltaTreeFV
|
opt.ToggleFeatures.UseOldDeltaProcess = flags.UseOldDeltaProcessFV
|
||||||
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
||||||
|
|
||||||
return opt
|
return opt
|
||||||
@ -58,7 +58,7 @@ func ParseBackupOptions() control.BackupConfig {
|
|||||||
opt.M365.DeltaPageSize = dps
|
opt.M365.DeltaPageSize = dps
|
||||||
opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
|
opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
|
||||||
opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
||||||
opt.M365.UseDriveDeltaTree = flags.UseDeltaTreeFV
|
opt.M365.UseOldDriveDeltaProcess = flags.UseOldDeltaProcessFV
|
||||||
opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
||||||
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
||||||
opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV
|
opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV
|
||||||
|
|||||||
@ -390,6 +390,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
|
|||||||
|
|
||||||
sel := selectors.NewSharePointBackup(siteIDs)
|
sel := selectors.NewSharePointBackup(siteIDs)
|
||||||
sel.Include(sel.LibraryFolders([]string{"foo"}, selectors.PrefixMatch()))
|
sel.Include(sel.LibraryFolders([]string{"foo"}, selectors.PrefixMatch()))
|
||||||
|
sel.Include(sel.Library("Documents"))
|
||||||
|
|
||||||
sel.SetDiscreteOwnerIDName(site.ID(), site.Name())
|
sel.SetDiscreteOwnerIDName(site.ID(), site.Name())
|
||||||
|
|
||||||
@ -407,21 +408,40 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
|
|||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
require.Len(t, cols, 2) // 1 collection, 1 path prefix directory to ensure the root path exists.
|
|
||||||
|
var (
|
||||||
|
hasDocumentsColl bool
|
||||||
|
hasMetadataColl bool
|
||||||
|
)
|
||||||
|
|
||||||
|
documentsColl, err := path.BuildPrefix(
|
||||||
|
suite.connector.tenant,
|
||||||
|
siteID,
|
||||||
|
path.SharePointService,
|
||||||
|
path.LibrariesCategory)
|
||||||
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
|
metadataColl, err := path.BuildMetadata(
|
||||||
|
suite.connector.tenant,
|
||||||
|
siteID,
|
||||||
|
path.SharePointService,
|
||||||
|
path.LibrariesCategory,
|
||||||
|
false)
|
||||||
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
|
for i, col := range cols {
|
||||||
|
fp := col.FullPath()
|
||||||
|
t.Logf("Collection %d: %s", i, fp)
|
||||||
|
|
||||||
|
hasDocumentsColl = hasDocumentsColl || fp.Equal(documentsColl)
|
||||||
|
hasMetadataColl = hasMetadataColl || fp.Equal(metadataColl)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.Truef(t, hasDocumentsColl, "found documents collection %s", documentsColl)
|
||||||
|
require.Truef(t, hasMetadataColl, "found metadata collection %s", metadataColl)
|
||||||
|
|
||||||
// No excludes yet as this isn't an incremental backup.
|
// No excludes yet as this isn't an incremental backup.
|
||||||
assert.True(t, excludes.Empty())
|
assert.True(t, excludes.Empty())
|
||||||
|
|
||||||
t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String())
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
path.SharePointMetadataService.String(),
|
|
||||||
cols[0].FullPath().Service().String())
|
|
||||||
|
|
||||||
t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String())
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
path.SharePointService.String(),
|
|
||||||
cols[1].FullPath().Service().String())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
|
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
|
||||||
|
|||||||
@ -285,7 +285,7 @@ func (c *Collections) Get(
|
|||||||
globalExcludeItemIDs *prefixmatcher.StringSetMatchBuilder,
|
globalExcludeItemIDs *prefixmatcher.StringSetMatchBuilder,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, bool, error) {
|
) ([]data.BackupCollection, bool, error) {
|
||||||
if c.ctrl.ToggleFeatures.UseDeltaTree {
|
if !c.ctrl.ToggleFeatures.UseOldDeltaProcess {
|
||||||
colls, canUsePrevBackup, err := c.getTree(ctx, prevMetadata, globalExcludeItemIDs, errs)
|
colls, canUsePrevBackup, err := c.getTree(ctx, prevMetadata, globalExcludeItemIDs, errs)
|
||||||
|
|
||||||
return colls,
|
return colls,
|
||||||
|
|||||||
@ -2634,7 +2634,9 @@ func (suite *CollectionsUnitSuite) TestGet() {
|
|||||||
tenant,
|
tenant,
|
||||||
idname.NewProvider(user, user),
|
idname.NewProvider(user, user),
|
||||||
func(*support.ControllerOperationStatus) {},
|
func(*support.ControllerOperationStatus) {},
|
||||||
control.Options{ToggleFeatures: control.Toggles{}},
|
control.Options{ToggleFeatures: control.Toggles{
|
||||||
|
UseOldDeltaProcess: true,
|
||||||
|
}},
|
||||||
count.New())
|
count.New())
|
||||||
|
|
||||||
prevDelta := "prev-delta"
|
prevDelta := "prev-delta"
|
||||||
|
|||||||
@ -207,9 +207,7 @@ func collWithMBH(mbh BackupHandler) *Collections {
|
|||||||
tenant,
|
tenant,
|
||||||
idname.NewProvider(user, user),
|
idname.NewProvider(user, user),
|
||||||
func(*support.ControllerOperationStatus) {},
|
func(*support.ControllerOperationStatus) {},
|
||||||
control.Options{ToggleFeatures: control.Toggles{
|
control.Options{ToggleFeatures: control.Toggles{}},
|
||||||
UseDeltaTree: true,
|
|
||||||
}},
|
|
||||||
count.New())
|
count.New())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -450,8 +450,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_noTree() {
|
|||||||
// check the expected items appear.
|
// check the expected items appear.
|
||||||
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_tree() {
|
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_tree() {
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
iterGetPreviewLimitsTests(suite, opts)
|
iterGetPreviewLimitsTests(suite, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -537,7 +535,7 @@ func runGetPreviewLimits(
|
|||||||
|
|
||||||
expectItemIDs := test.expectedItemIDsInCollection[folderPath]
|
expectItemIDs := test.expectedItemIDsInCollection[folderPath]
|
||||||
|
|
||||||
if opts.ToggleFeatures.UseDeltaTree && test.expectedItemIDsInCollectionTree != nil {
|
if !opts.ToggleFeatures.UseOldDeltaProcess && test.expectedItemIDsInCollectionTree != nil {
|
||||||
expectItemIDs = test.expectedItemIDsInCollectionTree[folderPath]
|
expectItemIDs = test.expectedItemIDsInCollectionTree[folderPath]
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -691,7 +689,6 @@ func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsNoTree() {
|
|||||||
// and such. Other tests are expected to provide more comprehensive checks.
|
// and such. Other tests are expected to provide more comprehensive checks.
|
||||||
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsWithTree() {
|
func (suite *LimiterUnitSuite) TestGet_PreviewLimits_defaultsWithTree() {
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
for _, test := range defaultLimitsTable() {
|
for _, test := range defaultLimitsTable() {
|
||||||
suite.Run(test.name, func() {
|
suite.Run(test.name, func() {
|
||||||
@ -815,7 +812,7 @@ func runGetPreviewLimitsDefaults(
|
|||||||
|
|
||||||
// Add one to account for the folder permissions item.
|
// Add one to account for the folder permissions item.
|
||||||
expected := test.expect.numItemsPerContainer + 1
|
expected := test.expect.numItemsPerContainer + 1
|
||||||
if opts.ToggleFeatures.UseDeltaTree {
|
if !opts.ToggleFeatures.UseOldDeltaProcess {
|
||||||
expected += test.expect.numItemsTreePadding
|
expected += test.expect.numItemsTreePadding
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -835,7 +832,7 @@ func runGetPreviewLimitsDefaults(
|
|||||||
|
|
||||||
// Add one to account for the folder permissions item.
|
// Add one to account for the folder permissions item.
|
||||||
expected := test.expect.numItems + test.expect.numContainers
|
expected := test.expect.numItems + test.expect.numContainers
|
||||||
if opts.ToggleFeatures.UseDeltaTree {
|
if !opts.ToggleFeatures.UseOldDeltaProcess {
|
||||||
expected += test.expect.numItemsTreePadding
|
expected += test.expect.numItemsTreePadding
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
"golang.org/x/exp/maps"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common/idname"
|
"github.com/alcionai/corso/src/internal/common/idname"
|
||||||
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
|
||||||
@ -1297,7 +1298,7 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
|
|||||||
runRestoreBackupTest(suite.T(), test, cfg)
|
runRestoreBackupTest(suite.T(), test, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
func (suite *ControllerIntegrationSuite) TestProduceBackupCollections_createsPrefixCollections() {
|
||||||
table := []struct {
|
table := []struct {
|
||||||
name string
|
name string
|
||||||
resourceCat resource.Category
|
resourceCat resource.Category
|
||||||
@ -1311,9 +1312,9 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
selectorFunc: func(t *testing.T) selectors.Selector {
|
selectorFunc: func(t *testing.T) selectors.Selector {
|
||||||
sel := selectors.NewExchangeBackup([]string{suite.user})
|
sel := selectors.NewExchangeBackup([]string{suite.user})
|
||||||
sel.Include(
|
sel.Include(
|
||||||
sel.ContactFolders([]string{selectors.NoneTgt}),
|
sel.ContactFolders(selectors.None()),
|
||||||
sel.EventCalendars([]string{selectors.NoneTgt}),
|
sel.EventCalendars(selectors.None()),
|
||||||
sel.MailFolders([]string{selectors.NoneTgt}))
|
sel.MailFolders(selectors.None()))
|
||||||
|
|
||||||
return sel.Selector
|
return sel.Selector
|
||||||
},
|
},
|
||||||
@ -1329,7 +1330,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
resourceCat: resource.Users,
|
resourceCat: resource.Users,
|
||||||
selectorFunc: func(t *testing.T) selectors.Selector {
|
selectorFunc: func(t *testing.T) selectors.Selector {
|
||||||
sel := selectors.NewOneDriveBackup([]string{suite.user})
|
sel := selectors.NewOneDriveBackup([]string{suite.user})
|
||||||
sel.Include(sel.Folders([]string{selectors.NoneTgt}))
|
sel.Include(sel.Folders(selectors.None()))
|
||||||
|
|
||||||
return sel.Selector
|
return sel.Selector
|
||||||
},
|
},
|
||||||
@ -1344,10 +1345,11 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
selectorFunc: func(t *testing.T) selectors.Selector {
|
selectorFunc: func(t *testing.T) selectors.Selector {
|
||||||
sel := selectors.NewSharePointBackup([]string{tconfig.M365SiteID(t)})
|
sel := selectors.NewSharePointBackup([]string{tconfig.M365SiteID(t)})
|
||||||
sel.Include(
|
sel.Include(
|
||||||
sel.LibraryFolders([]string{selectors.NoneTgt}),
|
sel.Library(selectors.NoneTgt),
|
||||||
|
sel.LibraryFolders(selectors.None()),
|
||||||
// not yet in use
|
// not yet in use
|
||||||
// sel.Pages([]string{selectors.NoneTgt}),
|
// sel.Pages(selectors.None()),
|
||||||
// sel.Lists([]string{selectors.NoneTgt}),
|
// sel.Lists(selectors.None()),
|
||||||
)
|
)
|
||||||
|
|
||||||
return sel.Selector
|
return sel.Selector
|
||||||
@ -1366,10 +1368,11 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
selectorFunc: func(t *testing.T) selectors.Selector {
|
selectorFunc: func(t *testing.T) selectors.Selector {
|
||||||
sel := selectors.NewGroupsBackup([]string{tconfig.M365TeamID(t)})
|
sel := selectors.NewGroupsBackup([]string{tconfig.M365TeamID(t)})
|
||||||
sel.Include(
|
sel.Include(
|
||||||
sel.LibraryFolders([]string{selectors.NoneTgt}),
|
sel.Library(selectors.NoneTgt),
|
||||||
|
sel.LibraryFolders(selectors.None()),
|
||||||
// not yet in use
|
// not yet in use
|
||||||
// sel.Pages([]string{selectors.NoneTgt}),
|
// sel.Pages(selectors.None()),
|
||||||
// sel.Lists([]string{selectors.NoneTgt}),
|
// sel.Lists(selectors.None()),
|
||||||
)
|
)
|
||||||
|
|
||||||
return sel.Selector
|
return sel.Selector
|
||||||
@ -1423,8 +1426,14 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
t.Logf("Backup enumeration complete in %v\n", time.Since(start))
|
t.Logf("Backup enumeration complete in %v\n", time.Since(start))
|
||||||
|
|
||||||
// Use a map to find duplicates.
|
// Use a map to find duplicates.
|
||||||
foundCategories := []string{}
|
foundCategories := map[string]struct{}{}
|
||||||
for _, col := range dcs {
|
var foundPrefixCollection bool
|
||||||
|
|
||||||
|
for i, col := range dcs {
|
||||||
|
fp := col.FullPath()
|
||||||
|
|
||||||
|
t.Logf("collection %d: %s", i, fp)
|
||||||
|
|
||||||
// TODO(ashmrtn): We should be able to remove the below if we change how
|
// TODO(ashmrtn): We should be able to remove the below if we change how
|
||||||
// status updates are done. Ideally we shouldn't have to fetch items in
|
// status updates are done. Ideally we shouldn't have to fetch items in
|
||||||
// these collections to avoid deadlocking.
|
// these collections to avoid deadlocking.
|
||||||
@ -1437,23 +1446,26 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Ignore metadata collections.
|
// Ignore metadata collections.
|
||||||
fullPath := col.FullPath()
|
if fp.Service() != test.service {
|
||||||
if fullPath.Service() != test.service {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.Empty(t, fullPath.Folders(), "non-prefix collection")
|
if len(fp.Folders()) == 0 {
|
||||||
assert.NotEqual(t, col.State(), data.NewState, "prefix collection marked as new")
|
t.Log("prefix collection tests")
|
||||||
foundCategories = append(foundCategories, fullPath.Category().String())
|
assert.NotEqual(t, col.State(), data.NewState, "prefix collection must have New status")
|
||||||
|
assert.Zero(t, found, "prefix collection must not contain items")
|
||||||
assert.Zero(t, found, "non-empty collection")
|
foundPrefixCollection = true
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.ElementsMatch(t, test.categories, foundCategories)
|
foundCategories[fp.Category().String()] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, test.categories, maps.Keys(foundCategories))
|
||||||
|
|
||||||
backupCtrl.Wait()
|
backupCtrl.Wait()
|
||||||
|
|
||||||
assert.NoError(t, errs.Failure())
|
assert.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
|
||||||
|
assert.True(t, foundPrefixCollection, "backup must generate a prefix collection")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -510,7 +510,6 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
|
|||||||
restoreCfg.IncludePermissions = true
|
restoreCfg.IncludePermissions = true
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
@ -763,7 +762,6 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
|
|||||||
restoreCfg.IncludePermissions = true
|
restoreCfg.IncludePermissions = true
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
@ -857,7 +855,6 @@ func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) {
|
|||||||
restoreCfg.IncludePermissions = false
|
restoreCfg.IncludePermissions = false
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
@ -1066,7 +1063,6 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
|
|||||||
restoreCfg.IncludePermissions = true
|
restoreCfg.IncludePermissions = true
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
@ -1265,7 +1261,6 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion
|
|||||||
restoreCfg.IncludePermissions = true
|
restoreCfg.IncludePermissions = true
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
@ -1385,7 +1380,6 @@ func testRestoreFolderNamedFolderRegression(
|
|||||||
restoreCfg.IncludePermissions = true
|
restoreCfg.IncludePermissions = true
|
||||||
|
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
cfg := m365Stub.ConfigInfo{
|
cfg := m365Stub.ConfigInfo{
|
||||||
Tenant: suite.Tenant(),
|
Tenant: suite.Tenant(),
|
||||||
|
|||||||
@ -409,6 +409,7 @@ func (suite *BackupOpUnitSuite) TestNewBackupOperation_configuredOptionsMatchInp
|
|||||||
RunMigrations: true,
|
RunMigrations: true,
|
||||||
DisableSlidingWindowLimiter: true,
|
DisableSlidingWindowLimiter: true,
|
||||||
UseDeltaTree: true,
|
UseDeltaTree: true,
|
||||||
|
UseOldDeltaProcess: true,
|
||||||
DisableLazyItemReader: true,
|
DisableLazyItemReader: true,
|
||||||
},
|
},
|
||||||
PreviewLimits: control.PreviewItemLimits{
|
PreviewLimits: control.PreviewItemLimits{
|
||||||
|
|||||||
@ -270,8 +270,6 @@ func RunMergeBaseGroupsUpdate(
|
|||||||
whatSet = deeTD.CategoryFromRepoRef
|
whatSet = deeTD.CategoryFromRepoRef
|
||||||
)
|
)
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
// Need outside the inner test case so bod lasts for the entire test.
|
// Need outside the inner test case so bod lasts for the entire test.
|
||||||
bo, bod := PrepNewTestBackupOp(
|
bo, bod := PrepNewTestBackupOp(
|
||||||
t,
|
t,
|
||||||
|
|||||||
@ -870,7 +870,6 @@ func RunDriveAssistBaseGroupsUpdate(
|
|||||||
opts = control.DefaultOptions()
|
opts = control.DefaultOptions()
|
||||||
)
|
)
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
opts.ItemExtensionFactory = []extensions.CreateItemExtensioner{
|
opts.ItemExtensionFactory = []extensions.CreateItemExtensioner{
|
||||||
NewCreateSingleFileFailExtension(),
|
NewCreateSingleFileFailExtension(),
|
||||||
}
|
}
|
||||||
|
|||||||
@ -108,8 +108,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeGroups() {
|
|||||||
|
|
||||||
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
|
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunBasicDriveishBackupTests(
|
RunBasicDriveishBackupTests(
|
||||||
suite,
|
suite,
|
||||||
path.GroupsService,
|
path.GroupsService,
|
||||||
@ -119,8 +117,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeGroups() {
|
|||||||
|
|
||||||
func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeIncrementalGroups() {
|
func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeIncrementalGroups() {
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
runGroupsIncrementalBackupTests(suite, suite.its, opts)
|
runGroupsIncrementalBackupTests(suite, suite.its, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -133,8 +129,6 @@ func (suite *GroupsBackupTreeIntgSuite) TestBackup_Run_treeExtensionsGroups() {
|
|||||||
|
|
||||||
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
|
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunDriveishBackupWithExtensionsTests(
|
RunDriveishBackupWithExtensionsTests(
|
||||||
suite,
|
suite,
|
||||||
path.GroupsService,
|
path.GroupsService,
|
||||||
|
|||||||
@ -116,8 +116,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeOneDrive() {
|
|||||||
|
|
||||||
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunBasicDriveishBackupTests(
|
RunBasicDriveishBackupTests(
|
||||||
suite,
|
suite,
|
||||||
path.OneDriveService,
|
path.OneDriveService,
|
||||||
@ -127,8 +125,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeOneDrive() {
|
|||||||
|
|
||||||
func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeIncrementalOneDrive() {
|
func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeIncrementalOneDrive() {
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
runOneDriveIncrementalBackupTests(suite, suite.its, opts)
|
runOneDriveIncrementalBackupTests(suite, suite.its, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -141,8 +137,6 @@ func (suite *OneDriveBackupTreeIntgSuite) TestBackup_Run_treeExtensionsOneDrive(
|
|||||||
|
|
||||||
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
sel.Include(selTD.OneDriveBackupFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunDriveishBackupWithExtensionsTests(
|
RunDriveishBackupWithExtensionsTests(
|
||||||
suite,
|
suite,
|
||||||
path.OneDriveService,
|
path.OneDriveService,
|
||||||
|
|||||||
@ -172,8 +172,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeSharePoint() {
|
|||||||
|
|
||||||
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunBasicDriveishBackupTests(
|
RunBasicDriveishBackupTests(
|
||||||
suite,
|
suite,
|
||||||
path.SharePointService,
|
path.SharePointService,
|
||||||
@ -183,8 +181,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeSharePoint() {
|
|||||||
|
|
||||||
func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeIncrementalSharePoint() {
|
func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeIncrementalSharePoint() {
|
||||||
opts := control.DefaultOptions()
|
opts := control.DefaultOptions()
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
runSharePointIncrementalBackupTests(suite, suite.its, opts)
|
runSharePointIncrementalBackupTests(suite, suite.its, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -197,8 +193,6 @@ func (suite *SharePointBackupTreeIntgSuite) TestBackup_Run_treeExtensionsSharePo
|
|||||||
|
|
||||||
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
sel.Include(selTD.SharePointBackupFolderScope(sel))
|
||||||
|
|
||||||
opts.ToggleFeatures.UseDeltaTree = true
|
|
||||||
|
|
||||||
RunDriveishBackupWithExtensionsTests(
|
RunDriveishBackupWithExtensionsTests(
|
||||||
suite,
|
suite,
|
||||||
path.SharePointService,
|
path.SharePointService,
|
||||||
|
|||||||
@ -162,7 +162,7 @@ func ReadTestConfig() (map[string]string, error) {
|
|||||||
TestCfgSiteID,
|
TestCfgSiteID,
|
||||||
os.Getenv(EnvCorsoM365TestSiteID),
|
os.Getenv(EnvCorsoM365TestSiteID),
|
||||||
vpr.GetString(TestCfgSiteID),
|
vpr.GetString(TestCfgSiteID),
|
||||||
"4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
|
"10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
|
||||||
fallbackTo(
|
fallbackTo(
|
||||||
testEnv,
|
testEnv,
|
||||||
TestCfgTeamID,
|
TestCfgTeamID,
|
||||||
@ -174,7 +174,7 @@ func ReadTestConfig() (map[string]string, error) {
|
|||||||
TestCfgTeamSiteID,
|
TestCfgTeamSiteID,
|
||||||
os.Getenv(EnvCorsoM365TestTeamSiteID),
|
os.Getenv(EnvCorsoM365TestTeamSiteID),
|
||||||
vpr.GetString(TestCfgTeamSiteID),
|
vpr.GetString(TestCfgTeamSiteID),
|
||||||
"8df7082a-62eb-4165-97f9-ead8cf04c0d8,11810f05-e8bd-45e2-9728-92e434260750")
|
"10rqc2.sharepoint.com,8df7082a-62eb-4165-97f9-ead8cf04c0d8,11810f05-e8bd-45e2-9728-92e434260750")
|
||||||
fallbackTo(
|
fallbackTo(
|
||||||
testEnv,
|
testEnv,
|
||||||
TestCfgSecondaryTeamID,
|
TestCfgSecondaryTeamID,
|
||||||
@ -204,7 +204,7 @@ func ReadTestConfig() (map[string]string, error) {
|
|||||||
TestCfgSecondarySiteID,
|
TestCfgSecondarySiteID,
|
||||||
os.Getenv(EnvCorsoSecondaryM365TestSiteID),
|
os.Getenv(EnvCorsoSecondaryM365TestSiteID),
|
||||||
vpr.GetString(TestCfgSecondarySiteID),
|
vpr.GetString(TestCfgSecondarySiteID),
|
||||||
"053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c")
|
"10rqc2.sharepoint.com,053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c")
|
||||||
fallbackTo(
|
fallbackTo(
|
||||||
testEnv,
|
testEnv,
|
||||||
TestCfgUnlicensedUserID,
|
TestCfgUnlicensedUserID,
|
||||||
|
|||||||
@ -64,7 +64,7 @@ type BackupM365Config struct {
|
|||||||
ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"`
|
ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"`
|
||||||
|
|
||||||
// see: https://github.com/alcionai/corso/issues/4688
|
// see: https://github.com/alcionai/corso/issues/4688
|
||||||
UseDriveDeltaTree bool `json:"useDriveDeltaTree"`
|
UseOldDriveDeltaProcess bool `json:"useOldDriveDeltaProcess"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Parallelism struct {
|
type Parallelism struct {
|
||||||
|
|||||||
@ -100,6 +100,7 @@ type Toggles struct {
|
|||||||
|
|
||||||
// see: https://github.com/alcionai/corso/issues/4688
|
// see: https://github.com/alcionai/corso/issues/4688
|
||||||
UseDeltaTree bool `json:"useDeltaTree"`
|
UseDeltaTree bool `json:"useDeltaTree"`
|
||||||
|
UseOldDeltaProcess bool `json:"useOldDeltaProcess"`
|
||||||
|
|
||||||
// AddDisableLazyItemReader disables lazy item reader, such that we fall
|
// AddDisableLazyItemReader disables lazy item reader, such that we fall
|
||||||
// back to prefetch reader. This flag is currently only meant for groups
|
// back to prefetch reader. This flag is currently only meant for groups
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user