larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add restore things container, move perms

Browse Source 
This PR includes two smaller changes that
cascaded to touch a lot of files:
first, introduces inject.RestoreConsumerConfig, which
is a container-of-things for holding common restore configs and options.
second, moves the restorePermissions flag from options
into the restoreConfig.
This commit is contained in:
ryanfkeepers 2023-07-19 14:05:17 -06:00
parent 3a02e3269b
commit 683fb248e3
38 changed files with 358 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/cli/backup/help_e2e_test.go
View File

@ -47,7 +47,7 @@ func prepM365Test(
vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force)
ctx = config.SetViper(ctx, vpr)
repo, err := repository.Initialize(ctx, acct, st, control.Defaults())
repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
return acct, st, repo, vpr, recorder, cfgFP

2
src/cli/repo/s3_e2e_test.go
View File

@ -200,7 +200,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
ctx = config.SetViper(ctx, vpr)
// init the repo first
_, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults())
_, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
// then test it

3
src/cli/utils/options.go
View File

@ -8,14 +8,13 @@ import (
// Control produces the control options based on the user's flags.
func Control() control.Options {
opt := control.Defaults()
opt := control.DefaultOptions()
if flags.FailFastFV {
opt.FailureHandling = control.FailFast
}
opt.DisableMetrics = flags.NoStatsFV
opt.RestorePermissions = flags.RestorePermissionsFV
opt.SkipReduce = flags.SkipReduceFV
opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV
opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV

12
src/cli/utils/restore_config.go
View File

@ -18,16 +18,18 @@ type RestoreCfgOpts struct {
// DTTMFormat is the timestamp format appended
// to the default folder name. Defaults to
// dttm.HumanReadable.
DTTMFormat dttm.TimeFormat
DTTMFormat dttm.TimeFormat
RestorePermissions bool
Populated flags.PopulatedFlags
}
func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts {
return RestoreCfgOpts{
Collisions: flags.CollisionsFV,
Destination: flags.DestinationFV,
DTTMFormat: dttm.HumanReadable,
Collisions: flags.CollisionsFV,
Destination: flags.DestinationFV,
DTTMFormat: dttm.HumanReadable,
RestorePermissions: flags.RestorePermissionsFV,
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
@ -67,6 +69,8 @@ func MakeRestoreConfig(
restoreCfg.Location = opts.Destination
}
restoreCfg.IncludePermissions = opts.RestorePermissions
Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
return restoreCfg

44
src/cli/utils/restore_config_test.go
View File

@ -68,18 +68,18 @@ func (suite *RestoreCfgUnitSuite) TestValidateRestoreConfigFlags() {
}
func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
rco := &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
}
table := []struct {
name string
rco *RestoreCfgOpts
populated flags.PopulatedFlags
expect control.RestoreConfig
}{
{
name: "not populated",
name: "not populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{},
expect: control.RestoreConfig{
OnCollision: control.Skip,
@ -88,6 +88,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
},
{
name: "collision populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{
flags.CollisionsFN: {},
},
@ -98,6 +102,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
},
{
name: "destination populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{
flags.DestinationFN: {},
},
@ -108,6 +116,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
},
{
name: "both populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{
flags.CollisionsFN: {},
flags.DestinationFN: {},
@ -117,6 +129,23 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
Location: "destination",
},
},
{
name: "with restore permissions",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
RestorePermissions: true,
},
populated: flags.PopulatedFlags{
flags.CollisionsFN: {},
flags.DestinationFN: {},
},
expect: control.RestoreConfig{
OnCollision: control.CollisionPolicy("collisions"),
Location: "destination",
IncludePermissions: true,
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -125,12 +154,13 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
ctx, flush := tester.NewContext(t)
defer flush()
opts := *rco
opts := *test.rco
opts.Populated = test.populated
result := MakeRestoreConfig(ctx, opts)
assert.Equal(t, test.expect.OnCollision, result.OnCollision)
assert.Contains(t, result.Location, test.expect.Location)
assert.Equal(t, test.expect.IncludePermissions, result.IncludePermissions)
})
}
}

30
src/cmd/factory/impl/common.go
View File

@ -21,12 +21,12 @@ import (
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
@ -104,7 +104,15 @@ func generateAndRestoreItems(
print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs, ctr)
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
return ctrl.ConsumeRestoreCollections(ctx, rcc, dataColls, errs, ctr)
}
// ------------------------------------------------------------------------------------------
@ -407,10 +415,8 @@ func generateAndRestoreDriveItems(
// input,
// version.Backup)
opts := control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
}
opts := control.DefaultOptions()
restoreCfg.IncludePermissions = true
config := m365Stub.ConfigInfo{
Opts: opts,
@ -418,7 +424,7 @@ func generateAndRestoreDriveItems(
Service: service,
Tenant: tenantID,
ResourceOwners: []string{resourceOwner},
RestoreCfg: testdata.DefaultRestoreConfig(""),
RestoreCfg: restoreCfg,
}
_, _, collections, _, err := m365Stub.GetCollectionsAndExpected(
@ -429,5 +435,13 @@ func generateAndRestoreDriveItems(
return nil, err
}
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs, ctr)
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
return ctrl.ConsumeRestoreCollections(ctx, rcc, collections, errs, ctr)
}

6
src/cmd/factory/impl/exchange.go
View File

@ -72,7 +72,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
subject, body, body,
now, now, now, now)
},
control.Defaults(),
control.DefaultOptions(),
errs,
count.New())
if err != nil {
@ -121,7 +121,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
exchMock.NoAttachments, exchMock.NoCancelledOccurrences,
exchMock.NoExceptionOccurrences)
},
control.Defaults(),
control.DefaultOptions(),
errs,
count.New())
if err != nil {
@ -172,7 +172,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
"123-456-7890",
)
},
control.Defaults(),
control.DefaultOptions(),
errs,
count.New())
if err != nil {

2
src/internal/events/events_test.go
View File

@ -52,7 +52,7 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
)
require.NoError(t, err, clues.ToCore(err))
b, err := events.NewBus(ctx, s, a.ID(), control.Defaults())
b, err := events.NewBus(ctx, s, a.ID(), control.DefaultOptions())
require.NotEmpty(t, b)
require.NoError(t, err, clues.ToCore(err))

10
src/internal/m365/backup_test.go
View File

@ -120,7 +120,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
sel := test.getSelector(t)
uidn := inMock.NewProvider(sel.ID(), sel.Name())
ctrlOpts := control.Defaults()
ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries
collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections(
@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner()
test.getSelector(t),
nil,
version.NoBackup,
control.Defaults(),
control.DefaultOptions(),
fault.New(true))
assert.Error(t, err, clues.ToCore(err))
assert.False(t, canUsePreviousBackup, "can use previous backup")
@ -296,7 +296,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
nil,
ctrl.credentials,
ctrl,
control.Defaults(),
control.DefaultOptions(),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -381,7 +381,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
sel.Selector,
nil,
version.NoBackup,
control.Defaults(),
control.DefaultOptions(),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -428,7 +428,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
sel.Selector,
nil,
version.NoBackup,
control.Defaults(),
control.DefaultOptions(),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup")

118
src/internal/m365/controller_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
"github.com/alcionai/corso/src/internal/data"
@ -20,6 +21,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/internal/version"
@ -384,15 +386,19 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() {
}
)
restoreCfg.IncludePermissions = true
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
deets, err := suite.ctrl.ConsumeRestoreCollections(
ctx,
version.Backup,
sel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
rcc,
nil,
fault.New(true),
count.New())
@ -407,6 +413,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() {
func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
restoreCfg := testdata.DefaultRestoreConfig("")
restoreCfg.IncludePermissions = true
table := []struct {
name string
col []data.RestoreCollection
@ -463,15 +471,17 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
ctx, flush := tester.NewContext(t)
defer flush()
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: test.sel,
RestoreConfig: restoreCfg,
Selector: test.sel,
}
deets, err := suite.ctrl.ConsumeRestoreCollections(
ctx,
version.Backup,
test.sel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
rcc,
test.col,
fault.New(true),
count.New())
@ -498,16 +508,24 @@ func runRestore(
sci.RestoreCfg.Location,
sci.ResourceOwners)
sci.RestoreCfg.IncludePermissions = true
start := time.Now()
restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService)
restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true)
rcc := inject.RestoreConsumerConfig{
BackupVersion: backupVersion,
Options: control.DefaultOptions(),
ProtectedResource: restoreSel,
RestoreConfig: sci.RestoreCfg,
Selector: restoreSel,
}
deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx,
backupVersion,
restoreSel,
sci.RestoreCfg,
sci.Opts,
rcc,
collections,
fault.New(true),
count.New())
@ -609,6 +627,7 @@ func runRestoreBackupTest(
tenant string,
resourceOwners []string,
opts control.Options,
restoreCfg control.RestoreConfig,
) {
ctx, flush := tester.NewContext(t)
defer flush()
@ -619,7 +638,7 @@ func runRestoreBackupTest(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
RestoreCfg: testdata.DefaultRestoreConfig(""),
RestoreCfg: restoreCfg,
}
totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected(
@ -654,6 +673,7 @@ func runRestoreTestWithVersion(
tenant string,
resourceOwners []string,
opts control.Options,
restoreCfg control.RestoreConfig,
) {
ctx, flush := tester.NewContext(t)
defer flush()
@ -664,7 +684,7 @@ func runRestoreTestWithVersion(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
RestoreCfg: testdata.DefaultRestoreConfig(""),
RestoreCfg: restoreCfg,
}
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
@ -691,6 +711,7 @@ func runRestoreBackupTestVersions(
tenant string,
resourceOwners []string,
opts control.Options,
restoreCfg control.RestoreConfig,
) {
ctx, flush := tester.NewContext(t)
defer flush()
@ -701,7 +722,7 @@ func runRestoreBackupTestVersions(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
RestoreCfg: testdata.DefaultRestoreConfig(""),
RestoreCfg: restoreCfg,
}
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
@ -739,6 +760,9 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
bodyText := "This email has some text. However, all the text is on the same line."
subjectText := "Test message for restore"
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
table := []restoreBackupInfo{
{
name: "EmailsWithAttachments",
@ -994,10 +1018,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
test,
suite.ctrl.tenant,
[]string{suite.user},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -1078,6 +1100,8 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
for i, collection := range test.collections {
// Get a restoreCfg per collection so they're independent.
restoreCfg := testdata.DefaultRestoreConfig("")
restoreCfg.IncludePermissions = true
expectedDests = append(expectedDests, destAndCats{
resourceOwner: suite.user,
dest: restoreCfg.Location,
@ -1110,15 +1134,18 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
)
restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService)
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: restoreSel,
RestoreConfig: restoreCfg,
Selector: restoreSel,
}
deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx,
version.Backup,
restoreSel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
rcc,
collections,
fault.New(true),
count.New())
@ -1150,10 +1177,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
backupSel,
nil,
version.NoBackup,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
control.DefaultOptions(),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -1162,10 +1186,13 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
t.Log("Backup enumeration complete")
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
ci := stub.ConfigInfo{
Opts: control.Options{RestorePermissions: true},
Opts: control.DefaultOptions(),
// Alright to be empty, needed for OneDrive.
RestoreCfg: control.RestoreConfig{},
RestoreCfg: restoreCfg,
}
// Pull the data prior to waiting for the status as otherwise it will
@ -1203,16 +1230,16 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
},
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTest(
suite.T(),
test,
suite.ctrl.tenant,
[]string{suite.user},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
)
control.DefaultOptions(),
restoreCfg)
}
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
@ -1306,10 +1333,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
backupSel,
nil,
version.NoBackup,
control.Options{
RestorePermissions: false,
ToggleFeatures: control.Toggles{},
},
control.DefaultOptions(),
fault.New(true))
require.NoError(t, err)
assert.True(t, canUsePreviousBackup, "can use previous backup")

14
src/internal/m365/exchange/backup_test.go
View File

@ -466,7 +466,7 @@ func (suite *BackupIntgSuite) TestMailFetch() {
ctx, flush := tester.NewContext(t)
defer flush()
ctrlOpts := control.Defaults()
ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries
collections, err := createCollections(
@ -554,7 +554,7 @@ func (suite *BackupIntgSuite) TestDelta() {
inMock.NewProvider(userID, userID),
test.scope,
DeltaPaths{},
control.Defaults(),
control.DefaultOptions(),
func(status *support.ControllerOperationStatus) {},
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
@ -587,7 +587,7 @@ func (suite *BackupIntgSuite) TestDelta() {
inMock.NewProvider(userID, userID),
test.scope,
dps,
control.Defaults(),
control.DefaultOptions(),
func(status *support.ControllerOperationStatus) {},
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
@ -633,7 +633,7 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() {
inMock.NewProvider(suite.user, suite.user),
sel.Scopes()[0],
DeltaPaths{},
control.Defaults(),
control.DefaultOptions(),
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
@ -709,7 +709,7 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() {
inMock.NewProvider(suite.user, suite.user),
test.scope,
DeltaPaths{},
control.Defaults(),
control.DefaultOptions(),
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
@ -834,7 +834,7 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() {
inMock.NewProvider(suite.user, suite.user),
test.scope,
DeltaPaths{},
control.Defaults(),
control.DefaultOptions(),
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
@ -1995,7 +1995,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
ctx, flush := tester.NewContext(t)
defer flush()
ctrlOpts := control.Defaults()
ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter
getter := test.getter

2
src/internal/m365/exchange/collection_test.go
View File

@ -178,7 +178,7 @@ func (suite *CollectionSuite) TestNewCollection_state() {
test.curr, test.prev, test.loc,
0,
&mockItemer{}, nil,
control.Defaults(),
control.DefaultOptions(),
false)
assert.Equal(t, test.expect, c.State(), "collection state")
assert.Equal(t, test.curr, c.fullPath, "full path")

9
src/internal/m365/exchange/restore.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
@ -28,7 +29,7 @@ import (
func ConsumeRestoreCollections(
ctx context.Context,
ac api.Client,
restoreCfg control.RestoreConfig,
rcc inject.RestoreConsumerConfig,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Bus,
@ -80,7 +81,7 @@ func ConsumeRestoreCollections(
containerID, gcc, err := createDestination(
ictx,
handler,
handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()),
handler.formatRestoreDestination(rcc.RestoreConfig.Location, dc.FullPath()),
userID,
directoryCache[category],
errs)
@ -105,7 +106,7 @@ func ConsumeRestoreCollections(
userID,
containerID,
collisionKeyToItemID,
restoreCfg.OnCollision,
rcc.RestoreConfig.OnCollision,
deets,
errs,
ctr)
@ -126,7 +127,7 @@ func ConsumeRestoreCollections(
support.Restore,
len(dcs),
metrics,
restoreCfg.Location)
rcc.RestoreConfig.Location)
return status, el.Failure()
}

2
src/internal/m365/helper_test.go
View File

@ -796,7 +796,7 @@ func compareDriveItem(
assert.Equal(t, expectedMeta.FileName, itemMeta.FileName)
}
if !mci.Opts.RestorePermissions {
if !mci.RestoreCfg.IncludePermissions {
assert.Equal(t, 0, len(itemMeta.Permissions))
return true
}

5
src/internal/m365/mock/connector.go
View File

@ -63,10 +63,7 @@ func (ctrl Controller) Wait() *data.CollectionStats {
func (ctrl Controller) ConsumeRestoreCollections(
_ context.Context,
_ int,
_ selectors.Selector,
_ control.RestoreConfig,
_ control.Options,
_ inject.RestoreConsumerConfig,
_ []data.RestoreCollection,
_ *fault.Bus,
_ *count.Bus,

2
src/internal/m365/onedrive/collection_test.go
View File

@ -945,7 +945,7 @@ func (suite *CollectionUnitTestSuite) TestItemExtensions() {
nil,
}
opts := control.Defaults()
opts := control.DefaultOptions()
opts.ItemExtensionFactory = append(
opts.ItemExtensionFactory,
test.factories...)

69
src/internal/m365/onedrive/restore.go
View File

@ -23,6 +23,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
@ -143,9 +144,7 @@ func NewRestoreCaches(
func ConsumeRestoreCollections(
ctx context.Context,
rh RestoreHandler,
backupVersion int,
restoreCfg control.RestoreConfig,
opts control.Options,
rcc inject.RestoreConsumerConfig,
backupDriveIDNames idname.Cacher,
dcs []data.RestoreCollection,
deets *details.Builder,
@ -160,7 +159,7 @@ func ConsumeRestoreCollections(
fallbackDriveName = "" // onedrive cannot create drives
)
ctx = clues.Add(ctx, "backup_version", backupVersion)
ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion)
err := caches.Populate(ctx, rh, protectedResourceID)
if err != nil {
@ -190,12 +189,10 @@ func ConsumeRestoreCollections(
metrics, err = RestoreCollection(
ictx,
rh,
restoreCfg,
backupVersion,
rcc,
dc,
caches,
deets,
opts.RestorePermissions,
fallbackDriveName,
errs,
ctr.Local())
@ -215,7 +212,7 @@ func ConsumeRestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
restoreCfg.Location)
rcc.RestoreConfig.Location)
return status, el.Failure()
}
@ -228,12 +225,10 @@ func ConsumeRestoreCollections(
func RestoreCollection(
ctx context.Context,
rh RestoreHandler,
restoreCfg control.RestoreConfig,
backupVersion int,
rcc inject.RestoreConsumerConfig,
dc data.RestoreCollection,
caches *restoreCaches,
deets *details.Builder,
restorePerms bool, // TODD: move into restoreConfig
fallbackDriveName string,
errs *fault.Bus,
ctr *count.Bus,
@ -281,8 +276,8 @@ func RestoreCollection(
// the drive into which this folder gets restored is tracked separately in drivePath.
restoreDir := &path.Builder{}
if len(restoreCfg.Location) > 0 {
restoreDir = restoreDir.Append(restoreCfg.Location)
if len(rcc.RestoreConfig.Location) > 0 {
restoreDir = restoreDir.Append(rcc.RestoreConfig.Location)
}
restoreDir = restoreDir.Append(drivePath.Folders...)
@ -301,8 +296,8 @@ func RestoreCollection(
drivePath,
dc,
caches,
backupVersion,
restorePerms)
rcc.BackupVersion,
rcc.RestoreConfig.IncludePermissions)
if err != nil {
return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx)
}
@ -316,7 +311,7 @@ func RestoreCollection(
dc.FullPath(),
colMeta,
caches,
restorePerms)
rcc.RestoreConfig.IncludePermissions)
if err != nil {
return metrics, clues.Wrap(err, "creating folders for restore")
}
@ -390,14 +385,12 @@ func RestoreCollection(
itemInfo, skipped, err := restoreItem(
ictx,
rh,
restoreCfg,
rcc,
dc,
backupVersion,
drivePath,
restoreFolderID,
copyBuffer,
caches,
restorePerms,
itemData,
itemPath,
ctr)
@ -440,14 +433,12 @@ func RestoreCollection(
func restoreItem(
ctx context.Context,
rh RestoreHandler,
restoreCfg control.RestoreConfig,
rcc inject.RestoreConsumerConfig,
fibn data.FetchItemByNamer,
backupVersion int,
drivePath *path.DrivePath,
restoreFolderID string,
copyBuffer []byte,
caches *restoreCaches,
restorePerms bool,
itemData data.Stream,
itemPath path.Path,
ctr *count.Bus,
@ -455,11 +446,11 @@ func restoreItem(
itemUUID := itemData.UUID()
ctx = clues.Add(ctx, "item_id", itemUUID)
if backupVersion < version.OneDrive1DataAndMetaFiles {
if rcc.BackupVersion < version.OneDrive1DataAndMetaFiles {
itemInfo, err := restoreV0File(
ctx,
rh,
restoreCfg,
rcc.RestoreConfig,
drivePath,
fibn,
restoreFolderID,
@ -468,7 +459,7 @@ func restoreItem(
itemData,
ctr)
if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil
}
@ -491,7 +482,7 @@ func restoreItem(
// Only the version.OneDrive1DataAndMetaFiles needed to deserialize the
// permission for child folders here. Later versions can request
// permissions inline when processing the collection.
if !restorePerms || backupVersion >= version.OneDrive4DirIncludesPermissions {
if !rcc.RestoreConfig.IncludePermissions || rcc.BackupVersion >= version.OneDrive4DirIncludesPermissions {
return details.ItemInfo{}, true, nil
}
@ -511,22 +502,21 @@ func restoreItem(
// only items with DataFileSuffix from this point on
if backupVersion < version.OneDrive6NameInMeta {
if rcc.BackupVersion < version.OneDrive6NameInMeta {
itemInfo, err := restoreV1File(
ctx,
rh,
restoreCfg,
rcc,
drivePath,
fibn,
restoreFolderID,
copyBuffer,
restorePerms,
caches,
itemPath,
itemData,
ctr)
if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil
}
@ -541,18 +531,17 @@ func restoreItem(
itemInfo, err := restoreV6File(
ctx,
rh,
restoreCfg,
rcc,
drivePath,
fibn,
restoreFolderID,
copyBuffer,
restorePerms,
caches,
itemPath,
itemData,
ctr)
if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil
}
@ -596,12 +585,11 @@ func restoreV0File(
func restoreV1File(
ctx context.Context,
rh RestoreHandler,
restoreCfg control.RestoreConfig,
rcc inject.RestoreConsumerConfig,
drivePath *path.DrivePath,
fibn data.FetchItemByNamer,
restoreFolderID string,
copyBuffer []byte,
restorePerms bool,
caches *restoreCaches,
itemPath path.Path,
itemData data.Stream,
@ -611,7 +599,7 @@ func restoreV1File(
itemID, itemInfo, err := restoreFile(
ctx,
restoreCfg,
rcc.RestoreConfig,
rh,
fibn,
trimmedName,
@ -627,7 +615,7 @@ func restoreV1File(
// Mark it as success without processing .meta
// file if we are not restoring permissions
if !restorePerms {
if !rcc.RestoreConfig.IncludePermissions {
return itemInfo, nil
}
@ -657,12 +645,11 @@ func restoreV1File(
func restoreV6File(
ctx context.Context,
rh RestoreHandler,
restoreCfg control.RestoreConfig,
rcc inject.RestoreConsumerConfig,
drivePath *path.DrivePath,
fibn data.FetchItemByNamer,
restoreFolderID string,
copyBuffer []byte,
restorePerms bool,
caches *restoreCaches,
itemPath path.Path,
itemData data.Stream,
@ -696,7 +683,7 @@ func restoreV6File(
itemID, itemInfo, err := restoreFile(
ctx,
restoreCfg,
rcc.RestoreConfig,
rh,
fibn,
meta.FileName,
@ -712,7 +699,7 @@ func restoreV6File(
// Mark it as success without processing .meta
// file if we are not restoring permissions
if !restorePerms {
if !rcc.RestoreConfig.IncludePermissions {
return itemInfo, nil
}

11
src/internal/m365/onedrive/restore_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/mock"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control"
@ -512,21 +513,25 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
ctr := count.New()
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
RestoreConfig: restoreCfg,
}
_, skip, err := restoreItem(
ctx,
rh,
restoreCfg,
rcc,
mock.FetchItemByName{
Item: &mock.Data{
Reader: mock.FileRespReadCloser(mock.DriveFileMetaData),
},
},
version.Backup,
dp,
"",
make([]byte, graph.CopyBufferSize),
caches,
false,
&mock.Data{
ID: uuid.NewString(),
Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData),

56
src/internal/m365/onedrive_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
@ -516,15 +517,16 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -763,15 +765,16 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -851,15 +854,16 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: false,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -1054,15 +1058,16 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -1247,15 +1252,16 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
})
control.DefaultOptions(),
restoreCfg)
})
}
}
@ -1362,16 +1368,16 @@ func testRestoreFolderNamedFolderRegression(
collectionsLatest: expected,
}
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreTestWithVersion(
t,
testData,
suite.Tenant(),
[]string{suite.ResourceOwner()},
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
)
control.DefaultOptions(),
restoreCfg)
})
}
}

40
src/internal/m365/restore.go
View File

@ -12,11 +12,11 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive"
"github.com/alcionai/corso/src/internal/m365/sharepoint"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/path"
)
// ConsumeRestoreCollections restores data from the specified collections
@ -24,10 +24,7 @@ import (
// SideEffect: status is updated at the completion of operation
func (ctrl *Controller) ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
sels selectors.Selector,
restoreCfg control.RestoreConfig,
opts control.Options,
rcc inject.RestoreConsumerConfig,
dcs []data.RestoreCollection,
errs *fault.Bus,
ctr *count.Bus,
@ -35,48 +32,45 @@ func (ctrl *Controller) ConsumeRestoreCollections(
ctx, end := diagnostics.Span(ctx, "m365:restore")
defer end()
ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()})
ctx = clues.Add(ctx, "restore_config", restoreCfg) // TODO(rkeepers): needs PII control
ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: rcc.Selector.PathService()})
ctx = clues.Add(ctx, "restore_config", rcc.RestoreConfig) // TODO(rkeepers): needs PII control
if len(dcs) == 0 {
return nil, clues.New("no collections to restore")
}
var (
status *support.ControllerOperationStatus
deets = &details.Builder{}
err error
service = rcc.Selector.PathService()
status *support.ControllerOperationStatus
deets = &details.Builder{}
err error
)
switch sels.Service {
case selectors.ServiceExchange:
status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, restoreCfg, dcs, deets, errs, ctr)
case selectors.ServiceOneDrive:
switch service {
case path.ExchangeService:
status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, rcc, dcs, deets, errs, ctr)
case path.OneDriveService:
status, err = onedrive.ConsumeRestoreCollections(
ctx,
onedrive.NewRestoreHandler(ctrl.AC),
backupVersion,
restoreCfg,
opts,
rcc,
ctrl.backupDriveIDNames,
dcs,
deets,
errs,
ctr)
case selectors.ServiceSharePoint:
case path.SharePointService:
status, err = sharepoint.ConsumeRestoreCollections(
ctx,
backupVersion,
rcc,
ctrl.AC,
restoreCfg,
opts,
ctrl.backupDriveIDNames,
dcs,
deets,
errs,
ctr)
default:
err = clues.Wrap(clues.New(sels.Service.String()), "service not supported")
err = clues.Wrap(clues.New(service.String()), "service not supported")
}
ctrl.incrementAwaitingMessages()

4
src/internal/m365/sharepoint/backup_test.go
View File

@ -107,7 +107,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
tenantID,
site,
nil,
control.Defaults())
control.DefaultOptions())
c.CollectionMap = collMap
@ -210,7 +210,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
ac,
mock.NewProvider(siteID, siteID),
&MockGraphService{},
control.Defaults(),
control.DefaultOptions(),
fault.New(true))
assert.NoError(t, err, clues.ToCore(err))
assert.NotEmpty(t, col)

2
src/internal/m365/sharepoint/collection_test.go
View File

@ -168,7 +168,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
suite.ac,
test.category,
nil,
control.Defaults())
control.DefaultOptions())
col.data <- test.getItem(t, test.itemName)
readItems := []data.Stream{}

17
src/internal/m365/sharepoint/restore.go
View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive"
betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
@ -31,10 +32,8 @@ import (
// ConsumeRestoreCollections will restore the specified data collections into OneDrive
func ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
rcc inject.RestoreConsumerConfig,
ac api.Client,
restoreCfg control.RestoreConfig,
opts control.Options,
backupDriveIDNames idname.Cacher,
dcs []data.RestoreCollection,
deets *details.Builder,
@ -70,7 +69,7 @@ func ConsumeRestoreCollections(
metrics support.CollectionMetrics
ictx = clues.Add(ctx,
"category", category,
"restore_location", restoreCfg.Location,
"restore_location", rcc.RestoreConfig.Location,
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"full_path", dc.FullPath())
)
@ -80,12 +79,10 @@ func ConsumeRestoreCollections(
metrics, err = onedrive.RestoreCollection(
ictx,
lrh,
restoreCfg,
backupVersion,
rcc,
dc,
caches,
deets,
opts.RestorePermissions,
control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem),
errs,
ctr)
@ -95,7 +92,7 @@ func ConsumeRestoreCollections(
ictx,
ac.Stable,
dc,
restoreCfg.Location,
rcc.RestoreConfig.Location,
deets,
errs)
@ -104,7 +101,7 @@ func ConsumeRestoreCollections(
ictx,
ac.Stable,
dc,
restoreCfg.Location,
rcc.RestoreConfig.Location,
deets,
errs)
@ -128,7 +125,7 @@ func ConsumeRestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
restoreCfg.Location)
rcc.RestoreConfig.Location)
return status, el.Failure()
}

4
src/internal/operations/backup_test.go
View File

@ -360,7 +360,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
op, err := NewBackupOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
kw,
sw,
ctrl,
@ -1241,7 +1241,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
sw = &store.Wrapper{}
ctrl = &mock.Controller{}
acct = tconfig.NewM365Account(suite.T())
opts = control.Defaults()
opts = control.DefaultOptions()
)
table := []struct {

2
src/internal/operations/help_test.go
View File

@ -27,7 +27,7 @@ func ControllerWithSelector(
ins idname.Cacher,
onFail func(),
) (*m365.Controller, selectors.Selector) {
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults())
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions())
if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil {
onFail()

18
src/internal/operations/inject/containers.go Normal file
View File

@ -0,0 +1,18 @@
package inject
import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/selectors"
)
// RestoreConsumerConfig container-of-things for holding options and
// configurations from various packages, which are widely used by all
// restore consumers independent of service or data category.
type RestoreConsumerConfig struct {
BackupVersion int
Options control.Options
ProtectedResource idname.Provider
RestoreConfig control.RestoreConfig
Selector selectors.Selector
}

5
src/internal/operations/inject/inject.go
View File

@ -36,10 +36,7 @@ type (
RestoreConsumer interface {
ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
selector selectors.Selector,
restoreCfg control.RestoreConfig,
opts control.Options,
rcc RestoreConsumerConfig,
dcs []data.RestoreCollection,
errs *fault.Bus,
ctr *count.Bus,

2
src/internal/operations/maintenance_test.go
View File

@ -54,7 +54,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() {
mo, err := NewMaintenanceOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
kw,
repository.Maintenance{
Type: repository.MetadataMaintenance,

4
src/internal/operations/operation_test.go
View File

@ -26,7 +26,7 @@ func TestOperationSuite(t *testing.T) {
func (suite *OperationSuite) TestNewOperation() {
t := suite.T()
op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil)
op := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, nil, nil)
assert.Greater(t, op.CreatedAt, time.Time{})
}
@ -46,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() {
}
for _, test := range table {
suite.Run(test.name, func() {
err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate()
err := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate()
test.errCheck(suite.T(), err, clues.ToCore(err))
})
}

35
src/internal/operations/restore.go
View File

@ -218,7 +218,7 @@ func (op *RestoreOperation) do(
return nil, clues.Wrap(err, "getting backup and details")
}
restoreProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector)
restoreToProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector)
if err != nil {
return nil, clues.Wrap(err, "getting destination protected resource")
}
@ -227,10 +227,10 @@ func (op *RestoreOperation) do(
ctx,
"backup_protected_resource_id", bup.Selector.ID(),
"backup_protected_resource_name", clues.Hide(bup.Selector.Name()),
"restore_protected_resource_id", restoreProtectedResource.ID(),
"restore_protected_resource_name", clues.Hide(restoreProtectedResource.Name()))
"restore_protected_resource_id", restoreToProtectedResource.ID(),
"restore_protected_resource_name", clues.Hide(restoreToProtectedResource.Name()))
observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreProtectedResource.Name()))
observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreToProtectedResource.Name()))
paths, err := formatDetailsForRestoration(
ctx,
@ -265,7 +265,12 @@ func (op *RestoreOperation) do(
kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository")
defer close(kopiaComplete)
dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors)
dcs, err := op.kopia.ProduceRestoreCollections(
ctx,
bup.SnapshotID,
paths,
opStats.bytesRead,
op.Errors)
if err != nil {
return nil, clues.Wrap(err, "producing collections to restore")
}
@ -282,6 +287,7 @@ func (op *RestoreOperation) do(
ctx,
op.rc,
bup.Version,
restoreToProtectedResource,
op.Selectors,
op.RestoreCfg,
op.Options,
@ -358,6 +364,7 @@ func consumeRestoreCollections(
ctx context.Context,
rc inject.RestoreConsumer,
backupVersion int,
toProtectedResoruce idname.Provider,
sel selectors.Selector,
restoreCfg control.RestoreConfig,
opts control.Options,
@ -371,15 +378,15 @@ func consumeRestoreCollections(
close(complete)
}()
deets, err := rc.ConsumeRestoreCollections(
ctx,
backupVersion,
sel,
restoreCfg,
opts,
dcs,
errs,
ctr)
rcc := inject.RestoreConsumerConfig{
BackupVersion: backupVersion,
Options: opts,
ProtectedResource: toProtectedResoruce,
RestoreConfig: restoreCfg,
Selector: sel,
}
deets, err := rc.ConsumeRestoreCollections(ctx, rcc, dcs, errs, ctr)
if err != nil {
return nil, clues.Wrap(err, "restoring collections")
}

12
src/internal/operations/restore_test.go
View File

@ -112,7 +112,7 @@ func (suite *RestoreOpUnitSuite) TestRestoreOperation_PersistResults() {
op, err := NewRestoreOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
kw,
sw,
ctrl,
@ -297,7 +297,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
sw = &store.Wrapper{}
ctrl = &mock.Controller{}
restoreCfg = testdata.DefaultRestoreConfig("")
opts = control.Defaults()
opts = control.DefaultOptions()
)
table := []struct {
@ -362,7 +362,7 @@ func setupExchangeBackup(
bo, err := NewBackupOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
kw,
sw,
ctrl,
@ -413,7 +413,7 @@ func setupSharePointBackup(
bo, err := NewBackupOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
kw,
sw,
ctrl,
@ -461,12 +461,12 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
suite.acct,
resource.Users,
rsel.PathService(),
control.Defaults())
control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
ro, err := NewRestoreOperation(
ctx,
control.Defaults(),
control.DefaultOptions(),
suite.kw,
suite.sw,
ctrl,

6
src/internal/operations/test/exchange_test.go
View File

@ -107,7 +107,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
var (
mb = evmock.NewBus()
sel = test.selector().Selector
opts = control.Defaults()
opts = control.DefaultOptions()
whatSet = deeTD.CategoryFromRepoRef
)
@ -260,7 +260,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
containers = []string{container1, container2, container3, containerRename}
sel = selectors.NewExchangeBackup([]string{suite.its.userID})
whatSet = deeTD.CategoryFromRepoRef
opts = control.Defaults()
opts = control.DefaultOptions()
)
opts.ToggleFeatures = toggles
@ -909,7 +909,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio
var (
mb = evmock.NewBus()
opts = control.Defaults()
opts = control.DefaultOptions()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup)

20
src/internal/operations/test/helper_test.go
View File

@ -25,6 +25,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/streamstore"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
@ -406,6 +407,7 @@ func generateContainerOfItems(
restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
restoreCfg.Location = destFldr
restoreCfg.IncludePermissions = true
dataColls := buildCollections(
t,
@ -414,15 +416,19 @@ func generateContainerOfItems(
restoreCfg,
collections)
opts := control.Defaults()
opts.RestorePermissions = true
opts := control.DefaultOptions()
rcc := inject.RestoreConsumerConfig{
BackupVersion: backupVersion,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
deets, err := ctrl.ConsumeRestoreCollections(
ctx,
backupVersion,
sel,
restoreCfg,
opts,
rcc,
dataColls,
fault.New(true),
count.New())
@ -541,7 +547,7 @@ func ControllerWithSelector(
ins idname.Cacher,
onFail func(*testing.T, context.Context),
) (*m365.Controller, selectors.Selector) {
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults())
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions())
if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil {
onFail(t, ctx)

14
src/internal/operations/test/onedrive_test.go
View File

@ -72,7 +72,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() {
osel = selectors.NewOneDriveBackup([]string{userID})
ws = deeTD.DriveIDFromRepoRef
svc = path.OneDriveService
opts = control.Defaults()
opts = control.DefaultOptions()
)
osel.Include(selTD.OneDriveBackupFolderScope(osel))
@ -166,7 +166,7 @@ func runDriveIncrementalTest(
var (
acct = tconfig.NewM365Account(t)
opts = control.Defaults()
opts = control.DefaultOptions()
mb = evmock.NewBus()
ws = deeTD.DriveIDFromRepoRef
@ -683,7 +683,7 @@ func runDriveIncrementalTest(
}
for _, test := range table {
suite.Run(test.name, func() {
cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults())
cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
bod.ctrl = cleanCtrl
@ -785,7 +785,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
var (
acct = tconfig.NewM365Account(t)
opts = control.Defaults()
opts = control.DefaultOptions()
mb = evmock.NewBus()
categories = map[path.CategoryType][]string{
@ -801,7 +801,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
acct,
resource.Users,
path.OneDriveService,
control.Defaults())
control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.userID)
@ -922,7 +922,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() {
osel = selectors.NewOneDriveBackup([]string{userID})
ws = deeTD.DriveIDFromRepoRef
svc = path.OneDriveService
opts = control.Defaults()
opts = control.DefaultOptions()
)
opts.ItemExtensionFactory = getTestExtensionFactories()
@ -1009,7 +1009,7 @@ func runDriveRestoreWithAdvancedOptions(
var (
mb = evmock.NewBus()
opts = control.Defaults()
opts = control.DefaultOptions()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup)

6
src/internal/operations/test/sharepoint_test.go
View File

@ -100,7 +100,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() {
var (
mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.siteID})
opts = control.Defaults()
opts = control.DefaultOptions()
)
sel.Include(selTD.SharePointBackupFolderScope(sel))
@ -129,7 +129,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() {
var (
mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.siteID})
opts = control.Defaults()
opts = control.DefaultOptions()
tenID = tconfig.M365TenantID(t)
svc = path.SharePointService
ws = deeTD.DriveIDFromRepoRef
@ -260,7 +260,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives
// run a backup
var (
mb = evmock.NewBus()
opts = control.Defaults()
opts = control.DefaultOptions()
graphClient = suite.its.ac.Stable.Client()
)

5
src/pkg/control/options.go
View File

@ -9,7 +9,6 @@ import (
type Options struct {
DisableMetrics bool `json:"disableMetrics"`
FailureHandling FailurePolicy `json:"failureHandling"`
RestorePermissions bool `json:"restorePermissions"`
SkipReduce bool `json:"skipReduce"`
ToggleFeatures Toggles `json:"toggleFeatures"`
Parallelism Parallelism `json:"parallelism"`
@ -35,8 +34,8 @@ const (
BestEffort FailurePolicy = "best-effort"
)
// Defaults provides an Options with the default values set.
func Defaults() Options {
// DefaultOptions provides an Options with the default values set.
func DefaultOptions() Options {
return Options{
FailureHandling: FailAfterRecovery,
ToggleFeatures: Toggles{},

4
src/pkg/control/restore.go
View File

@ -57,6 +57,10 @@ type RestoreConfig struct {
// up.
// Defaults to empty.
Drive string
// IncludePermissions toggles whether the restore will include the original
// folder- and item-level permissions.
IncludePermissions bool
}
func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig {

28
src/pkg/repository/repository_test.go
View File

@ -60,7 +60,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() {
st, err := test.storage()
assert.NoError(t, err, clues.ToCore(err))
_, err = Initialize(ctx, test.account, st, control.Defaults())
_, err = Initialize(ctx, test.account, st, control.DefaultOptions())
test.errCheck(t, err, clues.ToCore(err))
})
}
@ -94,7 +94,7 @@ func (suite *RepositoryUnitSuite) TestConnect() {
st, err := test.storage()
assert.NoError(t, err, clues.ToCore(err))
_, err = Connect(ctx, test.account, st, "not_found", control.Defaults())
_, err = Connect(ctx, test.account, st, "not_found", control.DefaultOptions())
test.errCheck(t, err, clues.ToCore(err))
})
}
@ -137,7 +137,7 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
defer flush()
st := test.storage(t)
r, err := Initialize(ctx, test.account, st, control.Defaults())
r, err := Initialize(ctx, test.account, st, control.DefaultOptions())
if err == nil {
defer func() {
err := r.Close(ctx)
@ -186,11 +186,11 @@ func (suite *RepositoryIntegrationSuite) TestConnect() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.Defaults())
repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
// now re-connect
_, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.Defaults())
_, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.DefaultOptions())
assert.NoError(t, err, clues.ToCore(err))
}
@ -203,7 +203,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, account.Account{}, st, control.Defaults())
r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
oldID := r.GetID()
@ -212,7 +212,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
require.NoError(t, err, clues.ToCore(err))
// now re-connect
r, err = Connect(ctx, account.Account{}, st, oldID, control.Defaults())
r, err = Connect(ctx, account.Account{}, st, oldID, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, oldID, r.GetID())
}
@ -228,7 +228,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults())
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
userID := tconfig.M365UserID(t)
@ -250,7 +250,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults())
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg)
@ -269,7 +269,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults())
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{})
@ -286,7 +286,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.Defaults())
repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err)
// now re-connect
@ -308,14 +308,14 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
{
name: "default options",
opts: func() control.Options {
return control.Defaults()
return control.DefaultOptions()
},
expectedLen: 0,
},
{
name: "options with an extension factory",
opts: func() control.Options {
o := control.Defaults()
o := control.DefaultOptions()
o.ItemExtensionFactory = append(
o.ItemExtensionFactory,
&extensions.MockItemExtensionFactory{})
@ -327,7 +327,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
{
name: "options with multiple extension factories",
opts: func() control.Options {
o := control.Defaults()
o := control.DefaultOptions()
f := []extensions.CreateItemExtensioner{
&extensions.MockItemExtensionFactory{},
&extensions.MockItemExtensionFactory{},