From 3613cb2aa03d87572bd42b2ec4a98af2381a3d17 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 26 Jul 2023 11:24:53 -0600 Subject: [PATCH] look up restore resource if specified (#3853) If the restore configuration specifies a protected resource as a restore target, use that as the destination for the restore. First step is to ensure the provided target can be retrieved and identified. --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #3562 #### Test Plan - [x] :zap: Unit test --- CHANGELOG.md | 4 + src/cli/backup/help_e2e_test.go | 2 +- src/cli/flags/restore_config.go | 7 +- src/cli/repo/s3_e2e_test.go | 2 +- src/cli/restore/exchange_test.go | 2 + src/cli/restore/onedrive_test.go | 6 + src/cli/restore/sharepoint_test.go | 8 + src/cli/utils/options.go | 3 +- src/cli/utils/restore_config.go | 15 +- src/cli/utils/restore_config_test.go | 44 +++- src/cli/utils/testdata/flags.go | 1 + src/cmd/factory/impl/common.go | 32 ++- src/cmd/factory/impl/exchange.go | 6 +- src/internal/common/idname/idname.go | 4 + src/internal/common/idname/idname_test.go | 60 +++++ src/internal/events/events_test.go | 2 +- src/internal/m365/backup.go | 13 +- src/internal/m365/backup_test.go | 16 +- src/internal/m365/controller.go | 19 +- src/internal/m365/controller_test.go | 132 +++++----- src/internal/m365/exchange/backup_test.go | 16 +- src/internal/m365/exchange/collection_test.go | 2 +- .../m365/exchange/container_resolver_test.go | 2 +- src/internal/m365/exchange/helper_test.go | 2 +- .../exchange/mail_container_cache_test.go | 2 +- src/internal/m365/exchange/restore.go | 40 ++- src/internal/m365/exchange/restore_test.go | 2 +- src/internal/m365/helper_test.go | 4 +- src/internal/m365/mock/connector.go | 19 +- src/internal/m365/onedrive/collection_test.go | 2 +- .../m365/onedrive/item_collector_test.go | 2 +- src/internal/m365/onedrive/restore.go | 205 ++++----------- src/internal/m365/onedrive/restore_caches.go | 116 +++++++++ src/internal/m365/onedrive/restore_test.go | 11 +- src/internal/m365/onedrive/service_test.go | 2 +- src/internal/m365/onedrive/url_cache_test.go | 2 +- src/internal/m365/onedrive_test.go | 87 +++---- src/internal/m365/restore.go | 59 +++-- src/internal/m365/sharepoint/backup_test.go | 6 +- .../m365/sharepoint/collection_test.go | 4 +- src/internal/m365/sharepoint/restore.go | 28 +- src/internal/m365/stub/stub.go | 3 +- src/internal/operations/backup_test.go | 6 +- src/internal/operations/export_test.go | 2 +- src/internal/operations/help_test.go | 4 +- src/internal/operations/inject/containers.go | 18 ++ src/internal/operations/inject/inject.go | 25 +- src/internal/operations/maintenance_test.go | 2 +- src/internal/operations/operation_test.go | 4 +- src/internal/operations/restore.go | 64 ++++- src/internal/operations/restore_test.go | 87 ++++++- src/internal/operations/test/exchange_test.go | 241 +++++++++++++++++- src/internal/operations/test/helper_test.go | 113 ++++---- src/internal/operations/test/onedrive_test.go | 204 +++++++++++++-- .../operations/test/sharepoint_test.go | 57 +++-- src/internal/tester/tconfig/config.go | 14 +- .../tester/tconfig/protected_resources.go | 11 + src/pkg/control/options.go | 5 +- src/pkg/control/restore.go | 13 +- src/pkg/repository/repository.go | 2 +- src/pkg/repository/repository_test.go | 28 +- src/pkg/services/m365/api/helper_test.go | 2 +- src/pkg/services/m365/api/sites_test.go | 17 +- src/pkg/services/m365/m365.go | 2 +- website/docs/setup/restore-options.md | 41 ++- 65 files changed, 1365 insertions(+), 591 deletions(-) create mode 100644 src/internal/common/idname/idname_test.go create mode 100644 src/internal/m365/onedrive/restore_caches.go create mode 100644 src/internal/operations/inject/containers.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 296b01a85..8c7123822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +### Added +- Restore commands now accept an optional resource override with the `--to-resource` flag. This allows restores to recreate backup data within different mailboxes, sites, and users. + ### Fixed - SharePoint document libraries deleted after the last backup can now be restored. +- Restore requires the protected resource to have access to the service being restored. ## [v0.11.1] (beta) - 2023-07-20 diff --git a/src/cli/backup/help_e2e_test.go b/src/cli/backup/help_e2e_test.go index d99d3769a..b7100d333 100644 --- a/src/cli/backup/help_e2e_test.go +++ b/src/cli/backup/help_e2e_test.go @@ -47,7 +47,7 @@ func prepM365Test( vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) ctx = config.SetViper(ctx, vpr) - repo, err := repository.Initialize(ctx, acct, st, control.Defaults()) + repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) return acct, st, repo, vpr, recorder, cfgFP diff --git a/src/cli/flags/restore_config.go b/src/cli/flags/restore_config.go index a2b8c3a86..4a1868d01 100644 --- a/src/cli/flags/restore_config.go +++ b/src/cli/flags/restore_config.go @@ -9,11 +9,13 @@ import ( const ( CollisionsFN = "collisions" DestinationFN = "destination" + ToResourceFN = "to-resource" ) var ( CollisionsFV string DestinationFV string + ToResourceFV string ) // AddRestoreConfigFlags adds the restore config flag set. @@ -25,5 +27,8 @@ func AddRestoreConfigFlags(cmd *cobra.Command) { "Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace)) fs.StringVar( &DestinationFV, DestinationFN, "", - "Overrides the destination where items get restored; '/' places items into their original location") + "Overrides the folder where items get restored; '/' places items into their original location") + fs.StringVar( + &ToResourceFV, ToResourceFN, "", + "Overrides the protected resource (mailbox, site, user, etc) where data gets restored") } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index 540d08836..4c3af6e5f 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -200,7 +200,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { ctx = config.SetViper(ctx, vpr) // init the repo first - _, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults()) + _, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_test.go b/src/cli/restore/exchange_test.go index 0dd022a81..a257fff7a 100644 --- a/src/cli/restore/exchange_test.go +++ b/src/cli/restore/exchange_test.go @@ -84,6 +84,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -125,6 +126,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) diff --git a/src/cli/restore/onedrive_test.go b/src/cli/restore/onedrive_test.go index a21f1191d..8a9fc7a94 100644 --- a/src/cli/restore/onedrive_test.go +++ b/src/cli/restore/onedrive_test.go @@ -70,6 +70,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -80,6 +81,9 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -99,6 +103,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -109,6 +114,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index b4547077f..6a8de8e57 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -75,6 +75,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -85,6 +86,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -111,6 +115,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -121,6 +126,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + + // bool flags + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 7f9176a90..932c56b6b 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -8,7 +8,7 @@ import ( // Control produces the control options based on the user's flags. func Control() control.Options { - opt := control.Defaults() + opt := control.DefaultOptions() if flags.FailFastFV { opt.FailureHandling = control.FailFast @@ -21,7 +21,6 @@ func Control() control.Options { opt.DeltaPageSize = dps opt.DisableMetrics = flags.NoStatsFV - opt.RestorePermissions = flags.RestorePermissionsFV opt.SkipReduce = flags.SkipReduceFV opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV diff --git a/src/cli/utils/restore_config.go b/src/cli/utils/restore_config.go index fa036e3f9..6be54f1ab 100644 --- a/src/cli/utils/restore_config.go +++ b/src/cli/utils/restore_config.go @@ -18,16 +18,20 @@ type RestoreCfgOpts struct { // DTTMFormat is the timestamp format appended // to the default folder name. Defaults to // dttm.HumanReadable. - DTTMFormat dttm.TimeFormat + DTTMFormat dttm.TimeFormat + ProtectedResource string + RestorePermissions bool Populated flags.PopulatedFlags } func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts { return RestoreCfgOpts{ - Collisions: flags.CollisionsFV, - Destination: flags.DestinationFV, - DTTMFormat: dttm.HumanReadable, + Collisions: flags.CollisionsFV, + Destination: flags.DestinationFV, + DTTMFormat: dttm.HumanReadable, + ProtectedResource: flags.ToResourceFV, + RestorePermissions: flags.RestorePermissionsFV, // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate @@ -67,6 +71,9 @@ func MakeRestoreConfig( restoreCfg.Location = opts.Destination } + restoreCfg.ProtectedResource = opts.ProtectedResource + restoreCfg.IncludePermissions = opts.RestorePermissions + Infof(ctx, "Restoring to folder %s", restoreCfg.Location) return restoreCfg diff --git a/src/cli/utils/restore_config_test.go b/src/cli/utils/restore_config_test.go index 1324c9571..c3509e360 100644 --- a/src/cli/utils/restore_config_test.go +++ b/src/cli/utils/restore_config_test.go @@ -68,18 +68,18 @@ func (suite *RestoreCfgUnitSuite) TestValidateRestoreConfigFlags() { } func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { - rco := &RestoreCfgOpts{ - Collisions: "collisions", - Destination: "destination", - } - table := []struct { name string + rco *RestoreCfgOpts populated flags.PopulatedFlags expect control.RestoreConfig }{ { - name: "not populated", + name: "not populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{}, expect: control.RestoreConfig{ OnCollision: control.Skip, @@ -88,6 +88,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "collision populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, }, @@ -98,6 +102,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "destination populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.DestinationFN: {}, }, @@ -108,6 +116,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "both populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, flags.DestinationFN: {}, @@ -117,6 +129,23 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { Location: "destination", }, }, + { + name: "with restore permissions", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + RestorePermissions: true, + }, + populated: flags.PopulatedFlags{ + flags.CollisionsFN: {}, + flags.DestinationFN: {}, + }, + expect: control.RestoreConfig{ + OnCollision: control.CollisionPolicy("collisions"), + Location: "destination", + IncludePermissions: true, + }, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -125,12 +154,13 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { ctx, flush := tester.NewContext(t) defer flush() - opts := *rco + opts := *test.rco opts.Populated = test.populated result := MakeRestoreConfig(ctx, opts) assert.Equal(t, test.expect.OnCollision, result.OnCollision) assert.Contains(t, result.Location, test.expect.Location) + assert.Equal(t, test.expect.IncludePermissions, result.IncludePermissions) }) } } diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index d29198072..85131a1e0 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -46,6 +46,7 @@ var ( Collisions = "collisions" Destination = "destination" + ToResource = "toResource" RestorePermissions = true DeltaPageSize = "deltaPageSize" diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 04967dc2a..837063b40 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -21,12 +21,12 @@ import ( odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" @@ -104,7 +104,15 @@ func generateAndRestoreItems( print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, dataColls, errs, ctr) } // ------------------------------------------------------------------------------------------ @@ -144,7 +152,7 @@ func getControllerAndVerifyResourceOwner( return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") } - id, _, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, resourceOwner, nil) + id, _, err := ctrl.PopulateProtectedResourceIDAndName(ctx, resourceOwner, nil) if err != nil { return nil, account.Account{}, nil, clues.Wrap(err, "verifying user") } @@ -407,10 +415,8 @@ func generateAndRestoreDriveItems( // input, // version.Backup) - opts := control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - } + opts := control.DefaultOptions() + restoreCfg.IncludePermissions = true config := m365Stub.ConfigInfo{ Opts: opts, @@ -418,7 +424,7 @@ func generateAndRestoreDriveItems( Service: service, Tenant: tenantID, ResourceOwners: []string{resourceOwner}, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } _, _, collections, _, err := m365Stub.GetCollectionsAndExpected( @@ -429,5 +435,13 @@ func generateAndRestoreDriveItems( return nil, err } - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, collections, errs, ctr) } diff --git a/src/cmd/factory/impl/exchange.go b/src/cmd/factory/impl/exchange.go index eb923969b..b7ad4840d 100644 --- a/src/cmd/factory/impl/exchange.go +++ b/src/cmd/factory/impl/exchange.go @@ -72,7 +72,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error { subject, body, body, now, now, now, now) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -121,7 +121,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error exchMock.NoAttachments, exchMock.NoCancelledOccurrences, exchMock.NoExceptionOccurrences) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -172,7 +172,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error { "123-456-7890", ) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index 0367d954b..e2a48fca3 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -28,6 +28,10 @@ type is struct { name string } +func NewProvider(id, name string) *is { + return &is{id, name} +} + func (is is) ID() string { return is.id } func (is is) Name() string { return is.name } diff --git a/src/internal/common/idname/idname_test.go b/src/internal/common/idname/idname_test.go new file mode 100644 index 000000000..229177d61 --- /dev/null +++ b/src/internal/common/idname/idname_test.go @@ -0,0 +1,60 @@ +package idname + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type IDNameUnitSuite struct { + tester.Suite +} + +func TestIDNameUnitSuite(t *testing.T) { + suite.Run(t, &IDNameUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *IDNameUnitSuite) TestAdd() { + table := []struct { + name string + inID string + inName string + searchID string + searchName string + }{ + { + name: "basic", + inID: "foo", + inName: "bar", + searchID: "foo", + searchName: "bar", + }, + { + name: "change casing", + inID: "FNORDS", + inName: "SMARF", + searchID: "fnords", + searchName: "smarf", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cache := NewCache(nil) + + cache.Add(test.inID, test.inName) + + id, found := cache.IDOf(test.searchName) + assert.True(t, found) + assert.Equal(t, test.inID, id) + + name, found := cache.NameOf(test.searchID) + assert.True(t, found) + assert.Equal(t, test.inName, name) + }) + } +} diff --git a/src/internal/events/events_test.go b/src/internal/events/events_test.go index 7cc47f607..f8b1d6beb 100644 --- a/src/internal/events/events_test.go +++ b/src/internal/events/events_test.go @@ -52,7 +52,7 @@ func (suite *EventsIntegrationSuite) TestNewBus() { ) require.NoError(t, err, clues.ToCore(err)) - b, err := events.NewBus(ctx, s, a.ID(), control.Defaults()) + b, err := events.NewBus(ctx, s, a.ID(), control.DefaultOptions()) require.NotEmpty(t, b) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 50514e4ad..a5a89f134 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -2,7 +2,6 @@ package m365 import ( "context" - "strings" "github.com/alcionai/clues" @@ -44,7 +43,7 @@ func (ctrl *Controller) ProduceBackupCollections( ctx, end := diagnostics.Span( ctx, "m365:produceBackupCollections", - diagnostics.Index("service", sels.Service.String())) + diagnostics.Index("service", sels.PathService().String())) defer end() ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) @@ -61,8 +60,8 @@ func (ctrl *Controller) ProduceBackupCollections( serviceEnabled, canMakeDeltaQueries, err := checkServiceEnabled( ctx, ctrl.AC.Users(), - path.ServiceType(sels.Service), - sels.DiscreteOwner) + sels.PathService(), + owner.ID()) if err != nil { return nil, nil, false, err } @@ -194,10 +193,8 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error { ids = siteIDs } - resourceOwner := strings.ToLower(sels.DiscreteOwner) - - if !filters.Equal(ids).Compare(resourceOwner) { - return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_resource_owner", sels.DiscreteOwner) + if !filters.Contains(ids).Compare(sels.ID()) { + return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner) } return nil diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index b80bd4ddc..87c9c766d 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -57,7 +57,7 @@ func (suite *DataCollectionIntgSuite) SetupSuite() { suite.tenantID = creds.AzureTenantID - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -120,7 +120,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { sel := test.getSelector(t) uidn := inMock.NewProvider(sel.ID(), sel.Name()) - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( @@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() test.getSelector(t), nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.Error(t, err, clues.ToCore(err)) assert.False(t, canUsePreviousBackup, "can use previous backup") @@ -296,7 +296,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { nil, ctrl.credentials, ctrl, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -367,7 +367,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -381,7 +381,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -414,7 +414,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -428,7 +428,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 051ab5fb2..174148a76 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -84,10 +84,11 @@ func NewController( AC: ac, IDNameLookup: idname.NewCache(nil), - credentials: creds, - ownerLookup: rCli, - tenant: acct.ID(), - wg: &sync.WaitGroup{}, + credentials: creds, + ownerLookup: rCli, + tenant: acct.ID(), + wg: &sync.WaitGroup{}, + backupDriveIDNames: idname.NewCache(nil), } return &ctrl, nil @@ -150,10 +151,6 @@ func (ctrl *Controller) incrementAwaitingMessages() { } func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { - if ctrl.backupDriveIDNames == nil { - ctrl.backupDriveIDNames = idname.NewCache(map[string]string{}) - } - if dii.SharePoint != nil { ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) } @@ -249,15 +246,15 @@ func (r resourceClient) getOwnerIDAndNameFrom( return id, name, nil } -// PopulateOwnerIDAndNamesFrom takes the provided owner identifier and produces +// PopulateProtectedResourceIDAndName takes the provided owner identifier and produces // the owner's name and ID from that value. Returns an error if the owner is // not recognized by the current tenant. // -// The id-name swapper is optional. Some processes will look up all owners in +// The id-name cacher is optional. Some processes will look up all owners in // the tenant before reaching this step. In that case, the data gets handed // down for this func to consume instead of performing further queries. The // data gets stored inside the controller instance for later re-use. -func (ctrl *Controller) PopulateOwnerIDAndNamesFrom( +func (ctrl *Controller) PopulateProtectedResourceIDAndName( ctx context.Context, owner string, // input value, can be either id or name ins idname.Cacher, diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 487603b39..f4ff3c032 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,15 +12,18 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" @@ -223,7 +226,7 @@ func (suite *ControllerUnitSuite) TestPopulateOwnerIDAndNamesFrom() { ctrl := &Controller{ownerLookup: test.rc} - rID, rName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, test.owner, test.ins) + rID, rName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, test.owner, test.ins) test.expectErr(t, err, clues.ToCore(err)) assert.Equal(t, test.expectID, rID, "id") assert.Equal(t, test.expectName, rName, "name") @@ -385,20 +388,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { } ) + restoreCfg.IncludePermissions = true + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, []data.RestoreCollection{&dataMock.Collection{}}, fault.New(true), count.New()) - assert.Error(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) + assert.Error(t, err, graph.ErrServiceNotEnabled, clues.ToCore(err)) + assert.Nil(t, deets) status := suite.ctrl.Wait() assert.Equal(t, 0, status.Objects) @@ -408,6 +415,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { func (suite *ControllerIntegrationSuite) TestEmptyCollections() { restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + table := []struct { name string col []data.RestoreCollection @@ -464,15 +473,17 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { ctx, flush := tester.NewContext(t) defer flush() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: test.sel, + RestoreConfig: restoreCfg, + Selector: test.sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - test.sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, test.col, fault.New(true), count.New()) @@ -503,12 +514,18 @@ func runRestore( restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService) restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: sci.RestoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - backupVersion, - restoreSel, - sci.RestoreCfg, - sci.Opts, + rcc, collections, fault.New(true), count.New()) @@ -610,6 +627,7 @@ func runRestoreBackupTest( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -620,7 +638,7 @@ func runRestoreBackupTest( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( @@ -655,6 +673,7 @@ func runRestoreTestWithVersion( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -665,7 +684,7 @@ func runRestoreTestWithVersion( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -692,7 +711,7 @@ func runRestoreBackupTestVersions( tenant string, resourceOwners []string, opts control.Options, - crc control.RestoreConfig, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -703,7 +722,7 @@ func runRestoreBackupTestVersions( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: crc, + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -737,7 +756,7 @@ func runRestoreBackupTestVersions( test.collectionsLatest) } -func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { +func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_core() { bodyText := "This email has some text. However, all the text is on the same line." subjectText := "Test message for restore" @@ -996,10 +1015,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)) }) } } @@ -1080,6 +1097,8 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { for i, collection := range test.collections { // Get a restoreCfg per collection so they're independent. restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + expectedDests = append(expectedDests, destAndCats{ resourceOwner: suite.user, dest: restoreCfg.Location, @@ -1112,15 +1131,18 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { ) restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: restoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - version.Backup, - restoreSel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, collections, fault.New(true), count.New()) @@ -1152,10 +1174,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -1164,10 +1183,13 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { t.Log("Backup enumeration complete") + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + ci := stub.ConfigInfo{ - Opts: control.Options{RestorePermissions: true}, + Opts: control.DefaultOptions(), // Alright to be empty, needed for OneDrive. - RestoreCfg: control.RestoreConfig{}, + RestoreCfg: restoreCfg, } // Pull the data prior to waiting for the status as otherwise it will @@ -1205,16 +1227,16 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen }, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreBackupTest( suite.T(), test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) } func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { @@ -1233,8 +1255,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { sel.Include( sel.ContactFolders([]string{selectors.NoneTgt}), sel.EventCalendars([]string{selectors.NoneTgt}), - sel.MailFolders([]string{selectors.NoneTgt}), - ) + sel.MailFolders([]string{selectors.NoneTgt})) return sel.Selector }, @@ -1297,23 +1318,20 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { start = time.Now() ) - id, name, err := backupCtrl.PopulateOwnerIDAndNamesFrom(ctx, backupSel.DiscreteOwner, nil) + id, name, err := backupCtrl.PopulateProtectedResourceIDAndName(ctx, backupSel.DiscreteOwner, nil) require.NoError(t, err, clues.ToCore(err)) backupSel.SetDiscreteOwnerIDName(id, name) dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( ctx, - inMock.NewProvider(id, name), + idname.NewProvider(id, name), backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) - require.NoError(t, err) + require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. assert.True(t, excludes.Empty()) diff --git a/src/internal/m365/exchange/backup_test.go b/src/internal/m365/exchange/backup_test.go index 34735eda8..fa4b87d9d 100644 --- a/src/internal/m365/exchange/backup_test.go +++ b/src/internal/m365/exchange/backup_test.go @@ -414,7 +414,7 @@ func (suite *BackupIntgSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.tenantID = creds.AzureTenantID @@ -466,7 +466,7 @@ func (suite *BackupIntgSuite) TestMailFetch() { ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries collections, err := createCollections( @@ -554,7 +554,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -587,7 +587,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, dps, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -633,7 +633,7 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() { inMock.NewProvider(suite.user, suite.user), sel.Scopes()[0], DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -709,7 +709,7 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -834,7 +834,7 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1995,7 +1995,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter getter := test.getter diff --git a/src/internal/m365/exchange/collection_test.go b/src/internal/m365/exchange/collection_test.go index 2c023d703..7c5a4adab 100644 --- a/src/internal/m365/exchange/collection_test.go +++ b/src/internal/m365/exchange/collection_test.go @@ -178,7 +178,7 @@ func (suite *CollectionSuite) TestNewCollection_state() { test.curr, test.prev, test.loc, 0, &mockItemer{}, nil, - control.Defaults(), + control.DefaultOptions(), false) assert.Equal(t, test.expect, c.State(), "collection state") assert.Equal(t, test.curr, c.fullPath, "full path") diff --git a/src/internal/m365/exchange/container_resolver_test.go b/src/internal/m365/exchange/container_resolver_test.go index 54cd23c67..b2ff30830 100644 --- a/src/internal/m365/exchange/container_resolver_test.go +++ b/src/internal/m365/exchange/container_resolver_test.go @@ -699,7 +699,7 @@ func (suite *ContainerResolverSuite) SetupSuite() { } func (suite *ContainerResolverSuite) TestPopulate() { - ac, err := api.NewClient(suite.credentials, control.Defaults()) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(suite.T(), err, clues.ToCore(err)) eventFunc := func(t *testing.T) graph.ContainerResolver { diff --git a/src/internal/m365/exchange/helper_test.go b/src/internal/m365/exchange/helper_test.go index f8cadd227..9b1583b9c 100644 --- a/src/internal/m365/exchange/helper_test.go +++ b/src/internal/m365/exchange/helper_test.go @@ -31,7 +31,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.creds = creds - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.userID = tconfig.GetM365UserID(ctx) diff --git a/src/internal/m365/exchange/mail_container_cache_test.go b/src/internal/m365/exchange/mail_container_cache_test.go index 64f453092..de0694749 100644 --- a/src/internal/m365/exchange/mail_container_cache_test.go +++ b/src/internal/m365/exchange/mail_container_cache_test.go @@ -84,7 +84,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { ctx, flush := tester.NewContext(t) defer flush() - ac, err := api.NewClient(suite.credentials, control.Defaults()) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) acm := ac.Mail() diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/exchange/restore.go index 5a5dbfcbc..7871f68d4 100644 --- a/src/internal/m365/exchange/restore.go +++ b/src/internal/m365/exchange/restore.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -28,7 +29,7 @@ import ( func ConsumeRestoreCollections( ctx context.Context, ac api.Client, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, @@ -39,16 +40,13 @@ func ConsumeRestoreCollections( } var ( - userID = dcs[0].FullPath().ResourceOwner() + resourceID = rcc.ProtectedResource.ID() directoryCache = make(map[path.CategoryType]graph.ContainerResolver) handlers = restoreHandlers(ac) metrics support.CollectionMetrics el = errs.Local() ) - // FIXME: should be user name - ctx = clues.Add(ctx, "resource_owner", clues.Hide(userID)) - for _, dc := range dcs { if el.Failure() != nil { break @@ -69,7 +67,7 @@ func ConsumeRestoreCollections( } if directoryCache[category] == nil { - gcr := handler.newContainerCache(userID) + gcr := handler.newContainerCache(resourceID) if err := gcr.Populate(ctx, errs, handler.defaultRootContainer()); err != nil { return nil, clues.Wrap(err, "populating container cache") } @@ -80,8 +78,8 @@ func ConsumeRestoreCollections( containerID, gcc, err := createDestination( ictx, handler, - handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()), - userID, + handler.formatRestoreDestination(rcc.RestoreConfig.Location, dc.FullPath()), + resourceID, directoryCache[category], errs) if err != nil { @@ -92,7 +90,7 @@ func ConsumeRestoreCollections( directoryCache[category] = gcc ictx = clues.Add(ictx, "restore_destination_id", containerID) - collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, userID, containerID) + collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, resourceID, containerID) if err != nil { el.AddRecoverable(ctx, clues.Wrap(err, "building item collision cache")) continue @@ -102,10 +100,10 @@ func ConsumeRestoreCollections( ictx, handler, dc, - userID, + resourceID, containerID, collisionKeyToItemID, - restoreCfg.OnCollision, + rcc.RestoreConfig.OnCollision, deets, errs, ctr) @@ -126,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), metrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -136,7 +134,7 @@ func restoreCollection( ctx context.Context, ir itemRestorer, dc data.RestoreCollection, - userID, destinationID string, + resourceID, destinationID string, collisionKeyToItemID map[string]string, collisionPolicy control.CollisionPolicy, deets *details.Builder, @@ -187,7 +185,7 @@ func restoreCollection( info, err := ir.restore( ictx, body, - userID, + resourceID, destinationID, collisionKeyToItemID, collisionPolicy, @@ -240,7 +238,7 @@ func createDestination( ctx context.Context, ca containerAPI, destination *path.Builder, - userID string, + resourceID string, gcr graph.ContainerResolver, errs *fault.Bus, ) (string, graph.ContainerResolver, error) { @@ -264,7 +262,7 @@ func createDestination( ca, cache, restoreLoc, - userID, + resourceID, containerParentID, container, errs) @@ -285,7 +283,7 @@ func getOrPopulateContainer( ca containerAPI, gcr graph.ContainerResolver, restoreLoc *path.Builder, - userID, containerParentID, containerName string, + resourceID, containerParentID, containerName string, errs *fault.Bus, ) (string, error) { cached, ok := gcr.LocationInCache(restoreLoc.String()) @@ -293,7 +291,7 @@ func getOrPopulateContainer( return cached, nil } - c, err := ca.CreateContainer(ctx, userID, containerParentID, containerName) + c, err := ca.CreateContainer(ctx, resourceID, containerParentID, containerName) // 409 handling case: // attempt to fetch the container by name and add that result to the cache. @@ -301,7 +299,7 @@ func getOrPopulateContainer( // sometimes the backend will create the folder despite the 5xx response, // leaving our local containerResolver with inconsistent state. if graph.IsErrFolderExists(err) { - cc, e := ca.GetContainerByName(ctx, userID, containerParentID, containerName) + cc, e := ca.GetContainerByName(ctx, resourceID, containerParentID, containerName) if e != nil { err = clues.Stack(err, e) } else { @@ -327,7 +325,7 @@ func uploadAttachments( ctx context.Context, ap attachmentPoster, as []models.Attachmentable, - userID, destinationID, itemID string, + resourceID, destinationID, itemID string, errs *fault.Bus, ) error { el := errs.Local() @@ -340,7 +338,7 @@ func uploadAttachments( err := uploadAttachment( ctx, ap, - userID, + resourceID, destinationID, itemID, a) diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/exchange/restore_test.go index 42e61a915..a30d56dd0 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/exchange/restore_test.go @@ -44,7 +44,7 @@ func (suite *RestoreIntgSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) suite.credentials = m365 - suite.ac, err = api.NewClient(m365, control.Defaults()) + suite.ac, err = api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index 25f4bb18c..a3f1d7d5e 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -796,8 +796,8 @@ func compareDriveItem( assert.Equal(t, expectedMeta.FileName, itemMeta.FileName) } - if !mci.Opts.RestorePermissions { - assert.Equal(t, 0, len(itemMeta.Permissions)) + if !mci.RestoreCfg.IncludePermissions { + assert.Empty(t, itemMeta.Permissions, "no permissions should be included in restore") return true } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 2c2ece635..64bc8deda 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -27,6 +27,10 @@ type Controller struct { Err error Stats data.CollectionStats + + ProtectedResourceID string + ProtectedResourceName string + ProtectedResourceErr error } func (ctrl Controller) ProduceBackupCollections( @@ -60,10 +64,7 @@ func (ctrl Controller) Wait() *data.CollectionStats { func (ctrl Controller) ConsumeRestoreCollections( _ context.Context, - _ int, - _ selectors.Selector, - _ control.RestoreConfig, - _ control.Options, + _ inject.RestoreConsumerConfig, _ []data.RestoreCollection, _ *fault.Bus, _ *count.Bus, @@ -84,3 +85,13 @@ func (ctrl Controller) ExportRestoreCollections( ) ([]export.Collection, error) { return nil, ctrl.Err } + +func (ctrl Controller) PopulateProtectedResourceIDAndName( + ctx context.Context, + protectedResource string, // input value, can be either id or name + ins idname.Cacher, +) (string, string, error) { + return ctrl.ProtectedResourceID, + ctrl.ProtectedResourceName, + ctrl.ProtectedResourceErr +} diff --git a/src/internal/m365/onedrive/collection_test.go b/src/internal/m365/onedrive/collection_test.go index bcd4da4b6..3c30cac22 100644 --- a/src/internal/m365/onedrive/collection_test.go +++ b/src/internal/m365/onedrive/collection_test.go @@ -945,7 +945,7 @@ func (suite *CollectionUnitTestSuite) TestItemExtensions() { nil, } - opts := control.Defaults() + opts := control.DefaultOptions() opts.ItemExtensionFactory = append( opts.ItemExtensionFactory, test.factories...) diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index fc2cccd62..6078517c7 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -313,7 +313,7 @@ func (suite *OneDriveIntgSuite) SetupSuite() { suite.creds = creds - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index f951419be..7c920e6ff 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -23,6 +23,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" @@ -38,114 +39,11 @@ const ( maxUploadRetries = 3 ) -type driveInfo struct { - id string - name string - rootFolderID string -} - -type restoreCaches struct { - BackupDriveIDName idname.Cacher - collisionKeyToItemID map[string]api.DriveItemIDType - DriveIDToDriveInfo map[string]driveInfo - DriveNameToDriveInfo map[string]driveInfo - Folders *folderCache - OldLinkShareIDToNewID map[string]string - OldPermIDToNewID map[string]string - ParentDirToMeta map[string]metadata.Metadata - - pool sync.Pool -} - -func (rc *restoreCaches) AddDrive( - ctx context.Context, - md models.Driveable, - grf GetRootFolderer, -) error { - di := driveInfo{ - id: ptr.Val(md.GetId()), - name: ptr.Val(md.GetName()), - } - - ctx = clues.Add(ctx, "drive_info", di) - - root, err := grf.GetRootFolder(ctx, di.id) - if err != nil { - return clues.Wrap(err, "getting drive root id") - } - - di.rootFolderID = ptr.Val(root.GetId()) - - rc.DriveIDToDriveInfo[di.id] = di - rc.DriveNameToDriveInfo[di.name] = di - - return nil -} - -// Populate looks up drive items available to the protectedResource -// and adds their info to the caches. -func (rc *restoreCaches) Populate( - ctx context.Context, - gdparf GetDrivePagerAndRootFolderer, - protectedResourceID string, -) error { - drives, err := api.GetAllDrives( - ctx, - gdparf.NewDrivePager(protectedResourceID, nil), - true, - maxDrivesRetries) - if err != nil { - return clues.Wrap(err, "getting drives") - } - - for _, md := range drives { - if err := rc.AddDrive(ctx, md, gdparf); err != nil { - return clues.Wrap(err, "caching drive") - } - } - - return nil -} - -type GetDrivePagerAndRootFolderer interface { - GetRootFolderer - NewDrivePagerer -} - -func NewRestoreCaches( - backupDriveIDNames idname.Cacher, -) *restoreCaches { - // avoid nil panics - if backupDriveIDNames == nil { - backupDriveIDNames = idname.NewCache(nil) - } - - return &restoreCaches{ - BackupDriveIDName: backupDriveIDNames, - collisionKeyToItemID: map[string]api.DriveItemIDType{}, - DriveIDToDriveInfo: map[string]driveInfo{}, - DriveNameToDriveInfo: map[string]driveInfo{}, - Folders: NewFolderCache(), - OldLinkShareIDToNewID: map[string]string{}, - OldPermIDToNewID: map[string]string{}, - ParentDirToMeta: map[string]metadata.Metadata{}, - // Buffer pool for uploads - pool: sync.Pool{ - New: func() any { - b := make([]byte, graph.CopyBufferSize) - return &b - }, - }, - } -} - // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, rh RestoreHandler, - backupVersion int, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, @@ -153,16 +51,15 @@ func ConsumeRestoreCollections( ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - el = errs.Local() - caches = NewRestoreCaches(backupDriveIDNames) - protectedResourceID = dcs[0].FullPath().ResourceOwner() - fallbackDriveName = restoreCfg.Location + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = NewRestoreCaches(backupDriveIDNames) + fallbackDriveName = rcc.RestoreConfig.Location ) - ctx = clues.Add(ctx, "backup_version", backupVersion) + ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion) - err := caches.Populate(ctx, rh, protectedResourceID) + err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID()) if err != nil { return nil, clues.Wrap(err, "initializing restore caches") } @@ -183,19 +80,16 @@ func ConsumeRestoreCollections( ictx = clues.Add( ctx, "category", dc.FullPath().Category(), - "resource_owner", clues.Hide(protectedResourceID), "full_path", dc.FullPath()) ) metrics, err = RestoreCollection( ictx, rh, - restoreCfg, - backupVersion, + rcc, dc, caches, deets, - opts.RestorePermissions, fallbackDriveName, errs, ctr.Local()) @@ -215,7 +109,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -228,26 +122,23 @@ func ConsumeRestoreCollections( func RestoreCollection( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, - backupVersion int, + rcc inject.RestoreConsumerConfig, dc data.RestoreCollection, caches *restoreCaches, deets *details.Builder, - restorePerms bool, // TODD: move into restoreConfig fallbackDriveName string, errs *fault.Bus, ctr *count.Bus, ) (support.CollectionMetrics, error) { var ( - metrics = support.CollectionMetrics{} - directory = dc.FullPath() - protectedResourceID = directory.ResourceOwner() - el = errs.Local() - metricsObjects int64 - metricsBytes int64 - metricsSuccess int64 - wg sync.WaitGroup - complete bool + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + el = errs.Local() + metricsObjects int64 + metricsBytes int64 + metricsSuccess int64 + wg sync.WaitGroup + complete bool ) ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory)) @@ -263,7 +154,7 @@ func RestoreCollection( rh, caches, drivePath, - protectedResourceID, + rcc.ProtectedResource.ID(), fallbackDriveName) if err != nil { return metrics, clues.Wrap(err, "ensuring drive exists") @@ -281,8 +172,8 @@ func RestoreCollection( // the drive into which this folder gets restored is tracked separately in drivePath. restoreDir := &path.Builder{} - if len(restoreCfg.Location) > 0 { - restoreDir = restoreDir.Append(restoreCfg.Location) + if len(rcc.RestoreConfig.Location) > 0 { + restoreDir = restoreDir.Append(rcc.RestoreConfig.Location) } restoreDir = restoreDir.Append(drivePath.Folders...) @@ -301,8 +192,8 @@ func RestoreCollection( drivePath, dc, caches, - backupVersion, - restorePerms) + rcc.BackupVersion, + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx) } @@ -316,7 +207,7 @@ func RestoreCollection( dc.FullPath(), colMeta, caches, - restorePerms) + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "creating folders for restore") } @@ -390,14 +281,12 @@ func RestoreCollection( itemInfo, skipped, err := restoreItem( ictx, rh, - restoreCfg, + rcc, dc, - backupVersion, drivePath, restoreFolderID, copyBuffer, caches, - restorePerms, itemData, itemPath, ctr) @@ -440,14 +329,12 @@ func RestoreCollection( func restoreItem( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, fibn data.FetchItemByNamer, - backupVersion int, drivePath *path.DrivePath, restoreFolderID string, copyBuffer []byte, caches *restoreCaches, - restorePerms bool, itemData data.Stream, itemPath path.Path, ctr *count.Bus, @@ -455,11 +342,11 @@ func restoreItem( itemUUID := itemData.UUID() ctx = clues.Add(ctx, "item_id", itemUUID) - if backupVersion < version.OneDrive1DataAndMetaFiles { + if rcc.BackupVersion < version.OneDrive1DataAndMetaFiles { itemInfo, err := restoreV0File( ctx, rh, - restoreCfg, + rcc.RestoreConfig, drivePath, fibn, restoreFolderID, @@ -468,7 +355,7 @@ func restoreItem( itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -491,7 +378,7 @@ func restoreItem( // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // permission for child folders here. Later versions can request // permissions inline when processing the collection. - if !restorePerms || backupVersion >= version.OneDrive4DirIncludesPermissions { + if !rcc.RestoreConfig.IncludePermissions || rcc.BackupVersion >= version.OneDrive4DirIncludesPermissions { return details.ItemInfo{}, true, nil } @@ -511,22 +398,21 @@ func restoreItem( // only items with DataFileSuffix from this point on - if backupVersion < version.OneDrive6NameInMeta { + if rcc.BackupVersion < version.OneDrive6NameInMeta { itemInfo, err := restoreV1File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -541,18 +427,17 @@ func restoreItem( itemInfo, err := restoreV6File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -596,12 +481,11 @@ func restoreV0File( func restoreV1File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -611,7 +495,7 @@ func restoreV1File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, trimmedName, @@ -627,7 +511,7 @@ func restoreV1File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } @@ -657,12 +541,11 @@ func restoreV1File( func restoreV6File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -696,7 +579,7 @@ func restoreV6File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, meta.FileName, @@ -712,10 +595,12 @@ func restoreV6File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } + fmt.Printf("\n-----\nrestorev6 %+v\n-----\n", rcc.RestoreConfig.IncludePermissions) + err = RestorePermissions( ctx, rh, @@ -765,6 +650,8 @@ func CreateRestoreFolders( return id, nil } + fmt.Printf("\n-----\ncreatefolders %+v\n-----\n", restorePerms) + err = RestorePermissions( ctx, rh, diff --git a/src/internal/m365/onedrive/restore_caches.go b/src/internal/m365/onedrive/restore_caches.go new file mode 100644 index 000000000..6951a8bfe --- /dev/null +++ b/src/internal/m365/onedrive/restore_caches.go @@ -0,0 +1,116 @@ +package onedrive + +import ( + "context" + "sync" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type driveInfo struct { + id string + name string + rootFolderID string +} + +type restoreCaches struct { + BackupDriveIDName idname.Cacher + collisionKeyToItemID map[string]api.DriveItemIDType + DriveIDToDriveInfo map[string]driveInfo + DriveNameToDriveInfo map[string]driveInfo + Folders *folderCache + OldLinkShareIDToNewID map[string]string + OldPermIDToNewID map[string]string + ParentDirToMeta map[string]metadata.Metadata + + pool sync.Pool +} + +func (rc *restoreCaches) AddDrive( + ctx context.Context, + md models.Driveable, + grf GetRootFolderer, +) error { + di := driveInfo{ + id: ptr.Val(md.GetId()), + name: ptr.Val(md.GetName()), + } + + ctx = clues.Add(ctx, "drive_info", di) + + root, err := grf.GetRootFolder(ctx, di.id) + if err != nil { + return clues.Wrap(err, "getting drive root id") + } + + di.rootFolderID = ptr.Val(root.GetId()) + + rc.DriveIDToDriveInfo[di.id] = di + rc.DriveNameToDriveInfo[di.name] = di + + return nil +} + +// Populate looks up drive items available to the protectedResource +// and adds their info to the caches. +func (rc *restoreCaches) Populate( + ctx context.Context, + gdparf GetDrivePagerAndRootFolderer, + protectedResourceID string, +) error { + drives, err := api.GetAllDrives( + ctx, + gdparf.NewDrivePager(protectedResourceID, nil), + true, + maxDrivesRetries) + if err != nil { + return clues.Wrap(err, "getting drives") + } + + for _, md := range drives { + if err := rc.AddDrive(ctx, md, gdparf); err != nil { + return clues.Wrap(err, "caching drive") + } + } + + return nil +} + +type GetDrivePagerAndRootFolderer interface { + GetRootFolderer + NewDrivePagerer +} + +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, + collisionKeyToItemID: map[string]api.DriveItemIDType{}, + DriveIDToDriveInfo: map[string]driveInfo{}, + DriveNameToDriveInfo: map[string]driveInfo{}, + Folders: NewFolderCache(), + OldLinkShareIDToNewID: map[string]string{}, + OldPermIDToNewID: map[string]string{}, + ParentDirToMeta: map[string]metadata.Metadata{}, + // Buffer pool for uploads + pool: sync.Pool{ + New: func() any { + b := make([]byte, graph.CopyBufferSize) + return &b + }, + }, + } +} diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index dbb7317c9..301a1b01e 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" @@ -512,21 +513,25 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { ctr := count.New() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + RestoreConfig: restoreCfg, + } + _, skip, err := restoreItem( ctx, rh, - restoreCfg, + rcc, mock.FetchItemByName{ Item: &mock.Data{ Reader: mock.FileRespReadCloser(mock.DriveFileMetaData), }, }, - version.Backup, dp, "", make([]byte, graph.CopyBufferSize), caches, - false, &mock.Data{ ID: uuid.NewString(), Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), diff --git a/src/internal/m365/onedrive/service_test.go b/src/internal/m365/onedrive/service_test.go index a39a65a76..a2766b8ee 100644 --- a/src/internal/m365/onedrive/service_test.go +++ b/src/internal/m365/onedrive/service_test.go @@ -21,7 +21,7 @@ type oneDriveService struct { } func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) { - ac, err := api.NewClient(credentials, control.Defaults()) + ac, err := api.NewClient(credentials, control.DefaultOptions()) if err != nil { return nil, err } diff --git a/src/internal/m365/onedrive/url_cache_test.go b/src/internal/m365/onedrive/url_cache_test.go index 7946da840..bf4f25350 100644 --- a/src/internal/m365/onedrive/url_cache_test.go +++ b/src/internal/m365/onedrive/url_cache_test.go @@ -53,7 +53,7 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user) diff --git a/src/internal/m365/onedrive_test.go b/src/internal/m365/onedrive_test.go index eade30c9d..ba81a477a 100644 --- a/src/internal/m365/onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" @@ -223,9 +224,9 @@ func (suite *SharePointIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *SharePointIntegrationSuite) TestPermissionsBackupAndNoRestore() { +func (suite *SharePointIntegrationSuite) TestRestoreNoPermissionsAndBackup() { suite.T().Skip("Temporarily disabled due to CI issues") - testPermissionsBackupAndNoRestore(suite, version.Backup) + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *SharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -290,8 +291,8 @@ func (suite *OneDriveIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *OneDriveIntegrationSuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.Backup) +func (suite *OneDriveIntegrationSuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *OneDriveIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -354,8 +355,8 @@ func (suite *OneDriveNightlySuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.OneDrive1DataAndMetaFiles) } -func (suite *OneDriveNightlySuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.OneDrive1DataAndMetaFiles) +func (suite *OneDriveNightlySuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.OneDrive1DataAndMetaFiles) } func (suite *OneDriveNightlySuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -517,19 +518,17 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -768,24 +767,22 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } -func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { +func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) { t := suite.T() ctx, flush := tester.NewContext(t) @@ -860,19 +857,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_backup_no_restore") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_backup_no_restore") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = false + + fmt.Printf("\n-----\nrcfg %+v\n-----\n", restoreCfg.IncludePermissions) runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1067,19 +1064,17 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1264,19 +1259,17 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion collectionsLatest: expected, } - rc := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") - rc.OnCollision = control.Replace + restoreCfg := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - rc) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1383,16 +1376,16 @@ func testRestoreFolderNamedFolderRegression( collectionsLatest: expected, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreTestWithVersion( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) }) } } diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 5d58fdb26..de9e0bb13 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -12,11 +12,11 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/sharepoint" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/path" ) // ConsumeRestoreCollections restores data from the specified collections @@ -24,10 +24,7 @@ import ( // SideEffect: status is updated at the completion of operation func (ctrl *Controller) ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - sels selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, @@ -35,48 +32,64 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctx, end := diagnostics.Span(ctx, "m365:restore") defer end() - ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) - ctx = clues.Add(ctx, "restore_config", restoreCfg) + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: rcc.Selector.PathService()}) + ctx = clues.Add(ctx, "restore_config", rcc.RestoreConfig) if len(dcs) == 0 { return nil, clues.New("no data collections to restore") } + serviceEnabled, _, err := checkServiceEnabled( + ctx, + ctrl.AC.Users(), + rcc.Selector.PathService(), + rcc.ProtectedResource.ID()) + if err != nil { + return nil, err + } + + if !serviceEnabled { + return nil, clues.Stack(graph.ErrServiceNotEnabled).WithClues(ctx) + } + var ( - status *support.ControllerOperationStatus - deets = &details.Builder{} - err error + service = rcc.Selector.PathService() + status *support.ControllerOperationStatus + deets = &details.Builder{} ) - switch sels.Service { - case selectors.ServiceExchange: - status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, restoreCfg, dcs, deets, errs, ctr) - case selectors.ServiceOneDrive: + switch service { + case path.ExchangeService: + status, err = exchange.ConsumeRestoreCollections( + ctx, + ctrl.AC, + rcc, + dcs, + deets, + errs, + ctr) + case path.OneDriveService: status, err = onedrive.ConsumeRestoreCollections( ctx, onedrive.NewRestoreHandler(ctrl.AC), - backupVersion, - restoreCfg, - opts, + rcc, ctrl.backupDriveIDNames, dcs, deets, errs, ctr) - case selectors.ServiceSharePoint: + case path.SharePointService: status, err = sharepoint.ConsumeRestoreCollections( ctx, - backupVersion, + rcc, ctrl.AC, - restoreCfg, - opts, ctrl.backupDriveIDNames, dcs, deets, errs, ctr) default: - err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + err = clues.Wrap(clues.New(service.String()), "service not supported") } ctrl.incrementAwaitingMessages() diff --git a/src/internal/m365/sharepoint/backup_test.go b/src/internal/m365/sharepoint/backup_test.go index 973a55670..348b15dfd 100644 --- a/src/internal/m365/sharepoint/backup_test.go +++ b/src/internal/m365/sharepoint/backup_test.go @@ -107,7 +107,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { tenantID, site, nil, - control.Defaults()) + control.DefaultOptions()) c.CollectionMap = collMap @@ -201,7 +201,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds, control.Defaults()) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) col, err := collectPages( @@ -210,7 +210,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { ac, mock.NewProvider(siteID, siteID), &MockGraphService{}, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) assert.NotEmpty(t, col) diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index 42f9ad9a1..0462a5c8e 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -43,7 +43,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() { suite.creds = m365 - ac, err := api.NewClient(m365, control.Defaults()) + ac, err := api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.ac = ac @@ -168,7 +168,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { suite.ac, test.category, nil, - control.Defaults()) + control.DefaultOptions()) col.data <- test.getItem(t, test.itemName) readItems := []data.Stream{} diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index c38b82e08..bb894f5ea 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -31,10 +32,8 @@ import ( // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, - backupVersion int, + rcc inject.RestoreConsumerConfig, ac api.Client, - restoreCfg control.RestoreConfig, - opts control.Options, backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, @@ -42,14 +41,13 @@ func ConsumeRestoreCollections( ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - lrh = libraryRestoreHandler{ac} - protectedResourceID = dcs[0].FullPath().ResourceOwner() - restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches(backupDriveIDNames) - el = errs.Local() + lrh = libraryRestoreHandler{ac} + restoreMetrics support.CollectionMetrics + caches = onedrive.NewRestoreCaches(backupDriveIDNames) + el = errs.Local() ) - err := caches.Populate(ctx, lrh, protectedResourceID) + err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) if err != nil { return nil, clues.Wrap(err, "initializing restore caches") } @@ -70,7 +68,7 @@ func ConsumeRestoreCollections( metrics support.CollectionMetrics ictx = clues.Add(ctx, "category", category, - "restore_location", restoreCfg.Location, + "restore_location", clues.Hide(rcc.RestoreConfig.Location), "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "full_path", dc.FullPath()) ) @@ -80,12 +78,10 @@ func ConsumeRestoreCollections( metrics, err = onedrive.RestoreCollection( ictx, lrh, - restoreCfg, - backupVersion, + rcc, dc, caches, deets, - opts.RestorePermissions, control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), errs, ctr) @@ -95,7 +91,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -104,7 +100,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -128,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index 601e57722..da3340f60 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -68,8 +68,7 @@ func GetCollectionsAndExpected( owner, config.RestoreCfg, testCollections, - backupVersion, - ) + backupVersion) if err != nil { return totalItems, totalKopiaItems, collections, expectedData, err } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 3aaeae45c..a2783e92e 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -362,7 +362,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() { op, err := NewBackupOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -1137,7 +1137,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds, control.Defaults()) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -1147,7 +1147,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} acct = tconfig.NewM365Account(suite.T()) - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go index c81114da9..10dec2ab1 100644 --- a/src/internal/operations/export_test.go +++ b/src/internal/operations/export_test.go @@ -99,7 +99,7 @@ func (suite *ExportOpSuite) TestExportOperation_PersistResults() { op, err := NewExportOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, diff --git a/src/internal/operations/help_test.go b/src/internal/operations/help_test.go index 0951572ba..bd8509a1a 100644 --- a/src/internal/operations/help_test.go +++ b/src/internal/operations/help_test.go @@ -27,7 +27,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() @@ -36,7 +36,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() diff --git a/src/internal/operations/inject/containers.go b/src/internal/operations/inject/containers.go new file mode 100644 index 000000000..f44bb7e66 --- /dev/null +++ b/src/internal/operations/inject/containers.go @@ -0,0 +1,18 @@ +package inject + +import ( + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// RestoreConsumerConfig is a container-of-things for holding options and +// configurations from various packages, which are widely used by all +// restore consumers independent of service or data category. +type RestoreConsumerConfig struct { + BackupVersion int + Options control.Options + ProtectedResource idname.Provider + RestoreConfig control.RestoreConfig + Selector selectors.Selector +} diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 5f6da9230..0a5e8581c 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -37,10 +37,7 @@ type ( RestoreConsumer interface { ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - selector selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, @@ -49,6 +46,7 @@ type ( Wait() *data.CollectionStats CacheItemInfoer + PopulateProtectedResourceIDAndNamer } CacheItemInfoer interface { @@ -76,6 +74,25 @@ type ( CacheItemInfoer } + PopulateProtectedResourceIDAndNamer interface { + // PopulateProtectedResourceIDAndName takes the provided owner identifier and produces + // the owner's name and ID from that value. Returns an error if the owner is + // not recognized by the current tenant. + // + // The id-name cacher should be optional. Some processes will look up all owners in + // the tenant before reaching this step. In that case, the data gets handed + // down for this func to consume instead of performing further queries. The + // data gets stored inside the controller instance for later re-use. + PopulateProtectedResourceIDAndName( + ctx context.Context, + owner string, // input value, can be either id or name + ins idname.Cacher, + ) ( + id, name string, + err error, + ) + } + RepoMaintenancer interface { RepoMaintenance(ctx context.Context, opts repository.Maintenance) error } diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index a625af482..4ec6a3ee9 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -54,7 +54,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { mo, err := NewMaintenanceOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, repository.Maintenance{ Type: repository.MetadataMaintenance, diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index 4cbbe2a0c..b615a492e 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -26,7 +26,7 @@ func TestOperationSuite(t *testing.T) { func (suite *OperationSuite) TestNewOperation() { t := suite.T() - op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) + op := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, nil, nil) assert.Greater(t, op.CreatedAt, time.Time{}) } @@ -46,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() { } for _, test := range table { suite.Run(test.name, func() { - err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() + err := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() test.errCheck(suite.T(), err, clues.ToCore(err)) }) } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 0f853a853..141300f6a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/events" @@ -172,7 +173,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De logger.CtxErr(ctx, err).Error("running restore") if errors.Is(err, kopia.ErrNoRestorePath) { - op.Errors.Fail(clues.New("empty backup or unknown path provided")) + op.Errors.Fail(clues.Wrap(err, "empty backup or unknown path provided")) } op.Errors.Fail(clues.Wrap(err, "running restore")) @@ -217,7 +218,19 @@ func (op *RestoreOperation) do( return nil, clues.Wrap(err, "getting backup and details") } - observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) + restoreToProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector) + if err != nil { + return nil, clues.Wrap(err, "getting destination protected resource") + } + + ctx = clues.Add( + ctx, + "backup_protected_resource_id", bup.Selector.ID(), + "backup_protected_resource_name", clues.Hide(bup.Selector.Name()), + "restore_protected_resource_id", restoreToProtectedResource.ID(), + "restore_protected_resource_name", clues.Hide(restoreToProtectedResource.Name())) + + observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreToProtectedResource.Name())) paths, err := formatDetailsForRestoration( ctx, @@ -232,8 +245,6 @@ func (op *RestoreOperation) do( ctx = clues.Add( ctx, - "resource_owner_id", bup.Selector.ID(), - "resource_owner_name", clues.Hide(bup.Selector.Name()), "details_entries", len(deets.Entries), "details_paths", len(paths), "backup_snapshot_id", bup.SnapshotID, @@ -254,7 +265,12 @@ func (op *RestoreOperation) do( kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") defer close(kopiaComplete) - dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) + dcs, err := op.kopia.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + opStats.bytesRead, + op.Errors) if err != nil { return nil, clues.Wrap(err, "producing collections to restore") } @@ -271,6 +287,7 @@ func (op *RestoreOperation) do( ctx, op.rc, bup.Version, + restoreToProtectedResource, op.Selectors, op.RestoreCfg, op.Options, @@ -321,6 +338,24 @@ func (op *RestoreOperation) persistResults( return op.Errors.Failure() } +func chooseRestoreResource( + ctx context.Context, + pprian inject.PopulateProtectedResourceIDAndNamer, + restoreCfg control.RestoreConfig, + orig idname.Provider, +) (idname.Provider, error) { + if len(restoreCfg.ProtectedResource) == 0 { + return orig, nil + } + + id, name, err := pprian.PopulateProtectedResourceIDAndName( + ctx, + restoreCfg.ProtectedResource, + nil) + + return idname.NewProvider(id, name), clues.Stack(err).OrNil() +} + // --------------------------------------------------------------------------- // Restorer funcs // --------------------------------------------------------------------------- @@ -329,6 +364,7 @@ func consumeRestoreCollections( ctx context.Context, rc inject.RestoreConsumer, backupVersion int, + toProtectedResource idname.Provider, sel selectors.Selector, restoreCfg control.RestoreConfig, opts control.Options, @@ -342,15 +378,15 @@ func consumeRestoreCollections( close(complete) }() - deets, err := rc.ConsumeRestoreCollections( - ctx, - backupVersion, - sel, - restoreCfg, - opts, - dcs, - errs, - ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: toProtectedResource, + RestoreConfig: restoreCfg, + Selector: sel, + } + + deets, err := rc.ConsumeRestoreCollections(ctx, rcc, dcs, errs, ctr) if err != nil { return nil, clues.Wrap(err, "restoring collections") } diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5a314aaf4..5b124ee64 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -37,15 +39,15 @@ import ( // unit // --------------------------------------------------------------------------- -type RestoreOpSuite struct { +type RestoreOpUnitSuite struct { tester.Suite } -func TestRestoreOpSuite(t *testing.T) { - suite.Run(t, &RestoreOpSuite{Suite: tester.NewUnitSuite(t)}) +func TestRestoreOpUnitSuite(t *testing.T) { + suite.Run(t, &RestoreOpUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { +func (suite *RestoreOpUnitSuite) TestRestoreOperation_PersistResults() { var ( kw = &kopia.Wrapper{} sw = &store.Wrapper{} @@ -107,7 +109,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { op, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -135,6 +137,75 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { } } +func (suite *RestoreOpUnitSuite) TestChooseRestoreResource() { + var ( + id = "id" + name = "name" + cfgWithPR = control.DefaultRestoreConfig(dttm.HumanReadable) + ) + + cfgWithPR.ProtectedResource = "cfgid" + + table := []struct { + name string + cfg control.RestoreConfig + ctrl *mock.Controller + orig idname.Provider + expectErr assert.ErrorAssertionFunc + expectProvider assert.ValueAssertionFunc + expectID string + expectName string + }{ + { + name: "use original", + cfg: control.DefaultRestoreConfig(dttm.HumanReadable), + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: "oid", + expectName: "oname", + }, + { + name: "look up resource with iface", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: id, + expectName: name, + }, + { + name: "error looking up protected resource", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceErr: assert.AnError, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.Error, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + result, err := chooseRestoreResource(ctx, test.ctrl, test.cfg, test.orig) + test.expectErr(t, err, clues.ToCore(err)) + require.NotNil(t, result) + assert.Equal(t, test.expectID, result.ID()) + assert.Equal(t, test.expectName, result.Name()) + }) + } +} + // --------------------------------------------------------------------------- // integration // --------------------------------------------------------------------------- @@ -216,7 +287,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} restoreCfg = testdata.DefaultRestoreConfig("") - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { @@ -275,12 +346,12 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { suite.acct, resource.Users, rsel.PathService(), - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) ro, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), suite.kw, suite.sw, ctrl, diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index e33cdd0ae..ddb16216b 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -67,9 +67,9 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Mail", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID return sel }, @@ -79,7 +79,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Contacts", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())) return sel }, @@ -89,7 +89,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Calendar Events", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) return sel }, @@ -107,7 +107,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { var ( mb = evmock.NewBus() sel = test.selector().Selector - opts = control.Defaults() + opts = control.DefaultOptions() whatSet = deeTD.CategoryFromRepoRef ) @@ -258,9 +258,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr // later on during the tests. Putting their identifiers into the selector // at this point is harmless. containers = []string{container1, container2, container3, containerRename} - sel = selectors.NewExchangeBackup([]string{suite.its.userID}) + sel = selectors.NewExchangeBackup([]string{suite.its.user.ID}) whatSet = deeTD.CategoryFromRepoRef - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ToggleFeatures = toggles @@ -278,7 +278,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds, control.Defaults()) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -295,7 +295,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr mailDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.MessageWith( - suite.its.userID, suite.its.userID, suite.its.userID, + suite.its.user.ID, suite.its.user.ID, suite.its.user.ID, subject, body, body, now, now, now, now) } @@ -312,7 +312,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr eventDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.EventWith( - suite.its.userID, subject, body, body, + suite.its.user.ID, subject, body, body, exchMock.NoOriginalStartDate, now, now, exchMock.NoRecurrence, exchMock.NoAttendees, exchMock.NoAttachments, exchMock.NoCancelledOccurrences, @@ -578,7 +578,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr service, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, - creds.AzureTenantID, suite.its.userID, "", container3, + creds.AzureTenantID, suite.its.user.ID, "", container3, 2, version.Backup, gen.dbf) @@ -897,7 +897,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // a backup is required to run restores - baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) baseSel.Include( // events cannot be run, for the same reason as incremental backups: the user needs // to have their account recycled. @@ -905,11 +905,11 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - baseSel.DiscreteOwner = suite.its.userID + baseSel.DiscreteOwner = suite.its.user.ID var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) @@ -1272,3 +1272,216 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio assert.Len(t, result, 0, "no items should have been added as copies") }) } + +func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtectedResource() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) + baseSel.Include( + // events cannot be run, for the same reason as incremental backups: the user needs + // to have their account recycled. + // base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + baseSel.DiscreteOwner = suite.its.user.ID + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + rsel, err := baseSel.ToExchangeRestore() + require.NoError(t, err, clues.ToCore(err)) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_restore_to_user") + sel = rsel.Selector + userID = suite.its.user.ID + secondaryUserID = suite.its.secondaryUser.ID + uid = userID + acCont = suite.its.ac.Contacts() + acMail = suite.its.ac.Mail() + // acEvts = suite.its.ac.Events() + firstCtr = count.New() + ) + + restoreCfg.OnCollision = control.Copy + mb = evmock.NewBus() + + // first restore to the current user + + ro1, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + firstCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro1, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + + var ( + userItemIDs = map[path.CategoryType]map[string]struct{}{} + userCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat := path.ContactsCategory + userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // then restore to the secondary user + + uid = secondaryUserID + mb = evmock.NewBus() + secondCtr := count.New() + restoreCfg.ProtectedResource = uid + + ro2, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + secondCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro2, mb, false) + + var ( + secondaryItemIDs = map[path.CategoryType]map[string]struct{}{} + secondaryCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat = path.ContactsCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // compare restore results + for _, cat := range []path.CategoryType{path.ContactsCategory, path.EmailCategory, path.EventsCategory} { + assert.Equal(t, len(userItemIDs[cat]), len(secondaryItemIDs[cat])) + assert.ElementsMatch(t, maps.Keys(userCollisionKeys[cat]), maps.Keys(secondaryCollisionKeys[cat])) + } +} + +type GetItemsKeysAndContainerByNameer interface { + GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, + ) (map[string]struct{}, error) + GetContainerByName( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]string, error) +} + +func getCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + gikacbn GetItemsKeysAndContainerByNameer, + userID, parentContainerID string, + containerNames ...string, +) (map[string]struct{}, map[string]string) { + var ( + c graph.Container + err error + cID string + ) + + for _, cn := range containerNames { + pcid := parentContainerID + + if len(cID) != 0 { + pcid = cID + } + + c, err = gikacbn.GetContainerByName(ctx, userID, pcid, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := gikacbn.GetItemIDsInContainer(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := gikacbn.GetItemsInContainerByCollisionKey(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index f1da62cbe..f1bf65261 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -25,6 +25,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" @@ -400,6 +401,7 @@ func generateContainerOfItems( restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg.Location = destFldr + restoreCfg.IncludePermissions = true dataColls := buildCollections( t, @@ -408,15 +410,19 @@ func generateContainerOfItems( restoreCfg, collections) - opts := control.Defaults() - opts.RestorePermissions = true + opts := control.DefaultOptions() + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } deets, err := ctrl.ConsumeRestoreCollections( ctx, - backupVersion, - sel, - restoreCfg, - opts, + rcc, dataColls, fault.New(true), count.New()) @@ -535,7 +541,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(*testing.T, context.Context), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -544,7 +550,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -562,15 +568,19 @@ func ControllerWithSelector( // Suite Setup // --------------------------------------------------------------------------- +type ids struct { + ID string + DriveID string + DriveRootFolderID string +} + type intgTesterSetup struct { - ac api.Client - gockAC api.Client - userID string - userDriveID string - userDriveRootFolderID string - siteID string - siteDriveID string - siteDriveRootFolderID string + ac api.Client + gockAC api.Client + user ids + secondaryUser ids + site ids + secondarySite ids } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -585,43 +595,58 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) require.NoError(t, err, clues.ToCore(err)) - // user drive - - its.userID = tconfig.M365UserID(t) - - userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveID = ptr.Val(userDrive.GetId()) - - userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId()) - - its.siteID = tconfig.M365SiteID(t) - - // site - - siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveID = ptr.Val(siteDrive.GetId()) - - siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) + its.user = userIDs(t, tconfig.M365UserID(t), its.ac) + its.secondaryUser = userIDs(t, tconfig.SecondaryM365UserID(t), its.ac) + its.site = siteIDs(t, tconfig.M365SiteID(t), its.ac) + its.secondarySite = siteIDs(t, tconfig.SecondaryM365SiteID(t), its.ac) return its } +func userIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Users().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + +func siteIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Sites().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + func getTestExtensionFactories() []extensions.CreateItemExtensioner { return []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index 41aab489d..b5057be31 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -72,7 +72,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) osel.Include(selTD.OneDriveBackupFolderScope(osel)) @@ -106,7 +106,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { } func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { - sel := selectors.NewOneDriveRestore([]string{suite.its.userID}) + sel := selectors.NewOneDriveRestore([]string{suite.its.user.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.Folders(cs, selectors.PrefixMatch())) @@ -117,10 +117,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.userID) + d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.user.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). - With("user", suite.its.userID) + With("user", suite.its.user.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -137,8 +137,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { runDriveIncrementalTest( suite, - suite.its.userID, - suite.its.userID, + suite.its.user.ID, + suite.its.user.ID, resource.Users, path.OneDriveService, path.FilesCategory, @@ -166,7 +166,7 @@ func runDriveIncrementalTest( var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() ws = deeTD.DriveIDFromRepoRef @@ -683,7 +683,7 @@ func runDriveIncrementalTest( } for _, test := range table { suite.Run(test.name, func() { - cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) + cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) bod.ctrl = cleanCtrl @@ -785,7 +785,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() categories = map[path.CategoryType][]string{ @@ -801,10 +801,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { acct, resource.Users, path.OneDriveService, - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.userID) + userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID) require.NoError(t, err, clues.ToCore(err)) uid := ptr.Val(userable.GetId()) @@ -922,7 +922,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ItemExtensionFactory = getTestExtensionFactories() @@ -982,17 +982,17 @@ func (suite *OneDriveRestoreIntgSuite) SetupSuite() { } func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { - sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) sel.Include(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.userDriveID, - suite.its.userDriveRootFolderID) + suite.its.user.DriveID, + suite.its.user.DriveRootFolderID) } func runDriveRestoreWithAdvancedOptions( @@ -1009,7 +1009,7 @@ func runDriveRestoreWithAdvancedOptions( var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) @@ -1250,3 +1250,173 @@ func runDriveRestoreWithAdvancedOptions( assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") }) } + +func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveAlternateProtectedResource() { + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.user.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.user, + suite.its.secondaryUser) +} + +func runDriveRestoreToAlternateProtectedResource( + t *testing.T, + suite tester.Suite, + ac api.Client, + sel selectors.Selector, // owner should match 'from', both Restore and Backup types work. + from, to ids, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("drive_restore_to_resource") + fromCollisionKeys map[string]api.DriveItemIDType + fromItemIDs map[string]api.DriveItemIDType + acd = ac.Drives() + ) + + // first restore to the 'from' resource + + suite.Run("restore original resource", func() { + mb = evmock.NewBus() + fromCtr := count.New() + driveID := from.DriveID + rootFolderID := from.DriveRootFolderID + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + fromCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + fromItemIDs, fromCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // then restore to the 'to' resource + var ( + toCollisionKeys map[string]api.DriveItemIDType + toItemIDs map[string]api.DriveItemIDType + ) + + suite.Run("restore to alternate resource", func() { + mb = evmock.NewBus() + toCtr := count.New() + driveID := to.DriveID + rootFolderID := to.DriveRootFolderID + restoreCfg.ProtectedResource = to.ID + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + toCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + toItemIDs, toCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // compare restore results + assert.Equal(t, len(fromItemIDs), len(toItemIDs)) + assert.ElementsMatch(t, maps.Keys(fromCollisionKeys), maps.Keys(toCollisionKeys)) +} + +type GetItemsKeysAndFolderByNameer interface { + GetItemIDsInContainer( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) + GetFolderByName( + ctx context.Context, + driveID, parentFolderID, folderName string, + ) (models.DriveItemable, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) +} + +func getDriveCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + gikafbn GetItemsKeysAndFolderByNameer, + driveID, parentContainerID string, + containerNames ...string, +) (map[string]api.DriveItemIDType, map[string]api.DriveItemIDType) { + var ( + c models.DriveItemable + err error + cID string + ) + + for _, cn := range containerNames { + pcid := parentContainerID + + if len(cID) != 0 { + pcid = cID + } + + c, err = gikafbn.GetFolderByName(ctx, driveID, pcid, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := gikafbn.GetItemIDsInContainer(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := gikafbn.GetItemsInContainerByCollisionKey(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys +} diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index ad2e5d79a..08cc4cb1c 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -49,7 +49,7 @@ func (suite *SharePointBackupIntgSuite) SetupSuite() { } func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { - sel := selectors.NewSharePointRestore([]string{suite.its.siteID}) + sel := selectors.NewSharePointRestore([]string{suite.its.site.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.LibraryFolders(cs, selectors.PrefixMatch())) @@ -60,10 +60,10 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.siteID) + d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.site.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). - With("site", suite.its.siteID) + With("site", suite.its.site.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -80,8 +80,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { runDriveIncrementalTest( suite, - suite.its.siteID, - suite.its.userID, + suite.its.site.ID, + suite.its.user.ID, resource.Sites, path.SharePointService, path.LibrariesCategory, @@ -91,7 +91,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { true) } -func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { +func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -99,8 +99,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() ) sel.Include(selTD.SharePointBackupFolderScope(sel)) @@ -116,7 +116,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) } @@ -128,8 +128,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() tenID = tconfig.M365TenantID(t) svc = path.SharePointService ws = deeTD.DriveIDFromRepoRef @@ -150,7 +150,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) bID := bo.Results.BackupID @@ -201,18 +201,33 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() { } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { - sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Filter(sel.Library("documents")) - sel.DiscreteOwner = suite.its.siteID + sel.DiscreteOwner = suite.its.site.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.siteDriveID, - suite.its.siteDriveRootFolderID) + suite.its.site.DriveID, + suite.its.site.DriveRootFolderID) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointAlternateProtectedResource() { + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.Filter(sel.Library("documents")) + sel.DiscreteOwner = suite.its.site.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.site, + suite.its.secondarySite) } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { @@ -229,7 +244,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives rc.OnCollision = control.Copy // create a new drive - md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.siteID, rc.Location) + md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.site.ID, rc.Location) require.NoError(t, err, clues.ToCore(err)) driveID := ptr.Val(md.GetId()) @@ -260,14 +275,14 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives // run a backup var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() graphClient = suite.its.ac.Stable.Client() ) - bsel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + bsel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) bsel.Include(selTD.SharePointBackupFolderScope(bsel)) bsel.Filter(bsel.Library(rc.Location)) - bsel.DiscreteOwner = suite.its.siteID + bsel.DiscreteOwner = suite.its.site.ID bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) defer bod.close(t, ctx) @@ -367,7 +382,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives pgr := suite.its.ac. Drives(). - NewSiteDrivePager(suite.its.siteID, []string{"id", "name"}) + NewSiteDrivePager(suite.its.site.ID, []string{"id", "name"}) drives, err := api.GetAllDrives(ctx, pgr, false, -1) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index 963f4f6b4..fe8597da9 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -23,6 +23,7 @@ const ( // M365 config TestCfgAzureTenantID = "azure_tenantid" + TestCfgSecondarySiteID = "secondarym365siteid" TestCfgSiteID = "m365siteid" TestCfgSiteURL = "m365siteurl" TestCfgUserID = "m365userid" @@ -36,13 +37,14 @@ const ( // test specific env vars const ( + EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" + EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" + EnvCorsoSecondaryM365TestSiteID = "CORSO_SECONDARY_M365_TEST_SITE_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" - EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" - EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE" EnvCorsoUnlicensedM365TestUserID = "CORSO_M365_TEST_UNLICENSED_USER" ) @@ -147,13 +149,19 @@ func ReadTestConfig() (map[string]string, error) { TestCfgSiteID, os.Getenv(EnvCorsoM365TestSiteID), vpr.GetString(TestCfgSiteID), - "10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") + "4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") fallbackTo( testEnv, TestCfgSiteURL, os.Getenv(EnvCorsoM365TestSiteURL), vpr.GetString(TestCfgSiteURL), "https://10rqc2.sharepoint.com/sites/CorsoCI") + fallbackTo( + testEnv, + TestCfgSecondarySiteID, + os.Getenv(EnvCorsoSecondaryM365TestSiteID), + vpr.GetString(TestCfgSecondarySiteID), + "053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c") fallbackTo( testEnv, TestCfgUnlicensedUserID, diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index b9e31ce06..bd2fded46 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -198,6 +198,17 @@ func GetM365SiteID(ctx context.Context) string { return strings.ToLower(cfg[TestCfgSiteID]) } +// SecondaryM365SiteID returns a siteID string representing the secondarym365SiteID described +// by either the env var CORSO_SECONDARY_M365_TEST_SITE_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func SecondaryM365SiteID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving secondary m365 site id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgSecondarySiteID]) +} + // UnlicensedM365UserID returns an userID string representing the m365UserID // described by either the env var CORSO_M365_TEST_UNLICENSED_USER, the // corso_test.toml config file or the default value (in that order of priority). diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index fbb3d08a9..01c88b5eb 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -15,7 +15,6 @@ type Options struct { ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` Parallelism Parallelism `json:"parallelism"` Repo repository.Options `json:"repo"` - RestorePermissions bool `json:"restorePermissions"` SkipReduce bool `json:"skipReduce"` ToggleFeatures Toggles `json:"toggleFeatures"` } @@ -38,8 +37,8 @@ const ( BestEffort FailurePolicy = "best-effort" ) -// Defaults provides an Options with the default values set. -func Defaults() Options { +// DefaultOptions provides an Options with the default values set. +func DefaultOptions() Options { return Options{ FailureHandling: FailAfterRecovery, DeltaPageSize: 500, diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 79d49ae20..c30b7d177 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -61,6 +61,10 @@ type RestoreConfig struct { // up. // Defaults to empty. Drive string `json:"drive"` + + // IncludePermissions toggles whether the restore will include the original + // folder- and item-level permissions. + IncludePermissions bool `json:"includePermissions"` } func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { @@ -120,10 +124,11 @@ func (rc RestoreConfig) marshal() string { func (rc RestoreConfig) concealed() RestoreConfig { return RestoreConfig{ - OnCollision: rc.OnCollision, - ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), - Location: path.LoggableDir(rc.Location), - Drive: clues.Hide(rc.Drive).Conceal(), + OnCollision: rc.OnCollision, + ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), + Location: path.LoggableDir(rc.Location), + Drive: clues.Hide(rc.Drive).Conceal(), + IncludePermissions: rc.IncludePermissions, } } diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 1e344bc7a..b26a6a2ef 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -335,7 +335,7 @@ func (r repository) NewBackupWithLookup( return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") } - ownerID, ownerName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if err != nil { return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") } diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 80c2b71f6..34c55d4e7 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -60,7 +60,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Initialize(ctx, test.account, st, control.Defaults()) + _, err = Initialize(ctx, test.account, st, control.DefaultOptions()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -94,7 +94,7 @@ func (suite *RepositoryUnitSuite) TestConnect() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Connect(ctx, test.account, st, "not_found", control.Defaults()) + _, err = Connect(ctx, test.account, st, "not_found", control.DefaultOptions()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -137,7 +137,7 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { defer flush() st := test.storage(t) - r, err := Initialize(ctx, test.account, st, control.Defaults()) + r, err := Initialize(ctx, test.account, st, control.DefaultOptions()) if err == nil { defer func() { err := r.Close(ctx) @@ -186,11 +186,11 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // now re-connect - _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.Defaults()) + _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.DefaultOptions()) assert.NoError(t, err, clues.ToCore(err)) } @@ -203,7 +203,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -212,7 +212,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { require.NoError(t, err, clues.ToCore(err)) // now re-connect - r, err = Connect(ctx, account.Account{}, st, oldID, control.Defaults()) + r, err = Connect(ctx, account.Account{}, st, oldID, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, oldID, r.GetID()) } @@ -228,7 +228,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -250,7 +250,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) @@ -269,7 +269,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize(ctx, acct, st, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -286,7 +286,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions()) require.NoError(t, err) // now re-connect @@ -308,14 +308,14 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "default options", opts: func() control.Options { - return control.Defaults() + return control.DefaultOptions() }, expectedLen: 0, }, { name: "options with an extension factory", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() o.ItemExtensionFactory = append( o.ItemExtensionFactory, &extensions.MockItemExtensionFactory{}) @@ -327,7 +327,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "options with multiple extension factories", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() f := []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{}, diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 8a98a5d56..cc97caa45 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -97,7 +97,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds, control.Defaults()) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) diff --git a/src/pkg/services/m365/api/sites_test.go b/src/pkg/services/m365/api/sites_test.go index d8f49614d..8c4ccf17f 100644 --- a/src/pkg/services/m365/api/sites_test.go +++ b/src/pkg/services/m365/api/sites_test.go @@ -143,12 +143,16 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { var ( t = suite.T() siteID = tconfig.M365SiteID(t) - host = strings.Split(siteID, ",")[0] - shortID = strings.TrimPrefix(siteID, host+",") + parts = strings.Split(siteID, ",") + uuids = siteID siteURL = tconfig.M365SiteURL(t) modifiedSiteURL = siteURL + "foo" ) + if len(parts) == 3 { + uuids = strings.Join(parts[1:], ",") + } + sitesAPI := suite.its.ac.Sites() table := []struct { @@ -165,7 +169,7 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { }, { name: "2 part id", - id: shortID, + id: uuids, expectErr: func(t *testing.T, err error) { assert.NoError(t, err, clues.ToCore(err)) }, @@ -191,13 +195,6 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { assert.NoError(t, err, clues.ToCore(err)) }, }, - { - name: "host only", - id: host, - expectErr: func(t *testing.T, err error) { - assert.NoError(t, err, clues.ToCore(err)) - }, - }, { name: "malformed url", id: "barunihlda", diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 9dd803cf5..5b61885e5 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -329,7 +329,7 @@ func makeAC( return api.Client{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds, control.Defaults()) + cli, err := api.NewClient(creds, control.DefaultOptions()) if err != nil { return api.Client{}, clues.Wrap(err, "constructing api client") } diff --git a/website/docs/setup/restore-options.md b/website/docs/setup/restore-options.md index 9072b2d3f..13265c378 100644 --- a/website/docs/setup/restore-options.md +++ b/website/docs/setup/restore-options.md @@ -10,7 +10,7 @@ manner to a new folder. When you need more control over the results you can use the advanced configuration options to change where and how your data gets restored. -## Destination +## Restore to target folder The `--destination` flag lets you select the top-level folder where Corso will write all of the restored data. @@ -18,7 +18,7 @@ write all of the restored data. ### The default destination { - `corso restore onedrive --backup abcd` + `corso restore onedrive --backup a422895c-c20c-4b06-883d-b866db9f86ef` } If the flag isn't provided, Corso will create a new folder with a standard name: @@ -29,7 +29,7 @@ data integrity then this is always the safest option. ### An alternate destination { - `corso restore onedrive --backup abcd --destination /my-latest-restore` + `corso restore onedrive --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a destination is manually specified, all restored will appear in that top-level @@ -41,14 +41,14 @@ folder multiple times. ### The original location { - `corso restore onedrive --backup abcd --destination /` + `corso restore onedrive --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } You can restore items back to their original location by setting the destination to `/`. This skips the creation of a top-level folder, and all restored items will appear back in their location at the time of backup. -### Limitations +### Destination Limitations * Destination won't create N-depth folder structures. `--destination a/b/c` doesn't create three folders; it creates a single, top-level folder named `a/b/c`. @@ -79,19 +79,19 @@ it still collides. Collisions can be handled with three different configurations: `Skip`, `Copy`, and `Replace`. -## Skip (default) +### Skip (default) { - `corso restore onedrive --backup abcd --collisions skip --destination /` + `corso restore onedrive --collisions skip --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a collision is identified, the item is skipped and no restore is attempted. -## Copy +### Copy { - `corso restore onedrive --backup abcd --collisions copy --destination /my-latest-restore` + `corso restore onedrive --collisions copy --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Item collisions create a copy of the item in the backup. The copy holds the backup @@ -99,12 +99,31 @@ version of the item, leaving the current version unchanged. If necessary, change item properties (such as filenames) to avoid additional collisions. Eg: the copy of`reports.txt` is named `reports 1.txt`. -## Replace +### Replace { - `corso restore onedrive --backup abcd --collisions replace --destination /` + `corso restore onedrive --collisions replace --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Collisions will entirely replace the current version of the item with the backup version. If multiple existing items collide with the backup item, only one of the existing items is replaced. + +## Restore to target resource + +The `--to-resource` flag lets you select which resource will receive the restored data. +A resource can be a mailbox, user, sharepoint site, or other owner of data. + +When restoring to a target resource, all other restore configuration behaves normally. +Data is restored into the default folder: `Corso_Restore_` (unless a +`--destination` flag is added). When restoring in-place, collision policies are followed. + +{ + `corso restore onedrive --to-resource adelev@alcion.ai --backup a422895c-c20c-4b06-883d-b866db9f86ef` +} + +### Resource Limitations + +* The resource must exist. Corso won't create new mailboxes, users, or sites. +* The resource must have access to the service being restored. No restore will be +performed for an unlicensed resource.