From 0a629e0807a32639ab1b79e0a53d2c012b773578 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Tue, 27 Sep 2022 09:06:15 -0700 Subject: [PATCH] Wire up RestoreDestination (#949) ## Description * No new functionality exposed to CLI users * generate restore folder names in CLI and pass down the stack * update tests for new parameters * centralize generation of restore container names ## Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :hamster: Trivial/Minor ## Issue(s) * #897 * #913 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cli/restore/exchange.go | 6 ++++- src/cli/restore/onedrive.go | 6 ++++- .../exchange/exchange_service_test.go | 3 ++- .../connector/exchange/service_restore.go | 2 +- src/internal/connector/graph_connector.go | 14 ++++++++--- src/internal/connector/onedrive/restore.go | 11 +++++---- src/internal/operations/restore.go | 23 +++++++++++-------- src/internal/operations/restore_test.go | 15 +++++++++--- src/pkg/control/options.go | 14 +++++++++++ src/pkg/repository/repository.go | 3 +++ src/pkg/repository/repository_load_test.go | 9 ++++++-- src/pkg/repository/repository_test.go | 4 +++- 12 files changed, 83 insertions(+), 27 deletions(-) diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index 2d28dd54c..3e65f94b4 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -9,6 +9,8 @@ import ( "github.com/alcionai/corso/src/cli/options" . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -192,7 +194,9 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error { sel.Include(sel.Users(selectors.Any())) } - ro, err := r.NewRestore(ctx, backupID, sel.Selector) + restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) + + ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore")) } diff --git a/src/cli/restore/onedrive.go b/src/cli/restore/onedrive.go index 5418b6a9d..b569f66e9 100644 --- a/src/cli/restore/onedrive.go +++ b/src/cli/restore/onedrive.go @@ -9,6 +9,8 @@ import ( "github.com/alcionai/corso/src/cli/options" . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -83,7 +85,9 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { sel.Include(sel.Users(selectors.Any())) } - ro, err := r.NewRestore(ctx, backupID, sel.Selector) + restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive) + + ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore")) } diff --git a/src/internal/connector/exchange/exchange_service_test.go b/src/internal/connector/exchange/exchange_service_test.go index efbf188dc..746655e30 100644 --- a/src/internal/connector/exchange/exchange_service_test.go +++ b/src/internal/connector/exchange/exchange_service_test.go @@ -450,6 +450,7 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() { // GraphConnector's Restore Workflow based on OptionIdentifier. func (suite *ExchangeServiceSuite) TestGetRestoreContainer() { ctx := context.Background() + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) tests := []struct { name string option path.CategoryType @@ -495,7 +496,7 @@ func (suite *ExchangeServiceSuite) TestGetRestoreContainer() { for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option) + containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option, dest.ContainerName) require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err))) if test.cleanupFunc != nil { diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index b6efe907d..224f38e27 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -24,8 +24,8 @@ func GetRestoreContainer( service graph.Service, user string, category path.CategoryType, + name string, ) (string, error) { - name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat)) option := categoryToOptionIdentifier(category) folderID, err := GetContainerID(ctx, service, name, user, option) diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 83fc9c41f..ccce8083a 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -245,13 +245,14 @@ func (gc *GraphConnector) ExchangeDataCollection( func (gc *GraphConnector) RestoreDataCollections( ctx context.Context, selector selectors.Selector, + dest control.RestoreDestination, dcs []data.Collection, ) error { switch selector.Service { case selectors.ServiceExchange: - return gc.RestoreExchangeDataCollections(ctx, dcs) + return gc.RestoreExchangeDataCollections(ctx, dest, dcs) case selectors.ServiceOneDrive: - status, err := onedrive.RestoreCollections(ctx, gc, dcs) + status, err := onedrive.RestoreCollections(ctx, gc, dest, dcs) if err != nil { return err } @@ -270,6 +271,7 @@ func (gc *GraphConnector) RestoreDataCollections( // into M365 func (gc *GraphConnector) RestoreExchangeDataCollections( ctx context.Context, + dest control.RestoreDestination, dcs []data.Collection, ) error { var ( @@ -294,7 +296,13 @@ func (gc *GraphConnector) RestoreExchangeDataCollections( if _, ok := pathCounter[directory.String()]; !ok { pathCounter[directory.String()] = true - folderID, errs = exchange.GetRestoreContainer(ctx, &gc.graphService, user, category) + folderID, errs = exchange.GetRestoreContainer( + ctx, + &gc.graphService, + user, + category, + dest.ContainerName, + ) if errs != nil { fmt.Println("RestoreContainer Failed") diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index d514a0aa6..0619964c7 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -2,15 +2,14 @@ package onedrive import ( "context" - "fmt" "io" "github.com/pkg/errors" - "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) @@ -45,13 +44,17 @@ func toOneDrivePath(p path.Path) (*drivePath, error) { } // RestoreCollections will restore the specified data collections into OneDrive -func RestoreCollections(ctx context.Context, service graph.Service, dcs []data.Collection, +func RestoreCollections( + ctx context.Context, + service graph.Service, + dest control.RestoreDestination, + dcs []data.Collection, ) (*support.ConnectorOperationStatus, error) { var ( total, restored int restoreErrors error copyBuffer = make([]byte, copyBufferSize) - restoreContainerName = fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormatOneDrive)) + restoreContainerName = dest.ContainerName ) // Iterate through the data collections and restore the contents of each diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 173e6073c..e8722487f 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -28,10 +28,11 @@ import ( type RestoreOperation struct { operation - BackupID model.StableID `json:"backupID"` - Results RestoreResults `json:"results"` - Selectors selectors.Selector `json:"selectors"` // todo: replace with Selectors - Version string `json:"version"` + BackupID model.StableID `json:"backupID"` + Results RestoreResults `json:"results"` + Selectors selectors.Selector `json:"selectors"` + Destination control.RestoreDestination `json:"destination"` + Version string `json:"version"` account account.Account } @@ -52,14 +53,16 @@ func NewRestoreOperation( acct account.Account, backupID model.StableID, sel selectors.Selector, + dest control.RestoreDestination, bus events.Eventer, ) (RestoreOperation, error) { op := RestoreOperation{ - operation: newOperation(opts, bus, kw, sw), - BackupID: backupID, - Selectors: sel, - Version: "v0", - account: acct, + operation: newOperation(opts, bus, kw, sw), + BackupID: backupID, + Selectors: sel, + Destination: dest, + Version: "v0", + account: acct, } if err := op.validate(); err != nil { return RestoreOperation{}, err @@ -191,7 +194,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { return err } - err = gc.RestoreDataCollections(ctx, op.Selectors, dcs) + err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs) if err != nil { err = errors.Wrap(err, "restoring service data") opStats.writeErr = err diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 34d8f8f28..aaa63aa5f 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -40,10 +41,10 @@ func TestRestoreOpSuite(t *testing.T) { // TODO: after modelStore integration is added, mock the store and/or // move this to an integration test. func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { - t := suite.T() - ctx := context.Background() - var ( + t = suite.T() + ctx = context.Background() + kw = &kopia.Wrapper{} sw = &store.Wrapper{} acct = account.Account{} @@ -61,6 +62,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { ObjectCount: 1, }, } + dest = control.DefaultRestoreDestination(common.SimpleDateTimeFormat) ) op, err := NewRestoreOperation( @@ -71,6 +73,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { acct, "foo", selectors.Selector{}, + dest, evmock.NewBus()) require.NoError(t, err) @@ -184,6 +187,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { kw := &kopia.Wrapper{} sw := &store.Wrapper{} acct := tester.NewM365Account(suite.T()) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) table := []struct { name string @@ -208,6 +212,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { test.acct, "backup-id", selectors.Selector{}, + dest, evmock.NewBus()) test.errCheck(t, err) }) @@ -221,6 +226,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { rsel := selectors.NewExchangeRestore() rsel.Include(rsel.Users([]string{tester.M365UserID(t)})) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) mb := evmock.NewBus() ro, err := NewRestoreOperation( @@ -231,6 +237,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { tester.NewM365Account(t), suite.backupID, rsel.Selector, + dest, mb) require.NoError(t, err) @@ -255,6 +262,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() { rsel := selectors.NewExchangeRestore() rsel.Include(rsel.Users(selectors.None())) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) mb := evmock.NewBus() ro, err := NewRestoreOperation( @@ -265,6 +273,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() { tester.NewM365Account(t), suite.backupID, rsel.Selector, + dest, mb) require.NoError(t, err) require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results") diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index c8a923cea..a2f34fad0 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -1,5 +1,13 @@ package control +import ( + "github.com/alcionai/corso/src/internal/common" +) + +const ( + defaultRestoreLocation = "Corso_Restore_" +) + // CollisionPolicy describes how the datalayer behaves in case of a collision. type CollisionPolicy int @@ -37,3 +45,9 @@ type RestoreDestination struct { // This field must be populated for a restore. ContainerName string } + +func DefaultRestoreDestination(timeFormat string) RestoreDestination { + return RestoreDestination{ + ContainerName: defaultRestoreLocation + common.FormatNow(timeFormat), + } +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index f2bc96b2e..dde0a9565 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -41,6 +41,7 @@ type Repository interface { ctx context.Context, backupID string, sel selectors.Selector, + dest control.RestoreDestination, ) (operations.RestoreOperation, error) DeleteBackup(ctx context.Context, id model.StableID) error BackupGetter @@ -194,6 +195,7 @@ func (r repository) NewRestore( ctx context.Context, backupID string, sel selectors.Selector, + dest control.RestoreDestination, ) (operations.RestoreOperation, error) { return operations.NewRestoreOperation( ctx, @@ -203,6 +205,7 @@ func (r repository) NewRestore( r.Account, model.StableID(backupID), sel, + dest, r.Bus) } diff --git a/src/pkg/repository/repository_load_test.go b/src/pkg/repository/repository_load_test.go index a9eafa575..8a68f8dcf 100644 --- a/src/pkg/repository/repository_load_test.go +++ b/src/pkg/repository/repository_load_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" @@ -200,7 +201,9 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() { rsel, err := bsel.ToExchangeRestore() require.NoError(t, err) - rst, err := r.NewRestore(ctx, bid, rsel.Selector) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) + + rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest) require.NoError(t, err) runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten) @@ -270,7 +273,9 @@ func (suite *RepositoryLoadTestOneDriveSuite) TestOneDrive() { rsel, err := bsel.ToOneDriveRestore() require.NoError(t, err) - rst, err := r.NewRestore(ctx, bid, rsel.Selector) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive) + + rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest) require.NoError(t, err) runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten) diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 67d366c5f..c10e14249 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -176,6 +177,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { ctx := context.Background() acct := tester.NewM365Account(t) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) // need to initialize the repository before we can test connecting to it. st := tester.NewPrefixedS3Storage(t) @@ -183,7 +185,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { r, err := repository.Initialize(ctx, acct, st, control.Options{}) require.NoError(t, err) - ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}) + ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}, dest) require.NoError(t, err) require.NotNil(t, ro) }