Wire up RestoreDestination (#949)

## Description

* No new functionality exposed to CLI users
* generate restore folder names in CLI and pass down the stack
* update tests for new parameters
* centralize generation of restore container names

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* #897 
* #913 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2022-09-27 09:06:15 -07:00 committed by GitHub
parent ed2dd9a026
commit 0a629e0807
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 83 additions and 27 deletions

View File

@ -9,6 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/options" "github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
) )
@ -192,7 +194,9 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
sel.Include(sel.Users(selectors.Any())) sel.Include(sel.Users(selectors.Any()))
} }
ro, err := r.NewRestore(ctx, backupID, sel.Selector) restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest)
if err != nil { if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore")) return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
} }

View File

@ -9,6 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/options" "github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
) )
@ -83,7 +85,9 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
sel.Include(sel.Users(selectors.Any())) sel.Include(sel.Users(selectors.Any()))
} }
ro, err := r.NewRestore(ctx, backupID, sel.Selector) restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive)
ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest)
if err != nil { if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore")) return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore"))
} }

View File

@ -450,6 +450,7 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() {
// GraphConnector's Restore Workflow based on OptionIdentifier. // GraphConnector's Restore Workflow based on OptionIdentifier.
func (suite *ExchangeServiceSuite) TestGetRestoreContainer() { func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
ctx := context.Background() ctx := context.Background()
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
tests := []struct { tests := []struct {
name string name string
option path.CategoryType option path.CategoryType
@ -495,7 +496,7 @@ func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
for _, test := range tests { for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option) containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option, dest.ContainerName)
require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err))) require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err)))
if test.cleanupFunc != nil { if test.cleanupFunc != nil {

View File

@ -24,8 +24,8 @@ func GetRestoreContainer(
service graph.Service, service graph.Service,
user string, user string,
category path.CategoryType, category path.CategoryType,
name string,
) (string, error) { ) (string, error) {
name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat))
option := categoryToOptionIdentifier(category) option := categoryToOptionIdentifier(category)
folderID, err := GetContainerID(ctx, service, name, user, option) folderID, err := GetContainerID(ctx, service, name, user, option)

View File

@ -245,13 +245,14 @@ func (gc *GraphConnector) ExchangeDataCollection(
func (gc *GraphConnector) RestoreDataCollections( func (gc *GraphConnector) RestoreDataCollections(
ctx context.Context, ctx context.Context,
selector selectors.Selector, selector selectors.Selector,
dest control.RestoreDestination,
dcs []data.Collection, dcs []data.Collection,
) error { ) error {
switch selector.Service { switch selector.Service {
case selectors.ServiceExchange: case selectors.ServiceExchange:
return gc.RestoreExchangeDataCollections(ctx, dcs) return gc.RestoreExchangeDataCollections(ctx, dest, dcs)
case selectors.ServiceOneDrive: case selectors.ServiceOneDrive:
status, err := onedrive.RestoreCollections(ctx, gc, dcs) status, err := onedrive.RestoreCollections(ctx, gc, dest, dcs)
if err != nil { if err != nil {
return err return err
} }
@ -270,6 +271,7 @@ func (gc *GraphConnector) RestoreDataCollections(
// into M365 // into M365
func (gc *GraphConnector) RestoreExchangeDataCollections( func (gc *GraphConnector) RestoreExchangeDataCollections(
ctx context.Context, ctx context.Context,
dest control.RestoreDestination,
dcs []data.Collection, dcs []data.Collection,
) error { ) error {
var ( var (
@ -294,7 +296,13 @@ func (gc *GraphConnector) RestoreExchangeDataCollections(
if _, ok := pathCounter[directory.String()]; !ok { if _, ok := pathCounter[directory.String()]; !ok {
pathCounter[directory.String()] = true pathCounter[directory.String()] = true
folderID, errs = exchange.GetRestoreContainer(ctx, &gc.graphService, user, category) folderID, errs = exchange.GetRestoreContainer(
ctx,
&gc.graphService,
user,
category,
dest.ContainerName,
)
if errs != nil { if errs != nil {
fmt.Println("RestoreContainer Failed") fmt.Println("RestoreContainer Failed")

View File

@ -2,15 +2,14 @@ package onedrive
import ( import (
"context" "context"
"fmt"
"io" "io"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -45,13 +44,17 @@ func toOneDrivePath(p path.Path) (*drivePath, error) {
} }
// RestoreCollections will restore the specified data collections into OneDrive // RestoreCollections will restore the specified data collections into OneDrive
func RestoreCollections(ctx context.Context, service graph.Service, dcs []data.Collection, func RestoreCollections(
ctx context.Context,
service graph.Service,
dest control.RestoreDestination,
dcs []data.Collection,
) (*support.ConnectorOperationStatus, error) { ) (*support.ConnectorOperationStatus, error) {
var ( var (
total, restored int total, restored int
restoreErrors error restoreErrors error
copyBuffer = make([]byte, copyBufferSize) copyBuffer = make([]byte, copyBufferSize)
restoreContainerName = fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormatOneDrive)) restoreContainerName = dest.ContainerName
) )
// Iterate through the data collections and restore the contents of each // Iterate through the data collections and restore the contents of each

View File

@ -28,10 +28,11 @@ import (
type RestoreOperation struct { type RestoreOperation struct {
operation operation
BackupID model.StableID `json:"backupID"` BackupID model.StableID `json:"backupID"`
Results RestoreResults `json:"results"` Results RestoreResults `json:"results"`
Selectors selectors.Selector `json:"selectors"` // todo: replace with Selectors Selectors selectors.Selector `json:"selectors"`
Version string `json:"version"` Destination control.RestoreDestination `json:"destination"`
Version string `json:"version"`
account account.Account account account.Account
} }
@ -52,14 +53,16 @@ func NewRestoreOperation(
acct account.Account, acct account.Account,
backupID model.StableID, backupID model.StableID,
sel selectors.Selector, sel selectors.Selector,
dest control.RestoreDestination,
bus events.Eventer, bus events.Eventer,
) (RestoreOperation, error) { ) (RestoreOperation, error) {
op := RestoreOperation{ op := RestoreOperation{
operation: newOperation(opts, bus, kw, sw), operation: newOperation(opts, bus, kw, sw),
BackupID: backupID, BackupID: backupID,
Selectors: sel, Selectors: sel,
Version: "v0", Destination: dest,
account: acct, Version: "v0",
account: acct,
} }
if err := op.validate(); err != nil { if err := op.validate(); err != nil {
return RestoreOperation{}, err return RestoreOperation{}, err
@ -191,7 +194,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
return err return err
} }
err = gc.RestoreDataCollections(ctx, op.Selectors, dcs) err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
if err != nil { if err != nil {
err = errors.Wrap(err, "restoring service data") err = errors.Wrap(err, "restoring service data")
opStats.writeErr = err opStats.writeErr = err

View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -40,10 +41,10 @@ func TestRestoreOpSuite(t *testing.T) {
// TODO: after modelStore integration is added, mock the store and/or // TODO: after modelStore integration is added, mock the store and/or
// move this to an integration test. // move this to an integration test.
func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
t := suite.T()
ctx := context.Background()
var ( var (
t = suite.T()
ctx = context.Background()
kw = &kopia.Wrapper{} kw = &kopia.Wrapper{}
sw = &store.Wrapper{} sw = &store.Wrapper{}
acct = account.Account{} acct = account.Account{}
@ -61,6 +62,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
ObjectCount: 1, ObjectCount: 1,
}, },
} }
dest = control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
) )
op, err := NewRestoreOperation( op, err := NewRestoreOperation(
@ -71,6 +73,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
acct, acct,
"foo", "foo",
selectors.Selector{}, selectors.Selector{},
dest,
evmock.NewBus()) evmock.NewBus())
require.NoError(t, err) require.NoError(t, err)
@ -184,6 +187,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
kw := &kopia.Wrapper{} kw := &kopia.Wrapper{}
sw := &store.Wrapper{} sw := &store.Wrapper{}
acct := tester.NewM365Account(suite.T()) acct := tester.NewM365Account(suite.T())
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
table := []struct { table := []struct {
name string name string
@ -208,6 +212,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
test.acct, test.acct,
"backup-id", "backup-id",
selectors.Selector{}, selectors.Selector{},
dest,
evmock.NewBus()) evmock.NewBus())
test.errCheck(t, err) test.errCheck(t, err)
}) })
@ -221,6 +226,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
rsel := selectors.NewExchangeRestore() rsel := selectors.NewExchangeRestore()
rsel.Include(rsel.Users([]string{tester.M365UserID(t)})) rsel.Include(rsel.Users([]string{tester.M365UserID(t)}))
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
mb := evmock.NewBus() mb := evmock.NewBus()
ro, err := NewRestoreOperation( ro, err := NewRestoreOperation(
@ -231,6 +237,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
tester.NewM365Account(t), tester.NewM365Account(t),
suite.backupID, suite.backupID,
rsel.Selector, rsel.Selector,
dest,
mb) mb)
require.NoError(t, err) require.NoError(t, err)
@ -255,6 +262,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
rsel := selectors.NewExchangeRestore() rsel := selectors.NewExchangeRestore()
rsel.Include(rsel.Users(selectors.None())) rsel.Include(rsel.Users(selectors.None()))
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
mb := evmock.NewBus() mb := evmock.NewBus()
ro, err := NewRestoreOperation( ro, err := NewRestoreOperation(
@ -265,6 +273,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
tester.NewM365Account(t), tester.NewM365Account(t),
suite.backupID, suite.backupID,
rsel.Selector, rsel.Selector,
dest,
mb) mb)
require.NoError(t, err) require.NoError(t, err)
require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results") require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results")

View File

@ -1,5 +1,13 @@
package control package control
import (
"github.com/alcionai/corso/src/internal/common"
)
const (
defaultRestoreLocation = "Corso_Restore_"
)
// CollisionPolicy describes how the datalayer behaves in case of a collision. // CollisionPolicy describes how the datalayer behaves in case of a collision.
type CollisionPolicy int type CollisionPolicy int
@ -37,3 +45,9 @@ type RestoreDestination struct {
// This field must be populated for a restore. // This field must be populated for a restore.
ContainerName string ContainerName string
} }
func DefaultRestoreDestination(timeFormat string) RestoreDestination {
return RestoreDestination{
ContainerName: defaultRestoreLocation + common.FormatNow(timeFormat),
}
}

View File

@ -41,6 +41,7 @@ type Repository interface {
ctx context.Context, ctx context.Context,
backupID string, backupID string,
sel selectors.Selector, sel selectors.Selector,
dest control.RestoreDestination,
) (operations.RestoreOperation, error) ) (operations.RestoreOperation, error)
DeleteBackup(ctx context.Context, id model.StableID) error DeleteBackup(ctx context.Context, id model.StableID) error
BackupGetter BackupGetter
@ -194,6 +195,7 @@ func (r repository) NewRestore(
ctx context.Context, ctx context.Context,
backupID string, backupID string,
sel selectors.Selector, sel selectors.Selector,
dest control.RestoreDestination,
) (operations.RestoreOperation, error) { ) (operations.RestoreOperation, error) {
return operations.NewRestoreOperation( return operations.NewRestoreOperation(
ctx, ctx,
@ -203,6 +205,7 @@ func (r repository) NewRestore(
r.Account, r.Account,
model.StableID(backupID), model.StableID(backupID),
sel, sel,
dest,
r.Bus) r.Bus)
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
@ -200,7 +201,9 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() {
rsel, err := bsel.ToExchangeRestore() rsel, err := bsel.ToExchangeRestore()
require.NoError(t, err) require.NoError(t, err)
rst, err := r.NewRestore(ctx, bid, rsel.Selector) dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest)
require.NoError(t, err) require.NoError(t, err)
runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten) runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten)
@ -270,7 +273,9 @@ func (suite *RepositoryLoadTestOneDriveSuite) TestOneDrive() {
rsel, err := bsel.ToOneDriveRestore() rsel, err := bsel.ToOneDriveRestore()
require.NoError(t, err) require.NoError(t, err)
rst, err := r.NewRestore(ctx, bid, rsel.Selector) dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive)
rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest)
require.NoError(t, err) require.NoError(t, err)
runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten) runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten)

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
@ -176,6 +177,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
ctx := context.Background() ctx := context.Background()
acct := tester.NewM365Account(t) acct := tester.NewM365Account(t)
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t) st := tester.NewPrefixedS3Storage(t)
@ -183,7 +185,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
r, err := repository.Initialize(ctx, acct, st, control.Options{}) r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err) require.NoError(t, err)
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}, dest)
require.NoError(t, err) require.NoError(t, err)
require.NotNil(t, ro) require.NotNil(t, ro)
} }