Wire up RestoreDestination (#949)

## Description

* No new functionality exposed to CLI users
* generate restore folder names in CLI and pass down the stack
* update tests for new parameters
* centralize generation of restore container names

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* #897 
* #913 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2022-09-27 09:06:15 -07:00 committed by GitHub
parent ed2dd9a026
commit 0a629e0807
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 83 additions and 27 deletions

View File

@ -9,6 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -192,7 +194,9 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
sel.Include(sel.Users(selectors.Any()))
}
ro, err := r.NewRestore(ctx, backupID, sel.Selector)
restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
}

View File

@ -9,6 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -83,7 +85,9 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
sel.Include(sel.Users(selectors.Any()))
}
ro, err := r.NewRestore(ctx, backupID, sel.Selector)
restoreDest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive)
ro, err := r.NewRestore(ctx, backupID, sel.Selector, restoreDest)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore"))
}

View File

@ -450,6 +450,7 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() {
// GraphConnector's Restore Workflow based on OptionIdentifier.
func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
ctx := context.Background()
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
tests := []struct {
name string
option path.CategoryType
@ -495,7 +496,7 @@ func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option)
containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option, dest.ContainerName)
require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err)))
if test.cleanupFunc != nil {

View File

@ -24,8 +24,8 @@ func GetRestoreContainer(
service graph.Service,
user string,
category path.CategoryType,
name string,
) (string, error) {
name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat))
option := categoryToOptionIdentifier(category)
folderID, err := GetContainerID(ctx, service, name, user, option)

View File

@ -245,13 +245,14 @@ func (gc *GraphConnector) ExchangeDataCollection(
func (gc *GraphConnector) RestoreDataCollections(
ctx context.Context,
selector selectors.Selector,
dest control.RestoreDestination,
dcs []data.Collection,
) error {
switch selector.Service {
case selectors.ServiceExchange:
return gc.RestoreExchangeDataCollections(ctx, dcs)
return gc.RestoreExchangeDataCollections(ctx, dest, dcs)
case selectors.ServiceOneDrive:
status, err := onedrive.RestoreCollections(ctx, gc, dcs)
status, err := onedrive.RestoreCollections(ctx, gc, dest, dcs)
if err != nil {
return err
}
@ -270,6 +271,7 @@ func (gc *GraphConnector) RestoreDataCollections(
// into M365
func (gc *GraphConnector) RestoreExchangeDataCollections(
ctx context.Context,
dest control.RestoreDestination,
dcs []data.Collection,
) error {
var (
@ -294,7 +296,13 @@ func (gc *GraphConnector) RestoreExchangeDataCollections(
if _, ok := pathCounter[directory.String()]; !ok {
pathCounter[directory.String()] = true
folderID, errs = exchange.GetRestoreContainer(ctx, &gc.graphService, user, category)
folderID, errs = exchange.GetRestoreContainer(
ctx,
&gc.graphService,
user,
category,
dest.ContainerName,
)
if errs != nil {
fmt.Println("RestoreContainer Failed")

View File

@ -2,15 +2,14 @@ package onedrive
import (
"context"
"fmt"
"io"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -45,13 +44,17 @@ func toOneDrivePath(p path.Path) (*drivePath, error) {
}
// RestoreCollections will restore the specified data collections into OneDrive
func RestoreCollections(ctx context.Context, service graph.Service, dcs []data.Collection,
func RestoreCollections(
ctx context.Context,
service graph.Service,
dest control.RestoreDestination,
dcs []data.Collection,
) (*support.ConnectorOperationStatus, error) {
var (
total, restored int
restoreErrors error
copyBuffer = make([]byte, copyBufferSize)
restoreContainerName = fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormatOneDrive))
restoreContainerName = dest.ContainerName
)
// Iterate through the data collections and restore the contents of each

View File

@ -30,7 +30,8 @@ type RestoreOperation struct {
BackupID model.StableID `json:"backupID"`
Results RestoreResults `json:"results"`
Selectors selectors.Selector `json:"selectors"` // todo: replace with Selectors
Selectors selectors.Selector `json:"selectors"`
Destination control.RestoreDestination `json:"destination"`
Version string `json:"version"`
account account.Account
@ -52,12 +53,14 @@ func NewRestoreOperation(
acct account.Account,
backupID model.StableID,
sel selectors.Selector,
dest control.RestoreDestination,
bus events.Eventer,
) (RestoreOperation, error) {
op := RestoreOperation{
operation: newOperation(opts, bus, kw, sw),
BackupID: backupID,
Selectors: sel,
Destination: dest,
Version: "v0",
account: acct,
}
@ -191,7 +194,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
return err
}
err = gc.RestoreDataCollections(ctx, op.Selectors, dcs)
err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
if err != nil {
err = errors.Wrap(err, "restoring service data")
opStats.writeErr = err

View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -40,10 +41,10 @@ func TestRestoreOpSuite(t *testing.T) {
// TODO: after modelStore integration is added, mock the store and/or
// move this to an integration test.
func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
t := suite.T()
ctx := context.Background()
var (
t = suite.T()
ctx = context.Background()
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
acct = account.Account{}
@ -61,6 +62,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
ObjectCount: 1,
},
}
dest = control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
)
op, err := NewRestoreOperation(
@ -71,6 +73,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
acct,
"foo",
selectors.Selector{},
dest,
evmock.NewBus())
require.NoError(t, err)
@ -184,6 +187,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
kw := &kopia.Wrapper{}
sw := &store.Wrapper{}
acct := tester.NewM365Account(suite.T())
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
table := []struct {
name string
@ -208,6 +212,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
test.acct,
"backup-id",
selectors.Selector{},
dest,
evmock.NewBus())
test.errCheck(t, err)
})
@ -221,6 +226,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
rsel := selectors.NewExchangeRestore()
rsel.Include(rsel.Users([]string{tester.M365UserID(t)}))
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
mb := evmock.NewBus()
ro, err := NewRestoreOperation(
@ -231,6 +237,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
tester.NewM365Account(t),
suite.backupID,
rsel.Selector,
dest,
mb)
require.NoError(t, err)
@ -255,6 +262,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
rsel := selectors.NewExchangeRestore()
rsel.Include(rsel.Users(selectors.None()))
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
mb := evmock.NewBus()
ro, err := NewRestoreOperation(
@ -265,6 +273,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
tester.NewM365Account(t),
suite.backupID,
rsel.Selector,
dest,
mb)
require.NoError(t, err)
require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results")

View File

@ -1,5 +1,13 @@
package control
import (
"github.com/alcionai/corso/src/internal/common"
)
const (
defaultRestoreLocation = "Corso_Restore_"
)
// CollisionPolicy describes how the datalayer behaves in case of a collision.
type CollisionPolicy int
@ -37,3 +45,9 @@ type RestoreDestination struct {
// This field must be populated for a restore.
ContainerName string
}
func DefaultRestoreDestination(timeFormat string) RestoreDestination {
return RestoreDestination{
ContainerName: defaultRestoreLocation + common.FormatNow(timeFormat),
}
}

View File

@ -41,6 +41,7 @@ type Repository interface {
ctx context.Context,
backupID string,
sel selectors.Selector,
dest control.RestoreDestination,
) (operations.RestoreOperation, error)
DeleteBackup(ctx context.Context, id model.StableID) error
BackupGetter
@ -194,6 +195,7 @@ func (r repository) NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
dest control.RestoreDestination,
) (operations.RestoreOperation, error) {
return operations.NewRestoreOperation(
ctx,
@ -203,6 +205,7 @@ func (r repository) NewRestore(
r.Account,
model.StableID(backupID),
sel,
dest,
r.Bus)
}

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
@ -200,7 +201,9 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() {
rsel, err := bsel.ToExchangeRestore()
require.NoError(t, err)
rst, err := r.NewRestore(ctx, bid, rsel.Selector)
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest)
require.NoError(t, err)
runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten)
@ -270,7 +273,9 @@ func (suite *RepositoryLoadTestOneDriveSuite) TestOneDrive() {
rsel, err := bsel.ToOneDriveRestore()
require.NoError(t, err)
rst, err := r.NewRestore(ctx, bid, rsel.Selector)
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive)
rst, err := r.NewRestore(ctx, bid, rsel.Selector, dest)
require.NoError(t, err)
runRestoreLoadTest(t, ctx, rst, service, b.Results.ItemsWritten)

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
@ -176,6 +177,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
ctx := context.Background()
acct := tester.NewM365Account(t)
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
@ -183,7 +185,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err)
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{})
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}, dest)
require.NoError(t, err)
require.NotNil(t, ro)
}