larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hook up restore end-to-end (#226)

Browse Source 
* hook up restore end-to-end

Now that GC and KW both provide restore operations for a
single message, we can hook up the end-to-end restore
process.  Integration tests for this change will follow in the
next PR.
This commit is contained in:
Keepers 2022-06-22 18:17:26 -06:00 committed by GitHub
parent 128e9274e9
commit 2b65ff80f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 68 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cli/backup/exchange.go
View File

@ -79,10 +79,11 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return errors.Wrap(err, "Failed to initialize Exchange backup")
}
if _, err := bo.Run(ctx); err != nil {
result, err := bo.Run(ctx)
if err != nil {
return errors.Wrap(err, "Failed to run Exchange backup")
}
fmt.Printf("Backed up Exchange in %s for user %s.\n", s.Provider, user)
fmt.Printf("Backed up restore point %s in %s for Exchange user %s.\n", result.SnapshotID, s.Provider, user)
return nil
}

27
src/cli/restore/exchange.go
View File

@ -15,9 +15,10 @@ import (
// exchange bucket info from flags
var (
folder string
mail string
user string
folder string
mail string
restorePointID string
user string
)
// called by restore.go to map parent subcommands to provider-specific handling.
@ -25,9 +26,11 @@ func addExchangeApp(parent *cobra.Command) *cobra.Command {
parent.AddCommand(exchangeCmd)
fs := exchangeCmd.Flags()
fs.StringVar(&user, "user", "", "ID of the user whose echange data will get restored.")
fs.StringVar(&folder, "folder", "", "Name of the mail folder being restored.")
fs.StringVar(&mail, "mail", "", "ID of the mail message being restored.")
fs.StringVar(&folder, "folder", "", "Name of the mail folder being restored")
fs.StringVar(&mail, "mail", "", "ID of the mail message being restored")
fs.StringVar(&restorePointID, "restore-point", "", "ID of the backup restore point")
exchangeCmd.MarkFlagRequired("restore-point")
fs.StringVar(&user, "user", "", "ID of the user whose exchange data will get restored")
return exchangeCmd
}
@ -48,7 +51,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return nil
}
if err := validateRestoreFlags(user, folder, mail); err != nil {
if err := validateRestoreFlags(user, folder, mail, restorePointID); err != nil {
return errors.Wrap(err, "Missing required flags")
}
@ -69,6 +72,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
logger.Ctx(ctx).Debugw(
"Called - "+cmd.CommandPath(),
"restorePointID", restorePointID,
"tenantID", m365.TenantID,
"clientID", m365.ClientID,
"hasClientSecret", len(m365.ClientSecret) > 0)
@ -79,12 +83,12 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
}
defer utils.CloseRepo(ctx, r)
ro, err := r.NewRestore(ctx, []string{user, folder, mail})
ro, err := r.NewRestore(ctx, restorePointID, []string{cfgTenantID, user, "mail", folder, mail})
if err != nil {
return errors.Wrap(err, "Failed to initialize Exchange restore")
}
if _, err := ro.Run(ctx); err != nil {
if err := ro.Run(ctx); err != nil {
return errors.Wrap(err, "Failed to run Exchange restore")
}
@ -92,7 +96,10 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return nil
}
func validateRestoreFlags(u, f, m string) error {
func validateRestoreFlags(u, f, m, rpid string) error {
if len(rpid) == 0 {
return errors.New("a restore point ID is requried")
}
lu, lf, lm := len(u), len(f), len(m)
if (lu == 0 || u == "*") && (lf+lm > 0) {
return errors.New("a specific --user must be provided if --folder or --mail is specified")

23
src/cli/restore/exchange_test.go
View File

@ -17,23 +17,24 @@ func TestRestoreSuite(t *testing.T) {
func (suite *RestoreSuite) TestValidateRestoreFlags() {
table := []struct {
name string
u, f, m string
errCheck assert.ErrorAssertionFunc
name string
u, f, m, rpid string
errCheck assert.ErrorAssertionFunc
}{
{"all populated", "u", "f", "m", assert.NoError},
{"folder missing user", "", "f", "m", assert.Error},
{"folder with wildcard user", "*", "f", "m", assert.Error},
{"mail missing user", "", "", "m", assert.Error},
{"mail missing folder", "u", "", "m", assert.Error},
{"mail with wildcard folder", "u", "*", "m", assert.Error},
{"all missing", "", "", "", assert.NoError},
{"all populated", "u", "f", "m", "rpid", assert.NoError},
{"folder missing user", "", "f", "m", "rpid", assert.Error},
{"folder with wildcard user", "*", "f", "m", "rpid", assert.Error},
{"mail missing user", "", "", "m", "rpid", assert.Error},
{"mail missing folder", "u", "", "m", "rpid", assert.Error},
{"mail with wildcard folder", "u", "*", "m", "rpid", assert.Error},
{"missing restore point id", "u", "f", "m", "", assert.Error},
{"all missing", "", "", "", "rpid", assert.NoError},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
test.errCheck(
t,
validateRestoreFlags(test.u, test.f, test.m),
validateRestoreFlags(test.u, test.f, test.m, test.rpid),
)
})
}

4
src/internal/connector/graph_connector.go
View File

@ -156,10 +156,10 @@ func optionsForMailFolders(moreOps []string) *msfolder.MailFoldersRequestBuilder
return options
}
// restoreMessages: Utility function to connect to M365 backstore
// RestoreMessages: Utility function to connect to M365 backstore
// and upload messages from DataCollection.
// FullPath: tenantId, userId, <mailCategory>, FolderId
func (gc *GraphConnector) restoreMessages(ctx context.Context, dc DataCollection) error {
func (gc *GraphConnector) RestoreMessages(ctx context.Context, dc DataCollection) error {
var errs error
// must be user.GetId(), PrimaryName no longer works 6-15-2022
user := dc.FullPath()[1]

2
src/internal/connector/graph_connector_test.go
View File

@ -88,7 +88,7 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages(
edc := NewExchangeDataCollection("tenant", []string{"tenantId", evs[user], mailCategory, "Inbox"})
edc.PopulateCollection(ds)
edc.FinishPopulation()
err = suite.connector.restoreMessages(context.Background(), &edc)
err = suite.connector.RestoreMessages(context.Background(), &edc)
assert.NoError(suite.T(), err)
}

2
src/internal/kopia/kopia.go
View File

@ -35,6 +35,7 @@ var (
)
type BackupStats struct {
SnapshotID string
TotalFileCount int
TotalDirectoryCount int
IgnoredErrorCount int
@ -45,6 +46,7 @@ type BackupStats struct {
func manifestToStats(man *snapshot.Manifest) BackupStats {
return BackupStats{
SnapshotID: string(man.ID),
TotalFileCount: int(man.Stats.TotalFileCount),
TotalDirectoryCount: int(man.Stats.TotalDirectoryCount),
IgnoredErrorCount: int(man.Stats.IgnoredErrorCount),

25
src/internal/kopia/kopia_test.go
View File

@ -9,9 +9,7 @@ import (
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/fs/virtualfs"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -131,7 +129,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
require.Len(suite.T(), subEntries, 1)
assert.Contains(suite.T(), subEntries[0].Name(), emails)
subDir, ok := subEntries[0].(fs.Directory)
subDir := subEntries[0].(fs.Directory)
emailFiles, err := fs.GetAllEntries(ctx, subDir)
require.NoError(suite.T(), err)
assert.Len(suite.T(), emailFiles, expectedFileCount[entry.Name()])
@ -294,25 +292,6 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
assert.False(suite.T(), stats.Incomplete)
}
func getSnapshotID(
t *testing.T,
ctx context.Context,
rep repo.Repository,
rootName string,
) manifest.ID {
si := snapshot.SourceInfo{
Host: kTestHost,
UserName: kTestUser,
Path: rootName,
}
manifests, err := snapshot.ListSnapshots(ctx, rep, si)
require.NoError(t, err)
require.Len(t, manifests, 1)
return manifests[0].ID
}
func setupSimpleRepo(t *testing.T, ctx context.Context, k *KopiaWrapper) manifest.ID {
collections := []connector.DataCollection{
&singleItemCollection{
@ -332,7 +311,7 @@ func setupSimpleRepo(t *testing.T, ctx context.Context, k *KopiaWrapper) manifes
require.Equal(t, stats.ErrorCount, 0)
require.False(t, stats.Incomplete)
return getSnapshotID(t, ctx, k.rep, testPath[0])
return manifest.ID(stats.SnapshotID)
}
func (suite *KopiaIntegrationSuite) TestBackupAndRestoreSingleItem() {

4
src/internal/operations/backup.go
View File

@ -57,13 +57,13 @@ func (op *BackupOperation) Run(ctx context.Context) (*kopia.BackupStats, error)
cs, err := gc.ExchangeDataCollection(ctx, op.Targets[0])
if err != nil {
return nil, errors.Wrap(err, "retrieving application data")
return nil, errors.Wrap(err, "retrieving service data")
}
// todo: utilize stats
stats, err := op.kopia.BackupCollections(ctx, cs)
if err != nil {
return nil, errors.Wrap(err, "backing up application data")
return nil, errors.Wrap(err, "backing up service data")
}
op.Status = Successful

9
src/internal/operations/operation.go
View File

@ -1,7 +1,6 @@
package operations
import (
"context"
"time"
"github.com/google/uuid"
@ -33,16 +32,8 @@ type operation struct {
Errors []error
}
type logger interface {
Debug(context.Context, string)
Info(context.Context, string)
Warn(context.Context, string)
Error(context.Context, string)
}
// OperationOpts configure some parameters of the operation
type OperationOpts struct {
Logger logger
}
func newOperation(

31
src/internal/operations/restore.go
View File

@ -5,6 +5,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/internal/connector"
"github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/pkg/credentials"
)
@ -14,7 +15,8 @@ type RestoreOperation struct {
operation
Version string
creds credentials.M365
restorePointID string
creds credentials.M365
Targets []string // something for targets/filter/source/app&users/etc
}
@ -25,13 +27,15 @@ func NewRestoreOperation(
opts OperationOpts,
kw *kopia.KopiaWrapper,
creds credentials.M365,
restorePointID string,
targets []string,
) (RestoreOperation, error) {
op := RestoreOperation{
operation: newOperation(opts, kw),
Version: "v0",
creds: creds,
Targets: targets,
operation: newOperation(opts, kw),
Version: "v0",
creds: creds,
restorePointID: restorePointID,
Targets: targets,
}
if err := op.validate(); err != nil {
return RestoreOperation{}, err
@ -49,10 +53,21 @@ func (op RestoreOperation) validate() error {
// Run begins a synchronous restore operation.
// todo (keepers): return stats block in first param.
func (op *RestoreOperation) Run(ctx context.Context) (any, error) {
func (op *RestoreOperation) Run(ctx context.Context) error {
dc, err := op.kopia.RestoreSingleItem(ctx, op.restorePointID, op.Targets)
if err != nil {
return errors.Wrap(err, "retrieving service data")
}
// todo: hook up with KW and GC restore operations.
gc, err := connector.NewGraphConnector(op.creds.TenantID, op.creds.ClientID, op.creds.ClientSecret)
if err != nil {
return errors.Wrap(err, "connecting to graph api")
}
if err := gc.RestoreMessages(ctx, dc); err != nil {
return errors.Wrap(err, "restoring service data")
}
op.Status = Successful
return nil, nil
return nil
}

1
src/internal/operations/restore_test.go
View File

@ -56,6 +56,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
operations.OperationOpts{},
test.kw,
test.creds,
"restore-point-id",
nil)
test.errCheck(t, err)
})

3
src/pkg/repository/repository.go
View File

@ -113,7 +113,7 @@ func (r Repository) NewBackup(ctx context.Context, targets []string) (operations
}
// NewRestore generates a restoreOperation runner.
func (r Repository) NewRestore(ctx context.Context, targets []string) (operations.RestoreOperation, error) {
func (r Repository) NewRestore(ctx context.Context, restorePointID string, targets []string) (operations.RestoreOperation, error) {
creds := credentials.M365{
ClientID: r.Account.ClientID,
ClientSecret: r.Account.ClientSecret,
@ -124,5 +124,6 @@ func (r Repository) NewRestore(ctx context.Context, targets []string) (operation
operations.OperationOpts{},
r.dataLayer,
creds,
restorePointID,
targets)
}

2
src/pkg/repository/repository_test.go
View File

@ -201,7 +201,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
r, err := repository.Initialize(ctx, acct, st)
require.NoError(t, err)
ro, err := r.NewRestore(ctx, []string{})
ro, err := r.NewRestore(ctx, "restore-point-id", []string{})
require.NoError(t, err)
require.NotNil(t, ro)
}