print restore details following successful op (#1030)

## Description

Now that GC is tracking the details entries during
a restore procedure, it can display the results in
the cli.  Due to an absence of itemInfo, only the
item shortRef is displayed.  But we can expand
on that from here.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #977

## Test Plan

- [x] 💪 Manual
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-10-04 14:43:38 -06:00 committed by GitHub
parent fa7f505b33
commit 3719c36bce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 105 additions and 81 deletions

View File

@ -213,11 +213,13 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore")) return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
} }
if err := ro.Run(ctx); err != nil { ds, err := ro.Run(ctx)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to run Exchange restore")) return Only(ctx, errors.Wrap(err, "Failed to run Exchange restore"))
} }
Infof(ctx, "Restored Exchange in %s for user %s.\n", s.Provider, user) Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources())
ds.PrintEntries(ctx)
return nil return nil
} }

View File

@ -153,11 +153,13 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore")) return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore"))
} }
if err := ro.Run(ctx); err != nil { ds, err := ro.Run(ctx)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to run OneDrive restore")) return Only(ctx, errors.Wrap(err, "Failed to run OneDrive restore"))
} }
Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources()) Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources())
ds.PrintEntries(ctx)
return nil return nil
} }

View File

@ -93,16 +93,16 @@ type restoreStats struct {
} }
// Run begins a synchronous restore operation. // Run begins a synchronous restore operation.
func (op *RestoreOperation) Run(ctx context.Context) (err error) { func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.Details, err error) {
defer trace.StartRegion(ctx, "operations:restore:run").End() defer trace.StartRegion(ctx, "operations:restore:run").End()
startTime := time.Now() var (
opStats = restoreStats{
// persist operation results to the model store on exit bytesRead: &stats.ByteCounter{},
opStats := restoreStats{ restoreID: uuid.NewString(),
bytesRead: &stats.ByteCounter{}, }
restoreID: uuid.NewString(), startTime = time.Now()
} )
defer func() { defer func() {
err = op.persistResults(ctx, startTime, &opStats) err = op.persistResults(ctx, startTime, &opStats)
@ -111,13 +111,12 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
} }
}() }()
// retrieve the restore point details deets, bup, err := op.store.GetDetailsFromBackupID(ctx, op.BackupID)
d, b, err := op.store.GetDetailsFromBackupID(ctx, op.BackupID)
if err != nil { if err != nil {
err = errors.Wrap(err, "getting backup details for restore") err = errors.Wrap(err, "getting backup details for restore")
opStats.readErr = err opStats.readErr = err
return err return nil, err
} }
op.bus.Event( op.bus.Event(
@ -126,96 +125,46 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
map[string]any{ map[string]any{
events.StartTime: startTime, events.StartTime: startTime,
events.BackupID: op.BackupID, events.BackupID: op.BackupID,
events.BackupCreateTime: b.CreationTime, events.BackupCreateTime: bup.CreationTime,
events.RestoreID: opStats.restoreID, events.RestoreID: opStats.restoreID,
// TODO: restore options, // TODO: restore options,
}, },
) )
var fds *details.Details paths, err := formatDetailsForRestoration(ctx, op.Selectors, deets)
if err != nil {
switch op.Selectors.Service { opStats.readErr = err
case selectors.ServiceExchange: return nil, err
er, err := op.Selectors.ToExchangeRestore()
if err != nil {
opStats.readErr = err
return err
}
// format the details and retrieve the items from kopia
fds = er.Reduce(ctx, d)
if len(fds.Entries) == 0 {
return selectors.ErrorNoMatchingItems
}
case selectors.ServiceOneDrive:
odr, err := op.Selectors.ToOneDriveRestore()
if err != nil {
opStats.readErr = err
return err
}
// format the details and retrieve the items from kopia
fds = odr.Reduce(ctx, d)
if len(fds.Entries) == 0 {
return selectors.ErrorNoMatchingItems
}
default:
return errors.Errorf("Service %s not supported", op.Selectors.Service)
} }
logger.Ctx(ctx).Infof("Discovered %d items in backup %s to restore", len(fds.Entries), op.BackupID) logger.Ctx(ctx).Infof("Discovered %d items in backup %s to restore", len(paths), op.BackupID)
fdsPaths := fds.Paths() dcs, err := op.kopia.RestoreMultipleItems(ctx, bup.SnapshotID, paths, opStats.bytesRead)
paths := make([]path.Path, len(fdsPaths))
var parseErrs *multierror.Error
for i := range fdsPaths {
p, err := path.FromDataLayerPath(fdsPaths[i], true)
if err != nil {
parseErrs = multierror.Append(
parseErrs,
errors.Wrap(err, "parsing details entry path"),
)
continue
}
paths[i] = p
}
dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths, opStats.bytesRead)
if err != nil { if err != nil {
err = errors.Wrap(err, "retrieving service data") err = errors.Wrap(err, "retrieving service data")
opStats.readErr = err
parseErrs = multierror.Append(parseErrs, err) return nil, err
opStats.readErr = parseErrs.ErrorOrNil()
return err
} }
opStats.readErr = parseErrs.ErrorOrNil()
opStats.cs = dcs opStats.cs = dcs
opStats.resourceCount = len(data.ResourceOwnerSet(dcs)) opStats.resourceCount = len(data.ResourceOwnerSet(dcs))
// restore those collections using graph // restore those collections using graph
gc, err := connector.NewGraphConnector(ctx, op.account) gc, err := connector.NewGraphConnector(ctx, op.account)
if err != nil { if err != nil {
err = errors.Wrap(err, "connecting to graph api") err = errors.Wrap(err, "connecting to microsoft servers")
opStats.writeErr = err opStats.writeErr = err
return err return nil, err
} }
// TODO: return details and print in CLI restoreDetails, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
_, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
if err != nil { if err != nil {
err = errors.Wrap(err, "restoring service data") err = errors.Wrap(err, "restoring service data")
opStats.writeErr = err opStats.writeErr = err
return err return nil, err
} }
opStats.started = true opStats.started = true
@ -223,7 +172,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
logger.Ctx(ctx).Debug(gc.PrintableStatus()) logger.Ctx(ctx).Debug(gc.PrintableStatus())
return nil return restoreDetails, nil
} }
// persists details and statistics about the restore operation. // persists details and statistics about the restore operation.
@ -274,3 +223,64 @@ func (op *RestoreOperation) persistResults(
return nil return nil
} }
// formatDetailsForRestoration reduces the provided detail entries according to the
// selector specifications.
func formatDetailsForRestoration(
ctx context.Context,
sel selectors.Selector,
deets *details.Details,
) ([]path.Path, error) {
var fds *details.Details
switch sel.Service {
case selectors.ServiceExchange:
er, err := sel.ToExchangeRestore()
if err != nil {
return nil, err
}
// format the details and retrieve the items from kopia
fds = er.Reduce(ctx, deets)
if len(fds.Entries) == 0 {
return nil, selectors.ErrorNoMatchingItems
}
case selectors.ServiceOneDrive:
odr, err := sel.ToOneDriveRestore()
if err != nil {
return nil, err
}
// format the details and retrieve the items from kopia
fds = odr.Reduce(ctx, deets)
if len(fds.Entries) == 0 {
return nil, selectors.ErrorNoMatchingItems
}
default:
return nil, errors.Errorf("Service %s not supported", sel.Service)
}
var (
errs *multierror.Error
fdsPaths = fds.Paths()
paths = make([]path.Path, len(fdsPaths))
)
for i := range fdsPaths {
p, err := path.FromDataLayerPath(fdsPaths[i], true)
if err != nil {
errs = multierror.Append(
errs,
errors.Wrap(err, "parsing details entry path"),
)
continue
}
paths[i] = p
}
return paths, nil
}

View File

@ -241,9 +241,13 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
mb) mb)
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, ro.Run(ctx), "restoreOp.Run()") ds, err := ro.Run(ctx)
require.NoError(t, err, "restoreOp.Run()")
require.NotEmpty(t, ro.Results, "restoreOp results") require.NotEmpty(t, ro.Results, "restoreOp results")
require.NotNil(t, ds, "restored details")
assert.Equal(t, ro.Status, Completed, "restoreOp status") assert.Equal(t, ro.Status, Completed, "restoreOp status")
assert.Equal(t, ro.Results.ItemsWritten, len(ds.Entries), "count of items written matches restored entries in details")
assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") assert.Less(t, 0, ro.Results.ItemsRead, "restore items read")
assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written") assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written")
assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read")
@ -276,7 +280,10 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
dest, dest,
mb) mb)
require.NoError(t, err) require.NoError(t, err)
require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results")
ds, err := ro.Run(ctx)
require.Error(t, err, "restoreOp.Run() should have errored")
require.Nil(t, ds, "restoreOp.Run() should not produce details")
assert.Zero(t, ro.Results.ResourceOwners, "resource owners") assert.Zero(t, ro.Results.ResourceOwners, "resource owners")
assert.Zero(t, ro.Results.BytesRead, "bytes read") assert.Zero(t, ro.Results.BytesRead, "bytes read")
assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events")

View File

@ -165,16 +165,19 @@ func runRestoreLoadTest(
t.Run("restore_"+name, func(t *testing.T) { t.Run("restore_"+name, func(t *testing.T) {
var ( var (
err error err error
ds *details.Details
labels = pprof.Labels("restore_load_test", name) labels = pprof.Labels("restore_load_test", name)
) )
pprof.Do(ctx, labels, func(ctx context.Context) { pprof.Do(ctx, labels, func(ctx context.Context) {
err = r.Run(ctx) ds, err = r.Run(ctx)
}) })
require.NoError(t, err, "running restore") require.NoError(t, err, "running restore")
require.NotEmpty(t, r.Results, "has results after run") require.NotEmpty(t, r.Results, "has results after run")
require.NotNil(t, ds, "has restored details")
assert.Equal(t, r.Status, operations.Completed, "restore status") assert.Equal(t, r.Status, operations.Completed, "restore status")
assert.Equal(t, r.Results.ItemsWritten, len(ds.Entries), "count of items written matches restored entries in details")
assert.Less(t, 0, r.Results.ItemsRead, "items read") assert.Less(t, 0, r.Results.ItemsRead, "items read")
assert.Less(t, 0, r.Results.ItemsWritten, "items written") assert.Less(t, 0, r.Results.ItemsWritten, "items written")
assert.Less(t, 0, r.Results.ResourceOwners, "resource owners") assert.Less(t, 0, r.Results.ResourceOwners, "resource owners")