print restore details following successful op (#1030)
## Description Now that GC is tracking the details entries during a restore procedure, it can display the results in the cli. Due to an absence of itemInfo, only the item shortRef is displayed. But we can expand on that from here. ## Type of change - [x] 🌻 Feature ## Issue(s) * #977 ## Test Plan - [x] 💪 Manual - [x] 💚 E2E
This commit is contained in:
parent
fa7f505b33
commit
3719c36bce
@ -213,11 +213,13 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
|
|||||||
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
|
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ro.Run(ctx); err != nil {
|
ds, err := ro.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
return Only(ctx, errors.Wrap(err, "Failed to run Exchange restore"))
|
return Only(ctx, errors.Wrap(err, "Failed to run Exchange restore"))
|
||||||
}
|
}
|
||||||
|
|
||||||
Infof(ctx, "Restored Exchange in %s for user %s.\n", s.Provider, user)
|
Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources())
|
||||||
|
ds.PrintEntries(ctx)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -153,11 +153,13 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
|
|||||||
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore"))
|
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ro.Run(ctx); err != nil {
|
ds, err := ro.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
return Only(ctx, errors.Wrap(err, "Failed to run OneDrive restore"))
|
return Only(ctx, errors.Wrap(err, "Failed to run OneDrive restore"))
|
||||||
}
|
}
|
||||||
|
|
||||||
Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources())
|
Infof(ctx, "Restored OneDrive in %s for user %s.\n", s.Provider, sel.ToPrintable().Resources())
|
||||||
|
ds.PrintEntries(ctx)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -93,16 +93,16 @@ type restoreStats struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run begins a synchronous restore operation.
|
// Run begins a synchronous restore operation.
|
||||||
func (op *RestoreOperation) Run(ctx context.Context) (err error) {
|
func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.Details, err error) {
|
||||||
defer trace.StartRegion(ctx, "operations:restore:run").End()
|
defer trace.StartRegion(ctx, "operations:restore:run").End()
|
||||||
|
|
||||||
startTime := time.Now()
|
var (
|
||||||
|
opStats = restoreStats{
|
||||||
// persist operation results to the model store on exit
|
bytesRead: &stats.ByteCounter{},
|
||||||
opStats := restoreStats{
|
restoreID: uuid.NewString(),
|
||||||
bytesRead: &stats.ByteCounter{},
|
}
|
||||||
restoreID: uuid.NewString(),
|
startTime = time.Now()
|
||||||
}
|
)
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
err = op.persistResults(ctx, startTime, &opStats)
|
err = op.persistResults(ctx, startTime, &opStats)
|
||||||
@ -111,13 +111,12 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// retrieve the restore point details
|
deets, bup, err := op.store.GetDetailsFromBackupID(ctx, op.BackupID)
|
||||||
d, b, err := op.store.GetDetailsFromBackupID(ctx, op.BackupID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = errors.Wrap(err, "getting backup details for restore")
|
err = errors.Wrap(err, "getting backup details for restore")
|
||||||
opStats.readErr = err
|
opStats.readErr = err
|
||||||
|
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
op.bus.Event(
|
op.bus.Event(
|
||||||
@ -126,96 +125,46 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
|
|||||||
map[string]any{
|
map[string]any{
|
||||||
events.StartTime: startTime,
|
events.StartTime: startTime,
|
||||||
events.BackupID: op.BackupID,
|
events.BackupID: op.BackupID,
|
||||||
events.BackupCreateTime: b.CreationTime,
|
events.BackupCreateTime: bup.CreationTime,
|
||||||
events.RestoreID: opStats.restoreID,
|
events.RestoreID: opStats.restoreID,
|
||||||
// TODO: restore options,
|
// TODO: restore options,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
var fds *details.Details
|
paths, err := formatDetailsForRestoration(ctx, op.Selectors, deets)
|
||||||
|
if err != nil {
|
||||||
switch op.Selectors.Service {
|
opStats.readErr = err
|
||||||
case selectors.ServiceExchange:
|
return nil, err
|
||||||
er, err := op.Selectors.ToExchangeRestore()
|
|
||||||
if err != nil {
|
|
||||||
opStats.readErr = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// format the details and retrieve the items from kopia
|
|
||||||
fds = er.Reduce(ctx, d)
|
|
||||||
if len(fds.Entries) == 0 {
|
|
||||||
return selectors.ErrorNoMatchingItems
|
|
||||||
}
|
|
||||||
|
|
||||||
case selectors.ServiceOneDrive:
|
|
||||||
odr, err := op.Selectors.ToOneDriveRestore()
|
|
||||||
if err != nil {
|
|
||||||
opStats.readErr = err
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// format the details and retrieve the items from kopia
|
|
||||||
fds = odr.Reduce(ctx, d)
|
|
||||||
if len(fds.Entries) == 0 {
|
|
||||||
return selectors.ErrorNoMatchingItems
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
return errors.Errorf("Service %s not supported", op.Selectors.Service)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Ctx(ctx).Infof("Discovered %d items in backup %s to restore", len(fds.Entries), op.BackupID)
|
logger.Ctx(ctx).Infof("Discovered %d items in backup %s to restore", len(paths), op.BackupID)
|
||||||
|
|
||||||
fdsPaths := fds.Paths()
|
dcs, err := op.kopia.RestoreMultipleItems(ctx, bup.SnapshotID, paths, opStats.bytesRead)
|
||||||
paths := make([]path.Path, len(fdsPaths))
|
|
||||||
|
|
||||||
var parseErrs *multierror.Error
|
|
||||||
|
|
||||||
for i := range fdsPaths {
|
|
||||||
p, err := path.FromDataLayerPath(fdsPaths[i], true)
|
|
||||||
if err != nil {
|
|
||||||
parseErrs = multierror.Append(
|
|
||||||
parseErrs,
|
|
||||||
errors.Wrap(err, "parsing details entry path"),
|
|
||||||
)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
paths[i] = p
|
|
||||||
}
|
|
||||||
|
|
||||||
dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths, opStats.bytesRead)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = errors.Wrap(err, "retrieving service data")
|
err = errors.Wrap(err, "retrieving service data")
|
||||||
|
opStats.readErr = err
|
||||||
|
|
||||||
parseErrs = multierror.Append(parseErrs, err)
|
return nil, err
|
||||||
opStats.readErr = parseErrs.ErrorOrNil()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
opStats.readErr = parseErrs.ErrorOrNil()
|
|
||||||
opStats.cs = dcs
|
opStats.cs = dcs
|
||||||
opStats.resourceCount = len(data.ResourceOwnerSet(dcs))
|
opStats.resourceCount = len(data.ResourceOwnerSet(dcs))
|
||||||
|
|
||||||
// restore those collections using graph
|
// restore those collections using graph
|
||||||
gc, err := connector.NewGraphConnector(ctx, op.account)
|
gc, err := connector.NewGraphConnector(ctx, op.account)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = errors.Wrap(err, "connecting to graph api")
|
err = errors.Wrap(err, "connecting to microsoft servers")
|
||||||
opStats.writeErr = err
|
opStats.writeErr = err
|
||||||
|
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: return details and print in CLI
|
restoreDetails, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
|
||||||
_, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = errors.Wrap(err, "restoring service data")
|
err = errors.Wrap(err, "restoring service data")
|
||||||
opStats.writeErr = err
|
opStats.writeErr = err
|
||||||
|
|
||||||
return err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
opStats.started = true
|
opStats.started = true
|
||||||
@ -223,7 +172,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
|
|||||||
|
|
||||||
logger.Ctx(ctx).Debug(gc.PrintableStatus())
|
logger.Ctx(ctx).Debug(gc.PrintableStatus())
|
||||||
|
|
||||||
return nil
|
return restoreDetails, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// persists details and statistics about the restore operation.
|
// persists details and statistics about the restore operation.
|
||||||
@ -274,3 +223,64 @@ func (op *RestoreOperation) persistResults(
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// formatDetailsForRestoration reduces the provided detail entries according to the
|
||||||
|
// selector specifications.
|
||||||
|
func formatDetailsForRestoration(
|
||||||
|
ctx context.Context,
|
||||||
|
sel selectors.Selector,
|
||||||
|
deets *details.Details,
|
||||||
|
) ([]path.Path, error) {
|
||||||
|
var fds *details.Details
|
||||||
|
|
||||||
|
switch sel.Service {
|
||||||
|
case selectors.ServiceExchange:
|
||||||
|
er, err := sel.ToExchangeRestore()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// format the details and retrieve the items from kopia
|
||||||
|
fds = er.Reduce(ctx, deets)
|
||||||
|
if len(fds.Entries) == 0 {
|
||||||
|
return nil, selectors.ErrorNoMatchingItems
|
||||||
|
}
|
||||||
|
|
||||||
|
case selectors.ServiceOneDrive:
|
||||||
|
odr, err := sel.ToOneDriveRestore()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// format the details and retrieve the items from kopia
|
||||||
|
fds = odr.Reduce(ctx, deets)
|
||||||
|
if len(fds.Entries) == 0 {
|
||||||
|
return nil, selectors.ErrorNoMatchingItems
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
return nil, errors.Errorf("Service %s not supported", sel.Service)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
errs *multierror.Error
|
||||||
|
fdsPaths = fds.Paths()
|
||||||
|
paths = make([]path.Path, len(fdsPaths))
|
||||||
|
)
|
||||||
|
|
||||||
|
for i := range fdsPaths {
|
||||||
|
p, err := path.FromDataLayerPath(fdsPaths[i], true)
|
||||||
|
if err != nil {
|
||||||
|
errs = multierror.Append(
|
||||||
|
errs,
|
||||||
|
errors.Wrap(err, "parsing details entry path"),
|
||||||
|
)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
paths[i] = p
|
||||||
|
}
|
||||||
|
|
||||||
|
return paths, nil
|
||||||
|
}
|
||||||
|
|||||||
@ -241,9 +241,13 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
|
|||||||
mb)
|
mb)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, ro.Run(ctx), "restoreOp.Run()")
|
ds, err := ro.Run(ctx)
|
||||||
|
|
||||||
|
require.NoError(t, err, "restoreOp.Run()")
|
||||||
require.NotEmpty(t, ro.Results, "restoreOp results")
|
require.NotEmpty(t, ro.Results, "restoreOp results")
|
||||||
|
require.NotNil(t, ds, "restored details")
|
||||||
assert.Equal(t, ro.Status, Completed, "restoreOp status")
|
assert.Equal(t, ro.Status, Completed, "restoreOp status")
|
||||||
|
assert.Equal(t, ro.Results.ItemsWritten, len(ds.Entries), "count of items written matches restored entries in details")
|
||||||
assert.Less(t, 0, ro.Results.ItemsRead, "restore items read")
|
assert.Less(t, 0, ro.Results.ItemsRead, "restore items read")
|
||||||
assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written")
|
assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written")
|
||||||
assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read")
|
assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read")
|
||||||
@ -276,7 +280,10 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
|
|||||||
dest,
|
dest,
|
||||||
mb)
|
mb)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results")
|
|
||||||
|
ds, err := ro.Run(ctx)
|
||||||
|
require.Error(t, err, "restoreOp.Run() should have errored")
|
||||||
|
require.Nil(t, ds, "restoreOp.Run() should not produce details")
|
||||||
assert.Zero(t, ro.Results.ResourceOwners, "resource owners")
|
assert.Zero(t, ro.Results.ResourceOwners, "resource owners")
|
||||||
assert.Zero(t, ro.Results.BytesRead, "bytes read")
|
assert.Zero(t, ro.Results.BytesRead, "bytes read")
|
||||||
assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events")
|
assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events")
|
||||||
|
|||||||
@ -165,16 +165,19 @@ func runRestoreLoadTest(
|
|||||||
t.Run("restore_"+name, func(t *testing.T) {
|
t.Run("restore_"+name, func(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
err error
|
err error
|
||||||
|
ds *details.Details
|
||||||
labels = pprof.Labels("restore_load_test", name)
|
labels = pprof.Labels("restore_load_test", name)
|
||||||
)
|
)
|
||||||
|
|
||||||
pprof.Do(ctx, labels, func(ctx context.Context) {
|
pprof.Do(ctx, labels, func(ctx context.Context) {
|
||||||
err = r.Run(ctx)
|
ds, err = r.Run(ctx)
|
||||||
})
|
})
|
||||||
|
|
||||||
require.NoError(t, err, "running restore")
|
require.NoError(t, err, "running restore")
|
||||||
require.NotEmpty(t, r.Results, "has results after run")
|
require.NotEmpty(t, r.Results, "has results after run")
|
||||||
|
require.NotNil(t, ds, "has restored details")
|
||||||
assert.Equal(t, r.Status, operations.Completed, "restore status")
|
assert.Equal(t, r.Status, operations.Completed, "restore status")
|
||||||
|
assert.Equal(t, r.Results.ItemsWritten, len(ds.Entries), "count of items written matches restored entries in details")
|
||||||
assert.Less(t, 0, r.Results.ItemsRead, "items read")
|
assert.Less(t, 0, r.Results.ItemsRead, "items read")
|
||||||
assert.Less(t, 0, r.Results.ItemsWritten, "items written")
|
assert.Less(t, 0, r.Results.ItemsWritten, "items written")
|
||||||
assert.Less(t, 0, r.Results.ResourceOwners, "resource owners")
|
assert.Less(t, 0, r.Results.ResourceOwners, "resource owners")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user