diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 8e40d820f..e0df72d55 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -2,6 +2,7 @@ package operations import ( "context" + "runtime/debug" "time" "github.com/alcionai/clues" @@ -106,7 +107,13 @@ type detailsWriter interface { // --------------------------------------------------------------------------- // Run begins a synchronous backup operation. -func (op *BackupOperation) Run(ctx context.Context) error { +func (op *BackupOperation) Run(ctx context.Context) (err error) { + defer func() { + if r := recover(); r != nil { + err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack()) + } + }() + ctx, end := D.Span(ctx, "operations:backup:run") defer func() { end() @@ -189,6 +196,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) { op.Errors.Fail(errors.Wrap(err, "collecting manifest heuristics")) opStats.readErr = op.Errors.Err() + logger.Ctx(ctx).With("err", err).Errorw("producing manifests and metadata", clues.InErr(err).Slice()...) + return opStats.readErr } @@ -197,6 +206,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) { op.Errors.Fail(errors.Wrap(err, "connecting to m365")) opStats.readErr = op.Errors.Err() + logger.Ctx(ctx).With("err", err).Errorw("connectng to m365", clues.InErr(err).Slice()...) + return opStats.readErr } @@ -205,6 +216,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) { op.Errors.Fail(errors.Wrap(err, "retrieving data to backup")) opStats.readErr = op.Errors.Err() + logger.Ctx(ctx).With("err", err).Errorw("producing backup data collections", clues.InErr(err).Slice()...) + return opStats.readErr } @@ -223,6 +236,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) { op.Errors.Fail(errors.Wrap(err, "backing up service data")) opStats.writeErr = op.Errors.Err() + logger.Ctx(ctx).With("err", err).Errorw("persisting collection backups", clues.InErr(err).Slice()...) + return opStats.writeErr } @@ -237,6 +252,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) { op.Errors.Fail(errors.Wrap(err, "merging backup details")) opStats.writeErr = op.Errors.Err() + logger.Ctx(ctx).With("err", err).Errorw("merging details", clues.InErr(err).Slice()...) + return opStats.writeErr } @@ -589,15 +606,21 @@ func (op *BackupOperation) persistResults( opStats.writeErr) } + op.Results.BytesRead = opStats.k.TotalHashedBytes + op.Results.BytesUploaded = opStats.k.TotalUploadedBytes + op.Results.ItemsWritten = opStats.k.TotalFileCount + op.Results.ResourceOwners = opStats.resourceCount + + if opStats.gc == nil { + op.Status = Failed + return errors.New("data population never completed") + } + if opStats.readErr == nil && opStats.writeErr == nil && opStats.gc.Successful == 0 { op.Status = NoData } - op.Results.BytesRead = opStats.k.TotalHashedBytes - op.Results.BytesUploaded = opStats.k.TotalUploadedBytes op.Results.ItemsRead = opStats.gc.Successful - op.Results.ItemsWritten = opStats.k.TotalFileCount - op.Results.ResourceOwners = opStats.resourceCount return nil } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 30bc303a9..aa9229336 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -3,6 +3,7 @@ package operations import ( "context" "fmt" + "runtime/debug" "time" "github.com/alcionai/clues" @@ -106,6 +107,12 @@ type restorer interface { // Run begins a synchronous restore operation. func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.Details, err error) { + defer func() { + if r := recover(); r != nil { + err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack()) + } + }() + ctx, end := D.Span(ctx, "operations:restore:run") defer func() { end() @@ -250,14 +257,20 @@ func (op *RestoreOperation) persistResults( opStats.writeErr) } + op.Results.BytesRead = opStats.bytesRead.NumBytes + op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count + op.Results.ResourceOwners = opStats.resourceCount + + if opStats.gc == nil { + op.Status = Failed + return errors.New("data restoration never completed") + } + if opStats.readErr == nil && opStats.writeErr == nil && opStats.gc.Successful == 0 { op.Status = NoData } - op.Results.BytesRead = opStats.bytesRead.NumBytes - op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count op.Results.ItemsWritten = opStats.gc.Successful - op.Results.ResourceOwners = opStats.resourceCount dur := op.Results.CompletedAt.Sub(op.Results.StartedAt) diff --git a/src/pkg/fault/fault.go b/src/pkg/fault/fault.go index 69017029c..b8f57108b 100644 --- a/src/pkg/fault/fault.go +++ b/src/pkg/fault/fault.go @@ -87,11 +87,12 @@ func (e *Errors) Fail(err error) *Errors { // setErr handles setting errors.err. Sync locking gets // handled upstream of this call. func (e *Errors) setErr(err error) *Errors { - if e.err != nil { - return e.addErr(err) + if e.err == nil { + e.err = err + return e } - e.err = err + e.errs = append(e.errs, err) return e } diff --git a/src/pkg/fault/fault_test.go b/src/pkg/fault/fault_test.go index 3f5ad127c..8fb2981d7 100644 --- a/src/pkg/fault/fault_test.go +++ b/src/pkg/fault/fault_test.go @@ -73,6 +73,8 @@ func (suite *FaultErrorsUnitSuite) TestErr() { suite.T().Run(test.name, func(t *testing.T) { n := fault.New(test.failFast) require.NotNil(t, n) + require.NoError(t, n.Err()) + require.Empty(t, n.Errs()) e := n.Fail(test.fail) require.NotNil(t, e) @@ -90,6 +92,8 @@ func (suite *FaultErrorsUnitSuite) TestFail() { n := fault.New(false) require.NotNil(t, n) + require.NoError(t, n.Err()) + require.Empty(t, n.Errs()) n.Fail(assert.AnError) assert.Error(t, n.Err())