handle error from bu.backupCollections (#2386)

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🐛 Bugfix

## Test Plan

- [x] 💚 E2E

src/internal/operations/backup.go

@@ -2,6 +2,7 @@ package operations
 
 import (
 	"context"
+	"runtime/debug"
 	"time"
 
 	"github.com/alcionai/clues"
@@ -106,7 +107,13 @@ type detailsWriter interface {
 // ---------------------------------------------------------------------------
 
 // Run begins a synchronous backup operation.
-func (op *BackupOperation) Run(ctx context.Context) error {
+func (op *BackupOperation) Run(ctx context.Context) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack())
+		}
+	}()
+
 	ctx, end := D.Span(ctx, "operations:backup:run")
 	defer func() {
 		end()
@@ -160,6 +167,12 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 
 	// persist operation results to the model store on exit
 	defer func() {
+		// panic recovery here prevents additional errors in op.persistResults()
+		if r := recover(); r != nil {
+			err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack())
+			return
+		}
+
 		err = op.persistResults(startTime, &opStats)
 		if err != nil {
 			op.Errors.Fail(errors.Wrap(err, "persisting backup results"))
@@ -189,6 +202,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 		op.Errors.Fail(errors.Wrap(err, "collecting manifest heuristics"))
 		opStats.readErr = op.Errors.Err()
 
+		logger.Ctx(ctx).With("err", err).Errorw("producing manifests and metadata", clues.InErr(err).Slice()...)
+
 		return opStats.readErr
 	}
 
@@ -197,6 +212,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 		op.Errors.Fail(errors.Wrap(err, "connecting to m365"))
 		opStats.readErr = op.Errors.Err()
 
+		logger.Ctx(ctx).With("err", err).Errorw("connectng to m365", clues.InErr(err).Slice()...)
+
 		return opStats.readErr
 	}
 
@@ -205,6 +222,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 		op.Errors.Fail(errors.Wrap(err, "retrieving data to backup"))
 		opStats.readErr = op.Errors.Err()
 
+		logger.Ctx(ctx).With("err", err).Errorw("producing backup data collections", clues.InErr(err).Slice()...)
+
 		return opStats.readErr
 	}
 
@@ -223,6 +242,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 		op.Errors.Fail(errors.Wrap(err, "backing up service data"))
 		opStats.writeErr = op.Errors.Err()
 
+		logger.Ctx(ctx).With("err", err).Errorw("persisting collection backups", clues.InErr(err).Slice()...)
+
 		return opStats.writeErr
 	}
 
@@ -237,6 +258,8 @@ func (op *BackupOperation) do(ctx context.Context) (err error) {
 		op.Errors.Fail(errors.Wrap(err, "merging backup details"))
 		opStats.writeErr = op.Errors.Err()
 
+		logger.Ctx(ctx).With("err", err).Errorw("merging details", clues.InErr(err).Slice()...)
+
 		return opStats.writeErr
 	}
 
@@ -428,24 +451,22 @@ func consumeBackupDataCollections(
 		cs,
 		nil,
 		tags,
-		isIncremental,
-	)
+		isIncremental)
+	if err != nil {
+		if kopiaStats == nil {
+			return nil, nil, nil, err
+		}
+
+		return nil, nil, nil, errors.Wrapf(
+			err,
+			"kopia snapshot failed with %v catastrophic errors and %v ignored errors",
+			kopiaStats.ErrorCount, kopiaStats.IgnoredErrorCount)
+	}
 
 	if kopiaStats.ErrorCount > 0 || kopiaStats.IgnoredErrorCount > 0 {
-		if err != nil {
-			err = errors.Wrapf(
-				err,
-				"kopia snapshot failed with %v catastrophic errors and %v ignored errors",
-				kopiaStats.ErrorCount,
-				kopiaStats.IgnoredErrorCount,
-			)
-		} else {
-			err = errors.Errorf(
-				"kopia snapshot failed with %v catastrophic errors and %v ignored errors",
-				kopiaStats.ErrorCount,
-				kopiaStats.IgnoredErrorCount,
-			)
-		}
+		err = errors.Errorf(
+			"kopia snapshot failed with %v catastrophic errors and %v ignored errors",
+			kopiaStats.ErrorCount, kopiaStats.IgnoredErrorCount)
 	}
 
 	return kopiaStats, deets, itemsSourcedFromBase, err
@@ -589,15 +610,21 @@ func (op *BackupOperation) persistResults(
 			opStats.writeErr)
 	}
 
+	op.Results.BytesRead = opStats.k.TotalHashedBytes
+	op.Results.BytesUploaded = opStats.k.TotalUploadedBytes
+	op.Results.ItemsWritten = opStats.k.TotalFileCount
+	op.Results.ResourceOwners = opStats.resourceCount
+
+	if opStats.gc == nil {
+		op.Status = Failed
+		return errors.New("data population never completed")
+	}
+
 	if opStats.readErr == nil && opStats.writeErr == nil && opStats.gc.Successful == 0 {
 		op.Status = NoData
 	}
 
-	op.Results.BytesRead = opStats.k.TotalHashedBytes
-	op.Results.BytesUploaded = opStats.k.TotalUploadedBytes
 	op.Results.ItemsRead = opStats.gc.Successful
-	op.Results.ItemsWritten = opStats.k.TotalFileCount
-	op.Results.ResourceOwners = opStats.resourceCount
 
 	return nil
 }

src/internal/operations/restore.go

@@ -3,6 +3,7 @@ package operations
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"time"
 
 	"github.com/alcionai/clues"
@@ -106,6 +107,12 @@ type restorer interface {
 
 // Run begins a synchronous restore operation.
 func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.Details, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack())
+		}
+	}()
+
 	ctx, end := D.Span(ctx, "operations:restore:run")
 	defer func() {
 		end()
@@ -143,6 +150,12 @@ func (op *RestoreOperation) do(ctx context.Context) (restoreDetails *details.Det
 	)
 
 	defer func() {
+		// panic recovery here prevents additional errors in op.persistResults()
+		if r := recover(); r != nil {
+			err = clues.Wrap(r.(error), "panic recovery").WithClues(ctx).With("stacktrace", debug.Stack())
+			return
+		}
+
 		err = op.persistResults(ctx, startTime, &opStats)
 		if err != nil {
 			return
@@ -250,14 +263,20 @@ func (op *RestoreOperation) persistResults(
 			opStats.writeErr)
 	}
 
+	op.Results.BytesRead = opStats.bytesRead.NumBytes
+	op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count
+	op.Results.ResourceOwners = opStats.resourceCount
+
+	if opStats.gc == nil {
+		op.Status = Failed
+		return errors.New("data restoration never completed")
+	}
+
 	if opStats.readErr == nil && opStats.writeErr == nil && opStats.gc.Successful == 0 {
 		op.Status = NoData
 	}
 
-	op.Results.BytesRead = opStats.bytesRead.NumBytes
-	op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count
 	op.Results.ItemsWritten = opStats.gc.Successful
-	op.Results.ResourceOwners = opStats.resourceCount
 
 	dur := op.Results.CompletedAt.Sub(op.Results.StartedAt)
 

src/pkg/fault/fault.go

@@ -87,11 +87,12 @@ func (e *Errors) Fail(err error) *Errors {
 // setErr handles setting errors.err.  Sync locking gets
 // handled upstream of this call.
 func (e *Errors) setErr(err error) *Errors {
-	if e.err != nil {
-		return e.addErr(err)
+	if e.err == nil {
+		e.err = err
+		return e
 	}
 
-	e.err = err
+	e.errs = append(e.errs, err)
 
 	return e
 }

src/pkg/fault/fault_test.go

@@ -73,6 +73,8 @@ func (suite *FaultErrorsUnitSuite) TestErr() {
 		suite.T().Run(test.name, func(t *testing.T) {
 			n := fault.New(test.failFast)
 			require.NotNil(t, n)
+			require.NoError(t, n.Err())
+			require.Empty(t, n.Errs())
 
 			e := n.Fail(test.fail)
 			require.NotNil(t, e)
@@ -90,6 +92,8 @@ func (suite *FaultErrorsUnitSuite) TestFail() {
 
 	n := fault.New(false)
 	require.NotNil(t, n)
+	require.NoError(t, n.Err())
+	require.Empty(t, n.Errs())
 
 	n.Fail(assert.AnError)
 	assert.Error(t, n.Err())