update operation/backup errs (#2239)

## Description

Begins updating operations/backup with the new
error handling procedures.  For backwards
compatibility, errors are currently duplicated in
the old stats.Errs and the new Errors struct.

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/operations/backup.go

@@ -144,6 +144,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 
 		err = op.persistResults(startTime, &opStats)
 		if err != nil {
+			op.Errors.Fail(errors.Wrap(err, "persisting backup results"))
 			return
 		}
 
@@ -153,7 +154,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 			opStats.k.SnapshotID,
 			backupDetails.Details())
 		if err != nil {
-			opStats.writeErr = err
+			op.Errors.Fail(errors.Wrap(err, "persisting backup"))
+			opStats.writeErr = op.Errors.Err()
 		}
 	}()
 
@@ -164,21 +166,27 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		reasons,
 		tenantID,
 		uib,
-	)
+		op.Errors)
 	if err != nil {
-		opStats.readErr = errors.Wrap(err, "connecting to M365")
+		op.Errors.Fail(errors.Wrap(err, "collecting manifest heuristics"))
+		opStats.readErr = op.Errors.Err()
+
 		return opStats.readErr
 	}
 
 	gc, err := connectToM365(ctx, op.Selectors, op.account)
 	if err != nil {
-		opStats.readErr = errors.Wrap(err, "connecting to M365")
+		op.Errors.Fail(errors.Wrap(err, "connecting to m365"))
+		opStats.readErr = op.Errors.Err()
+
 		return opStats.readErr
 	}
 
 	cs, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options)
 	if err != nil {
-		opStats.readErr = errors.Wrap(err, "retrieving data to backup")
+		op.Errors.Fail(errors.Wrap(err, "retrieving data to backup"))
+		opStats.readErr = op.Errors.Err()
+
 		return opStats.readErr
 	}
 
@@ -194,7 +202,9 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		op.Results.BackupID,
 		uib && canUseMetaData)
 	if err != nil {
-		opStats.writeErr = errors.Wrap(err, "backing up service data")
+		op.Errors.Fail(errors.Wrap(err, "backing up service data"))
+		opStats.writeErr = op.Errors.Err()
+
 		return opStats.writeErr
 	}
 
@@ -211,12 +221,15 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		toMerge,
 		backupDetails,
 	); err != nil {
-		opStats.writeErr = errors.Wrap(err, "merging backup details")
+		op.Errors.Fail(errors.Wrap(err, "merging backup details"))
+		opStats.writeErr = op.Errors.Err()
+
 		return opStats.writeErr
 	}
 
 	opStats.gc = gc.AwaitStatus()
 
+	// TODO(keepers): remove when fault.Errors handles all iterable error aggregation.
 	if opStats.gc.ErrorCount > 0 {
 		merr := multierror.Append(opStats.readErr, errors.Wrap(opStats.gc.Err, "retrieving data"))
 		opStats.readErr = merr.ErrorOrNil()
@@ -307,7 +320,9 @@ func selectorToReasons(sel selectors.Selector) []kopia.Reason {
 	return reasons
 }
 
-func builderFromReason(tenant string, r kopia.Reason) (*path.Builder, error) {
+func builderFromReason(ctx context.Context, tenant string, r kopia.Reason) (*path.Builder, error) {
+	ctx = clues.Add(ctx, "category", r.Category.String())
+
 	// This is hacky, but we want the path package to format the path the right
 	// way (e.x. proper order for service, category, etc), but we don't care about
 	// the folders after the prefix.
@@ -319,12 +334,7 @@ func builderFromReason(tenant string, r kopia.Reason) (*path.Builder, error) {
 		false,
 	)
 	if err != nil {
-		return nil, errors.Wrapf(
+		return nil, clues.Wrap(err, "building path").WithMap(clues.Values(ctx))
-			err,
-			"building path for service %s category %s",
-			r.Service.String(),
-			r.Category.String(),
-		)
 	}
 
 	return p.ToBuilder().Dir(), nil
@@ -367,7 +377,7 @@ func consumeBackupDataCollections(
 		categories := map[string]struct{}{}
 
 		for _, reason := range m.Reasons {
-			pb, err := builderFromReason(tenantID, reason)
+			pb, err := builderFromReason(ctx, tenantID, reason)
 			if err != nil {
 				return nil, nil, nil, errors.Wrap(err, "getting subtree paths for bases")
 			}
@@ -461,6 +471,8 @@ func mergeDetails(
 	var addedEntries int
 
 	for _, man := range mans {
+		mctx := clues.Add(ctx, "manifest_id", man.ID)
+
 		// For now skip snapshots that aren't complete. We will need to revisit this
 		// when we tackle restartability.
 		if len(man.IncompleteReason) > 0 {
@@ -469,9 +481,11 @@ func mergeDetails(
 
 		bID, ok := man.GetTag(kopia.TagBackupID)
 		if !ok {
-			return errors.Errorf("no backup ID in snapshot manifest with ID %s", man.ID)
+			return clues.New("no backup ID in snapshot manifest").WithMap(clues.Values(mctx))
 		}
 
+		mctx = clues.Add(mctx, "manifest_backup_id", bID)
+
 		_, baseDeets, err := getBackupAndDetailsFromID(
 			ctx,
 			model.StableID(bID),
@@ -479,18 +493,15 @@ func mergeDetails(
 			detailsStore,
 		)
 		if err != nil {
-			return errors.Wrapf(err, "backup fetching base details for backup %s", bID)
+			return clues.New("fetching base details for backup").WithMap(clues.Values(mctx))
 		}
 
 		for _, entry := range baseDeets.Items() {
 			rr, err := path.FromDataLayerPath(entry.RepoRef, true)
 			if err != nil {
-				return errors.Wrapf(
+				return clues.New("parsing base item info path").
-					err,
+					WithMap(clues.Values(mctx)).
-					"parsing base item info path %s in backup %s",
+					With("repo_ref", entry.RepoRef) // todo: pii
-					entry.RepoRef,
-					bID,
-				)
 			}
 
 			// Although this base has an entry it may not be the most recent. Check
@@ -513,11 +524,7 @@ func mergeDetails(
 			// Fixup paths in the item.
 			item := entry.ItemInfo
 			if err := details.UpdateItem(&item, newPath); err != nil {
-				return errors.Wrapf(
+				return clues.New("updating item details").WithMap(clues.Values(mctx))
-					err,
-					"updating item info for entry from backup %s",
-					bID,
-				)
 			}
 
 			// TODO(ashmrtn): This may need updated if we start using this merge
@@ -542,11 +549,9 @@ func mergeDetails(
 	}
 
 	if addedEntries != len(shortRefsFromPrevBackup) {
-		return errors.Errorf(
+		return clues.New("incomplete migration of backup details").
-			"incomplete migration of backup details: found %v of %v expected items",
+			WithMap(clues.Values(ctx)).
-			addedEntries,
+			WithAll("item_count", addedEntries, "expected_item_count", len(shortRefsFromPrevBackup))
-			len(shortRefsFromPrevBackup),
-		)
 	}
 
 	return nil
@@ -568,6 +573,7 @@ func (op *BackupOperation) persistResults(
 	if opStats.readErr != nil || opStats.writeErr != nil {
 		op.Status = Failed
 
+		// TODO(keepers): replace with fault.Errors handling.
 		return multierror.Append(
 			errors.New("errors prevented the operation from processing"),
 			opStats.readErr,
@@ -594,15 +600,18 @@ func (op *BackupOperation) createBackupModels(
 	snapID string,
 	backupDetails *details.Details,
 ) error {
+	ctx = clues.Add(ctx, "snapshot_id", snapID)
+
 	if backupDetails == nil {
-		return errors.New("no backup details to record")
+		return clues.New("no backup details to record").WithMap(clues.Values(ctx))
 	}
 
 	detailsID, err := detailsStore.WriteBackupDetails(ctx, backupDetails)
 	if err != nil {
-		return errors.Wrap(err, "creating backupdetails model")
+		return clues.Wrap(err, "creating backupDetails model").WithMap(clues.Values(ctx))
 	}
 
+	ctx = clues.Add(ctx, "details_id", detailsID)
 	b := backup.New(
 		snapID, detailsID, op.Status.String(),
 		op.Results.BackupID,
@@ -612,9 +621,8 @@ func (op *BackupOperation) createBackupModels(
 		op.Errors,
 	)
 
-	err = op.store.Put(ctx, model.BackupSchema, b)
+	if err = op.store.Put(ctx, model.BackupSchema, b); err != nil {
-	if err != nil {
+		return clues.Wrap(err, "creating backup model").WithMap(clues.Values(ctx))
-		return errors.Wrap(err, "creating backup model")
 	}
 
 	dur := op.Results.CompletedAt.Sub(op.Results.StartedAt)

src/internal/operations/backup_test.go

@@ -432,258 +432,6 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
 	}
 }
 
-func (suite *BackupOpSuite) TestBackupOperation_VerifyDistinctBases() {
-	const user = "a-user"
-
-	table := []struct {
-		name     string
-		input    []*kopia.ManifestEntry
-		errCheck assert.ErrorAssertionFunc
-	}{
-		{
-			name: "SingleManifestMultipleReasons",
-			input: []*kopia.ManifestEntry{
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id1",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EventsCategory,
-						},
-					},
-				},
-			},
-			errCheck: assert.NoError,
-		},
-		{
-			name: "MultipleManifestsDistinctReason",
-			input: []*kopia.ManifestEntry{
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id1",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-					},
-				},
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id2",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EventsCategory,
-						},
-					},
-				},
-			},
-			errCheck: assert.NoError,
-		},
-		{
-			name: "MultipleManifestsSameReason",
-			input: []*kopia.ManifestEntry{
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id1",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-					},
-				},
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id2",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-					},
-				},
-			},
-			errCheck: assert.Error,
-		},
-		{
-			name: "MultipleManifestsSameReasonOneIncomplete",
-			input: []*kopia.ManifestEntry{
-				{
-					Manifest: &snapshot.Manifest{
-						ID: "id1",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-					},
-				},
-				{
-					Manifest: &snapshot.Manifest{
-						ID:               "id2",
-						IncompleteReason: "checkpoint",
-					},
-					Reasons: []kopia.Reason{
-						{
-							ResourceOwner: user,
-							Service:       path.ExchangeService,
-							Category:      path.EmailCategory,
-						},
-					},
-				},
-			},
-			errCheck: assert.NoError,
-		},
-	}
-
-	for _, test := range table {
-		suite.T().Run(test.name, func(t *testing.T) {
-			test.errCheck(t, verifyDistinctBases(test.input))
-		})
-	}
-}
-
-func (suite *BackupOpSuite) TestBackupOperation_CollectMetadata() {
-	var (
-		tenant        = "a-tenant"
-		resourceOwner = "a-user"
-		fileNames     = []string{
-			"delta",
-			"paths",
-		}
-
-		emailDeltaPath = makeMetadataPath(
-			suite.T(),
-			tenant,
-			path.ExchangeService,
-			resourceOwner,
-			path.EmailCategory,
-			fileNames[0],
-		)
-		emailPathsPath = makeMetadataPath(
-			suite.T(),
-			tenant,
-			path.ExchangeService,
-			resourceOwner,
-			path.EmailCategory,
-			fileNames[1],
-		)
-		contactsDeltaPath = makeMetadataPath(
-			suite.T(),
-			tenant,
-			path.ExchangeService,
-			resourceOwner,
-			path.ContactsCategory,
-			fileNames[0],
-		)
-		contactsPathsPath = makeMetadataPath(
-			suite.T(),
-			tenant,
-			path.ExchangeService,
-			resourceOwner,
-			path.ContactsCategory,
-			fileNames[1],
-		)
-	)
-
-	table := []struct {
-		name       string
-		inputMan   *kopia.ManifestEntry
-		inputFiles []string
-		expected   []path.Path
-	}{
-		{
-			name: "SingleReasonSingleFile",
-			inputMan: &kopia.ManifestEntry{
-				Manifest: &snapshot.Manifest{},
-				Reasons: []kopia.Reason{
-					{
-						ResourceOwner: resourceOwner,
-						Service:       path.ExchangeService,
-						Category:      path.EmailCategory,
-					},
-				},
-			},
-			inputFiles: []string{fileNames[0]},
-			expected:   []path.Path{emailDeltaPath},
-		},
-		{
-			name: "SingleReasonMultipleFiles",
-			inputMan: &kopia.ManifestEntry{
-				Manifest: &snapshot.Manifest{},
-				Reasons: []kopia.Reason{
-					{
-						ResourceOwner: resourceOwner,
-						Service:       path.ExchangeService,
-						Category:      path.EmailCategory,
-					},
-				},
-			},
-			inputFiles: fileNames,
-			expected:   []path.Path{emailDeltaPath, emailPathsPath},
-		},
-		{
-			name: "MultipleReasonsMultipleFiles",
-			inputMan: &kopia.ManifestEntry{
-				Manifest: &snapshot.Manifest{},
-				Reasons: []kopia.Reason{
-					{
-						ResourceOwner: resourceOwner,
-						Service:       path.ExchangeService,
-						Category:      path.EmailCategory,
-					},
-					{
-						ResourceOwner: resourceOwner,
-						Service:       path.ExchangeService,
-						Category:      path.ContactsCategory,
-					},
-				},
-			},
-			inputFiles: fileNames,
-			expected: []path.Path{
-				emailDeltaPath,
-				emailPathsPath,
-				contactsDeltaPath,
-				contactsPathsPath,
-			},
-		},
-	}
-
-	for _, test := range table {
-		suite.T().Run(test.name, func(t *testing.T) {
-			ctx, flush := tester.NewContext()
-			defer flush()
-
-			mr := &mockRestorer{}
-
-			_, err := collectMetadata(ctx, mr, test.inputMan, test.inputFiles, tenant)
-			assert.NoError(t, err)
-
-			checkPaths(t, test.expected, mr.gotPaths)
-		})
-	}
-}
-
 func (suite *BackupOpSuite) TestBackupOperation_ConsumeBackupDataCollections_Paths() {
 	var (
 		tenant        = "a-tenant"

src/internal/operations/manifests.go

@@ -3,7 +3,7 @@ package operations
 import (
 	"context"
 
-	multierror "github.com/hashicorp/go-multierror"
+	"github.com/alcionai/clues"
 	"github.com/kopia/kopia/repo/manifest"
 	"github.com/pkg/errors"
 
@@ -12,6 +12,7 @@ import (
 	"github.com/alcionai/corso/src/internal/kopia"
 	"github.com/alcionai/corso/src/internal/model"
 	"github.com/alcionai/corso/src/pkg/backup"
+	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 )
@@ -44,6 +45,7 @@ func produceManifestsAndMetadata(
 	reasons []kopia.Reason,
 	tenantID string,
 	getMetadata bool,
+	errs fault.Adder,
 ) ([]*kopia.ManifestEntry, []data.Collection, bool, error) {
 	var (
 		metadataFiles = graph.AllMetadataFileNames()
@@ -68,12 +70,10 @@ func produceManifestsAndMetadata(
 	//
 	// TODO(ashmrtn): This may need updating if we start sourcing item backup
 	// details from previous snapshots when using kopia-assisted incrementals.
-	if err := verifyDistinctBases(ms); err != nil {
+	if err := verifyDistinctBases(ctx, ms, errs); err != nil {
-		logger.Ctx(ctx).Warnw(
+		logger.Ctx(ctx).With("error", err).Infow(
 			"base snapshot collision, falling back to full backup",
-			"error",
+			clues.Slice(ctx)...)
-			err,
-		)
 
 		return ms, nil, false, nil
 	}
@@ -83,40 +83,41 @@ func produceManifestsAndMetadata(
 			continue
 		}
 
+		mctx := clues.Add(ctx, "manifest_id", man.ID)
+
 		bID, ok := man.GetTag(kopia.TagBackupID)
 		if !ok {
-			return nil, nil, false, errors.New("snapshot manifest missing backup ID")
+			err = clues.New("snapshot manifest missing backup ID").WithMap(clues.Values(mctx))
+			return nil, nil, false, err
 		}
 
-		dID, _, err := gdi.GetDetailsIDFromBackupID(ctx, model.StableID(bID))
+		mctx = clues.Add(mctx, "manifest_backup_id", man.ID)
+
+		dID, _, err := gdi.GetDetailsIDFromBackupID(mctx, model.StableID(bID))
 		if err != nil {
 			// if no backup exists for any of the complete manifests, we want
 			// to fall back to a complete backup.
 			if errors.Is(err, kopia.ErrNotFound) {
-				logger.Ctx(ctx).Infow(
+				logger.Ctx(ctx).Infow("backup missing, falling back to full backup", clues.Slice(mctx)...)
-					"backup missing, falling back to full backup",
-					"backup_id", bID)
-
 				return ms, nil, false, nil
 			}
 
 			return nil, nil, false, errors.Wrap(err, "retrieving prior backup data")
 		}
 
+		mctx = clues.Add(mctx, "manifest_details_id", dID)
+
 		// if no detailsID exists for any of the complete manifests, we want
 		// to fall back to a complete backup.  This is a temporary prevention
 		// mechanism to keep backups from falling into a perpetually bad state.
 		// This makes an assumption that the ID points to a populated set of
 		// details; we aren't doing the work to look them up.
 		if len(dID) == 0 {
-			logger.Ctx(ctx).Infow(
+			logger.Ctx(ctx).Infow("backup missing details ID, falling back to full backup", clues.Slice(mctx)...)
-				"backup missing details ID, falling back to full backup",
-				"backup_id", bID)
-
 			return ms, nil, false, nil
 		}
 
-		colls, err := collectMetadata(ctx, mr, man, metadataFiles, tenantID)
+		colls, err := collectMetadata(mctx, mr, man, metadataFiles, tenantID)
 		if err != nil && !errors.Is(err, kopia.ErrNotFound) {
 			// prior metadata isn't guaranteed to exist.
 			// if it doesn't, we'll just have to do a
@@ -134,9 +135,9 @@ func produceManifestsAndMetadata(
 // of manifests, that each manifest's Reason (owner, service, category) is only
 // included once.  If a reason is duplicated by any two manifests, an error is
 // returned.
-func verifyDistinctBases(mans []*kopia.ManifestEntry) error {
+func verifyDistinctBases(ctx context.Context, mans []*kopia.ManifestEntry, errs fault.Adder) error {
 	var (
-		errs    *multierror.Error
+		failed  bool
 		reasons = map[string]manifest.ID{}
 	)
 
@@ -155,10 +156,11 @@ func verifyDistinctBases(mans []*kopia.ManifestEntry) error {
 			reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String()
 
 			if b, ok := reasons[reasonKey]; ok {
-				errs = multierror.Append(errs, errors.Errorf(
+				failed = true
-					"multiple base snapshots source data for %s %s. IDs: %s, %s",
+
-					reason.Service, reason.Category, b, man.ID,
+				errs.Add(clues.New("manifests have overlapping reasons").
-				))
+					WithMap(clues.Values(ctx)).
+					With("other_manifest_id", b))
 
 				continue
 			}
@@ -167,7 +169,11 @@ func verifyDistinctBases(mans []*kopia.ManifestEntry) error {
 		}
 	}
 
-	return errs.ErrorOrNil()
+	if failed {
+		return clues.New("multiple base snapshots qualify").WithMap(clues.Values(ctx))
+	}
+
+	return nil
 }
 
 // collectMetadata retrieves all metadata files associated with the manifest.
@@ -191,7 +197,9 @@ func collectMetadata(
 					reason.Category,
 					true)
 			if err != nil {
-				return nil, errors.Wrapf(err, "building metadata path")
+				return nil, clues.
+					Wrap(err, "building metadata path").
+					WithAll("metadata_file", fn, "category", reason.Category)
 			}
 
 			paths = append(paths, p)

src/internal/operations/manifests_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/alcionai/corso/src/internal/model"
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/pkg/backup"
+	"github.com/alcionai/corso/src/pkg/fault/mock"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
@@ -400,7 +401,10 @@ func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() {
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
-			err := verifyDistinctBases(test.mans)
+			ctx, flush := tester.NewContext()
+			defer flush()
+
+			err := verifyDistinctBases(ctx, test.mans, mock.NewAdder())
 			test.expect(t, err)
 		})
 	}
@@ -646,6 +650,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 			ctx, flush := tester.NewContext()
 			defer flush()
 
+			ma := mock.NewAdder()
+
 			mans, dcs, b, err := produceManifestsAndMetadata(
 				ctx,
 				&test.mr,
@@ -653,7 +659,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 				test.reasons,
 				tid,
 				test.getMeta,
-			)
+				ma)
 			test.assertErr(t, err)
 			test.assertB(t, b)
 
@@ -683,3 +689,270 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 		})
 	}
 }
+
+// ---------------------------------------------------------------------------
+// older tests
+// ---------------------------------------------------------------------------
+
+type BackupManifestSuite struct {
+	suite.Suite
+}
+
+func TestBackupManifestSuite(t *testing.T) {
+	suite.Run(t, new(BackupOpSuite))
+}
+
+func (suite *BackupManifestSuite) TestBackupOperation_VerifyDistinctBases() {
+	const user = "a-user"
+
+	table := []struct {
+		name     string
+		input    []*kopia.ManifestEntry
+		errCheck assert.ErrorAssertionFunc
+	}{
+		{
+			name: "SingleManifestMultipleReasons",
+			input: []*kopia.ManifestEntry{
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id1",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EventsCategory,
+						},
+					},
+				},
+			},
+			errCheck: assert.NoError,
+		},
+		{
+			name: "MultipleManifestsDistinctReason",
+			input: []*kopia.ManifestEntry{
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id1",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+					},
+				},
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id2",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EventsCategory,
+						},
+					},
+				},
+			},
+			errCheck: assert.NoError,
+		},
+		{
+			name: "MultipleManifestsSameReason",
+			input: []*kopia.ManifestEntry{
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id1",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+					},
+				},
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id2",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+					},
+				},
+			},
+			errCheck: assert.Error,
+		},
+		{
+			name: "MultipleManifestsSameReasonOneIncomplete",
+			input: []*kopia.ManifestEntry{
+				{
+					Manifest: &snapshot.Manifest{
+						ID: "id1",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+					},
+				},
+				{
+					Manifest: &snapshot.Manifest{
+						ID:               "id2",
+						IncompleteReason: "checkpoint",
+					},
+					Reasons: []kopia.Reason{
+						{
+							ResourceOwner: user,
+							Service:       path.ExchangeService,
+							Category:      path.EmailCategory,
+						},
+					},
+				},
+			},
+			errCheck: assert.NoError,
+		},
+	}
+
+	for _, test := range table {
+		suite.T().Run(test.name, func(t *testing.T) {
+			ctx, flush := tester.NewContext()
+			defer flush()
+
+			test.errCheck(t, verifyDistinctBases(ctx, test.input, mock.NewAdder()))
+		})
+	}
+}
+
+func (suite *BackupManifestSuite) TestBackupOperation_CollectMetadata() {
+	var (
+		tenant        = "a-tenant"
+		resourceOwner = "a-user"
+		fileNames     = []string{
+			"delta",
+			"paths",
+		}
+
+		emailDeltaPath = makeMetadataPath(
+			suite.T(),
+			tenant,
+			path.ExchangeService,
+			resourceOwner,
+			path.EmailCategory,
+			fileNames[0],
+		)
+		emailPathsPath = makeMetadataPath(
+			suite.T(),
+			tenant,
+			path.ExchangeService,
+			resourceOwner,
+			path.EmailCategory,
+			fileNames[1],
+		)
+		contactsDeltaPath = makeMetadataPath(
+			suite.T(),
+			tenant,
+			path.ExchangeService,
+			resourceOwner,
+			path.ContactsCategory,
+			fileNames[0],
+		)
+		contactsPathsPath = makeMetadataPath(
+			suite.T(),
+			tenant,
+			path.ExchangeService,
+			resourceOwner,
+			path.ContactsCategory,
+			fileNames[1],
+		)
+	)
+
+	table := []struct {
+		name       string
+		inputMan   *kopia.ManifestEntry
+		inputFiles []string
+		expected   []path.Path
+	}{
+		{
+			name: "SingleReasonSingleFile",
+			inputMan: &kopia.ManifestEntry{
+				Manifest: &snapshot.Manifest{},
+				Reasons: []kopia.Reason{
+					{
+						ResourceOwner: resourceOwner,
+						Service:       path.ExchangeService,
+						Category:      path.EmailCategory,
+					},
+				},
+			},
+			inputFiles: []string{fileNames[0]},
+			expected:   []path.Path{emailDeltaPath},
+		},
+		{
+			name: "SingleReasonMultipleFiles",
+			inputMan: &kopia.ManifestEntry{
+				Manifest: &snapshot.Manifest{},
+				Reasons: []kopia.Reason{
+					{
+						ResourceOwner: resourceOwner,
+						Service:       path.ExchangeService,
+						Category:      path.EmailCategory,
+					},
+				},
+			},
+			inputFiles: fileNames,
+			expected:   []path.Path{emailDeltaPath, emailPathsPath},
+		},
+		{
+			name: "MultipleReasonsMultipleFiles",
+			inputMan: &kopia.ManifestEntry{
+				Manifest: &snapshot.Manifest{},
+				Reasons: []kopia.Reason{
+					{
+						ResourceOwner: resourceOwner,
+						Service:       path.ExchangeService,
+						Category:      path.EmailCategory,
+					},
+					{
+						ResourceOwner: resourceOwner,
+						Service:       path.ExchangeService,
+						Category:      path.ContactsCategory,
+					},
+				},
+			},
+			inputFiles: fileNames,
+			expected: []path.Path{
+				emailDeltaPath,
+				emailPathsPath,
+				contactsDeltaPath,
+				contactsPathsPath,
+			},
+		},
+	}
+
+	for _, test := range table {
+		suite.T().Run(test.name, func(t *testing.T) {
+			ctx, flush := tester.NewContext()
+			defer flush()
+
+			mr := &mockRestorer{}
+
+			_, err := collectMetadata(ctx, mr, test.inputMan, test.inputFiles, tenant)
+			assert.NoError(t, err)
+
+			checkPaths(t, test.expected, mr.gotPaths)
+		})
+	}
+}

src/pkg/fault/fault.go

@@ -96,7 +96,9 @@ func (e *Errors) setErr(err error) *Errors {
 	return e
 }
 
-// TODO: introduce Adder interface
+type Adder interface {
+	Add(err error) *Errors
+}
 
 // Add appends the error to the slice of recoverable and
 // iterated errors (ie: errors.errs).  If failFast is true,

src/pkg/fault/mock/mock.go

@@ -0,0 +1,17 @@
+package mock
+
+import "github.com/alcionai/corso/src/pkg/fault"
+
+// Adder mocks an adder interface for testing.
+type Adder struct {
+	Errs []error
+}
+
+func NewAdder() *Adder {
+	return &Adder{Errs: []error{}}
+}
+
+func (ma *Adder) Add(err error) *fault.Errors {
+	ma.Errs = append(ma.Errs, err)
+	return fault.New(false)
+}