optionally look up and union fallback manifests (#3057)

Fallback manifests allow us to find manifests and
metadata from snapshots that utilized earlier
versions of kopia Reasons, primarily across
resource owner deltas.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #2825

#### Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test

src/internal/operations/backup.go

@@ -229,7 +229,11 @@ func (op *BackupOperation) do(
 	detailsStore streamstore.Streamer,
 	backupID model.StableID,
 ) (*details.Builder, error) {
-	reasons := selectorToReasons(op.Selectors)
+	var (
+		reasons         = selectorToReasons(op.Selectors, false)
+		fallbackReasons = makeFallbackReasons(op.Selectors)
+	)
+
 	logger.Ctx(ctx).With("selectors", op.Selectors).Info("backing up selection")
 
 	// should always be 1, since backups are 1:1 with resourceOwners.
@@ -239,7 +243,7 @@ func (op *BackupOperation) do(
 		ctx,
 		op.kopia,
 		op.store,
-		reasons,
+		reasons, fallbackReasons,
 		op.account.ID(),
 		op.incremental,
 		op.Errors)
@@ -297,6 +301,14 @@ func (op *BackupOperation) do(
 	return deets, nil
 }
 
+func makeFallbackReasons(sel selectors.Selector) []kopia.Reason {
+	if sel.PathService() != path.SharePointService {
+		return selectorToReasons(sel, true)
+	}
+
+	return nil
+}
+
 // checker to see if conditions are correct for incremental backup behavior such as
 // retrieving metadata like delta tokens and previous paths.
 func useIncrementalBackup(sel selectors.Selector, opts control.Options) bool {
@@ -338,7 +350,7 @@ func produceBackupDataCollections(
 // Consumer funcs
 // ---------------------------------------------------------------------------
 
-func selectorToReasons(sel selectors.Selector) []kopia.Reason {
+func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.Reason {
 	service := sel.PathService()
 	reasons := []kopia.Reason{}
 
@@ -349,10 +361,15 @@ func selectorToReasons(sel selectors.Selector) []kopia.Reason {
 		return nil
 	}
 
+	owner := sel.DiscreteOwner
+	if useOwnerNameForID {
+		owner = sel.DiscreteOwnerName
+	}
+
 	for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} {
 		for _, cat := range sl {
 			reasons = append(reasons, kopia.Reason{
-				ResourceOwner: sel.DiscreteOwner,
+				ResourceOwner: owner,
 				Service:       service,
 				Category:      cat,
 			})

src/internal/operations/manifests.go

@@ -43,24 +43,52 @@ func produceManifestsAndMetadata(
 	ctx context.Context,
 	mr manifestRestorer,
 	gb getBackuper,
-	reasons []kopia.Reason,
+	reasons, fallbackReasons []kopia.Reason,
 	tenantID string,
 	getMetadata bool,
 	errs *fault.Bus,
 ) ([]*kopia.ManifestEntry, []data.RestoreCollection, bool, error) {
 	var (
+		tags          = map[string]string{kopia.TagBackupCategory: ""}
 		metadataFiles = graph.AllMetadataFileNames()
 		collections   []data.RestoreCollection
 	)
 
-	ms, err := mr.FetchPrevSnapshotManifests(
+	ms, err := mr.FetchPrevSnapshotManifests(ctx, reasons, tags)
-		ctx,
-		reasons,
-		map[string]string{kopia.TagBackupCategory: ""})
 	if err != nil {
-		return nil, nil, false, err
+		return nil, nil, false, clues.Wrap(err, "looking up prior snapshots")
 	}
 
+	// We only need to check that we have 1:1 reason:base if we're doing an
+	// incremental with associated metadata. This ensures that we're only sourcing
+	// data from a single Point-In-Time (base) for each incremental backup.
+	//
+	// TODO(ashmrtn): This may need updating if we start sourcing item backup
+	// details from previous snapshots when using kopia-assisted incrementals.
+	if err := verifyDistinctBases(ctx, ms); err != nil {
+		logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup")
+		return ms, nil, false, nil
+	}
+
+	fbms, err := mr.FetchPrevSnapshotManifests(ctx, fallbackReasons, tags)
+	if err != nil {
+		return nil, nil, false, clues.Wrap(err, "looking up prior snapshots under alternate id")
+	}
+
+	// Also check distinct bases for the fallback set.
+	if err := verifyDistinctBases(ctx, fbms); err != nil {
+		logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup")
+		return ms, nil, false, nil
+	}
+
+	// one of three cases can occur when retrieving backups across reason migrations:
+	// 1. the current reasons don't match any manifests, and we use the fallback to
+	// look up the previous reason version.
+	// 2. the current reasons only contain an incomplete manifest, and the fallback
+	// can find a complete manifest.
+	// 3. the current reasons contain all the necessary manifests.
+	ms = unionManifests(reasons, ms, fbms)
+
 	if !getMetadata {
 		return ms, nil, false, nil
 	}
@@ -137,6 +165,98 @@ func produceManifestsAndMetadata(
 	return ms, collections, true, err
 }
 
+// unionManifests reduces the two manifest slices into a single slice.
+// Assumes fallback represents a prior manifest version (across some migration
+// that disrupts manifest lookup), and that mans contains the current version.
+// Also assumes the mans slice will have, at most, one complete and one incomplete
+// manifest per service+category tuple.
+//
+// Selection priority, for each reason, follows these rules:
+// 1. If the mans manifest is complete, ignore fallback manifests for that reason.
+// 2. If the mans manifest is only incomplete, look for a matching complete manifest in fallbacks.
+// 3. If mans has no entry for a reason, look for both complete and incomplete fallbacks.
+func unionManifests(
+	reasons []kopia.Reason,
+	mans []*kopia.ManifestEntry,
+	fallback []*kopia.ManifestEntry,
+) []*kopia.ManifestEntry {
+	if len(fallback) == 0 {
+		return mans
+	}
+
+	if len(mans) == 0 {
+		return fallback
+	}
+
+	type manTup struct {
+		complete   *kopia.ManifestEntry
+		incomplete *kopia.ManifestEntry
+	}
+
+	tups := map[string]manTup{}
+
+	for _, r := range reasons {
+		// no resource owner in the key.  Assume it's the same owner across all
+		// manifests, but that the identifier is different due to migration.
+		k := r.Service.String() + r.Category.String()
+		tups[k] = manTup{}
+	}
+
+	// track the manifests that were collected with the current lookup
+	for _, m := range mans {
+		for _, r := range m.Reasons {
+			k := r.Service.String() + r.Category.String()
+			t := tups[k]
+			// assume mans will have, at most, one complete and one incomplete per key
+			if len(m.IncompleteReason) > 0 {
+				t.incomplete = m
+			} else {
+				t.complete = m
+			}
+
+			tups[k] = t
+		}
+	}
+
+	// backfill from the fallback where necessary
+	for _, m := range fallback {
+		for _, r := range m.Reasons {
+			k := r.Service.String() + r.Category.String()
+			t := tups[k]
+
+			if t.complete != nil {
+				// assume fallbacks contains prior manifest versions.
+				// we don't want to stack a prior version incomplete onto
+				// a current version's complete snapshot.
+				continue
+			}
+
+			if len(m.IncompleteReason) > 0 && t.incomplete == nil {
+				t.incomplete = m
+			} else if len(m.IncompleteReason) == 0 {
+				t.complete = m
+			}
+
+			tups[k] = t
+		}
+	}
+
+	// collect the results into a single slice of manifests
+	ms := []*kopia.ManifestEntry{}
+
+	for _, m := range tups {
+		if m.complete != nil {
+			ms = append(ms, m.complete)
+		}
+
+		if m.incomplete != nil {
+			ms = append(ms, m.incomplete)
+		}
+	}
+
+	return ms
+}
+
 // verifyDistinctBases is a validation checker that ensures, for a given slice
 // of manifests, that each manifest's Reason (owner, service, category) is only
 // included once.  If a reason is duplicated by any two manifests, an error is

src/internal/operations/manifests_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/kopia/kopia/repo/manifest"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/internal/data"
@@ -34,7 +35,24 @@ func (mmr mockManifestRestorer) FetchPrevSnapshotManifests(
 	reasons []kopia.Reason,
 	tags map[string]string,
 ) ([]*kopia.ManifestEntry, error) {
-	return mmr.mans, mmr.mrErr
+	mans := []*kopia.ManifestEntry{}
+
+	for _, r := range reasons {
+		for _, m := range mmr.mans {
+			for _, mr := range m.Reasons {
+				if mr.ResourceOwner == r.ResourceOwner {
+					mans = append(mans, m)
+					break
+				}
+			}
+		}
+	}
+
+	if len(mans) == 0 && len(reasons) == 0 {
+		mans = mmr.mans
+	}
+
+	return mans, mmr.mrErr
 }
 
 type mockGetBackuper struct {
@@ -658,7 +676,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 				ctx,
 				&test.mr,
 				&test.gb,
-				test.reasons,
+				test.reasons, nil,
 				tid,
 				test.getMeta,
 				fault.New(true))
@@ -709,6 +727,137 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 	}
 }
 
+func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallbackReasons() {
+	const (
+		ro            = "resourceowner"
+		manComplete   = "complete"
+		manIncomplete = "incmpl"
+
+		fbro         = "fb_resourceowner"
+		fbComplete   = "fb_complete"
+		fbIncomplete = "fb_incmpl"
+	)
+
+	makeMan := func(owner, id, incmpl string) *kopia.ManifestEntry {
+		return &kopia.ManifestEntry{
+			Manifest: &snapshot.Manifest{
+				ID:               manifest.ID(id),
+				IncompleteReason: incmpl,
+				Tags:             map[string]string{},
+			},
+			Reasons: []kopia.Reason{
+				{
+					ResourceOwner: owner,
+					Service:       path.ExchangeService,
+					Category:      path.EmailCategory,
+				},
+			},
+		}
+	}
+
+	var (
+		manReason = kopia.Reason{
+			ResourceOwner: ro,
+			Service:       path.ExchangeService,
+			Category:      path.EmailCategory,
+		}
+		fbReason = kopia.Reason{
+			ResourceOwner: fbro,
+			Service:       path.ExchangeService,
+			Category:      path.EmailCategory,
+		}
+		mc  = makeMan(ro, manComplete, "")
+		mi  = makeMan(ro, manIncomplete, "ir")
+		fbc = makeMan(fbro, fbComplete, "")
+		fbi = makeMan(fbro, fbIncomplete, "ir")
+	)
+
+	table := []struct {
+		name          string
+		mr            mockManifestRestorer
+		assertErr     assert.ErrorAssertionFunc
+		expectMans    []string
+		expectNilMans bool
+	}{
+		{
+			name: "only mans, no fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mc, mi},
+			},
+			expectMans: []string{manComplete, manIncomplete},
+		},
+		{
+			name: "no mans, only fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{fbc, fbi},
+			},
+			expectMans: []string{fbComplete, fbIncomplete},
+		},
+		{
+			name: "complete mans and fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mc, fbc},
+			},
+			expectMans: []string{manComplete},
+		},
+		{
+			name: "incomplete mans and fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mi, fbi},
+			},
+			expectMans: []string{manIncomplete},
+		},
+		{
+			name: "complete and incomplete mans and fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mc, mi, fbc, fbi},
+			},
+			expectMans: []string{manComplete, manIncomplete},
+		},
+		{
+			name: "incomplete mans, complete fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mi, fbc},
+			},
+			expectMans: []string{fbComplete, manIncomplete},
+		},
+		{
+			name: "complete mans, incomplete fallbacks",
+			mr: mockManifestRestorer{
+				mans: []*kopia.ManifestEntry{mc, fbi},
+			},
+			expectMans: []string{manComplete},
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t := suite.T()
+
+			ctx, flush := tester.NewContext()
+			defer flush()
+
+			mans, _, b, err := produceManifestsAndMetadata(
+				ctx,
+				&test.mr,
+				nil,
+				[]kopia.Reason{manReason},
+				[]kopia.Reason{fbReason},
+				"tid",
+				false,
+				fault.New(true))
+			require.NoError(t, err, clues.ToCore(err))
+			assert.False(t, b, "no-metadata is forced for this test")
+
+			manIDs := []string{}
+			for _, m := range mans {
+				manIDs = append(manIDs, string(m.ID))
+			}
+
+			assert.ElementsMatch(t, test.expectMans, manIDs)
+		})
+	}
+}
+
 // ---------------------------------------------------------------------------
 // older tests
 // ---------------------------------------------------------------------------