diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 1c59b75dd..ad2e35e9f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -229,7 +229,11 @@ func (op *BackupOperation) do( detailsStore streamstore.Streamer, backupID model.StableID, ) (*details.Builder, error) { - reasons := selectorToReasons(op.Selectors) + var ( + reasons = selectorToReasons(op.Selectors, false) + fallbackReasons = makeFallbackReasons(op.Selectors) + ) + logger.Ctx(ctx).With("selectors", op.Selectors).Info("backing up selection") // should always be 1, since backups are 1:1 with resourceOwners. @@ -239,7 +243,7 @@ func (op *BackupOperation) do( ctx, op.kopia, op.store, - reasons, + reasons, fallbackReasons, op.account.ID(), op.incremental, op.Errors) @@ -297,6 +301,14 @@ func (op *BackupOperation) do( return deets, nil } +func makeFallbackReasons(sel selectors.Selector) []kopia.Reason { + if sel.PathService() != path.SharePointService { + return selectorToReasons(sel, true) + } + + return nil +} + // checker to see if conditions are correct for incremental backup behavior such as // retrieving metadata like delta tokens and previous paths. func useIncrementalBackup(sel selectors.Selector, opts control.Options) bool { @@ -338,7 +350,7 @@ func produceBackupDataCollections( // Consumer funcs // --------------------------------------------------------------------------- -func selectorToReasons(sel selectors.Selector) []kopia.Reason { +func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.Reason { service := sel.PathService() reasons := []kopia.Reason{} @@ -349,10 +361,15 @@ func selectorToReasons(sel selectors.Selector) []kopia.Reason { return nil } + owner := sel.DiscreteOwner + if useOwnerNameForID { + owner = sel.DiscreteOwnerName + } + for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} { for _, cat := range sl { reasons = append(reasons, kopia.Reason{ - ResourceOwner: sel.DiscreteOwner, + ResourceOwner: owner, Service: service, Category: cat, }) diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index f8f2c5041..5701561e8 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -43,24 +43,52 @@ func produceManifestsAndMetadata( ctx context.Context, mr manifestRestorer, gb getBackuper, - reasons []kopia.Reason, + reasons, fallbackReasons []kopia.Reason, tenantID string, getMetadata bool, errs *fault.Bus, ) ([]*kopia.ManifestEntry, []data.RestoreCollection, bool, error) { var ( + tags = map[string]string{kopia.TagBackupCategory: ""} metadataFiles = graph.AllMetadataFileNames() collections []data.RestoreCollection ) - ms, err := mr.FetchPrevSnapshotManifests( - ctx, - reasons, - map[string]string{kopia.TagBackupCategory: ""}) + ms, err := mr.FetchPrevSnapshotManifests(ctx, reasons, tags) if err != nil { - return nil, nil, false, err + return nil, nil, false, clues.Wrap(err, "looking up prior snapshots") } + // We only need to check that we have 1:1 reason:base if we're doing an + // incremental with associated metadata. This ensures that we're only sourcing + // data from a single Point-In-Time (base) for each incremental backup. + // + // TODO(ashmrtn): This may need updating if we start sourcing item backup + // details from previous snapshots when using kopia-assisted incrementals. + if err := verifyDistinctBases(ctx, ms); err != nil { + logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup") + return ms, nil, false, nil + } + + fbms, err := mr.FetchPrevSnapshotManifests(ctx, fallbackReasons, tags) + if err != nil { + return nil, nil, false, clues.Wrap(err, "looking up prior snapshots under alternate id") + } + + // Also check distinct bases for the fallback set. + if err := verifyDistinctBases(ctx, fbms); err != nil { + logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup") + return ms, nil, false, nil + } + + // one of three cases can occur when retrieving backups across reason migrations: + // 1. the current reasons don't match any manifests, and we use the fallback to + // look up the previous reason version. + // 2. the current reasons only contain an incomplete manifest, and the fallback + // can find a complete manifest. + // 3. the current reasons contain all the necessary manifests. + ms = unionManifests(reasons, ms, fbms) + if !getMetadata { return ms, nil, false, nil } @@ -137,6 +165,98 @@ func produceManifestsAndMetadata( return ms, collections, true, err } +// unionManifests reduces the two manifest slices into a single slice. +// Assumes fallback represents a prior manifest version (across some migration +// that disrupts manifest lookup), and that mans contains the current version. +// Also assumes the mans slice will have, at most, one complete and one incomplete +// manifest per service+category tuple. +// +// Selection priority, for each reason, follows these rules: +// 1. If the mans manifest is complete, ignore fallback manifests for that reason. +// 2. If the mans manifest is only incomplete, look for a matching complete manifest in fallbacks. +// 3. If mans has no entry for a reason, look for both complete and incomplete fallbacks. +func unionManifests( + reasons []kopia.Reason, + mans []*kopia.ManifestEntry, + fallback []*kopia.ManifestEntry, +) []*kopia.ManifestEntry { + if len(fallback) == 0 { + return mans + } + + if len(mans) == 0 { + return fallback + } + + type manTup struct { + complete *kopia.ManifestEntry + incomplete *kopia.ManifestEntry + } + + tups := map[string]manTup{} + + for _, r := range reasons { + // no resource owner in the key. Assume it's the same owner across all + // manifests, but that the identifier is different due to migration. + k := r.Service.String() + r.Category.String() + tups[k] = manTup{} + } + + // track the manifests that were collected with the current lookup + for _, m := range mans { + for _, r := range m.Reasons { + k := r.Service.String() + r.Category.String() + t := tups[k] + // assume mans will have, at most, one complete and one incomplete per key + if len(m.IncompleteReason) > 0 { + t.incomplete = m + } else { + t.complete = m + } + + tups[k] = t + } + } + + // backfill from the fallback where necessary + for _, m := range fallback { + for _, r := range m.Reasons { + k := r.Service.String() + r.Category.String() + t := tups[k] + + if t.complete != nil { + // assume fallbacks contains prior manifest versions. + // we don't want to stack a prior version incomplete onto + // a current version's complete snapshot. + continue + } + + if len(m.IncompleteReason) > 0 && t.incomplete == nil { + t.incomplete = m + } else if len(m.IncompleteReason) == 0 { + t.complete = m + } + + tups[k] = t + } + } + + // collect the results into a single slice of manifests + ms := []*kopia.ManifestEntry{} + + for _, m := range tups { + if m.complete != nil { + ms = append(ms, m.complete) + } + + if m.incomplete != nil { + ms = append(ms, m.incomplete) + } + } + + return ms +} + // verifyDistinctBases is a validation checker that ensures, for a given slice // of manifests, that each manifest's Reason (owner, service, category) is only // included once. If a reason is duplicated by any two manifests, an error is diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index 9f2a3b409..47c46a1fe 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -8,6 +8,7 @@ import ( "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" @@ -34,7 +35,24 @@ func (mmr mockManifestRestorer) FetchPrevSnapshotManifests( reasons []kopia.Reason, tags map[string]string, ) ([]*kopia.ManifestEntry, error) { - return mmr.mans, mmr.mrErr + mans := []*kopia.ManifestEntry{} + + for _, r := range reasons { + for _, m := range mmr.mans { + for _, mr := range m.Reasons { + if mr.ResourceOwner == r.ResourceOwner { + mans = append(mans, m) + break + } + } + } + } + + if len(mans) == 0 && len(reasons) == 0 { + mans = mmr.mans + } + + return mans, mmr.mrErr } type mockGetBackuper struct { @@ -658,7 +676,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { ctx, &test.mr, &test.gb, - test.reasons, + test.reasons, nil, tid, test.getMeta, fault.New(true)) @@ -709,6 +727,137 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { } } +func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallbackReasons() { + const ( + ro = "resourceowner" + manComplete = "complete" + manIncomplete = "incmpl" + + fbro = "fb_resourceowner" + fbComplete = "fb_complete" + fbIncomplete = "fb_incmpl" + ) + + makeMan := func(owner, id, incmpl string) *kopia.ManifestEntry { + return &kopia.ManifestEntry{ + Manifest: &snapshot.Manifest{ + ID: manifest.ID(id), + IncompleteReason: incmpl, + Tags: map[string]string{}, + }, + Reasons: []kopia.Reason{ + { + ResourceOwner: owner, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + }, + } + } + + var ( + manReason = kopia.Reason{ + ResourceOwner: ro, + Service: path.ExchangeService, + Category: path.EmailCategory, + } + fbReason = kopia.Reason{ + ResourceOwner: fbro, + Service: path.ExchangeService, + Category: path.EmailCategory, + } + mc = makeMan(ro, manComplete, "") + mi = makeMan(ro, manIncomplete, "ir") + fbc = makeMan(fbro, fbComplete, "") + fbi = makeMan(fbro, fbIncomplete, "ir") + ) + + table := []struct { + name string + mr mockManifestRestorer + assertErr assert.ErrorAssertionFunc + expectMans []string + expectNilMans bool + }{ + { + name: "only mans, no fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mc, mi}, + }, + expectMans: []string{manComplete, manIncomplete}, + }, + { + name: "no mans, only fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{fbc, fbi}, + }, + expectMans: []string{fbComplete, fbIncomplete}, + }, + { + name: "complete mans and fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mc, fbc}, + }, + expectMans: []string{manComplete}, + }, + { + name: "incomplete mans and fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mi, fbi}, + }, + expectMans: []string{manIncomplete}, + }, + { + name: "complete and incomplete mans and fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mc, mi, fbc, fbi}, + }, + expectMans: []string{manComplete, manIncomplete}, + }, + { + name: "incomplete mans, complete fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mi, fbc}, + }, + expectMans: []string{fbComplete, manIncomplete}, + }, + { + name: "complete mans, incomplete fallbacks", + mr: mockManifestRestorer{ + mans: []*kopia.ManifestEntry{mc, fbi}, + }, + expectMans: []string{manComplete}, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext() + defer flush() + + mans, _, b, err := produceManifestsAndMetadata( + ctx, + &test.mr, + nil, + []kopia.Reason{manReason}, + []kopia.Reason{fbReason}, + "tid", + false, + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + assert.False(t, b, "no-metadata is forced for this test") + + manIDs := []string{} + for _, m := range mans { + manIDs = append(manIDs, string(m.ID)) + } + + assert.ElementsMatch(t, test.expectMans, manIDs) + }) + } +} + // --------------------------------------------------------------------------- // older tests // ---------------------------------------------------------------------------