larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optionally look up and union fallback manifests (#3057)

Browse Source 
Fallback manifests allow us to find manifests and
metadata from snapshots that utilized earlier
versions of kopia Reasons, primarily across
resource owner deltas.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #2825

#### Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test
This commit is contained in:
Keepers 2023-04-10 20:47:11 -06:00 committed by GitHub
parent de3554d869
commit 7c9eada5a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 298 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/internal/operations/backup.go
View File

@ -229,7 +229,11 @@ func (op *BackupOperation) do(
detailsStore streamstore.Streamer,
backupID model.StableID,
) (*details.Builder, error) {
reasons := selectorToReasons(op.Selectors)
var (
reasons = selectorToReasons(op.Selectors, false)
fallbackReasons = makeFallbackReasons(op.Selectors)
)
logger.Ctx(ctx).With("selectors", op.Selectors).Info("backing up selection")
// should always be 1, since backups are 1:1 with resourceOwners.
@ -239,7 +243,7 @@ func (op *BackupOperation) do(
ctx,
op.kopia,
op.store,
reasons,
reasons, fallbackReasons,
op.account.ID(),
op.incremental,
op.Errors)
@ -297,6 +301,14 @@ func (op *BackupOperation) do(
return deets, nil
}
func makeFallbackReasons(sel selectors.Selector) []kopia.Reason {
if sel.PathService() != path.SharePointService {
return selectorToReasons(sel, true)
}
return nil
}
// checker to see if conditions are correct for incremental backup behavior such as
// retrieving metadata like delta tokens and previous paths.
func useIncrementalBackup(sel selectors.Selector, opts control.Options) bool {
@ -338,7 +350,7 @@ func produceBackupDataCollections(
// Consumer funcs
// ---------------------------------------------------------------------------
func selectorToReasons(sel selectors.Selector) []kopia.Reason {
func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.Reason {
service := sel.PathService()
reasons := []kopia.Reason{}
@ -349,10 +361,15 @@ func selectorToReasons(sel selectors.Selector) []kopia.Reason {
return nil
}
owner := sel.DiscreteOwner
if useOwnerNameForID {
owner = sel.DiscreteOwnerName
}
for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} {
for _, cat := range sl {
reasons = append(reasons, kopia.Reason{
ResourceOwner: sel.DiscreteOwner,
ResourceOwner: owner,
Service: service,
Category: cat,
})

132
src/internal/operations/manifests.go
View File

@ -43,24 +43,52 @@ func produceManifestsAndMetadata(
ctx context.Context,
mr manifestRestorer,
gb getBackuper,
reasons []kopia.Reason,
reasons, fallbackReasons []kopia.Reason,
tenantID string,
getMetadata bool,
errs *fault.Bus,
) ([]*kopia.ManifestEntry, []data.RestoreCollection, bool, error) {
var (
tags = map[string]string{kopia.TagBackupCategory: ""}
metadataFiles = graph.AllMetadataFileNames()
collections []data.RestoreCollection
)
ms, err := mr.FetchPrevSnapshotManifests(
ctx,
reasons,
map[string]string{kopia.TagBackupCategory: ""})
ms, err := mr.FetchPrevSnapshotManifests(ctx, reasons, tags)
if err != nil {
return nil, nil, false, err
return nil, nil, false, clues.Wrap(err, "looking up prior snapshots")
}
// We only need to check that we have 1:1 reason:base if we're doing an
// incremental with associated metadata. This ensures that we're only sourcing
// data from a single Point-In-Time (base) for each incremental backup.
//
// TODO(ashmrtn): This may need updating if we start sourcing item backup
// details from previous snapshots when using kopia-assisted incrementals.
if err := verifyDistinctBases(ctx, ms); err != nil {
logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup")
return ms, nil, false, nil
}
fbms, err := mr.FetchPrevSnapshotManifests(ctx, fallbackReasons, tags)
if err != nil {
return nil, nil, false, clues.Wrap(err, "looking up prior snapshots under alternate id")
}
// Also check distinct bases for the fallback set.
if err := verifyDistinctBases(ctx, fbms); err != nil {
logger.CtxErr(ctx, err).Info("base snapshot collision, falling back to full backup")
return ms, nil, false, nil
}
// one of three cases can occur when retrieving backups across reason migrations:
// 1. the current reasons don't match any manifests, and we use the fallback to
// look up the previous reason version.
// 2. the current reasons only contain an incomplete manifest, and the fallback
// can find a complete manifest.
// 3. the current reasons contain all the necessary manifests.
ms = unionManifests(reasons, ms, fbms)
if !getMetadata {
return ms, nil, false, nil
}
@ -137,6 +165,98 @@ func produceManifestsAndMetadata(
return ms, collections, true, err
}
// unionManifests reduces the two manifest slices into a single slice.
// Assumes fallback represents a prior manifest version (across some migration
// that disrupts manifest lookup), and that mans contains the current version.
// Also assumes the mans slice will have, at most, one complete and one incomplete
// manifest per service+category tuple.
//
// Selection priority, for each reason, follows these rules:
// 1. If the mans manifest is complete, ignore fallback manifests for that reason.
// 2. If the mans manifest is only incomplete, look for a matching complete manifest in fallbacks.
// 3. If mans has no entry for a reason, look for both complete and incomplete fallbacks.
func unionManifests(
reasons []kopia.Reason,
mans []*kopia.ManifestEntry,
fallback []*kopia.ManifestEntry,
) []*kopia.ManifestEntry {
if len(fallback) == 0 {
return mans
}
if len(mans) == 0 {
return fallback
}
type manTup struct {
complete *kopia.ManifestEntry
incomplete *kopia.ManifestEntry
}
tups := map[string]manTup{}
for _, r := range reasons {
// no resource owner in the key. Assume it's the same owner across all
// manifests, but that the identifier is different due to migration.
k := r.Service.String() + r.Category.String()
tups[k] = manTup{}
}
// track the manifests that were collected with the current lookup
for _, m := range mans {
for _, r := range m.Reasons {
k := r.Service.String() + r.Category.String()
t := tups[k]
// assume mans will have, at most, one complete and one incomplete per key
if len(m.IncompleteReason) > 0 {
t.incomplete = m
} else {
t.complete = m
}
tups[k] = t
}
}
// backfill from the fallback where necessary
for _, m := range fallback {
for _, r := range m.Reasons {
k := r.Service.String() + r.Category.String()
t := tups[k]
if t.complete != nil {
// assume fallbacks contains prior manifest versions.
// we don't want to stack a prior version incomplete onto
// a current version's complete snapshot.
continue
}
if len(m.IncompleteReason) > 0 && t.incomplete == nil {
t.incomplete = m
} else if len(m.IncompleteReason) == 0 {
t.complete = m
}
tups[k] = t
}
}
// collect the results into a single slice of manifests
ms := []*kopia.ManifestEntry{}
for _, m := range tups {
if m.complete != nil {
ms = append(ms, m.complete)
}
if m.incomplete != nil {
ms = append(ms, m.incomplete)
}
}
return ms
}
// verifyDistinctBases is a validation checker that ensures, for a given slice
// of manifests, that each manifest's Reason (owner, service, category) is only
// included once. If a reason is duplicated by any two manifests, an error is

153
src/internal/operations/manifests_test.go
View File

@ -8,6 +8,7 @@ import (
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/data"
@ -34,7 +35,24 @@ func (mmr mockManifestRestorer) FetchPrevSnapshotManifests(
reasons []kopia.Reason,
tags map[string]string,
) ([]*kopia.ManifestEntry, error) {
return mmr.mans, mmr.mrErr
mans := []*kopia.ManifestEntry{}
for _, r := range reasons {
for _, m := range mmr.mans {
for _, mr := range m.Reasons {
if mr.ResourceOwner == r.ResourceOwner {
mans = append(mans, m)
break
}
}
}
}
if len(mans) == 0 && len(reasons) == 0 {
mans = mmr.mans
}
return mans, mmr.mrErr
}
type mockGetBackuper struct {
@ -658,7 +676,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
ctx,
&test.mr,
&test.gb,
test.reasons,
test.reasons, nil,
tid,
test.getMeta,
fault.New(true))
@ -709,6 +727,137 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
}
}
func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallbackReasons() {
const (
ro = "resourceowner"
manComplete = "complete"
manIncomplete = "incmpl"
fbro = "fb_resourceowner"
fbComplete = "fb_complete"
fbIncomplete = "fb_incmpl"
)
makeMan := func(owner, id, incmpl string) *kopia.ManifestEntry {
return &kopia.ManifestEntry{
Manifest: &snapshot.Manifest{
ID: manifest.ID(id),
IncompleteReason: incmpl,
Tags: map[string]string{},
},
Reasons: []kopia.Reason{
{
ResourceOwner: owner,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
}
}
var (
manReason = kopia.Reason{
ResourceOwner: ro,
Service: path.ExchangeService,
Category: path.EmailCategory,
}
fbReason = kopia.Reason{
ResourceOwner: fbro,
Service: path.ExchangeService,
Category: path.EmailCategory,
}
mc = makeMan(ro, manComplete, "")
mi = makeMan(ro, manIncomplete, "ir")
fbc = makeMan(fbro, fbComplete, "")
fbi = makeMan(fbro, fbIncomplete, "ir")
)
table := []struct {
name string
mr mockManifestRestorer
assertErr assert.ErrorAssertionFunc
expectMans []string
expectNilMans bool
}{
{
name: "only mans, no fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mc, mi},
},
expectMans: []string{manComplete, manIncomplete},
},
{
name: "no mans, only fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{fbc, fbi},
},
expectMans: []string{fbComplete, fbIncomplete},
},
{
name: "complete mans and fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mc, fbc},
},
expectMans: []string{manComplete},
},
{
name: "incomplete mans and fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mi, fbi},
},
expectMans: []string{manIncomplete},
},
{
name: "complete and incomplete mans and fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mc, mi, fbc, fbi},
},
expectMans: []string{manComplete, manIncomplete},
},
{
name: "incomplete mans, complete fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mi, fbc},
},
expectMans: []string{fbComplete, manIncomplete},
},
{
name: "complete mans, incomplete fallbacks",
mr: mockManifestRestorer{
mans: []*kopia.ManifestEntry{mc, fbi},
},
expectMans: []string{manComplete},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
mans, _, b, err := produceManifestsAndMetadata(
ctx,
&test.mr,
nil,
[]kopia.Reason{manReason},
[]kopia.Reason{fbReason},
"tid",
false,
fault.New(true))
require.NoError(t, err, clues.ToCore(err))
assert.False(t, b, "no-metadata is forced for this test")
manIDs := []string{}
for _, m := range mans {
manIDs = append(manIDs, string(m.ID))
}
assert.ElementsMatch(t, test.expectMans, manIDs)
})
}
}
// ---------------------------------------------------------------------------
// older tests
// ---------------------------------------------------------------------------