larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix overlapping bases after fallback union

Browse Source 
In a case where the current manifests and the
fallback contain both distinct and overlapping
categories, the end result contains multiple
bases for the same category.  Verifydistinctresons
didn't catch this because it includes the resource
owner.
This commit is contained in:
ryanfkeepers 2023-04-13 09:17:54 -06:00
parent 9e692c7e2e
commit 0e3469c23e
2 changed files with 128 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/internal/operations/manifests.go
View File

@ -2,6 +2,7 @@ package operations
import (
"context"
"fmt"
"github.com/alcionai/clues"
"github.com/kopia/kopia/repo/manifest"
@ -93,20 +94,6 @@ func produceManifestsAndMetadata(
return ms, nil, false, nil
}
// We only need to check that we have 1:1 reason:base if we're doing an
// incremental with associated metadata. This ensures that we're only sourcing
// data from a single Point-In-Time (base) for each incremental backup.
//
// TODO(ashmrtn): This may need updating if we start sourcing item backup
// details from previous snapshots when using kopia-assisted incrementals.
if err := verifyDistinctBases(ctx, ms); err != nil {
logger.Ctx(ctx).With("error", err).Infow(
"unioned snapshot collision, falling back to full backup",
clues.In(ctx).Slice()...)
return ms, nil, false, nil
}
for _, man := range ms {
if len(man.IncompleteReason) > 0 {
continue
@ -163,6 +150,7 @@ func produceManifestsAndMetadata(
LogFaultErrors(ctx, fb.Errors(), "collecting metadata")
if err != nil && !errors.Is(err, data.ErrNotFound) {
fmt.Printf("\n-----\nCOLLECTING METADATA %+v\n-----\n", err)
// prior metadata isn't guaranteed to exist.
// if it doesn't, we'll just have to do a
// full backup for that data.
@ -234,6 +222,8 @@ func unionManifests(
// backfill from the fallback where necessary
for _, m := range fallback {
useReasons := []kopia.Reason{}
for _, r := range m.Reasons {
k := r.Service.String() + r.Category.String()
t := tups[k]
@ -245,6 +235,8 @@ func unionManifests(
continue
}
useReasons = append(useReasons, r)
if len(m.IncompleteReason) > 0 && t.incomplete == nil {
t.incomplete = m
} else if len(m.IncompleteReason) == 0 {
@ -253,6 +245,10 @@ func unionManifests(
tups[k] = t
}
if len(m.IncompleteReason) == 0 && len(useReasons) > 0 {
m.Reasons = useReasons
}
}
// collect the results into a single slice of manifests

145
src/internal/operations/manifests_test.go
View File

@ -757,18 +757,20 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
table := []struct {
name string
main []testInput
man []testInput
fallback []testInput
reasons []kopia.Reason
fallbackReasons []kopia.Reason
categories []path.CategoryType
manCategories []path.CategoryType
fbCategories []path.CategoryType
assertErr assert.ErrorAssertionFunc
expectManIDs []string
expectNilMans bool
expectReasons map[string][]path.CategoryType
}{
{
name: "only mans, no fallbacks",
main: []testInput{
man: []testInput{
{
id: manComplete,
},
@ -777,8 +779,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
incomplete: true,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete, manIncomplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete, manIncomplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.EmailCategory},
manIncomplete: {path.EmailCategory},
},
},
{
name: "no mans, only fallbacks",
@ -791,12 +798,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
incomplete: true,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{fbComplete, fbIncomplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{fbComplete, fbIncomplete},
expectReasons: map[string][]path.CategoryType{
fbComplete: {path.EmailCategory},
fbIncomplete: {path.EmailCategory},
},
},
{
name: "complete mans and fallbacks",
main: []testInput{
man: []testInput{
{
id: manComplete,
},
@ -806,12 +818,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
id: fbComplete,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.EmailCategory},
},
},
{
name: "incomplete mans and fallbacks",
main: []testInput{
man: []testInput{
{
id: manIncomplete,
incomplete: true,
@ -823,12 +839,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
incomplete: true,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manIncomplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manIncomplete},
expectReasons: map[string][]path.CategoryType{
manIncomplete: {path.EmailCategory},
},
},
{
name: "complete and incomplete mans and fallbacks",
main: []testInput{
man: []testInput{
{
id: manComplete,
},
@ -846,12 +866,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
incomplete: true,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete, manIncomplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete, manIncomplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.EmailCategory},
manIncomplete: {path.EmailCategory},
},
},
{
name: "incomplete mans, complete fallbacks",
main: []testInput{
man: []testInput{
{
id: manIncomplete,
incomplete: true,
@ -862,12 +887,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
id: fbComplete,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{fbComplete, manIncomplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{fbComplete, manIncomplete},
expectReasons: map[string][]path.CategoryType{
fbComplete: {path.EmailCategory},
manIncomplete: {path.EmailCategory},
},
},
{
name: "complete mans, incomplete fallbacks",
main: []testInput{
man: []testInput{
{
id: manComplete,
},
@ -878,12 +908,16 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
incomplete: true,
},
},
categories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete},
manCategories: []path.CategoryType{path.EmailCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.EmailCategory},
},
},
{
name: "complete mans, complete fallbacks, multiple reasons",
main: []testInput{
man: []testInput{
{
id: manComplete,
},
@ -893,8 +927,52 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
id: fbComplete,
},
},
categories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
expectManIDs: []string{manComplete},
manCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
fbCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory},
expectManIDs: []string{manComplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.EmailCategory, path.ContactsCategory},
},
},
{
name: "complete mans, complete fallbacks, distinct reasons",
man: []testInput{
{
id: manComplete,
},
},
fallback: []testInput{
{
id: fbComplete,
},
},
manCategories: []path.CategoryType{path.ContactsCategory},
fbCategories: []path.CategoryType{path.EmailCategory},
expectManIDs: []string{manComplete, fbComplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.ContactsCategory},
fbComplete: {path.EmailCategory},
},
},
{
name: "fb has superset of mans reasons",
man: []testInput{
{
id: manComplete,
},
},
fallback: []testInput{
{
id: fbComplete,
},
},
manCategories: []path.CategoryType{path.ContactsCategory},
fbCategories: []path.CategoryType{path.EmailCategory, path.ContactsCategory, path.EventsCategory},
expectManIDs: []string{manComplete, fbComplete},
expectReasons: map[string][]path.CategoryType{
manComplete: {path.ContactsCategory},
fbComplete: {path.EmailCategory, path.EventsCategory},
},
},
}
for _, test := range table {
@ -907,7 +985,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
mainReasons := []kopia.Reason{}
fbReasons := []kopia.Reason{}
for _, cat := range test.categories {
for _, cat := range test.manCategories {
mainReasons = append(
mainReasons,
kopia.Reason{
@ -915,7 +993,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
Service: path.ExchangeService,
Category: cat,
})
}
for _, cat := range test.fbCategories {
fbReasons = append(
fbReasons,
kopia.Reason{
@ -927,7 +1007,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
mans := []*kopia.ManifestEntry{}
for _, m := range test.main {
for _, m := range test.man {
incomplete := ""
if m.incomplete {
incomplete = "ir"
@ -959,8 +1039,19 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_fallb
assert.False(t, b, "no-metadata is forced for this test")
manIDs := []string{}
for _, m := range mans {
manIDs = append(manIDs, string(m.ID))
reasons, ok := test.expectReasons[string(m.ID)]
assert.True(t, ok, "unexpected manifest in result: ", m.ID)
mrs := []path.CategoryType{}
for _, r := range m.Reasons {
mrs = append(mrs, r.Category)
}
assert.ElementsMatch(t, reasons, mrs)
}
assert.ElementsMatch(t, test.expectManIDs, manIDs)