Return assist backup models during base selection process (#3907)

<!-- PR description-->
This PR adds support for selecting assist backup models. The intent here is to use the assist backups during merge details process. 

Changelist:
1. Assist bases now have a requirement to have a) an associated assist backup model b) details ssid. Please see [this code comment](https://github.com/alcionai/corso/pull/3907/files#diff-f9b6b7ab52c8e1148147909fd071d9b80b816d072203d8e9d2fa34ab93185db1R286) for more.
2. Any incomplete assist bases or assist bases not matching 1) are now discarded, i.e. they are not fed to kopia for kopia assisted incrementals. The impact here is that files cached by kopia might be now redownloaded if the backup failed without qualifying for an assist backup.


Unit tests added.


---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2023-08-10 00:16:08 +05:30 committed by GitHub
parent 23ff9cd08b
commit 7582e175e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 943 additions and 326 deletions

View File

@ -34,8 +34,8 @@ type backupBases struct {
// data. // data.
backups []BackupEntry backups []BackupEntry
mergeBases []ManifestEntry mergeBases []ManifestEntry
assistBases []ManifestEntry
assistBackups []BackupEntry assistBackups []BackupEntry
assistBases []ManifestEntry
} }
func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) { func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) {
@ -122,12 +122,12 @@ func (bb *backupBases) ClearAssistBases() {
// //
// Selection priority, for each reason key generated by reasonsToKey, follows // Selection priority, for each reason key generated by reasonsToKey, follows
// these rules: // these rules:
// 1. If the called BackupBases has an entry for a given resaon, ignore the // 1. If the called BackupBases has an entry for a given reason, ignore the
// other BackupBases matching that reason. // other BackupBases matching that reason.
// 2. If the the receiver BackupBases has only AssistBases, look for a matching // 2. If the called BackupBases has only AssistBases, look for a matching
// MergeBase manifest in the passed in BackupBases. // MergeBase manifest in the other BackupBases.
// 3. If the called BackupBases has no entry for a reason, look for both // 3. If the called BackupBases has no entry for a reason, look for a matching
// AssistBases and MergeBases in the passed in BackupBases. // MergeBase in the other BackupBases.
func (bb *backupBases) MergeBackupBases( func (bb *backupBases) MergeBackupBases(
ctx context.Context, ctx context.Context,
other BackupBases, other BackupBases,
@ -189,6 +189,10 @@ func (bb *backupBases) MergeBackupBases(
backups: bb.Backups(), backups: bb.Backups(),
mergeBases: bb.MergeBases(), mergeBases: bb.MergeBases(),
assistBases: bb.AssistBases(), assistBases: bb.AssistBases(),
// Note that assistBackups are a new feature and don't exist
// in prior versions where we were using UPN based reasons i.e.
// other won't have any assistBackups.
assistBackups: bb.AssistBackups(),
} }
// Add new mergeBases and backups. // Add new mergeBases and backups.
@ -211,39 +215,11 @@ func (bb *backupBases) MergeBackupBases(
res.backups = append(res.backups, bup) res.backups = append(res.backups, bup)
res.mergeBases = append(res.mergeBases, man) res.mergeBases = append(res.mergeBases, man)
// TODO(pandeyabs): Remove this once we remove overlap between
// between merge and assist bases as part of #3943.
res.assistBases = append(res.assistBases, man) res.assistBases = append(res.assistBases, man)
} }
// Add assistBases from other to this one as needed.
for _, m := range other.AssistBases() {
useReasons := []Reasoner{}
// Assume that all complete manifests in assist overlap with MergeBases.
if len(m.IncompleteReason) == 0 {
continue
}
for _, r := range m.Reasons {
k := reasonToKey(r)
if _, ok := assist[k]; ok {
// This reason is already covered by either:
// * complete manifest in bb
// * incomplete manifest in bb
//
// If it was already in the assist set then it must be the case that
// it's newer than any complete manifests in other for the same reason.
continue
}
useReasons = append(useReasons, r)
}
if len(useReasons) > 0 {
m.Reasons = useReasons
res.assistBases = append(res.assistBases, m)
}
}
return res return res
} }
@ -332,12 +308,16 @@ func getBackupByID(backups []BackupEntry, bID string) (BackupEntry, bool) {
// pull. On the other hand, *not* dropping them is unsafe as it will muck up // pull. On the other hand, *not* dropping them is unsafe as it will muck up
// merging when we add stuff to kopia (possibly multiple entries for the same // merging when we add stuff to kopia (possibly multiple entries for the same
// item etc). // item etc).
//
// TODO(pandeyabs): Refactor common code into a helper as part of #3943.
func (bb *backupBases) fixupAndVerify(ctx context.Context) { func (bb *backupBases) fixupAndVerify(ctx context.Context) {
toDrop := findNonUniqueManifests(ctx, bb.mergeBases) toDrop := findNonUniqueManifests(ctx, bb.mergeBases)
var ( var (
backupsToKeep []BackupEntry backupsToKeep []BackupEntry
assistBackupsToKeep []BackupEntry
mergeToKeep []ManifestEntry mergeToKeep []ManifestEntry
assistToKeep []ManifestEntry
) )
for _, man := range bb.mergeBases { for _, man := range bb.mergeBases {
@ -352,7 +332,7 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
toDrop[man.ID] = struct{}{} toDrop[man.ID] = struct{}{}
logger.Ctx(ctx).Info( logger.Ctx(ctx).Info(
"dropping manifest due to missing backup", "dropping merge base due to missing backup",
"manifest_id", man.ID) "manifest_id", man.ID)
continue continue
@ -367,7 +347,7 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
toDrop[man.ID] = struct{}{} toDrop[man.ID] = struct{}{}
logger.Ctx(ctx).Info( logger.Ctx(ctx).Info(
"dropping manifest due to invalid backup", "dropping merge base due to invalid backup",
"manifest_id", man.ID) "manifest_id", man.ID)
continue continue
@ -377,9 +357,9 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
mergeToKeep = append(mergeToKeep, man) mergeToKeep = append(mergeToKeep, man)
} }
var assistToKeep []ManifestEntry // Every merge base is also a kopia assist base.
// TODO(pandeyabs): This should be removed as part of #3943.
for _, man := range bb.assistBases { for _, man := range bb.mergeBases {
if _, ok := toDrop[man.ID]; ok { if _, ok := toDrop[man.ID]; ok {
continue continue
} }
@ -387,7 +367,48 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
assistToKeep = append(assistToKeep, man) assistToKeep = append(assistToKeep, man)
} }
// Drop assist snapshots with overlapping reasons.
toDropAssists := findNonUniqueManifests(ctx, bb.assistBases)
for _, man := range bb.assistBases {
if _, ok := toDropAssists[man.ID]; ok {
continue
}
bID, _ := man.GetTag(TagBackupID)
bup, ok := getBackupByID(bb.assistBackups, bID)
if !ok {
toDrop[man.ID] = struct{}{}
logger.Ctx(ctx).Info(
"dropping assist base due to missing backup",
"manifest_id", man.ID)
continue
}
deetsID := bup.StreamStoreID
if len(deetsID) == 0 {
deetsID = bup.DetailsID
}
if len(bup.SnapshotID) == 0 || len(deetsID) == 0 {
toDrop[man.ID] = struct{}{}
logger.Ctx(ctx).Info(
"dropping assist base due to invalid backup",
"manifest_id", man.ID)
continue
}
assistBackupsToKeep = append(assistBackupsToKeep, bup)
assistToKeep = append(assistToKeep, man)
}
bb.backups = backupsToKeep bb.backups = backupsToKeep
bb.mergeBases = mergeToKeep bb.mergeBases = mergeToKeep
bb.assistBases = assistToKeep bb.assistBases = assistToKeep
bb.assistBackups = assistBackupsToKeep
} }

View File

@ -207,35 +207,24 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
type testInput struct { type testInput struct {
id int id int
incomplete bool
cat []path.CategoryType cat []path.CategoryType
} }
// Make a function so tests can modify things without messing with each other. // Make a function so tests can modify things without messing with each other.
makeBackupBases := func(ti []testInput) *backupBases { makeBackupBases := func(mergeInputs []testInput, assistInputs []testInput) *backupBases {
res := &backupBases{} res := &backupBases{}
for _, i := range ti { for _, i := range mergeInputs {
baseID := fmt.Sprintf("id%d", i.id) baseID := fmt.Sprintf("id%d", i.id)
ir := ""
if i.incomplete {
ir = "checkpoint"
}
reasons := make([]Reasoner, 0, len(i.cat)) reasons := make([]Reasoner, 0, len(i.cat))
for _, c := range i.cat { for _, c := range i.cat {
reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) reasons = append(reasons, NewReason("", ro, path.ExchangeService, c))
} }
m := makeManifest(baseID, ir, "b"+baseID, reasons...) m := makeManifest(baseID, "", "b"+baseID, reasons...)
res.assistBases = append(res.assistBases, m) res.assistBases = append(res.assistBases, m)
if i.incomplete {
continue
}
b := BackupEntry{ b := BackupEntry{
Backup: &backup.Backup{ Backup: &backup.Backup{
BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)}, BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)},
@ -249,44 +238,95 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
res.mergeBases = append(res.mergeBases, m) res.mergeBases = append(res.mergeBases, m)
} }
for _, i := range assistInputs {
baseID := fmt.Sprintf("id%d", i.id)
reasons := make([]Reasoner, 0, len(i.cat))
for _, c := range i.cat {
reasons = append(reasons, NewReason("", ro, path.ExchangeService, c))
}
m := makeManifest(baseID, "", "a"+baseID, reasons...)
b := BackupEntry{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: model.StableID("a" + baseID),
Tags: map[string]string{model.BackupTypeTag: model.AssistBackup},
},
SnapshotID: baseID,
StreamStoreID: "ss" + baseID,
},
Reasons: reasons,
}
res.assistBackups = append(res.assistBackups, b)
res.assistBases = append(res.assistBases, m)
}
return res return res
} }
table := []struct { table := []struct {
name string name string
bb []testInput merge []testInput
other []testInput assist []testInput
expect []testInput otherMerge []testInput
otherAssist []testInput
expect func() *backupBases
}{ }{
{ {
name: "Other Empty", name: "Other Empty",
bb: []testInput{ merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
}, },
expect: []testInput{ assist: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
}, },
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}},
}, []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
})
return bs
},
}, },
{ {
name: "BB Empty", name: "current Empty",
other: []testInput{ otherMerge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
}, },
expect: []testInput{ otherAssist: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
}, },
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}},
}, []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
})
return bs
},
}, },
{ {
name: "Other overlaps Complete And Incomplete", name: "Other overlaps merge and assist",
bb: []testInput{ merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
}, },
other: []testInput{ assist: []testInput{
{
id: 4,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherMerge: []testInput{
{ {
id: 2, id: 2,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
@ -294,147 +334,121 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
{ {
id: 3, id: 3,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
}, },
expect: []testInput{ otherAssist: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {
id: 5,
cat: []path.CategoryType{path.EmailCategory},
},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
}, []testInput{
{
id: 4,
cat: []path.CategoryType{path.EmailCategory},
},
})
return bs
}, },
}, },
{ {
name: "Other Overlaps Complete", name: "Other overlaps merge",
bb: []testInput{ merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {
id: 1,
cat: []path.CategoryType{path.EmailCategory},
}, },
other: []testInput{ },
otherMerge: []testInput{
{ {
id: 2, id: 2,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
}, },
}, },
expect: []testInput{ expect: func() *backupBases {
{cat: []path.CategoryType{path.EmailCategory}}, bs := makeBackupBases([]testInput{
},
},
{
name: "Other Overlaps Incomplete",
bb: []testInput{
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true, },
}, nil)
return bs
}, },
}, },
other: []testInput{
{ {
id: 2, name: "Current assist overlaps with Other merge",
cat: []path.CategoryType{path.EmailCategory}, assist: []testInput{
},
{ {
id: 3, id: 3,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
}, },
expect: []testInput{ otherMerge: []testInput{
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
},
otherAssist: []testInput{
{ {
id: 2, id: 2,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
}, },
}, },
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
}, []testInput{
{
id: 3,
cat: []path.CategoryType{path.EmailCategory},
},
})
return bs
},
}, },
{ {
name: "Other Disjoint", name: "Other Disjoint",
bb: []testInput{ merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
}, },
other: []testInput{ otherMerge: []testInput{
{ {
id: 2, id: 2,
cat: []path.CategoryType{path.ContactsCategory}, cat: []path.CategoryType{path.ContactsCategory},
}, },
{
id: 3,
cat: []path.CategoryType{path.ContactsCategory},
incomplete: true,
}, },
}, expect: func() *backupBases {
expect: []testInput{ bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}}, {cat: []path.CategoryType{path.EmailCategory}},
{ {
id: 1, id: 1,
cat: []path.CategoryType{path.EmailCategory}, cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
}, },
{ {
id: 2, id: 2,
cat: []path.CategoryType{path.ContactsCategory}, cat: []path.CategoryType{path.ContactsCategory},
}, },
{ }, nil)
id: 3,
cat: []path.CategoryType{path.ContactsCategory}, return bs
incomplete: true,
},
},
},
{
name: "Other Reduced Reasons",
bb: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
},
},
other: []testInput{
{
id: 2,
cat: []path.CategoryType{
path.EmailCategory,
path.ContactsCategory,
},
},
{
id: 3,
cat: []path.CategoryType{
path.EmailCategory,
path.ContactsCategory,
},
incomplete: true,
},
},
expect: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
incomplete: true,
},
{
id: 2,
cat: []path.CategoryType{path.ContactsCategory},
},
{
id: 3,
cat: []path.CategoryType{path.ContactsCategory},
incomplete: true,
},
}, },
}, },
} }
@ -443,9 +457,9 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
bb := makeBackupBases(test.bb) bb := makeBackupBases(test.merge, test.assist)
other := makeBackupBases(test.other) other := makeBackupBases(test.otherMerge, test.otherAssist)
expect := makeBackupBases(test.expect) expected := test.expect()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -456,7 +470,7 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
func(r Reasoner) string { func(r Reasoner) string {
return r.Service().String() + r.Category().String() return r.Service().String() + r.Category().String()
}) })
AssertBackupBasesEqual(t, expect, got) AssertBackupBasesEqual(t, expected, got)
}) })
} }
} }
@ -486,8 +500,20 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
mergeBases: []ManifestEntry{ mergeBases: []ManifestEntry{
makeMan(path.EmailCategory, "id1", "", "bid1"), makeMan(path.EmailCategory, "id1", "", "bid1"),
}, },
assistBackups: []BackupEntry{
{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "bid2",
Tags: map[string]string{model.BackupTypeTag: model.AssistBackup},
},
SnapshotID: "id2",
StreamStoreID: "ssid2",
},
},
},
assistBases: []ManifestEntry{ assistBases: []ManifestEntry{
makeMan(path.EmailCategory, "id1", "", "bid1"), makeMan(path.EmailCategory, "id2", "", "bid2"),
}, },
} }
} }
@ -507,24 +533,77 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res := validMail1() res := validMail1()
res.backups = nil res.backups = nil
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
res.backups = nil
return res return res
}(), }(),
}, },
{ {
name: "Backup Missing Snapshot ID", name: "Merge Backup Missing Snapshot ID",
bb: func() *backupBases { bb: func() *backupBases {
res := validMail1() res := validMail1()
res.backups[0].SnapshotID = "" res.backups[0].SnapshotID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
res.backups = nil
return res return res
}(), }(),
}, },
{ {
name: "Backup Missing Deets ID", name: "Assist backup missing snapshot ID",
bb: func() *backupBases {
res := validMail1()
res.assistBackups[0].SnapshotID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = res.mergeBases
res.assistBackups = nil
return res
}(),
},
{
name: "Merge backup missing deets ID",
bb: func() *backupBases { bb: func() *backupBases {
res := validMail1() res := validMail1()
res.backups[0].StreamStoreID = "" res.backups[0].StreamStoreID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
res.backups = nil
return res
}(),
},
{
name: "Assist backup missing deets ID",
bb: func() *backupBases {
res := validMail1()
res.assistBackups[0].StreamStoreID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = res.mergeBases
res.assistBackups = nil
return res return res
}(), }(),
}, },
@ -545,15 +624,22 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons, res.mergeBases[0].Reasons,
res.mergeBases[0].Reasons[0]) res.mergeBases[0].Reasons[0])
res.assistBases = res.mergeBases
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
res.assistBases[0].Reasons[0])
return res return res
}(), }(),
}, },
{ {
name: "Single Valid Entry", name: "Single Valid Entry",
bb: validMail1(), bb: validMail1(),
expect: validMail1(), expect: func() *backupBases {
res := validMail1()
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
}, },
{ {
name: "Single Valid Entry With Incomplete Assist With Same Reason", name: "Single Valid Entry With Incomplete Assist With Same Reason",
@ -561,16 +647,14 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res := validMail1() res := validMail1()
res.assistBases = append( res.assistBases = append(
res.assistBases, res.assistBases,
makeMan(path.EmailCategory, "id2", "checkpoint", "bid2")) makeMan(path.EmailCategory, "id3", "checkpoint", "bid3"))
return res return res
}(), }(),
expect: func() *backupBases { expect: func() *backupBases {
res := validMail1() res := validMail1()
res.assistBases = append(
res.assistBases,
makeMan(path.EmailCategory, "id2", "checkpoint", "bid2"))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res return res
}(), }(),
}, },
@ -581,6 +665,9 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.backups[0].DetailsID = res.backups[0].StreamStoreID res.backups[0].DetailsID = res.backups[0].StreamStoreID
res.backups[0].StreamStoreID = "" res.backups[0].StreamStoreID = ""
res.assistBackups[0].DetailsID = res.assistBackups[0].StreamStoreID
res.assistBackups[0].StreamStoreID = ""
return res return res
}(), }(),
expect: func() *backupBases { expect: func() *backupBases {
@ -588,6 +675,11 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.backups[0].DetailsID = res.backups[0].StreamStoreID res.backups[0].DetailsID = res.backups[0].StreamStoreID
res.backups[0].StreamStoreID = "" res.backups[0].StreamStoreID = ""
res.assistBackups[0].DetailsID = res.assistBackups[0].StreamStoreID
res.assistBackups[0].StreamStoreID = ""
res.assistBases = append(res.mergeBases, res.assistBases...)
return res return res
}(), }(),
}, },
@ -598,7 +690,10 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons, res.mergeBases[0].Reasons,
NewReason("", ro, path.ExchangeService, path.ContactsCategory)) NewReason("", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases = res.mergeBases
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
return res return res
}(), }(),
@ -607,7 +702,12 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons, res.mergeBases[0].Reasons,
NewReason("", ro, path.ExchangeService, path.ContactsCategory)) NewReason("", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases = res.mergeBases
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res return res
}(), }(),
@ -618,14 +718,17 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res := validMail1() res := validMail1()
res.mergeBases = append( res.mergeBases = append(
res.mergeBases, res.mergeBases,
makeMan(path.EmailCategory, "id2", "", "bid2")) makeMan(path.EmailCategory, "id3", "", "bid3"))
res.assistBases = res.mergeBases
res.assistBases = append(
res.assistBases,
makeMan(path.EmailCategory, "id4", "", "bid4"))
return res return res
}(), }(),
}, },
{ {
name: "Three Entries One Invalid", name: "Merge Backup, Three Entries One Invalid",
bb: func() *backupBases { bb: func() *backupBases {
res := validMail1() res := validMail1()
res.backups = append( res.backups = append(
@ -633,24 +736,23 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
BackupEntry{ BackupEntry{
Backup: &backup.Backup{ Backup: &backup.Backup{
BaseModel: model.BaseModel{ BaseModel: model.BaseModel{
ID: "bid2", ID: "bid3",
}, },
}, },
}, },
BackupEntry{ BackupEntry{
Backup: &backup.Backup{ Backup: &backup.Backup{
BaseModel: model.BaseModel{ BaseModel: model.BaseModel{
ID: "bid3", ID: "bid4",
}, },
SnapshotID: "id3", SnapshotID: "id4",
StreamStoreID: "ssid3", StreamStoreID: "ssid4",
}, },
}) })
res.mergeBases = append( res.mergeBases = append(
res.mergeBases, res.mergeBases,
makeMan(path.ContactsCategory, "id2", "checkpoint", "bid2"), makeMan(path.ContactsCategory, "id3", "checkpoint", "bid3"),
makeMan(path.EventsCategory, "id3", "", "bid3")) makeMan(path.EventsCategory, "id4", "", "bid4"))
res.assistBases = res.mergeBases
return res return res
}(), }(),
@ -661,16 +763,70 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
BackupEntry{ BackupEntry{
Backup: &backup.Backup{ Backup: &backup.Backup{
BaseModel: model.BaseModel{ BaseModel: model.BaseModel{
ID: "bid3", ID: "bid4",
}, },
SnapshotID: "id3", SnapshotID: "id4",
StreamStoreID: "ssid3", StreamStoreID: "ssid4",
}, },
}) })
res.mergeBases = append( res.mergeBases = append(
res.mergeBases, res.mergeBases,
makeMan(path.EventsCategory, "id3", "", "bid3")) makeMan(path.EventsCategory, "id4", "", "bid4"))
res.assistBases = res.mergeBases res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
},
{
name: "Assist Backup, Three Entries One Invalid",
bb: func() *backupBases {
res := validMail1()
res.assistBackups = append(
res.assistBackups,
BackupEntry{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "bid3",
Tags: map[string]string{model.BackupTypeTag: model.AssistBackup},
},
},
},
BackupEntry{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "bid4",
Tags: map[string]string{model.BackupTypeTag: model.AssistBackup},
},
SnapshotID: "id4",
StreamStoreID: "ssid4",
},
})
res.assistBases = append(
res.assistBases,
makeMan(path.ContactsCategory, "id3", "checkpoint", "bid3"),
makeMan(path.EventsCategory, "id4", "", "bid4"))
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBackups = append(
res.assistBackups,
BackupEntry{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "bid4",
Tags: map[string]string{model.BackupTypeTag: model.AssistBackup},
},
SnapshotID: "id4",
StreamStoreID: "ssid4",
},
})
res.assistBases = append(
res.assistBases,
makeMan(path.EventsCategory, "id4", "", "bid4"))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res return res
}(), }(),

View File

@ -204,17 +204,20 @@ func (b *baseFinder) getBackupModel(
return bup, nil return bup, nil
} }
type backupBase struct {
backup BackupEntry
manifest ManifestEntry
}
// findBasesInSet goes through manifest metadata entries and sees if they're // findBasesInSet goes through manifest metadata entries and sees if they're
// incomplete or not. If an entry is incomplete and we don't already have a // incomplete or not. Manifests which don't have an associated backup
// complete or incomplete manifest add it to the set for kopia assisted // are discarded as incomplete. Manifests are then checked to see if they
// incrementals. If it's complete, fetch the backup model and see if it // are associated with an assist backup or merge backup.
// corresponds to a successful backup. If it does, return it as we only need the
// most recent complete backup as the base.
func (b *baseFinder) findBasesInSet( func (b *baseFinder) findBasesInSet(
ctx context.Context, ctx context.Context,
reason Reasoner, reason Reasoner,
metas []*manifest.EntryMetadata, metas []*manifest.EntryMetadata,
) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { ) (*backupBase, *backupBase, error) {
// Sort manifests by time so we can go through them sequentially. The code in // Sort manifests by time so we can go through them sequentially. The code in
// kopia appears to sort them already, but add sorting here just so we're not // kopia appears to sort them already, but add sorting here just so we're not
// reliant on undocumented behavior. // reliant on undocumented behavior.
@ -223,8 +226,8 @@ func (b *baseFinder) findBasesInSet(
}) })
var ( var (
kopiaAssistSnaps []ManifestEntry mergeBase *backupBase
foundIncomplete bool assistBase *backupBase
) )
for i := len(metas) - 1; i >= 0; i-- { for i := len(metas) - 1; i >= 0; i-- {
@ -240,16 +243,10 @@ func (b *baseFinder) findBasesInSet(
} }
if len(man.IncompleteReason) > 0 { if len(man.IncompleteReason) > 0 {
if !foundIncomplete { // Skip here since this snapshot cannot be considered an assist base.
foundIncomplete = true logger.Ctx(ictx).Debugw(
"Incomplete snapshot",
kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ "incomplete_reason", man.IncompleteReason)
Manifest: man,
Reasons: []Reasoner{reason},
})
logger.Ctx(ictx).Info("found incomplete backup")
}
continue continue
} }
@ -259,19 +256,7 @@ func (b *baseFinder) findBasesInSet(
if err != nil { if err != nil {
// Safe to continue here as we'll just end up attempting to use an older // Safe to continue here as we'll just end up attempting to use an older
// backup as the base. // backup as the base.
logger.CtxErr(ictx, err).Debug("searching for base backup") logger.CtxErr(ictx, err).Debug("searching for backup model")
if !foundIncomplete {
foundIncomplete = true
kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{
Manifest: man,
Reasons: []Reasoner{reason},
})
logger.Ctx(ictx).Info("found incomplete backup")
}
continue continue
} }
@ -285,49 +270,117 @@ func (b *baseFinder) findBasesInSet(
"empty backup stream store ID", "empty backup stream store ID",
"search_backup_id", bup.ID) "search_backup_id", bup.ID)
if !foundIncomplete {
foundIncomplete = true
kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{
Manifest: man,
Reasons: []Reasoner{reason},
})
logger.Ctx(ictx).Infow(
"found incomplete backup",
"search_backup_id", bup.ID)
}
continue continue
} }
// If we've made it to this point then we're considering the backup // If we've made it to this point then we're considering the backup
// complete as it has both an item data snapshot and a backup details // complete as it has both an item data snapshot and a backup details
// snapshot. // snapshot.
logger.Ctx(ictx).Infow("found complete backup", "base_backup_id", bup.ID) //
// Check first if this is an assist base. Criteria for selecting an
// assist base are:
// 1. most recent assist base for the reason.
// 2. at most one assist base per reason.
// 3. it must be more recent than the merge backup for the reason, if
// a merge backup exists.
me := ManifestEntry{ if b.isAssistBackupModel(ictx, bup) {
if assistBase == nil {
assistModel := BackupEntry{
Backup: bup,
Reasons: []Reasoner{reason},
}
assistSnap := ManifestEntry{
Manifest: man, Manifest: man,
Reasons: []Reasoner{reason}, Reasons: []Reasoner{reason},
} }
kopiaAssistSnaps = append(kopiaAssistSnaps, me)
return &BackupEntry{ assistBase = &backupBase{
Backup: bup, backup: assistModel,
Reasons: []Reasoner{reason}, manifest: assistSnap,
}, &me, kopiaAssistSnaps, nil
} }
logger.Ctx(ctx).Info("no base backups for reason") logger.Ctx(ictx).Infow(
"found assist base",
"search_backup_id", bup.ID,
"search_snapshot_id", meta.ID,
"ssid", ssid)
}
return nil, nil, kopiaAssistSnaps, nil // Skip if an assist base has already been selected.
continue
}
logger.Ctx(ictx).Infow("found merge base",
"search_backup_id", bup.ID,
"search_snapshot_id", meta.ID,
"ssid", ssid)
mergeSnap := ManifestEntry{
Manifest: man,
Reasons: []Reasoner{reason},
}
mergeModel := BackupEntry{
Backup: bup,
Reasons: []Reasoner{reason},
}
mergeBase = &backupBase{
backup: mergeModel,
manifest: mergeSnap,
}
break
}
if mergeBase == nil && assistBase == nil {
logger.Ctx(ctx).Info("no merge or assist base found for reason")
}
return mergeBase, assistBase, nil
}
// isAssistBackupModel checks if the provided backup is an assist backup.
func (b *baseFinder) isAssistBackupModel(
ctx context.Context,
bup *backup.Backup,
) bool {
allTags := map[string]string{
model.BackupTypeTag: model.AssistBackup,
}
for k, v := range allTags {
if bup.Tags[k] != v {
// This is not an assist backup so we can just exit here.
logger.Ctx(ctx).Debugw(
"assist backup model missing tags",
"backup_id", bup.ID,
"tag", k,
"expected_value", v,
"actual_value", bup.Tags[k])
return false
}
}
// Check if it has a valid streamstore id and snapshot id.
if len(bup.StreamStoreID) == 0 || len(bup.SnapshotID) == 0 {
logger.Ctx(ctx).Infow(
"nil ssid or snapshot id in assist base",
"ssid", bup.StreamStoreID,
"snapshot_id", bup.SnapshotID)
return false
}
return true
} }
func (b *baseFinder) getBase( func (b *baseFinder) getBase(
ctx context.Context, ctx context.Context,
r Reasoner, r Reasoner,
tags map[string]string, tags map[string]string,
) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { ) (*backupBase, *backupBase, error) {
allTags := map[string]string{} allTags := map[string]string{}
for _, k := range tagKeys(r) { for _, k := range tagKeys(r) {
@ -339,12 +392,12 @@ func (b *baseFinder) getBase(
metas, err := b.sm.FindManifests(ctx, allTags) metas, err := b.sm.FindManifests(ctx, allTags)
if err != nil { if err != nil {
return nil, nil, nil, clues.Wrap(err, "getting snapshots") return nil, nil, clues.Wrap(err, "getting snapshots")
} }
// No snapshots means no backups so we can just exit here. // No snapshots means no backups so we can just exit here.
if len(metas) == 0 { if len(metas) == 0 {
return nil, nil, nil, nil return nil, nil, nil
} }
return b.findBasesInSet(ctx, r, metas) return b.findBasesInSet(ctx, r, metas)
@ -360,9 +413,10 @@ func (b *baseFinder) FindBases(
// the reason for selecting something. Kopia assisted snapshots also use // the reason for selecting something. Kopia assisted snapshots also use
// ManifestEntry so we have the reasons for selecting them to aid in // ManifestEntry so we have the reasons for selecting them to aid in
// debugging. // debugging.
baseBups = map[model.StableID]BackupEntry{} mergeBups = map[model.StableID]BackupEntry{}
baseSnaps = map[manifest.ID]ManifestEntry{} assistBups = map[model.StableID]BackupEntry{}
kopiaAssistSnaps = map[manifest.ID]ManifestEntry{} mergeSnaps = map[manifest.ID]ManifestEntry{}
assistSnaps = map[manifest.ID]ManifestEntry{}
) )
for _, searchReason := range reasons { for _, searchReason := range reasons {
@ -372,7 +426,10 @@ func (b *baseFinder) FindBases(
"search_category", searchReason.Category().String()) "search_category", searchReason.Category().String())
logger.Ctx(ictx).Info("searching for previous manifests") logger.Ctx(ictx).Info("searching for previous manifests")
baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, searchReason, tags) mergeBase, assistBase, err := b.getBase(
ictx,
searchReason,
tags)
if err != nil { if err != nil {
logger.Ctx(ctx).Info( logger.Ctx(ctx).Info(
"getting base, falling back to full backup for reason", "getting base, falling back to full backup for reason",
@ -381,47 +438,60 @@ func (b *baseFinder) FindBases(
continue continue
} }
if baseBackup != nil { if mergeBase != nil {
bs, ok := baseBups[baseBackup.ID] mergeSnap := mergeBase.manifest
mergeBackup := mergeBase.backup
ms, ok := mergeSnaps[mergeSnap.ID]
if ok { if ok {
bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) ms.Reasons = append(ms.Reasons, mergeSnap.Reasons...)
} else { } else {
bs = *baseBackup ms = mergeSnap
} }
// Reassign since it's structs not pointers to structs. mergeSnaps[mergeSnap.ID] = ms
baseBups[baseBackup.ID] = bs
}
if baseSnap != nil { mb, ok := mergeBups[mergeBackup.ID]
bs, ok := baseSnaps[baseSnap.ID]
if ok { if ok {
bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) mb.Reasons = append(mb.Reasons, mergeSnap.Reasons...)
} else { } else {
bs = *baseSnap mb = mergeBackup
} }
// Reassign since it's structs not pointers to structs. mergeBups[mergeBackup.ID] = mb
baseSnaps[baseSnap.ID] = bs
} }
for _, s := range assistSnaps { if assistBase != nil {
bs, ok := kopiaAssistSnaps[s.ID] assistSnap := assistBase.manifest
assistBackup := assistBase.backup
as, ok := assistSnaps[assistSnap.ID]
if ok { if ok {
bs.Reasons = append(bs.Reasons, s.Reasons...) as.Reasons = append(as.Reasons, assistSnap.Reasons...)
} else { } else {
bs = s as = assistSnap
} }
// Reassign since it's structs not pointers to structs. assistSnaps[assistSnap.ID] = as
kopiaAssistSnaps[s.ID] = bs
ab, ok := assistBups[assistBackup.ID]
if ok {
ab.Reasons = append(ab.Reasons, assistBackup.Reasons...)
} else {
ab = assistBackup
}
assistBups[assistBackup.ID] = ab
} }
} }
// TODO(pandeyabs): Fix the terminology used in backupBases to go with
// new definitions i.e. mergeSnaps instead of mergeBases, etc.
res := &backupBases{ res := &backupBases{
backups: maps.Values(baseBups), backups: maps.Values(mergeBups),
mergeBases: maps.Values(baseSnaps), assistBackups: maps.Values(assistBups),
assistBases: maps.Values(kopiaAssistSnaps), mergeBases: maps.Values(mergeSnaps),
assistBases: maps.Values(assistSnaps),
} }
res.fixupAndVerify(ctx) res.fixupAndVerify(ctx)

View File

@ -25,12 +25,17 @@ const (
var ( var (
testT1 = time.Now() testT1 = time.Now()
testT2 = testT1.Add(1 * time.Hour) testT2 = testT1.Add(1 * time.Hour)
testT3 = testT2.Add(1 * time.Hour)
testT4 = testT3.Add(1 * time.Hour)
testID1 = manifest.ID("snap1") testID1 = manifest.ID("snap1")
testID2 = manifest.ID("snap2") testID2 = manifest.ID("snap2")
testID3 = manifest.ID("snap3")
testID4 = manifest.ID("snap4")
testBackup1 = "backupID1" testBackup1 = "backupID1"
testBackup2 = "backupID2" testBackup2 = "backupID2"
testBackup3 = "backupID3"
testBackup4 = "backupID4"
testMail = path.ExchangeService.String() + path.EmailCategory.String() testMail = path.ExchangeService.String() + path.EmailCategory.String()
testEvents = path.ExchangeService.String() + path.EventsCategory.String() testEvents = path.ExchangeService.String() + path.EventsCategory.String()
@ -212,12 +217,14 @@ func newBackupModel(
hasItemSnap bool, hasItemSnap bool,
hasDetailsSnap bool, hasDetailsSnap bool,
oldDetailsID bool, oldDetailsID bool,
tags map[string]string,
err error, err error,
) backupInfo { ) backupInfo {
res := backupInfo{ res := backupInfo{
b: backup.Backup{ b: backup.Backup{
BaseModel: model.BaseModel{ BaseModel: model.BaseModel{
ID: model.StableID(id), ID: model.StableID(id),
Tags: tags,
}, },
SnapshotID: "iid", SnapshotID: "iid",
}, },
@ -323,11 +330,14 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons map[int][]Reasoner expectedBaseReasons map[int][]Reasoner
// Use this to denote the Reasons a kopia assised incrementals manifest is // Use this to denote the Reasons a kopia assised incrementals manifest is
// selected. The int maps to the index of the manifest in data. // selected. The int maps to the index of the manifest in data.
// TODO(pandeyabs): Remove this once we have 1:1 mapping between snapshots
// and backup models.
expectedAssistManifestReasons map[int][]Reasoner expectedAssistManifestReasons map[int][]Reasoner
expectedAssistReasons map[int][]Reasoner
backupData []backupInfo backupData []backupInfo
}{ }{
{ {
name: "Return Older Base If Fail To Get Manifest", name: "Return Older Merge Base If Fail To Get Manifest",
input: testUser1Mail, input: testUser1Mail,
manifestData: []manifestInfo{ manifestData: []manifestInfo{
newManifestInfo( newManifestInfo(
@ -355,13 +365,55 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
1: testUser1Mail, 1: testUser1Mail,
}, },
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
}, },
}, },
{ {
name: "Return Older Base If Fail To Get Backup", name: "Return Older Assist Base If Fail To Get Manifest",
input: testUser1Mail,
manifestData: []manifestInfo{
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
assert.AnError,
testMail,
testUser1,
),
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testUser1,
),
},
expectedBaseReasons: map[int][]Reasoner{},
expectedAssistManifestReasons: map[int][]Reasoner{
1: testUser1Mail,
},
expectedAssistReasons: map[int][]Reasoner{
1: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(
testBackup1,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
},
},
{
name: "Return Older Merge Base If Fail To Get Backup",
input: testUser1Mail, input: testUser1Mail,
manifestData: []manifestInfo{ manifestData: []manifestInfo{
newManifestInfo( newManifestInfo(
@ -387,12 +439,11 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
1: testUser1Mail, 1: testUser1Mail,
}, },
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
1: testUser1Mail, 1: testUser1Mail,
}, },
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup2, false, false, false, assert.AnError), newBackupModel(testBackup2, false, false, false, nil, assert.AnError),
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
}, },
}, },
{ {
@ -422,12 +473,12 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
1: testUser1Mail, 1: testUser1Mail,
}, },
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
1: testUser1Mail, 1: testUser1Mail,
}, },
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup2, true, false, false, nil), newBackupModel(testBackup2, true, false, false, nil, nil),
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
}, },
}, },
{ {
@ -453,12 +504,13 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail, 0: testUser1Mail,
}, },
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup1, true, true, true, nil), newBackupModel(testBackup1, true, true, true, nil, nil),
}, },
}, },
{ {
name: "All One Snapshot", name: "All One Snapshot With Merge Base",
input: testAllUsersAllCats, input: testAllUsersAllCats,
manifestData: []manifestInfo{ manifestData: []manifestInfo{
newManifestInfo( newManifestInfo(
@ -480,8 +532,43 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
0: testAllUsersAllCats, 0: testAllUsersAllCats,
}, },
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
},
},
{
name: "All One Snapshot with Assist Base",
input: testAllUsersAllCats,
manifestData: []manifestInfo{
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testEvents,
testUser1,
testUser2,
testUser3,
),
},
expectedBaseReasons: map[int][]Reasoner{},
expectedAssistManifestReasons: map[int][]Reasoner{
0: testAllUsersAllCats,
},
expectedAssistReasons: map[int][]Reasoner{
0: testAllUsersAllCats,
},
backupData: []backupInfo{
newBackupModel(
testBackup1,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
}, },
}, },
{ {
@ -537,8 +624,96 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
}, },
}, },
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
},
},
{
name: "Unique assist bases with common merge Base, overlapping reasons",
input: testAllUsersAllCats,
manifestData: []manifestInfo{
newManifestInfo(
testID3,
testT3,
testCompleteMan,
testBackup3,
nil,
testEvents,
testUser1,
testUser2,
),
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
nil,
testMail,
testUser1,
testUser2,
),
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testEvents,
testUser1,
testUser2,
),
},
expectedBaseReasons: map[int][]Reasoner{
2: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
},
expectedAssistManifestReasons: map[int][]Reasoner{
0: {
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
1: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
},
2: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
},
expectedAssistReasons: map[int][]Reasoner{
0: {
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
1: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
},
},
backupData: []backupInfo{
newBackupModel(
testBackup3,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
newBackupModel(
testBackup2,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
newBackupModel(testBackup1, true, true, false, nil, nil),
}, },
}, },
{ {
@ -569,12 +744,11 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
}, },
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail, 0: testUser1Mail,
1: testUser1Mail,
}, },
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
// Shouldn't be returned but have here just so we can see. // Shouldn't be returned but have here just so we can see.
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
}, },
}, },
{ {
@ -608,8 +782,8 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
}, },
backupData: []backupInfo{ backupData: []backupInfo{
// Shouldn't be returned but have here just so we can see. // Shouldn't be returned but have here just so we can see.
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
}, },
}, },
{ {
@ -636,13 +810,11 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
), ),
}, },
expectedBaseReasons: map[int][]Reasoner{}, expectedBaseReasons: map[int][]Reasoner{},
expectedAssistManifestReasons: map[int][]Reasoner{ expectedAssistManifestReasons: map[int][]Reasoner{},
1: testUser1Mail,
},
backupData: []backupInfo{ backupData: []backupInfo{
// Shouldn't be returned but have here just so we can see. // Shouldn't be returned but have here just so we can see.
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
}, },
}, },
{ {
@ -666,7 +838,7 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
0: testUser1Mail, 0: testUser1Mail,
}, },
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
}, },
}, },
{ {
@ -701,9 +873,199 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
0: testUser1Mail, 0: testUser1Mail,
}, },
backupData: []backupInfo{ backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil), newBackupModel(testBackup2, true, true, false, nil, nil),
// Shouldn't be returned but here just so we can check. // Shouldn't be returned but here just so we can check.
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
},
},
{
name: "Return latest assist & merge base pair",
input: testUser1Mail,
manifestData: []manifestInfo{
newManifestInfo(
testID4,
testT4,
testCompleteMan,
testBackup4,
nil,
testMail,
testUser1,
),
newManifestInfo(
testID3,
testT3,
testCompleteMan,
testBackup3,
nil,
testMail,
testUser1,
),
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
nil,
testMail,
testUser1,
),
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testUser1,
),
},
expectedBaseReasons: map[int][]Reasoner{
2: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
2: testUser1Mail,
},
expectedAssistReasons: map[int][]Reasoner{
0: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(
testBackup4,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
newBackupModel(
testBackup3,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(testBackup1, true, true, false, nil, nil),
},
},
{
name: "Newer merge base than assist base",
input: testUser1Mail,
manifestData: []manifestInfo{
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
nil,
testMail,
testUser1,
),
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testUser1,
),
},
expectedBaseReasons: map[int][]Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(
testBackup1,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
},
},
{
name: "Only assist bases",
input: testUser1Mail,
manifestData: []manifestInfo{
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
nil,
testMail,
testUser1,
),
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testBackup1,
nil,
testMail,
testUser1,
),
},
expectedBaseReasons: map[int][]Reasoner{},
expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]Reasoner{
0: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(
testBackup2,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
newBackupModel(
testBackup1,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.AssistBackup},
nil),
},
},
{
name: "Merge base with tag",
input: testUser1Mail,
manifestData: []manifestInfo{
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testBackup2,
nil,
testMail,
testUser1,
),
},
expectedBaseReasons: map[int][]Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(
testBackup1,
true,
true,
false,
map[string]string{model.BackupTypeTag: model.MergeBackup},
nil),
}, },
}, },
} }
@ -730,6 +1092,12 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
bb.Backups(), bb.Backups(),
test.backupData, test.backupData,
test.expectedBaseReasons) test.expectedBaseReasons)
checkBackupEntriesMatch(
t,
bb.AssistBackups(),
test.backupData,
test.expectedAssistReasons)
checkManifestEntriesMatch( checkManifestEntriesMatch(
t, t,
bb.MergeBases(), bb.MergeBases(),
@ -759,7 +1127,7 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() {
), ),
} }
backupData := []backupInfo{ backupData := []backupInfo{
newBackupModel(testBackup1, true, true, false, nil), newBackupModel(testBackup1, true, true, false, nil, nil),
} }
table := []struct { table := []struct {

View File

@ -77,6 +77,8 @@ func getManifestsAndMetadata(
// 2. the current reasons only contain an incomplete manifest, and the fallback // 2. the current reasons only contain an incomplete manifest, and the fallback
// can find a complete manifest. // can find a complete manifest.
// 3. the current reasons contain all the necessary manifests. // 3. the current reasons contain all the necessary manifests.
// Note: This is not relevant for assist backups, since they are newly introduced
// and they don't exist with fallback reasons.
bb = bb.MergeBackupBases( bb = bb.MergeBackupBases(
ctx, ctx,
fbb, fbb,