Track assist bases and merge bases separately (#4181)

Do a few things to separate merge and assist bases
* use distinct sets for each base type
* make another function to return all base snapshots to pass to kopia assisted incrementals
* update function that removes a merge base to make it into an assist base

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #3943
* #4178

#### Test Plan

- [ ] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2023-09-11 17:19:21 -07:00 committed by GitHub
parent a610df9d68
commit db86ccabf8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 223 additions and 272 deletions

View File

@ -15,7 +15,9 @@ import (
// TODO(ashmrtn): Move this into some inject package. Here to avoid import
// cycles.
type BackupBases interface {
RemoveMergeBaseByManifestID(manifestID manifest.ID)
// ConvertToAssistBase converts the base with the given item data snapshot ID
// from a merge base to an assist base.
ConvertToAssistBase(manifestID manifest.ID)
Backups() []BackupEntry
UniqueAssistBackups() []BackupEntry
MinBackupVersion() int
@ -28,6 +30,11 @@ type BackupBases interface {
other BackupBases,
reasonToKey func(identity.Reasoner) string,
) BackupBases
// SnapshotAssistBases returns the set of bases to use for kopia assisted
// incremental snapshot operations. It consists of the union of merge bases
// and assist bases. If DisableAssistBases has been called then it returns
// nil.
SnapshotAssistBases() []ManifestEntry
}
type backupBases struct {
@ -46,35 +53,49 @@ type backupBases struct {
disableMergeBases bool
}
func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) {
func (bb *backupBases) SnapshotAssistBases() []ManifestEntry {
if bb.disableAssistBases {
return nil
}
// Need to use the actual variables here because the functions will return nil
// depending on what's been marked as disabled.
return append(slices.Clone(bb.assistBases), bb.mergeBases...)
}
func (bb *backupBases) ConvertToAssistBase(manifestID manifest.ID) {
var (
snapshotMan ManifestEntry
base BackupEntry
snapFound bool
)
idx := slices.IndexFunc(
bb.mergeBases,
func(man ManifestEntry) bool {
return man.ID == manifestID
})
if idx >= 0 {
snapFound = true
snapshotMan = bb.mergeBases[idx]
bb.mergeBases = slices.Delete(bb.mergeBases, idx, idx+1)
}
// TODO(ashmrtn): This may not be strictly necessary but is at least easier to
// reason about.
idx = slices.IndexFunc(
bb.assistBases,
func(man ManifestEntry) bool {
return man.ID == manifestID
})
if idx >= 0 {
bb.assistBases = slices.Delete(bb.assistBases, idx, idx+1)
}
idx = slices.IndexFunc(
bb.backups,
func(bup BackupEntry) bool {
return bup.SnapshotID == string(manifestID)
})
if idx >= 0 {
base = bb.backups[idx]
bb.backups = slices.Delete(bb.backups, idx, idx+1)
}
// Account for whether we found the backup.
if idx >= 0 && snapFound {
bb.assistBackups = append(bb.assistBackups, base)
bb.assistBases = append(bb.assistBases, snapshotMan)
}
}
func (bb backupBases) Backups() []BackupEntry {
@ -241,9 +262,6 @@ func (bb *backupBases) MergeBackupBases(
res.backups = append(res.backups, bup)
res.mergeBases = append(res.mergeBases, man)
// TODO(pandeyabs): Remove this once we remove overlap between
// between merge and assist bases as part of #3943.
res.assistBases = append(res.assistBases, man)
}
return res
@ -383,16 +401,6 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
mergeToKeep = append(mergeToKeep, man)
}
// Every merge base is also a kopia assist base.
// TODO(pandeyabs): This should be removed as part of #3943.
for _, man := range bb.mergeBases {
if _, ok := toDrop[man.ID]; ok {
continue
}
assistToKeep = append(assistToKeep, man)
}
// Drop assist snapshots with overlapping reasons.
toDropAssists := findNonUniqueManifests(ctx, bb.assistBases)

View File

@ -84,7 +84,7 @@ func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
}
}
func (suite *BackupBasesUnitSuite) TestRemoveMergeBaseByManifestID() {
func (suite *BackupBasesUnitSuite) TestConvertToAssistBase() {
backups := []BackupEntry{
{Backup: &backup.Backup{SnapshotID: "1"}},
{Backup: &backup.Backup{SnapshotID: "2"}},
@ -97,68 +97,71 @@ func (suite *BackupBasesUnitSuite) TestRemoveMergeBaseByManifestID() {
makeManifest("3", "", ""),
}
expected := &backupBases{
backups: []BackupEntry{backups[0], backups[1]},
mergeBases: []ManifestEntry{merges[0], merges[1]},
assistBases: []ManifestEntry{merges[0], merges[1]},
}
delID := manifest.ID("3")
table := []struct {
name string
// Below indices specify which items to add from the defined sets above.
backup []int
merge []int
assist []int
backup []int
merge []int
assist []int
expectAssist []int
}{
{
name: "Not In Bases",
backup: []int{0, 1},
merge: []int{0, 1},
assist: []int{0, 1},
name: "Not In Bases",
backup: []int{0, 1},
merge: []int{0, 1},
assist: []int{0, 1},
expectAssist: []int{0, 1},
},
{
name: "Different Indexes",
backup: []int{2, 0, 1},
merge: []int{0, 2, 1},
assist: []int{0, 1, 2},
name: "Different Indexes",
backup: []int{2, 0, 1},
merge: []int{0, 2, 1},
assist: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "First Item",
backup: []int{2, 0, 1},
merge: []int{2, 0, 1},
assist: []int{2, 0, 1},
name: "First Item",
backup: []int{2, 0, 1},
merge: []int{2, 0, 1},
assist: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Middle Item",
backup: []int{0, 2, 1},
merge: []int{0, 2, 1},
assist: []int{0, 2, 1},
name: "Middle Item",
backup: []int{0, 2, 1},
merge: []int{0, 2, 1},
assist: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Final Item",
backup: []int{0, 1, 2},
merge: []int{0, 1, 2},
assist: []int{0, 1, 2},
name: "Final Item",
backup: []int{0, 1, 2},
merge: []int{0, 1, 2},
assist: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Only In Backups",
backup: []int{0, 1, 2},
merge: []int{0, 1},
assist: []int{0, 1},
name: "Only In Backups",
backup: []int{0, 1, 2},
merge: []int{0, 1},
assist: []int{0, 1},
expectAssist: []int{0, 1},
},
{
name: "Only In Merges",
backup: []int{0, 1},
merge: []int{0, 1, 2},
assist: []int{0, 1},
name: "Only In Merges",
backup: []int{0, 1},
merge: []int{0, 1, 2},
assist: []int{0, 1},
expectAssist: []int{0, 1},
},
{
name: "Only In Assists",
backup: []int{0, 1},
merge: []int{0, 1},
assist: []int{0, 1, 2},
name: "Only In Assists Noops",
backup: []int{0, 1},
merge: []int{0, 1},
assist: []int{0, 1, 2},
expectAssist: []int{0, 1, 2},
},
}
@ -177,9 +180,20 @@ func (suite *BackupBasesUnitSuite) TestRemoveMergeBaseByManifestID() {
for _, i := range test.assist {
bb.assistBases = append(bb.assistBases, merges[i])
bb.assistBackups = append(bb.assistBackups, backups[i])
}
bb.RemoveMergeBaseByManifestID(delID)
expected := &backupBases{
backups: []BackupEntry{backups[0], backups[1]},
mergeBases: []ManifestEntry{merges[0], merges[1]},
}
for _, i := range test.expectAssist {
expected.assistBases = append(expected.assistBases, merges[i])
expected.assistBackups = append(expected.assistBackups, backups[i])
}
bb.ConvertToAssistBase(delID)
AssertBackupBasesEqual(t, expected, bb)
})
}
@ -201,6 +215,9 @@ func (suite *BackupBasesUnitSuite) TestDisableMergeBases() {
// Assist base set should be unchanged.
assert.Len(t, bb.UniqueAssistBackups(), 2)
assert.Len(t, bb.UniqueAssistBases(), 2)
// Merge bases should still appear in the assist base set passed in for kopia
// snapshots.
assert.Len(t, bb.SnapshotAssistBases(), 4)
}
func (suite *BackupBasesUnitSuite) TestDisableAssistBases() {
@ -215,6 +232,7 @@ func (suite *BackupBasesUnitSuite) TestDisableAssistBases() {
bb.DisableAssistBases()
assert.Empty(t, bb.UniqueAssistBases())
assert.Empty(t, bb.UniqueAssistBackups())
assert.Empty(t, bb.SnapshotAssistBases())
// Merge base should be unchanged.
assert.Len(t, bb.Backups(), 2)
@ -242,7 +260,6 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
}
m := makeManifest(baseID, "", "b"+baseID, reasons...)
res.assistBases = append(res.assistBases, m)
b := BackupEntry{
Backup: &backup.Backup{
@ -588,7 +605,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = res.mergeBases
res.assistBases = nil
res.assistBackups = nil
return res
@ -620,7 +637,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = res.mergeBases
res.assistBases = nil
res.assistBackups = nil
return res
@ -651,14 +668,9 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
}(),
},
{
name: "Single Valid Entry",
bb: validMail1(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
name: "Single Valid Entry",
bb: validMail1(),
expect: validMail1(),
},
{
name: "Single Valid Entry With Incomplete Assist With Same Reason",
@ -670,12 +682,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
expect: validMail1(),
},
{
name: "Single Valid Entry With Backup With Old Deets ID",
@ -697,8 +704,6 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.assistBackups[0].DetailsID = res.assistBackups[0].StreamStoreID
res.assistBackups[0].StreamStoreID = ""
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
},
@ -726,8 +731,6 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.assistBases[0].Reasons,
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
},
@ -791,7 +794,6 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.mergeBases = append(
res.mergeBases,
makeMan(path.EventsCategory, "id4", "", "bid4"))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
@ -845,8 +847,6 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
res.assistBases,
makeMan(path.EventsCategory, "id4", "", "bid4"))
res.assistBases = append(res.mergeBases, res.assistBases...)
return res
}(),
},

View File

@ -300,6 +300,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() {
bb := bf.FindBases(ctx, reasons, nil)
assert.Empty(t, bb.MergeBases())
assert.Empty(t, bb.UniqueAssistBases())
assert.Empty(t, bb.SnapshotAssistBases())
}
func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() {
@ -319,6 +320,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() {
bb := bf.FindBases(ctx, reasons, nil)
assert.Empty(t, bb.MergeBases())
assert.Empty(t, bb.UniqueAssistBases())
assert.Empty(t, bb.SnapshotAssistBases())
}
func (suite *BaseFinderUnitSuite) TestGetBases() {
@ -331,11 +333,8 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons map[int][]identity.Reasoner
// Use this to denote the Reasons a kopia assised incrementals manifest is
// selected. The int maps to the index of the manifest in data.
// TODO(pandeyabs): Remove this once we have 1:1 mapping between snapshots
// and backup models.
expectedAssistManifestReasons map[int][]identity.Reasoner
expectedAssistReasons map[int][]identity.Reasoner
backupData []backupInfo
expectedAssistReasons map[int][]identity.Reasoner
backupData []backupInfo
}{
{
name: "Return Older Merge Base If Fail To Get Manifest",
@ -361,10 +360,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(testBackup1, true, true, false, nil, nil),
@ -391,10 +386,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
testMail,
testUser1),
},
expectedBaseReasons: map[int][]identity.Reasoner{},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
@ -433,9 +424,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(testBackup2, false, false, false, nil, assert.AnError),
newBackupModel(testBackup1, true, true, false, nil, nil),
@ -465,10 +453,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, false, false, nil, nil),
newBackupModel(testBackup1, true, true, false, nil, nil),
@ -493,10 +477,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup1, true, true, true, nil, nil),
},
@ -520,10 +500,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testAllUsersAllCats,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testAllUsersAllCats,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil, nil),
},
@ -544,10 +520,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
testUser2,
testUser3),
},
expectedBaseReasons: map[int][]identity.Reasoner{},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testAllUsersAllCats,
},
expectedAssistReasons: map[int][]identity.Reasoner{
0: testAllUsersAllCats,
},
@ -599,18 +571,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
},
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
},
1: {
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
},
},
backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil, nil),
newBackupModel(testBackup2, true, true, false, nil, nil),
@ -657,22 +617,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: {
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
1: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
},
2: {
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
},
},
expectedAssistReasons: map[int][]identity.Reasoner{
0: {
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
@ -725,9 +669,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil, nil),
// Shouldn't be returned but have here just so we can see.
@ -758,9 +699,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
1: testUser1Mail,
},
backupData: []backupInfo{
// Shouldn't be returned but have here just so we can see.
newBackupModel(testBackup1, true, true, false, nil, nil),
@ -788,8 +726,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
testMail,
testUser1),
},
expectedBaseReasons: map[int][]identity.Reasoner{},
expectedAssistManifestReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
// Shouldn't be returned but have here just so we can see.
newBackupModel(testBackup1, true, true, false, nil, nil),
@ -812,9 +748,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(testBackup1, true, true, false, nil, nil),
},
@ -845,9 +778,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
// Shouldn't be returned but here just so we can check.
@ -894,10 +824,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
2: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
2: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
@ -944,10 +870,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(
@ -980,10 +902,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
testMail,
testUser1),
},
expectedBaseReasons: map[int][]identity.Reasoner{},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
@ -1020,10 +938,6 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
expectedBaseReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistManifestReasons: map[int][]identity.Reasoner{
0: testUser1Mail,
},
expectedAssistReasons: map[int][]identity.Reasoner{},
backupData: []backupInfo{
newBackupModel(testBackup2, true, true, false, nil, nil),
newBackupModel(
@ -1074,7 +988,7 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
t,
bb.UniqueAssistBases(),
test.manifestData,
test.expectedAssistManifestReasons)
test.expectedAssistReasons)
})
}
}

View File

@ -16,6 +16,7 @@ func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) {
assert.Empty(t, got.MergeBases(), "merge bases")
assert.Empty(t, got.UniqueAssistBackups(), "assist backups")
assert.Empty(t, got.UniqueAssistBases(), "assist bases")
assert.Empty(t, got.SnapshotAssistBases(), "snapshot assist bases")
return
}
@ -24,7 +25,8 @@ func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) {
if len(expect.Backups()) > 0 ||
len(expect.MergeBases()) > 0 ||
len(expect.UniqueAssistBackups()) > 0 ||
len(expect.UniqueAssistBases()) > 0 {
len(expect.UniqueAssistBases()) > 0 ||
len(expect.SnapshotAssistBases()) > 0 {
assert.Fail(t, "got was nil but expected non-nil result %v", expect)
}
@ -35,6 +37,7 @@ func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) {
assert.ElementsMatch(t, expect.MergeBases(), got.MergeBases(), "merge bases")
assert.ElementsMatch(t, expect.UniqueAssistBackups(), got.UniqueAssistBackups(), "assist backups")
assert.ElementsMatch(t, expect.UniqueAssistBases(), got.UniqueAssistBases(), "assist bases")
assert.ElementsMatch(t, expect.SnapshotAssistBases(), got.SnapshotAssistBases(), "snapshot assist bases")
}
func NewMockBackupBases() *MockBackupBases {
@ -52,8 +55,6 @@ func (bb *MockBackupBases) WithBackups(b ...BackupEntry) *MockBackupBases {
func (bb *MockBackupBases) WithMergeBases(m ...ManifestEntry) *MockBackupBases {
bb.backupBases.mergeBases = append(bb.MergeBases(), m...)
bb.backupBases.assistBases = append(bb.UniqueAssistBases(), m...)
return bb
}
@ -67,7 +68,12 @@ func (bb *MockBackupBases) WithAssistBases(m ...ManifestEntry) *MockBackupBases
return bb
}
func (bb *MockBackupBases) ClearMockAssistBases() *MockBackupBases {
bb.backupBases.DisableAssistBases()
func (bb *MockBackupBases) MockDisableAssistBases() *MockBackupBases {
bb.DisableAssistBases()
return bb
}
func (bb *MockBackupBases) MockDisableMergeBases() *MockBackupBases {
bb.DisableMergeBases()
return bb
}

View File

@ -178,7 +178,7 @@ func (w Wrapper) ConsumeBackupCollections(
mergeBase = bases.MergeBases()
}
assistBase = bases.UniqueAssistBases()
assistBase = bases.SnapshotAssistBases()
}
dirTree, err := inflateDirTree(

View File

@ -1002,7 +1002,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
{
name: "Merge Only",
baseBackups: func(base ManifestEntry) BackupBases {
return NewMockBackupBases().WithMergeBases(base).ClearMockAssistBases()
return NewMockBackupBases().WithMergeBases(base).MockDisableAssistBases()
},
// Pass in empty collections to force a backup. Otherwise we'll skip
// actually trying to do anything because we'll see there's nothing that

View File

@ -249,6 +249,19 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
}
}
makeBackup := func(snapID string, cats ...path.CategoryType) kopia.BackupEntry {
return kopia.BackupEntry{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: model.StableID(snapID + "bup"),
},
SnapshotID: snapID,
StreamStoreID: snapID + "store",
},
Reasons: buildReasons(ro, path.ExchangeService, cats...),
}
}
table := []struct {
name string
bf *mockBackupFinder
@ -276,8 +289,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
name: "don't get metadata",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
WithBackups(makeBackup("id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{},
@ -288,8 +302,10 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
assertErr: assert.NoError,
assertB: assert.False,
expectDCS: nil,
expectMans: kopia.NewMockBackupBases().WithAssistBases(
makeMan("id1", "", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
WithBackups(makeBackup("id1", path.EmailCategory)).
MockDisableMergeBases(),
},
{
name: "don't get metadata, incomplete manifest",
@ -359,9 +375,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
name: "one valid man, extra incomplete man",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)).WithAssistBases(
makeMan("id2", "checkpoint", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
WithAssistBases(makeMan("id2", "checkpoint", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -377,17 +393,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)).WithAssistBases(
makeMan("id2", "checkpoint", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
WithAssistBases(makeMan("id2", "checkpoint", path.EmailCategory)),
},
{
name: "one valid man, extra incomplete man, no assist bases",
name: "one valid man, extra incomplete man, drop assist bases",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)).WithAssistBases(
makeMan("id2", "checkpoint", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
WithAssistBases(makeMan("id2", "checkpoint", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -404,9 +420,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)).
ClearMockAssistBases(),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
MockDisableAssistBases(),
},
{
name: "multiple valid mans",
@ -438,8 +454,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
name: "error collecting metadata",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan("id1", "", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan("id1", "", path.EmailCategory)),
},
},
rp: mockRestoreProducer{err: assert.AnError},
@ -575,9 +591,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
name: "don't get metadata, only fallbacks",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{},
@ -586,8 +602,10 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.False,
expectDCS: nil,
expectMans: kopia.NewMockBackupBases().WithAssistBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)).
MockDisableMergeBases(),
},
{
name: "only fallbacks",
@ -608,17 +626,17 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
{
name: "only fallbacks, no assist",
name: "only fallbacks, drop assist",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -632,20 +650,20 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)).
ClearMockAssistBases(),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)).
MockDisableAssistBases(),
},
{
name: "complete mans and fallbacks",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(ro, "id1", "", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -692,13 +710,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
name: "complete and incomplete mans and fallbacks",
bf: &mockBackupFinder{
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory)).WithAssistBases(
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)).WithAssistBases(
makeMan(fbro, "fb_id2", "checkpoint", path.EmailCategory)),
ro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(ro, "id1", "", path.EmailCategory)).
WithAssistBases(makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)).
WithAssistBases(makeMan(fbro, "fb_id2", "checkpoint", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -715,9 +733,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory)).WithAssistBases(
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(ro, "id1", "", path.EmailCategory)).
WithAssistBases(makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
},
{
name: "incomplete mans and complete fallbacks",
@ -725,9 +743,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithAssistBases(
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -742,10 +760,10 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)).WithAssistBases(
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)).
WithAssistBases(makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
},
{
name: "incomplete mans and complete fallbacks, no assist bases",
@ -753,9 +771,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithAssistBases(
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
},
},
rp: mockRestoreProducer{
@ -771,10 +789,10 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory)).
ClearMockAssistBases(),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)).
MockDisableAssistBases(),
},
{
name: "complete mans and incomplete fallbacks",
@ -807,9 +825,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory, path.ContactsCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory, path.ContactsCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory, path.ContactsCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory, path.ContactsCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory, path.ContactsCategory)),
},
},
rp: mockRestoreProducer{
@ -839,9 +857,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.ContactsCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.ContactsCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(makeMan(fbro, "fb_id1", "", path.ContactsCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.ContactsCategory)),
},
},
rp: mockRestoreProducer{
@ -858,10 +876,11 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "id1"}, {id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory),
makeMan(fbro, "fb_id1", "", path.ContactsCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.ContactsCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory),
makeMan(fbro, "fb_id1", "", path.ContactsCategory)).
WithBackups(makeBackup(fbro, "fb_id1", path.ContactsCategory)),
},
{
name: "complete mans and complete fallbacks, fallback has superset of reasons",
@ -869,9 +888,11 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
data: map[string]kopia.BackupBases{
ro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory)),
fbro: kopia.NewMockBackupBases().WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory, path.ContactsCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory, path.ContactsCategory)),
fbro: kopia.NewMockBackupBases().
WithMergeBases(
makeMan(fbro, "fb_id1", "", path.EmailCategory, path.ContactsCategory)).
WithBackups(
makeBackup(fbro, "fb_id1", path.EmailCategory, path.ContactsCategory)),
},
},
rp: mockRestoreProducer{
@ -892,10 +913,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
assertErr: assert.NoError,
assertB: assert.True,
expectDCS: []mockColl{{id: "id1"}, {id: "fb_id1"}},
expectMans: kopia.NewMockBackupBases().WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory),
makeMan(fbro, "fb_id1", "", path.ContactsCategory)).WithBackups(
makeBackup(fbro, "fb_id1", path.ContactsCategory)),
expectMans: kopia.NewMockBackupBases().
WithMergeBases(
makeMan(ro, "id1", "", path.EmailCategory),
makeMan(fbro, "fb_id1", "", path.ContactsCategory)).
WithBackups(
makeBackup(fbro, "fb_id1", path.ContactsCategory)),
},
}