From 7582e175e6a70f0103ca8dd3e89ec9005b940441 Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Thu, 10 Aug 2023 00:16:08 +0530 Subject: [PATCH] Return assist backup models during base selection process (#3907) This PR adds support for selecting assist backup models. The intent here is to use the assist backups during merge details process. Changelist: 1. Assist bases now have a requirement to have a) an associated assist backup model b) details ssid. Please see [this code comment](https://github.com/alcionai/corso/pull/3907/files#diff-f9b6b7ab52c8e1148147909fd071d9b80b816d072203d8e9d2fa34ab93185db1R286) for more. 2. Any incomplete assist bases or assist bases not matching 1) are now discarded, i.e. they are not fed to kopia for kopia assisted incrementals. The impact here is that files cached by kopia might be now redownloaded if the backup failed without qualifying for an assist backup. Unit tests added. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/backup_bases.go | 107 +++--- src/internal/kopia/backup_bases_test.go | 492 ++++++++++++++++-------- src/internal/kopia/base_finder.go | 232 +++++++---- src/internal/kopia/base_finder_test.go | 436 +++++++++++++++++++-- src/internal/operations/manifests.go | 2 + 5 files changed, 943 insertions(+), 326 deletions(-) diff --git a/src/internal/kopia/backup_bases.go b/src/internal/kopia/backup_bases.go index dcfa1fc39..1631b9fa8 100644 --- a/src/internal/kopia/backup_bases.go +++ b/src/internal/kopia/backup_bases.go @@ -34,8 +34,8 @@ type backupBases struct { // data. backups []BackupEntry mergeBases []ManifestEntry - assistBases []ManifestEntry assistBackups []BackupEntry + assistBases []ManifestEntry } func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) { @@ -122,12 +122,12 @@ func (bb *backupBases) ClearAssistBases() { // // Selection priority, for each reason key generated by reasonsToKey, follows // these rules: -// 1. If the called BackupBases has an entry for a given resaon, ignore the +// 1. If the called BackupBases has an entry for a given reason, ignore the // other BackupBases matching that reason. -// 2. If the the receiver BackupBases has only AssistBases, look for a matching -// MergeBase manifest in the passed in BackupBases. -// 3. If the called BackupBases has no entry for a reason, look for both -// AssistBases and MergeBases in the passed in BackupBases. +// 2. If the called BackupBases has only AssistBases, look for a matching +// MergeBase manifest in the other BackupBases. +// 3. If the called BackupBases has no entry for a reason, look for a matching +// MergeBase in the other BackupBases. func (bb *backupBases) MergeBackupBases( ctx context.Context, other BackupBases, @@ -189,6 +189,10 @@ func (bb *backupBases) MergeBackupBases( backups: bb.Backups(), mergeBases: bb.MergeBases(), assistBases: bb.AssistBases(), + // Note that assistBackups are a new feature and don't exist + // in prior versions where we were using UPN based reasons i.e. + // other won't have any assistBackups. + assistBackups: bb.AssistBackups(), } // Add new mergeBases and backups. @@ -211,39 +215,11 @@ func (bb *backupBases) MergeBackupBases( res.backups = append(res.backups, bup) res.mergeBases = append(res.mergeBases, man) + // TODO(pandeyabs): Remove this once we remove overlap between + // between merge and assist bases as part of #3943. res.assistBases = append(res.assistBases, man) } - // Add assistBases from other to this one as needed. - for _, m := range other.AssistBases() { - useReasons := []Reasoner{} - - // Assume that all complete manifests in assist overlap with MergeBases. - if len(m.IncompleteReason) == 0 { - continue - } - - for _, r := range m.Reasons { - k := reasonToKey(r) - if _, ok := assist[k]; ok { - // This reason is already covered by either: - // * complete manifest in bb - // * incomplete manifest in bb - // - // If it was already in the assist set then it must be the case that - // it's newer than any complete manifests in other for the same reason. - continue - } - - useReasons = append(useReasons, r) - } - - if len(useReasons) > 0 { - m.Reasons = useReasons - res.assistBases = append(res.assistBases, m) - } - } - return res } @@ -332,12 +308,16 @@ func getBackupByID(backups []BackupEntry, bID string) (BackupEntry, bool) { // pull. On the other hand, *not* dropping them is unsafe as it will muck up // merging when we add stuff to kopia (possibly multiple entries for the same // item etc). +// +// TODO(pandeyabs): Refactor common code into a helper as part of #3943. func (bb *backupBases) fixupAndVerify(ctx context.Context) { toDrop := findNonUniqueManifests(ctx, bb.mergeBases) var ( - backupsToKeep []BackupEntry - mergeToKeep []ManifestEntry + backupsToKeep []BackupEntry + assistBackupsToKeep []BackupEntry + mergeToKeep []ManifestEntry + assistToKeep []ManifestEntry ) for _, man := range bb.mergeBases { @@ -352,7 +332,7 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) { toDrop[man.ID] = struct{}{} logger.Ctx(ctx).Info( - "dropping manifest due to missing backup", + "dropping merge base due to missing backup", "manifest_id", man.ID) continue @@ -367,7 +347,7 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) { toDrop[man.ID] = struct{}{} logger.Ctx(ctx).Info( - "dropping manifest due to invalid backup", + "dropping merge base due to invalid backup", "manifest_id", man.ID) continue @@ -377,9 +357,9 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) { mergeToKeep = append(mergeToKeep, man) } - var assistToKeep []ManifestEntry - - for _, man := range bb.assistBases { + // Every merge base is also a kopia assist base. + // TODO(pandeyabs): This should be removed as part of #3943. + for _, man := range bb.mergeBases { if _, ok := toDrop[man.ID]; ok { continue } @@ -387,7 +367,48 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) { assistToKeep = append(assistToKeep, man) } + // Drop assist snapshots with overlapping reasons. + toDropAssists := findNonUniqueManifests(ctx, bb.assistBases) + + for _, man := range bb.assistBases { + if _, ok := toDropAssists[man.ID]; ok { + continue + } + + bID, _ := man.GetTag(TagBackupID) + + bup, ok := getBackupByID(bb.assistBackups, bID) + if !ok { + toDrop[man.ID] = struct{}{} + + logger.Ctx(ctx).Info( + "dropping assist base due to missing backup", + "manifest_id", man.ID) + + continue + } + + deetsID := bup.StreamStoreID + if len(deetsID) == 0 { + deetsID = bup.DetailsID + } + + if len(bup.SnapshotID) == 0 || len(deetsID) == 0 { + toDrop[man.ID] = struct{}{} + + logger.Ctx(ctx).Info( + "dropping assist base due to invalid backup", + "manifest_id", man.ID) + + continue + } + + assistBackupsToKeep = append(assistBackupsToKeep, bup) + assistToKeep = append(assistToKeep, man) + } + bb.backups = backupsToKeep bb.mergeBases = mergeToKeep bb.assistBases = assistToKeep + bb.assistBackups = assistBackupsToKeep } diff --git a/src/internal/kopia/backup_bases_test.go b/src/internal/kopia/backup_bases_test.go index 04afb5408..4a222e912 100644 --- a/src/internal/kopia/backup_bases_test.go +++ b/src/internal/kopia/backup_bases_test.go @@ -206,36 +206,25 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { ro := "resource_owner" type testInput struct { - id int - incomplete bool - cat []path.CategoryType + id int + cat []path.CategoryType } // Make a function so tests can modify things without messing with each other. - makeBackupBases := func(ti []testInput) *backupBases { + makeBackupBases := func(mergeInputs []testInput, assistInputs []testInput) *backupBases { res := &backupBases{} - for _, i := range ti { + for _, i := range mergeInputs { baseID := fmt.Sprintf("id%d", i.id) - ir := "" - - if i.incomplete { - ir = "checkpoint" - } - reasons := make([]Reasoner, 0, len(i.cat)) for _, c := range i.cat { reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) } - m := makeManifest(baseID, ir, "b"+baseID, reasons...) + m := makeManifest(baseID, "", "b"+baseID, reasons...) res.assistBases = append(res.assistBases, m) - if i.incomplete { - continue - } - b := BackupEntry{ Backup: &backup.Backup{ BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)}, @@ -249,192 +238,217 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { res.mergeBases = append(res.mergeBases, m) } + for _, i := range assistInputs { + baseID := fmt.Sprintf("id%d", i.id) + + reasons := make([]Reasoner, 0, len(i.cat)) + + for _, c := range i.cat { + reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) + } + + m := makeManifest(baseID, "", "a"+baseID, reasons...) + + b := BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID("a" + baseID), + Tags: map[string]string{model.BackupTypeTag: model.AssistBackup}, + }, + SnapshotID: baseID, + StreamStoreID: "ss" + baseID, + }, + Reasons: reasons, + } + + res.assistBackups = append(res.assistBackups, b) + res.assistBases = append(res.assistBases, m) + } + return res } table := []struct { - name string - bb []testInput - other []testInput - expect []testInput + name string + merge []testInput + assist []testInput + otherMerge []testInput + otherAssist []testInput + expect func() *backupBases }{ { name: "Other Empty", - bb: []testInput{ + merge: []testInput{ {cat: []path.CategoryType{path.EmailCategory}}, }, - expect: []testInput{ + assist: []testInput{ {cat: []path.CategoryType{path.EmailCategory}}, }, + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }) + + return bs + }, }, { - name: "BB Empty", - other: []testInput{ + name: "current Empty", + otherMerge: []testInput{ {cat: []path.CategoryType{path.EmailCategory}}, }, - expect: []testInput{ + otherAssist: []testInput{ {cat: []path.CategoryType{path.EmailCategory}}, }, + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }) + + return bs + }, }, { - name: "Other overlaps Complete And Incomplete", - bb: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, + name: "Other overlaps merge and assist", + merge: []testInput{ { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, + id: 1, + cat: []path.CategoryType{path.EmailCategory}, }, }, - other: []testInput{ + assist: []testInput{ + { + id: 4, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, + otherMerge: []testInput{ { id: 2, cat: []path.CategoryType{path.EmailCategory}, }, { - id: 3, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, + id: 3, + cat: []path.CategoryType{path.EmailCategory}, }, }, - expect: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, + otherAssist: []testInput{ { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, + id: 5, + cat: []path.CategoryType{path.EmailCategory}, }, }, + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, []testInput{ + { + id: 4, + cat: []path.CategoryType{path.EmailCategory}, + }, + }) + + return bs + }, }, { - name: "Other Overlaps Complete", - bb: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, + name: "Other overlaps merge", + merge: []testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + }, }, - other: []testInput{ + otherMerge: []testInput{ { id: 2, cat: []path.CategoryType{path.EmailCategory}, }, }, - expect: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, nil) + + return bs }, }, { - name: "Other Overlaps Incomplete", - bb: []testInput{ + name: "Current assist overlaps with Other merge", + assist: []testInput{ { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, + id: 3, + cat: []path.CategoryType{path.EmailCategory}, }, }, - other: []testInput{ + otherMerge: []testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, + otherAssist: []testInput{ { id: 2, cat: []path.CategoryType{path.EmailCategory}, }, - { - id: 3, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, - }, }, - expect: []testInput{ - { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, - }, - { - id: 2, - cat: []path.CategoryType{path.EmailCategory}, - }, + + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, []testInput{ + { + id: 3, + cat: []path.CategoryType{path.EmailCategory}, + }, + }) + + return bs }, }, { name: "Other Disjoint", - bb: []testInput{ + merge: []testInput{ {cat: []path.CategoryType{path.EmailCategory}}, { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, + id: 1, + cat: []path.CategoryType{path.EmailCategory}, }, }, - other: []testInput{ + otherMerge: []testInput{ { id: 2, cat: []path.CategoryType{path.ContactsCategory}, }, - { - id: 3, - cat: []path.CategoryType{path.ContactsCategory}, - incomplete: true, - }, }, - expect: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, - { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, - }, - { - id: 2, - cat: []path.CategoryType{path.ContactsCategory}, - }, - { - id: 3, - cat: []path.CategoryType{path.ContactsCategory}, - incomplete: true, - }, - }, - }, - { - name: "Other Reduced Reasons", - bb: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, - { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, - }, - }, - other: []testInput{ - { - id: 2, - cat: []path.CategoryType{ - path.EmailCategory, - path.ContactsCategory, + expect: func() *backupBases { + bs := makeBackupBases([]testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, }, - }, - { - id: 3, - cat: []path.CategoryType{ - path.EmailCategory, - path.ContactsCategory, + { + id: 2, + cat: []path.CategoryType{path.ContactsCategory}, }, - incomplete: true, - }, - }, - expect: []testInput{ - {cat: []path.CategoryType{path.EmailCategory}}, - { - id: 1, - cat: []path.CategoryType{path.EmailCategory}, - incomplete: true, - }, - { - id: 2, - cat: []path.CategoryType{path.ContactsCategory}, - }, - { - id: 3, - cat: []path.CategoryType{path.ContactsCategory}, - incomplete: true, - }, + }, nil) + + return bs }, }, } @@ -443,9 +457,9 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { suite.Run(test.name, func() { t := suite.T() - bb := makeBackupBases(test.bb) - other := makeBackupBases(test.other) - expect := makeBackupBases(test.expect) + bb := makeBackupBases(test.merge, test.assist) + other := makeBackupBases(test.otherMerge, test.otherAssist) + expected := test.expect() ctx, flush := tester.NewContext(t) defer flush() @@ -456,7 +470,7 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { func(r Reasoner) string { return r.Service().String() + r.Category().String() }) - AssertBackupBasesEqual(t, expect, got) + AssertBackupBasesEqual(t, expected, got) }) } } @@ -486,8 +500,20 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { mergeBases: []ManifestEntry{ makeMan(path.EmailCategory, "id1", "", "bid1"), }, + assistBackups: []BackupEntry{ + { + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid2", + Tags: map[string]string{model.BackupTypeTag: model.AssistBackup}, + }, + SnapshotID: "id2", + StreamStoreID: "ssid2", + }, + }, + }, assistBases: []ManifestEntry{ - makeMan(path.EmailCategory, "id1", "", "bid1"), + makeMan(path.EmailCategory, "id2", "", "bid2"), }, } } @@ -507,24 +533,77 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.backups = nil + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.mergeBases = nil + res.backups = nil + return res }(), }, { - name: "Backup Missing Snapshot ID", + name: "Merge Backup Missing Snapshot ID", bb: func() *backupBases { res := validMail1() res.backups[0].SnapshotID = "" + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.mergeBases = nil + res.backups = nil + return res }(), }, { - name: "Backup Missing Deets ID", + name: "Assist backup missing snapshot ID", + bb: func() *backupBases { + res := validMail1() + res.assistBackups[0].SnapshotID = "" + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.assistBases = res.mergeBases + res.assistBackups = nil + + return res + }(), + }, + { + name: "Merge backup missing deets ID", bb: func() *backupBases { res := validMail1() res.backups[0].StreamStoreID = "" + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.mergeBases = nil + res.backups = nil + + return res + }(), + }, + { + name: "Assist backup missing deets ID", + bb: func() *backupBases { + res := validMail1() + res.assistBackups[0].StreamStoreID = "" + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.assistBases = res.mergeBases + res.assistBackups = nil + return res }(), }, @@ -545,15 +624,22 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, res.mergeBases[0].Reasons[0]) - res.assistBases = res.mergeBases + res.assistBases[0].Reasons = append( + res.assistBases[0].Reasons, + res.assistBases[0].Reasons[0]) return res }(), }, { - name: "Single Valid Entry", - bb: validMail1(), - expect: validMail1(), + name: "Single Valid Entry", + bb: validMail1(), + expect: func() *backupBases { + res := validMail1() + res.assistBases = append(res.mergeBases, res.assistBases...) + + return res + }(), }, { name: "Single Valid Entry With Incomplete Assist With Same Reason", @@ -561,16 +647,14 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.assistBases = append( res.assistBases, - makeMan(path.EmailCategory, "id2", "checkpoint", "bid2")) + makeMan(path.EmailCategory, "id3", "checkpoint", "bid3")) return res }(), expect: func() *backupBases { res := validMail1() - res.assistBases = append( - res.assistBases, - makeMan(path.EmailCategory, "id2", "checkpoint", "bid2")) + res.assistBases = append(res.mergeBases, res.assistBases...) return res }(), }, @@ -581,6 +665,9 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res.backups[0].DetailsID = res.backups[0].StreamStoreID res.backups[0].StreamStoreID = "" + res.assistBackups[0].DetailsID = res.assistBackups[0].StreamStoreID + res.assistBackups[0].StreamStoreID = "" + return res }(), expect: func() *backupBases { @@ -588,6 +675,11 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res.backups[0].DetailsID = res.backups[0].StreamStoreID res.backups[0].StreamStoreID = "" + res.assistBackups[0].DetailsID = res.assistBackups[0].StreamStoreID + res.assistBackups[0].StreamStoreID = "" + + res.assistBases = append(res.mergeBases, res.assistBases...) + return res }(), }, @@ -598,7 +690,10 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, NewReason("", ro, path.ExchangeService, path.ContactsCategory)) - res.assistBases = res.mergeBases + + res.assistBases[0].Reasons = append( + res.assistBases[0].Reasons, + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) return res }(), @@ -607,7 +702,12 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, NewReason("", ro, path.ExchangeService, path.ContactsCategory)) - res.assistBases = res.mergeBases + + res.assistBases[0].Reasons = append( + res.assistBases[0].Reasons, + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) + + res.assistBases = append(res.mergeBases, res.assistBases...) return res }(), @@ -618,14 +718,17 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases = append( res.mergeBases, - makeMan(path.EmailCategory, "id2", "", "bid2")) - res.assistBases = res.mergeBases + makeMan(path.EmailCategory, "id3", "", "bid3")) + + res.assistBases = append( + res.assistBases, + makeMan(path.EmailCategory, "id4", "", "bid4")) return res }(), }, { - name: "Three Entries One Invalid", + name: "Merge Backup, Three Entries One Invalid", bb: func() *backupBases { res := validMail1() res.backups = append( @@ -633,24 +736,23 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { BackupEntry{ Backup: &backup.Backup{ BaseModel: model.BaseModel{ - ID: "bid2", + ID: "bid3", }, }, }, BackupEntry{ Backup: &backup.Backup{ BaseModel: model.BaseModel{ - ID: "bid3", + ID: "bid4", }, - SnapshotID: "id3", - StreamStoreID: "ssid3", + SnapshotID: "id4", + StreamStoreID: "ssid4", }, }) res.mergeBases = append( res.mergeBases, - makeMan(path.ContactsCategory, "id2", "checkpoint", "bid2"), - makeMan(path.EventsCategory, "id3", "", "bid3")) - res.assistBases = res.mergeBases + makeMan(path.ContactsCategory, "id3", "checkpoint", "bid3"), + makeMan(path.EventsCategory, "id4", "", "bid4")) return res }(), @@ -661,16 +763,70 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { BackupEntry{ Backup: &backup.Backup{ BaseModel: model.BaseModel{ - ID: "bid3", + ID: "bid4", }, - SnapshotID: "id3", - StreamStoreID: "ssid3", + SnapshotID: "id4", + StreamStoreID: "ssid4", }, }) res.mergeBases = append( res.mergeBases, - makeMan(path.EventsCategory, "id3", "", "bid3")) - res.assistBases = res.mergeBases + makeMan(path.EventsCategory, "id4", "", "bid4")) + res.assistBases = append(res.mergeBases, res.assistBases...) + + return res + }(), + }, + { + name: "Assist Backup, Three Entries One Invalid", + bb: func() *backupBases { + res := validMail1() + res.assistBackups = append( + res.assistBackups, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid3", + Tags: map[string]string{model.BackupTypeTag: model.AssistBackup}, + }, + }, + }, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid4", + Tags: map[string]string{model.BackupTypeTag: model.AssistBackup}, + }, + SnapshotID: "id4", + StreamStoreID: "ssid4", + }, + }) + res.assistBases = append( + res.assistBases, + makeMan(path.ContactsCategory, "id3", "checkpoint", "bid3"), + makeMan(path.EventsCategory, "id4", "", "bid4")) + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.assistBackups = append( + res.assistBackups, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid4", + Tags: map[string]string{model.BackupTypeTag: model.AssistBackup}, + }, + SnapshotID: "id4", + StreamStoreID: "ssid4", + }, + }) + res.assistBases = append( + res.assistBases, + makeMan(path.EventsCategory, "id4", "", "bid4")) + + res.assistBases = append(res.mergeBases, res.assistBases...) return res }(), diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index 00561c833..b3ebf0d7f 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -204,17 +204,20 @@ func (b *baseFinder) getBackupModel( return bup, nil } +type backupBase struct { + backup BackupEntry + manifest ManifestEntry +} + // findBasesInSet goes through manifest metadata entries and sees if they're -// incomplete or not. If an entry is incomplete and we don't already have a -// complete or incomplete manifest add it to the set for kopia assisted -// incrementals. If it's complete, fetch the backup model and see if it -// corresponds to a successful backup. If it does, return it as we only need the -// most recent complete backup as the base. +// incomplete or not. Manifests which don't have an associated backup +// are discarded as incomplete. Manifests are then checked to see if they +// are associated with an assist backup or merge backup. func (b *baseFinder) findBasesInSet( ctx context.Context, reason Reasoner, metas []*manifest.EntryMetadata, -) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { +) (*backupBase, *backupBase, error) { // Sort manifests by time so we can go through them sequentially. The code in // kopia appears to sort them already, but add sorting here just so we're not // reliant on undocumented behavior. @@ -223,8 +226,8 @@ func (b *baseFinder) findBasesInSet( }) var ( - kopiaAssistSnaps []ManifestEntry - foundIncomplete bool + mergeBase *backupBase + assistBase *backupBase ) for i := len(metas) - 1; i >= 0; i-- { @@ -240,16 +243,10 @@ func (b *baseFinder) findBasesInSet( } if len(man.IncompleteReason) > 0 { - if !foundIncomplete { - foundIncomplete = true - - kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ - Manifest: man, - Reasons: []Reasoner{reason}, - }) - - logger.Ctx(ictx).Info("found incomplete backup") - } + // Skip here since this snapshot cannot be considered an assist base. + logger.Ctx(ictx).Debugw( + "Incomplete snapshot", + "incomplete_reason", man.IncompleteReason) continue } @@ -259,19 +256,7 @@ func (b *baseFinder) findBasesInSet( if err != nil { // Safe to continue here as we'll just end up attempting to use an older // backup as the base. - logger.CtxErr(ictx, err).Debug("searching for base backup") - - if !foundIncomplete { - foundIncomplete = true - - kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ - Manifest: man, - Reasons: []Reasoner{reason}, - }) - - logger.Ctx(ictx).Info("found incomplete backup") - } - + logger.CtxErr(ictx, err).Debug("searching for backup model") continue } @@ -285,49 +270,117 @@ func (b *baseFinder) findBasesInSet( "empty backup stream store ID", "search_backup_id", bup.ID) - if !foundIncomplete { - foundIncomplete = true - - kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ - Manifest: man, - Reasons: []Reasoner{reason}, - }) - - logger.Ctx(ictx).Infow( - "found incomplete backup", - "search_backup_id", bup.ID) - } - continue } // If we've made it to this point then we're considering the backup // complete as it has both an item data snapshot and a backup details // snapshot. - logger.Ctx(ictx).Infow("found complete backup", "base_backup_id", bup.ID) + // + // Check first if this is an assist base. Criteria for selecting an + // assist base are: + // 1. most recent assist base for the reason. + // 2. at most one assist base per reason. + // 3. it must be more recent than the merge backup for the reason, if + // a merge backup exists. - me := ManifestEntry{ + if b.isAssistBackupModel(ictx, bup) { + if assistBase == nil { + assistModel := BackupEntry{ + Backup: bup, + Reasons: []Reasoner{reason}, + } + assistSnap := ManifestEntry{ + Manifest: man, + Reasons: []Reasoner{reason}, + } + + assistBase = &backupBase{ + backup: assistModel, + manifest: assistSnap, + } + + logger.Ctx(ictx).Infow( + "found assist base", + "search_backup_id", bup.ID, + "search_snapshot_id", meta.ID, + "ssid", ssid) + } + + // Skip if an assist base has already been selected. + continue + } + + logger.Ctx(ictx).Infow("found merge base", + "search_backup_id", bup.ID, + "search_snapshot_id", meta.ID, + "ssid", ssid) + + mergeSnap := ManifestEntry{ Manifest: man, Reasons: []Reasoner{reason}, } - kopiaAssistSnaps = append(kopiaAssistSnaps, me) - - return &BackupEntry{ + mergeModel := BackupEntry{ Backup: bup, Reasons: []Reasoner{reason}, - }, &me, kopiaAssistSnaps, nil + } + + mergeBase = &backupBase{ + backup: mergeModel, + manifest: mergeSnap, + } + + break } - logger.Ctx(ctx).Info("no base backups for reason") + if mergeBase == nil && assistBase == nil { + logger.Ctx(ctx).Info("no merge or assist base found for reason") + } - return nil, nil, kopiaAssistSnaps, nil + return mergeBase, assistBase, nil +} + +// isAssistBackupModel checks if the provided backup is an assist backup. +func (b *baseFinder) isAssistBackupModel( + ctx context.Context, + bup *backup.Backup, +) bool { + allTags := map[string]string{ + model.BackupTypeTag: model.AssistBackup, + } + + for k, v := range allTags { + if bup.Tags[k] != v { + // This is not an assist backup so we can just exit here. + logger.Ctx(ctx).Debugw( + "assist backup model missing tags", + "backup_id", bup.ID, + "tag", k, + "expected_value", v, + "actual_value", bup.Tags[k]) + + return false + } + } + + // Check if it has a valid streamstore id and snapshot id. + if len(bup.StreamStoreID) == 0 || len(bup.SnapshotID) == 0 { + logger.Ctx(ctx).Infow( + "nil ssid or snapshot id in assist base", + "ssid", bup.StreamStoreID, + "snapshot_id", bup.SnapshotID) + + return false + } + + return true } func (b *baseFinder) getBase( ctx context.Context, r Reasoner, tags map[string]string, -) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { +) (*backupBase, *backupBase, error) { allTags := map[string]string{} for _, k := range tagKeys(r) { @@ -339,12 +392,12 @@ func (b *baseFinder) getBase( metas, err := b.sm.FindManifests(ctx, allTags) if err != nil { - return nil, nil, nil, clues.Wrap(err, "getting snapshots") + return nil, nil, clues.Wrap(err, "getting snapshots") } // No snapshots means no backups so we can just exit here. if len(metas) == 0 { - return nil, nil, nil, nil + return nil, nil, nil } return b.findBasesInSet(ctx, r, metas) @@ -360,9 +413,10 @@ func (b *baseFinder) FindBases( // the reason for selecting something. Kopia assisted snapshots also use // ManifestEntry so we have the reasons for selecting them to aid in // debugging. - baseBups = map[model.StableID]BackupEntry{} - baseSnaps = map[manifest.ID]ManifestEntry{} - kopiaAssistSnaps = map[manifest.ID]ManifestEntry{} + mergeBups = map[model.StableID]BackupEntry{} + assistBups = map[model.StableID]BackupEntry{} + mergeSnaps = map[manifest.ID]ManifestEntry{} + assistSnaps = map[manifest.ID]ManifestEntry{} ) for _, searchReason := range reasons { @@ -372,7 +426,10 @@ func (b *baseFinder) FindBases( "search_category", searchReason.Category().String()) logger.Ctx(ictx).Info("searching for previous manifests") - baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, searchReason, tags) + mergeBase, assistBase, err := b.getBase( + ictx, + searchReason, + tags) if err != nil { logger.Ctx(ctx).Info( "getting base, falling back to full backup for reason", @@ -381,47 +438,60 @@ func (b *baseFinder) FindBases( continue } - if baseBackup != nil { - bs, ok := baseBups[baseBackup.ID] + if mergeBase != nil { + mergeSnap := mergeBase.manifest + mergeBackup := mergeBase.backup + + ms, ok := mergeSnaps[mergeSnap.ID] if ok { - bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) + ms.Reasons = append(ms.Reasons, mergeSnap.Reasons...) } else { - bs = *baseBackup + ms = mergeSnap } - // Reassign since it's structs not pointers to structs. - baseBups[baseBackup.ID] = bs + mergeSnaps[mergeSnap.ID] = ms + + mb, ok := mergeBups[mergeBackup.ID] + if ok { + mb.Reasons = append(mb.Reasons, mergeSnap.Reasons...) + } else { + mb = mergeBackup + } + + mergeBups[mergeBackup.ID] = mb } - if baseSnap != nil { - bs, ok := baseSnaps[baseSnap.ID] + if assistBase != nil { + assistSnap := assistBase.manifest + assistBackup := assistBase.backup + + as, ok := assistSnaps[assistSnap.ID] if ok { - bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) + as.Reasons = append(as.Reasons, assistSnap.Reasons...) } else { - bs = *baseSnap + as = assistSnap } - // Reassign since it's structs not pointers to structs. - baseSnaps[baseSnap.ID] = bs - } + assistSnaps[assistSnap.ID] = as - for _, s := range assistSnaps { - bs, ok := kopiaAssistSnaps[s.ID] + ab, ok := assistBups[assistBackup.ID] if ok { - bs.Reasons = append(bs.Reasons, s.Reasons...) + ab.Reasons = append(ab.Reasons, assistBackup.Reasons...) } else { - bs = s + ab = assistBackup } - // Reassign since it's structs not pointers to structs. - kopiaAssistSnaps[s.ID] = bs + assistBups[assistBackup.ID] = ab } } + // TODO(pandeyabs): Fix the terminology used in backupBases to go with + // new definitions i.e. mergeSnaps instead of mergeBases, etc. res := &backupBases{ - backups: maps.Values(baseBups), - mergeBases: maps.Values(baseSnaps), - assistBases: maps.Values(kopiaAssistSnaps), + backups: maps.Values(mergeBups), + assistBackups: maps.Values(assistBups), + mergeBases: maps.Values(mergeSnaps), + assistBases: maps.Values(assistSnaps), } res.fixupAndVerify(ctx) diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index cb3239ca1..83c0876c9 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -23,14 +23,19 @@ const ( ) var ( - testT1 = time.Now() - testT2 = testT1.Add(1 * time.Hour) - + testT1 = time.Now() + testT2 = testT1.Add(1 * time.Hour) + testT3 = testT2.Add(1 * time.Hour) + testT4 = testT3.Add(1 * time.Hour) testID1 = manifest.ID("snap1") testID2 = manifest.ID("snap2") + testID3 = manifest.ID("snap3") + testID4 = manifest.ID("snap4") testBackup1 = "backupID1" testBackup2 = "backupID2" + testBackup3 = "backupID3" + testBackup4 = "backupID4" testMail = path.ExchangeService.String() + path.EmailCategory.String() testEvents = path.ExchangeService.String() + path.EventsCategory.String() @@ -212,12 +217,14 @@ func newBackupModel( hasItemSnap bool, hasDetailsSnap bool, oldDetailsID bool, + tags map[string]string, err error, ) backupInfo { res := backupInfo{ b: backup.Backup{ BaseModel: model.BaseModel{ - ID: model.StableID(id), + ID: model.StableID(id), + Tags: tags, }, SnapshotID: "iid", }, @@ -323,11 +330,14 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { expectedBaseReasons map[int][]Reasoner // Use this to denote the Reasons a kopia assised incrementals manifest is // selected. The int maps to the index of the manifest in data. + // TODO(pandeyabs): Remove this once we have 1:1 mapping between snapshots + // and backup models. expectedAssistManifestReasons map[int][]Reasoner + expectedAssistReasons map[int][]Reasoner backupData []backupInfo }{ { - name: "Return Older Base If Fail To Get Manifest", + name: "Return Older Merge Base If Fail To Get Manifest", input: testUser1Mail, manifestData: []manifestInfo{ newManifestInfo( @@ -355,13 +365,55 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, + expectedAssistReasons: map[int][]Reasoner{}, backupData: []backupInfo{ - newBackupModel(testBackup2, true, true, false, nil), - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), }, }, { - name: "Return Older Base If Fail To Get Backup", + name: "Return Older Assist Base If Fail To Get Manifest", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + assert.AnError, + testMail, + testUser1, + ), + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{ + 1: testUser1Mail, + }, + expectedAssistReasons: map[int][]Reasoner{ + 1: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil, nil), + newBackupModel( + testBackup1, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + }, + }, + { + name: "Return Older Merge Base If Fail To Get Backup", input: testUser1Mail, manifestData: []manifestInfo{ newManifestInfo( @@ -387,12 +439,11 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { 1: testUser1Mail, }, expectedAssistManifestReasons: map[int][]Reasoner{ - 0: testUser1Mail, 1: testUser1Mail, }, backupData: []backupInfo{ - newBackupModel(testBackup2, false, false, false, assert.AnError), - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, false, false, false, nil, assert.AnError), + newBackupModel(testBackup1, true, true, false, nil, nil), }, }, { @@ -422,12 +473,12 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { 1: testUser1Mail, }, expectedAssistManifestReasons: map[int][]Reasoner{ - 0: testUser1Mail, 1: testUser1Mail, }, + expectedAssistReasons: map[int][]Reasoner{}, backupData: []backupInfo{ - newBackupModel(testBackup2, true, false, false, nil), - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, true, false, false, nil, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), }, }, { @@ -453,12 +504,13 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, + expectedAssistReasons: map[int][]Reasoner{}, backupData: []backupInfo{ - newBackupModel(testBackup1, true, true, true, nil), + newBackupModel(testBackup1, true, true, true, nil, nil), }, }, { - name: "All One Snapshot", + name: "All One Snapshot With Merge Base", input: testAllUsersAllCats, manifestData: []manifestInfo{ newManifestInfo( @@ -480,8 +532,43 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { expectedAssistManifestReasons: map[int][]Reasoner{ 0: testAllUsersAllCats, }, + expectedAssistReasons: map[int][]Reasoner{}, backupData: []backupInfo{ - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + }, + }, + { + name: "All One Snapshot with Assist Base", + input: testAllUsersAllCats, + manifestData: []manifestInfo{ + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testEvents, + testUser1, + testUser2, + testUser3, + ), + }, + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: testAllUsersAllCats, + }, + expectedAssistReasons: map[int][]Reasoner{ + 0: testAllUsersAllCats, + }, + backupData: []backupInfo{ + newBackupModel( + testBackup1, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), }, }, { @@ -537,8 +624,96 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { }, }, backupData: []backupInfo{ - newBackupModel(testBackup1, true, true, false, nil), - newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), + }, + }, + { + name: "Unique assist bases with common merge Base, overlapping reasons", + input: testAllUsersAllCats, + manifestData: []manifestInfo{ + newManifestInfo( + testID3, + testT3, + testCompleteMan, + testBackup3, + nil, + testEvents, + testUser1, + testUser2, + ), + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + testUser2, + ), + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testEvents, + testUser1, + testUser2, + ), + }, + expectedBaseReasons: map[int][]Reasoner{ + 2: { + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + }, + }, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: { + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + }, + 1: { + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + }, + 2: { + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + }, + }, + expectedAssistReasons: map[int][]Reasoner{ + 0: { + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + }, + 1: { + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + }, + }, + backupData: []backupInfo{ + newBackupModel( + testBackup3, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + newBackupModel( + testBackup2, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + newBackupModel(testBackup1, true, true, false, nil, nil), }, }, { @@ -569,12 +744,11 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { }, expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, - 1: testUser1Mail, }, backupData: []backupInfo{ - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), // Shouldn't be returned but have here just so we can see. - newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), }, }, { @@ -608,8 +782,8 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { }, backupData: []backupInfo{ // Shouldn't be returned but have here just so we can see. - newBackupModel(testBackup1, true, true, false, nil), - newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), }, }, { @@ -635,14 +809,12 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reasoner{}, - expectedAssistManifestReasons: map[int][]Reasoner{ - 1: testUser1Mail, - }, + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{}, backupData: []backupInfo{ // Shouldn't be returned but have here just so we can see. - newBackupModel(testBackup1, true, true, false, nil), - newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), }, }, { @@ -666,7 +838,7 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { 0: testUser1Mail, }, backupData: []backupInfo{ - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), }, }, { @@ -701,9 +873,199 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { 0: testUser1Mail, }, backupData: []backupInfo{ - newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil, nil), // Shouldn't be returned but here just so we can check. - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + }, + }, + { + name: "Return latest assist & merge base pair", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo( + testID4, + testT4, + testCompleteMan, + testBackup4, + nil, + testMail, + testUser1, + ), + newManifestInfo( + testID3, + testT3, + testCompleteMan, + testBackup3, + nil, + testMail, + testUser1, + ), + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reasoner{ + 2: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: testUser1Mail, + 2: testUser1Mail, + }, + expectedAssistReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel( + testBackup4, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + newBackupModel( + testBackup3, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + newBackupModel(testBackup2, true, true, false, nil, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), + }, + }, + { + name: "Newer merge base than assist base", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + expectedAssistReasons: map[int][]Reasoner{}, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil, nil), + newBackupModel( + testBackup1, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + }, + }, + { + name: "Only assist bases", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + expectedAssistReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel( + testBackup2, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + newBackupModel( + testBackup1, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.AssistBackup}, + nil), + }, + }, + { + name: "Merge base with tag", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reasoner{ + 0: testUser1Mail, + }, + expectedAssistReasons: map[int][]Reasoner{}, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil, nil), + newBackupModel( + testBackup1, + true, + true, + false, + map[string]string{model.BackupTypeTag: model.MergeBackup}, + nil), }, }, } @@ -730,6 +1092,12 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { bb.Backups(), test.backupData, test.expectedBaseReasons) + checkBackupEntriesMatch( + t, + bb.AssistBackups(), + test.backupData, + test.expectedAssistReasons) + checkManifestEntriesMatch( t, bb.MergeBases(), @@ -759,7 +1127,7 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { ), } backupData := []backupInfo{ - newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup1, true, true, false, nil, nil), } table := []struct { diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 8ca339d26..d5ae6cdae 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -77,6 +77,8 @@ func getManifestsAndMetadata( // 2. the current reasons only contain an incomplete manifest, and the fallback // can find a complete manifest. // 3. the current reasons contain all the necessary manifests. + // Note: This is not relevant for assist backups, since they are newly introduced + // and they don't exist with fallback reasons. bb = bb.MergeBackupBases( ctx, fbb,