From 7120164db6b15df9ea503aab36e35aaff6f90371 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:16:07 -0700 Subject: [PATCH] Add struct functions for backup bases (#3595) Move most of the stuff that was acting on backup bases to be functions that are defined for backup bases. Other code can be removed at a later point New functions aren't called yet in other code --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3525 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/backup_bases.go | 387 +++++++++++++ src/internal/kopia/backup_bases_test.go | 705 ++++++++++++++++++++++++ src/internal/kopia/base_finder.go | 6 - src/internal/kopia/base_finder_test.go | 18 +- src/internal/kopia/mock_backup_base.go | 63 +++ 5 files changed, 1164 insertions(+), 15 deletions(-) create mode 100644 src/internal/kopia/backup_bases.go create mode 100644 src/internal/kopia/backup_bases_test.go create mode 100644 src/internal/kopia/mock_backup_base.go diff --git a/src/internal/kopia/backup_bases.go b/src/internal/kopia/backup_bases.go new file mode 100644 index 000000000..0505fc829 --- /dev/null +++ b/src/internal/kopia/backup_bases.go @@ -0,0 +1,387 @@ +package kopia + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/logger" +) + +// TODO(ashmrtn): Move this into some inject package. Here to avoid import +// cycles. +type BackupBases interface { + RemoveMergeBaseByManifestID(manifestID manifest.ID) + Backups() []BackupEntry + MinBackupVersion() int + MergeBases() []ManifestEntry + ClearMergeBases() + AssistBases() []ManifestEntry + ClearAssistBases() + MergeBackupBases( + ctx context.Context, + other BackupBases, + reasonToKey func(Reason) string, + ) BackupBases +} + +type backupBases struct { + // backups and mergeBases should be modified together as they relate similar + // data. + backups []BackupEntry + mergeBases []ManifestEntry + assistBases []ManifestEntry +} + +func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) { + idx := slices.IndexFunc( + bb.mergeBases, + func(man ManifestEntry) bool { + return man.ID == manifestID + }) + if idx >= 0 { + bb.mergeBases = slices.Delete(bb.mergeBases, idx, idx+1) + } + + // TODO(ashmrtn): This may not be strictly necessary but is at least easier to + // reason about. + idx = slices.IndexFunc( + bb.assistBases, + func(man ManifestEntry) bool { + return man.ID == manifestID + }) + if idx >= 0 { + bb.assistBases = slices.Delete(bb.assistBases, idx, idx+1) + } + + idx = slices.IndexFunc( + bb.backups, + func(bup BackupEntry) bool { + return bup.SnapshotID == string(manifestID) + }) + if idx >= 0 { + bb.backups = slices.Delete(bb.backups, idx, idx+1) + } +} + +func (bb backupBases) Backups() []BackupEntry { + return slices.Clone(bb.backups) +} + +func (bb *backupBases) MinBackupVersion() int { + min := version.NoBackup + + if bb == nil { + return min + } + + for _, bup := range bb.backups { + if min == version.NoBackup || bup.Version < min { + min = bup.Version + } + } + + return min +} + +func (bb backupBases) MergeBases() []ManifestEntry { + return slices.Clone(bb.mergeBases) +} + +func (bb *backupBases) ClearMergeBases() { + bb.mergeBases = nil + bb.backups = nil +} + +func (bb backupBases) AssistBases() []ManifestEntry { + return slices.Clone(bb.assistBases) +} + +func (bb *backupBases) ClearAssistBases() { + bb.assistBases = nil +} + +// MergeBackupBases reduces the two BackupBases into a single BackupBase. +// Assumes the passed in BackupBases represents a prior backup version (across +// some migration that disrupts lookup), and that the BackupBases used to call +// this function contains the current version. +// +// reasonToKey should be a function that, given a Reason, will produce some +// string that represents Reason in the context of the merge operation. For +// example, to merge BackupBases across a ResourceOwner migration, the Reason's +// service and category can be used as the key. +// +// Selection priority, for each reason key generated by reasonsToKey, follows +// these rules: +// 1. If the called BackupBases has an entry for a given resaon, ignore the +// other BackupBases matching that reason. +// 2. If the the receiver BackupBases has only AssistBases, look for a matching +// MergeBase manifest in the passed in BackupBases. +// 3. If the called BackupBases has no entry for a reason, look for both +// AssistBases and MergeBases in the passed in BackupBases. +func (bb *backupBases) MergeBackupBases( + ctx context.Context, + other BackupBases, + reasonToKey func(reason Reason) string, +) BackupBases { + if other == nil || (len(other.MergeBases()) == 0 && len(other.AssistBases()) == 0) { + return bb + } + + if bb == nil || (len(bb.MergeBases()) == 0 && len(bb.AssistBases()) == 0) { + return other + } + + toMerge := map[string]struct{}{} + assist := map[string]struct{}{} + + // Track the bases in bb. + for _, m := range bb.mergeBases { + for _, r := range m.Reasons { + k := reasonToKey(r) + + toMerge[k] = struct{}{} + assist[k] = struct{}{} + } + } + + for _, m := range bb.assistBases { + for _, r := range m.Reasons { + k := reasonToKey(r) + assist[k] = struct{}{} + } + } + + var toAdd []ManifestEntry + + // Calculate the set of mergeBases to pull from other into this one. + for _, m := range other.MergeBases() { + useReasons := []Reason{} + + for _, r := range m.Reasons { + k := reasonToKey(r) + if _, ok := toMerge[k]; ok { + // Assume other contains prior manifest versions. + // We don't want to stack a prior version incomplete onto + // a current version's complete snapshot. + continue + } + + useReasons = append(useReasons, r) + } + + if len(useReasons) > 0 { + m.Reasons = useReasons + toAdd = append(toAdd, m) + } + } + + res := &backupBases{ + backups: bb.Backups(), + mergeBases: bb.MergeBases(), + assistBases: bb.AssistBases(), + } + + // Add new mergeBases and backups. + for _, man := range toAdd { + // Will get empty string if not found which is fine, it'll fail one of the + // other checks. + bID, _ := man.GetTag(TagBackupID) + + bup, ok := getBackupByID(other.Backups(), bID) + if !ok { + logger.Ctx(ctx).Infow( + "not unioning snapshot missing backup", + "other_manifest_id", man.ID, + "other_backup_id", bID) + + continue + } + + bup.Reasons = man.Reasons + + res.backups = append(res.backups, bup) + res.mergeBases = append(res.mergeBases, man) + res.assistBases = append(res.assistBases, man) + } + + // Add assistBases from other to this one as needed. + for _, m := range other.AssistBases() { + useReasons := []Reason{} + + // Assume that all complete manifests in assist overlap with MergeBases. + if len(m.IncompleteReason) == 0 { + continue + } + + for _, r := range m.Reasons { + k := reasonToKey(r) + if _, ok := assist[k]; ok { + // This reason is already covered by either: + // * complete manifest in bb + // * incomplete manifest in bb + // + // If it was already in the assist set then it must be the case that + // it's newer than any complete manifests in other for the same reason. + continue + } + + useReasons = append(useReasons, r) + } + + if len(useReasons) > 0 { + m.Reasons = useReasons + res.assistBases = append(res.assistBases, m) + } + } + + return res +} + +func findNonUniqueManifests( + ctx context.Context, + manifests []ManifestEntry, +) map[manifest.ID]struct{} { + // ReasonKey -> manifests with that reason. + reasons := map[string][]ManifestEntry{} + toDrop := map[manifest.ID]struct{}{} + + for _, man := range manifests { + // Incomplete snapshots are used only for kopia-assisted incrementals. The + // fact that we need this check here makes it seem like this should live in + // the kopia code. However, keeping it here allows for better debugging as + // the kopia code only has access to a path builder which means it cannot + // remove the resource owner from the error/log output. That is also below + // the point where we decide if we should do a full backup or an incremental. + if len(man.IncompleteReason) > 0 { + logger.Ctx(ctx).Infow( + "dropping incomplete manifest", + "manifest_id", man.ID) + + toDrop[man.ID] = struct{}{} + + continue + } + + for _, reason := range man.Reasons { + reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String() + reasons[reasonKey] = append(reasons[reasonKey], man) + } + } + + for reason, mans := range reasons { + ictx := clues.Add(ctx, "reason", reason) + + if len(mans) == 0 { + // Not sure how this would happen but just in case... + continue + } else if len(mans) > 1 { + mIDs := make([]manifest.ID, 0, len(mans)) + for _, m := range mans { + toDrop[m.ID] = struct{}{} + mIDs = append(mIDs, m.ID) + } + + // TODO(ashmrtn): We should actually just remove this reason from the + // manifests and then if they have no reasons remaining drop them from the + // set. + logger.Ctx(ictx).Infow( + "dropping manifests with duplicate reason", + "manifest_ids", mIDs) + + continue + } + } + + return toDrop +} + +func getBackupByID(backups []BackupEntry, bID string) (BackupEntry, bool) { + if len(bID) == 0 { + return BackupEntry{}, false + } + + idx := slices.IndexFunc(backups, func(b BackupEntry) bool { + return string(b.ID) == bID + }) + + if idx < 0 || idx >= len(backups) { + return BackupEntry{}, false + } + + return backups[idx], true +} + +// fixupAndVerify goes through the set of backups and snapshots used for merging +// and ensures: +// - the reasons for selecting merge snapshots are distinct +// - all bases used for merging have a backup model with item and details +// snapshot ID +// +// Backups that have overlapping reasons or that are not complete are removed +// from the set. Dropping these is safe because it only affects how much data we +// pull. On the other hand, *not* dropping them is unsafe as it will muck up +// merging when we add stuff to kopia (possibly multiple entries for the same +// item etc). +func (bb *backupBases) fixupAndVerify(ctx context.Context) { + toDrop := findNonUniqueManifests(ctx, bb.mergeBases) + + var ( + backupsToKeep []BackupEntry + mergeToKeep []ManifestEntry + ) + + for _, man := range bb.mergeBases { + if _, ok := toDrop[man.ID]; ok { + continue + } + + bID, _ := man.GetTag(TagBackupID) + + bup, ok := getBackupByID(bb.backups, bID) + if !ok { + toDrop[man.ID] = struct{}{} + + logger.Ctx(ctx).Info( + "dropping manifest due to missing backup", + "manifest_id", man.ID) + + continue + } + + deetsID := bup.StreamStoreID + if len(deetsID) == 0 { + deetsID = bup.DetailsID + } + + if len(bup.SnapshotID) == 0 || len(deetsID) == 0 { + toDrop[man.ID] = struct{}{} + + logger.Ctx(ctx).Info( + "dropping manifest due to invalid backup", + "manifest_id", man.ID) + + continue + } + + backupsToKeep = append(backupsToKeep, bup) + mergeToKeep = append(mergeToKeep, man) + } + + var assistToKeep []ManifestEntry + + for _, man := range bb.assistBases { + if _, ok := toDrop[man.ID]; ok { + continue + } + + assistToKeep = append(assistToKeep, man) + } + + bb.backups = backupsToKeep + bb.mergeBases = mergeToKeep + bb.assistBases = assistToKeep +} diff --git a/src/internal/kopia/backup_bases_test.go b/src/internal/kopia/backup_bases_test.go new file mode 100644 index 000000000..f902d4e37 --- /dev/null +++ b/src/internal/kopia/backup_bases_test.go @@ -0,0 +1,705 @@ +package kopia + +import ( + "fmt" + "testing" + + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/path" +) + +func makeManifest(id, incmpl, bID string, reasons ...Reason) ManifestEntry { + bIDKey, _ := makeTagKV(TagBackupID) + + return ManifestEntry{ + Manifest: &snapshot.Manifest{ + ID: manifest.ID(id), + IncompleteReason: incmpl, + Tags: map[string]string{bIDKey: bID}, + }, + Reasons: reasons, + } +} + +type BackupBasesUnitSuite struct { + tester.Suite +} + +func TestBackupBasesUnitSuite(t *testing.T) { + suite.Run(t, &BackupBasesUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *BackupBasesUnitSuite) TestMinBackupVersion() { + table := []struct { + name string + bb *backupBases + expectedVersion int + }{ + { + name: "Nil BackupBase", + expectedVersion: version.NoBackup, + }, + { + name: "No Backups", + bb: &backupBases{}, + expectedVersion: version.NoBackup, + }, + { + name: "Unsorted Backups", + bb: &backupBases{ + backups: []BackupEntry{ + { + Backup: &backup.Backup{ + Version: 4, + }, + }, + { + Backup: &backup.Backup{ + Version: 0, + }, + }, + { + Backup: &backup.Backup{ + Version: 2, + }, + }, + }, + }, + expectedVersion: 0, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + assert.Equal(suite.T(), test.expectedVersion, test.bb.MinBackupVersion()) + }) + } +} + +func (suite *BackupBasesUnitSuite) TestRemoveMergeBaseByManifestID() { + backups := []BackupEntry{ + {Backup: &backup.Backup{SnapshotID: "1"}}, + {Backup: &backup.Backup{SnapshotID: "2"}}, + {Backup: &backup.Backup{SnapshotID: "3"}}, + } + + merges := []ManifestEntry{ + makeManifest("1", "", ""), + makeManifest("2", "", ""), + makeManifest("3", "", ""), + } + + expected := &backupBases{ + backups: []BackupEntry{backups[0], backups[1]}, + mergeBases: []ManifestEntry{merges[0], merges[1]}, + assistBases: []ManifestEntry{merges[0], merges[1]}, + } + + delID := manifest.ID("3") + + table := []struct { + name string + // Below indices specify which items to add from the defined sets above. + backup []int + merge []int + assist []int + }{ + { + name: "Not In Bases", + backup: []int{0, 1}, + merge: []int{0, 1}, + assist: []int{0, 1}, + }, + { + name: "Different Indexes", + backup: []int{2, 0, 1}, + merge: []int{0, 2, 1}, + assist: []int{0, 1, 2}, + }, + { + name: "First Item", + backup: []int{2, 0, 1}, + merge: []int{2, 0, 1}, + assist: []int{2, 0, 1}, + }, + { + name: "Middle Item", + backup: []int{0, 2, 1}, + merge: []int{0, 2, 1}, + assist: []int{0, 2, 1}, + }, + { + name: "Final Item", + backup: []int{0, 1, 2}, + merge: []int{0, 1, 2}, + assist: []int{0, 1, 2}, + }, + { + name: "Only In Backups", + backup: []int{0, 1, 2}, + merge: []int{0, 1}, + assist: []int{0, 1}, + }, + { + name: "Only In Merges", + backup: []int{0, 1}, + merge: []int{0, 1, 2}, + assist: []int{0, 1}, + }, + { + name: "Only In Assists", + backup: []int{0, 1}, + merge: []int{0, 1}, + assist: []int{0, 1, 2}, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + bb := &backupBases{} + + for _, i := range test.backup { + bb.backups = append(bb.backups, backups[i]) + } + + for _, i := range test.merge { + bb.mergeBases = append(bb.mergeBases, merges[i]) + } + + for _, i := range test.assist { + bb.assistBases = append(bb.assistBases, merges[i]) + } + + bb.RemoveMergeBaseByManifestID(delID) + AssertBackupBasesEqual(t, expected, bb) + }) + } +} + +func (suite *BackupBasesUnitSuite) TestClearMergeBases() { + bb := &backupBases{ + backups: make([]BackupEntry, 2), + mergeBases: make([]ManifestEntry, 2), + } + + bb.ClearMergeBases() + assert.Empty(suite.T(), bb.Backups()) + assert.Empty(suite.T(), bb.MergeBases()) +} + +func (suite *BackupBasesUnitSuite) TestClearAssistBases() { + bb := &backupBases{assistBases: make([]ManifestEntry, 2)} + + bb.ClearAssistBases() + assert.Empty(suite.T(), bb.AssistBases()) +} + +func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { + ro := "resource_owner" + + type testInput struct { + id int + incomplete bool + cat []path.CategoryType + } + + // Make a function so tests can modify things without messing with each other. + makeBackupBases := func(ti []testInput) *backupBases { + res := &backupBases{} + + for _, i := range ti { + baseID := fmt.Sprintf("id%d", i.id) + ir := "" + + if i.incomplete { + ir = "checkpoint" + } + + reasons := make([]Reason, 0, len(i.cat)) + + for _, c := range i.cat { + reasons = append(reasons, Reason{ + ResourceOwner: ro, + Service: path.ExchangeService, + Category: c, + }) + } + + m := makeManifest(baseID, ir, "b"+baseID, reasons...) + res.assistBases = append(res.assistBases, m) + + if i.incomplete { + continue + } + + b := BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)}, + SnapshotID: baseID, + StreamStoreID: "ss" + baseID, + }, + Reasons: reasons, + } + + res.backups = append(res.backups, b) + res.mergeBases = append(res.mergeBases, m) + } + + return res + } + + table := []struct { + name string + bb []testInput + other []testInput + expect []testInput + }{ + { + name: "Other Empty", + bb: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + }, + { + name: "BB Empty", + other: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + }, + { + name: "Other overlaps Complete And Incomplete", + bb: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + other: []testInput{ + { + id: 2, + cat: []path.CategoryType{path.EmailCategory}, + }, + { + id: 3, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + }, + { + name: "Other Overlaps Complete", + bb: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + other: []testInput{ + { + id: 2, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + }, + }, + { + name: "Other Overlaps Incomplete", + bb: []testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + other: []testInput{ + { + id: 2, + cat: []path.CategoryType{path.EmailCategory}, + }, + { + id: 3, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + expect: []testInput{ + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + { + id: 2, + cat: []path.CategoryType{path.EmailCategory}, + }, + }, + }, + { + name: "Other Disjoint", + bb: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + other: []testInput{ + { + id: 2, + cat: []path.CategoryType{path.ContactsCategory}, + }, + { + id: 3, + cat: []path.CategoryType{path.ContactsCategory}, + incomplete: true, + }, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + { + id: 2, + cat: []path.CategoryType{path.ContactsCategory}, + }, + { + id: 3, + cat: []path.CategoryType{path.ContactsCategory}, + incomplete: true, + }, + }, + }, + { + name: "Other Reduced Reasons", + bb: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + }, + other: []testInput{ + { + id: 2, + cat: []path.CategoryType{ + path.EmailCategory, + path.ContactsCategory, + }, + }, + { + id: 3, + cat: []path.CategoryType{ + path.EmailCategory, + path.ContactsCategory, + }, + incomplete: true, + }, + }, + expect: []testInput{ + {cat: []path.CategoryType{path.EmailCategory}}, + { + id: 1, + cat: []path.CategoryType{path.EmailCategory}, + incomplete: true, + }, + { + id: 2, + cat: []path.CategoryType{path.ContactsCategory}, + }, + { + id: 3, + cat: []path.CategoryType{path.ContactsCategory}, + incomplete: true, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + bb := makeBackupBases(test.bb) + other := makeBackupBases(test.other) + expect := makeBackupBases(test.expect) + + ctx, flush := tester.NewContext(t) + defer flush() + + got := bb.MergeBackupBases( + ctx, + other, + func(reason Reason) string { + return reason.Service.String() + reason.Category.String() + }) + AssertBackupBasesEqual(t, expect, got) + }) + } +} + +func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { + ro := "resource_owner" + + makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry { + reason := Reason{ + ResourceOwner: ro, + Service: path.ExchangeService, + Category: pct, + } + + return makeManifest(id, incmpl, bID, reason) + } + + // Make a function so tests can modify things without messing with each other. + validMail1 := func() *backupBases { + return &backupBases{ + backups: []BackupEntry{ + { + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid1", + }, + SnapshotID: "id1", + StreamStoreID: "ssid1", + }, + }, + }, + mergeBases: []ManifestEntry{ + makeMan(path.EmailCategory, "id1", "", "bid1"), + }, + assistBases: []ManifestEntry{ + makeMan(path.EmailCategory, "id1", "", "bid1"), + }, + } + } + + table := []struct { + name string + bb *backupBases + expect BackupBases + }{ + { + name: "empty BaseBackups", + bb: &backupBases{}, + }, + { + name: "Merge Base Without Backup", + bb: func() *backupBases { + res := validMail1() + res.backups = nil + + return res + }(), + }, + { + name: "Backup Missing Snapshot ID", + bb: func() *backupBases { + res := validMail1() + res.backups[0].SnapshotID = "" + + return res + }(), + }, + { + name: "Backup Missing Deets ID", + bb: func() *backupBases { + res := validMail1() + res.backups[0].StreamStoreID = "" + + return res + }(), + }, + { + name: "Incomplete Snapshot", + bb: func() *backupBases { + res := validMail1() + res.mergeBases[0].IncompleteReason = "ir" + res.assistBases[0].IncompleteReason = "ir" + + return res + }(), + }, + { + name: "Duplicate Reason", + bb: func() *backupBases { + res := validMail1() + res.mergeBases[0].Reasons = append( + res.mergeBases[0].Reasons, + res.mergeBases[0].Reasons[0]) + res.assistBases = res.mergeBases + + return res + }(), + }, + { + name: "Single Valid Entry", + bb: validMail1(), + expect: validMail1(), + }, + { + name: "Single Valid Entry With Incomplete Assist With Same Reason", + bb: func() *backupBases { + res := validMail1() + res.assistBases = append( + res.assistBases, + makeMan(path.EmailCategory, "id2", "checkpoint", "bid2")) + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.assistBases = append( + res.assistBases, + makeMan(path.EmailCategory, "id2", "checkpoint", "bid2")) + + return res + }(), + }, + { + name: "Single Valid Entry With Backup With Old Deets ID", + bb: func() *backupBases { + res := validMail1() + res.backups[0].DetailsID = res.backups[0].StreamStoreID + res.backups[0].StreamStoreID = "" + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.backups[0].DetailsID = res.backups[0].StreamStoreID + res.backups[0].StreamStoreID = "" + + return res + }(), + }, + { + name: "Single Valid Entry With Multiple Reasons", + bb: func() *backupBases { + res := validMail1() + res.mergeBases[0].Reasons = append( + res.mergeBases[0].Reasons, + Reason{ + ResourceOwner: ro, + Service: path.ExchangeService, + Category: path.ContactsCategory, + }) + res.assistBases = res.mergeBases + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.mergeBases[0].Reasons = append( + res.mergeBases[0].Reasons, + Reason{ + ResourceOwner: ro, + Service: path.ExchangeService, + Category: path.ContactsCategory, + }) + res.assistBases = res.mergeBases + + return res + }(), + }, + { + name: "Two Entries Overlapping Reasons", + bb: func() *backupBases { + res := validMail1() + res.mergeBases = append( + res.mergeBases, + makeMan(path.EmailCategory, "id2", "", "bid2")) + res.assistBases = res.mergeBases + + return res + }(), + }, + { + name: "Three Entries One Invalid", + bb: func() *backupBases { + res := validMail1() + res.backups = append( + res.backups, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid2", + }, + }, + }, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid3", + }, + SnapshotID: "id3", + StreamStoreID: "ssid3", + }, + }) + res.mergeBases = append( + res.mergeBases, + makeMan(path.ContactsCategory, "id2", "checkpoint", "bid2"), + makeMan(path.EventsCategory, "id3", "", "bid3")) + res.assistBases = res.mergeBases + + return res + }(), + expect: func() *backupBases { + res := validMail1() + res.backups = append( + res.backups, + BackupEntry{ + Backup: &backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid3", + }, + SnapshotID: "id3", + StreamStoreID: "ssid3", + }, + }) + res.mergeBases = append( + res.mergeBases, + makeMan(path.EventsCategory, "id3", "", "bid3")) + res.assistBases = res.mergeBases + + return res + }(), + }, + } + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext(suite.T()) + defer flush() + + test.bb.fixupAndVerify(ctx) + AssertBackupBasesEqual(suite.T(), test.expect, test.bb) + }) + } +} diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index b01c4401a..ebe8f3287 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -47,12 +47,6 @@ func (r Reason) Key() string { return r.ResourceOwner + r.Service.String() + r.Category.String() } -type backupBases struct { - backups []BackupEntry - mergeBases []ManifestEntry - assistBases []ManifestEntry -} - type BackupEntry struct { *backup.Backup Reasons []Reason diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index 2382063cd..ca84f5d94 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -342,8 +342,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { bb, err := bf.findBases(ctx, reasons, nil) assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) - assert.Empty(t, bb.mergeBases) - assert.Empty(t, bb.assistBases) + assert.Empty(t, bb.MergeBases()) + assert.Empty(t, bb.AssistBases()) } func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { @@ -366,8 +366,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { bb, err := bf.findBases(ctx, reasons, nil) assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) - assert.Empty(t, bb.mergeBases) - assert.Empty(t, bb.assistBases) + assert.Empty(t, bb.MergeBases()) + assert.Empty(t, bb.AssistBases()) } func (suite *BaseFinderUnitSuite) TestGetBases() { @@ -831,24 +831,24 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { checkBackupEntriesMatch( t, - bb.backups, + bb.Backups(), test.backupData, test.expectedBaseReasons) checkManifestEntriesMatch( t, - bb.mergeBases, + bb.MergeBases(), test.manifestData, test.expectedBaseReasons) checkManifestEntriesMatch( t, - bb.assistBases, + bb.AssistBases(), test.manifestData, test.expectedAssistManifestReasons) }) } } -func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() { +func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { manifestData := []manifestInfo{ newManifestInfo2( testID1, @@ -926,7 +926,7 @@ func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() { checkManifestEntriesMatch( t, - bb.mergeBases, + bb.MergeBases(), manifestData, test.expectedIdxs) }) diff --git a/src/internal/kopia/mock_backup_base.go b/src/internal/kopia/mock_backup_base.go new file mode 100644 index 000000000..84743486e --- /dev/null +++ b/src/internal/kopia/mock_backup_base.go @@ -0,0 +1,63 @@ +package kopia + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) { + if expect == nil && got == nil { + return + } + + if expect == nil { + assert.Empty(t, got.Backups(), "backups") + assert.Empty(t, got.MergeBases(), "merge bases") + assert.Empty(t, got.AssistBases(), "assist bases") + + return + } + + if got == nil { + if len(expect.Backups()) > 0 && len(expect.MergeBases()) > 0 && len(expect.AssistBases()) > 0 { + assert.Fail(t, "got was nil but expected non-nil result %v", expect) + } + + return + } + + assert.ElementsMatch(t, expect.Backups(), got.Backups(), "backups") + assert.ElementsMatch(t, expect.MergeBases(), got.MergeBases(), "merge bases") + assert.ElementsMatch(t, expect.AssistBases(), got.AssistBases(), "assist bases") +} + +func NewMockBackupBases() *MockBackupBases { + return &MockBackupBases{backupBases: &backupBases{}} +} + +type MockBackupBases struct { + *backupBases +} + +func (bb *MockBackupBases) WithBackups(b ...BackupEntry) *MockBackupBases { + bb.backupBases.backups = append(bb.Backups(), b...) + return bb +} + +func (bb *MockBackupBases) WithMergeBases(m ...ManifestEntry) *MockBackupBases { + bb.backupBases.mergeBases = append(bb.MergeBases(), m...) + bb.backupBases.assistBases = append(bb.AssistBases(), m...) + + return bb +} + +func (bb *MockBackupBases) WithAssistBases(m ...ManifestEntry) *MockBackupBases { + bb.backupBases.assistBases = append(bb.AssistBases(), m...) + return bb +} + +func (bb *MockBackupBases) ClearMockAssistBases() *MockBackupBases { + bb.backupBases.ClearAssistBases() + return bb +}