diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go new file mode 100644 index 000000000..e1adef823 --- /dev/null +++ b/src/internal/kopia/base_finder.go @@ -0,0 +1,259 @@ +package kopia + +import ( + "context" + "sort" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/snapshot" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/logger" +) + +type BackupBases struct { + Backups []BackupEntry + MergeBases []ManifestEntry + AssistBases []ManifestEntry +} + +type BackupEntry struct { + *backup.Backup + Reasons []Reason +} + +type baseFinder struct { + sm snapshotManager + bg inject.GetBackuper +} + +func NewBaseFinder( + sm snapshotManager, + bg inject.GetBackuper, +) (*baseFinder, error) { + if sm == nil { + return nil, clues.New("nil snapshotManager") + } + + if bg == nil { + return nil, clues.New("nil GetBackuper") + } + + return &baseFinder{ + sm: sm, + bg: bg, + }, nil +} + +func (b *baseFinder) getBackupModel( + ctx context.Context, + man *snapshot.Manifest, +) (*backup.Backup, error) { + k, _ := makeTagKV(TagBackupID) + bID := man.Tags[k] + + ctx = clues.Add(ctx, "search_backup_id", bID) + + bup, err := b.bg.GetBackup(ctx, model.StableID(bID)) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx) + } + + return bup, nil +} + +// findBasesInSet goes through manifest metadata entries and sees if they're +// incomplete or not. If an entry is incomplete and we don't already have a +// complete or incomplete manifest add it to the set for kopia assisted +// incrementals. If it's complete, fetch the backup model and see if it +// corresponds to a successful backup. If it does, return it as we only need the +// most recent complete backup as the base. +func (b *baseFinder) findBasesInSet( + ctx context.Context, + reason Reason, + metas []*manifest.EntryMetadata, +) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { + // Sort manifests by time so we can go through them sequentially. The code in + // kopia appears to sort them already, but add sorting here just so we're not + // reliant on undocumented behavior. + sort.Slice(metas, func(i, j int) bool { + return metas[i].ModTime.Before(metas[j].ModTime) + }) + + var ( + kopiaAssistSnaps []ManifestEntry + foundIncomplete bool + ) + + for i := len(metas) - 1; i >= 0; i-- { + meta := metas[i] + ictx := clues.Add(ctx, "search_snapshot_id", meta.ID) + + man, err := b.sm.LoadSnapshot(ictx, meta.ID) + if err != nil { + // Safe to continue here as we'll just end up attempting to use an older + // backup as the base. + logger.CtxErr(ictx, err).Info("attempting to get snapshot") + continue + } + + if len(man.IncompleteReason) > 0 { + if !foundIncomplete { + foundIncomplete = true + + kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ + Manifest: man, + Reasons: []Reason{reason}, + }) + } + + continue + } + + // This is a complete snapshot so see if we have a backup model for it. + bup, err := b.getBackupModel(ictx, man) + if err != nil { + // Safe to continue here as we'll just end up attempting to use an older + // backup as the base. + logger.CtxErr(ictx, err).Debug("searching for base backup") + continue + } + + ssid := bup.StreamStoreID + if len(ssid) == 0 { + ssid = bup.DetailsID + } + + if len(ssid) == 0 { + logger.Ctx(ictx).Debugw( + "empty backup stream store ID", + "search_backup_id", bup.ID) + + continue + } + + // If we've made it to this point then we're considering the backup + // complete as it has both an item data snapshot and a backup details + // snapshot. + me := ManifestEntry{ + Manifest: man, + Reasons: []Reason{reason}, + } + kopiaAssistSnaps = append(kopiaAssistSnaps, me) + + return &BackupEntry{ + Backup: bup, + Reasons: []Reason{reason}, + }, &me, kopiaAssistSnaps, nil + } + + logger.Ctx(ctx).Info("no base backups for reason") + + return nil, nil, kopiaAssistSnaps, nil +} + +func (b *baseFinder) getBase( + ctx context.Context, + reason Reason, + tags map[string]string, +) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { + allTags := map[string]string{} + + for _, k := range reason.TagKeys() { + allTags[k] = "" + } + + maps.Copy(allTags, tags) + allTags = normalizeTagKVs(allTags) + + metas, err := b.sm.FindManifests(ctx, allTags) + if err != nil { + return nil, nil, nil, clues.Wrap(err, "getting snapshots") + } + + // No snapshots means no backups so we can just exit here. + if len(metas) == 0 { + return nil, nil, nil, nil + } + + return b.findBasesInSet(ctx, reason, metas) +} + +func (b *baseFinder) FindBases( + ctx context.Context, + reasons []Reason, + tags map[string]string, +) (BackupBases, error) { + var ( + // All maps go from ID -> entry. We need to track by ID so we can coalesce + // the reason for selecting something. Kopia assisted snapshots also use + // ManifestEntry so we have the reasons for selecting them to aid in + // debugging. + baseBups = map[model.StableID]BackupEntry{} + baseSnaps = map[manifest.ID]ManifestEntry{} + kopiaAssistSnaps = map[manifest.ID]ManifestEntry{} + ) + + for _, reason := range reasons { + ictx := clues.Add( + ctx, + "search_service", reason.Service.String(), + "search_category", reason.Category.String()) + logger.Ctx(ictx).Info("searching for previous manifests") + + baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, reason, tags) + if err != nil { + logger.Ctx(ctx).Info( + "getting base, falling back to full backup for reason", + "error", err) + + continue + } + + if baseBackup != nil { + bs, ok := baseBups[baseBackup.ID] + if ok { + bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) + } else { + bs = *baseBackup + } + + // Reassign since it's structs not pointers to structs. + baseBups[baseBackup.ID] = bs + } + + if baseSnap != nil { + bs, ok := baseSnaps[baseSnap.ID] + if ok { + bs.Reasons = append(bs.Reasons, baseSnap.Reasons...) + } else { + bs = *baseSnap + } + + // Reassign since it's structs not pointers to structs. + baseSnaps[baseSnap.ID] = bs + } + + for _, s := range assistSnaps { + bs, ok := kopiaAssistSnaps[s.ID] + if ok { + bs.Reasons = append(bs.Reasons, s.Reasons...) + } else { + bs = s + } + + // Reassign since it's structs not pointers to structs. + kopiaAssistSnaps[s.ID] = bs + } + } + + return BackupBases{ + Backups: maps.Values(baseBups), + MergeBases: maps.Values(baseSnaps), + AssistBases: maps.Values(kopiaAssistSnaps), + }, nil +} diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go new file mode 100644 index 000000000..4a9f47100 --- /dev/null +++ b/src/internal/kopia/base_finder_test.go @@ -0,0 +1,1020 @@ +package kopia + +import ( + "context" + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/path" +) + +const ( + testCompleteMan = false + testIncompleteMan = !testCompleteMan +) + +var ( + testT1 = time.Now() + testT2 = testT1.Add(1 * time.Hour) + testT3 = testT2.Add(1 * time.Hour) + + testID1 = manifest.ID("snap1") + testID2 = manifest.ID("snap2") + testID3 = manifest.ID("snap3") + + testBackup1 = "backupID1" + testBackup2 = "backupID2" + + testMail = path.ExchangeService.String() + path.EmailCategory.String() + testEvents = path.ExchangeService.String() + path.EventsCategory.String() + + testUser1 = "user1" + testUser2 = "user2" + testUser3 = "user3" + + testAllUsersAllCats = []Reason{ + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + { + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + { + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + } + testAllUsersMail = []Reason{ + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + } + testUser1Mail = []Reason{ + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + } +) + +// ----------------------------------------------------------------------------- +// Empty mocks that return no data +// ----------------------------------------------------------------------------- +type mockEmptySnapshotManager struct{} + +func (sm mockEmptySnapshotManager) FindManifests( + context.Context, + map[string]string, +) ([]*manifest.EntryMetadata, error) { + return nil, nil +} + +func (sm mockEmptySnapshotManager) LoadSnapshots( + context.Context, + []manifest.ID, +) ([]*snapshot.Manifest, error) { + return nil, clues.New("not implemented") +} + +func (sm mockEmptySnapshotManager) LoadSnapshot( + context.Context, + manifest.ID, +) (*snapshot.Manifest, error) { + return nil, snapshot.ErrSnapshotNotFound +} + +type mockEmptyModelGetter struct{} + +func (mg mockEmptyModelGetter) GetBackup( + context.Context, + model.StableID, +) (*backup.Backup, error) { + return nil, data.ErrNotFound +} + +// ----------------------------------------------------------------------------- +// Mocks that return data or errors +// ----------------------------------------------------------------------------- +type manifestInfo struct { + // We don't currently use the values in the tags. + tags map[string]string + metadata *manifest.EntryMetadata + man *snapshot.Manifest + err error +} + +func newManifestInfo2( + id manifest.ID, + modTime time.Time, + incomplete bool, + backupID string, + err error, + tags ...string, +) manifestInfo { + incompleteStr := "" + if incomplete { + incompleteStr = "checkpoint" + } + + structTags := make(map[string]string, len(tags)) + + for _, t := range tags { + tk, _ := makeTagKV(t) + structTags[tk] = "" + } + + res := manifestInfo{ + tags: structTags, + err: err, + metadata: &manifest.EntryMetadata{ + ID: id, + ModTime: modTime, + Labels: structTags, + }, + man: &snapshot.Manifest{ + ID: id, + IncompleteReason: incompleteStr, + Tags: structTags, + }, + } + + if len(backupID) > 0 { + k, _ := makeTagKV(TagBackupID) + res.metadata.Labels[k] = backupID + res.man.Tags[k] = backupID + } + + return res +} + +type mockSnapshotManager2 struct { + data []manifestInfo + findErr error +} + +func matchesTags2(mi manifestInfo, tags map[string]string) bool { + for k := range tags { + if _, ok := mi.tags[k]; !ok { + return false + } + } + + return true +} + +func (msm *mockSnapshotManager2) FindManifests( + ctx context.Context, + tags map[string]string, +) ([]*manifest.EntryMetadata, error) { + if msm == nil { + return nil, assert.AnError + } + + if msm.findErr != nil { + return nil, msm.findErr + } + + res := []*manifest.EntryMetadata{} + + for _, mi := range msm.data { + if matchesTags2(mi, tags) { + res = append(res, mi.metadata) + } + } + + return res, nil +} + +func (msm *mockSnapshotManager2) LoadSnapshots( + ctx context.Context, + ids []manifest.ID, +) ([]*snapshot.Manifest, error) { + return nil, clues.New("not implemented") +} + +func (msm *mockSnapshotManager2) LoadSnapshot( + ctx context.Context, + id manifest.ID, +) (*snapshot.Manifest, error) { + if msm == nil { + return nil, assert.AnError + } + + for _, mi := range msm.data { + if mi.man.ID == id { + return mi.man, nil + } + } + + return nil, snapshot.ErrSnapshotNotFound +} + +type backupInfo struct { + b backup.Backup + err error +} + +func newBackupModel( + id string, + hasItemSnap bool, + hasDetailsSnap bool, + oldDetailsID bool, + err error, +) backupInfo { + res := backupInfo{ + b: backup.Backup{ + BaseModel: model.BaseModel{ + ID: model.StableID(id), + }, + SnapshotID: "iid", + }, + err: err, + } + + if !oldDetailsID { + res.b.StreamStoreID = "ssid" + } else { + res.b.DetailsID = "ssid" + } + + return res +} + +type mockModelGetter struct { + data []backupInfo +} + +func (mg mockModelGetter) GetBackup( + _ context.Context, + id model.StableID, +) (*backup.Backup, error) { + // Use struct here so we return a copy of the struct just in case the caller + // somehow ends up modifying it. + var res backup.Backup + + for _, bi := range mg.data { + if bi.b.ID != id { + continue + } + + if bi.err != nil { + return nil, bi.err + } + + res = bi.b + + return &res, nil + } + + return nil, data.ErrNotFound +} + +// ----------------------------------------------------------------------------- +// Tests for getting bases +// ----------------------------------------------------------------------------- +type BaseFinderUnitSuite struct { + tester.Suite +} + +func TestBaseFinderUnitSuite(t *testing.T) { + suite.Run(t, &BaseFinderUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + bf := baseFinder{ + sm: mockEmptySnapshotManager{}, + bg: mockEmptyModelGetter{}, + } + reasons := []Reason{ + { + ResourceOwner: "a-user", + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + } + + bb, err := bf.FindBases(ctx, reasons, nil) + assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) + assert.Empty(t, bb.MergeBases) + assert.Empty(t, bb.AssistBases) +} + +func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + bf := baseFinder{ + sm: &mockSnapshotManager2{findErr: assert.AnError}, + bg: mockEmptyModelGetter{}, + } + reasons := []Reason{ + { + ResourceOwner: "a-user", + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + } + + bb, err := bf.FindBases(ctx, reasons, nil) + assert.NoError(t, err, "getting bases: %v", clues.ToCore(err)) + assert.Empty(t, bb.MergeBases) + assert.Empty(t, bb.AssistBases) +} + +func (suite *BaseFinderUnitSuite) TestGetBases() { + table := []struct { + name string + input []Reason + manifestData []manifestInfo + // Use this to denote the Reasons a base backup or base manifest is + // selected. The int maps to the index of the backup or manifest in data. + expectedBaseReasons map[int][]Reason + // Use this to denote the Reasons a kopia assised incrementals manifest is + // selected. The int maps to the index of the manifest in data. + expectedAssistManifestReasons map[int][]Reason + backupData []backupInfo + }{ + { + name: "Return Older Base If Fail To Get Manifest", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + assert.AnError, + testMail, + testUser1, + ), + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, false, false, false, assert.AnError), + }, + }, + { + name: "Return Older Base If Fail To Get Backup", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, false, false, false, assert.AnError), + }, + }, + { + name: "Return Older Base If Missing Details", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil), + newBackupModel(testBackup1, true, false, false, nil), + }, + }, + { + name: "Old Backup Details Pointer", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testEvents, + testUser1, + testUser2, + testUser3, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup1, true, true, true, nil), + }, + }, + { + name: "All One Snapshot", + input: testAllUsersAllCats, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testEvents, + testUser1, + testUser2, + testUser3, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testAllUsersAllCats, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testAllUsersAllCats, + }, + backupData: []backupInfo{ + newBackupModel(testBackup1, true, true, false, nil), + }, + }, + { + name: "Multiple Bases Some Overlapping Reasons", + input: testAllUsersAllCats, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testEvents, + testUser1, + testUser2, + testUser3, + ), + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testEvents, + testUser1, + testUser2, + testUser3, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: { + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + }, + 1: { + Reason{ + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + Reason{ + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + Reason{ + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + }, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: { + { + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + { + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EmailCategory, + }, + }, + 1: { + Reason{ + ResourceOwner: testUser1, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + Reason{ + ResourceOwner: testUser2, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + Reason{ + ResourceOwner: testUser3, + Service: path.ExchangeService, + Category: path.EventsCategory, + }, + }, + }, + backupData: []backupInfo{ + newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil), + }, + }, + { + name: "Newer Incomplete Assist Snapshot", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID2, + testT2, + testIncompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + 1: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup1, true, true, false, nil), + // Shouldn't be returned but have here just so we can see. + newBackupModel(testBackup2, true, true, false, nil), + }, + }, + { + name: "Incomplete Older Than Complete", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testIncompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 1: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 1: testUser1Mail, + }, + backupData: []backupInfo{ + // Shouldn't be returned but have here just so we can see. + newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil), + }, + }, + { + name: "Newest Incomplete Only Incomplete", + input: testUser1Mail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testIncompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID2, + testT2, + testIncompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{}, + expectedAssistManifestReasons: map[int][]Reason{ + 1: testUser1Mail, + }, + backupData: []backupInfo{ + // Shouldn't be returned but have here just so we can see. + newBackupModel(testBackup1, true, true, false, nil), + newBackupModel(testBackup2, true, true, false, nil), + }, + }, + { + name: "Some Bases Not Found", + input: testAllUsersMail, + manifestData: []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup1, true, true, false, nil), + }, + }, + { + name: "Manifests Not Sorted", + input: testAllUsersMail, + // Manifests are currently returned in the order they're defined by the + // mock. + manifestData: []manifestInfo{ + newManifestInfo2( + testID2, + testT2, + testCompleteMan, + testBackup2, + nil, + testMail, + testUser1, + ), + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + ), + }, + expectedBaseReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + expectedAssistManifestReasons: map[int][]Reason{ + 0: testUser1Mail, + }, + backupData: []backupInfo{ + newBackupModel(testBackup2, true, true, false, nil), + // Shouldn't be returned but here just so we can check. + newBackupModel(testBackup1, true, true, false, nil), + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + bf := baseFinder{ + sm: &mockSnapshotManager2{data: test.manifestData}, + bg: &mockModelGetter{data: test.backupData}, + } + + bb, err := bf.FindBases( + ctx, + test.input, + nil) + require.NoError(t, err, "getting bases: %v", clues.ToCore(err)) + + checkBackupEntriesMatch( + t, + bb.Backups, + test.backupData, + test.expectedBaseReasons) + checkManifestEntriesMatch( + t, + bb.MergeBases, + test.manifestData, + test.expectedBaseReasons) + checkManifestEntriesMatch( + t, + bb.AssistBases, + test.manifestData, + test.expectedAssistManifestReasons) + }) + } +} + +func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() { + manifestData := []manifestInfo{ + newManifestInfo2( + testID1, + testT1, + testCompleteMan, + testBackup1, + nil, + testMail, + testUser1, + "fnords", + "smarf", + ), + } + backupData := []backupInfo{ + newBackupModel(testBackup1, true, true, false, nil), + } + + table := []struct { + name string + input []Reason + tags map[string]string + // Use this to denote which manifests in data should be expected. Allows + // defining data in a table while not repeating things between data and + // expected. + expectedIdxs map[int][]Reason + }{ + { + name: "no tags specified", + tags: nil, + expectedIdxs: map[int][]Reason{ + 0: testUser1Mail, + }, + }, + { + name: "all custom tags", + tags: map[string]string{ + "fnords": "", + "smarf": "", + }, + expectedIdxs: map[int][]Reason{ + 0: testUser1Mail, + }, + }, + { + name: "subset of custom tags", + tags: map[string]string{"fnords": ""}, + expectedIdxs: map[int][]Reason{ + 0: testUser1Mail, + }, + }, + { + name: "custom tag mismatch", + tags: map[string]string{"bojangles": ""}, + expectedIdxs: nil, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + bf := baseFinder{ + sm: &mockSnapshotManager2{data: manifestData}, + bg: &mockModelGetter{data: backupData}, + } + + bb, err := bf.FindBases( + ctx, + testAllUsersAllCats, + test.tags) + require.NoError(t, err, "getting bases: %v", clues.ToCore(err)) + + checkManifestEntriesMatch( + t, + bb.MergeBases, + manifestData, + test.expectedIdxs) + }) + } +} + +func checkManifestEntriesMatch( + t *testing.T, + retSnaps []ManifestEntry, + allExpected []manifestInfo, + expectedIdxsAndReasons map[int][]Reason, +) { + // Check the proper snapshot manifests were returned. + expected := make([]*snapshot.Manifest, 0, len(expectedIdxsAndReasons)) + for i := range expectedIdxsAndReasons { + expected = append(expected, allExpected[i].man) + } + + got := make([]*snapshot.Manifest, 0, len(retSnaps)) + for _, s := range retSnaps { + got = append(got, s.Manifest) + } + + assert.ElementsMatch(t, expected, got) + + // Check the reasons for selecting each manifest are correct. + expectedReasons := make(map[manifest.ID][]Reason, len(expectedIdxsAndReasons)) + for idx, reasons := range expectedIdxsAndReasons { + expectedReasons[allExpected[idx].man.ID] = reasons + } + + for _, found := range retSnaps { + reasons, ok := expectedReasons[found.ID] + if !ok { + // Missing or extra snapshots will be reported by earlier checks. + continue + } + + assert.ElementsMatch( + t, + reasons, + found.Reasons, + "incorrect reasons for snapshot with ID %s", + found.ID, + ) + } +} + +func checkBackupEntriesMatch( + t *testing.T, + retBups []BackupEntry, + allExpected []backupInfo, + expectedIdxsAndReasons map[int][]Reason, +) { + // Check the proper snapshot manifests were returned. + expected := make([]*backup.Backup, 0, len(expectedIdxsAndReasons)) + for i := range expectedIdxsAndReasons { + expected = append(expected, &allExpected[i].b) + } + + got := make([]*backup.Backup, 0, len(retBups)) + for _, s := range retBups { + got = append(got, s.Backup) + } + + assert.ElementsMatch(t, expected, got) + + // Check the reasons for selecting each manifest are correct. + expectedReasons := make(map[model.StableID][]Reason, len(expectedIdxsAndReasons)) + for idx, reasons := range expectedIdxsAndReasons { + expectedReasons[allExpected[idx].b.ID] = reasons + } + + for _, found := range retBups { + reasons, ok := expectedReasons[found.ID] + if !ok { + // Missing or extra snapshots will be reported by earlier checks. + continue + } + + assert.ElementsMatch( + t, + reasons, + found.Reasons, + "incorrect reasons for snapshot with ID %s", + found.ID, + ) + } +} diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index 09d4a34ff..3447e810f 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -406,6 +406,18 @@ func checkCompressor(compressor compression.Name) error { return clues.Stack(clues.New("unknown compressor type"), clues.New(string(compressor))) } +func (w *conn) LoadSnapshot( + ctx context.Context, + id manifest.ID, +) (*snapshot.Manifest, error) { + man, err := snapshot.LoadSnapshot(ctx, w.Repository, id) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx) + } + + return man, nil +} + func (w *conn) LoadSnapshots( ctx context.Context, ids []manifest.ID, diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index 9c1f8c697..be1d9a16b 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -33,4 +33,12 @@ type ( errs *fault.Bus, ) ([]data.RestoreCollection, error) } + + BaseFinder interface { + FindBases( + ctx context.Context, + reasons []kopia.Reason, + tags map[string]string, + ) (kopia.BackupBases, error) + } ) diff --git a/src/internal/kopia/snapshot_manager.go b/src/internal/kopia/snapshot_manager.go index a89eccbd5..5fa09223c 100644 --- a/src/internal/kopia/snapshot_manager.go +++ b/src/internal/kopia/snapshot_manager.go @@ -68,6 +68,8 @@ type snapshotManager interface { ctx context.Context, tags map[string]string, ) ([]*manifest.EntryMetadata, error) + LoadSnapshot(ctx context.Context, id manifest.ID) (*snapshot.Manifest, error) + // TODO(ashmrtn): Remove this when we switch to the new BaseFinder. LoadSnapshots(ctx context.Context, ids []manifest.ID) ([]*snapshot.Manifest, error) } diff --git a/src/internal/kopia/snapshot_manager_test.go b/src/internal/kopia/snapshot_manager_test.go index a407f09d7..9acc1735f 100644 --- a/src/internal/kopia/snapshot_manager_test.go +++ b/src/internal/kopia/snapshot_manager_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/alcionai/clues" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -15,85 +16,6 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -const ( - testCompleteMan = false - testIncompleteMan = !testCompleteMan -) - -var ( - testT1 = time.Now() - testT2 = testT1.Add(1 * time.Hour) - testT3 = testT2.Add(1 * time.Hour) - - testID1 = manifest.ID("snap1") - testID2 = manifest.ID("snap2") - testID3 = manifest.ID("snap3") - - testMail = path.ExchangeService.String() + path.EmailCategory.String() - testEvents = path.ExchangeService.String() + path.EventsCategory.String() - - testUser1 = "user1" - testUser2 = "user2" - testUser3 = "user3" - - testAllUsersAllCats = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - } - testAllUsersMail = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - } -) - -type manifestInfo struct { - // We don't currently use the values in the tags. - tags map[string]struct{} - metadata *manifest.EntryMetadata - man *snapshot.Manifest -} - func newManifestInfo( id manifest.ID, modTime time.Time, @@ -105,11 +27,11 @@ func newManifestInfo( incompleteStr = "checkpoint" } - structTags := make(map[string]struct{}, len(tags)) + structTags := make(map[string]string, len(tags)) for _, t := range tags { tk, _ := makeTagKV(t) - structTags[tk] = struct{}{} + structTags[tk] = "" } return manifestInfo{ @@ -186,6 +108,13 @@ func (msm *mockSnapshotManager) LoadSnapshots( return res, nil } +func (msm *mockSnapshotManager) LoadSnapshot( + ctx context.Context, + id manifest.ID, +) (*snapshot.Manifest, error) { + return nil, clues.New("not implemented") +} + type SnapshotFetchUnitSuite struct { tester.Suite } @@ -951,6 +880,13 @@ func (msm *mockErrorSnapshotManager) LoadSnapshots( return msm.sm.LoadSnapshots(ctx, ids) } +func (msm *mockErrorSnapshotManager) LoadSnapshot( + ctx context.Context, + id manifest.ID, +) (*snapshot.Manifest, error) { + return nil, clues.New("not implemented") +} + func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots_withErrors() { t := suite.T() diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index f262e3ac0..670a98293 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -6,7 +6,9 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" @@ -47,4 +49,11 @@ type ( RepoMaintenancer interface { RepoMaintenance(ctx context.Context, opts repository.Maintenance) error } + + GetBackuper interface { + GetBackup( + ctx context.Context, + backupID model.StableID, + ) (*backup.Backup, error) + } )