diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index ca657c171..d57e93e01 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -220,16 +220,6 @@ func produceManifestsAndMetadata( continue } - // TODO(ashmrtn): Uncomment this again when we need to fetch and merge - // backup details from previous snapshots. - // k, _ := kopia.MakeTagKV(kopia.TagBackupID) - // bupID := man.Tags[k] - - // bup, err := sw.GetBackup(ctx, model.StableID(bupID)) - // if err != nil { - // return nil, nil, err - // } - colls, err := collectMetadata(ctx, kw, man, metadataFiles, tenantID) if err != nil && !errors.Is(err, kopia.ErrNotFound) { // prior metadata isn't guaranteed to exist. @@ -412,6 +402,119 @@ func consumeBackupDataCollections( return bu.BackupCollections(ctx, bases, cs, sel.PathService(), oc, tags) } +func matchesReason(reasons []kopia.Reason, p path.Path) bool { + for _, reason := range reasons { + if p.ResourceOwner() == reason.ResourceOwner && + p.Service() == reason.Service && + p.Category() == reason.Category { + return true + } + } + + return false +} + +func mergeDetails( + ctx context.Context, + ms *store.Wrapper, + detailsStore detailsReader, + mans []*kopia.ManifestEntry, + shortRefsFromPrevBackup map[string]path.Path, + deets *details.Builder, +) error { + // Don't bother loading any of the base details if there's nothing we need to + // merge. + if len(shortRefsFromPrevBackup) == 0 { + return nil + } + + var addedEntries int + + for _, man := range mans { + // For now skip snapshots that aren't complete. We will need to revisit this + // when we tackle restartability. + if len(man.IncompleteReason) > 0 { + continue + } + + k, _ := kopia.MakeTagKV(kopia.TagBackupID) + bID := man.Tags[k] + + _, baseDeets, err := getBackupAndDetailsFromID( + ctx, + model.StableID(bID), + ms, + detailsStore, + ) + if err != nil { + return errors.Wrapf(err, "backup fetching base details for backup %s", bID) + } + + for _, entry := range baseDeets.Items() { + rr, err := path.FromDataLayerPath(entry.RepoRef, true) + if err != nil { + return errors.Wrapf( + err, + "parsing base item info path %s in backup %s", + entry.RepoRef, + bID, + ) + } + + // Although this base has an entry it may not be the most recent. Check + // the reasons a snapshot was returned to ensure we only choose the recent + // entries. + // + // TODO(ashmrtn): This logic will need expanded to cover entries from + // checkpoints if we start doing kopia-assisted incrementals for those. + if !matchesReason(man.Reasons, rr) { + continue + } + + newPath := shortRefsFromPrevBackup[rr.ShortRef()] + if newPath == nil { + // This entry was not sourced from a base snapshot or cached from a + // previous backup, skip it. + continue + } + + // Fixup paths in the item. + item := entry.ItemInfo + if err := details.UpdateItem(&item, newPath); err != nil { + return errors.Wrapf( + err, + "updating item info for entry from backup %s", + bID, + ) + } + + deets.Add( + newPath.String(), + newPath.ShortRef(), + newPath.ToBuilder().Dir().ShortRef(), + // TODO(ashmrtn): This may need updated if we start using this merge + // strategry for items that were cached in kopia. + newPath.String() != rr.String(), + item, + ) + + // Track how many entries we added so that we know if we got them all when + // we're done. + addedEntries++ + } + } + + if addedEntries != len(shortRefsFromPrevBackup) { + return errors.Errorf( + "incomplete migration of backup details: found %v of %v expected items", + addedEntries, + len(shortRefsFromPrevBackup), + ) + } + + return nil +} + // writes the results metrics to the operation results. // later stored in the manifest using createBackupModels. func (op *BackupOperation) persistResults( diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 12995e00d..2bf03b9c2 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -2,10 +2,13 @@ package operations import ( "context" + stdpath "path" "testing" "time" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -457,6 +460,664 @@ func (suite *BackupOpSuite) TestBackupOperation_ConsumeBackupDataCollections_Pat } } +type mockDetailsReader struct { + entries map[string]*details.Details +} + +func (mdr mockDetailsReader) ReadBackupDetails( + ctx context.Context, + detailsID string, +) (*details.Details, error) { + r := mdr.entries[detailsID] + + if r == nil { + return nil, errors.Errorf("no details for ID %s", detailsID) + } + + return r, nil +} + +type mockBackupStorer struct { + // Only using this to store backup models right now. + entries map[model.StableID]backup.Backup +} + +func (mbs mockBackupStorer) Get( + ctx context.Context, + s model.Schema, + id model.StableID, + toPopulate model.Model, +) error { + if s != model.BackupSchema { + return errors.Errorf("unexpected schema %s", s) + } + + r, ok := mbs.entries[id] + if !ok { + return errors.Errorf("model with id %s not found", id) + } + + bu, ok := toPopulate.(*backup.Backup) + if !ok { + return errors.Errorf("bad input type %T", toPopulate) + } + + *bu = r + + return nil +} + +// Functions we need to implement but don't care about. +func (mbs mockBackupStorer) Delete(context.Context, model.Schema, model.StableID) error { + return errors.New("not implemented") +} + +func (mbs mockBackupStorer) DeleteWithModelStoreID(context.Context, manifest.ID) error { + return errors.New("not implemented") +} + +func (mbs mockBackupStorer) GetIDsForType( + context.Context, + model.Schema, + map[string]string, +) ([]*model.BaseModel, error) { + return nil, errors.New("not implemented") +} + +func (mbs mockBackupStorer) GetWithModelStoreID( + context.Context, + model.Schema, + manifest.ID, + model.Model, +) error { + return errors.New("not implemented") +} + +func (mbs mockBackupStorer) Put(context.Context, model.Schema, model.Model) error { + return errors.New("not implemented") +} + +func (mbs mockBackupStorer) Update(context.Context, model.Schema, model.Model) error { + return errors.New("not implemented") +} + +// TODO(ashmrtn): Really need to factor a function like this out into some +// common file that is only compiled for tests. +func makePath( + t *testing.T, + elements []string, + isItem bool, +) path.Path { + t.Helper() + + p, err := path.FromDataLayerPath(stdpath.Join(elements...), isItem) + require.NoError(t, err) + + return p +} + +func makeDetailsEntry( + t *testing.T, + p path.Path, + size int, + updated bool, +) *details.DetailsEntry { + t.Helper() + + res := &details.DetailsEntry{ + RepoRef: p.String(), + ShortRef: p.ShortRef(), + ParentRef: p.ToBuilder().Dir().ShortRef(), + ItemInfo: details.ItemInfo{}, + Updated: updated, + } + + switch p.Service() { + case path.ExchangeService: + if p.Category() != path.EmailCategory { + assert.FailNowf( + t, + "category %s not supported in helper function", + p.Category().String(), + ) + } + + res.Exchange = &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + Size: int64(size), + } + + case path.OneDriveService: + parent, err := path.GetDriveFolderPath(p) + require.NoError(t, err) + + res.OneDrive = &details.OneDriveInfo{ + ItemType: details.OneDriveItem, + ParentPath: parent, + Size: int64(size), + } + + default: + assert.FailNowf( + t, + "service %s not supported in helper function", + p.Service().String(), + ) + } + + return res +} + +func makeManifest(backupID model.StableID, incompleteReason string) *snapshot.Manifest { + backupIDTagKey, _ := kopia.MakeTagKV(kopia.TagBackupID) + + return &snapshot.Manifest{ + Tags: map[string]string{ + backupIDTagKey: string(backupID), + }, + IncompleteReason: incompleteReason, + } +} + +func (suite *BackupOpSuite) TestBackupOperation_MergeBackupDetails() { + var ( + tenant = "a-tenant" + ro = "a-user" + + itemPath1 = makePath( + suite.T(), + []string{ + tenant, + path.OneDriveService.String(), + ro, + path.FilesCategory.String(), + "drives", + "drive-id", + "root:", + "work", + "item1", + }, + true, + ) + itemPath2 = makePath( + suite.T(), + []string{ + tenant, + path.OneDriveService.String(), + ro, + path.FilesCategory.String(), + "drives", + "drive-id", + "root:", + "personal", + "item2", + }, + true, + ) + itemPath3 = makePath( + suite.T(), + []string{ + tenant, + path.ExchangeService.String(), + ro, + path.EmailCategory.String(), + "personal", + "item3", + }, + true, + ) + + backup1 = backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid1", + }, + DetailsID: "did1", + } + + backup2 = backup.Backup{ + BaseModel: model.BaseModel{ + ID: "bid2", + }, + DetailsID: "did2", + } + + pathReason1 = kopia.Reason{ + ResourceOwner: itemPath1.ResourceOwner(), + Service: itemPath1.Service(), + Category: itemPath1.Category(), + } + pathReason3 = kopia.Reason{ + ResourceOwner: itemPath3.ResourceOwner(), + Service: itemPath3.Service(), + Category: itemPath3.Category(), + } + ) + + itemParents1, err := path.GetDriveFolderPath(itemPath1) + require.NoError(suite.T(), err) + + table := []struct { + name string + populatedModels map[model.StableID]backup.Backup + populatedDetails map[string]*details.Details + inputMans []*kopia.ManifestEntry + inputShortRefsFromPrevBackup map[string]path.Path + + errCheck assert.ErrorAssertionFunc + expectedEntries []*details.DetailsEntry + }{ + { + name: "NilShortRefsFromPrevBackup", + errCheck: assert.NoError, + // Use empty slice so we don't error out on nil != empty. + expectedEntries: []*details.DetailsEntry{}, + }, + { + name: "EmptyShortRefsFromPrevBackup", + inputShortRefsFromPrevBackup: map[string]path.Path{}, + errCheck: assert.NoError, + // Use empty slice so we don't error out on nil != empty. + expectedEntries: []*details.DetailsEntry{}, + }, + { + name: "BackupIDNotFound", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest("foo", ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + errCheck: assert.Error, + }, + { + name: "DetailsIDNotFound", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: { + BaseModel: model.BaseModel{ + ID: backup1.ID, + }, + DetailsID: "foo", + }, + }, + errCheck: assert.Error, + }, + { + name: "BaseMissingItems", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + itemPath2.ShortRef(): itemPath2, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + }, + errCheck: assert.Error, + }, + { + name: "TooManyItems", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + }, + errCheck: assert.Error, + }, + { + name: "BadBaseRepoRef", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + { + RepoRef: stdpath.Join( + append( + []string{ + itemPath1.Tenant(), + itemPath1.Service().String(), + itemPath1.ResourceOwner(), + path.UnknownCategory.String(), + }, + itemPath1.Folders()..., + )..., + ), + ItemInfo: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + ItemType: details.OneDriveItem, + ParentPath: itemParents1, + Size: 42, + }, + }, + }, + }, + }, + }, + }, + errCheck: assert.Error, + }, + { + name: "BadOneDrivePath", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): makePath( + suite.T(), + []string{ + itemPath1.Tenant(), + path.OneDriveService.String(), + itemPath1.ResourceOwner(), + path.FilesCategory.String(), + "personal", + "item1", + }, + true, + ), + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + }, + errCheck: assert.Error, + }, + { + name: "ItemMerged", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + }, + errCheck: assert.NoError, + expectedEntries: []*details.DetailsEntry{ + makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + { + name: "ItemMergedExtraItemsInBase", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + *makeDetailsEntry(suite.T(), itemPath2, 84, false), + }, + }, + }, + }, + errCheck: assert.NoError, + expectedEntries: []*details.DetailsEntry{ + makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + { + name: "ItemMoved", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath2, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + }, + errCheck: assert.NoError, + expectedEntries: []*details.DetailsEntry{ + makeDetailsEntry(suite.T(), itemPath2, 42, true), + }, + }, + { + name: "MultipleBases", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + itemPath3.ShortRef(): itemPath3, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + { + Manifest: makeManifest(backup2.ID, ""), + Reasons: []kopia.Reason{ + pathReason3, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + backup2.ID: backup2, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + backup2.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + // This entry should not be picked due to a mismatch on Reasons. + *makeDetailsEntry(suite.T(), itemPath1, 84, false), + // This item should be picked. + *makeDetailsEntry(suite.T(), itemPath3, 37, false), + }, + }, + }, + }, + errCheck: assert.NoError, + expectedEntries: []*details.DetailsEntry{ + makeDetailsEntry(suite.T(), itemPath1, 42, false), + makeDetailsEntry(suite.T(), itemPath3, 37, false), + }, + }, + { + name: "SomeBasesIncomplete", + inputShortRefsFromPrevBackup: map[string]path.Path{ + itemPath1.ShortRef(): itemPath1, + }, + inputMans: []*kopia.ManifestEntry{ + { + Manifest: makeManifest(backup1.ID, ""), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + { + Manifest: makeManifest(backup2.ID, "checkpoint"), + Reasons: []kopia.Reason{ + pathReason1, + }, + }, + }, + populatedModels: map[model.StableID]backup.Backup{ + backup1.ID: backup1, + backup2.ID: backup2, + }, + populatedDetails: map[string]*details.Details{ + backup1.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + *makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + }, + backup2.DetailsID: { + DetailsModel: details.DetailsModel{ + Entries: []details.DetailsEntry{ + // This entry should not be picked due to being incomplete. + *makeDetailsEntry(suite.T(), itemPath1, 84, false), + }, + }, + }, + }, + errCheck: assert.NoError, + expectedEntries: []*details.DetailsEntry{ + makeDetailsEntry(suite.T(), itemPath1, 42, false), + }, + }, + } + + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + ctx, flush := tester.NewContext() + defer flush() + + mdr := mockDetailsReader{entries: test.populatedDetails} + w := &store.Wrapper{Storer: mockBackupStorer{entries: test.populatedModels}} + + deets := details.Builder{} + + err := mergeDetails( + ctx, + w, + mdr, + test.inputMans, + test.inputShortRefsFromPrevBackup, + &deets, + ) + + test.errCheck(t, err) + if err != nil { + return + } + + assert.ElementsMatch(t, test.expectedEntries, deets.Details().Items()) + }) + } +} + // --------------------------------------------------------------------------- // integration // --------------------------------------------------------------------------- diff --git a/src/internal/operations/common.go b/src/internal/operations/common.go index 389db0fe7..addbeb5ac 100644 --- a/src/internal/operations/common.go +++ b/src/internal/operations/common.go @@ -5,31 +5,28 @@ import ( "github.com/pkg/errors" - "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" - "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/store" ) +type detailsReader interface { + ReadBackupDetails(ctx context.Context, detailsID string) (*details.Details, error) +} + func getBackupAndDetailsFromID( ctx context.Context, - tenant string, backupID model.StableID, - service path.ServiceType, ms *store.Wrapper, - kw *kopia.Wrapper, + detailsStore detailsReader, ) (*backup.Backup, *details.Details, error) { dID, bup, err := ms.GetDetailsIDFromBackupID(ctx, backupID) if err != nil { return nil, nil, errors.Wrap(err, "getting backup details ID") } - deets, err := streamstore. - New(kw, tenant, service). - ReadBackupDetails(ctx, dID) + deets, err := detailsStore.ReadBackupDetails(ctx, dID) if err != nil { return nil, nil, errors.Wrap(err, "getting backup details data") } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 29d02d8a1..3c85ae475 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" @@ -117,13 +118,13 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De } }() + detailsStore := streamstore.New(op.kopia, op.account.ID(), op.Selectors.PathService()) + bup, deets, err := getBackupAndDetailsFromID( ctx, - op.account.ID(), op.BackupID, - op.Selectors.PathService(), op.store, - op.kopia, + detailsStore, ) if err != nil { err = errors.Wrap(err, "restore")