Functions to merge a set of backup details (#1904)
## Description Go through the provided bases, load their backup details, and check if any of the items in them need to be merged into the details for the current backup. Has a small amount of logic to treat moved items as updated. Also include all the tests Viewing by commit may be useful ## Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No ## Type of change - [x] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🐹 Trivial/Minor ## Issue(s) * #1800 ## Test Plan - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
ashmrtn
GitHub
parent
b6a2227acc
commit
bc56d38970
@ -220,16 +220,6 @@ func produceManifestsAndMetadata(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(ashmrtn): Uncomment this again when we need to fetch and merge
|
|
||||||
// backup details from previous snapshots.
|
|
||||||
// k, _ := kopia.MakeTagKV(kopia.TagBackupID)
|
|
||||||
// bupID := man.Tags[k]
|
|
||||||
|
|
||||||
// bup, err := sw.GetBackup(ctx, model.StableID(bupID))
|
|
||||||
// if err != nil {
|
|
||||||
// return nil, nil, err
|
|
||||||
// }
|
|
||||||
|
|
||||||
colls, err := collectMetadata(ctx, kw, man, metadataFiles, tenantID)
|
colls, err := collectMetadata(ctx, kw, man, metadataFiles, tenantID)
|
||||||
if err != nil && !errors.Is(err, kopia.ErrNotFound) {
|
if err != nil && !errors.Is(err, kopia.ErrNotFound) {
|
||||||
// prior metadata isn't guaranteed to exist.
|
// prior metadata isn't guaranteed to exist.
|
||||||
@ -412,6 +402,119 @@ func consumeBackupDataCollections(
|
|||||||
return bu.BackupCollections(ctx, bases, cs, sel.PathService(), oc, tags)
|
return bu.BackupCollections(ctx, bases, cs, sel.PathService(), oc, tags)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func matchesReason(reasons []kopia.Reason, p path.Path) bool {
|
||||||
|
for _, reason := range reasons {
|
||||||
|
if p.ResourceOwner() == reason.ResourceOwner &&
|
||||||
|
p.Service() == reason.Service &&
|
||||||
|
p.Category() == reason.Category {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeDetails(
|
||||||
|
ctx context.Context,
|
||||||
|
ms *store.Wrapper,
|
||||||
|
detailsStore detailsReader,
|
||||||
|
mans []*kopia.ManifestEntry,
|
||||||
|
shortRefsFromPrevBackup map[string]path.Path,
|
||||||
|
deets *details.Builder,
|
||||||
|
) error {
|
||||||
|
// Don't bother loading any of the base details if there's nothing we need to
|
||||||
|
// merge.
|
||||||
|
if len(shortRefsFromPrevBackup) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var addedEntries int
|
||||||
|
|
||||||
|
for _, man := range mans {
|
||||||
|
// For now skip snapshots that aren't complete. We will need to revisit this
|
||||||
|
// when we tackle restartability.
|
||||||
|
if len(man.IncompleteReason) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
k, _ := kopia.MakeTagKV(kopia.TagBackupID)
|
||||||
|
bID := man.Tags[k]
|
||||||
|
|
||||||
|
_, baseDeets, err := getBackupAndDetailsFromID(
|
||||||
|
ctx,
|
||||||
|
model.StableID(bID),
|
||||||
|
ms,
|
||||||
|
detailsStore,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(err, "backup fetching base details for backup %s", bID)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range baseDeets.Items() {
|
||||||
|
rr, err := path.FromDataLayerPath(entry.RepoRef, true)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrapf(
|
||||||
|
err,
|
||||||
|
"parsing base item info path %s in backup %s",
|
||||||
|
entry.RepoRef,
|
||||||
|
bID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Although this base has an entry it may not be the most recent. Check
|
||||||
|
// the reasons a snapshot was returned to ensure we only choose the recent
|
||||||
|
// entries.
|
||||||
|
//
|
||||||
|
// TODO(ashmrtn): This logic will need expanded to cover entries from
|
||||||
|
// checkpoints if we start doing kopia-assisted incrementals for those.
|
||||||
|
if !matchesReason(man.Reasons, rr) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
newPath := shortRefsFromPrevBackup[rr.ShortRef()]
|
||||||
|
if newPath == nil {
|
||||||
|
// This entry was not sourced from a base snapshot or cached from a
|
||||||
|
// previous backup, skip it.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fixup paths in the item.
|
||||||
|
item := entry.ItemInfo
|
||||||
|
if err := details.UpdateItem(&item, newPath); err != nil {
|
||||||
|
return errors.Wrapf(
|
||||||
|
err,
|
||||||
|
"updating item info for entry from backup %s",
|
||||||
|
bID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
deets.Add(
|
||||||
|
newPath.String(),
|
||||||
|
newPath.ShortRef(),
|
||||||
|
newPath.ToBuilder().Dir().ShortRef(),
|
||||||
|
// TODO(ashmrtn): This may need updated if we start using this merge
|
||||||
|
// strategry for items that were cached in kopia.
|
||||||
|
newPath.String() != rr.String(),
|
||||||
|
item,
|
||||||
|
)
|
||||||
|
|
||||||
|
// Track how many entries we added so that we know if we got them all when
|
||||||
|
// we're done.
|
||||||
|
addedEntries++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if addedEntries != len(shortRefsFromPrevBackup) {
|
||||||
|
return errors.Errorf(
|
||||||
|
"incomplete migration of backup details: found %v of %v expected items",
|
||||||
|
addedEntries,
|
||||||
|
len(shortRefsFromPrevBackup),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// writes the results metrics to the operation results.
|
// writes the results metrics to the operation results.
|
||||||
// later stored in the manifest using createBackupModels.
|
// later stored in the manifest using createBackupModels.
|
||||||
func (op *BackupOperation) persistResults(
|
func (op *BackupOperation) persistResults(
|
||||||
|
|||||||
@ -2,10 +2,13 @@ package operations
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
stdpath "path"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/kopia/kopia/repo/manifest"
|
||||||
"github.com/kopia/kopia/snapshot"
|
"github.com/kopia/kopia/snapshot"
|
||||||
|
"github.com/pkg/errors"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
@ -457,6 +460,664 @@ func (suite *BackupOpSuite) TestBackupOperation_ConsumeBackupDataCollections_Pat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type mockDetailsReader struct {
|
||||||
|
entries map[string]*details.Details
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mdr mockDetailsReader) ReadBackupDetails(
|
||||||
|
ctx context.Context,
|
||||||
|
detailsID string,
|
||||||
|
) (*details.Details, error) {
|
||||||
|
r := mdr.entries[detailsID]
|
||||||
|
|
||||||
|
if r == nil {
|
||||||
|
return nil, errors.Errorf("no details for ID %s", detailsID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type mockBackupStorer struct {
|
||||||
|
// Only using this to store backup models right now.
|
||||||
|
entries map[model.StableID]backup.Backup
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) Get(
|
||||||
|
ctx context.Context,
|
||||||
|
s model.Schema,
|
||||||
|
id model.StableID,
|
||||||
|
toPopulate model.Model,
|
||||||
|
) error {
|
||||||
|
if s != model.BackupSchema {
|
||||||
|
return errors.Errorf("unexpected schema %s", s)
|
||||||
|
}
|
||||||
|
|
||||||
|
r, ok := mbs.entries[id]
|
||||||
|
if !ok {
|
||||||
|
return errors.Errorf("model with id %s not found", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
bu, ok := toPopulate.(*backup.Backup)
|
||||||
|
if !ok {
|
||||||
|
return errors.Errorf("bad input type %T", toPopulate)
|
||||||
|
}
|
||||||
|
|
||||||
|
*bu = r
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Functions we need to implement but don't care about.
|
||||||
|
func (mbs mockBackupStorer) Delete(context.Context, model.Schema, model.StableID) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) DeleteWithModelStoreID(context.Context, manifest.ID) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) GetIDsForType(
|
||||||
|
context.Context,
|
||||||
|
model.Schema,
|
||||||
|
map[string]string,
|
||||||
|
) ([]*model.BaseModel, error) {
|
||||||
|
return nil, errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) GetWithModelStoreID(
|
||||||
|
context.Context,
|
||||||
|
model.Schema,
|
||||||
|
manifest.ID,
|
||||||
|
model.Model,
|
||||||
|
) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) Put(context.Context, model.Schema, model.Model) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mbs mockBackupStorer) Update(context.Context, model.Schema, model.Model) error {
|
||||||
|
return errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO(ashmrtn): Really need to factor a function like this out into some
|
||||||
|
// common file that is only compiled for tests.
|
||||||
|
func makePath(
|
||||||
|
t *testing.T,
|
||||||
|
elements []string,
|
||||||
|
isItem bool,
|
||||||
|
) path.Path {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
p, err := path.FromDataLayerPath(stdpath.Join(elements...), isItem)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeDetailsEntry(
|
||||||
|
t *testing.T,
|
||||||
|
p path.Path,
|
||||||
|
size int,
|
||||||
|
updated bool,
|
||||||
|
) *details.DetailsEntry {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
res := &details.DetailsEntry{
|
||||||
|
RepoRef: p.String(),
|
||||||
|
ShortRef: p.ShortRef(),
|
||||||
|
ParentRef: p.ToBuilder().Dir().ShortRef(),
|
||||||
|
ItemInfo: details.ItemInfo{},
|
||||||
|
Updated: updated,
|
||||||
|
}
|
||||||
|
|
||||||
|
switch p.Service() {
|
||||||
|
case path.ExchangeService:
|
||||||
|
if p.Category() != path.EmailCategory {
|
||||||
|
assert.FailNowf(
|
||||||
|
t,
|
||||||
|
"category %s not supported in helper function",
|
||||||
|
p.Category().String(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Exchange = &details.ExchangeInfo{
|
||||||
|
ItemType: details.ExchangeMail,
|
||||||
|
Size: int64(size),
|
||||||
|
}
|
||||||
|
|
||||||
|
case path.OneDriveService:
|
||||||
|
parent, err := path.GetDriveFolderPath(p)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
res.OneDrive = &details.OneDriveInfo{
|
||||||
|
ItemType: details.OneDriveItem,
|
||||||
|
ParentPath: parent,
|
||||||
|
Size: int64(size),
|
||||||
|
}
|
||||||
|
|
||||||
|
default:
|
||||||
|
assert.FailNowf(
|
||||||
|
t,
|
||||||
|
"service %s not supported in helper function",
|
||||||
|
p.Service().String(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeManifest(backupID model.StableID, incompleteReason string) *snapshot.Manifest {
|
||||||
|
backupIDTagKey, _ := kopia.MakeTagKV(kopia.TagBackupID)
|
||||||
|
|
||||||
|
return &snapshot.Manifest{
|
||||||
|
Tags: map[string]string{
|
||||||
|
backupIDTagKey: string(backupID),
|
||||||
|
},
|
||||||
|
IncompleteReason: incompleteReason,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *BackupOpSuite) TestBackupOperation_MergeBackupDetails() {
|
||||||
|
var (
|
||||||
|
tenant = "a-tenant"
|
||||||
|
ro = "a-user"
|
||||||
|
|
||||||
|
itemPath1 = makePath(
|
||||||
|
suite.T(),
|
||||||
|
[]string{
|
||||||
|
tenant,
|
||||||
|
path.OneDriveService.String(),
|
||||||
|
ro,
|
||||||
|
path.FilesCategory.String(),
|
||||||
|
"drives",
|
||||||
|
"drive-id",
|
||||||
|
"root:",
|
||||||
|
"work",
|
||||||
|
"item1",
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
itemPath2 = makePath(
|
||||||
|
suite.T(),
|
||||||
|
[]string{
|
||||||
|
tenant,
|
||||||
|
path.OneDriveService.String(),
|
||||||
|
ro,
|
||||||
|
path.FilesCategory.String(),
|
||||||
|
"drives",
|
||||||
|
"drive-id",
|
||||||
|
"root:",
|
||||||
|
"personal",
|
||||||
|
"item2",
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
itemPath3 = makePath(
|
||||||
|
suite.T(),
|
||||||
|
[]string{
|
||||||
|
tenant,
|
||||||
|
path.ExchangeService.String(),
|
||||||
|
ro,
|
||||||
|
path.EmailCategory.String(),
|
||||||
|
"personal",
|
||||||
|
"item3",
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
|
||||||
|
backup1 = backup.Backup{
|
||||||
|
BaseModel: model.BaseModel{
|
||||||
|
ID: "bid1",
|
||||||
|
},
|
||||||
|
DetailsID: "did1",
|
||||||
|
}
|
||||||
|
|
||||||
|
backup2 = backup.Backup{
|
||||||
|
BaseModel: model.BaseModel{
|
||||||
|
ID: "bid2",
|
||||||
|
},
|
||||||
|
DetailsID: "did2",
|
||||||
|
}
|
||||||
|
|
||||||
|
pathReason1 = kopia.Reason{
|
||||||
|
ResourceOwner: itemPath1.ResourceOwner(),
|
||||||
|
Service: itemPath1.Service(),
|
||||||
|
Category: itemPath1.Category(),
|
||||||
|
}
|
||||||
|
pathReason3 = kopia.Reason{
|
||||||
|
ResourceOwner: itemPath3.ResourceOwner(),
|
||||||
|
Service: itemPath3.Service(),
|
||||||
|
Category: itemPath3.Category(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
itemParents1, err := path.GetDriveFolderPath(itemPath1)
|
||||||
|
require.NoError(suite.T(), err)
|
||||||
|
|
||||||
|
table := []struct {
|
||||||
|
name string
|
||||||
|
populatedModels map[model.StableID]backup.Backup
|
||||||
|
populatedDetails map[string]*details.Details
|
||||||
|
inputMans []*kopia.ManifestEntry
|
||||||
|
inputShortRefsFromPrevBackup map[string]path.Path
|
||||||
|
|
||||||
|
errCheck assert.ErrorAssertionFunc
|
||||||
|
expectedEntries []*details.DetailsEntry
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "NilShortRefsFromPrevBackup",
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
// Use empty slice so we don't error out on nil != empty.
|
||||||
|
expectedEntries: []*details.DetailsEntry{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "EmptyShortRefsFromPrevBackup",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
// Use empty slice so we don't error out on nil != empty.
|
||||||
|
expectedEntries: []*details.DetailsEntry{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "BackupIDNotFound",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest("foo", ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "DetailsIDNotFound",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: {
|
||||||
|
BaseModel: model.BaseModel{
|
||||||
|
ID: backup1.ID,
|
||||||
|
},
|
||||||
|
DetailsID: "foo",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "BaseMissingItems",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
itemPath2.ShortRef(): itemPath2,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "TooManyItems",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "BadBaseRepoRef",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
{
|
||||||
|
RepoRef: stdpath.Join(
|
||||||
|
append(
|
||||||
|
[]string{
|
||||||
|
itemPath1.Tenant(),
|
||||||
|
itemPath1.Service().String(),
|
||||||
|
itemPath1.ResourceOwner(),
|
||||||
|
path.UnknownCategory.String(),
|
||||||
|
},
|
||||||
|
itemPath1.Folders()...,
|
||||||
|
)...,
|
||||||
|
),
|
||||||
|
ItemInfo: details.ItemInfo{
|
||||||
|
OneDrive: &details.OneDriveInfo{
|
||||||
|
ItemType: details.OneDriveItem,
|
||||||
|
ParentPath: itemParents1,
|
||||||
|
Size: 42,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "BadOneDrivePath",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): makePath(
|
||||||
|
suite.T(),
|
||||||
|
[]string{
|
||||||
|
itemPath1.Tenant(),
|
||||||
|
path.OneDriveService.String(),
|
||||||
|
itemPath1.ResourceOwner(),
|
||||||
|
path.FilesCategory.String(),
|
||||||
|
"personal",
|
||||||
|
"item1",
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.Error,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ItemMerged",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
expectedEntries: []*details.DetailsEntry{
|
||||||
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ItemMergedExtraItemsInBase",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath2, 84, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
expectedEntries: []*details.DetailsEntry{
|
||||||
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ItemMoved",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath2,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
expectedEntries: []*details.DetailsEntry{
|
||||||
|
makeDetailsEntry(suite.T(), itemPath2, 42, true),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "MultipleBases",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
itemPath3.ShortRef(): itemPath3,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup2.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
backup2.ID: backup2,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
backup2.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
// This entry should not be picked due to a mismatch on Reasons.
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 84, false),
|
||||||
|
// This item should be picked.
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath3, 37, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
expectedEntries: []*details.DetailsEntry{
|
||||||
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
makeDetailsEntry(suite.T(), itemPath3, 37, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "SomeBasesIncomplete",
|
||||||
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
||||||
|
itemPath1.ShortRef(): itemPath1,
|
||||||
|
},
|
||||||
|
inputMans: []*kopia.ManifestEntry{
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup1.ID, ""),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Manifest: makeManifest(backup2.ID, "checkpoint"),
|
||||||
|
Reasons: []kopia.Reason{
|
||||||
|
pathReason1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
populatedModels: map[model.StableID]backup.Backup{
|
||||||
|
backup1.ID: backup1,
|
||||||
|
backup2.ID: backup2,
|
||||||
|
},
|
||||||
|
populatedDetails: map[string]*details.Details{
|
||||||
|
backup1.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
backup2.DetailsID: {
|
||||||
|
DetailsModel: details.DetailsModel{
|
||||||
|
Entries: []details.DetailsEntry{
|
||||||
|
// This entry should not be picked due to being incomplete.
|
||||||
|
*makeDetailsEntry(suite.T(), itemPath1, 84, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
errCheck: assert.NoError,
|
||||||
|
expectedEntries: []*details.DetailsEntry{
|
||||||
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range table {
|
||||||
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
|
ctx, flush := tester.NewContext()
|
||||||
|
defer flush()
|
||||||
|
|
||||||
|
mdr := mockDetailsReader{entries: test.populatedDetails}
|
||||||
|
w := &store.Wrapper{Storer: mockBackupStorer{entries: test.populatedModels}}
|
||||||
|
|
||||||
|
deets := details.Builder{}
|
||||||
|
|
||||||
|
err := mergeDetails(
|
||||||
|
ctx,
|
||||||
|
w,
|
||||||
|
mdr,
|
||||||
|
test.inputMans,
|
||||||
|
test.inputShortRefsFromPrevBackup,
|
||||||
|
&deets,
|
||||||
|
)
|
||||||
|
|
||||||
|
test.errCheck(t, err)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, test.expectedEntries, deets.Details().Items())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// integration
|
// integration
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@ -5,31 +5,28 @@ import (
|
|||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/kopia"
|
|
||||||
"github.com/alcionai/corso/src/internal/model"
|
"github.com/alcionai/corso/src/internal/model"
|
||||||
"github.com/alcionai/corso/src/internal/streamstore"
|
|
||||||
"github.com/alcionai/corso/src/pkg/backup"
|
"github.com/alcionai/corso/src/pkg/backup"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
|
||||||
"github.com/alcionai/corso/src/pkg/store"
|
"github.com/alcionai/corso/src/pkg/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type detailsReader interface {
|
||||||
|
ReadBackupDetails(ctx context.Context, detailsID string) (*details.Details, error)
|
||||||
|
}
|
||||||
|
|
||||||
func getBackupAndDetailsFromID(
|
func getBackupAndDetailsFromID(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tenant string,
|
|
||||||
backupID model.StableID,
|
backupID model.StableID,
|
||||||
service path.ServiceType,
|
|
||||||
ms *store.Wrapper,
|
ms *store.Wrapper,
|
||||||
kw *kopia.Wrapper,
|
detailsStore detailsReader,
|
||||||
) (*backup.Backup, *details.Details, error) {
|
) (*backup.Backup, *details.Details, error) {
|
||||||
dID, bup, err := ms.GetDetailsIDFromBackupID(ctx, backupID)
|
dID, bup, err := ms.GetDetailsIDFromBackupID(ctx, backupID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, errors.Wrap(err, "getting backup details ID")
|
return nil, nil, errors.Wrap(err, "getting backup details ID")
|
||||||
}
|
}
|
||||||
|
|
||||||
deets, err := streamstore.
|
deets, err := detailsStore.ReadBackupDetails(ctx, dID)
|
||||||
New(kw, tenant, service).
|
|
||||||
ReadBackupDetails(ctx, dID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, errors.Wrap(err, "getting backup details data")
|
return nil, nil, errors.Wrap(err, "getting backup details data")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/model"
|
"github.com/alcionai/corso/src/internal/model"
|
||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
"github.com/alcionai/corso/src/internal/stats"
|
"github.com/alcionai/corso/src/internal/stats"
|
||||||
|
"github.com/alcionai/corso/src/internal/streamstore"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
@ -117,13 +118,13 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
detailsStore := streamstore.New(op.kopia, op.account.ID(), op.Selectors.PathService())
|
||||||
|
|
||||||
bup, deets, err := getBackupAndDetailsFromID(
|
bup, deets, err := getBackupAndDetailsFromID(
|
||||||
ctx,
|
ctx,
|
||||||
op.account.ID(),
|
|
||||||
op.BackupID,
|
op.BackupID,
|
||||||
op.Selectors.PathService(),
|
|
||||||
op.store,
|
op.store,
|
||||||
op.kopia,
|
detailsStore,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = errors.Wrap(err, "restore")
|
err = errors.Wrap(err, "restore")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user