48e4b65165
## Description Adds mocked unit tests for produceManifestsAnd- Metadata. For cleanliness, moves that func, and any funcs called within it, to their own file within operations ## Does this PR need a docs update or release note? - [x] ⛔ No ## Type of change - [x] 🤖 Test ## Issue(s) * #2062 ## Test Plan - [x] ⚡ Unit test
1438 lines
32 KiB
Go
1438 lines
32 KiB
Go
package operations
|
|
|
|
import (
|
|
"context"
|
|
stdpath "path"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kopia/kopia/repo/manifest"
|
|
"github.com/kopia/kopia/snapshot"
|
|
"github.com/pkg/errors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/stretchr/testify/suite"
|
|
|
|
"github.com/alcionai/corso/src/internal/connector/support"
|
|
"github.com/alcionai/corso/src/internal/data"
|
|
evmock "github.com/alcionai/corso/src/internal/events/mock"
|
|
"github.com/alcionai/corso/src/internal/kopia"
|
|
"github.com/alcionai/corso/src/internal/model"
|
|
"github.com/alcionai/corso/src/internal/tester"
|
|
"github.com/alcionai/corso/src/pkg/account"
|
|
"github.com/alcionai/corso/src/pkg/backup"
|
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
|
"github.com/alcionai/corso/src/pkg/control"
|
|
"github.com/alcionai/corso/src/pkg/path"
|
|
"github.com/alcionai/corso/src/pkg/selectors"
|
|
"github.com/alcionai/corso/src/pkg/store"
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// mocks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// ----- restore producer
|
|
|
|
type mockRestorer struct {
|
|
gotPaths []path.Path
|
|
colls []data.Collection
|
|
collsByID map[string][]data.Collection // snapshotID: []Collection
|
|
err error
|
|
onRestore restoreFunc
|
|
}
|
|
|
|
type restoreFunc func(id string, ps []path.Path) ([]data.Collection, error)
|
|
|
|
func (mr *mockRestorer) buildRestoreFunc(
|
|
t *testing.T,
|
|
oid string,
|
|
ops []path.Path,
|
|
) {
|
|
mr.onRestore = func(id string, ps []path.Path) ([]data.Collection, error) {
|
|
assert.Equal(t, oid, id, "manifest id")
|
|
checkPaths(t, ops, ps)
|
|
|
|
return mr.colls, mr.err
|
|
}
|
|
}
|
|
|
|
func (mr *mockRestorer) RestoreMultipleItems(
|
|
ctx context.Context,
|
|
snapshotID string,
|
|
paths []path.Path,
|
|
bc kopia.ByteCounter,
|
|
) ([]data.Collection, error) {
|
|
mr.gotPaths = append(mr.gotPaths, paths...)
|
|
|
|
if mr.onRestore != nil {
|
|
return mr.onRestore(snapshotID, paths)
|
|
}
|
|
|
|
if len(mr.collsByID) > 0 {
|
|
return mr.collsByID[snapshotID], mr.err
|
|
}
|
|
|
|
return mr.colls, mr.err
|
|
}
|
|
|
|
func checkPaths(t *testing.T, expected, got []path.Path) {
|
|
assert.ElementsMatch(t, expected, got)
|
|
}
|
|
|
|
// ----- backup producer
|
|
|
|
type mockBackuper struct {
|
|
checkFunc func(
|
|
bases []kopia.IncrementalBase,
|
|
cs []data.Collection,
|
|
tags map[string]string,
|
|
buildTreeWithBase bool,
|
|
)
|
|
}
|
|
|
|
func (mbu mockBackuper) BackupCollections(
|
|
ctx context.Context,
|
|
bases []kopia.IncrementalBase,
|
|
cs []data.Collection,
|
|
tags map[string]string,
|
|
buildTreeWithBase bool,
|
|
) (*kopia.BackupStats, *details.Builder, map[string]path.Path, error) {
|
|
if mbu.checkFunc != nil {
|
|
mbu.checkFunc(bases, cs, tags, buildTreeWithBase)
|
|
}
|
|
|
|
return &kopia.BackupStats{}, &details.Builder{}, nil, nil
|
|
}
|
|
|
|
// ----- details
|
|
|
|
type mockDetailsReader struct {
|
|
entries map[string]*details.Details
|
|
}
|
|
|
|
func (mdr mockDetailsReader) ReadBackupDetails(
|
|
ctx context.Context,
|
|
detailsID string,
|
|
) (*details.Details, error) {
|
|
r := mdr.entries[detailsID]
|
|
|
|
if r == nil {
|
|
return nil, errors.Errorf("no details for ID %s", detailsID)
|
|
}
|
|
|
|
return r, nil
|
|
}
|
|
|
|
// ----- model store for backups
|
|
|
|
type mockBackupStorer struct {
|
|
// Only using this to store backup models right now.
|
|
entries map[model.StableID]backup.Backup
|
|
}
|
|
|
|
func (mbs mockBackupStorer) Get(
|
|
ctx context.Context,
|
|
s model.Schema,
|
|
id model.StableID,
|
|
toPopulate model.Model,
|
|
) error {
|
|
if s != model.BackupSchema {
|
|
return errors.Errorf("unexpected schema %s", s)
|
|
}
|
|
|
|
r, ok := mbs.entries[id]
|
|
if !ok {
|
|
return errors.Errorf("model with id %s not found", id)
|
|
}
|
|
|
|
bu, ok := toPopulate.(*backup.Backup)
|
|
if !ok {
|
|
return errors.Errorf("bad input type %T", toPopulate)
|
|
}
|
|
|
|
*bu = r
|
|
|
|
return nil
|
|
}
|
|
|
|
func (mbs mockBackupStorer) Delete(context.Context, model.Schema, model.StableID) error {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
func (mbs mockBackupStorer) DeleteWithModelStoreID(context.Context, manifest.ID) error {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
func (mbs mockBackupStorer) GetIDsForType(
|
|
context.Context,
|
|
model.Schema,
|
|
map[string]string,
|
|
) ([]*model.BaseModel, error) {
|
|
return nil, errors.New("not implemented")
|
|
}
|
|
|
|
func (mbs mockBackupStorer) GetWithModelStoreID(
|
|
context.Context,
|
|
model.Schema,
|
|
manifest.ID,
|
|
model.Model,
|
|
) error {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
func (mbs mockBackupStorer) Put(context.Context, model.Schema, model.Model) error {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
func (mbs mockBackupStorer) Update(context.Context, model.Schema, model.Model) error {
|
|
return errors.New("not implemented")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// helper funcs
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// expects you to Append your own file
|
|
func makeMetadataBasePath(
|
|
t *testing.T,
|
|
tenant string,
|
|
service path.ServiceType,
|
|
resourceOwner string,
|
|
category path.CategoryType,
|
|
) path.Path {
|
|
t.Helper()
|
|
|
|
p, err := path.Builder{}.ToServiceCategoryMetadataPath(
|
|
tenant,
|
|
resourceOwner,
|
|
service,
|
|
category,
|
|
false)
|
|
require.NoError(t, err)
|
|
|
|
return p
|
|
}
|
|
|
|
func makeMetadataPath(
|
|
t *testing.T,
|
|
tenant string,
|
|
service path.ServiceType,
|
|
resourceOwner string,
|
|
category path.CategoryType,
|
|
fileName string,
|
|
) path.Path {
|
|
t.Helper()
|
|
|
|
p, err := path.Builder{}.Append(fileName).ToServiceCategoryMetadataPath(
|
|
tenant,
|
|
resourceOwner,
|
|
service,
|
|
category,
|
|
true)
|
|
require.NoError(t, err)
|
|
|
|
return p
|
|
}
|
|
|
|
func makeFolderEntry(
|
|
t *testing.T,
|
|
pb *path.Builder,
|
|
size int64,
|
|
modTime time.Time,
|
|
) *details.DetailsEntry {
|
|
t.Helper()
|
|
|
|
return &details.DetailsEntry{
|
|
RepoRef: pb.String(),
|
|
ShortRef: pb.ShortRef(),
|
|
ParentRef: pb.Dir().ShortRef(),
|
|
ItemInfo: details.ItemInfo{
|
|
Folder: &details.FolderInfo{
|
|
ItemType: details.FolderItem,
|
|
DisplayName: pb.Elements()[len(pb.Elements())-1],
|
|
Modified: modTime,
|
|
Size: size,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// TODO(ashmrtn): Really need to factor a function like this out into some
|
|
// common file that is only compiled for tests.
|
|
func makePath(t *testing.T, elements []string, isItem bool) path.Path {
|
|
t.Helper()
|
|
|
|
p, err := path.FromDataLayerPath(stdpath.Join(elements...), isItem)
|
|
require.NoError(t, err)
|
|
|
|
return p
|
|
}
|
|
|
|
func makeDetailsEntry(
|
|
t *testing.T,
|
|
p path.Path,
|
|
size int,
|
|
updated bool,
|
|
) *details.DetailsEntry {
|
|
t.Helper()
|
|
|
|
res := &details.DetailsEntry{
|
|
RepoRef: p.String(),
|
|
ShortRef: p.ShortRef(),
|
|
ParentRef: p.ToBuilder().Dir().ShortRef(),
|
|
ItemInfo: details.ItemInfo{},
|
|
Updated: updated,
|
|
}
|
|
|
|
switch p.Service() {
|
|
case path.ExchangeService:
|
|
if p.Category() != path.EmailCategory {
|
|
assert.FailNowf(
|
|
t,
|
|
"category %s not supported in helper function",
|
|
p.Category().String(),
|
|
)
|
|
}
|
|
|
|
res.Exchange = &details.ExchangeInfo{
|
|
ItemType: details.ExchangeMail,
|
|
Size: int64(size),
|
|
}
|
|
|
|
case path.OneDriveService:
|
|
parent, err := path.GetDriveFolderPath(p)
|
|
require.NoError(t, err)
|
|
|
|
res.OneDrive = &details.OneDriveInfo{
|
|
ItemType: details.OneDriveItem,
|
|
ParentPath: parent,
|
|
Size: int64(size),
|
|
}
|
|
|
|
default:
|
|
assert.FailNowf(
|
|
t,
|
|
"service %s not supported in helper function",
|
|
p.Service().String(),
|
|
)
|
|
}
|
|
|
|
return res
|
|
}
|
|
|
|
// TODO(ashmrtn): This should belong to some code that lives in the kopia
|
|
// package that is only compiled when running tests.
|
|
func makeKopiaTagKey(k string) string {
|
|
return "tag:" + k
|
|
}
|
|
|
|
func makeManifest(t *testing.T, backupID model.StableID, incompleteReason string) *snapshot.Manifest {
|
|
t.Helper()
|
|
|
|
tagKey := makeKopiaTagKey(kopia.TagBackupID)
|
|
|
|
return &snapshot.Manifest{
|
|
Tags: map[string]string{
|
|
tagKey: string(backupID),
|
|
},
|
|
IncompleteReason: incompleteReason,
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// unit tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type BackupOpSuite struct {
|
|
suite.Suite
|
|
}
|
|
|
|
func TestBackupOpSuite(t *testing.T) {
|
|
suite.Run(t, new(BackupOpSuite))
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
|
|
ctx, flush := tester.NewContext()
|
|
defer flush()
|
|
|
|
var (
|
|
kw = &kopia.Wrapper{}
|
|
sw = &store.Wrapper{}
|
|
acct = account.Account{}
|
|
now = time.Now()
|
|
)
|
|
|
|
table := []struct {
|
|
expectStatus opStatus
|
|
expectErr assert.ErrorAssertionFunc
|
|
stats backupStats
|
|
}{
|
|
{
|
|
expectStatus: Completed,
|
|
expectErr: assert.NoError,
|
|
stats: backupStats{
|
|
started: true,
|
|
resourceCount: 1,
|
|
k: &kopia.BackupStats{
|
|
TotalFileCount: 1,
|
|
TotalHashedBytes: 1,
|
|
TotalUploadedBytes: 1,
|
|
},
|
|
gc: &support.ConnectorOperationStatus{
|
|
Successful: 1,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
expectStatus: Failed,
|
|
expectErr: assert.Error,
|
|
stats: backupStats{
|
|
started: false,
|
|
k: &kopia.BackupStats{},
|
|
gc: &support.ConnectorOperationStatus{},
|
|
},
|
|
},
|
|
{
|
|
expectStatus: NoData,
|
|
expectErr: assert.NoError,
|
|
stats: backupStats{
|
|
started: true,
|
|
k: &kopia.BackupStats{},
|
|
gc: &support.ConnectorOperationStatus{},
|
|
},
|
|
},
|
|
}
|
|
for _, test := range table {
|
|
suite.T().Run(test.expectStatus.String(), func(t *testing.T) {
|
|
sel := selectors.Selector{}
|
|
sel.DiscreteOwner = "bombadil"
|
|
|
|
op, err := NewBackupOperation(
|
|
ctx,
|
|
control.Options{},
|
|
kw,
|
|
sw,
|
|
acct,
|
|
sel,
|
|
evmock.NewBus())
|
|
require.NoError(t, err)
|
|
test.expectErr(t, op.persistResults(now, &test.stats))
|
|
|
|
assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status")
|
|
assert.Equal(t, test.stats.gc.Successful, op.Results.ItemsRead, "items read")
|
|
assert.Equal(t, test.stats.readErr, op.Results.ReadErrors, "read errors")
|
|
assert.Equal(t, test.stats.k.TotalFileCount, op.Results.ItemsWritten, "items written")
|
|
assert.Equal(t, test.stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read")
|
|
assert.Equal(t, test.stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written")
|
|
assert.Equal(t, test.stats.resourceCount, op.Results.ResourceOwners, "resource owners")
|
|
assert.Equal(t, test.stats.writeErr, op.Results.WriteErrors, "write errors")
|
|
assert.Equal(t, now, op.Results.StartedAt, "started at")
|
|
assert.Less(t, now, op.Results.CompletedAt, "completed at")
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_VerifyDistinctBases() {
|
|
const user = "a-user"
|
|
|
|
table := []struct {
|
|
name string
|
|
input []*kopia.ManifestEntry
|
|
errCheck assert.ErrorAssertionFunc
|
|
}{
|
|
{
|
|
name: "SingleManifestMultipleReasons",
|
|
input: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id1",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EventsCategory,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
},
|
|
{
|
|
name: "MultipleManifestsDistinctReason",
|
|
input: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id1",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id2",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EventsCategory,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
},
|
|
{
|
|
name: "MultipleManifestsSameReason",
|
|
input: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id1",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id2",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "MultipleManifestsSameReasonOneIncomplete",
|
|
input: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id1",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Manifest: &snapshot.Manifest{
|
|
ID: "id2",
|
|
IncompleteReason: "checkpoint",
|
|
},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: user,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
},
|
|
}
|
|
|
|
for _, test := range table {
|
|
suite.T().Run(test.name, func(t *testing.T) {
|
|
test.errCheck(t, verifyDistinctBases(test.input))
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_CollectMetadata() {
|
|
var (
|
|
tenant = "a-tenant"
|
|
resourceOwner = "a-user"
|
|
fileNames = []string{
|
|
"delta",
|
|
"paths",
|
|
}
|
|
|
|
emailDeltaPath = makeMetadataPath(
|
|
suite.T(),
|
|
tenant,
|
|
path.ExchangeService,
|
|
resourceOwner,
|
|
path.EmailCategory,
|
|
fileNames[0],
|
|
)
|
|
emailPathsPath = makeMetadataPath(
|
|
suite.T(),
|
|
tenant,
|
|
path.ExchangeService,
|
|
resourceOwner,
|
|
path.EmailCategory,
|
|
fileNames[1],
|
|
)
|
|
contactsDeltaPath = makeMetadataPath(
|
|
suite.T(),
|
|
tenant,
|
|
path.ExchangeService,
|
|
resourceOwner,
|
|
path.ContactsCategory,
|
|
fileNames[0],
|
|
)
|
|
contactsPathsPath = makeMetadataPath(
|
|
suite.T(),
|
|
tenant,
|
|
path.ExchangeService,
|
|
resourceOwner,
|
|
path.ContactsCategory,
|
|
fileNames[1],
|
|
)
|
|
)
|
|
|
|
table := []struct {
|
|
name string
|
|
inputMan *kopia.ManifestEntry
|
|
inputFiles []string
|
|
expected []path.Path
|
|
}{
|
|
{
|
|
name: "SingleReasonSingleFile",
|
|
inputMan: &kopia.ManifestEntry{
|
|
Manifest: &snapshot.Manifest{},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
inputFiles: []string{fileNames[0]},
|
|
expected: []path.Path{emailDeltaPath},
|
|
},
|
|
{
|
|
name: "SingleReasonMultipleFiles",
|
|
inputMan: &kopia.ManifestEntry{
|
|
Manifest: &snapshot.Manifest{},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
},
|
|
},
|
|
inputFiles: fileNames,
|
|
expected: []path.Path{emailDeltaPath, emailPathsPath},
|
|
},
|
|
{
|
|
name: "MultipleReasonsMultipleFiles",
|
|
inputMan: &kopia.ManifestEntry{
|
|
Manifest: &snapshot.Manifest{},
|
|
Reasons: []kopia.Reason{
|
|
{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
},
|
|
{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.ContactsCategory,
|
|
},
|
|
},
|
|
},
|
|
inputFiles: fileNames,
|
|
expected: []path.Path{
|
|
emailDeltaPath,
|
|
emailPathsPath,
|
|
contactsDeltaPath,
|
|
contactsPathsPath,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range table {
|
|
suite.T().Run(test.name, func(t *testing.T) {
|
|
ctx, flush := tester.NewContext()
|
|
defer flush()
|
|
|
|
mr := &mockRestorer{}
|
|
|
|
_, err := collectMetadata(ctx, mr, test.inputMan, test.inputFiles, tenant)
|
|
assert.NoError(t, err)
|
|
|
|
checkPaths(t, test.expected, mr.gotPaths)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_ConsumeBackupDataCollections_Paths() {
|
|
var (
|
|
tenant = "a-tenant"
|
|
resourceOwner = "a-user"
|
|
|
|
emailBuilder = path.Builder{}.Append(
|
|
tenant,
|
|
path.ExchangeService.String(),
|
|
resourceOwner,
|
|
path.EmailCategory.String(),
|
|
)
|
|
contactsBuilder = path.Builder{}.Append(
|
|
tenant,
|
|
path.ExchangeService.String(),
|
|
resourceOwner,
|
|
path.ContactsCategory.String(),
|
|
)
|
|
|
|
emailReason = kopia.Reason{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.EmailCategory,
|
|
}
|
|
contactsReason = kopia.Reason{
|
|
ResourceOwner: resourceOwner,
|
|
Service: path.ExchangeService,
|
|
Category: path.ContactsCategory,
|
|
}
|
|
|
|
manifest1 = &snapshot.Manifest{
|
|
ID: "id1",
|
|
}
|
|
manifest2 = &snapshot.Manifest{
|
|
ID: "id2",
|
|
}
|
|
)
|
|
|
|
table := []struct {
|
|
name string
|
|
inputMan []*kopia.ManifestEntry
|
|
expected []kopia.IncrementalBase
|
|
}{
|
|
{
|
|
name: "SingleManifestSingleReason",
|
|
inputMan: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: manifest1,
|
|
Reasons: []kopia.Reason{
|
|
emailReason,
|
|
},
|
|
},
|
|
},
|
|
expected: []kopia.IncrementalBase{
|
|
{
|
|
Manifest: manifest1,
|
|
SubtreePaths: []*path.Builder{
|
|
emailBuilder,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "SingleManifestMultipleReasons",
|
|
inputMan: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: manifest1,
|
|
Reasons: []kopia.Reason{
|
|
emailReason,
|
|
contactsReason,
|
|
},
|
|
},
|
|
},
|
|
expected: []kopia.IncrementalBase{
|
|
{
|
|
Manifest: manifest1,
|
|
SubtreePaths: []*path.Builder{
|
|
emailBuilder,
|
|
contactsBuilder,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "MultipleManifestsMultipleReasons",
|
|
inputMan: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: manifest1,
|
|
Reasons: []kopia.Reason{
|
|
emailReason,
|
|
contactsReason,
|
|
},
|
|
},
|
|
{
|
|
Manifest: manifest2,
|
|
Reasons: []kopia.Reason{
|
|
emailReason,
|
|
contactsReason,
|
|
},
|
|
},
|
|
},
|
|
expected: []kopia.IncrementalBase{
|
|
{
|
|
Manifest: manifest1,
|
|
SubtreePaths: []*path.Builder{
|
|
emailBuilder,
|
|
contactsBuilder,
|
|
},
|
|
},
|
|
{
|
|
Manifest: manifest2,
|
|
SubtreePaths: []*path.Builder{
|
|
emailBuilder,
|
|
contactsBuilder,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range table {
|
|
suite.T().Run(test.name, func(t *testing.T) {
|
|
ctx, flush := tester.NewContext()
|
|
defer flush()
|
|
|
|
mbu := &mockBackuper{
|
|
checkFunc: func(
|
|
bases []kopia.IncrementalBase,
|
|
cs []data.Collection,
|
|
tags map[string]string,
|
|
buildTreeWithBase bool,
|
|
) {
|
|
assert.ElementsMatch(t, test.expected, bases)
|
|
},
|
|
}
|
|
|
|
//nolint:errcheck
|
|
consumeBackupDataCollections(
|
|
ctx,
|
|
mbu,
|
|
tenant,
|
|
nil,
|
|
test.inputMan,
|
|
nil,
|
|
model.StableID(""),
|
|
true,
|
|
)
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_MergeBackupDetails_AddsItems() {
|
|
var (
|
|
tenant = "a-tenant"
|
|
ro = "a-user"
|
|
|
|
itemPath1 = makePath(
|
|
suite.T(),
|
|
[]string{
|
|
tenant,
|
|
path.OneDriveService.String(),
|
|
ro,
|
|
path.FilesCategory.String(),
|
|
"drives",
|
|
"drive-id",
|
|
"root:",
|
|
"work",
|
|
"item1",
|
|
},
|
|
true,
|
|
)
|
|
itemPath2 = makePath(
|
|
suite.T(),
|
|
[]string{
|
|
tenant,
|
|
path.OneDriveService.String(),
|
|
ro,
|
|
path.FilesCategory.String(),
|
|
"drives",
|
|
"drive-id",
|
|
"root:",
|
|
"personal",
|
|
"item2",
|
|
},
|
|
true,
|
|
)
|
|
itemPath3 = makePath(
|
|
suite.T(),
|
|
[]string{
|
|
tenant,
|
|
path.ExchangeService.String(),
|
|
ro,
|
|
path.EmailCategory.String(),
|
|
"personal",
|
|
"item3",
|
|
},
|
|
true,
|
|
)
|
|
|
|
backup1 = backup.Backup{
|
|
BaseModel: model.BaseModel{
|
|
ID: "bid1",
|
|
},
|
|
DetailsID: "did1",
|
|
}
|
|
|
|
backup2 = backup.Backup{
|
|
BaseModel: model.BaseModel{
|
|
ID: "bid2",
|
|
},
|
|
DetailsID: "did2",
|
|
}
|
|
|
|
pathReason1 = kopia.Reason{
|
|
ResourceOwner: itemPath1.ResourceOwner(),
|
|
Service: itemPath1.Service(),
|
|
Category: itemPath1.Category(),
|
|
}
|
|
pathReason3 = kopia.Reason{
|
|
ResourceOwner: itemPath3.ResourceOwner(),
|
|
Service: itemPath3.Service(),
|
|
Category: itemPath3.Category(),
|
|
}
|
|
)
|
|
|
|
itemParents1, err := path.GetDriveFolderPath(itemPath1)
|
|
require.NoError(suite.T(), err)
|
|
|
|
table := []struct {
|
|
name string
|
|
populatedModels map[model.StableID]backup.Backup
|
|
populatedDetails map[string]*details.Details
|
|
inputMans []*kopia.ManifestEntry
|
|
inputShortRefsFromPrevBackup map[string]path.Path
|
|
|
|
errCheck assert.ErrorAssertionFunc
|
|
expectedEntries []*details.DetailsEntry
|
|
}{
|
|
{
|
|
name: "NilShortRefsFromPrevBackup",
|
|
errCheck: assert.NoError,
|
|
// Use empty slice so we don't error out on nil != empty.
|
|
expectedEntries: []*details.DetailsEntry{},
|
|
},
|
|
{
|
|
name: "EmptyShortRefsFromPrevBackup",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{},
|
|
errCheck: assert.NoError,
|
|
// Use empty slice so we don't error out on nil != empty.
|
|
expectedEntries: []*details.DetailsEntry{},
|
|
},
|
|
{
|
|
name: "BackupIDNotFound",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), "foo", ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "DetailsIDNotFound",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: {
|
|
BaseModel: model.BaseModel{
|
|
ID: backup1.ID,
|
|
},
|
|
DetailsID: "foo",
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "BaseMissingItems",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
itemPath2.ShortRef(): itemPath2,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "TooManyItems",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "BadBaseRepoRef",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
{
|
|
RepoRef: stdpath.Join(
|
|
append(
|
|
[]string{
|
|
itemPath1.Tenant(),
|
|
itemPath1.Service().String(),
|
|
itemPath1.ResourceOwner(),
|
|
path.UnknownCategory.String(),
|
|
},
|
|
itemPath1.Folders()...,
|
|
)...,
|
|
),
|
|
ItemInfo: details.ItemInfo{
|
|
OneDrive: &details.OneDriveInfo{
|
|
ItemType: details.OneDriveItem,
|
|
ParentPath: itemParents1,
|
|
Size: 42,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "BadOneDrivePath",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): makePath(
|
|
suite.T(),
|
|
[]string{
|
|
itemPath1.Tenant(),
|
|
path.OneDriveService.String(),
|
|
itemPath1.ResourceOwner(),
|
|
path.FilesCategory.String(),
|
|
"personal",
|
|
"item1",
|
|
},
|
|
true,
|
|
),
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.Error,
|
|
},
|
|
{
|
|
name: "ItemMerged",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
expectedEntries: []*details.DetailsEntry{
|
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
{
|
|
name: "ItemMergedExtraItemsInBase",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
*makeDetailsEntry(suite.T(), itemPath2, 84, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
expectedEntries: []*details.DetailsEntry{
|
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
{
|
|
name: "ItemMoved",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath2,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
expectedEntries: []*details.DetailsEntry{
|
|
makeDetailsEntry(suite.T(), itemPath2, 42, true),
|
|
},
|
|
},
|
|
{
|
|
name: "MultipleBases",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
itemPath3.ShortRef(): itemPath3,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup2.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason3,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
backup2.ID: backup2,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
backup2.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
// This entry should not be picked due to a mismatch on Reasons.
|
|
*makeDetailsEntry(suite.T(), itemPath1, 84, false),
|
|
// This item should be picked.
|
|
*makeDetailsEntry(suite.T(), itemPath3, 37, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
expectedEntries: []*details.DetailsEntry{
|
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
makeDetailsEntry(suite.T(), itemPath3, 37, false),
|
|
},
|
|
},
|
|
{
|
|
name: "SomeBasesIncomplete",
|
|
inputShortRefsFromPrevBackup: map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
},
|
|
inputMans: []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
{
|
|
Manifest: makeManifest(suite.T(), backup2.ID, "checkpoint"),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
},
|
|
populatedModels: map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
backup2.ID: backup2,
|
|
},
|
|
populatedDetails: map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
},
|
|
backup2.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
// This entry should not be picked due to being incomplete.
|
|
*makeDetailsEntry(suite.T(), itemPath1, 84, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
errCheck: assert.NoError,
|
|
expectedEntries: []*details.DetailsEntry{
|
|
makeDetailsEntry(suite.T(), itemPath1, 42, false),
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range table {
|
|
suite.T().Run(test.name, func(t *testing.T) {
|
|
ctx, flush := tester.NewContext()
|
|
defer flush()
|
|
|
|
mdr := mockDetailsReader{entries: test.populatedDetails}
|
|
w := &store.Wrapper{Storer: mockBackupStorer{entries: test.populatedModels}}
|
|
|
|
deets := details.Builder{}
|
|
|
|
err := mergeDetails(
|
|
ctx,
|
|
w,
|
|
mdr,
|
|
test.inputMans,
|
|
test.inputShortRefsFromPrevBackup,
|
|
&deets,
|
|
)
|
|
|
|
test.errCheck(t, err)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
assert.ElementsMatch(t, test.expectedEntries, deets.Details().Items())
|
|
})
|
|
}
|
|
}
|
|
|
|
func (suite *BackupOpSuite) TestBackupOperation_MergeBackupDetails_AddsFolders() {
|
|
var (
|
|
t = suite.T()
|
|
|
|
tenant = "a-tenant"
|
|
ro = "a-user"
|
|
|
|
pathElems = []string{
|
|
tenant,
|
|
path.ExchangeService.String(),
|
|
ro,
|
|
path.EmailCategory.String(),
|
|
"work",
|
|
"item1",
|
|
}
|
|
|
|
itemPath1 = makePath(
|
|
t,
|
|
pathElems,
|
|
true,
|
|
)
|
|
|
|
backup1 = backup.Backup{
|
|
BaseModel: model.BaseModel{
|
|
ID: "bid1",
|
|
},
|
|
DetailsID: "did1",
|
|
}
|
|
|
|
pathReason1 = kopia.Reason{
|
|
ResourceOwner: itemPath1.ResourceOwner(),
|
|
Service: itemPath1.Service(),
|
|
Category: itemPath1.Category(),
|
|
}
|
|
|
|
inputToMerge = map[string]path.Path{
|
|
itemPath1.ShortRef(): itemPath1,
|
|
}
|
|
|
|
inputMans = []*kopia.ManifestEntry{
|
|
{
|
|
Manifest: makeManifest(t, backup1.ID, ""),
|
|
Reasons: []kopia.Reason{
|
|
pathReason1,
|
|
},
|
|
},
|
|
}
|
|
|
|
populatedModels = map[model.StableID]backup.Backup{
|
|
backup1.ID: backup1,
|
|
}
|
|
|
|
itemSize = 42
|
|
itemDetails = makeDetailsEntry(t, itemPath1, itemSize, false)
|
|
|
|
populatedDetails = map[string]*details.Details{
|
|
backup1.DetailsID: {
|
|
DetailsModel: details.DetailsModel{
|
|
Entries: []details.DetailsEntry{
|
|
*itemDetails,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
expectedEntries = []details.DetailsEntry{
|
|
*itemDetails,
|
|
}
|
|
)
|
|
|
|
itemDetails.Exchange.Modified = time.Now()
|
|
|
|
for i := 1; i < len(pathElems); i++ {
|
|
expectedEntries = append(expectedEntries, *makeFolderEntry(
|
|
t,
|
|
path.Builder{}.Append(pathElems[:i]...),
|
|
int64(itemSize),
|
|
itemDetails.Exchange.Modified,
|
|
))
|
|
}
|
|
|
|
ctx, flush := tester.NewContext()
|
|
defer flush()
|
|
|
|
mdr := mockDetailsReader{entries: populatedDetails}
|
|
w := &store.Wrapper{Storer: mockBackupStorer{entries: populatedModels}}
|
|
|
|
deets := details.Builder{}
|
|
|
|
err := mergeDetails(
|
|
ctx,
|
|
w,
|
|
mdr,
|
|
inputMans,
|
|
inputToMerge,
|
|
&deets,
|
|
)
|
|
|
|
assert.NoError(t, err)
|
|
assert.ElementsMatch(t, expectedEntries, deets.Details().Entries)
|
|
}
|