migrate, test manifest+meta producer (#2091)
## Description Adds mocked unit tests for produceManifestsAnd- Metadata. For cleanliness, moves that func, and any funcs called within it, to their own file within operations ## Does this PR need a docs update or release note? - [x] ⛔ No ## Type of change - [x] 🤖 Test ## Issue(s) * #2062 ## Test Plan - [x] ⚡ Unit test
This commit is contained in:
Keepers
GitHub
parent
45874abf7e
commit
48e4b65165
@ -6,12 +6,10 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common"
|
||||
"github.com/alcionai/corso/src/internal/connector"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
D "github.com/alcionai/corso/src/internal/diagnostics"
|
||||
@ -262,178 +260,6 @@ type backuper interface {
|
||||
) (*kopia.BackupStats, *details.Builder, map[string]path.Path, error)
|
||||
}
|
||||
|
||||
func verifyDistinctBases(mans []*kopia.ManifestEntry) error {
|
||||
var (
|
||||
errs *multierror.Error
|
||||
reasons = map[string]manifest.ID{}
|
||||
)
|
||||
|
||||
for _, man := range mans {
|
||||
// Incomplete snapshots are used only for kopia-assisted incrementals. The
|
||||
// fact that we need this check here makes it seem like this should live in
|
||||
// the kopia code. However, keeping it here allows for better debugging as
|
||||
// the kopia code only has access to a path builder which means it cannot
|
||||
// remove the resource owner from the error/log output. That is also below
|
||||
// the point where we decide if we should do a full backup or an
|
||||
// incremental.
|
||||
if len(man.IncompleteReason) > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, reason := range man.Reasons {
|
||||
reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String()
|
||||
|
||||
if b, ok := reasons[reasonKey]; ok {
|
||||
errs = multierror.Append(errs, errors.Errorf(
|
||||
"multiple base snapshots source data for %s %s. IDs: %s, %s",
|
||||
reason.Service.String(),
|
||||
reason.Category.String(),
|
||||
b,
|
||||
man.ID,
|
||||
))
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
reasons[reasonKey] = man.ID
|
||||
}
|
||||
}
|
||||
|
||||
return errs.ErrorOrNil()
|
||||
}
|
||||
|
||||
// calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics.
|
||||
func produceManifestsAndMetadata(
|
||||
ctx context.Context,
|
||||
kw *kopia.Wrapper,
|
||||
sw *store.Wrapper,
|
||||
reasons []kopia.Reason,
|
||||
tenantID string,
|
||||
getMetadata bool,
|
||||
) ([]*kopia.ManifestEntry, []data.Collection, bool, error) {
|
||||
var (
|
||||
metadataFiles = graph.AllMetadataFileNames()
|
||||
collections []data.Collection
|
||||
)
|
||||
|
||||
ms, err := kw.FetchPrevSnapshotManifests(
|
||||
ctx,
|
||||
reasons,
|
||||
map[string]string{kopia.TagBackupCategory: ""})
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
if !getMetadata {
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
// We only need to check that we have 1:1 reason:base if we're doing an
|
||||
// incremental with associated metadata. This ensures that we're only sourcing
|
||||
// data from a single Point-In-Time (base) for each incremental backup.
|
||||
//
|
||||
// TODO(ashmrtn): This may need updating if we start sourcing item backup
|
||||
// details from previous snapshots when using kopia-assisted incrementals.
|
||||
if err := verifyDistinctBases(ms); err != nil {
|
||||
logger.Ctx(ctx).Warnw(
|
||||
"base snapshot collision, falling back to full backup",
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
for _, man := range ms {
|
||||
if len(man.IncompleteReason) > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
bID, ok := man.GetTag(kopia.TagBackupID)
|
||||
if !ok {
|
||||
return nil, nil, false, errors.New("snapshot manifest missing backup ID")
|
||||
}
|
||||
|
||||
dID, _, err := sw.GetDetailsIDFromBackupID(ctx, model.StableID(bID))
|
||||
if err != nil {
|
||||
// if no backup exists for any of the complete manifests, we want
|
||||
// to fall back to a complete backup.
|
||||
if errors.Is(err, kopia.ErrNotFound) {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"backup missing, falling back to full backup",
|
||||
"backup_id", bID)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
return nil, nil, false, errors.Wrap(err, "retrieving prior backup data")
|
||||
}
|
||||
|
||||
// if no detailsID exists for any of the complete manifests, we want
|
||||
// to fall back to a complete backup. This is a temporary prevention
|
||||
// mechanism to keep backups from falling into a perpetually bad state.
|
||||
// This makes an assumption that the ID points to a populated set of
|
||||
// details; we aren't doing the work to look them up.
|
||||
if len(dID) == 0 {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"backup missing details ID, falling back to full backup",
|
||||
"backup_id", bID)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
colls, err := collectMetadata(ctx, kw, man, metadataFiles, tenantID)
|
||||
if err != nil && !errors.Is(err, kopia.ErrNotFound) {
|
||||
// prior metadata isn't guaranteed to exist.
|
||||
// if it doesn't, we'll just have to do a
|
||||
// full backup for that data.
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
collections = append(collections, colls...)
|
||||
}
|
||||
|
||||
return ms, collections, true, err
|
||||
}
|
||||
|
||||
func collectMetadata(
|
||||
ctx context.Context,
|
||||
r restorer,
|
||||
man *kopia.ManifestEntry,
|
||||
fileNames []string,
|
||||
tenantID string,
|
||||
) ([]data.Collection, error) {
|
||||
paths := []path.Path{}
|
||||
|
||||
for _, fn := range fileNames {
|
||||
for _, reason := range man.Reasons {
|
||||
p, err := path.Builder{}.
|
||||
Append(fn).
|
||||
ToServiceCategoryMetadataPath(
|
||||
tenantID,
|
||||
reason.ResourceOwner,
|
||||
reason.Service,
|
||||
reason.Category,
|
||||
true)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "building metadata path")
|
||||
}
|
||||
|
||||
paths = append(paths, p)
|
||||
}
|
||||
}
|
||||
|
||||
dcs, err := r.RestoreMultipleItems(ctx, string(man.ID), paths, nil)
|
||||
if err != nil {
|
||||
// Restore is best-effort and we want to keep it that way since we want to
|
||||
// return as much metadata as we can to reduce the work we'll need to do.
|
||||
// Just wrap the error here for better reporting/debugging.
|
||||
return dcs, errors.Wrap(err, "collecting prior metadata")
|
||||
}
|
||||
|
||||
return dcs, nil
|
||||
}
|
||||
|
||||
func selectorToReasons(sel selectors.Selector) []kopia.Reason {
|
||||
service := sel.PathService()
|
||||
reasons := []kopia.Reason{}
|
||||
|
||||
@ -36,6 +36,25 @@ import (
|
||||
|
||||
type mockRestorer struct {
|
||||
gotPaths []path.Path
|
||||
colls []data.Collection
|
||||
collsByID map[string][]data.Collection // snapshotID: []Collection
|
||||
err error
|
||||
onRestore restoreFunc
|
||||
}
|
||||
|
||||
type restoreFunc func(id string, ps []path.Path) ([]data.Collection, error)
|
||||
|
||||
func (mr *mockRestorer) buildRestoreFunc(
|
||||
t *testing.T,
|
||||
oid string,
|
||||
ops []path.Path,
|
||||
) {
|
||||
mr.onRestore = func(id string, ps []path.Path) ([]data.Collection, error) {
|
||||
assert.Equal(t, oid, id, "manifest id")
|
||||
checkPaths(t, ops, ps)
|
||||
|
||||
return mr.colls, mr.err
|
||||
}
|
||||
}
|
||||
|
||||
func (mr *mockRestorer) RestoreMultipleItems(
|
||||
@ -46,13 +65,19 @@ func (mr *mockRestorer) RestoreMultipleItems(
|
||||
) ([]data.Collection, error) {
|
||||
mr.gotPaths = append(mr.gotPaths, paths...)
|
||||
|
||||
return nil, nil
|
||||
if mr.onRestore != nil {
|
||||
return mr.onRestore(snapshotID, paths)
|
||||
}
|
||||
|
||||
func (mr mockRestorer) checkPaths(t *testing.T, expected []path.Path) {
|
||||
t.Helper()
|
||||
if len(mr.collsByID) > 0 {
|
||||
return mr.collsByID[snapshotID], mr.err
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expected, mr.gotPaths)
|
||||
return mr.colls, mr.err
|
||||
}
|
||||
|
||||
func checkPaths(t *testing.T, expected, got []path.Path) {
|
||||
assert.ElementsMatch(t, expected, got)
|
||||
}
|
||||
|
||||
// ----- backup producer
|
||||
@ -168,6 +193,27 @@ func (mbs mockBackupStorer) Update(context.Context, model.Schema, model.Model) e
|
||||
// helper funcs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// expects you to Append your own file
|
||||
func makeMetadataBasePath(
|
||||
t *testing.T,
|
||||
tenant string,
|
||||
service path.ServiceType,
|
||||
resourceOwner string,
|
||||
category path.CategoryType,
|
||||
) path.Path {
|
||||
t.Helper()
|
||||
|
||||
p, err := path.Builder{}.ToServiceCategoryMetadataPath(
|
||||
tenant,
|
||||
resourceOwner,
|
||||
service,
|
||||
category,
|
||||
false)
|
||||
require.NoError(t, err)
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
func makeMetadataPath(
|
||||
t *testing.T,
|
||||
tenant string,
|
||||
@ -183,8 +229,7 @@ func makeMetadataPath(
|
||||
resourceOwner,
|
||||
service,
|
||||
category,
|
||||
true,
|
||||
)
|
||||
true)
|
||||
require.NoError(t, err)
|
||||
|
||||
return p
|
||||
@ -635,7 +680,7 @@ func (suite *BackupOpSuite) TestBackupOperation_CollectMetadata() {
|
||||
_, err := collectMetadata(ctx, mr, test.inputMan, test.inputFiles, tenant)
|
||||
assert.NoError(t, err)
|
||||
|
||||
mr.checkPaths(t, test.expected)
|
||||
checkPaths(t, test.expected, mr.gotPaths)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
210
src/internal/operations/manifests.go
Normal file
210
src/internal/operations/manifests.go
Normal file
@ -0,0 +1,210 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
type manifestFetcher interface {
|
||||
FetchPrevSnapshotManifests(
|
||||
ctx context.Context,
|
||||
reasons []kopia.Reason,
|
||||
tags map[string]string,
|
||||
) ([]*kopia.ManifestEntry, error)
|
||||
}
|
||||
|
||||
type manifestRestorer interface {
|
||||
manifestFetcher
|
||||
restorer
|
||||
}
|
||||
|
||||
type getDetailsIDer interface {
|
||||
GetDetailsIDFromBackupID(
|
||||
ctx context.Context,
|
||||
backupID model.StableID,
|
||||
) (string, *backup.Backup, error)
|
||||
}
|
||||
|
||||
// calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics.
|
||||
func produceManifestsAndMetadata(
|
||||
ctx context.Context,
|
||||
mr manifestRestorer,
|
||||
gdi getDetailsIDer,
|
||||
reasons []kopia.Reason,
|
||||
tenantID string,
|
||||
getMetadata bool,
|
||||
) ([]*kopia.ManifestEntry, []data.Collection, bool, error) {
|
||||
var (
|
||||
metadataFiles = graph.AllMetadataFileNames()
|
||||
collections []data.Collection
|
||||
)
|
||||
|
||||
ms, err := mr.FetchPrevSnapshotManifests(
|
||||
ctx,
|
||||
reasons,
|
||||
map[string]string{kopia.TagBackupCategory: ""})
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
if !getMetadata {
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
// We only need to check that we have 1:1 reason:base if we're doing an
|
||||
// incremental with associated metadata. This ensures that we're only sourcing
|
||||
// data from a single Point-In-Time (base) for each incremental backup.
|
||||
//
|
||||
// TODO(ashmrtn): This may need updating if we start sourcing item backup
|
||||
// details from previous snapshots when using kopia-assisted incrementals.
|
||||
if err := verifyDistinctBases(ms); err != nil {
|
||||
logger.Ctx(ctx).Warnw(
|
||||
"base snapshot collision, falling back to full backup",
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
for _, man := range ms {
|
||||
if len(man.IncompleteReason) > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
bID, ok := man.GetTag(kopia.TagBackupID)
|
||||
if !ok {
|
||||
return nil, nil, false, errors.New("snapshot manifest missing backup ID")
|
||||
}
|
||||
|
||||
dID, _, err := gdi.GetDetailsIDFromBackupID(ctx, model.StableID(bID))
|
||||
if err != nil {
|
||||
// if no backup exists for any of the complete manifests, we want
|
||||
// to fall back to a complete backup.
|
||||
if errors.Is(err, kopia.ErrNotFound) {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"backup missing, falling back to full backup",
|
||||
"backup_id", bID)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
return nil, nil, false, errors.Wrap(err, "retrieving prior backup data")
|
||||
}
|
||||
|
||||
// if no detailsID exists for any of the complete manifests, we want
|
||||
// to fall back to a complete backup. This is a temporary prevention
|
||||
// mechanism to keep backups from falling into a perpetually bad state.
|
||||
// This makes an assumption that the ID points to a populated set of
|
||||
// details; we aren't doing the work to look them up.
|
||||
if len(dID) == 0 {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"backup missing details ID, falling back to full backup",
|
||||
"backup_id", bID)
|
||||
|
||||
return ms, nil, false, nil
|
||||
}
|
||||
|
||||
colls, err := collectMetadata(ctx, mr, man, metadataFiles, tenantID)
|
||||
if err != nil && !errors.Is(err, kopia.ErrNotFound) {
|
||||
// prior metadata isn't guaranteed to exist.
|
||||
// if it doesn't, we'll just have to do a
|
||||
// full backup for that data.
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
collections = append(collections, colls...)
|
||||
}
|
||||
|
||||
return ms, collections, true, err
|
||||
}
|
||||
|
||||
// verifyDistinctBases is a validation checker that ensures, for a given slice
|
||||
// of manifests, that each manifest's Reason (owner, service, category) is only
|
||||
// included once. If a reason is duplicated by any two manifests, an error is
|
||||
// returned.
|
||||
func verifyDistinctBases(mans []*kopia.ManifestEntry) error {
|
||||
var (
|
||||
errs *multierror.Error
|
||||
reasons = map[string]manifest.ID{}
|
||||
)
|
||||
|
||||
for _, man := range mans {
|
||||
// Incomplete snapshots are used only for kopia-assisted incrementals. The
|
||||
// fact that we need this check here makes it seem like this should live in
|
||||
// the kopia code. However, keeping it here allows for better debugging as
|
||||
// the kopia code only has access to a path builder which means it cannot
|
||||
// remove the resource owner from the error/log output. That is also below
|
||||
// the point where we decide if we should do a full backup or an incremental.
|
||||
if len(man.IncompleteReason) > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, reason := range man.Reasons {
|
||||
reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String()
|
||||
|
||||
if b, ok := reasons[reasonKey]; ok {
|
||||
errs = multierror.Append(errs, errors.Errorf(
|
||||
"multiple base snapshots source data for %s %s. IDs: %s, %s",
|
||||
reason.Service, reason.Category, b, man.ID,
|
||||
))
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
reasons[reasonKey] = man.ID
|
||||
}
|
||||
}
|
||||
|
||||
return errs.ErrorOrNil()
|
||||
}
|
||||
|
||||
// collectMetadata retrieves all metadata files associated with the manifest.
|
||||
func collectMetadata(
|
||||
ctx context.Context,
|
||||
r restorer,
|
||||
man *kopia.ManifestEntry,
|
||||
fileNames []string,
|
||||
tenantID string,
|
||||
) ([]data.Collection, error) {
|
||||
paths := []path.Path{}
|
||||
|
||||
for _, fn := range fileNames {
|
||||
for _, reason := range man.Reasons {
|
||||
p, err := path.Builder{}.
|
||||
Append(fn).
|
||||
ToServiceCategoryMetadataPath(
|
||||
tenantID,
|
||||
reason.ResourceOwner,
|
||||
reason.Service,
|
||||
reason.Category,
|
||||
true)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "building metadata path")
|
||||
}
|
||||
|
||||
paths = append(paths, p)
|
||||
}
|
||||
}
|
||||
|
||||
dcs, err := r.RestoreMultipleItems(ctx, string(man.ID), paths, nil)
|
||||
if err != nil {
|
||||
// Restore is best-effort and we want to keep it that way since we want to
|
||||
// return as much metadata as we can to reduce the work we'll need to do.
|
||||
// Just wrap the error here for better reporting/debugging.
|
||||
return dcs, errors.Wrap(err, "collecting prior metadata")
|
||||
}
|
||||
|
||||
return dcs, nil
|
||||
}
|
||||
685
src/internal/operations/manifests_test.go
Normal file
685
src/internal/operations/manifests_test.go
Normal file
@ -0,0 +1,685 @@
|
||||
package operations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// interfaces
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mockManifestRestorer struct {
|
||||
mockRestorer
|
||||
mans []*kopia.ManifestEntry
|
||||
mrErr error // err varname already claimed by mockRestorer
|
||||
}
|
||||
|
||||
func (mmr mockManifestRestorer) FetchPrevSnapshotManifests(
|
||||
ctx context.Context,
|
||||
reasons []kopia.Reason,
|
||||
tags map[string]string,
|
||||
) ([]*kopia.ManifestEntry, error) {
|
||||
return mmr.mans, mmr.mrErr
|
||||
}
|
||||
|
||||
type mockGetDetailsIDer struct {
|
||||
detailsID string
|
||||
err error
|
||||
}
|
||||
|
||||
func (mg mockGetDetailsIDer) GetDetailsIDFromBackupID(
|
||||
ctx context.Context,
|
||||
backupID model.StableID,
|
||||
) (string, *backup.Backup, error) {
|
||||
return mg.detailsID, nil, mg.err
|
||||
}
|
||||
|
||||
type mockColl struct {
|
||||
id string // for comparisons
|
||||
p path.Path
|
||||
prevP path.Path
|
||||
}
|
||||
|
||||
func (mc mockColl) Items() <-chan data.Stream {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mc mockColl) FullPath() path.Path {
|
||||
return mc.p
|
||||
}
|
||||
|
||||
func (mc mockColl) PreviousPath() path.Path {
|
||||
return mc.prevP
|
||||
}
|
||||
|
||||
func (mc mockColl) State() data.CollectionState {
|
||||
return data.NewState
|
||||
}
|
||||
|
||||
func (mc mockColl) DoNotMergeItems() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type OperationsManifestsUnitSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestOperationsManifestsUnitSuite(t *testing.T) {
|
||||
suite.Run(t, new(OperationsManifestsUnitSuite))
|
||||
}
|
||||
|
||||
func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() {
|
||||
const (
|
||||
ro = "owner"
|
||||
tid = "tenantid"
|
||||
)
|
||||
|
||||
var (
|
||||
emailPath = makeMetadataBasePath(
|
||||
suite.T(),
|
||||
tid,
|
||||
path.ExchangeService,
|
||||
ro,
|
||||
path.EmailCategory)
|
||||
contactPath = makeMetadataBasePath(
|
||||
suite.T(),
|
||||
tid,
|
||||
path.ExchangeService,
|
||||
ro,
|
||||
path.ContactsCategory)
|
||||
)
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
manID string
|
||||
reasons []kopia.Reason
|
||||
fileNames []string
|
||||
expectPaths func(*testing.T, []string) []path.Path
|
||||
expectErr error
|
||||
}{
|
||||
{
|
||||
name: "single reason, single file",
|
||||
manID: "single single",
|
||||
reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
ps := make([]path.Path, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
p, err := emailPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
}
|
||||
|
||||
return ps
|
||||
},
|
||||
fileNames: []string{"a"},
|
||||
},
|
||||
{
|
||||
name: "single reason, multiple files",
|
||||
manID: "single multi",
|
||||
reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
ps := make([]path.Path, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
p, err := emailPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
}
|
||||
|
||||
return ps
|
||||
},
|
||||
fileNames: []string{"a", "b"},
|
||||
},
|
||||
{
|
||||
name: "multiple reasons, single file",
|
||||
manID: "multi single",
|
||||
reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
},
|
||||
},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
ps := make([]path.Path, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
p, err := emailPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
p, err = contactPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
}
|
||||
|
||||
return ps
|
||||
},
|
||||
fileNames: []string{"a"},
|
||||
},
|
||||
{
|
||||
name: "multiple reasons, multiple file",
|
||||
manID: "multi multi",
|
||||
reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
},
|
||||
},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
ps := make([]path.Path, 0, len(files))
|
||||
|
||||
for _, f := range files {
|
||||
p, err := emailPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
p, err = contactPath.Append(f, true)
|
||||
assert.NoError(t, err)
|
||||
ps = append(ps, p)
|
||||
}
|
||||
|
||||
return ps
|
||||
},
|
||||
fileNames: []string{"a", "b"},
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.T().Run(test.name, func(t *testing.T) {
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
paths := test.expectPaths(t, test.fileNames)
|
||||
|
||||
mr := mockRestorer{err: test.expectErr}
|
||||
mr.buildRestoreFunc(t, test.manID, paths)
|
||||
|
||||
man := &kopia.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{ID: manifest.ID(test.manID)},
|
||||
Reasons: test.reasons,
|
||||
}
|
||||
|
||||
_, err := collectMetadata(ctx, &mr, man, test.fileNames, tid)
|
||||
assert.ErrorIs(t, err, test.expectErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *OperationsManifestsUnitSuite) TestVerifyDistinctBases() {
|
||||
ro := "resource_owner"
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
mans []*kopia.ManifestEntry
|
||||
expect assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "one manifest, one reason",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "one incomplete manifest",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{IncompleteReason: "ir"},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "one manifest, multiple reasons",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "one manifest, duplicate reasons",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "two manifests, non-overlapping reasons",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "two manifests, overlapping reasons",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "two manifests, overlapping reasons, one snapshot incomplete",
|
||||
mans: []*kopia.ManifestEntry{
|
||||
{
|
||||
Manifest: &snapshot.Manifest{},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Manifest: &snapshot.Manifest{IncompleteReason: "ir"},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.EmailCategory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.T().Run(test.name, func(t *testing.T) {
|
||||
err := verifyDistinctBases(test.mans)
|
||||
test.expect(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
const (
|
||||
ro = "resourceowner"
|
||||
tid = "tenantid"
|
||||
did = "detailsid"
|
||||
)
|
||||
|
||||
makeMan := func(pct path.CategoryType, id, incmpl, bid string) *kopia.ManifestEntry {
|
||||
tags := map[string]string{}
|
||||
if len(bid) > 0 {
|
||||
tags = map[string]string{"tag:" + kopia.TagBackupID: bid}
|
||||
}
|
||||
|
||||
return &kopia.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
IncompleteReason: incmpl,
|
||||
Tags: tags,
|
||||
},
|
||||
Reasons: []kopia.Reason{
|
||||
{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: pct,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
mr mockManifestRestorer
|
||||
gdi mockGetDetailsIDer
|
||||
reasons []kopia.Reason
|
||||
getMeta bool
|
||||
assertErr assert.ErrorAssertionFunc
|
||||
assertB assert.BoolAssertionFunc
|
||||
expectDCS []data.Collection
|
||||
expectNilMans bool
|
||||
}{
|
||||
{
|
||||
name: "don't get metadata, no mans",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: false,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "don't get metadata",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "", "", "")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: false,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "don't get metadata, incomplete manifest",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "", "ir", "")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: false,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "fetch manifests errors",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mrErr: assert.AnError,
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.Error,
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "verify distinct bases fails",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "", "", ""),
|
||||
makeMan(path.EmailCategory, "", "", ""),
|
||||
},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError, // No error, even though verify failed.
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "no manifests",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.True,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "only incomplete manifests",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "", "ir", ""),
|
||||
makeMan(path.ContactsCategory, "", "ir", ""),
|
||||
},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.True,
|
||||
expectDCS: nil,
|
||||
},
|
||||
{
|
||||
name: "man missing backup id",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{collsByID: map[string][]data.Collection{
|
||||
"id": {mockColl{id: "id_coll"}},
|
||||
}},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.Error,
|
||||
assertB: assert.False,
|
||||
expectNilMans: true,
|
||||
},
|
||||
{
|
||||
name: "backup missing details id",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "", "", "bid")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.False,
|
||||
},
|
||||
{
|
||||
name: "one complete, one incomplete",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{collsByID: map[string][]data.Collection{
|
||||
"id": {mockColl{id: "id_coll"}},
|
||||
"incmpl_id": {mockColl{id: "incmpl_id_coll"}},
|
||||
}},
|
||||
mans: []*kopia.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "id", "", "bid"),
|
||||
makeMan(path.EmailCategory, "incmpl_id", "ir", ""),
|
||||
},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.True,
|
||||
expectDCS: []data.Collection{mockColl{id: "id_coll"}},
|
||||
},
|
||||
{
|
||||
name: "single valid man",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{collsByID: map[string][]data.Collection{
|
||||
"id": {mockColl{id: "id_coll"}},
|
||||
}},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "id", "", "bid")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.True,
|
||||
expectDCS: []data.Collection{mockColl{id: "id_coll"}},
|
||||
},
|
||||
{
|
||||
name: "multiple valid mans",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{collsByID: map[string][]data.Collection{
|
||||
"mail": {mockColl{id: "mail_coll"}},
|
||||
"contact": {mockColl{id: "contact_coll"}},
|
||||
}},
|
||||
mans: []*kopia.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "mail", "", "bid"),
|
||||
makeMan(path.ContactsCategory, "contact", "", "bid"),
|
||||
},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
assertB: assert.True,
|
||||
expectDCS: []data.Collection{
|
||||
mockColl{id: "mail_coll"},
|
||||
mockColl{id: "contact_coll"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "error collecting metadata",
|
||||
mr: mockManifestRestorer{
|
||||
mockRestorer: mockRestorer{err: assert.AnError},
|
||||
mans: []*kopia.ManifestEntry{makeMan(path.EmailCategory, "", "", "bid")},
|
||||
},
|
||||
gdi: mockGetDetailsIDer{detailsID: did},
|
||||
reasons: []kopia.Reason{},
|
||||
getMeta: true,
|
||||
assertErr: assert.Error,
|
||||
assertB: assert.False,
|
||||
expectDCS: nil,
|
||||
expectNilMans: true,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.T().Run(test.name, func(t *testing.T) {
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
mans, dcs, b, err := produceManifestsAndMetadata(
|
||||
ctx,
|
||||
&test.mr,
|
||||
&test.gdi,
|
||||
test.reasons,
|
||||
tid,
|
||||
test.getMeta,
|
||||
)
|
||||
test.assertErr(t, err)
|
||||
test.assertB(t, b)
|
||||
|
||||
expectMans := test.mr.mans
|
||||
if test.expectNilMans {
|
||||
expectMans = nil
|
||||
}
|
||||
assert.Equal(t, expectMans, mans)
|
||||
|
||||
expect, got := []string{}, []string{}
|
||||
|
||||
for _, dc := range test.expectDCS {
|
||||
mc, ok := dc.(mockColl)
|
||||
assert.True(t, ok)
|
||||
|
||||
expect = append(expect, mc.id)
|
||||
}
|
||||
|
||||
for _, dc := range dcs {
|
||||
mc, ok := dc.(mockColl)
|
||||
assert.True(t, ok)
|
||||
|
||||
got = append(got, mc.id)
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expect, got, "expected collections are present")
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user