From ab0028bb0fa4e67278f254f5407f0820ad212b18 Mon Sep 17 00:00:00 2001 From: ryanfkeepers Date: Mon, 10 Jul 2023 11:07:39 -0600 Subject: [PATCH] setting aside temporarily --- src/internal/kopia/base_finder.go | 6 + src/internal/m365/controller_test.go | 4 +- src/internal/m365/helper_test.go | 34 -- src/internal/operations/test/backup_test.go | 462 ++++++++++++++++++ src/internal/operations/test/exchange_test.go | 330 ++++++------- src/internal/operations/test/helper_test.go | 451 ++--------------- src/internal/operations/test/onedrive_test.go | 208 ++++---- .../operations/test/sharepoint_test.go | 146 ++++-- src/pkg/backup/details/testdata/in_deets.go | 20 +- src/pkg/selectors/testdata/selectors.go | 44 ++ 10 files changed, 958 insertions(+), 747 deletions(-) create mode 100644 src/internal/operations/test/backup_test.go create mode 100644 src/pkg/selectors/testdata/selectors.go diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index 9ac651512..b1b86a708 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -2,6 +2,7 @@ package kopia import ( "context" + "fmt" "sort" "github.com/alcionai/clues" @@ -287,6 +288,11 @@ func (b *baseFinder) getBase( return nil, nil, nil, clues.Wrap(err, "getting snapshots") } + fmt.Printf("\n-----\nmetas %+v\n-----\n", len(metas)) + for _, m := range metas { + fmt.Println(m.ID, m.Labels) + } + // No snapshots means no backups so we can just exit here. if len(metas) == 0 { return nil, nil, nil, nil diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 6d04b7e9e..3dca2a4c7 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -428,7 +428,7 @@ func runRestore( start := time.Now() restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService) - restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true) + restoreSel := selTD.MakeSelector(t, sci.Service, sci.ResourceOwners, true) deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, backupVersion, @@ -997,7 +997,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { ctx, flush := tester.NewContext(t) defer flush() - restoreSel := getSelectorWith(t, test.service, []string{suite.user}, true) + restoreSel := selTD.MakeSelector(t, test.service, []string{suite.user}, true) expectedDests := make([]destAndCats, 0, len(test.collections)) allItems := 0 allExpectedData := map[string]map[string][]byte{} diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index 4befc991c..a79ecf0df 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -1157,40 +1157,6 @@ func backupSelectorForExpected( return selectors.Selector{} } -func getSelectorWith( - t *testing.T, - service path.ServiceType, - resourceOwners []string, - forRestore bool, -) selectors.Selector { - switch service { - case path.ExchangeService: - if forRestore { - return selectors.NewExchangeRestore(resourceOwners).Selector - } - - return selectors.NewExchangeBackup(resourceOwners).Selector - - case path.OneDriveService: - if forRestore { - return selectors.NewOneDriveRestore(resourceOwners).Selector - } - - return selectors.NewOneDriveBackup(resourceOwners).Selector - - case path.SharePointService: - if forRestore { - return selectors.NewSharePointRestore(resourceOwners).Selector - } - - return selectors.NewSharePointBackup(resourceOwners).Selector - - default: - require.FailNow(t, "unknown path service") - return selectors.Selector{} - } -} - func newController( ctx context.Context, t *testing.T, diff --git a/src/internal/operations/test/backup_test.go b/src/internal/operations/test/backup_test.go new file mode 100644 index 000000000..57a8a8834 --- /dev/null +++ b/src/internal/operations/test/backup_test.go @@ -0,0 +1,462 @@ +package test_test + +import ( + "context" + "fmt" + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/events" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/m365" + "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/storage" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" + "github.com/alcionai/corso/src/pkg/store" +) + +// --------------------------------------------------------------------------- +// singleton +// --------------------------------------------------------------------------- + +type backupInstance struct { + obo *operations.BackupOperation + bod *backupOpDependencies + // forms a linked list of incremental backups + incremental *backupInstance +} + +func (bi *backupInstance) close( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) { + if bi.incremental != nil { + bi.incremental.close(t, ctx) + } + + bi.bod.close(t, ctx) +} + +func (bi *backupInstance) runAndCheckBackup( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + mb *evmock.Bus, + acceptNoData bool, +) { + err := bi.obo.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, bi.obo.Results, "the backup had non-zero results") + require.NotEmpty(t, bi.obo.Results.BackupID, "the backup generated an ID") + + checkBackup(t, *bi.obo, mb, acceptNoData) +} + +func (bi *backupInstance) runAndCheckIncrementalBackup( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + mb *evmock.Bus, +) *backupInstance { + // bi.incremental = prepNewTestBackupOp(t, ctx, mb, bi.bod.sel, bi.obo.Options, bi.obo.BackupVersion) + incremental := &backupInstance{ + bod: &backupOpDependencies{}, + } + + // copy the old bod connection references + *incremental.bod = *bi.bod + + // generate a new controller to avoid statefulness + incremental.bod.renewController(t, ctx) + + incremental.obo = newTestBackupOp( + t, + ctx, + incremental.bod, + mb, + bi.obo.Options) + + err := bi.incremental.obo.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, bi.incremental.obo.Results, "the incremental backup had non-zero results") + require.NotEmpty(t, bi.incremental.obo.Results.BackupID, "the incremental backup generated an ID") + + return bi.incremental +} + +func checkBackup( + t *testing.T, + obo operations.BackupOperation, + mb *evmock.Bus, + acceptNoData bool, +) { + expectStatus := []operations.OpStatus{operations.Completed} + if acceptNoData { + expectStatus = append(expectStatus, operations.NoData) + } + + require.Contains( + t, + expectStatus, + obo.Status, + "backup doesn't match expectation, wanted any of %v, got %s", + expectStatus, + obo.Status) + + require.Less(t, 0, obo.Results.ItemsWritten) + assert.Less(t, 0, obo.Results.ItemsRead, "count of items read") + assert.Less(t, int64(0), obo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), obo.Results.BytesUploaded, "bytes uploaded") + assert.Equal(t, 1, obo.Results.ResourceOwners, "count of resource owners") + assert.NoErrorf( + t, + obo.Errors.Failure(), + "incremental non-recoverable error %+v", + clues.ToCore(obo.Errors.Failure())) + assert.Empty(t, obo.Errors.Recovered(), "incremental recoverable/iteration errors") + assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") + assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") + assert.Equal(t, + mb.CalledWith[events.BackupStart][0][events.BackupID], + obo.Results.BackupID, "incremental pre-run backupID event") +} + +func checkIncrementalBackup( + t *testing.T, + obo operations.BackupOperation, + mb *evmock.Bus, +) { + assert.NoError(t, obo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(obo.Errors.Failure())) + assert.Empty(t, obo.Errors.Recovered(), "incremental recoverable/iteration errors") + assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "incremental backup-start events") + assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "incremental backup-end events") + // FIXME: commented tests are flaky due to delta calls retaining data that is + // out of scope of the test data. + // we need to find a better way to make isolated assertions here. + // The addition of the deeTD package gives us enough coverage to comment + // out the tests for now and look to their improvemeng later. + + // do some additional checks to ensure the incremental dealt with fewer items. + // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. + // if !toggles.DisableDelta { + // assert.Equal(t, test.deltaItemsRead+4, incBO.Results.ItemsRead, "incremental items read") + // assert.Equal(t, test.deltaItemsWritten+4, incBO.Results.ItemsWritten, "incremental items written") + // } else { + // assert.Equal(t, test.nonDeltaItemsRead+4, incBO.Results.ItemsRead, "non delta items read") + // assert.Equal(t, test.nonDeltaItemsWritten+4, incBO.Results.ItemsWritten, "non delta items written") + // } + // assert.Equal(t, test.nonMetaItemsWritten, incBO.Results.ItemsWritten, "non meta incremental items write") + assert.Equal(t, + mb.CalledWith[events.BackupStart][0][events.BackupID], + obo.Results.BackupID, "incremental pre-run backupID event") +} + +// --------------------------------------------------------------------------- +// initialization and dependencies +// --------------------------------------------------------------------------- + +type backupOpDependencies struct { + acct account.Account + ctrl *m365.Controller + kms *kopia.ModelStore + kw *kopia.Wrapper + sel selectors.Selector + sss streamstore.Streamer + st storage.Storage + sw *store.Wrapper + + closer func() +} + +// prepNewTestBackupOp generates all clients required to run a backup operation, +// returning both a backup operation created with those clients, as well as +// the clients themselves. +func prepNewTestBackupOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + bus events.Eventer, + sel selectors.Selector, + opts control.Options, + backupVersion int, +) *backupInstance { + bod := &backupOpDependencies{ + acct: tconfig.NewM365Account(t), + st: storeTD.NewPrefixedS3Storage(t), + } + + k := kopia.NewConn(bod.st) + + err := k.Initialize(ctx, repository.Options{}) + require.NoError(t, err, clues.ToCore(err)) + + defer func() { + if err != nil { + bod.close(t, ctx) + t.FailNow() + } + }() + + // kopiaRef comes with a count of 1 and Wrapper bumps it again + // we're so safe to close here. + bod.closer = func() { + err := k.Close(ctx) + assert.NoErrorf(t, err, "k close: %+v", clues.ToCore(err)) + } + + bod.kw, err = kopia.NewWrapper(k) + require.NoError(t, err, clues.ToCore(err)) + + bod.kms, err = kopia.NewModelStore(k) + require.NoError(t, err, clues.ToCore(err)) + + bod.sw = store.NewKopiaStore(bod.kms) + + connectorResource := resource.Users + if sel.Service == selectors.ServiceSharePoint { + connectorResource = resource.Sites + } + + bod.ctrl, bod.sel = ControllerWithSelector( + t, + ctx, + bod.acct, + connectorResource, + sel, + nil, + bod.close) + + obo := newTestBackupOp( + t, + ctx, + bod, + bus, + opts) + + bod.sss = streamstore.NewStreamer( + bod.kw, + bod.acct.ID(), + bod.sel.PathService()) + + return &backupInstance{ + obo: obo, + bod: bod, + } +} + +func (bod *backupOpDependencies) close( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) { + bod.closer() + + if bod.kw != nil { + err := bod.kw.Close(ctx) + assert.NoErrorf(t, err, "kw close: %+v", clues.ToCore(err)) + } + + if bod.kms != nil { + err := bod.kw.Close(ctx) + assert.NoErrorf(t, err, "kms close: %+v", clues.ToCore(err)) + } +} + +// generates a new controller, and replaces bod.ctrl with that instance. +// useful for clearing controller state between runs. +func (bod *backupOpDependencies) renewController( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) { + rc := resource.Users + if bod.sel.PathService() == path.SharePointService { + rc = resource.Sites + } + + newCtrl, err := m365.NewController( + ctx, + bod.acct, + rc, + bod.sel.PathService(), + control.Defaults()) + require.NoError(t, err, clues.ToCore(err)) + + bod.ctrl = newCtrl +} + +// newTestBackupOp accepts the clients required to compose a backup operation, plus +// any other metadata, and uses them to generate a new backup operation. This +// allows backup chains to utilize the same temp directory and configuration +// details. +func newTestBackupOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + bod *backupOpDependencies, + bus events.Eventer, + opts control.Options, +) *operations.BackupOperation { + bod.ctrl.IDNameLookup = idname.NewCache(map[string]string{bod.sel.ID(): bod.sel.Name()}) + + bo, err := operations.NewBackupOperation( + ctx, + opts, + bod.kw, + bod.sw, + bod.ctrl, + bod.acct, + bod.sel, + bod.sel, + bus) + if !assert.NoError(t, err, clues.ToCore(err)) { + bod.close(t, ctx) + t.FailNow() + } + + return &bo +} + +func checkBackupIsInManifests( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + bod *backupOpDependencies, + bo *operations.BackupOperation, + sel selectors.Selector, + resourceOwner string, + categories ...path.CategoryType, +) { + for _, category := range categories { + t.Run("backup_in_manifests_"+category.String(), func(t *testing.T) { + var ( + reasons = []kopia.Reason{ + { + ResourceOwner: resourceOwner, + Service: sel.PathService(), + Category: category, + }, + } + tags = map[string]string{kopia.TagBackupCategory: ""} + found bool + ) + + bf, err := bod.kw.NewBaseFinder(bod.sw) + require.NoError(t, err, clues.ToCore(err)) + + fmt.Printf("\n-----\nR %+v\nT %+v\n-----\n", reasons, tags) + + mans := bf.FindBases(ctx, reasons, tags) + + mmb := mans.MergeBases() + require.NotEmpty(t, mmb, "should find at least one merge base") + + t.Log("Backup IDs from merge bases:") + + for _, man := range mmb { + bID, ok := man.GetTag(kopia.TagBackupID) + if !assert.Truef(t, ok, "snapshot manifest %s missing backup ID tag", man.ID) { + continue + } + + t.Log("-", bID) + + if bID == string(bo.Results.BackupID) { + found = true + break + } + } + + assert.True(t, found, "backup %q retrieved by previous snapshot manifest", bo.Results.BackupID) + }) + } +} + +func checkMetadataFilesExist( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupID model.StableID, + bod *backupOpDependencies, + tenant, resourceOwner string, + service path.ServiceType, + filesByCat map[path.CategoryType][]string, +) { + for category, files := range filesByCat { + t.Run("metadata_files_exist_"+category.String(), func(t *testing.T) { + bup := &backup.Backup{} + + err := bod.kms.Get(ctx, model.BackupSchema, backupID, bup) + if !assert.NoError(t, err, clues.ToCore(err)) { + return + } + + paths := []path.RestorePaths{} + pathsByRef := map[string][]string{} + + for _, fName := range files { + p, err := path.Builder{}. + Append(fName). + ToServiceCategoryMetadataPath(tenant, resourceOwner, service, category, true) + if !assert.NoError(t, err, "bad metadata path", clues.ToCore(err)) { + continue + } + + dir, err := p.Dir() + if !assert.NoError(t, err, "parent path", clues.ToCore(err)) { + continue + } + + paths = append( + paths, + path.RestorePaths{StoragePath: p, RestorePath: dir}) + pathsByRef[dir.ShortRef()] = append(pathsByRef[dir.ShortRef()], fName) + } + + cols, err := bod.kw.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + nil, + fault.New(true)) + assert.NoError(t, err, clues.ToCore(err)) + + for _, col := range cols { + itemNames := []string{} + + for item := range col.Items(ctx, fault.New(true)) { + assert.Implements(t, (*data.StreamSize)(nil), item) + + s := item.(data.StreamSize) + assert.Greaterf( + t, + s.Size(), + int64(0), + "empty metadata file: %s/%s", + col.FullPath(), + item.UUID(), + ) + + itemNames = append(itemNames, item.UUID()) + } + + assert.ElementsMatchf( + t, + pathsByRef[col.FullPath().ShortRef()], + itemNames, + "collection %s missing expected files", + col.FullPath(), + ) + } + }) + } +} diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index 1facd0f46..a1e2d1bda 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -8,7 +8,6 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" @@ -16,7 +15,6 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/m365/exchange" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" @@ -34,64 +32,81 @@ import ( storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) -type ExchangeBackupIntgSuite struct { +type ExchangeIntgSuite struct { tester.Suite its intgTesterSetup + // the goal of backupInstances is to run a single backup at the start of + // the suite, and re-use that backup throughout the rest of the suite. + bi *backupInstance } -func TestExchangeBackupIntgSuite(t *testing.T) { - suite.Run(t, &ExchangeBackupIntgSuite{ +func TestExchangeIntgSuite(t *testing.T) { + suite.Run(t, &ExchangeIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), }) } -func (suite *ExchangeBackupIntgSuite) SetupSuite() { - suite.its = newIntegrationTesterSetup(suite.T()) +func (suite *ExchangeIntgSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + suite.its = newIntegrationTesterSetup(t) + + sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel.Include( + sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + sel.DiscreteOwner = suite.its.userID + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + suite.bi = prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) + suite.bi.runAndCheckBackup(t, ctx, mb, false) +} + +func (suite *ExchangeIntgSuite) TeardownSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + if suite.bi != nil { + suite.bi.close(t, ctx) + } } // TestBackup_Run ensures that Integration Testing works // for the following scopes: Contacts, Events, and Mail -func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { +func (suite *ExchangeIntgSuite) TestBackup_Run_exchange() { tests := []struct { name string - selector func() *selectors.ExchangeBackup category path.CategoryType metadataFiles []string }{ { - name: "Mail", - selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) - sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - sel.DiscreteOwner = suite.its.userID - - return sel - }, + name: "Mail", category: path.EmailCategory, metadataFiles: exchange.MetadataFileNames(path.EmailCategory), }, - { - name: "Contacts", - selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) - sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())) - return sel - }, - category: path.ContactsCategory, - metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), - }, - { - name: "Calendar Events", - selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) - sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) - return sel - }, - category: path.EventsCategory, - metadataFiles: exchange.MetadataFileNames(path.EventsCategory), - }, + // { + // name: "Contacts", + // category: path.ContactsCategory, + // metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), + // }, + // { + // name: "Events", + // category: path.EventsCategory, + // metadataFiles: exchange.MetadataFileNames(path.EventsCategory), + // }, } for _, test := range tests { suite.Run(test.name, func() { @@ -101,50 +116,45 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { defer flush() var ( - mb = evmock.NewBus() - sel = test.selector().Selector - opts = control.Defaults() + bod = suite.bi.bod + sel = suite.bi.bod.sel + obo = suite.bi.obo + userID = suite.its.userID whatSet = deeTD.CategoryFromRepoRef ) - bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) - defer bod.close(t, ctx) + fmt.Printf("\n-----\n%s BUPs\n", test.name) + ibii := suite.bi + for ibii != nil { + fmt.Println(ibii.obo.Results.BackupID) + ibii = ibii.incremental + } - sel = bod.sel + fmt.Printf("-----\n") - userID := sel.ID() - - m365, err := bod.acct.M365Config() - require.NoError(t, err, clues.ToCore(err)) - - // run the tests - runAndCheckBackup(t, ctx, &bo, mb, false) checkBackupIsInManifests( t, ctx, - bod.kw, - bod.sw, - &bo, + bod, + obo, sel, userID, test.category) checkMetadataFilesExist( t, ctx, - bo.Results.BackupID, - bod.kw, - bod.kms, - m365.AzureTenantID, + obo.Results.BackupID, + bod, + suite.its.acct.ID(), userID, path.ExchangeService, map[path.CategoryType][]string{test.category: test.metadataFiles}) - _, expectDeets := deeTD.GetDeetsInBackup( t, ctx, - bo.Results.BackupID, + obo.Results.BackupID, bod.acct.ID(), - userID, + sel, path.ExchangeService, whatSet, bod.kms, @@ -152,82 +162,82 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { deeTD.CheckBackupDetails( t, ctx, - bo.Results.BackupID, + obo.Results.BackupID, whatSet, bod.kms, bod.sss, expectDeets, false) - - // Basic, happy path incremental test. No changes are dictated or expected. - // This only tests that an incremental backup is runnable at all, and that it - // produces fewer results than the last backup. - var ( - incMB = evmock.NewBus() - incBO = newTestBackupOp( - t, - ctx, - bod, - incMB, - opts) - ) - - runAndCheckBackup(t, ctx, &incBO, incMB, true) - checkBackupIsInManifests( - t, - ctx, - bod.kw, - bod.sw, - &incBO, - sel, - userID, - test.category) - checkMetadataFilesExist( - t, - ctx, - incBO.Results.BackupID, - bod.kw, - bod.kms, - m365.AzureTenantID, - userID, - path.ExchangeService, - map[path.CategoryType][]string{test.category: test.metadataFiles}) - deeTD.CheckBackupDetails( - t, - ctx, - incBO.Results.BackupID, - whatSet, - bod.kms, - bod.sss, - expectDeets, - false) - - // do some additional checks to ensure the incremental dealt with fewer items. - assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written") - assert.Greater(t, bo.Results.ItemsRead, incBO.Results.ItemsRead, "incremental items read") - assert.Greater(t, bo.Results.BytesRead, incBO.Results.BytesRead, "incremental bytes read") - assert.Greater(t, bo.Results.BytesUploaded, incBO.Results.BytesUploaded, "incremental bytes uploaded") - assert.Equal(t, bo.Results.ResourceOwners, incBO.Results.ResourceOwners, "incremental backup resource owner") - assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) - assert.Empty(t, incBO.Errors.Recovered(), "count incremental recoverable/iteration errors") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") - assert.Equal(t, - incMB.CalledWith[events.BackupStart][0][events.BackupID], - incBO.Results.BackupID, "incremental backupID pre-declaration") }) } + + // // Basic, happy path incremental test. No changes are dictated or expected. + // // This only tests that an incremental backup is runnable at all, and that it + // // produces fewer results than the last backup. + // var ( + // incMB = evmock.NewBus() + // incBO = newTestBackupOp( + // t, + // ctx, + // bod, + // incMB, + // opts) + // ) + + // runAndCheckBackup(t, ctx, &incBO, incMB, true) + // checkBackupIsInManifests( + // t, + // ctx, + // bod.kw, + // bod.sw, + // &incBO, + // sel, + // userID, + // test.category) + // checkMetadataFilesExist( + // t, + // ctx, + // incBO.Results.BackupID, + // bod.kw, + // bod.kms, + // m365.AzureTenantID, + // userID, + // path.ExchangeService, + // map[path.CategoryType][]string{test.category: test.metadataFiles}) + // deeTD.CheckBackupDetails( + // t, + // ctx, + // incBO.Results.BackupID, + // whatSet, + // bod.kms, + // bod.sss, + // expectDeets, + // false) + + // // do some additional checks to ensure the incremental dealt with fewer items. + // assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written") + // assert.Greater(t, bo.Results.ItemsRead, incBO.Results.ItemsRead, "incremental items read") + // assert.Greater(t, bo.Results.BytesRead, incBO.Results.BytesRead, "incremental bytes read") + // assert.Greater(t, bo.Results.BytesUploaded, incBO.Results.BytesUploaded, "incremental bytes uploaded") + // assert.Equal(t, bo.Results.ResourceOwners, incBO.Results.ResourceOwners, "incremental backup resource owner") + // assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) + // assert.Empty(t, incBO.Errors.Recovered(), "count incremental recoverable/iteration errors") + // assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") + // assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") + // assert.Equal(t, + // incMB.CalledWith[events.BackupStart][0][events.BackupID], + // incBO.Results.BackupID, "incremental backupID pre-declaration") } -func (suite *ExchangeBackupIntgSuite) TestBackup_Run_incrementalExchange() { +func (suite *ExchangeIntgSuite) TestBackup_Run_incrementalExchange() { testExchangeContinuousBackups(suite, control.Toggles{}) } -func (suite *ExchangeBackupIntgSuite) TestBackup_Run_incrementalNonDeltaExchange() { +func (suite *ExchangeIntgSuite) TestBackup_Run_incrementalNonDeltaExchange() { testExchangeContinuousBackups(suite, control.Toggles{DisableDelta: true}) } -func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles control.Toggles) { +func testExchangeContinuousBackups(suite *ExchangeIntgSuite, toggles control.Toggles) { t := suite.T() ctx, flush := tester.NewContext(t) @@ -346,6 +356,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr // populate initial test data for category, gen := range dataset { for destName := range gen.dests { + rc := control.DefaultRestoreConfig("") + rc.Location = destName + deets := generateContainerOfItems( t, ctx, @@ -356,7 +369,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr creds.AzureTenantID, uidn.ID(), "", - destName, + rc, 2, version.Backup, gen.dbf) @@ -380,11 +393,10 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr } } - bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) - defer bod.close(t, ctx) - - // run the initial backup - runAndCheckBackup(t, ctx, &bo, mb, false) + // run the initial incremental backup + ibi := suite.bi.runAndCheckIncrementalBackup(t, ctx, mb) + obo := ibi.obo + bod := ibi.bod rrPfx, err := path.ServicePrefix(acct.ID(), uidn.ID(), service, path.EmailCategory) require.NoError(t, err, clues.ToCore(err)) @@ -394,9 +406,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr bupDeets, _ := deeTD.GetDeetsInBackup( t, ctx, - bo.Results.BackupID, + obo.Results.BackupID, acct.ID(), - uidn.ID(), + uidn, service, whatSet, bod.kms, @@ -467,7 +479,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr deeTD.CheckBackupDetails( t, ctx, - bo.Results.BackupID, + obo.Results.BackupID, whatSet, bod.kms, bod.sss, @@ -553,6 +565,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr name: "add a new folder", updateUserData: func(t *testing.T, ctx context.Context) { for category, gen := range dataset { + rc := control.DefaultRestoreConfig("") + rc.Location = container3 + deets := generateContainerOfItems( t, ctx, @@ -560,7 +575,10 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr service, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, - creds.AzureTenantID, suite.its.userID, "", container3, + creds.AzureTenantID, + suite.its.userID, + "", + rc, 2, version.Backup, gen.dbf) @@ -763,15 +781,17 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr for _, test := range table { suite.Run(test.name, func() { var ( - t = suite.T() - incMB = evmock.NewBus() - atid = creds.AzureTenantID + t = suite.T() + mb = evmock.NewBus() + atid = creds.AzureTenantID + ctx, flush = tester.WithContext(t, ctx) ) - ctx, flush := tester.WithContext(t, ctx) defer flush() - incBO := newTestBackupOp(t, ctx, bod, incMB, opts) + ibi = ibi.runAndCheckIncrementalBackup(t, ctx, mb) + obo := ibi.obo + bod := ibi.bod suite.Run("PreTestSetup", func() { t := suite.T() @@ -782,17 +802,16 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr test.updateUserData(t, ctx) }) - err := incBO.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) + bupID := obo.Results.BackupID - bupID := incBO.Results.BackupID + err := obo.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) checkBackupIsInManifests( t, ctx, - bod.kw, - bod.sw, - &incBO, + bod, + obo, sels, uidn.ID(), maps.Keys(categories)...) @@ -800,8 +819,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr t, ctx, bupID, - bod.kw, - bod.kms, + bod, atid, uidn.ID(), service, @@ -815,30 +833,6 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr bod.sss, expectDeets, true) - - // FIXME: commented tests are flaky due to delta calls retaining data that is - // out of scope of the test data. - // we need to find a better way to make isolated assertions here. - // The addition of the deeTD package gives us enough coverage to comment - // out the tests for now and look to their improvemeng later. - - // do some additional checks to ensure the incremental dealt with fewer items. - // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. - // if !toggles.DisableDelta { - // assert.Equal(t, test.deltaItemsRead+4, incBO.Results.ItemsRead, "incremental items read") - // assert.Equal(t, test.deltaItemsWritten+4, incBO.Results.ItemsWritten, "incremental items written") - // } else { - // assert.Equal(t, test.nonDeltaItemsRead+4, incBO.Results.ItemsRead, "non delta items read") - // assert.Equal(t, test.nonDeltaItemsWritten+4, incBO.Results.ItemsWritten, "non delta items written") - // } - // assert.Equal(t, test.nonMetaItemsWritten, incBO.Results.ItemsWritten, "non meta incremental items write") - assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(incBO.Errors.Failure())) - assert.Empty(t, incBO.Errors.Recovered(), "incremental recoverable/iteration errors") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") - assert.Equal(t, - incMB.CalledWith[events.BackupStart][0][events.BackupID], - bupID, "incremental backupID pre-declaration") }) } } diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 009713bda..843a594d8 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -2,7 +2,6 @@ package test_test import ( "context" - "fmt" "testing" "time" @@ -15,24 +14,16 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/events" - evmock "github.com/alcionai/corso/src/internal/events/mock" - "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/model" - "github.com/alcionai/corso/src/internal/operations" - "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/extensions" "github.com/alcionai/corso/src/pkg/fault" @@ -40,9 +31,6 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/mock" - "github.com/alcionai/corso/src/pkg/storage" - storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" - "github.com/alcionai/corso/src/pkg/store" ) // Does not use the tester.DefaultTestRestoreDestination syntax as some of these @@ -50,308 +38,69 @@ import ( // they get clearly selected without accidental overlap. const incrementalsDestContainerPrefix = "incrementals_ci_" -type backupOpDependencies struct { - acct account.Account - ctrl *m365.Controller - kms *kopia.ModelStore - kw *kopia.Wrapper - sel selectors.Selector - sss streamstore.Streamer - st storage.Storage - sw *store.Wrapper +// --------------------------------------------------------------------------- +// Suite Setup +// --------------------------------------------------------------------------- - closer func() +type intgTesterSetup struct { + ac api.Client + gockAC api.Client + acct account.Account + userID string + userDriveID string + userDriveRootFolderID string + siteID string + siteDriveID string + siteDriveRootFolderID string } -func (bod *backupOpDependencies) close( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument -) { - bod.closer() +func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { + its := intgTesterSetup{} - if bod.kw != nil { - err := bod.kw.Close(ctx) - assert.NoErrorf(t, err, "kw close: %+v", clues.ToCore(err)) - } + ctx, flush := tester.NewContext(t) + defer flush() - if bod.kms != nil { - err := bod.kw.Close(ctx) - assert.NoErrorf(t, err, "kms close: %+v", clues.ToCore(err)) - } -} + graph.InitializeConcurrencyLimiter(ctx, true, 4) -// prepNewTestBackupOp generates all clients required to run a backup operation, -// returning both a backup operation created with those clients, as well as -// the clients themselves. -func prepNewTestBackupOp( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument - bus events.Eventer, - sel selectors.Selector, - opts control.Options, - backupVersion int, -) ( - operations.BackupOperation, - *backupOpDependencies, -) { - bod := &backupOpDependencies{ - acct: tconfig.NewM365Account(t), - st: storeTD.NewPrefixedS3Storage(t), - } - - k := kopia.NewConn(bod.st) - - err := k.Initialize(ctx, repository.Options{}) + its.acct = tconfig.NewM365Account(t) + creds, err := its.acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - defer func() { - if err != nil { - bod.close(t, ctx) - t.FailNow() - } - }() - - // kopiaRef comes with a count of 1 and Wrapper bumps it again - // we're so safe to close here. - bod.closer = func() { - err := k.Close(ctx) - assert.NoErrorf(t, err, "k close: %+v", clues.ToCore(err)) - } - - bod.kw, err = kopia.NewWrapper(k) - if !assert.NoError(t, err, clues.ToCore(err)) { - return operations.BackupOperation{}, nil - } - - bod.kms, err = kopia.NewModelStore(k) - if !assert.NoError(t, err, clues.ToCore(err)) { - return operations.BackupOperation{}, nil - } - - bod.sw = store.NewKopiaStore(bod.kms) - - connectorResource := resource.Users - if sel.Service == selectors.ServiceSharePoint { - connectorResource = resource.Sites - } - - bod.ctrl, bod.sel = ControllerWithSelector( - t, - ctx, - bod.acct, - connectorResource, - sel, - nil, - bod.close) - - bo := newTestBackupOp( - t, - ctx, - bod, - bus, - opts) - - bod.sss = streamstore.NewStreamer( - bod.kw, - bod.acct.ID(), - bod.sel.PathService()) - - return bo, bod -} - -// newTestBackupOp accepts the clients required to compose a backup operation, plus -// any other metadata, and uses them to generate a new backup operation. This -// allows backup chains to utilize the same temp directory and configuration -// details. -func newTestBackupOp( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument - bod *backupOpDependencies, - bus events.Eventer, - opts control.Options, -) operations.BackupOperation { - bod.ctrl.IDNameLookup = idname.NewCache(map[string]string{bod.sel.ID(): bod.sel.Name()}) - - bo, err := operations.NewBackupOperation( - ctx, - opts, - bod.kw, - bod.sw, - bod.ctrl, - bod.acct, - bod.sel, - bod.sel, - bus) - if !assert.NoError(t, err, clues.ToCore(err)) { - bod.close(t, ctx) - t.FailNow() - } - - return bo -} - -func runAndCheckBackup( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument - bo *operations.BackupOperation, - mb *evmock.Bus, - acceptNoData bool, -) { - err := bo.Run(ctx) + its.ac, err = api.NewClient(creds) require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results, "the backup had non-zero results") - require.NotEmpty(t, bo.Results.BackupID, "the backup generated an ID") - expectStatus := []operations.OpStatus{operations.Completed} - if acceptNoData { - expectStatus = append(expectStatus, operations.NoData) - } + its.gockAC, err = mock.NewClient(creds) + require.NoError(t, err, clues.ToCore(err)) - require.Contains( - t, - expectStatus, - bo.Status, - "backup doesn't match expectation, wanted any of %v, got %s", - expectStatus, - bo.Status) + // user drive - require.Less(t, 0, bo.Results.ItemsWritten) - assert.Less(t, 0, bo.Results.ItemsRead, "count of items read") - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") - assert.Equal(t, 1, bo.Results.ResourceOwners, "count of resource owners") - assert.NoError(t, bo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) - assert.Empty(t, bo.Errors.Recovered(), "incremental recoverable/iteration errors") - assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") - assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") - assert.Equal(t, - mb.CalledWith[events.BackupStart][0][events.BackupID], - bo.Results.BackupID, "backupID pre-declaration") -} + its.userID = tconfig.M365UserID(t) -func checkBackupIsInManifests( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument - kw *kopia.Wrapper, - sw *store.Wrapper, - bo *operations.BackupOperation, - sel selectors.Selector, - resourceOwner string, - categories ...path.CategoryType, -) { - for _, category := range categories { - t.Run(category.String(), func(t *testing.T) { - var ( - reasons = []kopia.Reason{ - { - ResourceOwner: resourceOwner, - Service: sel.PathService(), - Category: category, - }, - } - tags = map[string]string{kopia.TagBackupCategory: ""} - found bool - ) + userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) + require.NoError(t, err, clues.ToCore(err)) - bf, err := kw.NewBaseFinder(sw) - require.NoError(t, err, clues.ToCore(err)) + its.userDriveID = ptr.Val(userDrive.GetId()) - mans := bf.FindBases(ctx, reasons, tags) - for _, man := range mans.MergeBases() { - bID, ok := man.GetTag(kopia.TagBackupID) - if !assert.Truef(t, ok, "snapshot manifest %s missing backup ID tag", man.ID) { - continue - } + userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID) + require.NoError(t, err, clues.ToCore(err)) - if bID == string(bo.Results.BackupID) { - found = true - break - } - } + its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId()) - assert.True(t, found, "backup retrieved by previous snapshot manifest") - }) - } -} + its.siteID = tconfig.M365SiteID(t) -func checkMetadataFilesExist( - t *testing.T, - ctx context.Context, //revive:disable-line:context-as-argument - backupID model.StableID, - kw *kopia.Wrapper, - ms *kopia.ModelStore, - tenant, resourceOwner string, - service path.ServiceType, - filesByCat map[path.CategoryType][]string, -) { - for category, files := range filesByCat { - t.Run(category.String(), func(t *testing.T) { - bup := &backup.Backup{} + // site - err := ms.Get(ctx, model.BackupSchema, backupID, bup) - if !assert.NoError(t, err, clues.ToCore(err)) { - return - } + siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) + require.NoError(t, err, clues.ToCore(err)) - paths := []path.RestorePaths{} - pathsByRef := map[string][]string{} + its.siteDriveID = ptr.Val(siteDrive.GetId()) - for _, fName := range files { - p, err := path.Builder{}. - Append(fName). - ToServiceCategoryMetadataPath(tenant, resourceOwner, service, category, true) - if !assert.NoError(t, err, "bad metadata path", clues.ToCore(err)) { - continue - } + siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID) + require.NoError(t, err, clues.ToCore(err)) - dir, err := p.Dir() - if !assert.NoError(t, err, "parent path", clues.ToCore(err)) { - continue - } + its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) - paths = append( - paths, - path.RestorePaths{StoragePath: p, RestorePath: dir}) - pathsByRef[dir.ShortRef()] = append(pathsByRef[dir.ShortRef()], fName) - } - - cols, err := kw.ProduceRestoreCollections( - ctx, - bup.SnapshotID, - paths, - nil, - fault.New(true)) - assert.NoError(t, err, clues.ToCore(err)) - - for _, col := range cols { - itemNames := []string{} - - for item := range col.Items(ctx, fault.New(true)) { - assert.Implements(t, (*data.StreamSize)(nil), item) - - s := item.(data.StreamSize) - assert.Greaterf( - t, - s.Size(), - int64(0), - "empty metadata file: %s/%s", - col.FullPath(), - item.UUID(), - ) - - itemNames = append(itemNames, item.UUID()) - } - - assert.ElementsMatchf( - t, - pathsByRef[col.FullPath().ShortRef()], - itemNames, - "collection %s missing expected files", - col.FullPath(), - ) - } - }) - } + return its } // --------------------------------------------------------------------------- @@ -373,7 +122,8 @@ func generateContainerOfItems( service path.ServiceType, cat path.CategoryType, sel selectors.Selector, - tenantID, resourceOwner, driveID, destFldr string, + tenantID, resourceOwner, driveID string, + rc control.RestoreConfig, howManyItems int, backupVersion int, dbf dataBuilderFunc, @@ -391,11 +141,11 @@ func generateContainerOfItems( }) } - pathFolders := []string{destFldr} + pathFolders := []string{} switch service { case path.OneDriveService, path.SharePointService: - pathFolders = []string{odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, destFldr} + pathFolders = []string{odConsts.DrivesPathDir, driveID, odConsts.RootPathDir} } collections := []incrementalCollection{{ @@ -404,14 +154,10 @@ func generateContainerOfItems( items: items, }} - restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) - restoreCfg.Location = destFldr - dataColls := buildCollections( t, service, tenantID, resourceOwner, - restoreCfg, collections) opts := control.Defaults() @@ -421,7 +167,7 @@ func generateContainerOfItems( ctx, backupVersion, sel, - restoreCfg, + rc, opts, dataColls, fault.New(true), @@ -467,7 +213,6 @@ func buildCollections( t *testing.T, service path.ServiceType, tenant, user string, - restoreCfg control.RestoreConfig, colls []incrementalCollection, ) []data.RestoreCollection { t.Helper() @@ -475,14 +220,8 @@ func buildCollections( collections := make([]data.RestoreCollection, 0, len(colls)) for _, c := range colls { - pth := toDataLayerPath( - t, - service, - tenant, - user, - c.category, - c.pathFolders, - false) + pth, err := path.Build(tenant, user, service, c.category, false, c.pathFolders...) + require.NoError(t, err, clues.ToCore(err)) mc := exchMock.NewCollection(pth, pth, len(c.items)) @@ -497,38 +236,6 @@ func buildCollections( return collections } -func toDataLayerPath( - t *testing.T, - service path.ServiceType, - tenant, resourceOwner string, - category path.CategoryType, - elements []string, - isItem bool, -) path.Path { - t.Helper() - - var ( - pb = path.Builder{}.Append(elements...) - p path.Path - err error - ) - - switch service { - case path.ExchangeService: - p, err = pb.ToDataLayerExchangePathForCategory(tenant, resourceOwner, category, isItem) - case path.OneDriveService: - p, err = pb.ToDataLayerOneDrivePath(tenant, resourceOwner, isItem) - case path.SharePointService: - p, err = pb.ToDataLayerSharePointPath(tenant, resourceOwner, category, isItem) - default: - err = clues.New(fmt.Sprintf("unknown service: %s", service)) - } - - require.NoError(t, err, clues.ToCore(err)) - - return p -} - // A QoL builder for live instances that updates // the selector's owner id and name in the process // to help avoid gotchas. @@ -564,70 +271,6 @@ func ControllerWithSelector( return ctrl, sel } -// --------------------------------------------------------------------------- -// Suite Setup -// --------------------------------------------------------------------------- - -type intgTesterSetup struct { - ac api.Client - gockAC api.Client - userID string - userDriveID string - userDriveRootFolderID string - siteID string - siteDriveID string - siteDriveRootFolderID string -} - -func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { - its := intgTesterSetup{} - - ctx, flush := tester.NewContext(t) - defer flush() - - graph.InitializeConcurrencyLimiter(ctx, true, 4) - - a := tconfig.NewM365Account(t) - creds, err := a.M365Config() - require.NoError(t, err, clues.ToCore(err)) - - its.ac, err = api.NewClient(creds) - require.NoError(t, err, clues.ToCore(err)) - - its.gockAC, err = mock.NewClient(creds) - require.NoError(t, err, clues.ToCore(err)) - - // user drive - - its.userID = tconfig.M365UserID(t) - - userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveID = ptr.Val(userDrive.GetId()) - - userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId()) - - its.siteID = tconfig.M365SiteID(t) - - // site - - siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveID = ptr.Val(siteDrive.GetId()) - - siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) - - return its -} - func getTestExtensionFactories() []extensions.CreateItemExtensioner { return []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index 3c58f9ae4..63a07b9b0 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -43,6 +43,9 @@ import ( type OneDriveBackupIntgSuite struct { tester.Suite its intgTesterSetup + // the goal of backupInstances is to run a single backup at the start of + // the suite, and re-use that backup throughout the rest of the suite. + bi *backupInstance } func TestOneDriveBackupIntgSuite(t *testing.T) { @@ -54,7 +57,24 @@ func TestOneDriveBackupIntgSuite(t *testing.T) { } func (suite *OneDriveBackupIntgSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + suite.its = newIntegrationTesterSetup(suite.T()) + + sel := selectors.NewOneDriveBackup([]string{suite.its.siteID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.userID + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + suite.bi = prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) + suite.bi.runAndCheckBackup(t, ctx, mb, false) } func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { @@ -64,39 +84,37 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { defer flush() var ( - tenID = tconfig.M365TenantID(t) - mb = evmock.NewBus() - userID = tconfig.SecondaryM365UserID(t) - osel = selectors.NewOneDriveBackup([]string{userID}) - ws = deeTD.DriveIDFromRepoRef - svc = path.OneDriveService - opts = control.Defaults() + bod = suite.bi.bod + sel = suite.bi.bod.sel + obo = suite.bi.obo + siteID = suite.its.siteID + whatSet = deeTD.DriveIDFromRepoRef ) - osel.Include(selTD.OneDriveBackupFolderScope(osel)) - - bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup) - defer bod.close(t, ctx) - - runAndCheckBackup(t, ctx, &bo, mb, false) - - bID := bo.Results.BackupID + checkBackupIsInManifests( + t, + ctx, + bod, + obo, + sel, + siteID, + path.LibrariesCategory) _, expectDeets := deeTD.GetDeetsInBackup( t, ctx, - bID, - tenID, - bod.sel.ID(), - svc, - ws, + obo.Results.BackupID, + bod.acct.ID(), + sel, + path.OneDriveService, + whatSet, bod.kms, bod.sss) deeTD.CheckBackupDetails( t, ctx, - bID, - ws, + obo.Results.BackupID, + whatSet, bod.kms, bod.sss, expectDeets, @@ -135,6 +153,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { runDriveIncrementalTest( suite, + suite.bi, suite.its.userID, suite.its.userID, resource.Users, @@ -148,6 +167,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { func runDriveIncrementalTest( suite tester.Suite, + bi *backupInstance, owner, permissionsUser string, rc resource.Category, service path.ServiceType, @@ -164,7 +184,6 @@ func runDriveIncrementalTest( var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() mb = evmock.NewBus() ws = deeTD.DriveIDFromRepoRef @@ -223,6 +242,9 @@ func runDriveIncrementalTest( // through the changes. This should be enough to cover most delta // actions. for _, destName := range genDests { + rc := control.DefaultRestoreConfig("") + rc.Location = destName + deets := generateContainerOfItems( t, ctx, @@ -230,7 +252,10 @@ func runDriveIncrementalTest( service, category, sel, - atid, roidn.ID(), driveID, destName, + atid, + roidn.ID(), + driveID, + rc, 2, // Use an old backup version so we don't need metadata files. 0, @@ -260,20 +285,19 @@ func runDriveIncrementalTest( containerIDs[destName] = ptr.Val(resp.GetId()) } - bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) - defer bod.close(t, ctx) + // run the initial incremental backup + ibi := bi.runAndCheckIncrementalBackup(t, ctx, mb) + obo := ibi.obo + bod := ibi.bod sel = bod.sel - // run the initial backup - runAndCheckBackup(t, ctx, &bo, mb, false) - // precheck to ensure the expectedDeets are correct. // if we fail here, the expectedDeets were populated incorrectly. deeTD.CheckBackupDetails( t, ctx, - bo.Results.BackupID, + obo.Results.BackupID, ws, bod.kms, bod.sss, @@ -568,6 +592,9 @@ func runDriveIncrementalTest( { name: "add a new folder", updateFiles: func(t *testing.T, ctx context.Context) { + rc := control.DefaultRestoreConfig("") + rc.Location = container3 + generateContainerOfItems( t, ctx, @@ -575,7 +602,10 @@ func runDriveIncrementalTest( service, category, sel, - atid, roidn.ID(), driveID, container3, + atid, + roidn.ID(), + driveID, + rc, 2, 0, fileDBF) @@ -600,25 +630,23 @@ func runDriveIncrementalTest( } for _, test := range table { suite.Run(test.name, func() { - cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) - require.NoError(t, err, clues.ToCore(err)) + // cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) + // require.NoError(t, err, clues.ToCore(err)) - bod.ctrl = cleanCtrl + // bod.ctrl = cleanCtrl var ( - t = suite.T() - incMB = evmock.NewBus() - incBO = newTestBackupOp( - t, - ctx, - bod, - incMB, - opts) + t = suite.T() + mb = evmock.NewBus() ) ctx, flush := tester.WithContext(t, ctx) defer flush() + ibi = ibi.runAndCheckIncrementalBackup(t, ctx, mb) + obo := ibi.obo + bod := ibi.bod + suite.Run("PreTestSetup", func() { t := suite.T() @@ -628,17 +656,16 @@ func runDriveIncrementalTest( test.updateFiles(t, ctx) }) - err = incBO.Run(ctx) + err = obo.Run(ctx) require.NoError(t, err, clues.ToCore(err)) - bupID := incBO.Results.BackupID + bupID := obo.Results.BackupID checkBackupIsInManifests( t, ctx, - bod.kw, - bod.sw, - &incBO, + bod, + obo, sel, roidn.ID(), maps.Keys(categories)...) @@ -646,8 +673,7 @@ func runDriveIncrementalTest( t, ctx, bupID, - bod.kw, - bod.kms, + bod, atid, roidn.ID(), service, @@ -679,17 +705,9 @@ func runDriveIncrementalTest( assertReadWrite = assert.LessOrEqual } - assertReadWrite(t, expectWrites, incBO.Results.ItemsWritten, "incremental items written") - assertReadWrite(t, expectNonMetaWrites, incBO.Results.NonMetaItemsWritten, "incremental non-meta items written") - assertReadWrite(t, expectReads, incBO.Results.ItemsRead, "incremental items read") - - assert.NoError(t, incBO.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(incBO.Errors.Failure())) - assert.Empty(t, incBO.Errors.Recovered(), "incremental recoverable/iteration errors") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupStart], "incremental backup-start events") - assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") - assert.Equal(t, - incMB.CalledWith[events.BackupStart][0][events.BackupID], - bupID, "incremental backupID pre-declaration") + assertReadWrite(t, expectWrites, obo.Results.ItemsWritten, "incremental items written") + assertReadWrite(t, expectNonMetaWrites, obo.Results.NonMetaItemsWritten, "incremental non-meta items written") + assertReadWrite(t, expectReads, obo.Results.ItemsRead, "incremental items read") }) } } @@ -730,26 +748,30 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { oldsel := selectors.NewOneDriveBackup([]string{uname}) oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel)) - bo, bod := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, opts, 0) - defer bod.close(t, ctx) + // don't re-use the suite.bi for this case because we need + // to control for the backup version. + bi := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, opts, 0) + defer bi.close(t, ctx) + obo := bi.obo + bod := bi.bod sel := bod.sel // ensure the initial owner uses name in both cases - bo.ResourceOwner = sel.SetDiscreteOwnerIDName(uname, uname) + obo.ResourceOwner = sel.SetDiscreteOwnerIDName(uname, uname) // required, otherwise we don't run the migration - bo.BackupVersion = version.All8MigrateUserPNToID - 1 + obo.BackupVersion = version.All8MigrateUserPNToID - 1 require.Equalf( t, - bo.ResourceOwner.Name(), - bo.ResourceOwner.ID(), + obo.ResourceOwner.Name(), + obo.ResourceOwner.ID(), "historical representation of user id [%s] should match pn [%s]", - bo.ResourceOwner.ID(), - bo.ResourceOwner.Name()) + obo.ResourceOwner.ID(), + obo.ResourceOwner.Name()) // run the initial backup - runAndCheckBackup(t, ctx, &bo, mb, false) + bi.runAndCheckBackup(t, ctx, mb, false) newsel := selectors.NewOneDriveBackup([]string{uid}) newsel.Include(selTD.OneDriveBackupFolderScope(newsel)) @@ -758,7 +780,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { var ( incMB = evmock.NewBus() // the incremental backup op should have a proper user ID for the id. - incBO = newTestBackupOp(t, ctx, bod, incMB, opts) + incBO = newTestBackupOp(t, ctx, bi.bod, incMB, opts) ) require.NotEqualf( @@ -774,9 +796,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { checkBackupIsInManifests( t, ctx, - bod.kw, - bod.sw, - &incBO, + bod, + obo, sel, uid, maps.Keys(categories)...) @@ -784,8 +805,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { t, ctx, incBO.Results.BackupID, - bod.kw, - bod.kms, + bod, creds.AzureTenantID, uid, path.OneDriveService, @@ -806,13 +826,13 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { bid := incBO.Results.BackupID bup := &backup.Backup{} - err = bod.kms.Get(ctx, model.BackupSchema, bid, bup) + err = bi.bod.kms.Get(ctx, model.BackupSchema, bid, bup) require.NoError(t, err, clues.ToCore(err)) var ( ssid = bup.StreamStoreID deets details.Details - ss = streamstore.NewStreamer(bod.kw, creds.AzureTenantID, path.OneDriveService) + ss = streamstore.NewStreamer(bi.bod.kw, creds.AzureTenantID, path.OneDriveService) ) err = ss.Read(ctx, ssid, streamstore.DetailsReader(details.UnmarshalTo(&deets)), fault.New(true)) @@ -836,7 +856,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { tenID = tconfig.M365TenantID(t) mb = evmock.NewBus() userID = tconfig.SecondaryM365UserID(t) - osel = selectors.NewOneDriveBackup([]string{userID}) + sel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService opts = control.Defaults() @@ -844,32 +864,44 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { opts.ItemExtensionFactory = getTestExtensionFactories() - osel.Include(selTD.OneDriveBackupFolderScope(osel)) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) - bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup) - defer bod.close(t, ctx) + // TODO: use the existing backupInstance for this test + bi := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) + defer bi.bod.close(t, ctx) - runAndCheckBackup(t, ctx, &bo, mb, false) + bi.runAndCheckBackup(t, ctx, mb, false) - bID := bo.Results.BackupID + bod := bi.bod + obo := bi.obo + bID := obo.Results.BackupID + + checkBackupIsInManifests( + t, + ctx, + bod, + obo, + bod.sel, + suite.its.siteID, + path.LibrariesCategory) deets, expectDeets := deeTD.GetDeetsInBackup( t, ctx, bID, tenID, - bod.sel.ID(), + bod.sel, svc, ws, - bod.kms, - bod.sss) + bi.bod.kms, + bi.bod.sss) deeTD.CheckBackupDetails( t, ctx, bID, ws, - bod.kms, - bod.sss, + bi.bod.kms, + bi.bod.sss, expectDeets, false) diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index a3221f937..82f699933 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -26,24 +26,99 @@ import ( storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) -type SharePointBackupIntgSuite struct { +type SharePointIntgSuite struct { tester.Suite its intgTesterSetup + // the goal of backupInstances is to run a single backup at the start of + // the suite, and re-use that backup throughout the rest of the suite. + bi *backupInstance } -func TestSharePointBackupIntgSuite(t *testing.T) { - suite.Run(t, &SharePointBackupIntgSuite{ +func TestSharePointIntgSuite(t *testing.T) { + suite.Run(t, &SharePointIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), }) } -func (suite *SharePointBackupIntgSuite) SetupSuite() { +func (suite *SharePointIntgSuite) SetupSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + suite.its = newIntegrationTesterSetup(suite.T()) + + sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.siteID + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + suite.bi = prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) + suite.bi.runAndCheckBackup(t, ctx, mb, false) } -func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { +func (suite *SharePointIntgSuite) TeardownSuite() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + if suite.bi != nil { + suite.bi.close(t, ctx) + } +} + +func (suite *SharePointIntgSuite) TestBackup_Run_sharePoint() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + bod = suite.bi.bod + sel = suite.bi.bod.sel + obo = suite.bi.obo + siteID = suite.its.siteID + whatSet = deeTD.DriveIDFromRepoRef + ) + + checkBackupIsInManifests( + t, + ctx, + bod, + obo, + sel, + siteID, + path.LibrariesCategory) + + _, expectDeets := deeTD.GetDeetsInBackup( + t, + ctx, + obo.Results.BackupID, + bod.acct.ID(), + sel, + path.SharePointService, + whatSet, + bod.kms, + bod.sss) + deeTD.CheckBackupDetails( + t, + ctx, + obo.Results.BackupID, + whatSet, + bod.kms, + bod.sss, + expectDeets, + false) +} + +func (suite *SharePointIntgSuite) TestBackup_Run_incrementalSharePoint() { sel := selectors.NewSharePointRestore([]string{suite.its.siteID}) ic := func(cs []string) selectors.Selector { @@ -75,6 +150,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { runDriveIncrementalTest( suite, + suite.bi, suite.its.siteID, suite.its.userID, resource.Sites, @@ -86,36 +162,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { true) } -func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() - ) - - sel.Include(selTD.SharePointBackupFolderScope(sel)) - - bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) - defer bod.close(t, ctx) - - runAndCheckBackup(t, ctx, &bo, mb, false) - checkBackupIsInManifests( - t, - ctx, - bod.kw, - bod.sw, - &bo, - bod.sel, - suite.its.siteID, - path.LibrariesCategory) -} - -func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { +func (suite *SharePointIntgSuite) TestBackup_Run_sharePointExtensions() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -134,39 +181,42 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { sel.Include(selTD.SharePointBackupFolderScope(sel)) - bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) - defer bod.close(t, ctx) + // TODO: use the existing backupInstance for this test + bi := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) + defer bi.bod.close(t, ctx) + + bi.runAndCheckBackup(t, ctx, mb, false) + + bod := bi.bod + obo := bi.obo + bID := obo.Results.BackupID - runAndCheckBackup(t, ctx, &bo, mb, false) checkBackupIsInManifests( t, ctx, - bod.kw, - bod.sw, - &bo, + bod, + obo, bod.sel, suite.its.siteID, path.LibrariesCategory) - bID := bo.Results.BackupID - deets, expectDeets := deeTD.GetDeetsInBackup( t, ctx, bID, tenID, - bod.sel.ID(), + bod.sel, svc, ws, - bod.kms, - bod.sss) + bi.bod.kms, + bi.bod.sss) deeTD.CheckBackupDetails( t, ctx, bID, ws, - bod.kms, - bod.sss, + bi.bod.kms, + bi.bod.sss, expectDeets, false) diff --git a/src/pkg/backup/details/testdata/in_deets.go b/src/pkg/backup/details/testdata/in_deets.go index b15c50f17..11fd5e53c 100644 --- a/src/pkg/backup/details/testdata/in_deets.go +++ b/src/pkg/backup/details/testdata/in_deets.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/exp/maps" + "github.com/alcionai/corso/src/internal/common/idname" + idnMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/streamstore" @@ -293,7 +295,16 @@ func CheckBackupDetails( // of data. mustEqualFolders bool, ) { - deets, result := GetDeetsInBackup(t, ctx, backupID, "", "", path.UnknownService, ws, ms, ssr) + deets, result := GetDeetsInBackup( + t, + ctx, + backupID, + "", + idnMock.NewProvider("", ""), + path.UnknownService, + ws, + ms, + ssr) t.Log("details entries in result") @@ -339,7 +350,8 @@ func GetDeetsInBackup( t *testing.T, ctx context.Context, //revive:disable-line:context-as-argument backupID model.StableID, - tid, resourceOwner string, + tid string, + protectedResource idname.Provider, service path.ServiceType, ws whatSet, ms *kopia.ModelStore, @@ -361,7 +373,9 @@ func GetDeetsInBackup( fault.New(true)) require.NoError(t, err, clues.ToCore(err)) - id := NewInDeets(path.Builder{}.Append(tid, service.String(), resourceOwner).String()) + pb := path.Builder{}.Append(tid, service.String(), protectedResource.ID()) + + id := NewInDeets(pb.String()) id.AddAll(deets, ws) return deets, id diff --git a/src/pkg/selectors/testdata/selectors.go b/src/pkg/selectors/testdata/selectors.go new file mode 100644 index 000000000..1213adb85 --- /dev/null +++ b/src/pkg/selectors/testdata/selectors.go @@ -0,0 +1,44 @@ +package testdata + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" +) + +func MakeSelector( + t *testing.T, + service path.ServiceType, + resourceOwners []string, + forRestore bool, +) selectors.Selector { + switch service { + case path.ExchangeService: + if forRestore { + return selectors.NewExchangeRestore(resourceOwners).Selector + } + + return selectors.NewExchangeBackup(resourceOwners).Selector + + case path.OneDriveService: + if forRestore { + return selectors.NewOneDriveRestore(resourceOwners).Selector + } + + return selectors.NewOneDriveBackup(resourceOwners).Selector + + case path.SharePointService: + if forRestore { + return selectors.NewSharePointRestore(resourceOwners).Selector + } + + return selectors.NewSharePointBackup(resourceOwners).Selector + + default: + require.FailNow(t, "unknown path service") + return selectors.Selector{} + } +}