From 0e6fce7d1e8037997c293665d5cfa9df900ff27a Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 28 Dec 2022 13:23:07 -0700 Subject: [PATCH] another cleanup refinement (#1971) ## Description A little code consolidation in ops/backup integration tests, to prepare testing incrementals. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :robot: Test ## Issue(s) * #1966 ## Test Plan - [x] :green_heart: E2E --- src/internal/events/mock/mock.go | 12 +- .../operations/backup_integration_test.go | 216 +++++++++--------- 2 files changed, 110 insertions(+), 118 deletions(-) diff --git a/src/internal/events/mock/mock.go b/src/internal/events/mock/mock.go index 076b5ac57..0c72748a7 100644 --- a/src/internal/events/mock/mock.go +++ b/src/internal/events/mock/mock.go @@ -6,20 +6,20 @@ import ( "github.com/pkg/errors" ) -type mockBus struct { +type Bus struct { TimesCalled map[string]int CalledWith map[string][]map[string]any TimesClosed int } -func NewBus() *mockBus { - return &mockBus{ +func NewBus() *Bus { + return &Bus{ TimesCalled: map[string]int{}, CalledWith: map[string][]map[string]any{}, } } -func (b *mockBus) Event(ctx context.Context, key string, data map[string]any) { +func (b *Bus) Event(ctx context.Context, key string, data map[string]any) { b.TimesCalled[key] = b.TimesCalled[key] + 1 cw := b.CalledWith[key] @@ -31,11 +31,11 @@ func (b *mockBus) Event(ctx context.Context, key string, data map[string]any) { b.CalledWith[key] = cw } -func (b *mockBus) Close() error { +func (b *Bus) Close() error { b.TimesClosed++ if b.TimesClosed > 1 { - return errors.New("multiple closes on mockBus") + return errors.New("multiple closes on Bus") } return nil diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 0d60d8447..c9fea5efd 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -88,6 +88,74 @@ func prepNewBackupOp( return bo, acct, kw, ms, closer } +//revive:disable:context-as-argument +func runAndCheckBackup( + t *testing.T, + ctx context.Context, + bo *BackupOperation, + mb *evmock.Bus, +) { + //revive:enable:context-as-argument + require.NoError(t, bo.Run(ctx)) + require.NotEmpty(t, bo.Results) + require.NotEmpty(t, bo.Results.BackupID) + require.Equalf( + t, + Completed, + bo.Status, + "backup status %s is not Completed", + bo.Status, + ) + require.Less(t, 0, bo.Results.ItemsWritten) + + assert.Less(t, 0, bo.Results.ItemsRead) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") + assert.Equal(t, 1, bo.Results.ResourceOwners) + assert.NoError(t, bo.Results.ReadErrors) + assert.NoError(t, bo.Results.WriteErrors) + assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") + assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") + assert.Equal(t, + mb.CalledWith[events.BackupStart][0][events.BackupID], + bo.Results.BackupID, "backupID pre-declaration") +} + +//revive:disable:context-as-argument +func checkBackupIsInManifests( + t *testing.T, + ctx context.Context, + kw *kopia.Wrapper, + bo *BackupOperation, + sel selectors.Selector, + resourceOwner string, + category path.CategoryType, +) { + //revive:enable:context-as-argument + var ( + sck, scv = kopia.MakeServiceCat(sel.PathService(), category) + oc = &kopia.OwnersCats{ + ResourceOwners: map[string]struct{}{resourceOwner: {}}, + ServiceCats: map[string]kopia.ServiceCat{sck: scv}, + } + tags = map[string]string{kopia.TagBackupCategory: ""} + found bool + ) + + mans, err := kw.FetchPrevSnapshotManifests(ctx, oc, tags) + require.NoError(t, err) + + for _, man := range mans { + tk, _ := kopia.MakeTagKV(kopia.TagBackupID) + if man.Tags[tk] == string(bo.Results.BackupID) { + found = true + break + } + } + + assert.True(t, found, "backup retrieved by previous snapshot manifest") +} + //revive:disable:context-as-argument func checkMetadataFilesExist( t *testing.T, @@ -166,6 +234,7 @@ func checkMetadataFilesExist( type BackupOpIntegrationSuite struct { suite.Suite + user, site string } func TestBackupOpIntegrationSuite(t *testing.T) { @@ -185,6 +254,9 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() { tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs) require.NoError(suite.T(), err) + + suite.user = tester.M365UserID(suite.T()) + suite.site = tester.M365SiteID(suite.T()) } func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { @@ -229,30 +301,31 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { ctx, flush := tester.NewContext() defer flush() - m365UserID := tester.M365UserID(suite.T()) - users := []string{m365UserID} + users := []string{suite.user} tests := []struct { - name string - selectFunc func() *selectors.ExchangeBackup - resourceOwner string - category path.CategoryType - metadataFiles []string + name string + selector func() *selectors.ExchangeBackup + resourceOwner string + category path.CategoryType + metadataFiles []string + runIncremental bool }{ { name: "Mail", - selectFunc: func() *selectors.ExchangeBackup { + selector: func() *selectors.ExchangeBackup { sel := selectors.NewExchangeBackup(users) sel.Include(sel.MailFolders(users, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) return sel }, - resourceOwner: m365UserID, - category: path.EmailCategory, - metadataFiles: exchange.MetadataFileNames(path.EmailCategory), + resourceOwner: suite.user, + category: path.EmailCategory, + metadataFiles: exchange.MetadataFileNames(path.EmailCategory), + runIncremental: true, }, { name: "Contacts", - selectFunc: func() *selectors.ExchangeBackup { + selector: func() *selectors.ExchangeBackup { sel := selectors.NewExchangeBackup(users) sel.Include(sel.ContactFolders( users, @@ -261,88 +334,36 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { return sel }, - resourceOwner: m365UserID, - category: path.ContactsCategory, - metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), + resourceOwner: suite.user, + category: path.ContactsCategory, + metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), + runIncremental: true, }, { name: "Calendar Events", - selectFunc: func() *selectors.ExchangeBackup { + selector: func() *selectors.ExchangeBackup { sel := selectors.NewExchangeBackup(users) sel.Include(sel.EventCalendars(users, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) return sel }, - resourceOwner: m365UserID, + resourceOwner: suite.user, category: path.EventsCategory, metadataFiles: exchange.MetadataFileNames(path.EventsCategory), }, } for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - mb := evmock.NewBus() - sel := test.selectFunc() - bo, acct, kw, ms, closer := prepNewBackupOp(t, ctx, mb, sel.Selector) - defer closer() - - failed := false - - require.NoError(t, bo.Run(ctx)) - require.NotEmpty(t, bo.Results) - require.NotEmpty(t, bo.Results.BackupID) - - if !assert.Equalf( - t, - Completed, - bo.Status, - "backup status %s is not Completed", - bo.Status, - ) { - failed = true - } - - if !assert.Less(t, 0, bo.Results.ItemsWritten) { - failed = true - } - - assert.Less(t, 0, bo.Results.ItemsRead) - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") - assert.Equal(t, 1, bo.Results.ResourceOwners) - assert.NoError(t, bo.Results.ReadErrors) - assert.NoError(t, bo.Results.WriteErrors) - assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") - assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") - assert.Equal(t, - mb.CalledWith[events.BackupStart][0][events.BackupID], - bo.Results.BackupID, "backupID pre-declaration") - - // verify that we can find the new backup id in the manifests var ( - sck, scv = kopia.MakeServiceCat(sel.PathService(), test.category) - oc = &kopia.OwnersCats{ - ResourceOwners: map[string]struct{}{test.resourceOwner: {}}, - ServiceCats: map[string]kopia.ServiceCat{sck: scv}, - } - tags = map[string]string{kopia.TagBackupCategory: ""} - found bool + mb = evmock.NewBus() + sel = test.selector() + bo, acct, kw, ms, closer = prepNewBackupOp(t, ctx, mb, sel.Selector) ) - mans, err := kw.FetchPrevSnapshotManifests(ctx, oc, tags) - assert.NoError(t, err) + defer closer() - for _, man := range mans { - tk, _ := kopia.MakeTagKV(kopia.TagBackupID) - if man.Tags[tk] == string(bo.Results.BackupID) { - found = true - break - } - } + runAndCheckBackup(t, ctx, &bo, mb) - assert.True(t, found, "backup retrieved by previous snapshot manifest") - - if failed { - return - } + checkBackupIsInManifests(t, ctx, kw, &bo, sel.Selector, test.resourceOwner, test.category) m365, err := acct.M365Config() require.NoError(t, err) @@ -354,7 +375,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { kw, ms, m365.AzureTenantID, - m365UserID, + test.resourceOwner, path.ExchangeService, test.category, test.metadataFiles, @@ -379,21 +400,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() { bo, _, _, _, closer := prepNewBackupOp(t, ctx, mb, sel.Selector) defer closer() - require.NoError(t, bo.Run(ctx)) - require.NotEmpty(t, bo.Results) - require.NotEmpty(t, bo.Results.BackupID) - assert.Equalf(t, Completed, bo.Status, "backup status %s is not Completed", bo.Status) - assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") - assert.Equal(t, 1, bo.Results.ResourceOwners) - assert.NoError(t, bo.Results.ReadErrors) - assert.NoError(t, bo.Results.WriteErrors) - assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") - assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") - assert.Equal(t, - mb.CalledWith[events.BackupStart][0][events.BackupID], - bo.Results.BackupID, "backupID pre-declaration") + runAndCheckBackup(t, ctx, &bo, mb) } func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { @@ -401,30 +408,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { defer flush() var ( - t = suite.T() - mb = evmock.NewBus() - siteID = tester.M365SiteID(t) - sel = selectors.NewSharePointBackup([]string{siteID}) + t = suite.T() + mb = evmock.NewBus() + sel = selectors.NewSharePointBackup([]string{suite.site}) ) - sel.Include(sel.Sites([]string{siteID})) + sel.Include(sel.Sites([]string{suite.site})) bo, _, _, _, closer := prepNewBackupOp(t, ctx, mb, sel.Selector) defer closer() - require.NoError(t, bo.Run(ctx)) - require.NotEmpty(t, bo.Results) - require.NotEmpty(t, bo.Results.BackupID) - assert.Equalf(t, Completed, bo.Status, "backup status %s is not Completed", bo.Status) - assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") - assert.Equal(t, 1, bo.Results.ResourceOwners) - assert.NoError(t, bo.Results.ReadErrors) - assert.NoError(t, bo.Results.WriteErrors) - assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") - assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") - assert.Equal(t, - mb.CalledWith[events.BackupStart][0][events.BackupID], - bo.Results.BackupID, "backupID pre-declaration") + runAndCheckBackup(t, ctx, &bo, mb) }