From 5a78f478a1560d85c9579b7af187c87eb42ae689 Mon Sep 17 00:00:00 2001 From: Keepers Date: Sun, 16 Jul 2023 21:56:08 -0600 Subject: [PATCH] add operations test for adv rest (#3783) ads operations level tests for advanced restore configuration on all three services. Code is largely boilerplate between each service, but with just enough quirks that full consolidation would require excess jumping through hoops. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) * #3562 #### Test Plan - [x] :green_heart: E2E --- src/internal/kopia/conn.go | 2 +- src/internal/kopia/model_store.go | 4 +- .../m365/exchange/contacts_restore.go | 6 + .../m365/exchange/contacts_restore_test.go | 21 +- src/internal/m365/exchange/events_restore.go | 6 + .../m365/exchange/events_restore_test.go | 21 +- src/internal/m365/exchange/mail_restore.go | 6 + .../m365/exchange/mail_restore_test.go | 21 +- src/internal/m365/exchange/restore.go | 2 +- src/internal/m365/graph/middleware_test.go | 2 +- src/internal/m365/onedrive/handlers.go | 2 +- src/internal/m365/onedrive/item_handler.go | 2 +- src/internal/m365/onedrive/mock/handlers.go | 4 +- src/internal/m365/onedrive/restore.go | 16 +- src/internal/m365/onedrive/restore_test.go | 47 +- .../m365/sharepoint/library_handler.go | 2 +- src/internal/operations/backup.go | 3 +- src/internal/operations/common.go | 2 +- src/internal/operations/maintenance.go | 3 +- src/internal/operations/operation.go | 3 +- src/internal/operations/operation_test.go | 5 +- src/internal/operations/restore.go | 3 +- src/internal/operations/restore_test.go | 13 +- src/internal/operations/test/exchange_test.go | 404 ++++++++++++++++++ src/internal/operations/test/helper_test.go | 8 +- src/internal/operations/test/onedrive_test.go | 289 +++++++++++++ .../operations/test/restore_helper_test.go | 279 ++++++++++++ .../operations/test/sharepoint_test.go | 32 ++ src/pkg/count/keys.go | 7 +- src/pkg/repository/repository.go | 4 +- src/pkg/selectors/testdata/onedrive.go | 4 +- src/pkg/selectors/testdata/sharepoint.go | 2 +- src/pkg/services/m365/api/contacts_pager.go | 21 + .../services/m365/api/contacts_pager_test.go | 44 ++ src/pkg/services/m365/api/drive_pager.go | 32 +- src/pkg/services/m365/api/drive_pager_test.go | 77 +++- src/pkg/services/m365/api/mail_pager.go | 21 + src/pkg/services/m365/api/mail_pager_test.go | 48 +++ src/pkg/services/m365/m365_test.go | 3 - 39 files changed, 1414 insertions(+), 57 deletions(-) create mode 100644 src/internal/operations/test/restore_helper_test.go diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index f0ea3bd36..d28001f3f 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -239,7 +239,7 @@ func (w *conn) wrap() error { defer w.mu.Unlock() if w.refCount == 0 { - return clues.New("conn already closed") + return clues.New("conn not established or already closed") } w.refCount++ diff --git a/src/internal/kopia/model_store.go b/src/internal/kopia/model_store.go index ee7d32c13..54e7b67b5 100644 --- a/src/internal/kopia/model_store.go +++ b/src/internal/kopia/model_store.go @@ -506,11 +506,11 @@ func (ms *ModelStore) DeleteWithModelStoreID(ctx context.Context, id manifest.ID } opts := repo.WriteSessionOptions{Purpose: "ModelStoreDelete"} - ctr := func(innerCtx context.Context, w repo.RepositoryWriter) error { + cb := func(innerCtx context.Context, w repo.RepositoryWriter) error { return w.DeleteManifest(innerCtx, id) } - if err := repo.WriteSession(ctx, ms.c, opts, ctr); err != nil { + if err := repo.WriteSession(ctx, ms.c, opts, cb); err != nil { return clues.Wrap(err, "deleting model").WithClues(ctx) } diff --git a/src/internal/m365/exchange/contacts_restore.go b/src/internal/m365/exchange/contacts_restore.go index 7f7ffc498..ea5dc2441 100644 --- a/src/internal/m365/exchange/contacts_restore.go +++ b/src/internal/m365/exchange/contacts_restore.go @@ -154,6 +154,12 @@ func restoreContact( info := api.ContactInfo(item) info.Size = int64(len(body)) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return info, nil } diff --git a/src/internal/m365/exchange/contacts_restore_test.go b/src/internal/m365/exchange/contacts_restore_test.go index 0476cd35b..d55c1d261 100644 --- a/src/internal/m365/exchange/contacts_restore_test.go +++ b/src/internal/m365/exchange/contacts_restore_test.go @@ -90,6 +90,12 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { collisionKey := api.ContactCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *contactRestoreMock @@ -97,6 +103,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *contactRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -110,6 +117,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -123,6 +131,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -136,6 +145,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -149,6 +159,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -162,6 +173,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -175,6 +187,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -188,6 +201,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -197,6 +211,8 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreContact( ctx, test.apiMock, @@ -206,10 +222,13 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/events_restore.go b/src/internal/m365/exchange/events_restore.go index b8eb03e1f..922d7a0b0 100644 --- a/src/internal/m365/exchange/events_restore.go +++ b/src/internal/m365/exchange/events_restore.go @@ -196,6 +196,12 @@ func restoreEvent( info := api.EventInfo(event) info.Size = int64(len(body)) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return info, nil } diff --git a/src/internal/m365/exchange/events_restore_test.go b/src/internal/m365/exchange/events_restore_test.go index e280a0686..b8db6f052 100644 --- a/src/internal/m365/exchange/events_restore_test.go +++ b/src/internal/m365/exchange/events_restore_test.go @@ -138,6 +138,12 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { collisionKey := api.EventCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *eventRestoreMock @@ -145,6 +151,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *eventRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -158,6 +165,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -171,6 +179,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -184,6 +193,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -197,6 +207,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -210,6 +221,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -223,6 +235,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -236,6 +249,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -245,6 +259,8 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreEvent( ctx, test.apiMock, @@ -254,10 +270,13 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/mail_restore.go b/src/internal/m365/exchange/mail_restore.go index 67010c55d..6828361f8 100644 --- a/src/internal/m365/exchange/mail_restore.go +++ b/src/internal/m365/exchange/mail_restore.go @@ -174,6 +174,12 @@ func restoreMail( size = int64(len(bc)) } + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return api.MailInfo(msg, size), nil } diff --git a/src/internal/m365/exchange/mail_restore_test.go b/src/internal/m365/exchange/mail_restore_test.go index ec6fd69cc..5b85321b6 100644 --- a/src/internal/m365/exchange/mail_restore_test.go +++ b/src/internal/m365/exchange/mail_restore_test.go @@ -107,6 +107,12 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { collisionKey := api.MailCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *mailRestoreMock @@ -114,6 +120,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *mailRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -127,6 +134,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -140,6 +148,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -153,6 +162,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -166,6 +176,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -179,6 +190,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -192,6 +204,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -205,6 +218,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -214,6 +228,8 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreMail( ctx, test.apiMock, @@ -223,10 +239,13 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/exchange/restore.go index b65c21f99..5a5dbfcbc 100644 --- a/src/internal/m365/exchange/restore.go +++ b/src/internal/m365/exchange/restore.go @@ -108,7 +108,7 @@ func ConsumeRestoreCollections( restoreCfg.OnCollision, deets, errs, - ctr.Local()) + ctr) metrics = support.CombineMetrics(metrics, temp) diff --git a/src/internal/m365/graph/middleware_test.go b/src/internal/m365/graph/middleware_test.go index 0a5dc8eb5..a85e98605 100644 --- a/src/internal/m365/graph/middleware_test.go +++ b/src/internal/m365/graph/middleware_test.go @@ -237,7 +237,7 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() { newMWReturns(test.status, nil, test.providedErr)) mw.repeatReturn0 = true - adpt, err := mockAdapter(suite.creds, mw, 15*time.Second) + adpt, err := mockAdapter(suite.creds, mw, 25*time.Second) require.NoError(t, err, clues.ToCore(err)) // url doesn't fit the builder, but that shouldn't matter diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/onedrive/handlers.go index 0bdd1edfa..dfea5ee17 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/onedrive/handlers.go @@ -117,7 +117,7 @@ type GetItemsByCollisionKeyser interface { GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, - ) (map[string]api.DriveCollisionItem, error) + ) (map[string]api.DriveItemIDType, error) } type NewItemContentUploader interface { diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/onedrive/item_handler.go index 9c7292bfd..0b1420cf0 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/onedrive/item_handler.go @@ -164,7 +164,7 @@ func (h itemRestoreHandler) DeleteItemPermission( func (h itemRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/onedrive/mock/handlers.go index 7ede53a25..92b4573e6 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/onedrive/mock/handlers.go @@ -239,7 +239,7 @@ func (m GetsItemPermission) GetItemPermission( type RestoreHandler struct { ItemInfo details.ItemInfo - CollisionKeyMap map[string]api.DriveCollisionItem + CollisionKeyMap map[string]api.DriveItemIDType CalledDeleteItem bool CalledDeleteItemOn string @@ -264,7 +264,7 @@ func (h *RestoreHandler) AugmentItemInfo( func (h *RestoreHandler) GetItemsInContainerByCollisionKey( context.Context, string, string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { return h.CollisionKeyMap, nil } diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 66b4c6a56..84b8f1cd0 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -38,7 +38,7 @@ const ( ) type restoreCaches struct { - collisionKeyToItemID map[string]api.DriveCollisionItem + collisionKeyToItemID map[string]api.DriveItemIDType DriveIDToRootFolderID map[string]string Folders *folderCache OldLinkShareIDToNewID map[string]string @@ -50,7 +50,7 @@ type restoreCaches struct { func NewRestoreCaches() *restoreCaches { return &restoreCaches{ - collisionKeyToItemID: map[string]api.DriveCollisionItem{}, + collisionKeyToItemID: map[string]api.DriveItemIDType{}, DriveIDToRootFolderID: map[string]string{}, Folders: NewFolderCache(), OldLinkShareIDToNewID: map[string]string{}, @@ -478,7 +478,7 @@ func restoreV0File( fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - collisionKeyToItemID map[string]api.DriveCollisionItem, + collisionKeyToItemID map[string]api.DriveItemIDType, itemData data.Stream, ctr *count.Bus, ) (details.ItemInfo, error) { @@ -808,7 +808,7 @@ func restoreFile( name string, itemData data.Stream, driveID, parentFolderID string, - collisionKeyToItemID map[string]api.DriveCollisionItem, + collisionKeyToItemID map[string]api.DriveItemIDType, copyBuffer []byte, ctr *count.Bus, ) (string, details.ItemInfo, error) { @@ -826,7 +826,7 @@ func restoreFile( var ( item = newItem(name, false) collisionKey = api.DriveItemCollisionKey(item) - collision api.DriveCollisionItem + collision api.DriveItemIDType shouldDeleteOriginal bool ) @@ -937,6 +937,12 @@ func restoreFile( dii := ir.AugmentItemInfo(details.ItemInfo{}, newItem, written, nil) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return ptr.Val(newItem.GetId()), dii, nil } diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index b9ff87715..4128661f5 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -329,47 +329,57 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { const mndiID = "mndi-id" + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string - collisionKeys map[string]api.DriveCollisionItem + collisionKeys map[string]api.DriveItemIDType onCollision control.CollisionPolicy deleteErr error expectSkipped assert.BoolAssertionFunc expectMock func(*testing.T, *mock.RestoreHandler) + expectCounts counts }{ { name: "no collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Copy, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Replace, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Skip, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Copy, @@ -378,10 +388,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Replace, @@ -391,10 +402,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledDeleteItem, "new item deleted") assert.Equal(t, mndiID, rh.CalledDeleteItemOn, "deleted the correct item") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision, replace - err already deleted", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: "smarf"}, }, onCollision: control.Replace, @@ -404,10 +416,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Skip, @@ -416,10 +429,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "file-folder collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -431,10 +445,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "file-folder collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -446,10 +461,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "file-folder collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -461,6 +477,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{1, 0, 0}, }, } for _, test := range table { @@ -491,6 +508,8 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { dp, err := path.ToDrivePath(dpp) require.NoError(t, err) + ctr := count.New() + _, skip, err := restoreItem( ctx, rh, @@ -511,10 +530,14 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), }, nil, - count.New()) + ctr) + require.NoError(t, err, clues.ToCore(err)) test.expectSkipped(t, skip) test.expectMock(t, rh) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/sharepoint/library_handler.go index ca58b8bac..07c997fcb 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/sharepoint/library_handler.go @@ -190,7 +190,7 @@ func (h libraryRestoreHandler) DeleteItemPermission( func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 19c451009..00eb82884 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -28,6 +28,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -79,7 +80,7 @@ func NewBackupOperation( bus events.Eventer, ) (BackupOperation, error) { op := BackupOperation{ - operation: newOperation(opts, bus, kw, sw), + operation: newOperation(opts, bus, count.New(), kw, sw), ResourceOwner: owner, Selectors: selector, Version: "v0", diff --git a/src/internal/operations/common.go b/src/internal/operations/common.go index 57a40d2de..ffef59023 100644 --- a/src/internal/operations/common.go +++ b/src/internal/operations/common.go @@ -22,7 +22,7 @@ func getBackupAndDetailsFromID( ) (*backup.Backup, *details.Details, error) { bup, err := ms.GetBackup(ctx, backupID) if err != nil { - return nil, nil, clues.Wrap(err, "getting backup") + return nil, nil, clues.Stack(err) } deets, err := getDetailsFromBackup(ctx, bup, detailsStore, errs) diff --git a/src/internal/operations/maintenance.go b/src/internal/operations/maintenance.go index 9233cc0b2..a00f13272 100644 --- a/src/internal/operations/maintenance.go +++ b/src/internal/operations/maintenance.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" ) // MaintenanceOperation wraps an operation with restore-specific props. @@ -36,7 +37,7 @@ func NewMaintenanceOperation( bus events.Eventer, ) (MaintenanceOperation, error) { op := MaintenanceOperation{ - operation: newOperation(opts, bus, kw, nil), + operation: newOperation(opts, bus, count.New(), kw, nil), mOpts: mOpts, } diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index 834874130..854fb2e17 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -63,13 +63,14 @@ type operation struct { func newOperation( opts control.Options, bus events.Eventer, + ctr *count.Bus, kw *kopia.Wrapper, sw *store.Wrapper, ) operation { return operation{ CreatedAt: time.Now(), Errors: fault.New(opts.FailureHandling == control.FailFast), - Counter: count.New(), + Counter: ctr, Options: opts, bus: bus, diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index e95f942b5..4cbbe2a0c 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/store" ) @@ -25,7 +26,7 @@ func TestOperationSuite(t *testing.T) { func (suite *OperationSuite) TestNewOperation() { t := suite.T() - op := newOperation(control.Defaults(), events.Bus{}, nil, nil) + op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) assert.Greater(t, op.CreatedAt, time.Time{}) } @@ -45,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() { } for _, test := range table { suite.Run(test.name, func() { - err := newOperation(control.Defaults(), events.Bus{}, test.kw, test.sw).validate() + err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() test.errCheck(suite.T(), err, clues.ToCore(err)) }) } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 4e5f8d8af..e77b1104b 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -65,9 +65,10 @@ func NewRestoreOperation( sel selectors.Selector, restoreCfg control.RestoreConfig, bus events.Eventer, + ctr *count.Bus, ) (RestoreOperation, error) { op := RestoreOperation{ - operation: newOperation(opts, bus, kw, sw), + operation: newOperation(opts, bus, ctr, kw, sw), acct: acct, BackupID: backupID, RestoreCfg: control.EnsureRestoreConfigDefaults(ctx, restoreCfg), diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index cc5634234..f02ee1731 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -30,6 +30,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -118,7 +119,8 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { "foo", selectors.Selector{DiscreteOwner: "test"}, restoreCfg, - evmock.NewBus()) + evmock.NewBus(), + count.New()) require.NoError(t, err, clues.ToCore(err)) op.Errors.Fail(test.fail) @@ -258,7 +260,8 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg, - evmock.NewBus()) + evmock.NewBus(), + count.New()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -427,7 +430,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { bup.backupID, test.getSelector(t, bup.selectorResourceOwners), test.restoreCfg, - mb) + mb, + count.New()) require.NoError(t, err, clues.ToCore(err)) ds, err := ro.Run(ctx) @@ -481,7 +485,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { "backupID", rsel.Selector, restoreCfg, - mb) + mb, + count.New()) require.NoError(t, err, clues.ToCore(err)) ds, err := ro.Run(ctx) diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index db3beca94..647c7a397 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -26,8 +26,11 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -868,3 +871,404 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr }) } } + +type ExchangeRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestExchangeRestoreIntgSuite(t *testing.T) { + suite.Run(t, &ExchangeRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *ExchangeRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) + baseSel.Include( + // events cannot be run, for the same reason as incremental backups: the user needs + // to have their account recycled. + // base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + baseSel.DiscreteOwner = suite.its.userID + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + rsel, err := baseSel.ToExchangeRestore() + require.NoError(t, err, clues.ToCore(err)) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") + sel = rsel.Selector + userID = sel.ID() + cIDs = map[path.CategoryType]string{ + path.ContactsCategory: "", + path.EmailCategory: "", + path.EventsCategory: "", + } + collKeys = map[path.CategoryType]map[string]string{} + countContactsInRestore int + acCont = suite.its.ac.Contacts() + contactIDs map[string]struct{} + countEmailsInRestore int + acMail = suite.its.ac.Mail() + mailIDs map[string]struct{} + countItemsInRestore int + // countEventsInRestore int + // acEvts = suite.its.ac.Events() + // eventIDs = []string{} + ) + + // initial restore + + suite.Run("baseline", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr1 := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr1, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + + // --- contacts + + contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId()) + + collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey( + ctx, + userID, + cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + countContactsInRestore = len(collKeys[path.ContactsCategory]) + t.Log(countContactsInRestore, "contacts restored") + + contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + // --- events + + // gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location) + // require.NoError(t, err, clues.ToCore(err)) + + // restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId()) + + // collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey( + // ctx, + // userID, + // cIDs[path.EventsCategory]) + // require.NoError(t, err, clues.ToCore(err)) + // countEventsInRestore = len(collKeys[path.EventsCategory]) + // t.Log(countContactsInRestore, "events restored") + + mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox) + require.NoError(t, err, clues.ToCore(err)) + + cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId()) + + // --- mail + + collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey( + ctx, + userID, + cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + countEmailsInRestore = len(collKeys[path.EmailCategory]) + t.Log(countContactsInRestore, "emails restored") + + mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore + checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore) + }) + + // skip restore + + suite.Run("skip collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr2 := count.New() + + restoreCfg.OnCollision = control.Skip + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr2, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + + assert.Zero( + t, + len(deets.Entries), + "no items should have been restored") + + checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0) + + // --- contacts + + // get all files in folder, use these as the base + // set of files to compare against. + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, contactIDs, currentContactIDs, "ids are equal") + + // --- events + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + // --- mail + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, mailIDs, currentMailIDs, "ids are equal") + + assert.Len(t, result, 0, "no new items should get added") + }) + + // replace restore + + suite.Run("replace collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr3 := count.New() + + restoreCfg.OnCollision = control.Replace + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr3, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been replaced") + + // --- contacts + + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal") + for orig := range contactIDs { + assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement") + } + + contactIDs = currentContactIDs + + // --- events + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + // --- mail + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal") + for orig := range mailIDs { + assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement") + } + + mailIDs = currentMailIDs + + assert.Len(t, result, 0, "all items should have been replaced") + }) + + // copy restore + + suite.Run("copy collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr4 := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr4, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been copied") + + checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore) + + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy") + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy") + + // TODO: we have the option of modifying copy creations in exchange + // so that the results don't collide. But we haven't made that + // decision yet. + assert.Len(t, result, 0, "no items should have been added as copies") + }) +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 009713bda..93a609365 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -215,10 +215,10 @@ func runAndCheckBackup( expectStatus, bo.Status) - require.Less(t, 0, bo.Results.ItemsWritten) - assert.Less(t, 0, bo.Results.ItemsRead, "count of items read") - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") + require.NotZero(t, bo.Results.ItemsWritten) + assert.NotZero(t, bo.Results.ItemsRead, "count of items read") + assert.NotZero(t, bo.Results.BytesRead, "bytes read") + assert.NotZero(t, bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners, "count of resource owners") assert.NoError(t, bo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) assert.Empty(t, bo.Errors.Recovered(), "incremental recoverable/iteration errors") diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index d99268e34..41aab489d 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -32,6 +32,8 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -961,3 +963,290 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { } } } + +type OneDriveRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestOneDriveRestoreIntgSuite(t *testing.T) { + suite.Run(t, &OneDriveRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *OneDriveRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { + sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.userID + + runDriveRestoreWithAdvancedOptions( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.userDriveID, + suite.its.userDriveRootFolderID) +} + +func runDriveRestoreWithAdvancedOptions( + t *testing.T, + suite tester.Suite, + ac api.Client, + sel selectors.Selector, // both Restore and Backup types work. + driveID, rootFolderID string, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("drive_adv_restore") + containerID string + countItemsInRestore int + collKeys = map[string]api.DriveItemIDType{} + fileIDs map[string]api.DriveItemIDType + acd = ac.Drives() + ) + + // initial restore + + suite.Run("baseline", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + contGC, err := acd.GetFolderByName(ctx, driveID, rootFolderID, restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + // the folder containing the files is a child of the folder created by the restore. + contGC, err = acd.GetFolderByName(ctx, driveID, ptr.Val(contGC.GetId()), selTD.TestFolderName) + require.NoError(t, err, clues.ToCore(err)) + + containerID = ptr.Val(contGC.GetId()) + + collKeys, err = acd.GetItemsInContainerByCollisionKey( + ctx, + driveID, + containerID) + require.NoError(t, err, clues.ToCore(err)) + + countItemsInRestore = len(collKeys) + + checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore) + + fileIDs, err = acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + }) + + // skip restore + + suite.Run("skip collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Skip + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + + checkRestoreCounts(t, ctr, countItemsInRestore, 0, 0) + assert.Zero( + t, + len(deets.Entries), + "no items should have been restored") + + // get all files in folder, use these as the base + // set of files to compare against. + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, 0, "no new items should get added") + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, fileIDs, currentFileIDs, "ids are equal") + }) + + // replace restore + + suite.Run("replace collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Replace + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + checkRestoreCounts(t, ctr, 0, countItemsInRestore, 0) + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been replaced") + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, 0, "all items should have been replaced") + + for k, v := range result { + assert.NotEqual(t, v, collKeys[k], "replaced items should have new IDs") + } + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(fileIDs), len(currentFileIDs), "count of ids ids are equal") + for orig := range fileIDs { + assert.NotContains(t, currentFileIDs, orig, "original item should not exist after replacement") + } + + fileIDs = currentFileIDs + }) + + // copy restore + + suite.Run("copy collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore) + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been copied") + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, len(collKeys), "all items should have been added as copies") + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(fileIDs), len(currentFileIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") + }) +} diff --git a/src/internal/operations/test/restore_helper_test.go b/src/internal/operations/test/restore_helper_test.go new file mode 100644 index 000000000..0c397318f --- /dev/null +++ b/src/internal/operations/test/restore_helper_test.go @@ -0,0 +1,279 @@ +package test_test + +import ( + "context" + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/events" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/m365" + "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/storage" + "github.com/alcionai/corso/src/pkg/store" +) + +type restoreOpDependencies struct { + acct account.Account + ctrl *m365.Controller + kms *kopia.ModelStore + kw *kopia.Wrapper + sel selectors.Selector + sss streamstore.Streamer + st storage.Storage + sw *store.Wrapper + + closer func() +} + +func (rod *restoreOpDependencies) close( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) { + if rod.closer != nil { + rod.closer() + } + + if rod.kw != nil { + err := rod.kw.Close(ctx) + assert.NoErrorf(t, err, "kw close: %+v", clues.ToCore(err)) + } + + if rod.kms != nil { + err := rod.kw.Close(ctx) + assert.NoErrorf(t, err, "kms close: %+v", clues.ToCore(err)) + } +} + +// prepNewTestRestoreOp generates all clients required to run a restore operation, +// returning both a restore operation created with those clients, as well as +// the clients themselves. +func prepNewTestRestoreOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupStore storage.Storage, + backupID model.StableID, + bus events.Eventer, + ctr *count.Bus, + sel selectors.Selector, + opts control.Options, + restoreCfg control.RestoreConfig, +) ( + operations.RestoreOperation, + *restoreOpDependencies, +) { + var ( + rod = &restoreOpDependencies{ + acct: tconfig.NewM365Account(t), + st: backupStore, + } + k = kopia.NewConn(rod.st) + ) + + err := k.Connect(ctx, repository.Options{}) + require.NoError(t, err, clues.ToCore(err)) + + // kopiaRef comes with a count of 1 and Wrapper bumps it again + // we're so safe to close here. + defer func() { + err := k.Close(ctx) + assert.NoErrorf(t, err, "k close: %+v", clues.ToCore(err)) + }() + + rod.kw, err = kopia.NewWrapper(k) + if !assert.NoError(t, err, clues.ToCore(err)) { + return operations.RestoreOperation{}, rod + } + + rod.kms, err = kopia.NewModelStore(k) + if !assert.NoError(t, err, clues.ToCore(err)) { + return operations.RestoreOperation{}, rod + } + + rod.sw = store.NewKopiaStore(rod.kms) + + connectorResource := resource.Users + if sel.Service == selectors.ServiceSharePoint { + connectorResource = resource.Sites + } + + rod.ctrl, rod.sel = ControllerWithSelector( + t, + ctx, + rod.acct, + connectorResource, + sel, + nil, + rod.close) + + ro := newTestRestoreOp( + t, + ctx, + rod, + backupID, + bus, + ctr, + opts, + restoreCfg) + + rod.sss = streamstore.NewStreamer( + rod.kw, + rod.acct.ID(), + rod.sel.PathService()) + + return ro, rod +} + +// newTestRestoreOp accepts the clients required to compose a restore operation, plus +// any other metadata, and uses them to generate a new restore operation. This +// allows restore chains to utilize the same temp directory and configuration +// details. +func newTestRestoreOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + rod *restoreOpDependencies, + backupID model.StableID, + bus events.Eventer, + ctr *count.Bus, + opts control.Options, + restoreCfg control.RestoreConfig, +) operations.RestoreOperation { + rod.ctrl.IDNameLookup = idname.NewCache(map[string]string{rod.sel.ID(): rod.sel.Name()}) + + ro, err := operations.NewRestoreOperation( + ctx, + opts, + rod.kw, + rod.sw, + rod.ctrl, + rod.acct, + backupID, + rod.sel, + restoreCfg, + bus, + ctr) + if !assert.NoError(t, err, clues.ToCore(err)) { + rod.close(t, ctx) + t.FailNow() + } + + return ro +} + +func runAndCheckRestore( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + ro *operations.RestoreOperation, + mb *evmock.Bus, + acceptNoData bool, +) *details.Details { + deets, err := ro.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, ro.Results, "the restore had non-zero results") + require.NotNil(t, deets, "restore details") + + expectStatus := []operations.OpStatus{operations.Completed} + if acceptNoData { + expectStatus = append(expectStatus, operations.NoData) + } + + require.Contains( + t, + expectStatus, + ro.Status, + "restore doesn't match expectation, wanted any of %v, got %s", + expectStatus, + ro.Status) + + assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure())) + + if assert.Empty(t, ro.Errors.Recovered(), "recoverable/iteration errors") { + allErrs := ro.Errors.Errors() + for i, err := range allErrs.Recovered { + t.Log("recovered from test err", i, err) + } + } + + assert.NotZero(t, ro.Results.ItemsRead, "count of items read") + assert.NotZero(t, ro.Results.BytesRead, "bytes read") + assert.Equal(t, 1, ro.Results.ResourceOwners, "count of resource owners") + assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") + assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events") + + return deets +} + +type GetItemsInContainerByCollisionKeyer[T any] interface { + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]T, error) +} + +func filterCollisionKeyResults[T any]( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + protectedResourceID, containerID string, + giicbck GetItemsInContainerByCollisionKeyer[T], + filterOut map[string]T, +) map[string]T { + m, err := giicbck.GetItemsInContainerByCollisionKey( + ctx, + protectedResourceID, + containerID) + require.NoError(t, err, clues.ToCore(err)) + + for k := range filterOut { + delete(m, k) + } + + return m +} + +func checkRestoreCounts( + t *testing.T, + ctr *count.Bus, + expectSkips, expectReplaces, expectNew int, +) { + t.Log("counted values", ctr.Values()) + t.Log("counted totals", ctr.TotalValues()) + + if expectSkips >= 0 { + assert.Equal( + t, + int64(expectSkips), + ctr.Total(count.CollisionSkip), + "count of collisions resolved by skip") + } + + if expectReplaces >= 0 { + assert.Equal( + t, + int64(expectReplaces), + ctr.Total(count.CollisionReplace), + "count of collisions resolved by replace") + } + + if expectNew >= 0 { + assert.Equal( + t, + int64(expectNew), + ctr.Total(count.NewItemCreated), + "count of new items or collisions resolved by copy") + } +} diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index a3221f937..1b5a52dc2 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -177,3 +177,35 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { } } } + +type SharePointRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestSharePointRestoreIntgSuite(t *testing.T) { + suite.Run(t, &SharePointRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *SharePointRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { + sel := selectors.NewSharePointBackup([]string{suite.its.userID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.Filter(sel.Library("documents")) + sel.DiscreteOwner = suite.its.siteID + + runDriveRestoreWithAdvancedOptions( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.siteDriveID, + suite.its.siteDriveRootFolderID) +} diff --git a/src/pkg/count/keys.go b/src/pkg/count/keys.go index 96513b056..593af4b46 100644 --- a/src/pkg/count/keys.go +++ b/src/pkg/count/keys.go @@ -3,5 +3,10 @@ package count type key string const ( - CollisionSkip key = "collision-skip" + // NewItemCreated should be used for non-skip, non-replace, + // non-meta item creation counting. IE: use it specifically + // for counting new items (no collision) or copied items. + NewItemCreated key = "new-item-created" + CollisionReplace key = "collision-replace" + CollisionSkip key = "collision-skip" ) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 54471149f..8a8abb33e 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -26,6 +26,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" rep "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -370,7 +371,8 @@ func (r repository) NewRestore( model.StableID(backupID), sel, restoreCfg, - r.Bus) + r.Bus, + count.New()) } func (r repository) NewMaintenance( diff --git a/src/pkg/selectors/testdata/onedrive.go b/src/pkg/selectors/testdata/onedrive.go index 8592d3d80..677bdf840 100644 --- a/src/pkg/selectors/testdata/onedrive.go +++ b/src/pkg/selectors/testdata/onedrive.go @@ -2,8 +2,10 @@ package testdata import "github.com/alcionai/corso/src/pkg/selectors" +const TestFolderName = "test" + // OneDriveBackupFolderScope is the standard folder scope that should be used // in integration backups with onedrive. func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope { - return sel.Folders([]string{"test"}, selectors.PrefixMatch()) + return sel.Folders([]string{TestFolderName}, selectors.PrefixMatch()) } diff --git a/src/pkg/selectors/testdata/sharepoint.go b/src/pkg/selectors/testdata/sharepoint.go index c6cc414ff..e6c3be58b 100644 --- a/src/pkg/selectors/testdata/sharepoint.go +++ b/src/pkg/selectors/testdata/sharepoint.go @@ -7,5 +7,5 @@ import ( // SharePointBackupFolderScope is the standard folder scope that should be used // in integration backups with sharepoint. func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope { - return sel.LibraryFolders([]string{"test"}, selectors.PrefixMatch()) + return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch()) } diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index 69caa110a..f997bd2e7 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -166,6 +166,27 @@ func (c Contacts) GetItemsInContainerByCollisionKey( return m, nil } +func (c Contacts) GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, +) (map[string]struct{}, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewContactsPager(userID, containerID, "id") + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]struct{}{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = struct{}{} + } + + return m, nil +} + // --------------------------------------------------------------------------- // item ID pager // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/contacts_pager_test.go b/src/pkg/services/m365/api/contacts_pager_test.go index cc826aa06..5f3561e6d 100644 --- a/src/pkg/services/m365/api/contacts_pager_test.go +++ b/src/pkg/services/m365/api/contacts_pager_test.go @@ -83,3 +83,47 @@ func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsInContainerByCollision assert.Truef(t, ok, "expected results to contain collision key: %s", e) } } + +func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsIDsInContainer() { + t := suite.T() + ac := suite.its.ac.Contacts() + + ctx, flush := tester.NewContext(t) + defer flush() + + container, err := ac.GetContainerByID(ctx, suite.its.userID, api.DefaultContacts) + require.NoError(t, err, clues.ToCore(err)) + + msgs, err := ac.Stable. + Client(). + Users(). + ByUserId(suite.its.userID). + ContactFolders(). + ByContactFolderId(ptr.Val(container.GetId())). + Contacts(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(err)) + + ms := msgs.GetValue() + expect := map[string]struct{}{} + + for _, m := range ms { + expect[ptr.Val(m.GetId())] = struct{}{} + } + + results, err := suite.its.ac.Contacts(). + GetItemIDsInContainer(ctx, suite.its.userID, api.DefaultContacts) + require.NoError(t, err, clues.ToCore(err)) + require.Less(t, 0, len(results), "requires at least one result") + require.Equal(t, len(expect), len(results), "must have same count of items") + + for _, k := range expect { + t.Log("expects key", k) + } + + for k := range results { + t.Log("results key", k) + } + + assert.Equal(t, expect, results) +} diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go index 0b073571b..3ba6e4b46 100644 --- a/src/pkg/services/m365/api/drive_pager.go +++ b/src/pkg/services/m365/api/drive_pager.go @@ -67,7 +67,7 @@ func (p *driveItemPageCtrl) setNext(nextLink string) { p.builder = drives.NewItemItemsItemChildrenRequestBuilder(nextLink, p.gs.Adapter()) } -type DriveCollisionItem struct { +type DriveItemIDType struct { ItemID string IsFolder bool } @@ -75,7 +75,7 @@ type DriveCollisionItem struct { func (c Drives) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]DriveCollisionItem, error) { +) (map[string]DriveItemIDType, error) { ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...) @@ -84,10 +84,34 @@ func (c Drives) GetItemsInContainerByCollisionKey( return nil, graph.Wrap(ctx, err, "enumerating drive items") } - m := map[string]DriveCollisionItem{} + m := map[string]DriveItemIDType{} for _, item := range items { - m[DriveItemCollisionKey(item)] = DriveCollisionItem{ + m[DriveItemCollisionKey(item)] = DriveItemIDType{ + ItemID: ptr.Val(item.GetId()), + IsFolder: item.GetFolder() != nil, + } + } + + return m, nil +} + +func (c Drives) GetItemIDsInContainer( + ctx context.Context, + driveID, containerID string, +) (map[string]DriveItemIDType, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewDriveItemPager(driveID, containerID, idAnd("file", "folder")...) + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]DriveItemIDType{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = DriveItemIDType{ ItemID: ptr.Val(item.GetId()), IsFolder: item.GetFolder() != nil, } diff --git a/src/pkg/services/m365/api/drive_pager_test.go b/src/pkg/services/m365/api/drive_pager_test.go index 2eb1f8e27..71177e2c8 100644 --- a/src/pkg/services/m365/api/drive_pager_test.go +++ b/src/pkg/services/m365/api/drive_pager_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -68,7 +69,7 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey() require.NoError(t, err, clues.ToCore(err)) ims := items.GetValue() - expect := make([]api.DriveCollisionItem, 0, len(ims)) + expect := make([]api.DriveItemIDType, 0, len(ims)) assert.NotEmptyf( t, @@ -103,3 +104,77 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey() }) } } + +func (suite *DrivePagerIntgSuite) TestDrives_GetItemIDsInContainer() { + table := []struct { + name string + driveID string + rootFolderID string + }{ + { + name: "user drive", + driveID: suite.its.userDriveID, + rootFolderID: suite.its.userDriveRootFolderID, + }, + { + name: "site drive", + driveID: suite.its.siteDriveID, + rootFolderID: suite.its.siteDriveRootFolderID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + t.Log("drive", test.driveID) + t.Log("rootFolder", test.rootFolderID) + + items, err := suite.its.ac.Stable. + Client(). + Drives(). + ByDriveId(test.driveID). + Items(). + ByDriveItemId(test.rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(err)) + + igv := items.GetValue() + expect := map[string]api.DriveItemIDType{} + + assert.NotEmptyf( + t, + igv, + "need at least one item to compare in user %s drive %s folder %s", + suite.its.userID, test.driveID, test.rootFolderID) + + for _, itm := range igv { + expect[ptr.Val(itm.GetId())] = api.DriveItemIDType{ + ItemID: ptr.Val(itm.GetId()), + IsFolder: itm.GetFolder() != nil, + } + } + + results, err := suite.its.ac. + Drives(). + GetItemIDsInContainer(ctx, test.driveID, test.rootFolderID) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, results) + require.Equal(t, len(expect), len(results), "must have same count of items") + + for k := range expect { + t.Log("expects key", k) + } + + for k, v := range results { + t.Log("results key", k) + assert.NotEmpty(t, v, "all values should be populated") + } + + assert.Equal(t, expect, results) + }) + } +} diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index caf6a2934..5472239f8 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -247,6 +247,27 @@ func (c Mail) GetItemsInContainerByCollisionKey( return m, nil } +func (c Mail) GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, +) (map[string]struct{}, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewMailPager(userID, containerID, "id") + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]struct{}{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = struct{}{} + } + + return m, nil +} + // --------------------------------------------------------------------------- // delta item ID pager // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/mail_pager_test.go b/src/pkg/services/m365/api/mail_pager_test.go index 476e13401..d99c428a2 100644 --- a/src/pkg/services/m365/api/mail_pager_test.go +++ b/src/pkg/services/m365/api/mail_pager_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/users" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -83,3 +84,50 @@ func (suite *MailPagerIntgSuite) TestMail_GetItemsInContainerByCollisionKey() { assert.Truef(t, ok, "expected results to contain collision key: %s", e) } } + +func (suite *MailPagerIntgSuite) TestMail_GetItemsIDsInContainer() { + t := suite.T() + ac := suite.its.ac.Mail() + + ctx, flush := tester.NewContext(t) + defer flush() + + config := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{ + Top: ptr.To[int32](1000), + }, + } + + msgs, err := ac.Stable. + Client(). + Users(). + ByUserId(suite.its.userID). + MailFolders(). + ByMailFolderId(api.MailInbox). + Messages(). + Get(ctx, config) + require.NoError(t, err, clues.ToCore(err)) + + ms := msgs.GetValue() + expect := map[string]struct{}{} + + for _, m := range ms { + expect[ptr.Val(m.GetId())] = struct{}{} + } + + results, err := suite.its.ac.Mail(). + GetItemIDsInContainer(ctx, suite.its.userID, api.MailInbox) + require.NoError(t, err, clues.ToCore(err)) + require.Less(t, 0, len(results), "requires at least one result") + require.Equal(t, len(expect), len(results), "must have same count of items") + + for k := range expect { + t.Log("expects key", k) + } + + for k := range results { + t.Log("results key", k) + } + + assert.Equal(t, expect, results) +} diff --git a/src/pkg/services/m365/m365_test.go b/src/pkg/services/m365/m365_test.go index e83dcf609..59812ebbd 100644 --- a/src/pkg/services/m365/m365_test.go +++ b/src/pkg/services/m365/m365_test.go @@ -2,7 +2,6 @@ package m365 import ( "context" - "fmt" "testing" "github.com/alcionai/clues" @@ -508,8 +507,6 @@ func (suite *DiscoveryIntgSuite) TestGetUserInfo() { } for _, test := range table { suite.Run(test.name, func() { - fmt.Printf("\n-----\n%+v\n-----\n", test.name) - t := suite.T() ctx, flush := tester.NewContext(t)