diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index f0ea3bd36..d28001f3f 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -239,7 +239,7 @@ func (w *conn) wrap() error { defer w.mu.Unlock() if w.refCount == 0 { - return clues.New("conn already closed") + return clues.New("conn not established or already closed") } w.refCount++ diff --git a/src/internal/kopia/model_store.go b/src/internal/kopia/model_store.go index ee7d32c13..54e7b67b5 100644 --- a/src/internal/kopia/model_store.go +++ b/src/internal/kopia/model_store.go @@ -506,11 +506,11 @@ func (ms *ModelStore) DeleteWithModelStoreID(ctx context.Context, id manifest.ID } opts := repo.WriteSessionOptions{Purpose: "ModelStoreDelete"} - ctr := func(innerCtx context.Context, w repo.RepositoryWriter) error { + cb := func(innerCtx context.Context, w repo.RepositoryWriter) error { return w.DeleteManifest(innerCtx, id) } - if err := repo.WriteSession(ctx, ms.c, opts, ctr); err != nil { + if err := repo.WriteSession(ctx, ms.c, opts, cb); err != nil { return clues.Wrap(err, "deleting model").WithClues(ctx) } diff --git a/src/internal/m365/exchange/contacts_restore.go b/src/internal/m365/exchange/contacts_restore.go index 7f7ffc498..ea5dc2441 100644 --- a/src/internal/m365/exchange/contacts_restore.go +++ b/src/internal/m365/exchange/contacts_restore.go @@ -154,6 +154,12 @@ func restoreContact( info := api.ContactInfo(item) info.Size = int64(len(body)) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return info, nil } diff --git a/src/internal/m365/exchange/contacts_restore_test.go b/src/internal/m365/exchange/contacts_restore_test.go index 0476cd35b..d55c1d261 100644 --- a/src/internal/m365/exchange/contacts_restore_test.go +++ b/src/internal/m365/exchange/contacts_restore_test.go @@ -90,6 +90,12 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { collisionKey := api.ContactCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *contactRestoreMock @@ -97,6 +103,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *contactRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -110,6 +117,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -123,6 +131,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -136,6 +145,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -149,6 +159,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -162,6 +173,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -175,6 +187,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -188,6 +201,7 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -197,6 +211,8 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreContact( ctx, test.apiMock, @@ -206,10 +222,13 @@ func (suite *ContactsRestoreIntgSuite) TestRestoreContact() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/events_restore.go b/src/internal/m365/exchange/events_restore.go index b8eb03e1f..922d7a0b0 100644 --- a/src/internal/m365/exchange/events_restore.go +++ b/src/internal/m365/exchange/events_restore.go @@ -196,6 +196,12 @@ func restoreEvent( info := api.EventInfo(event) info.Size = int64(len(body)) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return info, nil } diff --git a/src/internal/m365/exchange/events_restore_test.go b/src/internal/m365/exchange/events_restore_test.go index e280a0686..b8db6f052 100644 --- a/src/internal/m365/exchange/events_restore_test.go +++ b/src/internal/m365/exchange/events_restore_test.go @@ -138,6 +138,12 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { collisionKey := api.EventCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *eventRestoreMock @@ -145,6 +151,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *eventRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -158,6 +165,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -171,6 +179,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -184,6 +193,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -197,6 +207,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -210,6 +221,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -223,6 +235,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -236,6 +249,7 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -245,6 +259,8 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreEvent( ctx, test.apiMock, @@ -254,10 +270,13 @@ func (suite *EventsRestoreIntgSuite) TestRestoreEvent() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/mail_restore.go b/src/internal/m365/exchange/mail_restore.go index 67010c55d..6828361f8 100644 --- a/src/internal/m365/exchange/mail_restore.go +++ b/src/internal/m365/exchange/mail_restore.go @@ -174,6 +174,12 @@ func restoreMail( size = int64(len(bc)) } + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return api.MailInfo(msg, size), nil } diff --git a/src/internal/m365/exchange/mail_restore_test.go b/src/internal/m365/exchange/mail_restore_test.go index ec6fd69cc..5b85321b6 100644 --- a/src/internal/m365/exchange/mail_restore_test.go +++ b/src/internal/m365/exchange/mail_restore_test.go @@ -107,6 +107,12 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { collisionKey := api.MailCollisionKey(stub) + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string apiMock *mailRestoreMock @@ -114,6 +120,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { onCollision control.CollisionPolicy expectErr func(*testing.T, error) expectMock func(*testing.T, *mailRestoreMock) + expectCounts counts }{ { name: "no collision: skip", @@ -127,6 +134,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: copy", @@ -140,6 +148,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision: replace", @@ -153,6 +162,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: skip", @@ -166,6 +176,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.False(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "collision: copy", @@ -179,6 +190,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.False(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision: replace", @@ -192,6 +204,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision: replace - err already deleted", @@ -205,6 +218,7 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { assert.True(t, m.calledPost, "new item posted") assert.True(t, m.calledDelete, "old item deleted") }, + expectCounts: counts{0, 1, 0}, }, } for _, test := range table { @@ -214,6 +228,8 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { ctx, flush := tester.NewContext(t) defer flush() + ctr := count.New() + _, err := restoreMail( ctx, test.apiMock, @@ -223,10 +239,13 @@ func (suite *MailRestoreIntgSuite) TestRestoreMail() { test.collisionMap, test.onCollision, fault.New(true), - count.New()) + ctr) test.expectErr(t, err) test.expectMock(t, test.apiMock) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/exchange/restore.go index b65c21f99..5a5dbfcbc 100644 --- a/src/internal/m365/exchange/restore.go +++ b/src/internal/m365/exchange/restore.go @@ -108,7 +108,7 @@ func ConsumeRestoreCollections( restoreCfg.OnCollision, deets, errs, - ctr.Local()) + ctr) metrics = support.CombineMetrics(metrics, temp) diff --git a/src/internal/m365/graph/middleware_test.go b/src/internal/m365/graph/middleware_test.go index 0a5dc8eb5..a85e98605 100644 --- a/src/internal/m365/graph/middleware_test.go +++ b/src/internal/m365/graph/middleware_test.go @@ -237,7 +237,7 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() { newMWReturns(test.status, nil, test.providedErr)) mw.repeatReturn0 = true - adpt, err := mockAdapter(suite.creds, mw, 15*time.Second) + adpt, err := mockAdapter(suite.creds, mw, 25*time.Second) require.NoError(t, err, clues.ToCore(err)) // url doesn't fit the builder, but that shouldn't matter diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/onedrive/handlers.go index 0bdd1edfa..dfea5ee17 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/onedrive/handlers.go @@ -117,7 +117,7 @@ type GetItemsByCollisionKeyser interface { GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, - ) (map[string]api.DriveCollisionItem, error) + ) (map[string]api.DriveItemIDType, error) } type NewItemContentUploader interface { diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/onedrive/item_handler.go index 9c7292bfd..0b1420cf0 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/onedrive/item_handler.go @@ -164,7 +164,7 @@ func (h itemRestoreHandler) DeleteItemPermission( func (h itemRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/onedrive/mock/handlers.go index 7ede53a25..92b4573e6 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/onedrive/mock/handlers.go @@ -239,7 +239,7 @@ func (m GetsItemPermission) GetItemPermission( type RestoreHandler struct { ItemInfo details.ItemInfo - CollisionKeyMap map[string]api.DriveCollisionItem + CollisionKeyMap map[string]api.DriveItemIDType CalledDeleteItem bool CalledDeleteItemOn string @@ -264,7 +264,7 @@ func (h *RestoreHandler) AugmentItemInfo( func (h *RestoreHandler) GetItemsInContainerByCollisionKey( context.Context, string, string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { return h.CollisionKeyMap, nil } diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 66b4c6a56..84b8f1cd0 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -38,7 +38,7 @@ const ( ) type restoreCaches struct { - collisionKeyToItemID map[string]api.DriveCollisionItem + collisionKeyToItemID map[string]api.DriveItemIDType DriveIDToRootFolderID map[string]string Folders *folderCache OldLinkShareIDToNewID map[string]string @@ -50,7 +50,7 @@ type restoreCaches struct { func NewRestoreCaches() *restoreCaches { return &restoreCaches{ - collisionKeyToItemID: map[string]api.DriveCollisionItem{}, + collisionKeyToItemID: map[string]api.DriveItemIDType{}, DriveIDToRootFolderID: map[string]string{}, Folders: NewFolderCache(), OldLinkShareIDToNewID: map[string]string{}, @@ -478,7 +478,7 @@ func restoreV0File( fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - collisionKeyToItemID map[string]api.DriveCollisionItem, + collisionKeyToItemID map[string]api.DriveItemIDType, itemData data.Stream, ctr *count.Bus, ) (details.ItemInfo, error) { @@ -808,7 +808,7 @@ func restoreFile( name string, itemData data.Stream, driveID, parentFolderID string, - collisionKeyToItemID map[string]api.DriveCollisionItem, + collisionKeyToItemID map[string]api.DriveItemIDType, copyBuffer []byte, ctr *count.Bus, ) (string, details.ItemInfo, error) { @@ -826,7 +826,7 @@ func restoreFile( var ( item = newItem(name, false) collisionKey = api.DriveItemCollisionKey(item) - collision api.DriveCollisionItem + collision api.DriveItemIDType shouldDeleteOriginal bool ) @@ -937,6 +937,12 @@ func restoreFile( dii := ir.AugmentItemInfo(details.ItemInfo{}, newItem, written, nil) + if shouldDeleteOriginal { + ctr.Inc(count.CollisionReplace) + } else { + ctr.Inc(count.NewItemCreated) + } + return ptr.Val(newItem.GetId()), dii, nil } diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index b9ff87715..4128661f5 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -329,47 +329,57 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { const mndiID = "mndi-id" + type counts struct { + skip int64 + replace int64 + new int64 + } + table := []struct { name string - collisionKeys map[string]api.DriveCollisionItem + collisionKeys map[string]api.DriveItemIDType onCollision control.CollisionPolicy deleteErr error expectSkipped assert.BoolAssertionFunc expectMock func(*testing.T, *mock.RestoreHandler) + expectCounts counts }{ { name: "no collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Copy, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Replace, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "no collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{}, + collisionKeys: map[string]api.DriveItemIDType{}, onCollision: control.Skip, expectSkipped: assert.False, expectMock: func(t *testing.T, rh *mock.RestoreHandler) { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Copy, @@ -378,10 +388,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Replace, @@ -391,10 +402,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledDeleteItem, "new item deleted") assert.Equal(t, mndiID, rh.CalledDeleteItemOn, "deleted the correct item") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision, replace - err already deleted", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: "smarf"}, }, onCollision: control.Replace, @@ -404,10 +416,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.True(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 1, 0}, }, { name: "collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: {ItemID: mndiID}, }, onCollision: control.Skip, @@ -416,10 +429,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{1, 0, 0}, }, { name: "file-folder collision, copy", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -431,10 +445,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "file-folder collision, replace", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -446,10 +461,11 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.True(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{0, 0, 1}, }, { name: "file-folder collision, skip", - collisionKeys: map[string]api.DriveCollisionItem{ + collisionKeys: map[string]api.DriveItemIDType{ mock.DriveItemFileName: { ItemID: mndiID, IsFolder: true, @@ -461,6 +477,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { assert.False(t, rh.CalledPostItem, "new item posted") assert.False(t, rh.CalledDeleteItem, "new item deleted") }, + expectCounts: counts{1, 0, 0}, }, } for _, test := range table { @@ -491,6 +508,8 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { dp, err := path.ToDrivePath(dpp) require.NoError(t, err) + ctr := count.New() + _, skip, err := restoreItem( ctx, rh, @@ -511,10 +530,14 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), }, nil, - count.New()) + ctr) + require.NoError(t, err, clues.ToCore(err)) test.expectSkipped(t, skip) test.expectMock(t, rh) + assert.Equal(t, test.expectCounts.skip, ctr.Get(count.CollisionSkip), "skips") + assert.Equal(t, test.expectCounts.replace, ctr.Get(count.CollisionReplace), "replaces") + assert.Equal(t, test.expectCounts.new, ctr.Get(count.NewItemCreated), "new items") }) } } diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/sharepoint/library_handler.go index ca58b8bac..07c997fcb 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/sharepoint/library_handler.go @@ -190,7 +190,7 @@ func (h libraryRestoreHandler) DeleteItemPermission( func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]api.DriveCollisionItem, error) { +) (map[string]api.DriveItemIDType, error) { m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 19c451009..00eb82884 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -28,6 +28,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -79,7 +80,7 @@ func NewBackupOperation( bus events.Eventer, ) (BackupOperation, error) { op := BackupOperation{ - operation: newOperation(opts, bus, kw, sw), + operation: newOperation(opts, bus, count.New(), kw, sw), ResourceOwner: owner, Selectors: selector, Version: "v0", diff --git a/src/internal/operations/common.go b/src/internal/operations/common.go index 57a40d2de..ffef59023 100644 --- a/src/internal/operations/common.go +++ b/src/internal/operations/common.go @@ -22,7 +22,7 @@ func getBackupAndDetailsFromID( ) (*backup.Backup, *details.Details, error) { bup, err := ms.GetBackup(ctx, backupID) if err != nil { - return nil, nil, clues.Wrap(err, "getting backup") + return nil, nil, clues.Stack(err) } deets, err := getDetailsFromBackup(ctx, bup, detailsStore, errs) diff --git a/src/internal/operations/maintenance.go b/src/internal/operations/maintenance.go index 9233cc0b2..a00f13272 100644 --- a/src/internal/operations/maintenance.go +++ b/src/internal/operations/maintenance.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" ) // MaintenanceOperation wraps an operation with restore-specific props. @@ -36,7 +37,7 @@ func NewMaintenanceOperation( bus events.Eventer, ) (MaintenanceOperation, error) { op := MaintenanceOperation{ - operation: newOperation(opts, bus, kw, nil), + operation: newOperation(opts, bus, count.New(), kw, nil), mOpts: mOpts, } diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index 834874130..854fb2e17 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -63,13 +63,14 @@ type operation struct { func newOperation( opts control.Options, bus events.Eventer, + ctr *count.Bus, kw *kopia.Wrapper, sw *store.Wrapper, ) operation { return operation{ CreatedAt: time.Now(), Errors: fault.New(opts.FailureHandling == control.FailFast), - Counter: count.New(), + Counter: ctr, Options: opts, bus: bus, diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index e95f942b5..4cbbe2a0c 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/store" ) @@ -25,7 +26,7 @@ func TestOperationSuite(t *testing.T) { func (suite *OperationSuite) TestNewOperation() { t := suite.T() - op := newOperation(control.Defaults(), events.Bus{}, nil, nil) + op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) assert.Greater(t, op.CreatedAt, time.Time{}) } @@ -45,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() { } for _, test := range table { suite.Run(test.name, func() { - err := newOperation(control.Defaults(), events.Bus{}, test.kw, test.sw).validate() + err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() test.errCheck(suite.T(), err, clues.ToCore(err)) }) } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 4e5f8d8af..e77b1104b 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -65,9 +65,10 @@ func NewRestoreOperation( sel selectors.Selector, restoreCfg control.RestoreConfig, bus events.Eventer, + ctr *count.Bus, ) (RestoreOperation, error) { op := RestoreOperation{ - operation: newOperation(opts, bus, kw, sw), + operation: newOperation(opts, bus, ctr, kw, sw), acct: acct, BackupID: backupID, RestoreCfg: control.EnsureRestoreConfigDefaults(ctx, restoreCfg), diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index cc5634234..f02ee1731 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -30,6 +30,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -118,7 +119,8 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { "foo", selectors.Selector{DiscreteOwner: "test"}, restoreCfg, - evmock.NewBus()) + evmock.NewBus(), + count.New()) require.NoError(t, err, clues.ToCore(err)) op.Errors.Fail(test.fail) @@ -258,7 +260,8 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg, - evmock.NewBus()) + evmock.NewBus(), + count.New()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -427,7 +430,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { bup.backupID, test.getSelector(t, bup.selectorResourceOwners), test.restoreCfg, - mb) + mb, + count.New()) require.NoError(t, err, clues.ToCore(err)) ds, err := ro.Run(ctx) @@ -481,7 +485,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { "backupID", rsel.Selector, restoreCfg, - mb) + mb, + count.New()) require.NoError(t, err, clues.ToCore(err)) ds, err := ro.Run(ctx) diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index db3beca94..647c7a397 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -26,8 +26,11 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -868,3 +871,404 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr }) } } + +type ExchangeRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestExchangeRestoreIntgSuite(t *testing.T) { + suite.Run(t, &ExchangeRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *ExchangeRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) + baseSel.Include( + // events cannot be run, for the same reason as incremental backups: the user needs + // to have their account recycled. + // base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + baseSel.DiscreteOwner = suite.its.userID + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + rsel, err := baseSel.ToExchangeRestore() + require.NoError(t, err, clues.ToCore(err)) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") + sel = rsel.Selector + userID = sel.ID() + cIDs = map[path.CategoryType]string{ + path.ContactsCategory: "", + path.EmailCategory: "", + path.EventsCategory: "", + } + collKeys = map[path.CategoryType]map[string]string{} + countContactsInRestore int + acCont = suite.its.ac.Contacts() + contactIDs map[string]struct{} + countEmailsInRestore int + acMail = suite.its.ac.Mail() + mailIDs map[string]struct{} + countItemsInRestore int + // countEventsInRestore int + // acEvts = suite.its.ac.Events() + // eventIDs = []string{} + ) + + // initial restore + + suite.Run("baseline", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr1 := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr1, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + + // --- contacts + + contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId()) + + collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey( + ctx, + userID, + cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + countContactsInRestore = len(collKeys[path.ContactsCategory]) + t.Log(countContactsInRestore, "contacts restored") + + contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + // --- events + + // gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location) + // require.NoError(t, err, clues.ToCore(err)) + + // restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId()) + + // collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey( + // ctx, + // userID, + // cIDs[path.EventsCategory]) + // require.NoError(t, err, clues.ToCore(err)) + // countEventsInRestore = len(collKeys[path.EventsCategory]) + // t.Log(countContactsInRestore, "events restored") + + mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox) + require.NoError(t, err, clues.ToCore(err)) + + cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId()) + + // --- mail + + collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey( + ctx, + userID, + cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + countEmailsInRestore = len(collKeys[path.EmailCategory]) + t.Log(countContactsInRestore, "emails restored") + + mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore + checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore) + }) + + // skip restore + + suite.Run("skip collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr2 := count.New() + + restoreCfg.OnCollision = control.Skip + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr2, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + + assert.Zero( + t, + len(deets.Entries), + "no items should have been restored") + + checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0) + + // --- contacts + + // get all files in folder, use these as the base + // set of files to compare against. + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, contactIDs, currentContactIDs, "ids are equal") + + // --- events + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + // --- mail + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, mailIDs, currentMailIDs, "ids are equal") + + assert.Len(t, result, 0, "no new items should get added") + }) + + // replace restore + + suite.Run("replace collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr3 := count.New() + + restoreCfg.OnCollision = control.Replace + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr3, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been replaced") + + // --- contacts + + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal") + for orig := range contactIDs { + assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement") + } + + contactIDs = currentContactIDs + + // --- events + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + // --- mail + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal") + for orig := range mailIDs { + assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement") + } + + mailIDs = currentMailIDs + + assert.Len(t, result, 0, "all items should have been replaced") + }) + + // copy restore + + suite.Run("copy collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr4 := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr4, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been copied") + + checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore) + + result := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.ContactsCategory], + GetItemsInContainerByCollisionKeyer[string](acCont), + collKeys[path.ContactsCategory]) + + currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy") + + // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) + // maps.Copy(result, m) + + m := filterCollisionKeyResults( + t, + ctx, + userID, + cIDs[path.EmailCategory], + GetItemsInContainerByCollisionKeyer[string](acMail), + collKeys[path.EmailCategory]) + maps.Copy(result, m) + + currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy") + + // TODO: we have the option of modifying copy creations in exchange + // so that the results don't collide. But we haven't made that + // decision yet. + assert.Len(t, result, 0, "no items should have been added as copies") + }) +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 009713bda..93a609365 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -215,10 +215,10 @@ func runAndCheckBackup( expectStatus, bo.Status) - require.Less(t, 0, bo.Results.ItemsWritten) - assert.Less(t, 0, bo.Results.ItemsRead, "count of items read") - assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") - assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") + require.NotZero(t, bo.Results.ItemsWritten) + assert.NotZero(t, bo.Results.ItemsRead, "count of items read") + assert.NotZero(t, bo.Results.BytesRead, "bytes read") + assert.NotZero(t, bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners, "count of resource owners") assert.NoError(t, bo.Errors.Failure(), "incremental non-recoverable error", clues.ToCore(bo.Errors.Failure())) assert.Empty(t, bo.Errors.Recovered(), "incremental recoverable/iteration errors") diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index d99268e34..41aab489d 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -32,6 +32,8 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -961,3 +963,290 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { } } } + +type OneDriveRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestOneDriveRestoreIntgSuite(t *testing.T) { + suite.Run(t, &OneDriveRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *OneDriveRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { + sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.userID + + runDriveRestoreWithAdvancedOptions( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.userDriveID, + suite.its.userDriveRootFolderID) +} + +func runDriveRestoreWithAdvancedOptions( + t *testing.T, + suite tester.Suite, + ac api.Client, + sel selectors.Selector, // both Restore and Backup types work. + driveID, rootFolderID string, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + var ( + mb = evmock.NewBus() + opts = control.Defaults() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("drive_adv_restore") + containerID string + countItemsInRestore int + collKeys = map[string]api.DriveItemIDType{} + fileIDs map[string]api.DriveItemIDType + acd = ac.Drives() + ) + + // initial restore + + suite.Run("baseline", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + contGC, err := acd.GetFolderByName(ctx, driveID, rootFolderID, restoreCfg.Location) + require.NoError(t, err, clues.ToCore(err)) + + // the folder containing the files is a child of the folder created by the restore. + contGC, err = acd.GetFolderByName(ctx, driveID, ptr.Val(contGC.GetId()), selTD.TestFolderName) + require.NoError(t, err, clues.ToCore(err)) + + containerID = ptr.Val(contGC.GetId()) + + collKeys, err = acd.GetItemsInContainerByCollisionKey( + ctx, + driveID, + containerID) + require.NoError(t, err, clues.ToCore(err)) + + countItemsInRestore = len(collKeys) + + checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore) + + fileIDs, err = acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + }) + + // skip restore + + suite.Run("skip collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Skip + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + + checkRestoreCounts(t, ctr, countItemsInRestore, 0, 0) + assert.Zero( + t, + len(deets.Entries), + "no items should have been restored") + + // get all files in folder, use these as the base + // set of files to compare against. + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, 0, "no new items should get added") + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, fileIDs, currentFileIDs, "ids are equal") + }) + + // replace restore + + suite.Run("replace collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Replace + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + checkRestoreCounts(t, ctr, 0, countItemsInRestore, 0) + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been replaced") + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, 0, "all items should have been replaced") + + for k, v := range result { + assert.NotEqual(t, v, collKeys[k], "replaced items should have new IDs") + } + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(fileIDs), len(currentFileIDs), "count of ids ids are equal") + for orig := range fileIDs { + assert.NotContains(t, currentFileIDs, orig, "original item should not exist after replacement") + } + + fileIDs = currentFileIDs + }) + + // copy restore + + suite.Run("copy collisions", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + mb := evmock.NewBus() + ctr := count.New() + + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + sel, + opts, + restoreCfg) + + deets := runAndCheckRestore(t, ctx, &ro, mb, false) + filtEnts := []details.Entry{} + + for _, e := range deets.Entries { + if e.Folder == nil { + filtEnts = append(filtEnts, e) + } + } + + checkRestoreCounts(t, ctr, 0, 0, countItemsInRestore) + assert.Len( + t, + filtEnts, + countItemsInRestore, + "every item should have been copied") + + result := filterCollisionKeyResults( + t, + ctx, + driveID, + containerID, + GetItemsInContainerByCollisionKeyer[api.DriveItemIDType](acd), + collKeys) + + assert.Len(t, result, len(collKeys), "all items should have been added as copies") + + currentFileIDs, err := acd.GetItemIDsInContainer(ctx, driveID, containerID) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, 2*len(fileIDs), len(currentFileIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") + }) +} diff --git a/src/internal/operations/test/restore_helper_test.go b/src/internal/operations/test/restore_helper_test.go new file mode 100644 index 000000000..0c397318f --- /dev/null +++ b/src/internal/operations/test/restore_helper_test.go @@ -0,0 +1,279 @@ +package test_test + +import ( + "context" + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/events" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/m365" + "github.com/alcionai/corso/src/internal/m365/resource" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/storage" + "github.com/alcionai/corso/src/pkg/store" +) + +type restoreOpDependencies struct { + acct account.Account + ctrl *m365.Controller + kms *kopia.ModelStore + kw *kopia.Wrapper + sel selectors.Selector + sss streamstore.Streamer + st storage.Storage + sw *store.Wrapper + + closer func() +} + +func (rod *restoreOpDependencies) close( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) { + if rod.closer != nil { + rod.closer() + } + + if rod.kw != nil { + err := rod.kw.Close(ctx) + assert.NoErrorf(t, err, "kw close: %+v", clues.ToCore(err)) + } + + if rod.kms != nil { + err := rod.kw.Close(ctx) + assert.NoErrorf(t, err, "kms close: %+v", clues.ToCore(err)) + } +} + +// prepNewTestRestoreOp generates all clients required to run a restore operation, +// returning both a restore operation created with those clients, as well as +// the clients themselves. +func prepNewTestRestoreOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupStore storage.Storage, + backupID model.StableID, + bus events.Eventer, + ctr *count.Bus, + sel selectors.Selector, + opts control.Options, + restoreCfg control.RestoreConfig, +) ( + operations.RestoreOperation, + *restoreOpDependencies, +) { + var ( + rod = &restoreOpDependencies{ + acct: tconfig.NewM365Account(t), + st: backupStore, + } + k = kopia.NewConn(rod.st) + ) + + err := k.Connect(ctx, repository.Options{}) + require.NoError(t, err, clues.ToCore(err)) + + // kopiaRef comes with a count of 1 and Wrapper bumps it again + // we're so safe to close here. + defer func() { + err := k.Close(ctx) + assert.NoErrorf(t, err, "k close: %+v", clues.ToCore(err)) + }() + + rod.kw, err = kopia.NewWrapper(k) + if !assert.NoError(t, err, clues.ToCore(err)) { + return operations.RestoreOperation{}, rod + } + + rod.kms, err = kopia.NewModelStore(k) + if !assert.NoError(t, err, clues.ToCore(err)) { + return operations.RestoreOperation{}, rod + } + + rod.sw = store.NewKopiaStore(rod.kms) + + connectorResource := resource.Users + if sel.Service == selectors.ServiceSharePoint { + connectorResource = resource.Sites + } + + rod.ctrl, rod.sel = ControllerWithSelector( + t, + ctx, + rod.acct, + connectorResource, + sel, + nil, + rod.close) + + ro := newTestRestoreOp( + t, + ctx, + rod, + backupID, + bus, + ctr, + opts, + restoreCfg) + + rod.sss = streamstore.NewStreamer( + rod.kw, + rod.acct.ID(), + rod.sel.PathService()) + + return ro, rod +} + +// newTestRestoreOp accepts the clients required to compose a restore operation, plus +// any other metadata, and uses them to generate a new restore operation. This +// allows restore chains to utilize the same temp directory and configuration +// details. +func newTestRestoreOp( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + rod *restoreOpDependencies, + backupID model.StableID, + bus events.Eventer, + ctr *count.Bus, + opts control.Options, + restoreCfg control.RestoreConfig, +) operations.RestoreOperation { + rod.ctrl.IDNameLookup = idname.NewCache(map[string]string{rod.sel.ID(): rod.sel.Name()}) + + ro, err := operations.NewRestoreOperation( + ctx, + opts, + rod.kw, + rod.sw, + rod.ctrl, + rod.acct, + backupID, + rod.sel, + restoreCfg, + bus, + ctr) + if !assert.NoError(t, err, clues.ToCore(err)) { + rod.close(t, ctx) + t.FailNow() + } + + return ro +} + +func runAndCheckRestore( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + ro *operations.RestoreOperation, + mb *evmock.Bus, + acceptNoData bool, +) *details.Details { + deets, err := ro.Run(ctx) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, ro.Results, "the restore had non-zero results") + require.NotNil(t, deets, "restore details") + + expectStatus := []operations.OpStatus{operations.Completed} + if acceptNoData { + expectStatus = append(expectStatus, operations.NoData) + } + + require.Contains( + t, + expectStatus, + ro.Status, + "restore doesn't match expectation, wanted any of %v, got %s", + expectStatus, + ro.Status) + + assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure())) + + if assert.Empty(t, ro.Errors.Recovered(), "recoverable/iteration errors") { + allErrs := ro.Errors.Errors() + for i, err := range allErrs.Recovered { + t.Log("recovered from test err", i, err) + } + } + + assert.NotZero(t, ro.Results.ItemsRead, "count of items read") + assert.NotZero(t, ro.Results.BytesRead, "bytes read") + assert.Equal(t, 1, ro.Results.ResourceOwners, "count of resource owners") + assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") + assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events") + + return deets +} + +type GetItemsInContainerByCollisionKeyer[T any] interface { + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]T, error) +} + +func filterCollisionKeyResults[T any]( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + protectedResourceID, containerID string, + giicbck GetItemsInContainerByCollisionKeyer[T], + filterOut map[string]T, +) map[string]T { + m, err := giicbck.GetItemsInContainerByCollisionKey( + ctx, + protectedResourceID, + containerID) + require.NoError(t, err, clues.ToCore(err)) + + for k := range filterOut { + delete(m, k) + } + + return m +} + +func checkRestoreCounts( + t *testing.T, + ctr *count.Bus, + expectSkips, expectReplaces, expectNew int, +) { + t.Log("counted values", ctr.Values()) + t.Log("counted totals", ctr.TotalValues()) + + if expectSkips >= 0 { + assert.Equal( + t, + int64(expectSkips), + ctr.Total(count.CollisionSkip), + "count of collisions resolved by skip") + } + + if expectReplaces >= 0 { + assert.Equal( + t, + int64(expectReplaces), + ctr.Total(count.CollisionReplace), + "count of collisions resolved by replace") + } + + if expectNew >= 0 { + assert.Equal( + t, + int64(expectNew), + ctr.Total(count.NewItemCreated), + "count of new items or collisions resolved by copy") + } +} diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index a3221f937..1b5a52dc2 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -177,3 +177,35 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { } } } + +type SharePointRestoreIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestSharePointRestoreIntgSuite(t *testing.T) { + suite.Run(t, &SharePointRestoreIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs, storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *SharePointRestoreIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { + sel := selectors.NewSharePointBackup([]string{suite.its.userID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.Filter(sel.Library("documents")) + sel.DiscreteOwner = suite.its.siteID + + runDriveRestoreWithAdvancedOptions( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.siteDriveID, + suite.its.siteDriveRootFolderID) +} diff --git a/src/pkg/count/keys.go b/src/pkg/count/keys.go index 96513b056..593af4b46 100644 --- a/src/pkg/count/keys.go +++ b/src/pkg/count/keys.go @@ -3,5 +3,10 @@ package count type key string const ( - CollisionSkip key = "collision-skip" + // NewItemCreated should be used for non-skip, non-replace, + // non-meta item creation counting. IE: use it specifically + // for counting new items (no collision) or copied items. + NewItemCreated key = "new-item-created" + CollisionReplace key = "collision-replace" + CollisionSkip key = "collision-skip" ) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 54471149f..8a8abb33e 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -26,6 +26,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" rep "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -370,7 +371,8 @@ func (r repository) NewRestore( model.StableID(backupID), sel, restoreCfg, - r.Bus) + r.Bus, + count.New()) } func (r repository) NewMaintenance( diff --git a/src/pkg/selectors/testdata/onedrive.go b/src/pkg/selectors/testdata/onedrive.go index 8592d3d80..677bdf840 100644 --- a/src/pkg/selectors/testdata/onedrive.go +++ b/src/pkg/selectors/testdata/onedrive.go @@ -2,8 +2,10 @@ package testdata import "github.com/alcionai/corso/src/pkg/selectors" +const TestFolderName = "test" + // OneDriveBackupFolderScope is the standard folder scope that should be used // in integration backups with onedrive. func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope { - return sel.Folders([]string{"test"}, selectors.PrefixMatch()) + return sel.Folders([]string{TestFolderName}, selectors.PrefixMatch()) } diff --git a/src/pkg/selectors/testdata/sharepoint.go b/src/pkg/selectors/testdata/sharepoint.go index c6cc414ff..e6c3be58b 100644 --- a/src/pkg/selectors/testdata/sharepoint.go +++ b/src/pkg/selectors/testdata/sharepoint.go @@ -7,5 +7,5 @@ import ( // SharePointBackupFolderScope is the standard folder scope that should be used // in integration backups with sharepoint. func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope { - return sel.LibraryFolders([]string{"test"}, selectors.PrefixMatch()) + return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch()) } diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index 69caa110a..f997bd2e7 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -166,6 +166,27 @@ func (c Contacts) GetItemsInContainerByCollisionKey( return m, nil } +func (c Contacts) GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, +) (map[string]struct{}, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewContactsPager(userID, containerID, "id") + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]struct{}{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = struct{}{} + } + + return m, nil +} + // --------------------------------------------------------------------------- // item ID pager // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/contacts_pager_test.go b/src/pkg/services/m365/api/contacts_pager_test.go index cc826aa06..5f3561e6d 100644 --- a/src/pkg/services/m365/api/contacts_pager_test.go +++ b/src/pkg/services/m365/api/contacts_pager_test.go @@ -83,3 +83,47 @@ func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsInContainerByCollision assert.Truef(t, ok, "expected results to contain collision key: %s", e) } } + +func (suite *ContactsPagerIntgSuite) TestContacts_GetItemsIDsInContainer() { + t := suite.T() + ac := suite.its.ac.Contacts() + + ctx, flush := tester.NewContext(t) + defer flush() + + container, err := ac.GetContainerByID(ctx, suite.its.userID, api.DefaultContacts) + require.NoError(t, err, clues.ToCore(err)) + + msgs, err := ac.Stable. + Client(). + Users(). + ByUserId(suite.its.userID). + ContactFolders(). + ByContactFolderId(ptr.Val(container.GetId())). + Contacts(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(err)) + + ms := msgs.GetValue() + expect := map[string]struct{}{} + + for _, m := range ms { + expect[ptr.Val(m.GetId())] = struct{}{} + } + + results, err := suite.its.ac.Contacts(). + GetItemIDsInContainer(ctx, suite.its.userID, api.DefaultContacts) + require.NoError(t, err, clues.ToCore(err)) + require.Less(t, 0, len(results), "requires at least one result") + require.Equal(t, len(expect), len(results), "must have same count of items") + + for _, k := range expect { + t.Log("expects key", k) + } + + for k := range results { + t.Log("results key", k) + } + + assert.Equal(t, expect, results) +} diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go index 0b073571b..3ba6e4b46 100644 --- a/src/pkg/services/m365/api/drive_pager.go +++ b/src/pkg/services/m365/api/drive_pager.go @@ -67,7 +67,7 @@ func (p *driveItemPageCtrl) setNext(nextLink string) { p.builder = drives.NewItemItemsItemChildrenRequestBuilder(nextLink, p.gs.Adapter()) } -type DriveCollisionItem struct { +type DriveItemIDType struct { ItemID string IsFolder bool } @@ -75,7 +75,7 @@ type DriveCollisionItem struct { func (c Drives) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, -) (map[string]DriveCollisionItem, error) { +) (map[string]DriveItemIDType, error) { ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...) @@ -84,10 +84,34 @@ func (c Drives) GetItemsInContainerByCollisionKey( return nil, graph.Wrap(ctx, err, "enumerating drive items") } - m := map[string]DriveCollisionItem{} + m := map[string]DriveItemIDType{} for _, item := range items { - m[DriveItemCollisionKey(item)] = DriveCollisionItem{ + m[DriveItemCollisionKey(item)] = DriveItemIDType{ + ItemID: ptr.Val(item.GetId()), + IsFolder: item.GetFolder() != nil, + } + } + + return m, nil +} + +func (c Drives) GetItemIDsInContainer( + ctx context.Context, + driveID, containerID string, +) (map[string]DriveItemIDType, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewDriveItemPager(driveID, containerID, idAnd("file", "folder")...) + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]DriveItemIDType{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = DriveItemIDType{ ItemID: ptr.Val(item.GetId()), IsFolder: item.GetFolder() != nil, } diff --git a/src/pkg/services/m365/api/drive_pager_test.go b/src/pkg/services/m365/api/drive_pager_test.go index 2eb1f8e27..71177e2c8 100644 --- a/src/pkg/services/m365/api/drive_pager_test.go +++ b/src/pkg/services/m365/api/drive_pager_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -68,7 +69,7 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey() require.NoError(t, err, clues.ToCore(err)) ims := items.GetValue() - expect := make([]api.DriveCollisionItem, 0, len(ims)) + expect := make([]api.DriveItemIDType, 0, len(ims)) assert.NotEmptyf( t, @@ -103,3 +104,77 @@ func (suite *DrivePagerIntgSuite) TestDrives_GetItemsInContainerByCollisionKey() }) } } + +func (suite *DrivePagerIntgSuite) TestDrives_GetItemIDsInContainer() { + table := []struct { + name string + driveID string + rootFolderID string + }{ + { + name: "user drive", + driveID: suite.its.userDriveID, + rootFolderID: suite.its.userDriveRootFolderID, + }, + { + name: "site drive", + driveID: suite.its.siteDriveID, + rootFolderID: suite.its.siteDriveRootFolderID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + t.Log("drive", test.driveID) + t.Log("rootFolder", test.rootFolderID) + + items, err := suite.its.ac.Stable. + Client(). + Drives(). + ByDriveId(test.driveID). + Items(). + ByDriveItemId(test.rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(err)) + + igv := items.GetValue() + expect := map[string]api.DriveItemIDType{} + + assert.NotEmptyf( + t, + igv, + "need at least one item to compare in user %s drive %s folder %s", + suite.its.userID, test.driveID, test.rootFolderID) + + for _, itm := range igv { + expect[ptr.Val(itm.GetId())] = api.DriveItemIDType{ + ItemID: ptr.Val(itm.GetId()), + IsFolder: itm.GetFolder() != nil, + } + } + + results, err := suite.its.ac. + Drives(). + GetItemIDsInContainer(ctx, test.driveID, test.rootFolderID) + require.NoError(t, err, clues.ToCore(err)) + require.NotEmpty(t, results) + require.Equal(t, len(expect), len(results), "must have same count of items") + + for k := range expect { + t.Log("expects key", k) + } + + for k, v := range results { + t.Log("results key", k) + assert.NotEmpty(t, v, "all values should be populated") + } + + assert.Equal(t, expect, results) + }) + } +} diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index caf6a2934..5472239f8 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -247,6 +247,27 @@ func (c Mail) GetItemsInContainerByCollisionKey( return m, nil } +func (c Mail) GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, +) (map[string]struct{}, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewMailPager(userID, containerID, "id") + + items, err := enumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating contacts") + } + + m := map[string]struct{}{} + + for _, item := range items { + m[ptr.Val(item.GetId())] = struct{}{} + } + + return m, nil +} + // --------------------------------------------------------------------------- // delta item ID pager // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/mail_pager_test.go b/src/pkg/services/m365/api/mail_pager_test.go index 476e13401..d99c428a2 100644 --- a/src/pkg/services/m365/api/mail_pager_test.go +++ b/src/pkg/services/m365/api/mail_pager_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/users" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -83,3 +84,50 @@ func (suite *MailPagerIntgSuite) TestMail_GetItemsInContainerByCollisionKey() { assert.Truef(t, ok, "expected results to contain collision key: %s", e) } } + +func (suite *MailPagerIntgSuite) TestMail_GetItemsIDsInContainer() { + t := suite.T() + ac := suite.its.ac.Mail() + + ctx, flush := tester.NewContext(t) + defer flush() + + config := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{ + Top: ptr.To[int32](1000), + }, + } + + msgs, err := ac.Stable. + Client(). + Users(). + ByUserId(suite.its.userID). + MailFolders(). + ByMailFolderId(api.MailInbox). + Messages(). + Get(ctx, config) + require.NoError(t, err, clues.ToCore(err)) + + ms := msgs.GetValue() + expect := map[string]struct{}{} + + for _, m := range ms { + expect[ptr.Val(m.GetId())] = struct{}{} + } + + results, err := suite.its.ac.Mail(). + GetItemIDsInContainer(ctx, suite.its.userID, api.MailInbox) + require.NoError(t, err, clues.ToCore(err)) + require.Less(t, 0, len(results), "requires at least one result") + require.Equal(t, len(expect), len(results), "must have same count of items") + + for k := range expect { + t.Log("expects key", k) + } + + for k := range results { + t.Log("results key", k) + } + + assert.Equal(t, expect, results) +} diff --git a/src/pkg/services/m365/m365_test.go b/src/pkg/services/m365/m365_test.go index e83dcf609..59812ebbd 100644 --- a/src/pkg/services/m365/m365_test.go +++ b/src/pkg/services/m365/m365_test.go @@ -2,7 +2,6 @@ package m365 import ( "context" - "fmt" "testing" "github.com/alcionai/clues" @@ -508,8 +507,6 @@ func (suite *DiscoveryIntgSuite) TestGetUserInfo() { } for _, test := range table { suite.Run(test.name, func() { - fmt.Printf("\n-----\n%+v\n-----\n", test.name) - t := suite.T() ctx, flush := tester.NewContext(t)