From 2bc3c8988523979767c3ea824dd16a0fd42b8fac Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 25 Oct 2023 19:27:49 -0600 Subject: [PATCH] fix api/mock circular ref (#4544) No logic changes. Just code movement/renaming. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup --- src/cli/backup/helpers_test.go | 28 ++++++++- .../m365/collection/drive/collections.go | 3 +- .../m365/collection/drive/collections_test.go | 59 ++++++++++--------- .../m365/collection/drive/handlers.go | 5 +- .../m365/collection/drive/restore_test.go | 3 +- .../m365/collection/drive/site_handler.go | 5 +- .../m365/collection/drive/url_cache_test.go | 3 +- .../collection/drive/user_drive_handler.go | 5 +- .../m365/collection/exchange/backup.go | 4 +- .../m365/collection/exchange/backup_test.go | 23 ++++---- .../m365/collection/exchange/handlers.go | 3 +- .../m365/collection/groups/backup_test.go | 5 +- .../m365/collection/groups/channel_handler.go | 3 +- .../m365/collection/groups/handlers.go | 4 +- .../m365/service/onedrive/mock/handlers.go | 21 +++---- .../m365/service/sharepoint/backup_test.go | 3 +- src/internal/operations/test/helper_test.go | 28 ++++++++- src/pkg/services/m365/api/channels_pager.go | 29 ++++----- src/pkg/services/m365/api/contacts_pager.go | 31 +++++----- src/pkg/services/m365/api/drive_pager.go | 35 ++++++----- src/pkg/services/m365/api/events_pager.go | 31 +++++----- src/pkg/services/m365/api/helper_test.go | 29 ++++++++- src/pkg/services/m365/api/mail_pager.go | 31 +++++----- src/pkg/services/m365/api/mock/mail.go | 28 --------- src/pkg/services/m365/api/mock/pager.go | 10 ++-- .../api/{item_pager.go => pagers/pagers.go} | 42 +++++++------ .../pagers_test.go} | 52 ++++++++-------- 27 files changed, 295 insertions(+), 228 deletions(-) delete mode 100644 src/pkg/services/m365/api/mock/mail.go rename src/pkg/services/m365/api/{item_pager.go => pagers/pagers.go} (94%) rename src/pkg/services/m365/api/{item_pager_test.go => pagers/pagers_test.go} (90%) diff --git a/src/cli/backup/helpers_test.go b/src/cli/backup/helpers_test.go index e3023f834..03bea250b 100644 --- a/src/cli/backup/helpers_test.go +++ b/src/cli/backup/helpers_test.go @@ -17,6 +17,7 @@ import ( cliTD "github.com/alcionai/corso/src/cli/testdata" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" + gmock "github.com/alcionai/corso/src/internal/m365/graph/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" @@ -24,11 +25,34 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/services/m365/api" - "github.com/alcionai/corso/src/pkg/services/m365/api/mock" "github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage/testdata" ) +// --------------------------------------------------------------------------- +// Gockable client +// --------------------------------------------------------------------------- + +// GockClient produces a new exchange api client that can be +// mocked using gock. +func gockClient(creds account.M365Config) (api.Client, error) { + s, err := gmock.NewService(creds) + if err != nil { + return api.Client{}, err + } + + li, err := gmock.NewService(creds, graph.NoTimeout()) + if err != nil { + return api.Client{}, err + } + + return api.Client{ + Credentials: creds, + Stable: s, + LargeItem: li, + }, nil +} + // --------------------------------------------------------------------------- // Suite Setup // --------------------------------------------------------------------------- @@ -64,7 +88,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - its.gockAC, err = mock.NewClient(creds) + its.gockAC, err = gockClient(creds) require.NoError(t, err, clues.ToCore(err)) // user drive diff --git a/src/internal/m365/collection/drive/collections.go b/src/internal/m365/collection/drive/collections.go index ff4bdc85d..efcead63b 100644 --- a/src/internal/m365/collection/drive/collections.go +++ b/src/internal/m365/collection/drive/collections.go @@ -27,6 +27,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) const restrictedDirectory = "Site Pages" @@ -676,7 +677,7 @@ func (c *Collections) PopulateDriveCollections( topLevelPackages map[string]struct{}, prevDeltaLink string, errs *fault.Bus, -) (api.DeltaUpdate, map[string]string, error) { +) (pagers.DeltaUpdate, map[string]string, error) { var ( el = errs.Local() newPrevPaths = map[string]string{} diff --git a/src/internal/m365/collection/drive/collections_test.go b/src/internal/m365/collection/drive/collections_test.go index fa8023224..5eba2c657 100644 --- a/src/internal/m365/collection/drive/collections_test.go +++ b/src/internal/m365/collection/drive/collections_test.go @@ -31,6 +31,7 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type statePath struct { @@ -843,7 +844,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestPopulateDriveCollections() { var ( mbh = mock.DefaultOneDriveBH(user) - du = api.DeltaUpdate{ + du = pagers.DeltaUpdate{ URL: "notempty", Reset: false, } @@ -1375,7 +1376,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { delItem("file", driveBasePath1, "root", true, false, false), }, }}, - DeltaUpdate: api.DeltaUpdate{URL: delta}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta}, }, }, }, @@ -1409,7 +1410,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("file", "file", driveBasePath1, "root", true, false, false), }, }}, - DeltaUpdate: api.DeltaUpdate{URL: delta}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta}, }, }, }, @@ -1444,7 +1445,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), }, }}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1484,7 +1485,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("file", "file2", driveBasePath1+"/folder", "folder", true, false, false), }, }}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1522,7 +1523,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), driveItem("file", "file2", driveBasePath1, "root", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta}, }, }, }, @@ -1561,7 +1562,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: empty, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: empty, Reset: true}, }, }, }, @@ -1609,7 +1610,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1671,7 +1672,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1729,7 +1730,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1771,7 +1772,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, driveID2: { Pages: []mock.NextPage{{Items: []models.DriveItemable{ @@ -1779,7 +1780,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("folder2", "folder", driveBasePath2, "root2", false, true, false), driveItem("file2", "file", driveBasePath2+"/folder", "folder2", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta2, Reset: true}, }, }, }, @@ -1831,7 +1832,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("folder", "folder", driveBasePath1, "root", false, true, false), driveItem("file", "file", driveBasePath1+"/folder", "folder", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, driveID2: { Pages: []mock.NextPage{{Items: []models.DriveItemable{ @@ -1839,7 +1840,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveItem("folder", "folder", driveBasePath2, "root", false, true, false), driveItem("file2", "file", driveBasePath2+"/folder", "folder", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta2, Reset: true}, }, }, }, @@ -1884,7 +1885,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { DrivePagers: map[string]*mock.DriveItemsDeltaPager{ driveID1: { Pages: []mock.NextPage{{Items: []models.DriveItemable{}}}, - DeltaUpdate: api.DeltaUpdate{}, + DeltaUpdate: pagers.DeltaUpdate{}, Err: assert.AnError, }, }, @@ -1918,7 +1919,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -1970,7 +1971,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2033,7 +2034,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2088,7 +2089,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2148,7 +2149,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2206,7 +2207,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta2, Reset: true}, }, }, }, @@ -2253,7 +2254,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, Reset: true, }}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2298,7 +2299,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { Reset: true, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2347,7 +2348,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta2, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta2, Reset: true}, }, }, }, @@ -2393,7 +2394,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, }, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2431,7 +2432,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveRootItem("root"), delItem("folder", driveBasePath1, "root", false, true, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2466,7 +2467,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { driveRootItem("root"), delItem("file", driveBasePath1, "root", true, false, false), }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta, Reset: true}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta, Reset: true}, }, }, }, @@ -2500,7 +2501,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { Pages: []mock.NextPage{{Items: []models.DriveItemable{ driveRootItem("root"), // will be present }}}, - DeltaUpdate: api.DeltaUpdate{URL: delta}, + DeltaUpdate: pagers.DeltaUpdate{URL: delta}, }, }, }, @@ -2794,7 +2795,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestAddURLCacheToDriveCollections() { ctx, flush := tester.NewContext(t) defer flush() - itemPagers := map[string]api.DeltaPager[models.DriveItemable]{} + itemPagers := map[string]pagers.DeltaHandler[models.DriveItemable]{} itemPagers[driveID] = &apiMock.DeltaPager[models.DriveItemable]{} mbh := mock.DefaultOneDriveBH("test-user") diff --git a/src/internal/m365/collection/drive/handlers.go b/src/internal/m365/collection/drive/handlers.go index 0b5211050..76c147144 100644 --- a/src/internal/m365/collection/drive/handlers.go +++ b/src/internal/m365/collection/drive/handlers.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type ItemInfoAugmenter interface { @@ -65,7 +66,7 @@ type BackupHandler interface { } type NewDrivePagerer interface { - NewDrivePager(resourceOwner string, fields []string) api.Pager[models.Driveable] + NewDrivePager(resourceOwner string, fields []string) pagers.NonDeltaHandler[models.Driveable] } type GetItemPermissioner interface { @@ -87,7 +88,7 @@ type EnumerateDriveItemsDeltaer interface { ctx context.Context, driveID, prevDeltaLink string, cc api.CallConfig, - ) api.NextPageResulter[models.DriveItemable] + ) pagers.NextPageResulter[models.DriveItemable] } // --------------------------------------------------------------------------- diff --git a/src/internal/m365/collection/drive/restore_test.go b/src/internal/m365/collection/drive/restore_test.go index 00aff96e0..b7efd6f11 100644 --- a/src/internal/m365/collection/drive/restore_test.go +++ b/src/internal/m365/collection/drive/restore_test.go @@ -27,6 +27,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type RestoreUnitSuite struct { @@ -423,7 +424,7 @@ func (m *mockGDPARF) GetRootFolder( func (m *mockGDPARF) NewDrivePager( string, []string, -) api.Pager[models.Driveable] { +) pagers.NonDeltaHandler[models.Driveable] { return m.pager } diff --git a/src/internal/m365/collection/drive/site_handler.go b/src/internal/m365/collection/drive/site_handler.go index 126f480c3..2500b565a 100644 --- a/src/internal/m365/collection/drive/site_handler.go +++ b/src/internal/m365/collection/drive/site_handler.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type baseSiteHandler struct { @@ -25,7 +26,7 @@ type baseSiteHandler struct { func (h baseSiteHandler) NewDrivePager( resourceOwner string, fields []string, -) api.Pager[models.Driveable] { +) pagers.NonDeltaHandler[models.Driveable] { return h.ac.NewSiteDrivePager(resourceOwner, fields) } @@ -176,7 +177,7 @@ func (h siteBackupHandler) EnumerateDriveItemsDelta( ctx context.Context, driveID, prevDeltaLink string, cc api.CallConfig, -) api.NextPageResulter[models.DriveItemable] { +) pagers.NextPageResulter[models.DriveItemable] { return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink, cc) } diff --git a/src/internal/m365/collection/drive/url_cache_test.go b/src/internal/m365/collection/drive/url_cache_test.go index 62f2d2e30..99327d8fa 100644 --- a/src/internal/m365/collection/drive/url_cache_test.go +++ b/src/internal/m365/collection/drive/url_cache_test.go @@ -26,6 +26,7 @@ import ( "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -556,7 +557,7 @@ func (suite *URLCacheUnitSuite) TestGetItemProperties() { driveID: { Pages: test.pages, Err: test.pagerErr, - DeltaUpdate: api.DeltaUpdate{URL: deltaString}, + DeltaUpdate: pagers.DeltaUpdate{URL: deltaString}, }, }, } diff --git a/src/internal/m365/collection/drive/user_drive_handler.go b/src/internal/m365/collection/drive/user_drive_handler.go index 3d19cd23c..dad4a1f6c 100644 --- a/src/internal/m365/collection/drive/user_drive_handler.go +++ b/src/internal/m365/collection/drive/user_drive_handler.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -29,7 +30,7 @@ type baseUserDriveHandler struct { func (h baseUserDriveHandler) NewDrivePager( resourceOwner string, fields []string, -) api.Pager[models.Driveable] { +) pagers.NonDeltaHandler[models.Driveable] { return h.ac.NewUserDrivePager(resourceOwner, fields) } @@ -176,7 +177,7 @@ func (h userDriveBackupHandler) EnumerateDriveItemsDelta( ctx context.Context, driveID, prevDeltaLink string, cc api.CallConfig, -) api.NextPageResulter[models.DriveItemable] { +) pagers.NextPageResulter[models.DriveItemable] { return h.ac.EnumerateDriveItemsDelta(ctx, driveID, prevDeltaLink, cc) } diff --git a/src/internal/m365/collection/exchange/backup.go b/src/internal/m365/collection/exchange/backup.go index 5999002ad..ae6ef7276 100644 --- a/src/internal/m365/collection/exchange/backup.go +++ b/src/internal/m365/collection/exchange/backup.go @@ -18,7 +18,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) func CreateCollections( @@ -176,7 +176,7 @@ func populateCollections( // to reset. This prevents any old items from being retained in // storage. If the container (or its children) are sill missing // on the next backup, they'll get tombstoned. - newDelta = api.DeltaUpdate{Reset: true} + newDelta = pagers.DeltaUpdate{Reset: true} } if len(newDelta.URL) > 0 { diff --git a/src/internal/m365/collection/exchange/backup_test.go b/src/internal/m365/collection/exchange/backup_test.go index 6e661983a..55d3c9b17 100644 --- a/src/internal/m365/collection/exchange/backup_test.go +++ b/src/internal/m365/collection/exchange/backup_test.go @@ -31,6 +31,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -65,7 +66,7 @@ type ( mockGetterResults struct { added []string removed []string - newDelta api.DeltaUpdate + newDelta pagers.DeltaUpdate err error } ) @@ -79,12 +80,12 @@ func (mg mockGetter) GetAddedAndRemovedItemIDs( map[string]time.Time, bool, []string, - api.DeltaUpdate, + pagers.DeltaUpdate, error, ) { results, ok := mg.results[cID] if !ok { - return nil, false, nil, api.DeltaUpdate{}, clues.New("mock not found for " + cID) + return nil, false, nil, pagers.DeltaUpdate{}, clues.New("mock not found for " + cID) } delta := results.newDelta @@ -919,18 +920,18 @@ func (suite *CollectionPopulationSuite) TestPopulateCollections() { commonResult = mockGetterResults{ added: []string{"a1", "a2", "a3"}, removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, + newDelta: pagers.DeltaUpdate{URL: "delta_url"}, } errorResult = mockGetterResults{ added: []string{"a1", "a2", "a3"}, removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, + newDelta: pagers.DeltaUpdate{URL: "delta_url"}, err: assert.AnError, } deletedInFlightResult = mockGetterResults{ added: []string{"a1", "a2", "a3"}, removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, + newDelta: pagers.DeltaUpdate{URL: "delta_url"}, err: graph.ErrDeletedInFlight, } container1 = mockContainer{ @@ -1235,12 +1236,12 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D result1 = mockGetterResults{ added: []string{"a1", "a2", "a3"}, removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, + newDelta: pagers.DeltaUpdate{URL: "delta_url"}, } result2 = mockGetterResults{ added: []string{"a4", "a5", "a6"}, removed: []string{"r4", "r5", "r6"}, - newDelta: api.DeltaUpdate{URL: "delta_url2"}, + newDelta: pagers.DeltaUpdate{URL: "delta_url2"}, } container1 = mockContainer{ @@ -1523,7 +1524,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D } func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_repeatedItems() { - newDelta := api.DeltaUpdate{URL: "delta_url"} + newDelta := pagers.DeltaUpdate{URL: "delta_url"} table := []struct { name string @@ -1693,11 +1694,11 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i allScope = selectors.NewExchangeBackup(nil).MailFolders(selectors.Any())[0] commonResults = mockGetterResults{ added: []string{"added"}, - newDelta: api.DeltaUpdate{URL: "new_delta_url"}, + newDelta: pagers.DeltaUpdate{URL: "new_delta_url"}, } expiredResults = mockGetterResults{ added: []string{"added"}, - newDelta: api.DeltaUpdate{ + newDelta: pagers.DeltaUpdate{ URL: "new_delta_url", Reset: true, }, diff --git a/src/internal/m365/collection/exchange/handlers.go b/src/internal/m365/collection/exchange/handlers.go index 8bedb8c10..9265c4c9b 100644 --- a/src/internal/m365/collection/exchange/handlers.go +++ b/src/internal/m365/collection/exchange/handlers.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -31,7 +32,7 @@ type addedAndRemovedItemGetter interface { user, containerID, oldDeltaToken string, immutableIDs bool, canMakeDeltaQueries bool, - ) (map[string]time.Time, bool, []string, api.DeltaUpdate, error) + ) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) } type itemGetterSerializer interface { diff --git a/src/internal/m365/collection/groups/backup_test.go b/src/internal/m365/collection/groups/backup_test.go index 1d4465ecc..7976bf5ca 100644 --- a/src/internal/m365/collection/groups/backup_test.go +++ b/src/internal/m365/collection/groups/backup_test.go @@ -29,6 +29,7 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -57,14 +58,14 @@ func (bh mockBackupHandler) getContainerItemIDs( _ context.Context, _, _ string, _ bool, -) (map[string]time.Time, bool, []string, api.DeltaUpdate, error) { +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { idRes := make(map[string]time.Time, len(bh.messageIDs)) for _, id := range bh.messageIDs { idRes[id] = time.Time{} } - return idRes, true, bh.deletedMsgIDs, api.DeltaUpdate{}, bh.messagesErr + return idRes, true, bh.deletedMsgIDs, pagers.DeltaUpdate{}, bh.messagesErr } func (bh mockBackupHandler) includeContainer( diff --git a/src/internal/m365/collection/groups/channel_handler.go b/src/internal/m365/collection/groups/channel_handler.go index 488e504b7..f136272f2 100644 --- a/src/internal/m365/collection/groups/channel_handler.go +++ b/src/internal/m365/collection/groups/channel_handler.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) var _ backupHandler = &channelsBackupHandler{} @@ -41,7 +42,7 @@ func (bh channelsBackupHandler) getContainerItemIDs( ctx context.Context, channelID, prevDelta string, canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, api.DeltaUpdate, error) { +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, canMakeDeltaQueries) } diff --git a/src/internal/m365/collection/groups/handlers.go b/src/internal/m365/collection/groups/handlers.go index 67c0211c8..ce859bad3 100644 --- a/src/internal/m365/collection/groups/handlers.go +++ b/src/internal/m365/collection/groups/handlers.go @@ -10,7 +10,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type backupHandler interface { @@ -26,7 +26,7 @@ type backupHandler interface { ctx context.Context, containerID, prevDelta string, canMakeDeltaQueries bool, - ) (map[string]time.Time, bool, []string, api.DeltaUpdate, error) + ) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) // includeContainer evaluates whether the container is included // in the provided scope. diff --git a/src/internal/m365/service/onedrive/mock/handlers.go b/src/internal/m365/service/onedrive/mock/handlers.go index d9b944478..cbb304730 100644 --- a/src/internal/m365/service/onedrive/mock/handlers.go +++ b/src/internal/m365/service/onedrive/mock/handlers.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -45,9 +46,9 @@ type BackupHandler[T any] struct { Service path.ServiceType Category path.CategoryType - DrivePagerV api.Pager[models.Driveable] + DrivePagerV pagers.NonDeltaHandler[models.Driveable] // driveID -> itemPager - ItemPagerV map[string]api.DeltaPager[models.DriveItemable] + ItemPagerV map[string]pagers.DeltaHandler[models.DriveItemable] LocationIDFn locationIDer @@ -136,7 +137,7 @@ func (h BackupHandler[T]) ServiceCat() (path.ServiceType, path.CategoryType) { return h.Service, h.Category } -func (h BackupHandler[T]) NewDrivePager(string, []string) api.Pager[models.Driveable] { +func (h BackupHandler[T]) NewDrivePager(string, []string) pagers.NonDeltaHandler[models.Driveable] { return h.DrivePagerV } @@ -174,7 +175,7 @@ func (h BackupHandler[T]) EnumerateDriveItemsDelta( ctx context.Context, driveID, prevDeltaLink string, cc api.CallConfig, -) api.NextPageResulter[models.DriveItemable] { +) pagers.NextPageResulter[models.DriveItemable] { return h.DriveItemEnumeration.EnumerateDriveItemsDelta( ctx, driveID, @@ -301,12 +302,12 @@ type EnumerateItemsDeltaByDrive struct { DrivePagers map[string]*DriveItemsDeltaPager } -var _ api.NextPageResulter[models.DriveItemable] = &DriveItemsDeltaPager{} +var _ pagers.NextPageResulter[models.DriveItemable] = &DriveItemsDeltaPager{} type DriveItemsDeltaPager struct { Idx int Pages []NextPage - DeltaUpdate api.DeltaUpdate + DeltaUpdate pagers.DeltaUpdate Err error } @@ -314,7 +315,7 @@ func (edibd EnumerateItemsDeltaByDrive) EnumerateDriveItemsDelta( _ context.Context, driveID, _ string, _ api.CallConfig, -) api.NextPageResulter[models.DriveItemable] { +) pagers.NextPageResulter[models.DriveItemable] { didp := edibd.DrivePagers[driveID] return didp } @@ -330,7 +331,7 @@ func (edi *DriveItemsDeltaPager) NextPage() ([]models.DriveItemable, bool, bool) return np.Items, np.Reset, false } -func (edi *DriveItemsDeltaPager) Results() (api.DeltaUpdate, error) { +func (edi *DriveItemsDeltaPager) Results() (pagers.DeltaUpdate, error) { return edi.DeltaUpdate, edi.Err } @@ -367,7 +368,7 @@ type RestoreHandler struct { PostItemResp models.DriveItemable PostItemErr error - DrivePagerV api.Pager[models.Driveable] + DrivePagerV pagers.NonDeltaHandler[models.Driveable] PostDriveResp models.Driveable PostDriveErr error @@ -382,7 +383,7 @@ func (h RestoreHandler) PostDrive( return h.PostDriveResp, h.PostDriveErr } -func (h RestoreHandler) NewDrivePager(string, []string) api.Pager[models.Driveable] { +func (h RestoreHandler) NewDrivePager(string, []string) pagers.NonDeltaHandler[models.Driveable] { return h.DrivePagerV } diff --git a/src/internal/m365/service/sharepoint/backup_test.go b/src/internal/m365/service/sharepoint/backup_test.go index 35e48edf6..198e6247b 100644 --- a/src/internal/m365/service/sharepoint/backup_test.go +++ b/src/internal/m365/service/sharepoint/backup_test.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- @@ -92,7 +93,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { var ( mbh = mock.DefaultSharePointBH(siteID) - du = api.DeltaUpdate{ + du = pagers.DeltaUpdate{ URL: "notempty", Reset: false, } diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 84b8b8431..8deddc4d0 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/m365" "github.com/alcionai/corso/src/internal/m365/graph" + gmock "github.com/alcionai/corso/src/internal/m365/graph/mock" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/internal/model" @@ -40,12 +41,35 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" - "github.com/alcionai/corso/src/pkg/services/m365/api/mock" "github.com/alcionai/corso/src/pkg/storage" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" "github.com/alcionai/corso/src/pkg/store" ) +// --------------------------------------------------------------------------- +// Gockable client +// --------------------------------------------------------------------------- + +// GockClient produces a new exchange api client that can be +// mocked using gock. +func gockClient(creds account.M365Config) (api.Client, error) { + s, err := gmock.NewService(creds) + if err != nil { + return api.Client{}, err + } + + li, err := gmock.NewService(creds, graph.NoTimeout()) + if err != nil { + return api.Client{}, err + } + + return api.Client{ + Credentials: creds, + Stable: s, + LargeItem: li, + }, nil +} + // Does not use the tester.DefaultTestRestoreDestination syntax as some of these // items are created directly, not as a result of restoration, and we want to ensure // they get clearly selected without accidental overlap. @@ -601,7 +625,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - its.gockAC, err = mock.NewClient(creds) + its.gockAC, err = gockClient(creds) require.NoError(t, err, clues.ToCore(err)) its.user = userIDs(t, tconfig.M365UserID(t), its.ac) diff --git a/src/pkg/services/m365/api/channels_pager.go b/src/pkg/services/m365/api/channels_pager.go index 2f069a5bf..21b284d36 100644 --- a/src/pkg/services/m365/api/channels_pager.go +++ b/src/pkg/services/m365/api/channels_pager.go @@ -10,13 +10,14 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- // channel message pager // --------------------------------------------------------------------------- -var _ Pager[models.ChatMessageable] = &channelMessagePageCtrl{} +var _ pagers.NonDeltaHandler[models.ChatMessageable] = &channelMessagePageCtrl{} type channelMessagePageCtrl struct { resourceID, channelID string @@ -31,7 +32,7 @@ func (p *channelMessagePageCtrl) SetNextLink(nextLink string) { func (p *channelMessagePageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.ChatMessageable], error) { +) (pagers.NextLinkValuer[models.ChatMessageable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -83,7 +84,7 @@ func (c Channels) GetChannelMessages( ) ([]models.ChatMessageable, error) { ctx = clues.Add(ctx, "channel_id", channelID) pager := c.NewChannelMessagePager(teamID, channelID, cc) - items, err := batchEnumerateItems[models.ChatMessageable](ctx, pager) + items, err := pagers.BatchEnumerateItems[models.ChatMessageable](ctx, pager) return items, graph.Stack(ctx, err).OrNil() } @@ -92,7 +93,7 @@ func (c Channels) GetChannelMessages( // channel message delta pager // --------------------------------------------------------------------------- -var _ DeltaPager[models.ChatMessageable] = &channelMessageDeltaPageCtrl{} +var _ pagers.DeltaHandler[models.ChatMessageable] = &channelMessageDeltaPageCtrl{} type channelMessageDeltaPageCtrl struct { resourceID, channelID string @@ -107,7 +108,7 @@ func (p *channelMessageDeltaPageCtrl) SetNextLink(nextLink string) { func (p *channelMessageDeltaPageCtrl) GetPage( ctx context.Context, -) (DeltaLinkValuer[models.ChatMessageable], error) { +) (pagers.DeltaLinkValuer[models.ChatMessageable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -187,14 +188,14 @@ func (c Channels) GetChannelMessageIDs( ctx context.Context, teamID, channelID, prevDeltaLink string, canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, DeltaUpdate, error) { - added, validModTimes, removed, du, err := getAddedAndRemovedItemIDs[models.ChatMessageable]( +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { + added, validModTimes, removed, du, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable]( ctx, c.NewChannelMessagePager(teamID, channelID, CallConfig{}), c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink), prevDeltaLink, canMakeDeltaQueries, - addedAndRemovedByDeletedDateTime[models.ChatMessageable], + pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable], FilterOutSystemMessages) return added, validModTimes, removed, du, clues.Stack(err).OrNil() @@ -204,7 +205,7 @@ func (c Channels) GetChannelMessageIDs( // channel message replies pager // --------------------------------------------------------------------------- -var _ Pager[models.ChatMessageable] = &channelMessageRepliesPageCtrl{} +var _ pagers.NonDeltaHandler[models.ChatMessageable] = &channelMessageRepliesPageCtrl{} type channelMessageRepliesPageCtrl struct { gs graph.Servicer @@ -218,7 +219,7 @@ func (p *channelMessageRepliesPageCtrl) SetNextLink(nextLink string) { func (p *channelMessageRepliesPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.ChatMessageable], error) { +) (pagers.NextLinkValuer[models.ChatMessageable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -265,7 +266,7 @@ func (c Channels) GetChannelMessageReplies( ctx context.Context, teamID, channelID, messageID string, ) ([]models.ChatMessageable, error) { - return batchEnumerateItems[models.ChatMessageable]( + return pagers.BatchEnumerateItems[models.ChatMessageable]( ctx, c.NewChannelMessageRepliesPager(teamID, channelID, messageID)) } @@ -274,7 +275,7 @@ func (c Channels) GetChannelMessageReplies( // channel pager // --------------------------------------------------------------------------- -var _ Pager[models.Channelable] = &channelPageCtrl{} +var _ pagers.NonDeltaHandler[models.Channelable] = &channelPageCtrl{} type channelPageCtrl struct { gs graph.Servicer @@ -288,7 +289,7 @@ func (p *channelPageCtrl) SetNextLink(nextLink string) { func (p *channelPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.Channelable], error) { +) (pagers.NextLinkValuer[models.Channelable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -322,5 +323,5 @@ func (c Channels) GetChannels( ctx context.Context, teamID string, ) ([]models.Channelable, error) { - return batchEnumerateItems[models.Channelable](ctx, c.NewChannelPager(teamID)) + return pagers.BatchEnumerateItems[models.Channelable](ctx, c.NewChannelPager(teamID)) } diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index 688650f99..e2dfe6622 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -11,13 +11,14 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- // container pager // --------------------------------------------------------------------------- -var _ Pager[models.ContactFolderable] = &contactsFoldersPageCtrl{} +var _ pagers.NonDeltaHandler[models.ContactFolderable] = &contactsFoldersPageCtrl{} type contactsFoldersPageCtrl struct { gs graph.Servicer @@ -29,7 +30,7 @@ func (c Contacts) NewContactFoldersPager( userID, baseContainerID string, immutableIDs bool, selectProps ...string, -) Pager[models.ContactFolderable] { +) pagers.NonDeltaHandler[models.ContactFolderable] { options := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{}, @@ -53,7 +54,7 @@ func (c Contacts) NewContactFoldersPager( func (p *contactsFoldersPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.ContactFolderable], error) { +) (pagers.NextLinkValuer[models.ContactFolderable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -72,7 +73,7 @@ func (c Contacts) EnumerateContainers( userID, baseContainerID string, immutableIDs bool, ) ([]models.ContactFolderable, error) { - containers, err := batchEnumerateItems(ctx, c.NewContactFoldersPager(userID, baseContainerID, immutableIDs)) + containers, err := pagers.BatchEnumerateItems(ctx, c.NewContactFoldersPager(userID, baseContainerID, immutableIDs)) return containers, graph.Stack(ctx, err).OrNil() } @@ -80,7 +81,7 @@ func (c Contacts) EnumerateContainers( // item pager // --------------------------------------------------------------------------- -var _ Pager[models.Contactable] = &contactsPageCtrl{} +var _ pagers.NonDeltaHandler[models.Contactable] = &contactsPageCtrl{} type contactsPageCtrl struct { gs graph.Servicer @@ -92,7 +93,7 @@ func (c Contacts) NewContactsPager( userID, containerID string, immutableIDs bool, selectProps ...string, -) Pager[models.Contactable] { +) pagers.NonDeltaHandler[models.Contactable] { options := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{}, @@ -116,7 +117,7 @@ func (c Contacts) NewContactsPager( func (p *contactsPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.Contactable], error) { +) (pagers.NextLinkValuer[models.Contactable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -136,7 +137,7 @@ func (c Contacts) GetItemsInContainerByCollisionKey( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewContactsPager(userID, containerID, false, contactCollisionKeyProps()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating contacts") } @@ -157,7 +158,7 @@ func (c Contacts) GetItemIDsInContainer( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewContactsPager(userID, containerID, false, idAnd()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating contacts") } @@ -175,7 +176,7 @@ func (c Contacts) GetItemIDsInContainer( // delta item ID pager // --------------------------------------------------------------------------- -var _ DeltaPager[models.Contactable] = &contactDeltaPager{} +var _ pagers.DeltaHandler[models.Contactable] = &contactDeltaPager{} type contactDeltaPager struct { gs graph.Servicer @@ -206,7 +207,7 @@ func (c Contacts) NewContactsDeltaPager( userID, containerID, prevDeltaLink string, immutableIDs bool, selectProps ...string, -) DeltaPager[models.Contactable] { +) pagers.DeltaHandler[models.Contactable] { options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{ // do NOT set Top. It limits the total items received. QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{}, @@ -229,7 +230,7 @@ func (c Contacts) NewContactsDeltaPager( func (p *contactDeltaPager) GetPage( ctx context.Context, -) (DeltaLinkValuer[models.Contactable], error) { +) (pagers.DeltaLinkValuer[models.Contactable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -251,7 +252,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink string, immutableIDs bool, canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, DeltaUpdate, error) { +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ctx = clues.Add( ctx, "data_category", path.ContactsCategory, @@ -270,11 +271,11 @@ func (c Contacts) GetAddedAndRemovedItemIDs( immutableIDs, idAnd(lastModifiedDateTime)...) - return getAddedAndRemovedItemIDs[models.Contactable]( + return pagers.GetAddedAndRemovedItemIDs[models.Contactable]( ctx, pager, deltaPager, prevDeltaLink, canMakeDeltaQueries, - addedAndRemovedByAddtlData[models.Contactable]) + pagers.AddedAndRemovedByAddtlData[models.Contactable]) } diff --git a/src/pkg/services/m365/api/drive_pager.go b/src/pkg/services/m365/api/drive_pager.go index 708fa3439..e0bdbf944 100644 --- a/src/pkg/services/m365/api/drive_pager.go +++ b/src/pkg/services/m365/api/drive_pager.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" onedrive "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) type DriveItemIDType struct { @@ -24,7 +25,7 @@ type DriveItemIDType struct { // non-delta item pager // --------------------------------------------------------------------------- -var _ Pager[models.DriveItemable] = &driveItemPageCtrl{} +var _ pagers.NonDeltaHandler[models.DriveItemable] = &driveItemPageCtrl{} type driveItemPageCtrl struct { gs graph.Servicer @@ -35,7 +36,7 @@ type driveItemPageCtrl struct { func (c Drives) NewDriveItemPager( driveID, containerID string, selectProps ...string, -) Pager[models.DriveItemable] { +) pagers.NonDeltaHandler[models.DriveItemable] { options := &drives.ItemItemsItemChildrenRequestBuilderGetRequestConfiguration{ QueryParameters: &drives.ItemItemsItemChildrenRequestBuilderGetQueryParameters{}, } @@ -57,7 +58,7 @@ func (c Drives) NewDriveItemPager( func (p *driveItemPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.DriveItemable], error) { +) (pagers.NextLinkValuer[models.DriveItemable], error) { page, err := p.builder.Get(ctx, p.options) return page, graph.Stack(ctx, err).OrNil() } @@ -77,7 +78,7 @@ func (c Drives) GetItemsInContainerByCollisionKey( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewDriveItemPager(driveID, containerID, idAnd("name")...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating drive items") } @@ -101,7 +102,7 @@ func (c Drives) GetItemIDsInContainer( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewDriveItemPager(driveID, containerID, idAnd("file", "folder")...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating contacts") } @@ -122,7 +123,7 @@ func (c Drives) GetItemIDsInContainer( // delta item pager // --------------------------------------------------------------------------- -var _ DeltaPager[models.DriveItemable] = &DriveItemDeltaPageCtrl{} +var _ pagers.DeltaHandler[models.DriveItemable] = &DriveItemDeltaPageCtrl{} type DriveItemDeltaPageCtrl struct { gs graph.Servicer @@ -175,7 +176,7 @@ func (c Drives) newDriveItemDeltaPager( func (p *DriveItemDeltaPageCtrl) GetPage( ctx context.Context, -) (DeltaLinkValuer[models.DriveItemable], error) { +) (pagers.DeltaLinkValuer[models.DriveItemable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -204,20 +205,18 @@ func (c Drives) EnumerateDriveItemsDelta( driveID string, prevDeltaLink string, cc CallConfig, -) NextPageResulter[models.DriveItemable] { +) pagers.NextPageResulter[models.DriveItemable] { deltaPager := c.newDriveItemDeltaPager( driveID, prevDeltaLink, cc) - npr := &nextPageResults[models.DriveItemable]{ - pages: make(chan nextPage[models.DriveItemable]), - } + npr := pagers.NewNextPageResults[models.DriveItemable]() // asynchronously enumerate pages on the caller's behalf. // they only need to consume the pager and call Results at // the end. - go deltaEnumerateItems[models.DriveItemable]( + go pagers.DeltaEnumerateItems[models.DriveItemable]( ctx, deltaPager, npr, @@ -230,7 +229,7 @@ func (c Drives) EnumerateDriveItemsDelta( // user's drives pager // --------------------------------------------------------------------------- -var _ Pager[models.Driveable] = &userDrivePager{} +var _ pagers.NonDeltaHandler[models.Driveable] = &userDrivePager{} type userDrivePager struct { userID string @@ -277,7 +276,7 @@ func (nl nopUserDrivePage) GetOdataNextLink() *string { func (p *userDrivePager) GetPage( ctx context.Context, -) (NextLinkValuer[models.Driveable], error) { +) (pagers.NextLinkValuer[models.Driveable], error) { // we only ever want to return the user's default drive. d, err := p.gs. Client(). @@ -301,7 +300,7 @@ func (p *userDrivePager) ValidModTimes() bool { // site's libraries pager // --------------------------------------------------------------------------- -var _ Pager[models.Driveable] = &siteDrivePager{} +var _ pagers.NonDeltaHandler[models.Driveable] = &siteDrivePager{} type siteDrivePager struct { gs graph.Servicer @@ -339,7 +338,7 @@ func (c Drives) NewSiteDrivePager( func (p *siteDrivePager) GetPage( ctx context.Context, -) (NextLinkValuer[models.Driveable], error) { +) (pagers.NextLinkValuer[models.Driveable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -359,9 +358,9 @@ func (p *siteDrivePager) ValidModTimes() bool { // GetAllDrives fetches all drives for the given pager func GetAllDrives( ctx context.Context, - pager Pager[models.Driveable], + pager pagers.NonDeltaHandler[models.Driveable], ) ([]models.Driveable, error) { - ds, err := batchEnumerateItems(ctx, pager) + ds, err := pagers.BatchEnumerateItems(ctx, pager) // no license or drives available. // return a non-error and let the caller assume an empty result set. diff --git a/src/pkg/services/m365/api/events_pager.go b/src/pkg/services/m365/api/events_pager.go index 624e5fad6..278125b53 100644 --- a/src/pkg/services/m365/api/events_pager.go +++ b/src/pkg/services/m365/api/events_pager.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) const eventBetaDeltaURLTemplate = "https://graph.microsoft.com/beta/users/%s/calendars/%s/events/delta" @@ -20,7 +21,7 @@ const eventBetaDeltaURLTemplate = "https://graph.microsoft.com/beta/users/%s/cal // container pager // --------------------------------------------------------------------------- -var _ Pager[models.Calendarable] = &eventsCalendarsPageCtrl{} +var _ pagers.NonDeltaHandler[models.Calendarable] = &eventsCalendarsPageCtrl{} type eventsCalendarsPageCtrl struct { gs graph.Servicer @@ -32,7 +33,7 @@ func (c Events) NewEventCalendarsPager( userID string, immutableIDs bool, selectProps ...string, -) Pager[models.Calendarable] { +) pagers.NonDeltaHandler[models.Calendarable] { options := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{}, @@ -54,7 +55,7 @@ func (c Events) NewEventCalendarsPager( func (p *eventsCalendarsPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.Calendarable], error) { +) (pagers.NextLinkValuer[models.Calendarable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -73,7 +74,7 @@ func (c Events) EnumerateContainers( userID, _ string, // baseContainerID not needed here immutableIDs bool, ) ([]models.Calendarable, error) { - containers, err := batchEnumerateItems(ctx, c.NewEventCalendarsPager(userID, immutableIDs)) + containers, err := pagers.BatchEnumerateItems(ctx, c.NewEventCalendarsPager(userID, immutableIDs)) return containers, graph.Stack(ctx, err).OrNil() } @@ -81,7 +82,7 @@ func (c Events) EnumerateContainers( // item pager // --------------------------------------------------------------------------- -var _ Pager[models.Eventable] = &eventsPageCtrl{} +var _ pagers.NonDeltaHandler[models.Eventable] = &eventsPageCtrl{} type eventsPageCtrl struct { gs graph.Servicer @@ -93,7 +94,7 @@ func (c Events) NewEventsPager( userID, containerID string, immutableIDs bool, selectProps ...string, -) Pager[models.Eventable] { +) pagers.NonDeltaHandler[models.Eventable] { options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{}, @@ -117,7 +118,7 @@ func (c Events) NewEventsPager( func (p *eventsPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.Eventable], error) { +) (pagers.NextLinkValuer[models.Eventable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -137,7 +138,7 @@ func (c Events) GetItemsInContainerByCollisionKey( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewEventsPager(userID, containerID, false, eventCollisionKeyProps()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating events") } @@ -158,7 +159,7 @@ func (c Events) GetItemIDsInContainer( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewEventsPager(userID, containerID, false, idAnd()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating events") } @@ -176,7 +177,7 @@ func (c Events) GetItemIDsInContainer( // delta item ID pager // --------------------------------------------------------------------------- -var _ DeltaPager[models.Eventable] = &eventDeltaPager{} +var _ pagers.DeltaHandler[models.Eventable] = &eventDeltaPager{} type eventDeltaPager struct { gs graph.Servicer @@ -200,7 +201,7 @@ func (c Events) NewEventsDeltaPager( userID, containerID, prevDeltaLink string, immutableIDs bool, selectProps ...string, -) DeltaPager[models.Eventable] { +) pagers.DeltaHandler[models.Eventable] { options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ // do NOT set Top. It limits the total items received. QueryParameters: &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{}, @@ -224,7 +225,7 @@ func (c Events) NewEventsDeltaPager( func (p *eventDeltaPager) GetPage( ctx context.Context, -) (DeltaLinkValuer[models.Eventable], error) { +) (pagers.DeltaLinkValuer[models.Eventable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -246,7 +247,7 @@ func (c Events) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink string, immutableIDs bool, canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, DeltaUpdate, error) { +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ctx = clues.Add( ctx, "data_category", path.EventsCategory, @@ -265,11 +266,11 @@ func (c Events) GetAddedAndRemovedItemIDs( immutableIDs, idAnd(lastModifiedDateTime)...) - return getAddedAndRemovedItemIDs[models.Eventable]( + return pagers.GetAddedAndRemovedItemIDs[models.Eventable]( ctx, pager, deltaPager, prevDeltaLink, canMakeDeltaQueries, - addedAndRemovedByAddtlData[models.Eventable]) + pagers.AddedAndRemovedByAddtlData[models.Eventable]) } diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index ac30e3f74..4c8f97152 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -14,13 +14,38 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" + gmock "github.com/alcionai/corso/src/internal/m365/graph/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" - "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) +// --------------------------------------------------------------------------- +// Gockable client +// --------------------------------------------------------------------------- + +// GockClient produces a new exchange api client that can be +// mocked using gock. +func gockClient(creds account.M365Config) (api.Client, error) { + s, err := gmock.NewService(creds) + if err != nil { + return api.Client{}, err + } + + li, err := gmock.NewService(creds, graph.NoTimeout()) + if err != nil { + return api.Client{}, err + } + + return api.Client{ + Credentials: creds, + Stable: s, + LargeItem: li, + }, nil +} + // --------------------------------------------------------------------------- // Intercepting calls with Gock // --------------------------------------------------------------------------- @@ -107,7 +132,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - its.gockAC, err = mock.NewClient(creds) + its.gockAC, err = gockClient(creds) require.NoError(t, err, clues.ToCore(err)) // user drive diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index 7bfa2b516..13f9e849e 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -12,13 +12,14 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) // --------------------------------------------------------------------------- // container pager // --------------------------------------------------------------------------- -var _ Pager[models.MailFolderable] = &mailFoldersPageCtrl{} +var _ pagers.NonDeltaHandler[models.MailFolderable] = &mailFoldersPageCtrl{} type mailFoldersPageCtrl struct { gs graph.Servicer @@ -30,7 +31,7 @@ func (c Mail) NewMailFoldersPager( userID string, immutableIDs bool, selectProps ...string, -) Pager[models.MailFolderable] { +) pagers.NonDeltaHandler[models.MailFolderable] { options := &users.ItemMailFoldersRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemMailFoldersRequestBuilderGetQueryParameters{}, @@ -50,7 +51,7 @@ func (c Mail) NewMailFoldersPager( func (p *mailFoldersPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.MailFolderable], error) { +) (pagers.NextLinkValuer[models.MailFolderable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -69,7 +70,7 @@ func (c Mail) EnumerateContainers( userID, _ string, // baseContainerID not needed here immutableIDs bool, ) ([]models.MailFolderable, error) { - containers, err := batchEnumerateItems(ctx, c.NewMailFoldersPager(userID, immutableIDs)) + containers, err := pagers.BatchEnumerateItems(ctx, c.NewMailFoldersPager(userID, immutableIDs)) return containers, graph.Stack(ctx, err).OrNil() } @@ -77,7 +78,7 @@ func (c Mail) EnumerateContainers( // item pager // --------------------------------------------------------------------------- -var _ Pager[models.Messageable] = &mailsPageCtrl{} +var _ pagers.NonDeltaHandler[models.Messageable] = &mailsPageCtrl{} type mailsPageCtrl struct { gs graph.Servicer @@ -89,7 +90,7 @@ func (c Mail) NewMailPager( userID, containerID string, immutableIDs bool, selectProps ...string, -) Pager[models.Messageable] { +) pagers.NonDeltaHandler[models.Messageable] { options := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{}, @@ -113,7 +114,7 @@ func (c Mail) NewMailPager( func (p *mailsPageCtrl) GetPage( ctx context.Context, -) (NextLinkValuer[models.Messageable], error) { +) (pagers.NextLinkValuer[models.Messageable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -133,7 +134,7 @@ func (c Mail) GetItemsInContainerByCollisionKey( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewMailPager(userID, containerID, false, mailCollisionKeyProps()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating mails") } @@ -154,7 +155,7 @@ func (c Mail) GetItemIDsInContainer( ctx = clues.Add(ctx, "container_id", containerID) pager := c.NewMailPager(userID, containerID, false, idAnd()...) - items, err := batchEnumerateItems(ctx, pager) + items, err := pagers.BatchEnumerateItems(ctx, pager) if err != nil { return nil, graph.Wrap(ctx, err, "enumerating mails") } @@ -172,7 +173,7 @@ func (c Mail) GetItemIDsInContainer( // delta item ID pager // --------------------------------------------------------------------------- -var _ DeltaPager[models.Messageable] = &mailDeltaPager{} +var _ pagers.DeltaHandler[models.Messageable] = &mailDeltaPager{} type mailDeltaPager struct { gs graph.Servicer @@ -203,7 +204,7 @@ func (c Mail) NewMailDeltaPager( userID, containerID, prevDeltaLink string, immutableIDs bool, selectProps ...string, -) DeltaPager[models.Messageable] { +) pagers.DeltaHandler[models.Messageable] { options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{ // do NOT set Top. It limits the total items received. QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{}, @@ -226,7 +227,7 @@ func (c Mail) NewMailDeltaPager( func (p *mailDeltaPager) GetPage( ctx context.Context, -) (DeltaLinkValuer[models.Messageable], error) { +) (pagers.DeltaLinkValuer[models.Messageable], error) { resp, err := p.builder.Get(ctx, p.options) return resp, graph.Stack(ctx, err).OrNil() } @@ -248,7 +249,7 @@ func (c Mail) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink string, immutableIDs bool, canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, DeltaUpdate, error) { +) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ctx = clues.Add( ctx, "data_category", path.EmailCategory, @@ -267,11 +268,11 @@ func (c Mail) GetAddedAndRemovedItemIDs( immutableIDs, idAnd(lastModifiedDateTime)...) - return getAddedAndRemovedItemIDs[models.Messageable]( + return pagers.GetAddedAndRemovedItemIDs[models.Messageable]( ctx, pager, deltaPager, prevDeltaLink, canMakeDeltaQueries, - addedAndRemovedByAddtlData[models.Messageable]) + pagers.AddedAndRemovedByAddtlData[models.Messageable]) } diff --git a/src/pkg/services/m365/api/mock/mail.go b/src/pkg/services/m365/api/mock/mail.go deleted file mode 100644 index 8a71d1067..000000000 --- a/src/pkg/services/m365/api/mock/mail.go +++ /dev/null @@ -1,28 +0,0 @@ -package mock - -import ( - "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/internal/m365/graph/mock" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -// NewClient produces a new exchange api client that can be -// mocked using gock. -func NewClient(creds account.M365Config) (api.Client, error) { - s, err := mock.NewService(creds) - if err != nil { - return api.Client{}, err - } - - li, err := mock.NewService(creds, graph.NoTimeout()) - if err != nil { - return api.Client{}, err - } - - return api.Client{ - Credentials: creds, - Stable: s, - LargeItem: li, - }, nil -} diff --git a/src/pkg/services/m365/api/mock/pager.go b/src/pkg/services/m365/api/mock/pager.go index bccf5b428..5a1e9ab4b 100644 --- a/src/pkg/services/m365/api/mock/pager.go +++ b/src/pkg/services/m365/api/mock/pager.go @@ -5,12 +5,12 @@ import ( "github.com/alcionai/clues" - "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) var ( - _ api.Pager[any] = &Pager[any]{} - _ api.DeltaPager[any] = &DeltaPager[any]{} + _ pagers.NonDeltaHandler[any] = &Pager[any]{} + _ pagers.DeltaHandler[any] = &DeltaPager[any]{} ) type DeltaNextLinkValues[T any] struct { @@ -50,7 +50,7 @@ type Pager[T any] struct { func (p *Pager[T]) GetPage( context.Context, -) (api.NextLinkValuer[T], error) { +) (pagers.NextLinkValuer[T], error) { if len(p.ToReturn) <= p.getIdx { return nil, clues.New("index out of bounds"). With("index", p.getIdx, "values", p.ToReturn) @@ -81,7 +81,7 @@ type DeltaPager[T any] struct { func (p *DeltaPager[T]) GetPage( context.Context, -) (api.DeltaLinkValuer[T], error) { +) (pagers.DeltaLinkValuer[T], error) { if len(p.ToReturn) <= p.getIdx { return nil, clues.New("index out of bounds"). With("index", p.getIdx, "values", p.ToReturn) diff --git a/src/pkg/services/m365/api/item_pager.go b/src/pkg/services/m365/api/pagers/pagers.go similarity index 94% rename from src/pkg/services/m365/api/item_pager.go rename to src/pkg/services/m365/api/pagers/pagers.go index ef15836a4..daf31dce5 100644 --- a/src/pkg/services/m365/api/item_pager.go +++ b/src/pkg/services/m365/api/pagers/pagers.go @@ -1,4 +1,4 @@ -package api +package pagers import ( "context" @@ -59,6 +59,12 @@ type nextPageResults[T any] struct { err error } +func NewNextPageResults[T any]() *nextPageResults[T] { + return &nextPageResults[T]{ + pages: make(chan nextPage[T]), + } +} + func (npr *nextPageResults[T]) writeNextPage( ctx context.Context, items []T, @@ -171,15 +177,15 @@ func NextAndDeltaLink(pl DeltaLinker) (string, string) { // non-delta item paging // --------------------------------------------------------------------------- -type Pager[T any] interface { +type NonDeltaHandler[T any] interface { GetPager[NextLinkValuer[T]] SetNextLinker ValidModTimer } -func enumerateItems[T any]( +func EnumerateItems[T any]( ctx context.Context, - pager Pager[T], + pager NonDeltaHandler[T], npr *nextPageResults[T], ) { defer npr.close() @@ -211,9 +217,9 @@ func enumerateItems[T any]( logger.Ctx(ctx).Infow("completed delta item enumeration", "result_count", len(result)) } -func batchEnumerateItems[T any]( +func BatchEnumerateItems[T any]( ctx context.Context, - pager Pager[T], + pager NonDeltaHandler[T], ) ([]T, error) { var ( npr = nextPageResults[T]{ @@ -222,7 +228,7 @@ func batchEnumerateItems[T any]( items = []T{} ) - go enumerateItems[T](ctx, pager, &npr) + go EnumerateItems[T](ctx, pager, &npr) for page, _, done := npr.NextPage(); !done; page, _, done = npr.NextPage() { items = append(items, page...) @@ -237,7 +243,7 @@ func batchEnumerateItems[T any]( // generic handler for delta-based item paging // --------------------------------------------------------------------------- -type DeltaPager[T any] interface { +type DeltaHandler[T any] interface { GetPager[DeltaLinkValuer[T]] Resetter SetNextLinker @@ -247,9 +253,9 @@ type DeltaPager[T any] interface { // enumerates pages of items, streaming each page to the provided channel. // the DeltaUpdate, reset notifications, and any errors are also fed to the // same channel. -func deltaEnumerateItems[T any]( +func DeltaEnumerateItems[T any]( ctx context.Context, - pager DeltaPager[T], + pager DeltaHandler[T], npr *nextPageResults[T], prevDeltaLink string, ) { @@ -335,7 +341,7 @@ func deltaEnumerateItems[T any]( func batchDeltaEnumerateItems[T any]( ctx context.Context, - pager DeltaPager[T], + pager DeltaHandler[T], prevDeltaLink string, ) ([]T, DeltaUpdate, error) { var ( @@ -345,7 +351,7 @@ func batchDeltaEnumerateItems[T any]( results = []T{} ) - go deltaEnumerateItems[T](ctx, pager, &npr, prevDeltaLink) + go DeltaEnumerateItems[T](ctx, pager, &npr, prevDeltaLink) for page, reset, done := npr.NextPage(); !done; page, reset, done = npr.NextPage() { if reset { @@ -373,10 +379,10 @@ type addedAndRemovedHandler[T any] func( error, ) -func getAddedAndRemovedItemIDs[T any]( +func GetAddedAndRemovedItemIDs[T any]( ctx context.Context, - pager Pager[T], - deltaPager DeltaPager[T], + pager NonDeltaHandler[T], + deltaPager DeltaHandler[T], prevDeltaLink string, canMakeDeltaQueries bool, aarh addedAndRemovedHandler[T], @@ -396,7 +402,7 @@ func getAddedAndRemovedItemIDs[T any]( du := DeltaUpdate{Reset: true} - ts, err := batchEnumerateItems(ctx, pager) + ts, err := BatchEnumerateItems(ctx, pager) if err != nil { return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err) } @@ -425,7 +431,7 @@ type getModTimer interface { GetLastModifiedDateTime() *time.Time } -func addedAndRemovedByAddtlData[T any]( +func AddedAndRemovedByAddtlData[T any]( items []T, filters ...func(T) bool, ) (map[string]time.Time, []string, error) { @@ -484,7 +490,7 @@ type getIDModAndDeletedDateTimer interface { GetDeletedDateTime() *time.Time } -func addedAndRemovedByDeletedDateTime[T any]( +func AddedAndRemovedByDeletedDateTime[T any]( items []T, filters ...func(T) bool, ) (map[string]time.Time, []string, error) { diff --git a/src/pkg/services/m365/api/item_pager_test.go b/src/pkg/services/m365/api/pagers/pagers_test.go similarity index 90% rename from src/pkg/services/m365/api/item_pager_test.go rename to src/pkg/services/m365/api/pagers/pagers_test.go index f3f260f68..c62f04ab2 100644 --- a/src/pkg/services/m365/api/item_pager_test.go +++ b/src/pkg/services/m365/api/pagers/pagers_test.go @@ -1,4 +1,4 @@ -package api +package pagers import ( "context" @@ -86,7 +86,7 @@ func (p testPage) GetValue() []testItem { // mock item pager -var _ Pager[testItem] = &testPager{} +var _ NonDeltaHandler[testItem] = &testPager{} type testPager struct { t *testing.T @@ -104,7 +104,7 @@ func (p testPager) ValidModTimes() bool { return true } // mock id pager -var _ Pager[testItem] = &testIDsPager{} +var _ NonDeltaHandler[testItem] = &testIDsPager{} type testIDsPager struct { t *testing.T @@ -166,7 +166,7 @@ func (p testIDsPager) ValidModTimes() bool { return p.validModTimes } -var _ DeltaPager[testItem] = &testIDsDeltaPager{} +var _ DeltaHandler[testItem] = &testIDsDeltaPager{} type testIDsDeltaPager struct { t *testing.T @@ -243,7 +243,7 @@ func TestPagerUnitSuite(t *testing.T) { func (suite *PagerUnitSuite) TestEnumerateItems() { tests := []struct { name string - getPager func(*testing.T, context.Context) Pager[testItem] + getPager func(*testing.T, context.Context) NonDeltaHandler[testItem] expect []testItem expectErr require.ErrorAssertionFunc }{ @@ -252,7 +252,7 @@ func (suite *PagerUnitSuite) TestEnumerateItems() { getPager: func( t *testing.T, ctx context.Context, - ) Pager[testItem] { + ) NonDeltaHandler[testItem] { return &testPager{ t: t, pager: testPage{[]testItem{{id: "foo"}, {id: "bar"}}}, @@ -266,7 +266,7 @@ func (suite *PagerUnitSuite) TestEnumerateItems() { getPager: func( t *testing.T, ctx context.Context, - ) Pager[testItem] { + ) NonDeltaHandler[testItem] { return &testPager{ t: t, pageErr: assert.AnError, @@ -284,7 +284,7 @@ func (suite *PagerUnitSuite) TestEnumerateItems() { ctx, flush := tester.NewContext(t) defer flush() - result, err := batchEnumerateItems(ctx, test.getPager(t, ctx)) + result, err := BatchEnumerateItems(ctx, test.getPager(t, ctx)) test.expectErr(t, err, clues.ToCore(err)) require.EqualValues(t, test.expect, result) @@ -308,10 +308,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { name string pagerGetter func( *testing.T, - ) Pager[testItem] + ) NonDeltaHandler[testItem] deltaPagerGetter func( *testing.T, - ) DeltaPager[testItem] + ) DeltaHandler[testItem] prevDelta string filter func(a testItem) bool expect expected @@ -320,10 +320,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }{ { name: "no prev delta", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -347,10 +347,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "no prev delta invalid mod times", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -372,10 +372,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "with prev delta", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -400,10 +400,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "delta expired", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -430,7 +430,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "delta not allowed", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return &testIDsPager{ t: t, added: map[string]time.Time{ @@ -441,7 +441,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { validModTimes: true, } }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return nil }, expect: expected{ @@ -457,10 +457,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "no prev delta and fail all filter", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -482,10 +482,10 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { }, { name: "with prev delta and fail all filter", - pagerGetter: func(t *testing.T) Pager[testItem] { + pagerGetter: func(t *testing.T) NonDeltaHandler[testItem] { return nil }, - deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] { + deltaPagerGetter: func(t *testing.T) DeltaHandler[testItem] { return &testIDsDeltaPager{ t: t, added: map[string]time.Time{ @@ -520,13 +520,13 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { filters = append(filters, test.filter) } - added, validModTimes, removed, deltaUpdate, err := getAddedAndRemovedItemIDs[testItem]( + added, validModTimes, removed, deltaUpdate, err := GetAddedAndRemovedItemIDs[testItem]( ctx, test.pagerGetter(t), test.deltaPagerGetter(t), test.prevDelta, test.canDelta, - addedAndRemovedByAddtlData[testItem], + AddedAndRemovedByAddtlData[testItem], filters...) require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err))