From ea2bf19bd1b50fac303c2967ef3c1807fdf8da14 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:33:07 -0800 Subject: [PATCH] Wire up config for pager result limit (#4657) Allow callers of GetAddedRemovedItems to set the number of added items they'd like returned from the call. Will eventually be leveraged for Exchange preview backups Doesn't change existing limits for pagers since everything uses the default of zero for "get everything" --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../m365/collection/exchange/backup.go | 21 +++++++++++-------- .../m365/collection/exchange/handlers.go | 2 +- src/pkg/services/m365/api/client.go | 8 +++++++ src/pkg/services/m365/api/contacts_pager.go | 10 ++++----- src/pkg/services/m365/api/events_pager.go | 10 ++++----- src/pkg/services/m365/api/mail_pager.go | 10 ++++----- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/internal/m365/collection/exchange/backup.go b/src/internal/m365/collection/exchange/backup.go index 8fab92834..4d4ebe0cd 100644 --- a/src/internal/m365/collection/exchange/backup.go +++ b/src/internal/m365/collection/exchange/backup.go @@ -122,10 +122,13 @@ func populateCollections( return nil, el.Failure() } - cID := ptr.Val(c.GetId()) - var ( - err error + err error + itemConfig = api.CallConfig{ + CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta, + UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs, + } + cID = ptr.Val(c.GetId()) dp = dps[cID] prevDelta = dp.Delta prevPathStr = dp.Path // do not log: pii; log prevPath instead @@ -146,6 +149,11 @@ func populateCollections( continue } + ictx = clues.Add( + ictx, + "current_path", currPath, + "current_location", locPath) + delete(tombstones, cID) if len(prevPathStr) > 0 { @@ -158,18 +166,13 @@ func populateCollections( ictx = clues.Add(ictx, "previous_path", prevPath) - cc := api.CallConfig{ - CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta, - UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs, - } - addAndRem, err := bh.itemEnumerator(). GetAddedAndRemovedItemIDs( ictx, qp.ProtectedResource.ID(), cID, prevDelta, - cc) + itemConfig) if err != nil { if !graph.IsErrDeletedInFlight(err) { el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) diff --git a/src/internal/m365/collection/exchange/handlers.go b/src/internal/m365/collection/exchange/handlers.go index 35fb04c57..7924ecb1b 100644 --- a/src/internal/m365/collection/exchange/handlers.go +++ b/src/internal/m365/collection/exchange/handlers.go @@ -29,7 +29,7 @@ type addedAndRemovedItemGetter interface { GetAddedAndRemovedItemIDs( ctx context.Context, user, containerID, oldDeltaToken string, - cc api.CallConfig, + config api.CallConfig, ) (pagers.AddedAndRemoved, error) } diff --git a/src/pkg/services/m365/api/client.go b/src/pkg/services/m365/api/client.go index 60e8cbfbb..5b010d9b1 100644 --- a/src/pkg/services/m365/api/client.go +++ b/src/pkg/services/m365/api/client.go @@ -161,11 +161,19 @@ func (c Client) Post( // per-call config // --------------------------------------------------------------------------- +// CallConfig defines additional parameters to add to the query like field +// selection and limiting results. +// +// Not all calls may support all options. Check individual implementations to +// see what's supported and what's not. type CallConfig struct { Expand []string Select []string CanMakeDeltaQueries bool UseImmutableIDs bool + // LimitResults limits the returned results to the given number. If 0, returns + // all results. + LimitResults int } // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index a369d368f..42483ecf0 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -249,7 +249,7 @@ func (p *contactDeltaPager) ValidModTimes() bool { func (c Contacts) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - cc CallConfig, + config CallConfig, ) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, @@ -261,12 +261,12 @@ func (c Contacts) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd(lastModifiedDateTime)...) pager := c.NewContactsPager( userID, containerID, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Contactable]( @@ -274,7 +274,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - cc.CanMakeDeltaQueries, - 0, + config.CanMakeDeltaQueries, + config.LimitResults, pagers.AddedAndRemovedByAddtlData[models.Contactable]) } diff --git a/src/pkg/services/m365/api/events_pager.go b/src/pkg/services/m365/api/events_pager.go index 9ce2490ef..a5cc1a5e2 100644 --- a/src/pkg/services/m365/api/events_pager.go +++ b/src/pkg/services/m365/api/events_pager.go @@ -244,7 +244,7 @@ func (p *eventDeltaPager) ValidModTimes() bool { func (c Events) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - cc CallConfig, + config CallConfig, ) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, @@ -256,12 +256,12 @@ func (c Events) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd()...) pager := c.NewEventsPager( userID, containerID, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Eventable]( @@ -269,7 +269,7 @@ func (c Events) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - cc.CanMakeDeltaQueries, - 0, + config.CanMakeDeltaQueries, + config.LimitResults, pagers.AddedAndRemovedByAddtlData[models.Eventable]) } diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index 366812c6e..f99f55b93 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -246,7 +246,7 @@ func (p *mailDeltaPager) ValidModTimes() bool { func (c Mail) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - cc CallConfig, + config CallConfig, ) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, @@ -258,12 +258,12 @@ func (c Mail) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd(lastModifiedDateTime)...) pager := c.NewMailPager( userID, containerID, - cc.UseImmutableIDs, + config.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Messageable]( @@ -271,7 +271,7 @@ func (c Mail) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - cc.CanMakeDeltaQueries, - 0, + config.CanMakeDeltaQueries, + config.LimitResults, pagers.AddedAndRemovedByAddtlData[models.Messageable]) }