Add progress logging to delta queries (#2345)
## Description Adds logging to delta enumeration to log every 1000 items or so (each page is set to 200 items) ## Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No ## Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup ## Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #2329 ## Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
7ee1575dc2
commit
4ab5040569
@ -12,10 +12,12 @@ import (
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph/api"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -234,6 +236,11 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
|
||||
resetDelta bool
|
||||
)
|
||||
|
||||
ctx = clues.AddAll(
|
||||
ctx,
|
||||
"category", selectors.ExchangeContact,
|
||||
"folder_id", directoryID)
|
||||
|
||||
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
|
||||
if err != nil {
|
||||
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/alcionai/corso/src/internal/common"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph/api"
|
||||
@ -19,6 +20,7 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -271,6 +273,11 @@ func (c Events) GetAddedAndRemovedItemIDs(
|
||||
errs *multierror.Error
|
||||
)
|
||||
|
||||
ctx = clues.AddAll(
|
||||
ctx,
|
||||
"category", selectors.ExchangeEvent,
|
||||
"calendar_id", calendarID)
|
||||
|
||||
if len(oldDelta) > 0 {
|
||||
builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
|
||||
pgr := &eventPager{service, builder, nil}
|
||||
|
||||
@ -12,11 +12,13 @@ import (
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph/api"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -259,6 +261,11 @@ func (c Mail) GetAddedAndRemovedItemIDs(
|
||||
resetDelta bool
|
||||
)
|
||||
|
||||
ctx = clues.AddAll(
|
||||
ctx,
|
||||
"category", selectors.ExchangeMail,
|
||||
"folder_id", directoryID)
|
||||
|
||||
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
|
||||
if err != nil {
|
||||
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
|
||||
|
||||
@ -65,6 +65,9 @@ func getItemsAddedAndRemovedFromContainer(
|
||||
deltaURL string
|
||||
)
|
||||
|
||||
itemCount := 0
|
||||
page := 0
|
||||
|
||||
for {
|
||||
// get the next page of data, check for standard errors
|
||||
resp, err := pager.getPage(ctx)
|
||||
@ -83,7 +86,13 @@ func getItemsAddedAndRemovedFromContainer(
|
||||
return nil, nil, "", err
|
||||
}
|
||||
|
||||
logger.Ctx(ctx).Infow("Got page", "items", len(items))
|
||||
itemCount += len(items)
|
||||
page++
|
||||
|
||||
// Log every ~1000 items (the page size we use is 200)
|
||||
if page%5 == 0 {
|
||||
logger.Ctx(ctx).Infow("queried items", "count", itemCount)
|
||||
}
|
||||
|
||||
// iterate through the items in the page
|
||||
for _, item := range items {
|
||||
@ -117,5 +126,7 @@ func getItemsAddedAndRemovedFromContainer(
|
||||
pager.setNext(nextLink)
|
||||
}
|
||||
|
||||
logger.Ctx(ctx).Infow("completed enumeration", "count", itemCount)
|
||||
|
||||
return addedIDs, removedIDs, deltaURL, nil
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user