Add progress logging to delta queries (#2345)

## Description

Adds logging to delta enumeration to log every 1000 items or so (each page is set to 200 items)

## Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #2329 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Vaibhav Kamra 2023-01-31 19:39:00 -08:00 committed by GitHub
parent 7ee1575dc2
commit 4ab5040569
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 33 additions and 1 deletions

View File

@ -12,10 +12,12 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
@ -234,6 +236,11 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
resetDelta bool
)
ctx = clues.AddAll(
ctx,
"category", selectors.ExchangeContact,
"folder_id", directoryID)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")

View File

@ -12,6 +12,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
@ -19,6 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
@ -271,6 +273,11 @@ func (c Events) GetAddedAndRemovedItemIDs(
errs *multierror.Error
)
ctx = clues.AddAll(
ctx,
"category", selectors.ExchangeEvent,
"calendar_id", calendarID)
if len(oldDelta) > 0 {
builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr := &eventPager{service, builder, nil}

View File

@ -12,11 +12,13 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
@ -259,6 +261,11 @@ func (c Mail) GetAddedAndRemovedItemIDs(
resetDelta bool
)
ctx = clues.AddAll(
ctx,
"category", selectors.ExchangeMail,
"folder_id", directoryID)
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")

View File

@ -65,6 +65,9 @@ func getItemsAddedAndRemovedFromContainer(
deltaURL string
)
itemCount := 0
page := 0
for {
// get the next page of data, check for standard errors
resp, err := pager.getPage(ctx)
@ -83,7 +86,13 @@ func getItemsAddedAndRemovedFromContainer(
return nil, nil, "", err
}
logger.Ctx(ctx).Infow("Got page", "items", len(items))
itemCount += len(items)
page++
// Log every ~1000 items (the page size we use is 200)
if page%5 == 0 {
logger.Ctx(ctx).Infow("queried items", "count", itemCount)
}
// iterate through the items in the page
for _, item := range items {
@ -117,5 +126,7 @@ func getItemsAddedAndRemovedFromContainer(
pager.setNext(nextLink)
}
logger.Ctx(ctx).Infow("completed enumeration", "count", itemCount)
return addedIDs, removedIDs, deltaURL, nil
}