From 4ab504056900ed709041736aead9c69356ce52e8 Mon Sep 17 00:00:00 2001
From: Vaibhav Kamra <vkamra@alcion.ai>
Date: Tue, 31 Jan 2023 19:39:00 -0800
Subject: [PATCH] Add progress logging to delta queries (#2345)

## Description

Adds logging to delta enumeration to log every 1000 items or so (each page is set to 200 items)

## Does this PR need a docs update or release note?

- [ ] :white_check_mark: Yes, it's included
- [ ] :clock1: Yes, but in a later PR
- [x] :no_entry: No

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] :sunflower: Feature
- [x] :bug: Bugfix
- [ ] :world_map: Documentation
- [ ] :robot: Test
- [ ] :computer: CI/Deployment
- [ ] :broom: Tech Debt/Cleanup

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #2329

## Test Plan

<!-- How will this be tested prior to merging.-->
- [x] :muscle: Manual
- [ ] :zap: Unit test
- [ ] :green_heart: E2E
---
 src/internal/connector/exchange/api/contacts.go |  7 +++++++
 src/internal/connector/exchange/api/events.go   |  7 +++++++
 src/internal/connector/exchange/api/mail.go     |  7 +++++++
 src/internal/connector/exchange/api/shared.go   | 13 ++++++++++++-
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go
index 33f05d2a3..185bf6e22 100644
--- a/src/internal/connector/exchange/api/contacts.go
+++ b/src/internal/connector/exchange/api/contacts.go
@@ -12,10 +12,12 @@ import (
 	"github.com/microsoftgraph/msgraph-sdk-go/users"
 	"github.com/pkg/errors"
 
+	"github.com/alcionai/clues"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/graph/api"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 // ---------------------------------------------------------------------------
@@ -234,6 +236,11 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
 		resetDelta bool
 	)
 
+	ctx = clues.AddAll(
+		ctx,
+		"category", selectors.ExchangeContact,
+		"folder_id", directoryID)
+
 	options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
 	if err != nil {
 		return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go
index 63545143d..962b2e576 100644
--- a/src/internal/connector/exchange/api/events.go
+++ b/src/internal/connector/exchange/api/events.go
@@ -12,6 +12,7 @@ import (
 	"github.com/microsoftgraph/msgraph-sdk-go/users"
 	"github.com/pkg/errors"
 
+	"github.com/alcionai/clues"
 	"github.com/alcionai/corso/src/internal/common"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/graph/api"
@@ -19,6 +20,7 @@ import (
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 // ---------------------------------------------------------------------------
@@ -271,6 +273,11 @@ func (c Events) GetAddedAndRemovedItemIDs(
 		errs       *multierror.Error
 	)
 
+	ctx = clues.AddAll(
+		ctx,
+		"category", selectors.ExchangeEvent,
+		"calendar_id", calendarID)
+
 	if len(oldDelta) > 0 {
 		builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
 		pgr := &eventPager{service, builder, nil}
diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go
index 597676874..29c03b82f 100644
--- a/src/internal/connector/exchange/api/mail.go
+++ b/src/internal/connector/exchange/api/mail.go
@@ -12,11 +12,13 @@ import (
 	"github.com/microsoftgraph/msgraph-sdk-go/users"
 	"github.com/pkg/errors"
 
+	"github.com/alcionai/clues"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/graph/api"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 // ---------------------------------------------------------------------------
@@ -259,6 +261,11 @@ func (c Mail) GetAddedAndRemovedItemIDs(
 		resetDelta bool
 	)
 
+	ctx = clues.AddAll(
+		ctx,
+		"category", selectors.ExchangeMail,
+		"folder_id", directoryID)
+
 	options, err := optionsForFolderMessagesDelta([]string{"isRead"})
 	if err != nil {
 		return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
diff --git a/src/internal/connector/exchange/api/shared.go b/src/internal/connector/exchange/api/shared.go
index 9cb4d8ab0..e4d563e90 100644
--- a/src/internal/connector/exchange/api/shared.go
+++ b/src/internal/connector/exchange/api/shared.go
@@ -65,6 +65,9 @@ func getItemsAddedAndRemovedFromContainer(
 		deltaURL   string
 	)
 
+	itemCount := 0
+	page := 0
+
 	for {
 		// get the next page of data, check for standard errors
 		resp, err := pager.getPage(ctx)
@@ -83,7 +86,13 @@ func getItemsAddedAndRemovedFromContainer(
 			return nil, nil, "", err
 		}
 
-		logger.Ctx(ctx).Infow("Got page", "items", len(items))
+		itemCount += len(items)
+		page++
+
+		// Log every ~1000 items (the page size we use is 200)
+		if page%5 == 0 {
+			logger.Ctx(ctx).Infow("queried items", "count", itemCount)
+		}
 
 		// iterate through the items in the page
 		for _, item := range items {
@@ -117,5 +126,7 @@ func getItemsAddedAndRemovedFromContainer(
 		pager.setNext(nextLink)
 	}
 
+	logger.Ctx(ctx).Infow("completed enumeration", "count", itemCount)
+
 	return addedIDs, removedIDs, deltaURL, nil
 }