From 34832591ce50b9f93e688922ddf4ed542206aa55 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 1 Dec 2022 19:57:15 -0800 Subject: [PATCH] Use delta endpoint to fetch exchange mail items (#1620) ## Description Use delta endpoint to fetch exchange mail items. Disables picking only some item attributes as query parameters are not implemented in this version of Graph SDK. Switches to custom pagination iteration method so that next links and delta tokens can be extracted later on. ## Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :hamster: Trivial/Minor ## Issue(s) * #1612 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/connector/exchange/delta_get.go | 2 - .../connector/exchange/query_options.go | 7 +- .../connector/exchange/service_iterators.go | 70 +++++++++---------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/internal/connector/exchange/delta_get.go b/src/internal/connector/exchange/delta_get.go index 5d026e37c..cebc0329b 100644 --- a/src/internal/connector/exchange/delta_get.go +++ b/src/internal/connector/exchange/delta_get.go @@ -18,7 +18,6 @@ const ( // The following functions are based off the code in v0.41.0 of msgraph-sdk-go // for sending delta requests with query parameters. -//nolint:unused func createGetRequestInformationWithRequestConfiguration( baseRequestInfoFunc func() (*abs.RequestInformation, error), requestConfig *DeltaRequestBuilderGetRequestConfiguration, @@ -43,7 +42,6 @@ func createGetRequestInformationWithRequestConfiguration( return requestInfo, nil } -//nolint:unused func sendMessagesDeltaGet( ctx context.Context, m *msmaildelta.DeltaRequestBuilder, diff --git a/src/internal/connector/exchange/query_options.go b/src/internal/connector/exchange/query_options.go index 51e4a2bc0..42b7c499b 100644 --- a/src/internal/connector/exchange/query_options.go +++ b/src/internal/connector/exchange/query_options.go @@ -14,7 +14,6 @@ import ( msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events" msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders" msfolderitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders/item" - msmfmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders/item/messages" msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" msitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages/item" "github.com/pkg/errors" @@ -144,16 +143,16 @@ type DeltaRequestBuilderGetRequestConfiguration struct { QueryParameters *DeltaRequestBuilderGetQueryParameters } -func optionsForFolderMessages(moreOps []string) (*msmfmessage.MessagesRequestBuilderGetRequestConfiguration, error) { +func optionsForFolderMessages(moreOps []string) (*DeltaRequestBuilderGetRequestConfiguration, error) { selecting, err := buildOptions(moreOps, messages) if err != nil { return nil, err } - requestParameters := &msmfmessage.MessagesRequestBuilderGetQueryParameters{ + requestParameters := &DeltaRequestBuilderGetQueryParameters{ Select: selecting, } - options := &msmfmessage.MessagesRequestBuilderGetRequestConfiguration{ + options := &DeltaRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index f3d8f0b41..53e6001e4 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -8,6 +8,7 @@ import ( multierror "github.com/hashicorp/go-multierror" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" + mdelta "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders/item/messages/delta" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -16,6 +17,8 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" ) +const nextLinkKey = "@odata.nextLink" + // FilterContainersAndFillCollections is a utility function // that places the M365 object ids belonging to specific directories // into a Collection. Messages outside of those directories are omitted. @@ -281,57 +284,54 @@ func FetchMessageIDsFromDirectory( gs graph.Service, user, directoryID string, ) ([]string, error) { - ids := []string{} + var ( + errs *multierror.Error + ids []string + ) options, err := optionsForFolderMessages([]string{"id"}) if err != nil { return nil, errors.Wrap(err, "getting query options") } - response, err := gs.Client(). + builder := gs.Client(). UsersById(user). MailFoldersById(directoryID). Messages(). - Get(ctx, options) - if err != nil { - return nil, - errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - } + Delta() - pageIter, err := msgraphgocore.NewPageIterator( - response, - gs.Adapter(), - models.CreateMessageCollectionResponseFromDiscriminatorValue, - ) - if err != nil { - return nil, errors.Wrap(err, "creating graph iterator") - } - - var errs *multierror.Error - - err = pageIter.Iterate(ctx, func(pageItem any) bool { - item, ok := pageItem.(graph.Idable) - if !ok { - errs = multierror.Append(errs, errors.New("item without ID function")) - return true + for { + // TODO(ashmrtn): Update to pass options once graph SDK dependency is updated. + resp, err := sendMessagesDeltaGet(ctx, builder, options, gs.Adapter()) + if err != nil { + return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } - if item.GetId() == nil { - errs = multierror.Append(errs, errors.New("item with nil ID")) - return true + for _, item := range resp.GetValue() { + if item.GetId() == nil { + errs = multierror.Append( + errs, + errors.Errorf("item with nil ID in folder %s", directoryID), + ) + + // TODO(ashmrtn): Handle fail-fast. + continue + } + + ids = append(ids, *item.GetId()) } - ids = append(ids, *item.GetId()) + nextLinkIface := resp.GetAdditionalData()[nextLinkKey] + if nextLinkIface == nil { + break + } - return true - }) + nextLink := nextLinkIface.(*string) + if len(*nextLink) == 0 { + break + } - if err != nil { - return nil, errors.Wrap( - err, - support.ConnectorStackErrorTrace(err)+ - " :fetching messages from directory "+directoryID, - ) + builder = mdelta.NewDeltaRequestBuilder(*nextLink, gs.Adapter()) } return ids, errs.ErrorOrNil()