From 87427a52aa2dc14d8cb56f76e39813d48e983a81 Mon Sep 17 00:00:00 2001 From: Danny Date: Tue, 2 Aug 2022 13:31:53 -0400 Subject: [PATCH] Gc iterate abstraction (#459) Abstraction functions and iterate functions created for the `serializeMessages` move to createCollections. Graph Connector refactor still underway. --- .../connector/exchange/service_functions.go | 39 +++++++++++++++++- .../connector/exchange/service_query.go | 40 +++++++++++++++++++ src/internal/connector/graph_connector.go | 25 ++---------- .../graph_connector_disconnected_test.go | 12 +++--- .../connector/optionidentifier_string.go | 26 ------------ src/internal/connector/query.go | 36 ----------------- 6 files changed, 87 insertions(+), 91 deletions(-) delete mode 100644 src/internal/connector/optionidentifier_string.go delete mode 100644 src/internal/connector/query.go diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 8ab740bc7..fdfe4c22a 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -1,15 +1,52 @@ package exchange import ( + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/internal/connector/graph" "github.com/alcionai/corso/internal/connector/support" + "github.com/alcionai/corso/pkg/account" ) +type exchangeService struct { + client msgraphsdk.GraphServiceClient + adapter msgraphsdk.GraphRequestAdapter + failFast bool // if true service will exit sequence upon encountering an error + credentials account.M365Config +} + +func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient { + return &es.client +} + +func (es *exchangeService) Adapter() *msgraphsdk.GraphRequestAdapter { + return &es.adapter +} + +func (es *exchangeService) ErrPolicy() bool { + return es.failFast +} + +func createService(credentials account.M365Config, shouldFailFast bool) (*exchangeService, error) { + adapter, err := graph.CreateAdapter( + credentials.TenantID, + credentials.ClientID, + credentials.ClientSecret, + ) + if err != nil { + return nil, err + } + service := exchangeService{ + adapter: *adapter, + client: *msgraphsdk.NewGraphServiceClient(adapter), + failFast: shouldFailFast, + } + return &service, err +} + // CreateMailFolder makes a mail folder iff a folder of the same name does not exist // Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http func CreateMailFolder(gs graph.Service, user, folder string) (models.MailFolderable, error) { diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index f111e2475..347683e41 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -2,15 +2,20 @@ package exchange import ( absser "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-sdk-go/models" msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders" msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" "github.com/pkg/errors" "github.com/alcionai/corso/internal/connector/graph" + "github.com/alcionai/corso/internal/connector/support" + "github.com/alcionai/corso/pkg/account" ) type optionIdentifier int +const mailCategory = "mail" + //go:generate stringer -type=optionIdentifier const ( unknown optionIdentifier = iota @@ -34,6 +39,38 @@ func GetAllMessagesForUser(gs graph.Service, identities []string) (absser.Parsab return gs.Client().UsersById(identities[0]).Messages().GetWithRequestConfigurationAndResponseHandler(options, nil) } +// IterateMessageCollection utility function for Iterating through MessagesCollectionResponse +// During iteration, Collections are added to the map based on the parent folder +func IterateMessagesCollection( + tenant, user string, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + return func(messageItem any) bool { + message, ok := messageItem.(models.Messageable) + if !ok { + errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs) + return true + } + // Saving to messages to list. Indexed by folder + directory := *message.GetParentFolderId() + if _, ok = collections[directory]; !ok { + service, err := createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return true + } + edc := NewCollection(user, []string{tenant, user, mailCategory, directory}, service, statusCh) + collections[directory] = &edc + } + collections[directory].AddJob(*message.GetId()) + return true + } +} + //--------------------------------------------------- // exchange.Query Option Section //------------------------------------------------ @@ -83,6 +120,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) { "displayName": 1, "isHidden": 2, "parentFolderId": 3, + "id": 4, } fieldsForUsers := map[string]int{ @@ -93,6 +131,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) { "department": 5, "displayName": 6, "employeeId": 7, + "id": 8, } fieldsForMessages := map[string]int{ @@ -101,6 +140,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) { "parentFolderId": 3, "subject": 4, "webLink": 5, + "id": 6, } returnedOptions := []string{"id"} diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index b165018b0..00dae8012 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -202,6 +202,7 @@ func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector s errs) continue } + // TODO: Creates a map of collections based on scope dcs, err := gc.serializeMessages(ctx, user) if err != nil { return nil, support.WrapAndAppend(user, err, errs) @@ -301,8 +302,7 @@ func (gc *GraphConnector) RestoreMessages(ctx context.Context, dcs []data.Collec // to the GraphConnector struct. func (gc *GraphConnector) serializeMessages(ctx context.Context, user string) (map[string]*exchange.Collection, error) { var transformer absser.ParsableFactory - options := optionsForMessageSnapshot() - response, err := gc.graphService.client.UsersById(user).Messages().GetWithRequestConfigurationAndResponseHandler(options, nil) + response, err := exchange.GetAllMessagesForUser(&gc.graphService, []string{user}) //TODO: Selector to be used for exchange.query if err != nil { return nil, err } @@ -316,26 +316,7 @@ func (gc *GraphConnector) serializeMessages(ctx context.Context, user string) (m var errs error // callbackFunc iterates through all models.Messageable and fills exchange.Collection.jobs[] // with corresponding messageIDs. New collections are created for each directory - callbackFunc := func(messageItem any) bool { - message, ok := messageItem.(models.Messageable) - if !ok { - errs = support.WrapAndAppendf(gc.graphService.adapter.GetBaseUrl(), errors.New("message iteration failure"), err) - return true - } - // Saving to messages to list. Indexed by folder - directory := *message.GetParentFolderId() - if _, ok = collections[directory]; !ok { - service, err := gc.createService(gc.failFast) - if err != nil { - errs = support.WrapAndAppend(user, err, errs) - return true - } - edc := exchange.NewCollection(user, []string{gc.tenant, user, mailCategory, directory}, service, gc.statusCh) - collections[directory] = &edc - } - collections[directory].AddJob(*message.GetId()) - return true - } + callbackFunc := exchange.IterateMessagesCollection(gc.tenant, user, errs, gc.failFast, gc.credentials, collections, gc.statusCh) iterateError := pageIterator.Iterate(callbackFunc) if iterateError != nil { errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs) diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 73fce3efd..18e98c879 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -75,12 +75,12 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() { names["Axel"] = "Foley" first := buildFromMap(true, names) last := buildFromMap(false, names) - suite.True(Contains(first, "Al")) - suite.True(Contains(first, "Ellen")) - suite.True(Contains(first, "Axel")) - suite.True(Contains(last, "Bundy")) - suite.True(Contains(last, "Ripley")) - suite.True(Contains(last, "Foley")) + suite.Contains(first, "Al") + suite.Contains(first, "Ellen") + suite.Contains(first, "Axel") + suite.Contains(last, "Bundy") + suite.Contains(last, "Ripley") + suite.Contains(last, "Foley") } diff --git a/src/internal/connector/optionidentifier_string.go b/src/internal/connector/optionidentifier_string.go deleted file mode 100644 index e2f2a70ab..000000000 --- a/src/internal/connector/optionidentifier_string.go +++ /dev/null @@ -1,26 +0,0 @@ -// Code generated by "stringer -type=optionIdentifier"; DO NOT EDIT. - -package connector - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[unknown-0] - _ = x[folders-1] - _ = x[messages-2] - _ = x[users-3] -} - -const _optionIdentifier_name = "unknownfoldersmessagesusers" - -var _optionIdentifier_index = [...]uint8{0, 7, 14, 22, 27} - -func (i optionIdentifier) String() string { - if i < 0 || i >= optionIdentifier(len(_optionIdentifier_index)-1) { - return "optionIdentifier(" + strconv.FormatInt(int64(i), 10) + ")" - } - return _optionIdentifier_name[_optionIdentifier_index[i]:_optionIdentifier_index[i+1]] -} diff --git a/src/internal/connector/query.go b/src/internal/connector/query.go deleted file mode 100644 index d39d64678..000000000 --- a/src/internal/connector/query.go +++ /dev/null @@ -1,36 +0,0 @@ -package connector - -import ( - msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" -) - -type optionIdentifier int - -//go:generate stringer -type=optionIdentifier -const ( - unknown optionIdentifier = iota - folders - messages - users -) - -// Contains is a helper method for verifying if element -// is contained within the slice -func Contains(elems []string, value string) bool { - for _, s := range elems { - if value == s { - return true - } - } - return false -} - -func optionsForMessageSnapshot() *msmessage.MessagesRequestBuilderGetRequestConfiguration { - selecting := []string{"id", "parentFolderId"} - options := &msmessage.MessagesRequestBuilderGetRequestConfiguration{ - QueryParameters: &msmessage.MessagesRequestBuilderGetQueryParameters{ - Select: selecting, - }, - } - return options -}