Gc iterate abstraction (#459)

Abstraction functions and iterate functions created for the `serializeMessages` move to createCollections. 
Graph Connector refactor still underway.

src/internal/connector/exchange/service_functions.go

@@ -1,15 +1,52 @@
 package exchange
 
 import (
+	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
 	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
-
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/internal/connector/graph"
 	"github.com/alcionai/corso/internal/connector/support"
+	"github.com/alcionai/corso/pkg/account"
 )
 
+type exchangeService struct {
+	client      msgraphsdk.GraphServiceClient
+	adapter     msgraphsdk.GraphRequestAdapter
+	failFast    bool // if true service will exit sequence upon encountering an error
+	credentials account.M365Config
+}
+
+func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient {
+	return &es.client
+}
+
+func (es *exchangeService) Adapter() *msgraphsdk.GraphRequestAdapter {
+	return &es.adapter
+}
+
+func (es *exchangeService) ErrPolicy() bool {
+	return es.failFast
+}
+
+func createService(credentials account.M365Config, shouldFailFast bool) (*exchangeService, error) {
+	adapter, err := graph.CreateAdapter(
+		credentials.TenantID,
+		credentials.ClientID,
+		credentials.ClientSecret,
+	)
+	if err != nil {
+		return nil, err
+	}
+	service := exchangeService{
+		adapter:  *adapter,
+		client:   *msgraphsdk.NewGraphServiceClient(adapter),
+		failFast: shouldFailFast,
+	}
+	return &service, err
+}
+
 // CreateMailFolder makes a mail folder iff a folder of the same name does not exist
 // Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http
 func CreateMailFolder(gs graph.Service, user, folder string) (models.MailFolderable, error) {

src/internal/connector/exchange/service_query.go

@@ -2,15 +2,20 @@ package exchange
 
 import (
 	absser "github.com/microsoft/kiota-abstractions-go/serialization"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders"
 	msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages"
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/internal/connector/graph"
+	"github.com/alcionai/corso/internal/connector/support"
+	"github.com/alcionai/corso/pkg/account"
 )
 
 type optionIdentifier int
 
+const mailCategory = "mail"
+
 //go:generate stringer -type=optionIdentifier
 const (
 	unknown optionIdentifier = iota
@@ -34,6 +39,38 @@ func GetAllMessagesForUser(gs graph.Service, identities []string) (absser.Parsab
 	return gs.Client().UsersById(identities[0]).Messages().GetWithRequestConfigurationAndResponseHandler(options, nil)
 }
 
+// IterateMessageCollection utility function for Iterating through MessagesCollectionResponse
+// During iteration, Collections are added to the map based on the parent folder
+func IterateMessagesCollection(
+	tenant, user string,
+	errs error,
+	failFast bool,
+	credentials account.M365Config,
+	collections map[string]*Collection,
+	statusCh chan<- *support.ConnectorOperationStatus,
+) func(any) bool {
+	return func(messageItem any) bool {
+		message, ok := messageItem.(models.Messageable)
+		if !ok {
+			errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs)
+			return true
+		}
+		// Saving to messages to list. Indexed by folder
+		directory := *message.GetParentFolderId()
+		if _, ok = collections[directory]; !ok {
+			service, err := createService(credentials, failFast)
+			if err != nil {
+				errs = support.WrapAndAppend(user, err, errs)
+				return true
+			}
+			edc := NewCollection(user, []string{tenant, user, mailCategory, directory}, service, statusCh)
+			collections[directory] = &edc
+		}
+		collections[directory].AddJob(*message.GetId())
+		return true
+	}
+}
+
 //---------------------------------------------------
 // exchange.Query Option Section
 //------------------------------------------------
@@ -83,6 +120,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) {
 		"displayName":    1,
 		"isHidden":       2,
 		"parentFolderId": 3,
+		"id":             4,
 	}
 
 	fieldsForUsers := map[string]int{
@@ -93,6 +131,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) {
 		"department":     5,
 		"displayName":    6,
 		"employeeId":     7,
+		"id":             8,
 	}
 
 	fieldsForMessages := map[string]int{
@@ -101,6 +140,7 @@ func buildOptions(options []string, optId optionIdentifier) ([]string, error) {
 		"parentFolderId":    3,
 		"subject":           4,
 		"webLink":           5,
+		"id":                6,
 	}
 	returnedOptions := []string{"id"}
 

src/internal/connector/graph_connector.go

@@ -202,6 +202,7 @@ func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector s
 					errs)
 				continue
 			}
+			// TODO: Creates a map of collections based on scope
 			dcs, err := gc.serializeMessages(ctx, user)
 			if err != nil {
 				return nil, support.WrapAndAppend(user, err, errs)
@@ -301,8 +302,7 @@ func (gc *GraphConnector) RestoreMessages(ctx context.Context, dcs []data.Collec
 // to the GraphConnector struct.
 func (gc *GraphConnector) serializeMessages(ctx context.Context, user string) (map[string]*exchange.Collection, error) {
 	var transformer absser.ParsableFactory
-	options := optionsForMessageSnapshot()
+	response, err := exchange.GetAllMessagesForUser(&gc.graphService, []string{user}) //TODO: Selector to be used for exchange.query
-	response, err := gc.graphService.client.UsersById(user).Messages().GetWithRequestConfigurationAndResponseHandler(options, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -316,26 +316,7 @@ func (gc *GraphConnector) serializeMessages(ctx context.Context, user string) (m
 	var errs error
 	// callbackFunc iterates through all models.Messageable and fills exchange.Collection.jobs[]
 	// with corresponding messageIDs. New collections are created for each directory
-	callbackFunc := func(messageItem any) bool {
+	callbackFunc := exchange.IterateMessagesCollection(gc.tenant, user, errs, gc.failFast, gc.credentials, collections, gc.statusCh)
-		message, ok := messageItem.(models.Messageable)
-		if !ok {
-			errs = support.WrapAndAppendf(gc.graphService.adapter.GetBaseUrl(), errors.New("message iteration failure"), err)
-			return true
-		}
-		// Saving to messages to list. Indexed by folder
-		directory := *message.GetParentFolderId()
-		if _, ok = collections[directory]; !ok {
-			service, err := gc.createService(gc.failFast)
-			if err != nil {
-				errs = support.WrapAndAppend(user, err, errs)
-				return true
-			}
-			edc := exchange.NewCollection(user, []string{gc.tenant, user, mailCategory, directory}, service, gc.statusCh)
-			collections[directory] = &edc
-		}
-		collections[directory].AddJob(*message.GetId())
-		return true
-	}
 	iterateError := pageIterator.Iterate(callbackFunc)
 	if iterateError != nil {
 		errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs)

src/internal/connector/graph_connector_disconnected_test.go

@@ -75,12 +75,12 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() {
 	names["Axel"] = "Foley"
 	first := buildFromMap(true, names)
 	last := buildFromMap(false, names)
-	suite.True(Contains(first, "Al"))
+	suite.Contains(first, "Al")
-	suite.True(Contains(first, "Ellen"))
+	suite.Contains(first, "Ellen")
-	suite.True(Contains(first, "Axel"))
+	suite.Contains(first, "Axel")
-	suite.True(Contains(last, "Bundy"))
+	suite.Contains(last, "Bundy")
-	suite.True(Contains(last, "Ripley"))
+	suite.Contains(last, "Ripley")
-	suite.True(Contains(last, "Foley"))
+	suite.Contains(last, "Foley")
 
 }
 

src/internal/connector/optionidentifier_string.go

@@ -1,26 +0,0 @@
-// Code generated by "stringer -type=optionIdentifier"; DO NOT EDIT.
-
-package connector
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[unknown-0]
-	_ = x[folders-1]
-	_ = x[messages-2]
-	_ = x[users-3]
-}
-
-const _optionIdentifier_name = "unknownfoldersmessagesusers"
-
-var _optionIdentifier_index = [...]uint8{0, 7, 14, 22, 27}
-
-func (i optionIdentifier) String() string {
-	if i < 0 || i >= optionIdentifier(len(_optionIdentifier_index)-1) {
-		return "optionIdentifier(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _optionIdentifier_name[_optionIdentifier_index[i]:_optionIdentifier_index[i+1]]
-}

src/internal/connector/query.go

@@ -1,36 +0,0 @@
-package connector
-
-import (
-	msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages"
-)
-
-type optionIdentifier int
-
-//go:generate stringer -type=optionIdentifier
-const (
-	unknown optionIdentifier = iota
-	folders
-	messages
-	users
-)
-
-// Contains is a helper method for verifying if element
-// is contained within the slice
-func Contains(elems []string, value string) bool {
-	for _, s := range elems {
-		if value == s {
-			return true
-		}
-	}
-	return false
-}
-
-func optionsForMessageSnapshot() *msmessage.MessagesRequestBuilderGetRequestConfiguration {
-	selecting := []string{"id", "parentFolderId"}
-	options := &msmessage.MessagesRequestBuilderGetRequestConfiguration{
-		QueryParameters: &msmessage.MessagesRequestBuilderGetQueryParameters{
-			Select: selecting,
-		},
-	}
-	return options
-}