From 02c2b0b4d68c4d2f37c6cbe50989843242afb7bc Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 26 Aug 2022 13:47:49 -0400 Subject: [PATCH] GC: Iterate-Functions moved to separate file (#657) * Issue #654: Prepwork: `service_iterate.go` contains iterate functions for the package. --- .../connector/exchange/service_iterate.go | 295 ++++++++++++++++++ .../connector/exchange/service_query.go | 282 ----------------- 2 files changed, 295 insertions(+), 282 deletions(-) create mode 100644 src/internal/connector/exchange/service_iterate.go diff --git a/src/internal/connector/exchange/service_iterate.go b/src/internal/connector/exchange/service_iterate.go new file mode 100644 index 000000000..7bcb53d23 --- /dev/null +++ b/src/internal/connector/exchange/service_iterate.go @@ -0,0 +1,295 @@ +package exchange + +import ( + "fmt" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/pkg/errors" + + "github.com/alcionai/corso/internal/connector/graph" + "github.com/alcionai/corso/internal/connector/support" + "github.com/alcionai/corso/pkg/account" + "github.com/alcionai/corso/pkg/selectors" +) + +const ( + mailCategory = "mail" + contactsCategory = "contacts" + eventsCategory = "events" +) + +// GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator) +// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function +type GraphIterateFunc func( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + graphStatusChannel chan<- *support.ConnectorOperationStatus, +) func(any) bool + +// IterateSelectAllMessageForCollection utility function for +// Iterating through MessagesCollectionResponse +// During iteration, messages belonging to any folder are +// placed into a Collection based on the parent folder +func IterateSelectAllMessagesForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + return func(messageItem any) bool { + // Defines the type of collection being created within the function + collectionType := messages + user := scope.Get(selectors.ExchangeUser)[0] + + message, ok := messageItem.(models.Messageable) + if !ok { + errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs) + return true + } + // Saving to messages to list. Indexed by folder + directory := *message.GetParentFolderId() + if _, ok = collections[directory]; !ok { + service, err := createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return true + } + edc := NewCollection( + user, + []string{tenant, user, mailCategory, directory}, + collectionType, + service, + statusCh, + ) + collections[directory] = &edc + } + collections[directory].AddJob(*message.GetId()) + return true + } +} + +// IterateSelectAllEventsForCollections +// utility function for iterating through events +// and storing events in collections based on +// the calendarID which originates from M365. +func IterateSelectAllEventsForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + return func(eventItem any) bool { + user := scope.Get(selectors.ExchangeUser)[0] + + event, ok := eventItem.(models.Eventable) + if !ok { + errs = support.WrapAndAppend( + user, + errors.New("event iteration failure"), + errs, + ) + return true + } + + adtl := event.GetAdditionalData() + value, ok := adtl["calendar@odata.associationLink"] + if !ok { + errs = support.WrapAndAppend( + user, + fmt.Errorf("%s: does not support calendar look up", *event.GetId()), + errs, + ) + return true + } + link, ok := value.(*string) + if !ok || link == nil { + errs = support.WrapAndAppend( + user, + fmt.Errorf("%s: unable to obtain calendar event data", *event.GetId()), + errs, + ) + return true + } + // calendars and events are not easily correlated + // helper function retrieves calendarID from url + directory, err := parseCalendarIDFromEvent(*link) + if err != nil { + errs = support.WrapAndAppend( + user, + errors.Wrap(err, *event.GetId()), + errs, + ) + return true + } + + if _, ok := collections[directory]; !ok { + + service, err := createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return true + } + edc := NewCollection( + user, + []string{tenant, user, eventsCategory, directory}, + events, + service, + statusCh, + ) + collections[directory] = &edc + } + + collections[directory].AddJob(*event.GetId()) + return true + } +} + +// IterateAllContactsForCollection GraphIterateFunc for moving through +// a ContactsCollectionsResponse using the msgraphgocore paging interface. +// Contacts Ids are placed into a collection based upon the parent folder +func IterateAllContactsForCollection( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + return func(contactsItem any) bool { + user := scope.Get(selectors.ExchangeUser)[0] + + contact, ok := contactsItem.(models.Contactable) + if !ok { + errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs) + return true + } + directory := *contact.GetParentFolderId() + if _, ok := collections[directory]; !ok { + service, err := createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return true + } + edc := NewCollection( + user, + []string{tenant, user, contactsCategory, directory}, + contacts, + service, + statusCh, + ) + collections[directory] = &edc + } + collections[directory].AddJob(*contact.GetId()) + return true + } +} + +func IterateAndFilterMessagesForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + var isFilterSet bool + return func(messageItem any) bool { + user := scope.Get(selectors.ExchangeUser)[0] + if !isFilterSet { + + err := CollectMailFolders( + scope, + tenant, + user, + collections, + credentials, + failFast, + statusCh, + ) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return false + } + isFilterSet = true + } + + message, ok := messageItem.(models.Messageable) + if !ok { + errs = support.WrapAndAppend(user, errors.New("message iteration failure"), errs) + return true + } + // Saving only messages for the created directories + directory := *message.GetParentFolderId() + if _, ok = collections[directory]; !ok { + return true + } + collections[directory].AddJob(*message.GetId()) + return true + } +} + +func IterateFilterFolderDirectoriesForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + var ( + service graph.Service + err error + ) + return func(folderItem any) bool { + user := scope.Get(selectors.ExchangeUser)[0] + folder, ok := folderItem.(models.MailFolderable) + if !ok { + errs = support.WrapAndAppend( + user, + errors.New("unable to transform folderable item"), + errs, + ) + + return true + } + if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) { + return true + } + directory := *folder.GetId() + service, err = createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend( + *folder.GetDisplayName(), + errors.Wrap( + err, + "unable to create service a folder query service for "+user, + ), + errs, + ) + return true + } + temp := NewCollection( + user, + []string{tenant, user, mailCategory, directory}, + messages, + service, + statusCh, + ) + collections[directory] = &temp + + return true + } +} diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index 65eae8f52..5b060fcbf 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -14,12 +14,6 @@ import ( "github.com/alcionai/corso/pkg/selectors" ) -const ( - mailCategory = "mail" - contactsCategory = "contacts" - eventsCategory = "events" -) - // GraphQuery represents functions which perform exchange-specific queries // into M365 backstore. Responses -> returned items will only contain the information // that is included in the options @@ -105,282 +99,6 @@ func RetrieveMessageDataForUser(gs graph.Service, user, m365ID string) (absser.P return gs.Client().UsersById(user).MessagesById(m365ID).Get() } -// GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator) -// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function -type GraphIterateFunc func( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - graphStatusChannel chan<- *support.ConnectorOperationStatus, -) func(any) bool - -// IterateSelectAllMessageForCollection utility function for -// Iterating through MessagesCollectionResponse -// During iteration, messages belonging to any folder are -// placed into a Collection based on the parent folder -func IterateSelectAllMessagesForCollections( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - statusCh chan<- *support.ConnectorOperationStatus, -) func(any) bool { - return func(messageItem any) bool { - // Defines the type of collection being created within the function - collectionType := messages - user := scope.Get(selectors.ExchangeUser)[0] - - message, ok := messageItem.(models.Messageable) - if !ok { - errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs) - return true - } - // Saving to messages to list. Indexed by folder - directory := *message.GetParentFolderId() - if _, ok = collections[directory]; !ok { - service, err := createService(credentials, failFast) - if err != nil { - errs = support.WrapAndAppend(user, err, errs) - return true - } - edc := NewCollection( - user, - []string{tenant, user, mailCategory, directory}, - collectionType, - service, - statusCh, - ) - collections[directory] = &edc - } - collections[directory].AddJob(*message.GetId()) - return true - } -} - -// IterateSelectAllEventsForCollections -// utility function for iterating through events -// and storing events in collections based on -// the calendarID which originates from M365. -func IterateSelectAllEventsForCollections( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - statusCh chan<- *support.ConnectorOperationStatus, -) func(any) bool { - return func(eventItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - - event, ok := eventItem.(models.Eventable) - if !ok { - errs = support.WrapAndAppend( - user, - errors.New("event iteration failure"), - errs, - ) - return true - } - - adtl := event.GetAdditionalData() - value, ok := adtl["calendar@odata.associationLink"] - if !ok { - errs = support.WrapAndAppend( - user, - fmt.Errorf("%s: does not support calendar look up", *event.GetId()), - errs, - ) - return true - } - link, ok := value.(*string) - if !ok || link == nil { - errs = support.WrapAndAppend( - user, - fmt.Errorf("%s: unable to obtain calendar event data", *event.GetId()), - errs, - ) - return true - } - // calendars and events are not easily correlated - // helper function retrieves calendarID from url - directory, err := parseCalendarIDFromEvent(*link) - if err != nil { - errs = support.WrapAndAppend( - user, - errors.Wrap(err, *event.GetId()), - errs, - ) - return true - } - - if _, ok := collections[directory]; !ok { - - service, err := createService(credentials, failFast) - if err != nil { - errs = support.WrapAndAppend(user, err, errs) - return true - } - edc := NewCollection( - user, - []string{tenant, user, eventsCategory, directory}, - events, - service, - statusCh, - ) - collections[directory] = &edc - } - - collections[directory].AddJob(*event.GetId()) - return true - } -} - -// IterateAllContactsForCollection GraphIterateFunc for moving through -// a ContactsCollectionsResponse using the msgraphgocore paging interface. -// Contacts Ids are placed into a collection based upon the parent folder -func IterateAllContactsForCollection( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - statusCh chan<- *support.ConnectorOperationStatus, -) func(any) bool { - return func(contactsItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - - contact, ok := contactsItem.(models.Contactable) - if !ok { - errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs) - return true - } - directory := *contact.GetParentFolderId() - if _, ok := collections[directory]; !ok { - service, err := createService(credentials, failFast) - if err != nil { - errs = support.WrapAndAppend(user, err, errs) - return true - } - edc := NewCollection( - user, - []string{tenant, user, contactsCategory, directory}, - contacts, - service, - statusCh, - ) - collections[directory] = &edc - } - collections[directory].AddJob(*contact.GetId()) - return true - } -} - -func IterateAndFilterMessagesForCollections( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - statusCh chan<- *support.ConnectorOperationStatus, -) func(any) bool { - var isFilterSet bool - return func(messageItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - if !isFilterSet { - - err := CollectMailFolders( - scope, - tenant, - user, - collections, - credentials, - failFast, - statusCh, - ) - if err != nil { - errs = support.WrapAndAppend(user, err, errs) - return false - } - isFilterSet = true - } - - message, ok := messageItem.(models.Messageable) - if !ok { - errs = support.WrapAndAppend(user, errors.New("message iteration failure"), errs) - return true - } - // Saving only messages for the created directories - directory := *message.GetParentFolderId() - if _, ok = collections[directory]; !ok { - return true - } - collections[directory].AddJob(*message.GetId()) - return true - } -} - -func IterateFilterFolderDirectoriesForCollections( - tenant string, - scope selectors.ExchangeScope, - errs error, - failFast bool, - credentials account.M365Config, - collections map[string]*Collection, - statusCh chan<- *support.ConnectorOperationStatus, -) func(any) bool { - var ( - service graph.Service - err error - ) - return func(folderItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - folder, ok := folderItem.(models.MailFolderable) - if !ok { - errs = support.WrapAndAppend( - user, - errors.New("unable to transform folderable item"), - errs, - ) - - return true - } - if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) { - return true - } - directory := *folder.GetId() - service, err = createService(credentials, failFast) - if err != nil { - errs = support.WrapAndAppend( - *folder.GetDisplayName(), - errors.Wrap( - err, - "unable to create service a folder query service for "+user, - ), - errs, - ) - return true - } - temp := NewCollection( - user, - []string{tenant, user, mailCategory, directory}, - messages, - service, - statusCh, - ) - collections[directory] = &temp - - return true - } -} - func CollectMailFolders( scope selectors.ExchangeScope, tenant string,