GC: Iterate-Functions moved to separate file (#657)
* Issue #654: Prepwork: `service_iterate.go` contains iterate functions for the package.
This commit is contained in:
Danny
GitHub
parent
a2a5260d3f
commit
02c2b0b4d6
295
src/internal/connector/exchange/service_iterate.go
Normal file
295
src/internal/connector/exchange/service_iterate.go
Normal file
@ -0,0 +1,295 @@
|
||||
package exchange
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/internal/connector/graph"
|
||||
"github.com/alcionai/corso/internal/connector/support"
|
||||
"github.com/alcionai/corso/pkg/account"
|
||||
"github.com/alcionai/corso/pkg/selectors"
|
||||
)
|
||||
|
||||
const (
|
||||
mailCategory = "mail"
|
||||
contactsCategory = "contacts"
|
||||
eventsCategory = "events"
|
||||
)
|
||||
|
||||
// GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator)
|
||||
// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function
|
||||
type GraphIterateFunc func(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
graphStatusChannel chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool
|
||||
|
||||
// IterateSelectAllMessageForCollection utility function for
|
||||
// Iterating through MessagesCollectionResponse
|
||||
// During iteration, messages belonging to any folder are
|
||||
// placed into a Collection based on the parent folder
|
||||
func IterateSelectAllMessagesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(messageItem any) bool {
|
||||
// Defines the type of collection being created within the function
|
||||
collectionType := messages
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
message, ok := messageItem.(models.Messageable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
// Saving to messages to list. Indexed by folder
|
||||
directory := *message.GetParentFolderId()
|
||||
if _, ok = collections[directory]; !ok {
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, mailCategory, directory},
|
||||
collectionType,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
collections[directory].AddJob(*message.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// IterateSelectAllEventsForCollections
|
||||
// utility function for iterating through events
|
||||
// and storing events in collections based on
|
||||
// the calendarID which originates from M365.
|
||||
func IterateSelectAllEventsForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(eventItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
event, ok := eventItem.(models.Eventable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.New("event iteration failure"),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
adtl := event.GetAdditionalData()
|
||||
value, ok := adtl["calendar@odata.associationLink"]
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
fmt.Errorf("%s: does not support calendar look up", *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
link, ok := value.(*string)
|
||||
if !ok || link == nil {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
fmt.Errorf("%s: unable to obtain calendar event data", *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
// calendars and events are not easily correlated
|
||||
// helper function retrieves calendarID from url
|
||||
directory, err := parseCalendarIDFromEvent(*link)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.Wrap(err, *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
if _, ok := collections[directory]; !ok {
|
||||
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, eventsCategory, directory},
|
||||
events,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
|
||||
collections[directory].AddJob(*event.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// IterateAllContactsForCollection GraphIterateFunc for moving through
|
||||
// a ContactsCollectionsResponse using the msgraphgocore paging interface.
|
||||
// Contacts Ids are placed into a collection based upon the parent folder
|
||||
func IterateAllContactsForCollection(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(contactsItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
contact, ok := contactsItem.(models.Contactable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
directory := *contact.GetParentFolderId()
|
||||
if _, ok := collections[directory]; !ok {
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, contactsCategory, directory},
|
||||
contacts,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
collections[directory].AddJob(*contact.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func IterateAndFilterMessagesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
var isFilterSet bool
|
||||
return func(messageItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
if !isFilterSet {
|
||||
|
||||
err := CollectMailFolders(
|
||||
scope,
|
||||
tenant,
|
||||
user,
|
||||
collections,
|
||||
credentials,
|
||||
failFast,
|
||||
statusCh,
|
||||
)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return false
|
||||
}
|
||||
isFilterSet = true
|
||||
}
|
||||
|
||||
message, ok := messageItem.(models.Messageable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(user, errors.New("message iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
// Saving only messages for the created directories
|
||||
directory := *message.GetParentFolderId()
|
||||
if _, ok = collections[directory]; !ok {
|
||||
return true
|
||||
}
|
||||
collections[directory].AddJob(*message.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func IterateFilterFolderDirectoriesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
var (
|
||||
service graph.Service
|
||||
err error
|
||||
)
|
||||
return func(folderItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
folder, ok := folderItem.(models.MailFolderable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.New("unable to transform folderable item"),
|
||||
errs,
|
||||
)
|
||||
|
||||
return true
|
||||
}
|
||||
if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) {
|
||||
return true
|
||||
}
|
||||
directory := *folder.GetId()
|
||||
service, err = createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
*folder.GetDisplayName(),
|
||||
errors.Wrap(
|
||||
err,
|
||||
"unable to create service a folder query service for "+user,
|
||||
),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
temp := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, mailCategory, directory},
|
||||
messages,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &temp
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
@ -14,12 +14,6 @@ import (
|
||||
"github.com/alcionai/corso/pkg/selectors"
|
||||
)
|
||||
|
||||
const (
|
||||
mailCategory = "mail"
|
||||
contactsCategory = "contacts"
|
||||
eventsCategory = "events"
|
||||
)
|
||||
|
||||
// GraphQuery represents functions which perform exchange-specific queries
|
||||
// into M365 backstore. Responses -> returned items will only contain the information
|
||||
// that is included in the options
|
||||
@ -105,282 +99,6 @@ func RetrieveMessageDataForUser(gs graph.Service, user, m365ID string) (absser.P
|
||||
return gs.Client().UsersById(user).MessagesById(m365ID).Get()
|
||||
}
|
||||
|
||||
// GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator)
|
||||
// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function
|
||||
type GraphIterateFunc func(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
graphStatusChannel chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool
|
||||
|
||||
// IterateSelectAllMessageForCollection utility function for
|
||||
// Iterating through MessagesCollectionResponse
|
||||
// During iteration, messages belonging to any folder are
|
||||
// placed into a Collection based on the parent folder
|
||||
func IterateSelectAllMessagesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(messageItem any) bool {
|
||||
// Defines the type of collection being created within the function
|
||||
collectionType := messages
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
message, ok := messageItem.(models.Messageable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppendf(user, errors.New("message iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
// Saving to messages to list. Indexed by folder
|
||||
directory := *message.GetParentFolderId()
|
||||
if _, ok = collections[directory]; !ok {
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, mailCategory, directory},
|
||||
collectionType,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
collections[directory].AddJob(*message.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// IterateSelectAllEventsForCollections
|
||||
// utility function for iterating through events
|
||||
// and storing events in collections based on
|
||||
// the calendarID which originates from M365.
|
||||
func IterateSelectAllEventsForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(eventItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
event, ok := eventItem.(models.Eventable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.New("event iteration failure"),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
adtl := event.GetAdditionalData()
|
||||
value, ok := adtl["calendar@odata.associationLink"]
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
fmt.Errorf("%s: does not support calendar look up", *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
link, ok := value.(*string)
|
||||
if !ok || link == nil {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
fmt.Errorf("%s: unable to obtain calendar event data", *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
// calendars and events are not easily correlated
|
||||
// helper function retrieves calendarID from url
|
||||
directory, err := parseCalendarIDFromEvent(*link)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.Wrap(err, *event.GetId()),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
|
||||
if _, ok := collections[directory]; !ok {
|
||||
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, eventsCategory, directory},
|
||||
events,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
|
||||
collections[directory].AddJob(*event.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// IterateAllContactsForCollection GraphIterateFunc for moving through
|
||||
// a ContactsCollectionsResponse using the msgraphgocore paging interface.
|
||||
// Contacts Ids are placed into a collection based upon the parent folder
|
||||
func IterateAllContactsForCollection(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
return func(contactsItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
|
||||
contact, ok := contactsItem.(models.Contactable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
directory := *contact.GetParentFolderId()
|
||||
if _, ok := collections[directory]; !ok {
|
||||
service, err := createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return true
|
||||
}
|
||||
edc := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, contactsCategory, directory},
|
||||
contacts,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &edc
|
||||
}
|
||||
collections[directory].AddJob(*contact.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func IterateAndFilterMessagesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
var isFilterSet bool
|
||||
return func(messageItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
if !isFilterSet {
|
||||
|
||||
err := CollectMailFolders(
|
||||
scope,
|
||||
tenant,
|
||||
user,
|
||||
collections,
|
||||
credentials,
|
||||
failFast,
|
||||
statusCh,
|
||||
)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(user, err, errs)
|
||||
return false
|
||||
}
|
||||
isFilterSet = true
|
||||
}
|
||||
|
||||
message, ok := messageItem.(models.Messageable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(user, errors.New("message iteration failure"), errs)
|
||||
return true
|
||||
}
|
||||
// Saving only messages for the created directories
|
||||
directory := *message.GetParentFolderId()
|
||||
if _, ok = collections[directory]; !ok {
|
||||
return true
|
||||
}
|
||||
collections[directory].AddJob(*message.GetId())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func IterateFilterFolderDirectoriesForCollections(
|
||||
tenant string,
|
||||
scope selectors.ExchangeScope,
|
||||
errs error,
|
||||
failFast bool,
|
||||
credentials account.M365Config,
|
||||
collections map[string]*Collection,
|
||||
statusCh chan<- *support.ConnectorOperationStatus,
|
||||
) func(any) bool {
|
||||
var (
|
||||
service graph.Service
|
||||
err error
|
||||
)
|
||||
return func(folderItem any) bool {
|
||||
user := scope.Get(selectors.ExchangeUser)[0]
|
||||
folder, ok := folderItem.(models.MailFolderable)
|
||||
if !ok {
|
||||
errs = support.WrapAndAppend(
|
||||
user,
|
||||
errors.New("unable to transform folderable item"),
|
||||
errs,
|
||||
)
|
||||
|
||||
return true
|
||||
}
|
||||
if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) {
|
||||
return true
|
||||
}
|
||||
directory := *folder.GetId()
|
||||
service, err = createService(credentials, failFast)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
*folder.GetDisplayName(),
|
||||
errors.Wrap(
|
||||
err,
|
||||
"unable to create service a folder query service for "+user,
|
||||
),
|
||||
errs,
|
||||
)
|
||||
return true
|
||||
}
|
||||
temp := NewCollection(
|
||||
user,
|
||||
[]string{tenant, user, mailCategory, directory},
|
||||
messages,
|
||||
service,
|
||||
statusCh,
|
||||
)
|
||||
collections[directory] = &temp
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func CollectMailFolders(
|
||||
scope selectors.ExchangeScope,
|
||||
tenant string,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user