diff --git a/src/internal/connector/exchange/exchange_service_test.go b/src/internal/connector/exchange/exchange_service_test.go index 033a4be2e..eef03a6d3 100644 --- a/src/internal/connector/exchange/exchange_service_test.go +++ b/src/internal/connector/exchange/exchange_service_test.go @@ -353,6 +353,7 @@ func (suite *ExchangeServiceSuite) TestIterativeFunctions() { // with corresponding item IDs. New collections are created for each directory callbackFunc := test.iterativeFunction( "testingTenant", + userID, test.scope, errs, false, suite.es.credentials, diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 2090c20ec..c09cc8364 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -227,7 +227,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) ( nil } - if scope.IncludesCategory(selectors.ExchangeContactFolder) { + if scope.IncludesCategory(selectors.ExchangeContact) { return models.CreateContactFromDiscriminatorValue, GetAllContactsForUser, IterateAllContactsForCollection, diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index a5621a908..fa0d01009 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -36,6 +36,7 @@ type displayable interface { // @returns a callback func that works with msgraphgocore.PageIterator.Iterate function type GraphIterateFunc func( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -50,6 +51,7 @@ type GraphIterateFunc func( // placed into a Collection based on the parent folder func IterateSelectAllMessagesForCollections( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -95,6 +97,7 @@ func IterateSelectAllMessagesForCollections( // the calendarID which originates from M365. func IterateSelectAllEventsForCollections( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -103,8 +106,6 @@ func IterateSelectAllEventsForCollections( statusCh chan<- *support.ConnectorOperationStatus, ) func(any) bool { return func(eventItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - event, ok := eventItem.(models.Eventable) if !ok { errs = support.WrapAndAppend( @@ -173,6 +174,7 @@ func IterateSelectAllEventsForCollections( // Contacts Ids are placed into a collection based upon the parent folder func IterateAllContactsForCollection( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -181,8 +183,6 @@ func IterateAllContactsForCollection( statusCh chan<- *support.ConnectorOperationStatus, ) func(any) bool { return func(contactsItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] - contact, ok := contactsItem.(models.Contactable) if !ok { errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs) @@ -209,8 +209,12 @@ func IterateAllContactsForCollection( } } +// IterateAndFilterMessagesForCollections is a filtering GraphIterateFunc +// that places exchange mail message ids belonging to specific directories +// into a Collection. Messages outside of those directories are omitted. func IterateAndFilterMessagesForCollections( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -220,7 +224,6 @@ func IterateAndFilterMessagesForCollections( ) func(any) bool { var isFilterSet bool return func(messageItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] if !isFilterSet { err := CollectMailFolders( @@ -256,6 +259,7 @@ func IterateAndFilterMessagesForCollections( func IterateFilterFolderDirectoriesForCollections( tenant string, + user string, scope selectors.ExchangeScope, errs error, failFast bool, @@ -268,7 +272,6 @@ func IterateFilterFolderDirectoriesForCollections( err error ) return func(folderItem any) bool { - user := scope.Get(selectors.ExchangeUser)[0] folder, ok := folderItem.(models.MailFolderable) if !ok { errs = support.WrapAndAppend( @@ -279,6 +282,10 @@ func IterateFilterFolderDirectoriesForCollections( return true } + // Continue to iterate if folder name is empty + if folder.GetDisplayName() == nil { + return true + } if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) { return true } diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index 5b060fcbf..270a09275 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -133,6 +133,7 @@ func CollectMailFolders( callbackFunc := IterateFilterFolderDirectoriesForCollections( tenant, + user, scope, err, failFast, diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 358962592..f74514f41 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -201,39 +201,23 @@ func (gc *GraphConnector) ExchangeDataCollection( ) ([]data.Collection, error) { eb, err := selector.ToExchangeBackup() if err != nil { - return nil, errors.Wrap(err, "collecting exchange data") + return nil, errors.Wrap(err, "exchangeDataCollection: unable to parse selector") } - + scopes := eb.DiscreteScopes(gc.GetUsers()) collections := []data.Collection{} - scopes := eb.Scopes() var errs error - - // for each scope that includes mail messages, get all for _, scope := range scopes { - for _, user := range scope.Get(selectors.ExchangeUser) { - // TODO: handle "get mail for all users" - // this would probably no-op without this check, - // but we want it made obvious that we're punting. - if user == selectors.AnyTgt { - errs = support.WrapAndAppend( - "all-users", - errors.New("all users selector currently not handled"), - errs) - continue - } - // Creates a map of collections based on scope - dcs, err := gc.createCollections(ctx, scope) - if err != nil { - return nil, support.WrapAndAppend(user, err, errs) - } - - if len(dcs) > 0 { - for _, collection := range dcs { - collections = append(collections, collection) - } - } + // Creates a map of collections based on scope + dcs, err := gc.createCollections(ctx, scope) + if err != nil { + user := scope.Get(selectors.ExchangeUser) + return nil, support.WrapAndAppend(user[0], err, errs) + } + for _, collection := range dcs { + collections = append(collections, collection) } } + return collections, errs } @@ -307,48 +291,52 @@ func (gc *GraphConnector) RestoreMessages(ctx context.Context, dcs []data.Collec func (gc *GraphConnector) createCollections( ctx context.Context, scope selectors.ExchangeScope, -) (map[string]*exchange.Collection, error) { +) ([]*exchange.Collection, error) { var ( transformer absser.ParsableFactory query exchange.GraphQuery gIter exchange.GraphIterateFunc + errs error ) - user := scope.Get(selectors.ExchangeUser)[0] transformer, query, gIter, err := exchange.SetupExchangeCollectionVars(scope) if err != nil { - return nil, support.WrapAndAppend(user, err, nil) - } - response, err := query(&gc.graphService, user) - if err != nil { - return nil, errors.Wrapf( - err, - "user %s M365 query: %s", - user, support.ConnectorStackErrorTrace(err)) + return nil, support.WrapAndAppend(gc.Service().Adapter().GetBaseUrl(), err, nil) } + users := scope.Get(selectors.ExchangeUser) + allCollections := make([]*exchange.Collection, 0) + // Create collection of ExchangeDataCollection + for _, user := range users { + collections := make(map[string]*exchange.Collection) + response, err := query(&gc.graphService, user) + if err != nil { + return nil, errors.Wrapf( + err, + "user %s M365 query: %s", + user, support.ConnectorStackErrorTrace(err)) + } - pageIterator, err := msgraphgocore.NewPageIterator(response, &gc.graphService.adapter, transformer) - if err != nil { - return nil, err - } - // Create collection of ExchangeDataCollection and create data Holder - collections := make(map[string]*exchange.Collection) - var errs error - // callbackFunc iterates through all models.Messageable and fills exchange.Collection.jobs[] - // with corresponding item IDs. New collections are created for each directory - callbackFunc := gIter(gc.tenant, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh) - iterateError := pageIterator.Iterate(callbackFunc) - if iterateError != nil { - errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs) - } - if errs != nil { - return nil, errs // return error if snapshot is incomplete - } + pageIterator, err := msgraphgocore.NewPageIterator(response, &gc.graphService.adapter, transformer) + if err != nil { + return nil, err + } - for range collections { - gc.incrementAwaitingMessages() + // callbackFunc iterates through all M365 object target and fills exchange.Collection.jobs[] + // with corresponding item M365IDs. New collections are created for each directory. + // Each directory used the M365 Identifier. The use of ID stops collisions betweens users + callbackFunc := gIter(gc.tenant, user, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh) + iterateError := pageIterator.Iterate(callbackFunc) + if iterateError != nil { + errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs) + } + if errs != nil { + return nil, errs // return error if snapshot is incomplete + } + for _, collection := range collections { + gc.incrementAwaitingMessages() + allCollections = append(allCollections, collection) + } } - - return collections, errs + return allCollections, errs } // AwaitStatus updates status field based on item within statusChannel. diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 97922e497..0d38b683a 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/internal/connector/mockconnector" "github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/data" + "github.com/alcionai/corso/internal/tester" "github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/credentials" ) @@ -24,6 +25,7 @@ type DisconnectedGraphConnectorSuite struct { } func TestDisconnectedGraphSuite(t *testing.T) { + tester.LogTimeOfTest(t) suite.Run(t, new(DisconnectedGraphConnectorSuite)) } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 2e4398fb2..975d3e012 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -80,6 +80,8 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { // GraphConnector remains stable to receive a non-zero amount of Collections // for the Exchange Package. Enabled exchange applications: // - mail +// - contacts +// - events func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { t := suite.T() connector := loadConnector(t) @@ -248,35 +250,24 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() { assert.Equal(t, status.FolderCount, 1) } -// TestGraphConnector_SingleMailFolderCollectionQuery verifies single folder support -// for Backup operation -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolderCollectionQuery() { +// TestAccessOfInboxAllUsers verifies that GraphConnector can +// support `--all-users` for backup operations. Selector.DiscreteScopes +// returns all of the users within one scope. Only users who have +// messages in their inbox will have a collection returned. +// The final test insures that more than a 75% of the user collections are +// returned. If an error was experienced, the test will fail overall +func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() { t := suite.T() + connector := loadConnector(t) sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{suite.user}, []string{"Inbox"})) - scopes := sel.Scopes() + sel.Include(sel.MailFolders(selectors.Any(), []string{"Inbox"})) + scopes := sel.DiscreteScopes(connector.GetUsers()) for _, scope := range scopes { - collections, err := suite.connector.createCollections(context.Background(), scope) + users := scope.Get(selectors.ExchangeUser) + standard := (len(users) / 4) * 3 + collections, err := connector.createCollections(context.Background(), scope) require.NoError(t, err) - suite.Equal(len(collections), 1) - for _, edc := range collections { - number := 0 - streamChannel := edc.Items() - // Verify that each message can be restored - for stream := range streamChannel { - testName := fmt.Sprintf("%s_InboxMessage_%d", edc.FullPath()[1], number) - suite.T().Run(testName, func(t *testing.T) { - buf := &bytes.Buffer{} - read, err := buf.ReadFrom(stream.ToReader()) - assert.NoError(t, err) - assert.NotZero(t, read) - message, err := support.CreateMessageFromBytes(buf.Bytes()) - assert.NotNil(t, message) - assert.NoError(t, err) - number++ - }) - } - } + suite.Greater(len(collections), standard) } }