From 97113aa80b804b60e53352566ed479d1b8da855b Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 5 Aug 2022 11:20:36 -0400 Subject: [PATCH] Folder Granularity scope Added to Collections Retrieval (#482) Backup function `DataExchangeCollections` supports the ability to backup a single file using the `selectors` package. Support files and documentation updated accordingly --- docs/docs/cli/corso.md | 2 +- docs/docs/cli/corso_backup.md | 2 +- docs/docs/cli/corso_backup_create.md | 2 +- docs/docs/cli/corso_backup_create_exchange.md | 2 +- docs/docs/cli/corso_backup_details.md | 2 +- .../docs/cli/corso_backup_details_exchange.md | 2 +- docs/docs/cli/corso_backup_list.md | 2 +- docs/docs/cli/corso_backup_list_exchange.md | 2 +- docs/docs/cli/corso_repo.md | 2 +- docs/docs/cli/corso_repo_connect.md | 2 +- docs/docs/cli/corso_repo_connect_s3.md | 2 +- docs/docs/cli/corso_repo_init.md | 2 +- docs/docs/cli/corso_repo_init_s3.md | 2 +- docs/docs/cli/corso_restore.md | 2 +- docs/docs/cli/corso_restore_exchange.md | 2 +- .../connector/exchange/service_functions.go | 9 +- .../connector/exchange/service_query.go | 135 ++++++++++++++++++ .../connector/graph_connector_test.go | 28 ++++ 18 files changed, 186 insertions(+), 16 deletions(-) diff --git a/docs/docs/cli/corso.md b/docs/docs/cli/corso.md index 5c09e683e..df5896419 100644 --- a/docs/docs/cli/corso.md +++ b/docs/docs/cli/corso.md @@ -25,4 +25,4 @@ corso [flags] * [corso repo](corso_repo.md) - Manage your repositories * [corso restore](corso_restore.md) - Restore your service data -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup.md b/docs/docs/cli/corso_backup.md index 0a40be963..92461f008 100644 --- a/docs/docs/cli/corso_backup.md +++ b/docs/docs/cli/corso_backup.md @@ -30,4 +30,4 @@ corso backup [flags] * [corso backup details](corso_backup_details.md) - Shows the details of a backup for a service * [corso backup list](corso_backup_list.md) - List the history of backups for a service -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_create.md b/docs/docs/cli/corso_backup_create.md index dd7dd1d5e..8065069ca 100644 --- a/docs/docs/cli/corso_backup_create.md +++ b/docs/docs/cli/corso_backup_create.md @@ -24,4 +24,4 @@ corso backup create [flags] * [corso backup](corso_backup.md) - Backup your service data * [corso backup create exchange](corso_backup_create_exchange.md) - Backup M365 Exchange service data -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_create_exchange.md b/docs/docs/cli/corso_backup_create_exchange.md index 3816f6e2c..01f9cb2d2 100644 --- a/docs/docs/cli/corso_backup_create_exchange.md +++ b/docs/docs/cli/corso_backup_create_exchange.md @@ -26,4 +26,4 @@ corso backup create exchange [flags] * [corso backup create](corso_backup_create.md) - Backup an M365 Service -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_details.md b/docs/docs/cli/corso_backup_details.md index 6f42ae1f3..cb6564a93 100644 --- a/docs/docs/cli/corso_backup_details.md +++ b/docs/docs/cli/corso_backup_details.md @@ -24,4 +24,4 @@ corso backup details [flags] * [corso backup](corso_backup.md) - Backup your service data * [corso backup details exchange](corso_backup_details_exchange.md) - Shows the details of a M365 Exchange service backup -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_details_exchange.md b/docs/docs/cli/corso_backup_details_exchange.md index 1f66dac22..241f4ac5c 100644 --- a/docs/docs/cli/corso_backup_details_exchange.md +++ b/docs/docs/cli/corso_backup_details_exchange.md @@ -31,4 +31,4 @@ corso backup details exchange [flags] * [corso backup details](corso_backup_details.md) - Shows the details of a backup for a service -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_list.md b/docs/docs/cli/corso_backup_list.md index 3cdc1ba95..2e7851c9a 100644 --- a/docs/docs/cli/corso_backup_list.md +++ b/docs/docs/cli/corso_backup_list.md @@ -24,4 +24,4 @@ corso backup list [flags] * [corso backup](corso_backup.md) - Backup your service data * [corso backup list exchange](corso_backup_list_exchange.md) - List the history of M365 Exchange service backups -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_backup_list_exchange.md b/docs/docs/cli/corso_backup_list_exchange.md index d8b30d8b7..277253f89 100644 --- a/docs/docs/cli/corso_backup_list_exchange.md +++ b/docs/docs/cli/corso_backup_list_exchange.md @@ -23,4 +23,4 @@ corso backup list exchange [flags] * [corso backup list](corso_backup_list.md) - List the history of backups for a service -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_repo.md b/docs/docs/cli/corso_repo.md index e4892b56b..653603173 100644 --- a/docs/docs/cli/corso_repo.md +++ b/docs/docs/cli/corso_repo.md @@ -29,4 +29,4 @@ corso repo [flags] * [corso repo connect](corso_repo_connect.md) - Connect to a repository. * [corso repo init](corso_repo_init.md) - Initialize a repository. -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_repo_connect.md b/docs/docs/cli/corso_repo_connect.md index d1aade143..a4eb38362 100644 --- a/docs/docs/cli/corso_repo_connect.md +++ b/docs/docs/cli/corso_repo_connect.md @@ -28,4 +28,4 @@ corso repo connect [flags] * [corso repo](corso_repo.md) - Manage your repositories * [corso repo connect s3](corso_repo_connect_s3.md) - Connect to a S3 repository -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_repo_connect_s3.md b/docs/docs/cli/corso_repo_connect_s3.md index 864b01721..ed10a4d58 100644 --- a/docs/docs/cli/corso_repo_connect_s3.md +++ b/docs/docs/cli/corso_repo_connect_s3.md @@ -31,4 +31,4 @@ corso repo connect s3 [flags] * [corso repo connect](corso_repo_connect.md) - Connect to a repository. -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_repo_init.md b/docs/docs/cli/corso_repo_init.md index f3bcb5898..3fa254d8d 100644 --- a/docs/docs/cli/corso_repo_init.md +++ b/docs/docs/cli/corso_repo_init.md @@ -28,4 +28,4 @@ corso repo init [flags] * [corso repo](corso_repo.md) - Manage your repositories * [corso repo init s3](corso_repo_init_s3.md) - Initialize a S3 repository -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_repo_init_s3.md b/docs/docs/cli/corso_repo_init_s3.md index e4a77b1bf..5a349a46d 100644 --- a/docs/docs/cli/corso_repo_init_s3.md +++ b/docs/docs/cli/corso_repo_init_s3.md @@ -31,4 +31,4 @@ corso repo init s3 [flags] * [corso repo init](corso_repo_init.md) - Initialize a repository. -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_restore.md b/docs/docs/cli/corso_restore.md index 9850d00a0..c5e7ee7e3 100644 --- a/docs/docs/cli/corso_restore.md +++ b/docs/docs/cli/corso_restore.md @@ -28,4 +28,4 @@ corso restore [flags] * [corso](corso.md) - Protect your Microsoft 365 data. * [corso restore exchange](corso_restore_exchange.md) - Restore M365 Exchange service data -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/docs/docs/cli/corso_restore_exchange.md b/docs/docs/cli/corso_restore_exchange.md index a84853c61..d79af2e8a 100644 --- a/docs/docs/cli/corso_restore_exchange.md +++ b/docs/docs/cli/corso_restore_exchange.md @@ -31,4 +31,4 @@ corso restore exchange [flags] * [corso restore](corso_restore.md) - Restore your service data -###### Auto generated by spf13/cobra on 4-Aug-2022 +###### Auto generated by spf13/cobra on 5-Aug-2022 diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 4798f1dfb..5e4a38237 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -182,10 +182,17 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) ( GraphIterateFunc, ) { if scope.IncludesCategory(selectors.ExchangeMail) { + folders := scope.Get(selectors.ExchangeMailFolder) + if folders[0] == selectors.AnyTgt { + return models.CreateMessageCollectionResponseFromDiscriminatorValue, + GetAllMessagesForUser, + IterateSelectAllMessagesForCollections + } return models.CreateMessageCollectionResponseFromDiscriminatorValue, GetAllMessagesForUser, - IterateSelectAllMessagesForCollections + IterateAndFilterMessagesForCollections + } return nil, nil, nil diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index a1d661fbb..71deca476 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -1,7 +1,10 @@ package exchange import ( + "fmt" + absser "github.com/microsoft/kiota-abstractions-go/serialization" + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders" msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" @@ -35,12 +38,27 @@ type GraphQuery func(graph.Service, []string) (absser.Parsable, error) func GetAllMessagesForUser(gs graph.Service, identities []string) (absser.Parsable, error) { selecting := []string{"id", "parentFolderId"} options, err := optionsForMessages(selecting) + if err != nil { return nil, err } + return gs.Client().UsersById(identities[0]).Messages().GetWithRequestConfigurationAndResponseHandler(options, nil) } +// GetAllFolderDisplayNamesForUser is a GraphQuery function for getting FolderId and display +// names for Mail Folder. All other information for the MailFolder object is omitted. +func GetAllFolderNamesForUser(gs graph.Service, identities []string) (absser.Parsable, error) { + options, err := optionsForMailFolders([]string{"id", "displayName"}) + + if err != nil { + return nil, err + } + + return gs.Client().UsersById(identities[0]).MailFolders().GetWithRequestConfigurationAndResponseHandler(options, nil) + +} + // GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator) // @returns a callback func that works with msgraphgocore.PageIterator.Iterate function type GraphIterateFunc func( @@ -98,6 +116,123 @@ func IterateSelectAllMessagesForCollections( } } +func IterateAndFilterMessagesForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + var isFilterSet bool + return func(messageItem any) bool { + user := scope.Get(selectors.ExchangeUser)[0] + if !isFilterSet { + + err := CollectMailFolders( + scope, + tenant, + user, + collections, + credentials, + failFast, + statusCh, + ) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return false + } + isFilterSet = true + } + + message, ok := messageItem.(models.Messageable) + if !ok { + errs = support.WrapAndAppend(user, errors.New("message iteration failure"), errs) + return true + } + // Saving only messages for the created directories + directory := *message.GetParentFolderId() + if _, ok = collections[directory]; !ok { + return true + } + collections[directory].AddJob(*message.GetId()) + return true + + } +} + +func CollectMailFolders( + scope selectors.ExchangeScope, + tenant string, + user string, + collections map[string]*Collection, + credentials account.M365Config, + failFast bool, + statusCh chan<- *support.ConnectorOperationStatus, +) error { + queryService, err := createService(credentials, failFast) + if err != nil { + return errors.New("unable to create a mail folder query service for " + user) + } + + query, err := GetAllFolderNamesForUser(queryService, []string{user}) + if err != nil { + return fmt.Errorf( + "unable to query mail folder for %s: details: %s", + user, + support.ConnectorStackErrorTrace(err), + ) + } + // Iterator required to ensure all potential folders are inspected + // when the breadth of the folder space is large + pageIterator, err := msgraphgocore.NewPageIterator( + query, + &queryService.adapter, + models.CreateMailFolderCollectionResponseFromDiscriminatorValue) + if err != nil { + return errors.Wrap(err, "unable to create iterator during mail folder query service") + } + var service graph.Service + callbackFunc := func(pageItem any) bool { + folder, ok := pageItem.(models.MailFolderable) + if !ok { + err = support.WrapAndAppend(user, errors.New("unable to transform folderable item"), err) + return true + } + if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) { + return true + } + directory := *folder.GetId() + service, err = createService(credentials, failFast) + if err != nil { + err = support.WrapAndAppend( + *folder.GetDisplayName(), + errors.New("unable to create service a folder query service for "+user), + err, + ) + return true + } + temp := NewCollection( + user, + []string{tenant, user, mailCategory, directory}, + messages, + service, + statusCh, + ) + collections[directory] = &temp + + return true + } + + iterateFailure := pageIterator.Iterate(callbackFunc) + if iterateFailure != nil { + err = support.WrapAndAppend(user+" iterate failure", iterateFailure, err) + } + return err + +} + //--------------------------------------------------- // exchange.Query Option Section //------------------------------------------------ diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 310497083..157528316 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -139,6 +139,34 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages( assert.NoError(suite.T(), err) } +// TestGraphConnector_SingleMailFolderCollectionQuery verifies that single folder support +// enabled createCollections +func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolderCollectionQuery() { + t := suite.T() + sel := selectors.NewExchangeBackup() + sel.Include(sel.MailFolders([]string{suite.user}, []string{"Inbox"})) + scopes := sel.Scopes() + for _, scope := range scopes { + collections, err := suite.connector.createCollections(context.Background(), scope) + require.NoError(t, err) + suite.Equal(len(collections), 1) + for _, edc := range collections { + streamChannel := edc.Items() + // Verify that each message can be restored + for stream := range streamChannel { + buf := &bytes.Buffer{} + read, err := buf.ReadFrom(stream.ToReader()) + suite.NoError(err) + suite.NotZero(read) + message, err := support.CreateMessageFromBytes(buf.Bytes()) + suite.NotNil(message) + suite.NoError(err) + + } + } + } +} + ///------------------------------------------------------------ // Exchange Functions //-------------------------------------------------------