diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 7caf0fba8..4d915c976 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/cli/utils" "github.com/alcionai/corso/pkg/logger" "github.com/alcionai/corso/pkg/repository" + "github.com/alcionai/corso/pkg/selectors" ) // exchange bucket info from flags @@ -72,7 +73,10 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { } defer utils.CloseRepo(ctx, r) - bo, err := r.NewBackup(ctx, []string{user}) + sel := selectors.NewExchangeBackup() + sel.Include(sel.Users(user)) + + bo, err := r.NewBackup(ctx, sel.Selector) if err != nil { return errors.Wrap(err, "Failed to initialize Exchange backup") } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 58a616a7d..782f50c77 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -8,7 +8,6 @@ import ( "fmt" az "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - "github.com/alcionai/corso/internal/connector/support" ka "github.com/microsoft/kiota-authentication-azure-go" kw "github.com/microsoft/kiota-serialization-json-go" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" @@ -18,8 +17,10 @@ import ( msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders" "github.com/pkg/errors" + "github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/logger" + "github.com/alcionai/corso/pkg/selectors" ) const ( @@ -139,11 +140,47 @@ func buildFromMap(isKey bool, mapping map[string]string) []string { // Assumption: User exists // TODO: https://github.com/alcionai/corso/issues/135 // Add iota to this call -> mail, contacts, calendar, etc. -func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, user string) ([]DataCollection, error) { +func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector selectors.Selector) ([]DataCollection, error) { + eb, err := selector.ToExchangeBackup() + if err != nil { + return nil, errors.Wrap(err, "collecting exchange data") + } + + collections := []DataCollection{} + scopes := eb.Scopes() + var errs error + + // for each scope that includes mail messages, get all + for _, scope := range scopes { + if !scope.IncludesCategory(selectors.ExchangeMail) { + continue + } + + for _, user := range scope.Get(selectors.ExchangeUser) { + // TODO: handle "get mail for all users" + // this would probably no-op without this check, + // but we want it made obvious that we're punting. + if user == selectors.All { + errs = support.WrapAndAppend( + "all-users", + errors.New("all users selector currently not handled"), + errs) + continue + } + dcs, err := gc.serializeMessages(ctx, user) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + } + if len(dcs) > 0 { + collections = append(collections, dcs...) + } + } + } + // TODO replace with completion of Issue 124: //TODO: Retry handler to convert return: (DataCollection, error) - return gc.serializeMessages(ctx, user) + return collections, errs } // optionsForMailFolders creates transforms the 'select' into a more dynamic call for MailFolders. diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index acbe343e0..44b3e846f 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -13,6 +13,7 @@ import ( ctesting "github.com/alcionai/corso/internal/testing" "github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/credentials" + "github.com/alcionai/corso/pkg/selectors" ) type GraphConnectorIntegrationSuite struct { @@ -31,6 +32,10 @@ func TestGraphConnectorIntetgrationSuite(t *testing.T) { } func (suite *GraphConnectorIntegrationSuite) SetupSuite() { + if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil { + suite.T().Skip(err) + } + _, err := ctesting.GetRequiredEnvVars(ctesting.M365AcctCredEnvs...) require.NoError(suite.T(), err) @@ -46,14 +51,6 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector() { suite.NotNil(suite.connector) } -type DisconnectedGraphConnectorSuite struct { - suite.Suite -} - -func TestDisconnectedGraphSuite(t *testing.T) { - suite.Run(t, new(DisconnectedGraphConnectorSuite)) -} - func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() { err := suite.connector.setTenantUsers() assert.NoError(suite.T(), err) @@ -61,14 +58,17 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() } func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataCollection() { - if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil { - suite.T().Skip(err) - } - collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), "lidiah@8qzvrj.onmicrosoft.com") - assert.NotNil(suite.T(), collectionList, "collection list") - assert.Error(suite.T(), err) // TODO Remove after https://github.com/alcionai/corso/issues/140 - assert.NotNil(suite.T(), suite.connector.status, "connector status") - suite.NotContains(err.Error(), "attachment failed") // TODO Create Retry Exceeded Error + t := suite.T() + + sel := selectors.NewExchangeBackup() + sel.Include(sel.Users("lidiah@8qzvrj.onmicrosoft.com")) + collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), sel.Selector) + + require.NotNil(t, collectionList, "collection list") + assert.Error(t, err) // TODO Remove after https://github.com/alcionai/corso/issues/140 + assert.NotNil(t, suite.connector.status, "connector status") + assert.NotContains(t, err.Error(), "attachment failed") // TODO Create Retry Exceeded Error + exchangeData := collectionList[0] suite.Greater(len(exchangeData.FullPath()), 2) } @@ -92,6 +92,16 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages( assert.NoError(suite.T(), err) } +// --------------------------------------------------------------------------- + +type DisconnectedGraphConnectorSuite struct { + suite.Suite +} + +func TestDisconnectedGraphSuite(t *testing.T) { + suite.Run(t, new(DisconnectedGraphConnectorSuite)) +} + func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() { table := []struct { diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 6f160b1ea..91435bb26 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -10,15 +10,16 @@ import ( "github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/pkg/account" + "github.com/alcionai/corso/pkg/selectors" ) // BackupOperation wraps an operation with backup-specific props. type BackupOperation struct { operation - Results BackupResults `json:"results"` - Targets []string `json:"selectors"` // todo: replace with Selectors - Version string `json:"version"` + Results BackupResults `json:"results"` + Selectors selectors.Selector `json:"selectors"` + Version string `json:"version"` account account.Account } @@ -36,11 +37,11 @@ func NewBackupOperation( opts Options, kw *kopia.Wrapper, acct account.Account, - targets []string, + selector selectors.Selector, ) (BackupOperation, error) { op := BackupOperation{ operation: newOperation(opts, kw), - Targets: targets, + Selectors: selector, Version: "v0", account: acct, } @@ -81,7 +82,7 @@ func (op *BackupOperation) Run(ctx context.Context) error { } var cs []connector.DataCollection - cs, err = gc.ExchangeDataCollection(ctx, op.Targets[0]) + cs, err = gc.ExchangeDataCollection(ctx, op.Selectors) if err != nil { stats.readErr = err return errors.Wrap(err, "retrieving service data") diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 928ae2be0..a4c6bab52 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/internal/kopia" ctesting "github.com/alcionai/corso/internal/testing" "github.com/alcionai/corso/pkg/account" + "github.com/alcionai/corso/pkg/selectors" ) // --------------------------------------------------------------------------- @@ -50,7 +51,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { } ) - op, err := NewBackupOperation(ctx, Options{}, kw, acct, nil) + op, err := NewBackupOperation(ctx, Options{}, kw, acct, selectors.Selector{}) require.NoError(t, err) op.persistResults(now, &stats) @@ -112,7 +113,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { Options{}, test.kw, test.acct, - nil) + selectors.Selector{}) test.errCheck(t, err) }) } @@ -143,12 +144,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { w, err := kopia.NewWrapper(k) require.NoError(t, err) + sel := selectors.NewExchangeBackup() + sel.Include(sel.Users(m365User)) + bo, err := NewBackupOperation( ctx, Options{}, w, acct, - []string{m365User}) + sel.Selector) require.NoError(t, err) require.NoError(t, bo.Run(ctx)) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index dc132460b..5428e3fd5 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/internal/operations" "github.com/alcionai/corso/pkg/account" + "github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/storage" ) @@ -109,13 +110,13 @@ func (r *Repository) Close(ctx context.Context) error { } // NewBackup generates a backupOperation runner. -func (r Repository) NewBackup(ctx context.Context, targets []string) (operations.BackupOperation, error) { +func (r Repository) NewBackup(ctx context.Context, selector selectors.Selector) (operations.BackupOperation, error) { return operations.NewBackupOperation( ctx, operations.Options{}, r.dataLayer, r.Account, - targets) + selector) } // NewRestore generates a restoreOperation runner. diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index e6d8f0e85..a8fb09f9e 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -11,6 +11,7 @@ import ( ctesting "github.com/alcionai/corso/internal/testing" "github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/repository" + "github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/storage" ) @@ -170,7 +171,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { r, err := repository.Initialize(ctx, acct, st) require.NoError(t, err) - bo, err := r.NewBackup(ctx, []string{}) + bo, err := r.NewBackup(ctx, selectors.Selector{}) require.NoError(t, err) require.NotNil(t, bo) }