wire selectors up through backup handling (#278)

* wire selectors up through backup handling

Selectors are implemented enough to add them end-
to-end in some places.  This starts with backup
creation, since that's the most stable set of code in
the repo at the moment.

src/cli/backup/exchange.go

@@ -9,6 +9,7 @@ import (
 	"github.com/alcionai/corso/cli/utils"
 	"github.com/alcionai/corso/pkg/logger"
 	"github.com/alcionai/corso/pkg/repository"
+	"github.com/alcionai/corso/pkg/selectors"
 )
 
 // exchange bucket info from flags
@@ -72,7 +73,10 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 	}
 	defer utils.CloseRepo(ctx, r)
 
-	bo, err := r.NewBackup(ctx, []string{user})
+	sel := selectors.NewExchangeBackup()
+	sel.Include(sel.Users(user))
+
+	bo, err := r.NewBackup(ctx, sel.Selector)
 	if err != nil {
 		return errors.Wrap(err, "Failed to initialize Exchange backup")
 	}

src/internal/connector/graph_connector.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 
 	az "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
-	"github.com/alcionai/corso/internal/connector/support"
 	ka "github.com/microsoft/kiota-authentication-azure-go"
 	kw "github.com/microsoft/kiota-serialization-json-go"
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
@@ -18,8 +17,10 @@ import (
 	msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders"
 	"github.com/pkg/errors"
 
+	"github.com/alcionai/corso/internal/connector/support"
 	"github.com/alcionai/corso/pkg/account"
 	"github.com/alcionai/corso/pkg/logger"
+	"github.com/alcionai/corso/pkg/selectors"
 )
 
 const (
@@ -139,11 +140,47 @@ func buildFromMap(isKey bool, mapping map[string]string) []string {
 // Assumption: User exists
 // TODO: https://github.com/alcionai/corso/issues/135
 //  Add iota to this call -> mail, contacts, calendar,  etc.
-func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, user string) ([]DataCollection, error) {
+func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector selectors.Selector) ([]DataCollection, error) {
+	eb, err := selector.ToExchangeBackup()
+	if err != nil {
+		return nil, errors.Wrap(err, "collecting exchange data")
+	}
+
+	collections := []DataCollection{}
+	scopes := eb.Scopes()
+	var errs error
+
+	// for each scope that includes mail messages, get all
+	for _, scope := range scopes {
+		if !scope.IncludesCategory(selectors.ExchangeMail) {
+			continue
+		}
+
+		for _, user := range scope.Get(selectors.ExchangeUser) {
+			// TODO: handle "get mail for all users"
+			// this would probably no-op without this check,
+			// but we want it made obvious that we're punting.
+			if user == selectors.All {
+				errs = support.WrapAndAppend(
+					"all-users",
+					errors.New("all users selector currently not handled"),
+					errs)
+				continue
+			}
+			dcs, err := gc.serializeMessages(ctx, user)
+			if err != nil {
+				errs = support.WrapAndAppend(user, err, errs)
+			}
+			if len(dcs) > 0 {
+				collections = append(collections, dcs...)
+			}
+		}
+	}
+
 	// TODO replace with completion of Issue 124:
 
 	//TODO: Retry handler to convert return: (DataCollection, error)
-	return gc.serializeMessages(ctx, user)
+	return collections, errs
 }
 
 // optionsForMailFolders creates transforms the 'select' into a more dynamic call for MailFolders.

src/internal/connector/graph_connector_test.go

@@ -13,6 +13,7 @@ import (
 	ctesting "github.com/alcionai/corso/internal/testing"
 	"github.com/alcionai/corso/pkg/account"
 	"github.com/alcionai/corso/pkg/credentials"
+	"github.com/alcionai/corso/pkg/selectors"
 )
 
 type GraphConnectorIntegrationSuite struct {
@@ -31,6 +32,10 @@ func TestGraphConnectorIntetgrationSuite(t *testing.T) {
 }
 
 func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
+	if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil {
+		suite.T().Skip(err)
+	}
+
 	_, err := ctesting.GetRequiredEnvVars(ctesting.M365AcctCredEnvs...)
 	require.NoError(suite.T(), err)
 
@@ -46,14 +51,6 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector() {
 	suite.NotNil(suite.connector)
 }
 
-type DisconnectedGraphConnectorSuite struct {
-	suite.Suite
-}
-
-func TestDisconnectedGraphSuite(t *testing.T) {
-	suite.Run(t, new(DisconnectedGraphConnectorSuite))
-}
-
 func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() {
 	err := suite.connector.setTenantUsers()
 	assert.NoError(suite.T(), err)
@@ -61,14 +58,17 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers()
 }
 
 func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataCollection() {
-	if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil {
+	t := suite.T()
-		suite.T().Skip(err)
+
-	}
+	sel := selectors.NewExchangeBackup()
-	collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), "lidiah@8qzvrj.onmicrosoft.com")
+	sel.Include(sel.Users("lidiah@8qzvrj.onmicrosoft.com"))
-	assert.NotNil(suite.T(), collectionList, "collection list")
+	collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), sel.Selector)
-	assert.Error(suite.T(), err) // TODO Remove after https://github.com/alcionai/corso/issues/140
+
-	assert.NotNil(suite.T(), suite.connector.status, "connector status")
+	require.NotNil(t, collectionList, "collection list")
-	suite.NotContains(err.Error(), "attachment failed") // TODO Create Retry Exceeded Error
+	assert.Error(t, err) // TODO Remove after https://github.com/alcionai/corso/issues/140
+	assert.NotNil(t, suite.connector.status, "connector status")
+	assert.NotContains(t, err.Error(), "attachment failed") // TODO Create Retry Exceeded Error
+
 	exchangeData := collectionList[0]
 	suite.Greater(len(exchangeData.FullPath()), 2)
 }
@@ -92,6 +92,16 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages(
 	assert.NoError(suite.T(), err)
 }
 
+// ---------------------------------------------------------------------------
+
+type DisconnectedGraphConnectorSuite struct {
+	suite.Suite
+}
+
+func TestDisconnectedGraphSuite(t *testing.T) {
+	suite.Run(t, new(DisconnectedGraphConnectorSuite))
+}
+
 func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() {
 
 	table := []struct {

src/internal/operations/backup.go

@@ -10,15 +10,16 @@ import (
 	"github.com/alcionai/corso/internal/connector/support"
 	"github.com/alcionai/corso/internal/kopia"
 	"github.com/alcionai/corso/pkg/account"
+	"github.com/alcionai/corso/pkg/selectors"
 )
 
 // BackupOperation wraps an operation with backup-specific props.
 type BackupOperation struct {
 	operation
 
-	Results BackupResults `json:"results"`
+	Results   BackupResults      `json:"results"`
-	Targets []string      `json:"selectors"` // todo: replace with Selectors
+	Selectors selectors.Selector `json:"selectors"`
-	Version string        `json:"version"`
+	Version   string             `json:"version"`
 
 	account account.Account
 }
@@ -36,11 +37,11 @@ func NewBackupOperation(
 	opts Options,
 	kw *kopia.Wrapper,
 	acct account.Account,
-	targets []string,
+	selector selectors.Selector,
 ) (BackupOperation, error) {
 	op := BackupOperation{
 		operation: newOperation(opts, kw),
-		Targets:   targets,
+		Selectors: selector,
 		Version:   "v0",
 		account:   acct,
 	}
@@ -81,7 +82,7 @@ func (op *BackupOperation) Run(ctx context.Context) error {
 	}
 
 	var cs []connector.DataCollection
-	cs, err = gc.ExchangeDataCollection(ctx, op.Targets[0])
+	cs, err = gc.ExchangeDataCollection(ctx, op.Selectors)
 	if err != nil {
 		stats.readErr = err
 		return errors.Wrap(err, "retrieving service data")

src/internal/operations/backup_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/alcionai/corso/internal/kopia"
 	ctesting "github.com/alcionai/corso/internal/testing"
 	"github.com/alcionai/corso/pkg/account"
+	"github.com/alcionai/corso/pkg/selectors"
 )
 
 // ---------------------------------------------------------------------------
@@ -50,7 +51,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
 		}
 	)
 
-	op, err := NewBackupOperation(ctx, Options{}, kw, acct, nil)
+	op, err := NewBackupOperation(ctx, Options{}, kw, acct, selectors.Selector{})
 	require.NoError(t, err)
 
 	op.persistResults(now, &stats)
@@ -112,7 +113,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
 				Options{},
 				test.kw,
 				test.acct,
-				nil)
+				selectors.Selector{})
 			test.errCheck(t, err)
 		})
 	}
@@ -143,12 +144,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
 	w, err := kopia.NewWrapper(k)
 	require.NoError(t, err)
 
+	sel := selectors.NewExchangeBackup()
+	sel.Include(sel.Users(m365User))
+
 	bo, err := NewBackupOperation(
 		ctx,
 		Options{},
 		w,
 		acct,
-		[]string{m365User})
+		sel.Selector)
 	require.NoError(t, err)
 
 	require.NoError(t, bo.Run(ctx))

src/pkg/repository/repository.go

@@ -10,6 +10,7 @@ import (
 	"github.com/alcionai/corso/internal/kopia"
 	"github.com/alcionai/corso/internal/operations"
 	"github.com/alcionai/corso/pkg/account"
+	"github.com/alcionai/corso/pkg/selectors"
 	"github.com/alcionai/corso/pkg/storage"
 )
 
@@ -109,13 +110,13 @@ func (r *Repository) Close(ctx context.Context) error {
 }
 
 // NewBackup generates a backupOperation runner.
-func (r Repository) NewBackup(ctx context.Context, targets []string) (operations.BackupOperation, error) {
+func (r Repository) NewBackup(ctx context.Context, selector selectors.Selector) (operations.BackupOperation, error) {
 	return operations.NewBackupOperation(
 		ctx,
 		operations.Options{},
 		r.dataLayer,
 		r.Account,
-		targets)
+		selector)
 }
 
 // NewRestore generates a restoreOperation runner.

src/pkg/repository/repository_test.go

@@ -11,6 +11,7 @@ import (
 	ctesting "github.com/alcionai/corso/internal/testing"
 	"github.com/alcionai/corso/pkg/account"
 	"github.com/alcionai/corso/pkg/repository"
+	"github.com/alcionai/corso/pkg/selectors"
 	"github.com/alcionai/corso/pkg/storage"
 )
 
@@ -170,7 +171,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
 	r, err := repository.Initialize(ctx, acct, st)
 	require.NoError(t, err)
 
-	bo, err := r.NewBackup(ctx, []string{})
+	bo, err := r.NewBackup(ctx, selectors.Selector{})
 	require.NoError(t, err)
 	require.NotNil(t, bo)
 }