larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wire selectors up through backup handling (#278)

Browse Source 
* wire selectors up through backup handling

Selectors are implemented enough to add them end-
to-end in some places.  This starts with backup
creation, since that's the most stable set of code in
the repo at the moment.
This commit is contained in:
Keepers 2022-07-07 17:02:25 -06:00 committed by GitHub
parent 1143a33ce6
commit c29a4fffe0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 90 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cli/backup/exchange.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/cli/utils"
"github.com/alcionai/corso/pkg/logger"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/selectors"
)
// exchange bucket info from flags
@ -72,7 +73,10 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
}
defer utils.CloseRepo(ctx, r)
bo, err := r.NewBackup(ctx, []string{user})
sel := selectors.NewExchangeBackup()
sel.Include(sel.Users(user))
bo, err := r.NewBackup(ctx, sel.Selector)
if err != nil {
return errors.Wrap(err, "Failed to initialize Exchange backup")
}

43
src/internal/connector/graph_connector.go
View File

@ -8,7 +8,6 @@ import (
"fmt"
az "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/alcionai/corso/internal/connector/support"
ka "github.com/microsoft/kiota-authentication-azure-go"
kw "github.com/microsoft/kiota-serialization-json-go"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
@ -18,8 +17,10 @@ import (
msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders"
"github.com/pkg/errors"
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/logger"
"github.com/alcionai/corso/pkg/selectors"
)
const (
@ -139,11 +140,47 @@ func buildFromMap(isKey bool, mapping map[string]string) []string {
// Assumption: User exists
// TODO: https://github.com/alcionai/corso/issues/135
// Add iota to this call -> mail, contacts, calendar, etc.
func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, user string) ([]DataCollection, error) {
func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector selectors.Selector) ([]DataCollection, error) {
eb, err := selector.ToExchangeBackup()
if err != nil {
return nil, errors.Wrap(err, "collecting exchange data")
}
collections := []DataCollection{}
scopes := eb.Scopes()
var errs error
// for each scope that includes mail messages, get all
for _, scope := range scopes {
if !scope.IncludesCategory(selectors.ExchangeMail) {
continue
}
for _, user := range scope.Get(selectors.ExchangeUser) {
// TODO: handle "get mail for all users"
// this would probably no-op without this check,
// but we want it made obvious that we're punting.
if user == selectors.All {
errs = support.WrapAndAppend(
"all-users",
errors.New("all users selector currently not handled"),
errs)
continue
}
dcs, err := gc.serializeMessages(ctx, user)
if err != nil {
errs = support.WrapAndAppend(user, err, errs)
}
if len(dcs) > 0 {
collections = append(collections, dcs...)
}
}
}
// TODO replace with completion of Issue 124:
//TODO: Retry handler to convert return: (DataCollection, error)
return gc.serializeMessages(ctx, user)
return collections, errs
}
// optionsForMailFolders creates transforms the 'select' into a more dynamic call for MailFolders.

42
src/internal/connector/graph_connector_test.go
View File

@ -13,6 +13,7 @@ import (
ctesting "github.com/alcionai/corso/internal/testing"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/selectors"
)
type GraphConnectorIntegrationSuite struct {
@ -31,6 +32,10 @@ func TestGraphConnectorIntetgrationSuite(t *testing.T) {
}
func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil {
suite.T().Skip(err)
}
_, err := ctesting.GetRequiredEnvVars(ctesting.M365AcctCredEnvs...)
require.NoError(suite.T(), err)
@ -46,14 +51,6 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector() {
suite.NotNil(suite.connector)
}
type DisconnectedGraphConnectorSuite struct {
suite.Suite
}
func TestDisconnectedGraphSuite(t *testing.T) {
suite.Run(t, new(DisconnectedGraphConnectorSuite))
}
func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() {
err := suite.connector.setTenantUsers()
assert.NoError(suite.T(), err)
@ -61,14 +58,17 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers()
}
func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataCollection() {
if err := ctesting.RunOnAny(ctesting.CorsoCITests); err != nil {
suite.T().Skip(err)
}
collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), "lidiah@8qzvrj.onmicrosoft.com")
assert.NotNil(suite.T(), collectionList, "collection list")
assert.Error(suite.T(), err) // TODO Remove after https://github.com/alcionai/corso/issues/140
assert.NotNil(suite.T(), suite.connector.status, "connector status")
suite.NotContains(err.Error(), "attachment failed") // TODO Create Retry Exceeded Error
t := suite.T()
sel := selectors.NewExchangeBackup()
sel.Include(sel.Users("lidiah@8qzvrj.onmicrosoft.com"))
collectionList, err := suite.connector.ExchangeDataCollection(context.Background(), sel.Selector)
require.NotNil(t, collectionList, "collection list")
assert.Error(t, err) // TODO Remove after https://github.com/alcionai/corso/issues/140
assert.NotNil(t, suite.connector.status, "connector status")
assert.NotContains(t, err.Error(), "attachment failed") // TODO Create Retry Exceeded Error
exchangeData := collectionList[0]
suite.Greater(len(exchangeData.FullPath()), 2)
}
@ -92,6 +92,16 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages(
assert.NoError(suite.T(), err)
}
// ---------------------------------------------------------------------------
type DisconnectedGraphConnectorSuite struct {
suite.Suite
}
func TestDisconnectedGraphSuite(t *testing.T) {
suite.Run(t, new(DisconnectedGraphConnectorSuite))
}
func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() {
table := []struct {

13
src/internal/operations/backup.go
View File

@ -10,15 +10,16 @@ import (
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/selectors"
)
// BackupOperation wraps an operation with backup-specific props.
type BackupOperation struct {
operation
Results BackupResults `json:"results"`
Targets []string `json:"selectors"` // todo: replace with Selectors
Version string `json:"version"`
Results BackupResults `json:"results"`
Selectors selectors.Selector `json:"selectors"`
Version string `json:"version"`
account account.Account
}
@ -36,11 +37,11 @@ func NewBackupOperation(
opts Options,
kw *kopia.Wrapper,
acct account.Account,
targets []string,
selector selectors.Selector,
) (BackupOperation, error) {
op := BackupOperation{
operation: newOperation(opts, kw),
Targets: targets,
Selectors: selector,
Version: "v0",
account: acct,
}
@ -81,7 +82,7 @@ func (op *BackupOperation) Run(ctx context.Context) error {
}
var cs []connector.DataCollection
cs, err = gc.ExchangeDataCollection(ctx, op.Targets[0])
cs, err = gc.ExchangeDataCollection(ctx, op.Selectors)
if err != nil {
stats.readErr = err
return errors.Wrap(err, "retrieving service data")

10
src/internal/operations/backup_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/internal/kopia"
ctesting "github.com/alcionai/corso/internal/testing"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/selectors"
)
// ---------------------------------------------------------------------------
@ -50,7 +51,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
}
)
op, err := NewBackupOperation(ctx, Options{}, kw, acct, nil)
op, err := NewBackupOperation(ctx, Options{}, kw, acct, selectors.Selector{})
require.NoError(t, err)
op.persistResults(now, &stats)
@ -112,7 +113,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
Options{},
test.kw,
test.acct,
nil)
selectors.Selector{})
test.errCheck(t, err)
})
}
@ -143,12 +144,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
w, err := kopia.NewWrapper(k)
require.NoError(t, err)
sel := selectors.NewExchangeBackup()
sel.Include(sel.Users(m365User))
bo, err := NewBackupOperation(
ctx,
Options{},
w,
acct,
[]string{m365User})
sel.Selector)
require.NoError(t, err)
require.NoError(t, bo.Run(ctx))

5
src/pkg/repository/repository.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/internal/operations"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/storage"
)
@ -109,13 +110,13 @@ func (r *Repository) Close(ctx context.Context) error {
}
// NewBackup generates a backupOperation runner.
func (r Repository) NewBackup(ctx context.Context, targets []string) (operations.BackupOperation, error) {
func (r Repository) NewBackup(ctx context.Context, selector selectors.Selector) (operations.BackupOperation, error) {
return operations.NewBackupOperation(
ctx,
operations.Options{},
r.dataLayer,
r.Account,
targets)
selector)
}
// NewRestore generates a restoreOperation runner.

3
src/pkg/repository/repository_test.go
View File

@ -11,6 +11,7 @@ import (
ctesting "github.com/alcionai/corso/internal/testing"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/storage"
)
@ -170,7 +171,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
r, err := repository.Initialize(ctx, acct, st)
require.NoError(t, err)
bo, err := r.NewBackup(ctx, []string{})
bo, err := r.NewBackup(ctx, selectors.Selector{})
require.NoError(t, err)
require.NotNil(t, bo)
}