From 2104e66feaec79a959550f9c6685860de6f4b929 Mon Sep 17 00:00:00 2001 From: Keepers <104464746+ryanfkeepers@users.noreply.github.com> Date: Wed, 8 Jun 2022 17:32:14 -0600 Subject: [PATCH] add graph connector to backupOp (#165) Adds graph connector to the backup operation implementation. This includes feeding the account credentials into the backup as well. --- .github/workflows/ci.yml | 4 +++ src/go.mod | 2 +- src/internal/operations/backup.go | 20 +++++++++++--- src/internal/operations/backup_test.go | 33 +++++++++++++++++------ src/internal/operations/operation.go | 13 ++++----- src/internal/operations/operation_test.go | 6 +++-- src/pkg/credentials/credentials.go | 5 ++++ src/pkg/credentials/m365.go | 20 +++++++++++++- src/pkg/repository/repository.go | 7 +++++ 9 files changed, 89 insertions(+), 21 deletions(-) create mode 100644 src/pkg/credentials/credentials.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da3aec2d..4f00e63f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ permissions: jobs: Run-All: + environment: Testing runs-on: ubuntu-latest defaults: run: @@ -50,4 +51,7 @@ jobs: env: CORSO_CI_TESTS: true CORSO_PASSWORD: ${{ secrets.INTEGRATION_TEST_CORSO_PASSWORD }} + CLIENT_ID: ${{ secrets.CLIENT_ID }} + CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} + TENANT_ID: ${{ secrets.TENANT_ID }} run: go test ./... \ No newline at end of file diff --git a/src/go.mod b/src/go.mod index eac37ec50..50d5047cf 100644 --- a/src/go.mod +++ b/src/go.mod @@ -7,6 +7,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2022060816 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 github.com/google/uuid v1.3.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/kopia/kopia v0.10.7 github.com/microsoft/kiota-authentication-azure-go v0.3.0 github.com/microsoft/kiota-serialization-json-go v0.5.1 @@ -48,7 +49,6 @@ require ( github.com/golang-jwt/jwt v3.2.1+incompatible // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index f128d1d51..a28f818c9 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -3,13 +3,20 @@ package operations import ( "context" + "github.com/pkg/errors" + + "github.com/alcionai/corso/internal/connector" "github.com/alcionai/corso/internal/kopia" + "github.com/alcionai/corso/pkg/credentials" ) // BackupOperation wraps an operation with backup-specific props. type BackupOperation struct { operation Version string + + creds credentials.M365 + Targets []string // something for targets/filter/source/app&users/etc Work []string // something to reference the artifacts created, or at least their count @@ -22,14 +29,13 @@ func NewBackupOperation( ctx context.Context, opts OperationOpts, kw *kopia.KopiaWrapper, + creds credentials.M365, targets []string, ) (BackupOperation, error) { - // todo - initialize a graphConnector - // gc, err := graphConnector.Connect(bo.account) - bo := BackupOperation{ operation: newOperation(opts, kw), Version: "v0", + creds: creds, Targets: targets, Work: []string{}, } @@ -41,11 +47,19 @@ func NewBackupOperation( } func (bo BackupOperation) validate() error { + if err := bo.creds.Validate(); err != nil { + return errors.Wrap(err, "invalid credentials") + } return bo.operation.validate() } // Run begins a synchronous backup operation. func (bo BackupOperation) Run(ctx context.Context) error { + _, err := connector.NewGraphConnector(bo.creds.TenantID, bo.creds.ClientID, bo.creds.ClientSecret) + if err != nil { + return errors.Wrap(err, "connecting to graph api") + } + // todo - use the graphConnector to create datastreams // dStreams, err := bo.gc.BackupOp(bo.Targets) diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 3ea416816..c2f03eb03 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/internal/operations" ctesting "github.com/alcionai/corso/internal/testing" + "github.com/alcionai/corso/pkg/credentials" ) type BackupOpIntegrationSuite struct { @@ -23,24 +24,40 @@ func TestBackupOpIntegrationSuite(t *testing.T) { suite.Run(t, new(BackupOpIntegrationSuite)) } +func (suite *BackupOpIntegrationSuite) SetupSuite() { + if _, err := ctesting.GetRequiredEnvVars( + credentials.TenantID, + credentials.ClientID, + credentials.ClientSecret, + ); err != nil { + suite.T().Fatal(err) + } +} + func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { + kw := &kopia.KopiaWrapper{} + creds := credentials.GetM365() table := []struct { - name string - opts operations.OperationOpts - kw *kopia.KopiaWrapper - targets []string + name string + opts operations.OperationOpts + kw *kopia.KopiaWrapper + creds credentials.M365 + targets []string + errCheck assert.ErrorAssertionFunc }{ - {"good", operations.OperationOpts{}, new(kopia.KopiaWrapper), nil}, - {"missing kopia", operations.OperationOpts{}, nil, nil}, + {"good", operations.OperationOpts{}, kw, creds, nil, assert.NoError}, + {"missing kopia", operations.OperationOpts{}, nil, creds, nil, assert.Error}, + {"invalid creds", operations.OperationOpts{}, kw, credentials.M365{}, nil, assert.Error}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { _, err := operations.NewBackupOperation( context.Background(), operations.OperationOpts{}, - new(kopia.KopiaWrapper), + test.kw, + test.creds, nil) - assert.NoError(t, err) + test.errCheck(t, err) }) } } diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index bbcd36269..f852d2107 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -2,11 +2,12 @@ package operations import ( "context" - "errors" "time" - "github.com/alcionai/corso/internal/kopia" "github.com/google/uuid" + "github.com/pkg/errors" + + "github.com/alcionai/corso/internal/kopia" ) type opStatus int @@ -28,9 +29,6 @@ type operation struct { options OperationOpts kopia *kopia.KopiaWrapper - // TODO(rkeepers) deal with circular dependencies here - // graphConn GraphConnector // m365 details - Status opStatus Errors []error } @@ -47,7 +45,10 @@ type OperationOpts struct { Logger logger } -func newOperation(opts OperationOpts, kw *kopia.KopiaWrapper) operation { +func newOperation( + opts OperationOpts, + kw *kopia.KopiaWrapper, +) operation { return operation{ ID: uuid.New(), CreatedAt: time.Now(), diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index 36846a7c5..32926ace9 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -3,9 +3,10 @@ package operations import ( "testing" - "github.com/alcionai/corso/internal/kopia" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/internal/kopia" ) type OperationSuite struct { @@ -24,12 +25,13 @@ func (suite *OperationSuite) TestNewOperation() { } func (suite *OperationSuite) TestOperation_Validate() { + kwStub := &kopia.KopiaWrapper{} table := []struct { name string kw *kopia.KopiaWrapper errCheck assert.ErrorAssertionFunc }{ - {"good", new(kopia.KopiaWrapper), assert.NoError}, + {"good", kwStub, assert.NoError}, {"missing kopia", nil, assert.Error}, } for _, test := range table { diff --git a/src/pkg/credentials/credentials.go b/src/pkg/credentials/credentials.go new file mode 100644 index 000000000..feffa749a --- /dev/null +++ b/src/pkg/credentials/credentials.go @@ -0,0 +1,5 @@ +package credentials + +import "errors" + +var errMissingRequired = errors.New("missing required storage configuration") diff --git a/src/pkg/credentials/m365.go b/src/pkg/credentials/m365.go index c3425b79d..52d24bda8 100644 --- a/src/pkg/credentials/m365.go +++ b/src/pkg/credentials/m365.go @@ -1,6 +1,10 @@ package credentials -import "os" +import ( + "os" + + "github.com/pkg/errors" +) // envvar consts const ( @@ -26,3 +30,17 @@ func GetM365() M365 { TenantID: os.Getenv(TenantID), } } + +func (c M365) Validate() error { + check := map[string]string{ + ClientID: c.ClientID, + ClientSecret: c.ClientSecret, + TenantID: c.TenantID, + } + for k, v := range check { + if len(v) == 0 { + return errors.Wrap(errMissingRequired, k) + } + } + return nil +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 117c2cd3b..1c65d1b92 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/internal/operations" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/storage" ) @@ -106,9 +107,15 @@ func (r *Repository) Close(ctx context.Context) error { // NewBackup generates a backupOperation runner. func (r Repository) NewBackup(ctx context.Context, targets []string) (operations.BackupOperation, error) { + creds := credentials.M365{ + ClientID: r.Account.ClientID, + ClientSecret: r.Account.ClientSecret, + TenantID: r.Account.TenantID, + } return operations.NewBackupOperation( ctx, operations.OperationOpts{}, r.dataLayer, + creds, targets) }