diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index d86f6c9fd..f7ed855e7 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -6,10 +6,10 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/alcionai/corso/cli/config" "github.com/alcionai/corso/cli/utils" "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/repository" - "github.com/alcionai/corso/pkg/storage" ) // exchange bucket info from flags @@ -40,7 +40,21 @@ var exchangeCreateCmd = &cobra.Command{ // initializes a s3 repo. func createExchangeCmd(cmd *cobra.Command, args []string) error { + s, cfgTenantID, err := config.MakeS3Config(true, nil) + if err != nil { + return err + } + m365 := credentials.GetM365() + a := repository.Account{ + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, + } + if len(cfgTenantID) > 0 { + a.TenantID = cfgTenantID + } + fmt.Printf( "Called - %s\n\t365TenantID:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n", cmd.CommandPath(), @@ -48,17 +62,6 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { m365.ClientID, len(m365.ClientSecret) > 0) - a := repository.Account{ - TenantID: m365.TenantID, - ClientID: m365.ClientID, - ClientSecret: m365.ClientSecret, - } - // todo (rkeepers) - retrieve storage details from corso config - s, err := storage.NewStorage(storage.ProviderUnknown) - if err != nil { - return errors.Wrap(err, "Failed to configure storage provider") - } - r, err := repository.Connect(cmd.Context(), a, s) if err != nil { return errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider) diff --git a/src/cli/config/config.go b/src/cli/config/config.go index 239344b74..21de643e1 100644 --- a/src/cli/config/config.go +++ b/src/cli/config/config.go @@ -5,6 +5,8 @@ import ( "path" "strings" + "github.com/alcionai/corso/cli/utils" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/repository" "github.com/alcionai/corso/pkg/storage" "github.com/pkg/errors" @@ -92,3 +94,70 @@ func ReadRepoConfig() (s3Config storage.S3Config, account repository.Account, er return s3Config, account, nil } + +// MakeS3Config creates a storage instance by mediating all the possible +// data sources (config file, env vars, flag overrides) in the config. +func MakeS3Config(readFromFile bool, overrides map[string]string) (storage.Storage, string, error) { + var ( + s3Cfg storage.S3Config + account repository.Account + err error + ) + + // possibly read the prior config from a .corso file + if readFromFile { + s3Cfg, account, err = ReadRepoConfig() + if err != nil { + return storage.Storage{}, "", errors.Wrap(err, "reading corso config file") + } + } + + // compose the s3 storage config and credentials + aws := credentials.GetAWS(overrides) + if err := aws.Validate(); err != nil { + return storage.Storage{}, "", errors.Wrap(err, "validating aws credentials") + } + s3Cfg = storage.S3Config{ + AWS: aws, + Bucket: first(overrides[storage.Bucket], s3Cfg.Bucket), + Endpoint: first(overrides[storage.Endpoint], s3Cfg.Endpoint), + Prefix: first(overrides[storage.Prefix], s3Cfg.Prefix), + } + + // compose the common config and credentials + corso := credentials.GetCorso() + if err := corso.Validate(); err != nil { + return storage.Storage{}, "", errors.Wrap(err, "validating corso credentials") + } + cCfg := storage.CommonConfig{ + Corso: corso, + } + + // ensure requried properties are present + if err := utils.RequireProps(map[string]string{ + credentials.AWSAccessKeyID: aws.AccessKey, + storage.Bucket: s3Cfg.Bucket, + credentials.AWSSecretAccessKey: aws.SecretKey, + credentials.AWSSessionToken: aws.SessionToken, + credentials.CorsoPassword: corso.CorsoPassword, + }); err != nil { + return storage.Storage{}, "", err + } + + // return a complete storage + s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg) + if err != nil { + return storage.Storage{}, "", errors.Wrap(err, "configuring repository storage") + } + return s, account.TenantID, nil +} + +// returns the first non-zero valued string +func first(vs ...string) string { + for _, v := range vs { + if len(v) > 0 { + return v + } + } + return "" +} diff --git a/src/cli/config/config_test.go b/src/cli/config/config_test.go index bdeb39950..76ec9da6e 100644 --- a/src/cli/config/config_test.go +++ b/src/cli/config/config_test.go @@ -6,12 +6,13 @@ import ( "path" "testing" - "github.com/alcionai/corso/cli/config" - "github.com/alcionai/corso/pkg/repository" - "github.com/alcionai/corso/pkg/storage" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/cli/config" + "github.com/alcionai/corso/pkg/repository" + "github.com/alcionai/corso/pkg/storage" ) const ( @@ -34,7 +35,7 @@ func TestConfigSuite(t *testing.T) { func (suite *ConfigSuite) TestReadRepoConfigBasic() { // Generate test config file - b := "test-bucket" + b := "read-repo-config-basic-bucket" tID := "6f34ac30-8196-469b-bf8f-d83deadbbbba" testConfigData := fmt.Sprintf(configFileTemplate, b, tID) testConfigFilePath := path.Join(suite.T().TempDir(), "corso.toml") @@ -58,7 +59,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() { err := config.InitConfig(testConfigFilePath) assert.NoError(suite.T(), err) - s3Cfg := storage.S3Config{Bucket: "bucket"} + s3Cfg := storage.S3Config{Bucket: "write-read-config-bucket"} account := repository.Account{TenantID: "6f34ac30-8196-469b-bf8f-d83deadbbbbd"} err = config.WriteRepoConfig(s3Cfg, account) assert.NoError(suite.T(), err) diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 5d5a75aba..6ffbe833f 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -5,7 +5,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/alcionai/corso/cli/config" "github.com/alcionai/corso/cli/utils" @@ -55,11 +54,30 @@ var s3InitCmd = &cobra.Command{ func initS3Cmd(cmd *cobra.Command, args []string) error { log := logger.Ctx(cmd.Context()) - m365 := credentials.GetM365() - s3Cfg, commonCfg, err := makeS3Config() + overrides := map[string]string{ + credentials.AWSAccessKeyID: accessKey, + storage.Bucket: bucket, + storage.Endpoint: endpoint, + storage.Prefix: prefix, + } + s, cfgTenantID, err := config.MakeS3Config(false, overrides) if err != nil { return err } + s3Cfg, err := s.S3Config() + if err != nil { + return errors.Wrap(err, "Retrieving s3 configuration") + } + + m365 := credentials.GetM365() + a := repository.Account{ + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, + } + if len(cfgTenantID) > 0 { + a.TenantID = cfgTenantID + } log.Debugw( "Called - "+cmd.CommandPath(), @@ -69,16 +87,6 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { "accessKey", s3Cfg.AccessKey, "hasSecretKey", len(s3Cfg.SecretKey) > 0) - a := repository.Account{ - TenantID: m365.TenantID, - ClientID: m365.ClientID, - ClientSecret: m365.ClientSecret, - } - s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg) - if err != nil { - return errors.Wrap(err, "Failed to configure storage provider") - } - r, err := repository.Initialize(cmd.Context(), a, s) if err != nil { return errors.Wrap(err, "Failed to initialize a new S3 repository") @@ -106,11 +114,30 @@ var s3ConnectCmd = &cobra.Command{ func connectS3Cmd(cmd *cobra.Command, args []string) error { log := logger.Ctx(cmd.Context()) - m365 := credentials.GetM365() - s3Cfg, commonCfg, err := makeS3Config() + overrides := map[string]string{ + credentials.AWSAccessKeyID: accessKey, + storage.Bucket: bucket, + storage.Endpoint: endpoint, + storage.Prefix: prefix, + } + s, cfgTenantID, err := config.MakeS3Config(true, overrides) if err != nil { return err } + s3Cfg, err := s.S3Config() + if err != nil { + return errors.Wrap(err, "Retrieving s3 configuration") + } + + m365 := credentials.GetM365() + a := repository.Account{ + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, + } + if len(cfgTenantID) > 0 { + a.TenantID = cfgTenantID + } log.Debugw( "Called - "+cmd.CommandPath(), @@ -120,23 +147,6 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { "accessKey", s3Cfg.AccessKey, "hasSecretKey", len(s3Cfg.SecretKey) > 0) - // TODO: Merge/Validate any local configuration here to make sure there are no conflicts - // For now - just reading/logging the local config here (a successful repo connect will overwrite) - localS3Cfg, localAccount, err := config.ReadRepoConfig() - if err == nil { - fmt.Printf("ConfigFile - %s\n\tbucket:\t%s\n\ttenantID:\t%s\n", viper.ConfigFileUsed(), localS3Cfg.Bucket, localAccount.TenantID) - } - - a := repository.Account{ - TenantID: m365.TenantID, - ClientID: m365.ClientID, - ClientSecret: m365.ClientSecret, - } - s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg) - if err != nil { - return errors.Wrap(err, "Failed to configure storage provider") - } - r, err := repository.Connect(cmd.Context(), a, s) if err != nil { return errors.Wrap(err, "Failed to connect to the S3 repository") @@ -150,25 +160,3 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { } return nil } - -// helper for aggregating aws connection details. -func makeS3Config() (storage.S3Config, storage.CommonConfig, error) { - aws := credentials.GetAWS(map[string]string{credentials.AWSAccessKeyID: accessKey}) - corso := credentials.GetCorso() - return storage.S3Config{ - AWS: aws, - Bucket: bucket, - Endpoint: endpoint, - Prefix: prefix, - }, - storage.CommonConfig{ - Corso: corso, - }, - utils.RequireProps(map[string]string{ - credentials.AWSAccessKeyID: aws.AccessKey, - "bucket": bucket, - credentials.AWSSecretAccessKey: aws.SecretKey, - credentials.AWSSessionToken: aws.SessionToken, - credentials.CorsoPassword: corso.CorsoPassword, - }) -} diff --git a/src/internal/testing/integration_runners.go b/src/internal/testing/integration_runners.go index 90a5bcc61..6538cf44b 100644 --- a/src/internal/testing/integration_runners.go +++ b/src/internal/testing/integration_runners.go @@ -12,8 +12,8 @@ import ( const ( CorsoCITests = "CORSO_CI_TESTS" CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS" - CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS" CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS" + CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS" ) // RunOnAny takes in a list of env variable names and returns diff --git a/src/pkg/credentials/aws.go b/src/pkg/credentials/aws.go index f1f194e0e..16c946bba 100644 --- a/src/pkg/credentials/aws.go +++ b/src/pkg/credentials/aws.go @@ -2,6 +2,8 @@ package credentials import ( "os" + + "github.com/pkg/errors" ) // envvar consts @@ -35,3 +37,17 @@ func GetAWS(override map[string]string) AWS { SessionToken: sessToken, } } + +func (c AWS) Validate() error { + check := map[string]string{ + AWSAccessKeyID: c.AccessKey, + AWSSecretAccessKey: c.SecretKey, + AWSSessionToken: c.SessionToken, + } + for k, v := range check { + if len(v) == 0 { + return errors.Wrap(errMissingRequired, k) + } + } + return nil +} diff --git a/src/pkg/credentials/corso.go b/src/pkg/credentials/corso.go index 87a2cb393..9449c37a3 100644 --- a/src/pkg/credentials/corso.go +++ b/src/pkg/credentials/corso.go @@ -1,6 +1,10 @@ package credentials -import "os" +import ( + "os" + + "github.com/pkg/errors" +) // envvar consts const ( @@ -21,3 +25,15 @@ func GetCorso() Corso { CorsoPassword: corsoPasswd, } } + +func (c Corso) Validate() error { + check := map[string]string{ + CorsoPassword: c.CorsoPassword, + } + for k, v := range check { + if len(v) == 0 { + return errors.Wrap(errMissingRequired, k) + } + } + return nil +} diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index e6f3f4e30..26cf8e732 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -24,6 +24,13 @@ const ( keyS3SessionToken = "s3_sessionToken" ) +// config exported name consts +const ( + Bucket = "bucket" + Endpoint = "endpoint" + Prefix = "prefix" +) + func (c S3Config) Config() (config, error) { cfg := config{ keyS3AccessKey: c.AccessKey, @@ -55,7 +62,7 @@ func (c S3Config) validate() error { credentials.AWSAccessKeyID: c.AccessKey, credentials.AWSSecretAccessKey: c.SecretKey, credentials.AWSSessionToken: c.SessionToken, - "bucket": c.Bucket, + Bucket: c.Bucket, } for k, v := range check { if len(v) == 0 {