From b0fe4220355eadf79bc376f5bb709ebefdef8404 Mon Sep 17 00:00:00 2001 From: Keepers <104464746+ryanfkeepers@users.noreply.github.com> Date: Tue, 7 Jun 2022 10:59:56 -0600 Subject: [PATCH] add pkg/credentials (#147) credentials requriements surface in many places thorughout corso: they can be sourced from many locations (envs, files, manually), and used in many more (cli, repo, kopia). This usage could blossom into all kinds of duplicate structs sharing similar info. The goal of this change is to centralize where credentials are declared and managed, and how they then cascade out to other packages. --- src/cli/backup/exchange.go | 15 +++++---- src/cli/repo/s3.go | 58 ++++++++++++++------------------- src/cli/utils/utils.go | 19 ----------- src/internal/testing/storage.go | 17 +++++----- src/pkg/credentials/aws.go | 37 +++++++++++++++++++++ src/pkg/credentials/corso.go | 23 +++++++++++++ src/pkg/credentials/m365.go | 21 ++++++++++++ src/pkg/storage/common.go | 15 ++++----- src/pkg/storage/common_test.go | 7 +++- src/pkg/storage/s3.go | 32 ++++++++---------- src/pkg/storage/s3_test.go | 33 ++++++++++++++++--- 11 files changed, 177 insertions(+), 100 deletions(-) create mode 100644 src/pkg/credentials/aws.go create mode 100644 src/pkg/credentials/corso.go create mode 100644 src/pkg/credentials/m365.go diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index a73d4fb28..d86f6c9fd 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/alcionai/corso/cli/utils" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/repository" "github.com/alcionai/corso/pkg/storage" ) @@ -39,18 +40,18 @@ var exchangeCreateCmd = &cobra.Command{ // initializes a s3 repo. func createExchangeCmd(cmd *cobra.Command, args []string) error { - mv := utils.GetM365Vars() + m365 := credentials.GetM365() fmt.Printf( "Called - %s\n\t365TenantID:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n", cmd.CommandPath(), - mv.TenantID, - mv.ClientID, - len(mv.ClientSecret) > 0) + m365.TenantID, + m365.ClientID, + len(m365.ClientSecret) > 0) a := repository.Account{ - TenantID: mv.TenantID, - ClientID: mv.ClientID, - ClientSecret: mv.ClientSecret, + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, } // todo (rkeepers) - retrieve storage details from corso config s, err := storage.NewStorage(storage.ProviderUnknown) diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 21460fe48..7a08a1db9 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -2,12 +2,12 @@ package repo import ( "fmt" - "os" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/alcionai/corso/cli/utils" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/repository" "github.com/alcionai/corso/pkg/storage" ) @@ -50,7 +50,7 @@ var s3InitCmd = &cobra.Command{ // initializes a s3 repo. func initS3Cmd(cmd *cobra.Command, args []string) error { - mv := utils.GetM365Vars() + m365 := credentials.GetM365() s3Cfg, commonCfg, err := makeS3Config() if err != nil { return err @@ -61,14 +61,14 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { cmd.CommandPath(), s3Cfg.Bucket, s3Cfg.AccessKey, - mv.ClientID, - len(mv.ClientSecret) > 0, + m365.ClientID, + len(m365.ClientSecret) > 0, len(s3Cfg.SecretKey) > 0) a := repository.Account{ - TenantID: mv.TenantID, - ClientID: mv.ClientID, - ClientSecret: mv.ClientSecret, + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, } s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg) if err != nil { @@ -96,7 +96,7 @@ var s3ConnectCmd = &cobra.Command{ // connects to an existing s3 repo. func connectS3Cmd(cmd *cobra.Command, args []string) error { - mv := utils.GetM365Vars() + m365 := credentials.GetM365() s3Cfg, commonCfg, err := makeS3Config() if err != nil { return err @@ -107,14 +107,14 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { cmd.CommandPath(), s3Cfg.Bucket, s3Cfg.AccessKey, - mv.ClientID, - len(mv.ClientSecret) > 0, + m365.ClientID, + len(m365.ClientSecret) > 0, len(s3Cfg.SecretKey) > 0) a := repository.Account{ - TenantID: mv.TenantID, - ClientID: mv.ClientID, - ClientSecret: mv.ClientSecret, + TenantID: m365.TenantID, + ClientID: m365.ClientID, + ClientSecret: m365.ClientSecret, } s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg) if err != nil { @@ -133,30 +133,22 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { // helper for aggregating aws connection details. func makeS3Config() (storage.S3Config, storage.CommonConfig, error) { - ak := os.Getenv(storage.AWS_ACCESS_KEY_ID) - if len(accessKey) > 0 { - ak = accessKey - } - secretKey := os.Getenv(storage.AWS_SECRET_ACCESS_KEY) - sessToken := os.Getenv(storage.AWS_SESSION_TOKEN) - corsoPasswd := os.Getenv(storage.CORSO_PASSWORD) - + aws := credentials.GetAWS(map[string]string{credentials.AWS_ACCESS_KEY_ID: accessKey}) + corso := credentials.GetCorso() return storage.S3Config{ - AccessKey: ak, - Bucket: bucket, - Endpoint: endpoint, - Prefix: prefix, - SecretKey: secretKey, - SessionToken: sessToken, + AWS: aws, + Bucket: bucket, + Endpoint: endpoint, + Prefix: prefix, }, storage.CommonConfig{ - CorsoPassword: corsoPasswd, + Corso: corso, }, utils.RequireProps(map[string]string{ - storage.AWS_ACCESS_KEY_ID: ak, - "bucket": bucket, - storage.AWS_SECRET_ACCESS_KEY: secretKey, - storage.AWS_SESSION_TOKEN: sessToken, - storage.CORSO_PASSWORD: corsoPasswd, + credentials.AWS_ACCESS_KEY_ID: aws.AccessKey, + "bucket": bucket, + credentials.AWS_SECRET_ACCESS_KEY: aws.SecretKey, + credentials.AWS_SESSION_TOKEN: aws.SessionToken, + credentials.CORSO_PASSWORD: corso.CorsoPassword, }) } diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 3e53967da..8db3cb1c3 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "os" "github.com/alcionai/corso/pkg/repository" ) @@ -20,24 +19,6 @@ func RequireProps(props map[string]string) error { return nil } -// aggregates m365 details from flag and env_var values. -type m365Vars struct { - ClientID string - ClientSecret string - TenantID string -} - -// GetM365Vars is a helper for aggregating m365 connection details. -func GetM365Vars() m365Vars { - // todo (rkeeprs): read from either corso config file or env vars. - // https://github.com/alcionai/corso/issues/120 - return m365Vars{ - ClientID: os.Getenv("CLIENT_ID"), - ClientSecret: os.Getenv("CLIENT_SECRET"), - TenantID: os.Getenv("TENANT_ID"), - } -} - // CloseRepo handles closing a repo. func CloseRepo(ctx context.Context, r *repository.Repository) { if err := r.Close(ctx); err != nil { diff --git a/src/internal/testing/storage.go b/src/internal/testing/storage.go index 0d62849de..eeab5ad76 100644 --- a/src/internal/testing/storage.go +++ b/src/internal/testing/storage.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/storage" ) @@ -13,9 +14,9 @@ import ( // variables with S3. func CheckS3EnvVars() error { s3Envs := []string{ - storage.AWS_ACCESS_KEY_ID, - storage.AWS_SECRET_ACCESS_KEY, - storage.AWS_SESSION_TOKEN, + credentials.AWS_ACCESS_KEY_ID, + credentials.AWS_SECRET_ACCESS_KEY, + credentials.AWS_SESSION_TOKEN, } for _, env := range s3Envs { if os.Getenv(env) == "" { @@ -32,14 +33,12 @@ func NewS3Storage(prefix string) (storage.Storage, error) { return storage.NewStorage( storage.ProviderS3, storage.S3Config{ - AccessKey: os.Getenv(storage.AWS_ACCESS_KEY_ID), - Bucket: "test-corso-repo-init", - Prefix: prefix, - SecretKey: os.Getenv(storage.AWS_SECRET_ACCESS_KEY), - SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN), + AWS: credentials.GetAWS(nil), + Bucket: "test-corso-repo-init", + Prefix: prefix, }, storage.CommonConfig{ - CorsoPassword: os.Getenv(storage.CORSO_PASSWORD), + Corso: credentials.GetCorso(), }, ) } diff --git a/src/pkg/credentials/aws.go b/src/pkg/credentials/aws.go new file mode 100644 index 000000000..e774de70e --- /dev/null +++ b/src/pkg/credentials/aws.go @@ -0,0 +1,37 @@ +package credentials + +import ( + "os" +) + +// envvar consts +const ( + AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" + AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" + AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN" +) + +// AWS aggregates aws credentials from flag and env_var values. +type AWS struct { + AccessKey string // required + SecretKey string // required + SessionToken string // required +} + +// GetAWS is a helper for aggregating aws secrets and credentials. +func GetAWS(override map[string]string) AWS { + accessKey := os.Getenv(AWS_ACCESS_KEY_ID) + if ovr, ok := override[AWS_ACCESS_KEY_ID]; ok { + accessKey = ovr + } + secretKey := os.Getenv(AWS_SECRET_ACCESS_KEY) + sessToken := os.Getenv(AWS_SESSION_TOKEN) + + // todo (rkeeprs): read from either corso config file or env vars. + // https://github.com/alcionai/corso/issues/120 + return AWS{ + AccessKey: accessKey, + SecretKey: secretKey, + SessionToken: sessToken, + } +} diff --git a/src/pkg/credentials/corso.go b/src/pkg/credentials/corso.go new file mode 100644 index 000000000..bc76bfb92 --- /dev/null +++ b/src/pkg/credentials/corso.go @@ -0,0 +1,23 @@ +package credentials + +import "os" + +// envvar consts +const ( + CORSO_PASSWORD = "CORSO_PASSWORD" +) + +// Corso aggregates corso credentials from flag and env_var values. +type Corso struct { + CorsoPassword string // required +} + +// GetCorso is a helper for aggregating Corso secrets and credentials. +func GetCorso() Corso { + // todo (rkeeprs): read from either corso config file or env vars. + // https://github.com/alcionai/corso/issues/120 + corsoPasswd := os.Getenv(CORSO_PASSWORD) + return Corso{ + CorsoPassword: corsoPasswd, + } +} diff --git a/src/pkg/credentials/m365.go b/src/pkg/credentials/m365.go new file mode 100644 index 000000000..64219c0e9 --- /dev/null +++ b/src/pkg/credentials/m365.go @@ -0,0 +1,21 @@ +package credentials + +import "os" + +// M365 aggregates m365 credentials from flag and env_var values. +type M365 struct { + ClientID string + ClientSecret string + TenantID string +} + +// M365 is a helper for aggregating m365 secrets and credentials. +func GetM365() M365 { + // todo (rkeeprs): read from either corso config file or env vars. + // https://github.com/alcionai/corso/issues/120 + return M365{ + ClientID: os.Getenv("CLIENT_ID"), + ClientSecret: os.Getenv("CLIENT_SECRET"), + TenantID: os.Getenv("TENANT_ID"), + } +} diff --git a/src/pkg/storage/common.go b/src/pkg/storage/common.go index c89550147..1fa2fb91d 100644 --- a/src/pkg/storage/common.go +++ b/src/pkg/storage/common.go @@ -1,16 +1,15 @@ package storage -import "github.com/pkg/errors" +import ( + "github.com/pkg/errors" + + "github.com/alcionai/corso/pkg/credentials" +) type CommonConfig struct { - CorsoPassword string // required + credentials.Corso // requires: CorsoPassword } -// envvar consts -const ( - CORSO_PASSWORD = "CORSO_PASSWORD" -) - // config key consts const ( keyCommonCorsoPassword = "common_corsoPassword" @@ -35,7 +34,7 @@ func (s Storage) CommonConfig() (CommonConfig, error) { // ensures all required properties are present func (c CommonConfig) validate() error { if len(c.CorsoPassword) == 0 { - return errors.Wrap(errMissingRequired, CORSO_PASSWORD) + return errors.Wrap(errMissingRequired, credentials.CORSO_PASSWORD) } return nil } diff --git a/src/pkg/storage/common_test.go b/src/pkg/storage/common_test.go index 8f5ddd807..5aec0b7e3 100644 --- a/src/pkg/storage/common_test.go +++ b/src/pkg/storage/common_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/storage" ) @@ -17,7 +18,11 @@ func TestCommonCfgSuite(t *testing.T) { suite.Run(t, new(CommonCfgSuite)) } -var goodCommonConfig = storage.CommonConfig{"passwd"} +var goodCommonConfig = storage.CommonConfig{ + Corso: credentials.Corso{ + CorsoPassword: "passwd", + }, +} func (suite *CommonCfgSuite) TestCommonConfig_Config() { cfg := goodCommonConfig diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index a6b3553d3..845f6e7e2 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -1,22 +1,18 @@ package storage -import "github.com/pkg/errors" +import ( + "github.com/pkg/errors" + + "github.com/alcionai/corso/pkg/credentials" +) type S3Config struct { - AccessKey string // required - Bucket string // required - Endpoint string - Prefix string - SecretKey string // required - SessionToken string // required -} + credentials.AWS // requires: AccessKey, SecretKey, SessionToken -// envvar consts -const ( - AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" - AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" - AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN" -) + Bucket string // required + Endpoint string + Prefix string +} // config key consts const ( @@ -56,10 +52,10 @@ func (s Storage) S3Config() (S3Config, error) { func (c S3Config) validate() error { check := map[string]string{ - AWS_ACCESS_KEY_ID: c.AccessKey, - AWS_SECRET_ACCESS_KEY: c.SecretKey, - AWS_SESSION_TOKEN: c.SessionToken, - "bucket": c.Bucket, + credentials.AWS_ACCESS_KEY_ID: c.AccessKey, + credentials.AWS_SECRET_ACCESS_KEY: c.SecretKey, + credentials.AWS_SESSION_TOKEN: c.SessionToken, + "bucket": c.Bucket, } for k, v := range check { if len(v) == 0 { diff --git a/src/pkg/storage/s3_test.go b/src/pkg/storage/s3_test.go index 6c5bc16df..985fd2161 100644 --- a/src/pkg/storage/s3_test.go +++ b/src/pkg/storage/s3_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/storage" ) @@ -17,7 +18,16 @@ func TestS3CfgSuite(t *testing.T) { suite.Run(t, new(S3CfgSuite)) } -var goodS3Config = storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"} +var goodS3Config = storage.S3Config{ + AWS: credentials.AWS{ + AccessKey: "ak", + SecretKey: "sk", + SessionToken: "tkn", + }, + Bucket: "bkt", + Endpoint: "end", + Prefix: "pre", +} func (suite *S3CfgSuite) TestS3Config_Config() { s3 := goodS3Config @@ -57,16 +67,29 @@ func (suite *S3CfgSuite) TestStorage_S3Config() { assert.Equal(t, in.SessionToken, out.SessionToken) } +func makeTestS3Cfg(ak, bkt, end, pre, sk, tkn string) storage.S3Config { + return storage.S3Config{ + AWS: credentials.AWS{ + AccessKey: ak, + SecretKey: sk, + SessionToken: tkn, + }, + Bucket: bkt, + Endpoint: end, + Prefix: pre, + } +} + func (suite *S3CfgSuite) TestStorage_S3Config_InvalidCases() { // missing required properties table := []struct { name string cfg storage.S3Config }{ - {"missing access key", storage.S3Config{"", "bkt", "end", "pre", "sk", "tkn"}}, - {"missing bucket", storage.S3Config{"ak", "", "end", "pre", "sk", "tkn"}}, - {"missing secret key", storage.S3Config{"ak", "bkt", "end", "pre", "", "tkn"}}, - {"missing session token", storage.S3Config{"ak", "bkt", "end", "pre", "sk", ""}}, + {"missing access key", makeTestS3Cfg("", "bkt", "end", "pre", "sk", "tkn")}, + {"missing bucket", makeTestS3Cfg("ak", "", "end", "pre", "sk", "tkn")}, + {"missing secret key", makeTestS3Cfg("ak", "bkt", "end", "pre", "", "tkn")}, + {"missing session token", makeTestS3Cfg("ak", "bkt", "end", "pre", "sk", "")}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) {