larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add pkg/credentials (#147)

Browse Source 
credentials requriements surface in many places thorughout corso:
they can be sourced from many locations (envs, files, manually),
and used in many more (cli, repo, kopia).  This usage could blossom
into all kinds of duplicate structs sharing similar info.  The goal
of this change is to centralize where credentials are declared
and managed, and how they then cascade out to other packages.
This commit is contained in:
Keepers 2022-06-07 10:59:56 -06:00 committed by GitHub
parent 3be698eab9
commit b0fe422035
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 177 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/cli/backup/exchange.go
View File

@ -7,6 +7,7 @@ import (
"github.com/spf13/cobra"
"github.com/alcionai/corso/cli/utils"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
)
@ -39,18 +40,18 @@ var exchangeCreateCmd = &cobra.Command{
// initializes a s3 repo.
func createExchangeCmd(cmd *cobra.Command, args []string) error {
mv := utils.GetM365Vars()
m365 := credentials.GetM365()
fmt.Printf(
"Called - %s\n\t365TenantID:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n",
cmd.CommandPath(),
mv.TenantID,
mv.ClientID,
len(mv.ClientSecret) > 0)
m365.TenantID,
m365.ClientID,
len(m365.ClientSecret) > 0)
a := repository.Account{
TenantID: mv.TenantID,
ClientID: mv.ClientID,
ClientSecret: mv.ClientSecret,
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
// todo (rkeepers) - retrieve storage details from corso config
s, err := storage.NewStorage(storage.ProviderUnknown)

58
src/cli/repo/s3.go
View File

@ -2,12 +2,12 @@ package repo
import (
"fmt"
"os"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/alcionai/corso/cli/utils"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
)
@ -50,7 +50,7 @@ var s3InitCmd = &cobra.Command{
// initializes a s3 repo.
func initS3Cmd(cmd *cobra.Command, args []string) error {
mv := utils.GetM365Vars()
m365 := credentials.GetM365()
s3Cfg, commonCfg, err := makeS3Config()
if err != nil {
return err
@ -61,14 +61,14 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
cmd.CommandPath(),
s3Cfg.Bucket,
s3Cfg.AccessKey,
mv.ClientID,
len(mv.ClientSecret) > 0,
m365.ClientID,
len(m365.ClientSecret) > 0,
len(s3Cfg.SecretKey) > 0)
a := repository.Account{
TenantID: mv.TenantID,
ClientID: mv.ClientID,
ClientSecret: mv.ClientSecret,
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if err != nil {
@ -96,7 +96,7 @@ var s3ConnectCmd = &cobra.Command{
// connects to an existing s3 repo.
func connectS3Cmd(cmd *cobra.Command, args []string) error {
mv := utils.GetM365Vars()
m365 := credentials.GetM365()
s3Cfg, commonCfg, err := makeS3Config()
if err != nil {
return err
@ -107,14 +107,14 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
cmd.CommandPath(),
s3Cfg.Bucket,
s3Cfg.AccessKey,
mv.ClientID,
len(mv.ClientSecret) > 0,
m365.ClientID,
len(m365.ClientSecret) > 0,
len(s3Cfg.SecretKey) > 0)
a := repository.Account{
TenantID: mv.TenantID,
ClientID: mv.ClientID,
ClientSecret: mv.ClientSecret,
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if err != nil {
@ -133,30 +133,22 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
// helper for aggregating aws connection details.
func makeS3Config() (storage.S3Config, storage.CommonConfig, error) {
ak := os.Getenv(storage.AWS_ACCESS_KEY_ID)
if len(accessKey) > 0 {
ak = accessKey
}
secretKey := os.Getenv(storage.AWS_SECRET_ACCESS_KEY)
sessToken := os.Getenv(storage.AWS_SESSION_TOKEN)
corsoPasswd := os.Getenv(storage.CORSO_PASSWORD)
aws := credentials.GetAWS(map[string]string{credentials.AWS_ACCESS_KEY_ID: accessKey})
corso := credentials.GetCorso()
return storage.S3Config{
AccessKey: ak,
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
SecretKey: secretKey,
SessionToken: sessToken,
AWS: aws,
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
},
storage.CommonConfig{
CorsoPassword: corsoPasswd,
Corso: corso,
},
utils.RequireProps(map[string]string{
storage.AWS_ACCESS_KEY_ID: ak,
"bucket": bucket,
storage.AWS_SECRET_ACCESS_KEY: secretKey,
storage.AWS_SESSION_TOKEN: sessToken,
storage.CORSO_PASSWORD: corsoPasswd,
credentials.AWS_ACCESS_KEY_ID: aws.AccessKey,
"bucket": bucket,
credentials.AWS_SECRET_ACCESS_KEY: aws.SecretKey,
credentials.AWS_SESSION_TOKEN: aws.SessionToken,
credentials.CORSO_PASSWORD: corso.CorsoPassword,
})
}

19
src/cli/utils/utils.go
View File

@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"os"
"github.com/alcionai/corso/pkg/repository"
)
@ -20,24 +19,6 @@ func RequireProps(props map[string]string) error {
return nil
}
// aggregates m365 details from flag and env_var values.
type m365Vars struct {
ClientID string
ClientSecret string
TenantID string
}
// GetM365Vars is a helper for aggregating m365 connection details.
func GetM365Vars() m365Vars {
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
return m365Vars{
ClientID: os.Getenv("CLIENT_ID"),
ClientSecret: os.Getenv("CLIENT_SECRET"),
TenantID: os.Getenv("TENANT_ID"),
}
}
// CloseRepo handles closing a repo.
func CloseRepo(ctx context.Context, r *repository.Repository) {
if err := r.Close(ctx); err != nil {

17
src/internal/testing/storage.go
View File

@ -5,6 +5,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/storage"
)
@ -13,9 +14,9 @@ import (
// variables with S3.
func CheckS3EnvVars() error {
s3Envs := []string{
storage.AWS_ACCESS_KEY_ID,
storage.AWS_SECRET_ACCESS_KEY,
storage.AWS_SESSION_TOKEN,
credentials.AWS_ACCESS_KEY_ID,
credentials.AWS_SECRET_ACCESS_KEY,
credentials.AWS_SESSION_TOKEN,
}
for _, env := range s3Envs {
if os.Getenv(env) == "" {
@ -32,14 +33,12 @@ func NewS3Storage(prefix string) (storage.Storage, error) {
return storage.NewStorage(
storage.ProviderS3,
storage.S3Config{
AccessKey: os.Getenv(storage.AWS_ACCESS_KEY_ID),
Bucket: "test-corso-repo-init",
Prefix: prefix,
SecretKey: os.Getenv(storage.AWS_SECRET_ACCESS_KEY),
SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN),
AWS: credentials.GetAWS(nil),
Bucket: "test-corso-repo-init",
Prefix: prefix,
},
storage.CommonConfig{
CorsoPassword: os.Getenv(storage.CORSO_PASSWORD),
Corso: credentials.GetCorso(),
},
)
}

37
src/pkg/credentials/aws.go Normal file
View File

@ -0,0 +1,37 @@
package credentials
import (
"os"
)
// envvar consts
const (
AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN"
)
// AWS aggregates aws credentials from flag and env_var values.
type AWS struct {
AccessKey string // required
SecretKey string // required
SessionToken string // required
}
// GetAWS is a helper for aggregating aws secrets and credentials.
func GetAWS(override map[string]string) AWS {
accessKey := os.Getenv(AWS_ACCESS_KEY_ID)
if ovr, ok := override[AWS_ACCESS_KEY_ID]; ok {
accessKey = ovr
}
secretKey := os.Getenv(AWS_SECRET_ACCESS_KEY)
sessToken := os.Getenv(AWS_SESSION_TOKEN)
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
return AWS{
AccessKey: accessKey,
SecretKey: secretKey,
SessionToken: sessToken,
}
}

23
src/pkg/credentials/corso.go Normal file
View File

@ -0,0 +1,23 @@
package credentials
import "os"
// envvar consts
const (
CORSO_PASSWORD = "CORSO_PASSWORD"
)
// Corso aggregates corso credentials from flag and env_var values.
type Corso struct {
CorsoPassword string // required
}
// GetCorso is a helper for aggregating Corso secrets and credentials.
func GetCorso() Corso {
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
corsoPasswd := os.Getenv(CORSO_PASSWORD)
return Corso{
CorsoPassword: corsoPasswd,
}
}

21
src/pkg/credentials/m365.go Normal file
View File

@ -0,0 +1,21 @@
package credentials
import "os"
// M365 aggregates m365 credentials from flag and env_var values.
type M365 struct {
ClientID string
ClientSecret string
TenantID string
}
// M365 is a helper for aggregating m365 secrets and credentials.
func GetM365() M365 {
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
return M365{
ClientID: os.Getenv("CLIENT_ID"),
ClientSecret: os.Getenv("CLIENT_SECRET"),
TenantID: os.Getenv("TENANT_ID"),
}
}

15
src/pkg/storage/common.go
View File

@ -1,16 +1,15 @@
package storage
import "github.com/pkg/errors"
import (
"github.com/pkg/errors"
"github.com/alcionai/corso/pkg/credentials"
)
type CommonConfig struct {
CorsoPassword string // required
credentials.Corso // requires: CorsoPassword
}
// envvar consts
const (
CORSO_PASSWORD = "CORSO_PASSWORD"
)
// config key consts
const (
keyCommonCorsoPassword = "common_corsoPassword"
@ -35,7 +34,7 @@ func (s Storage) CommonConfig() (CommonConfig, error) {
// ensures all required properties are present
func (c CommonConfig) validate() error {
if len(c.CorsoPassword) == 0 {
return errors.Wrap(errMissingRequired, CORSO_PASSWORD)
return errors.Wrap(errMissingRequired, credentials.CORSO_PASSWORD)
}
return nil
}

7
src/pkg/storage/common_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/storage"
)
@ -17,7 +18,11 @@ func TestCommonCfgSuite(t *testing.T) {
suite.Run(t, new(CommonCfgSuite))
}
var goodCommonConfig = storage.CommonConfig{"passwd"}
var goodCommonConfig = storage.CommonConfig{
Corso: credentials.Corso{
CorsoPassword: "passwd",
},
}
func (suite *CommonCfgSuite) TestCommonConfig_Config() {
cfg := goodCommonConfig

32
src/pkg/storage/s3.go
View File

@ -1,22 +1,18 @@
package storage
import "github.com/pkg/errors"
import (
"github.com/pkg/errors"
"github.com/alcionai/corso/pkg/credentials"
)
type S3Config struct {
AccessKey string // required
Bucket string // required
Endpoint string
Prefix string
SecretKey string // required
SessionToken string // required
}
credentials.AWS // requires: AccessKey, SecretKey, SessionToken
// envvar consts
const (
AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN"
)
Bucket string // required
Endpoint string
Prefix string
}
// config key consts
const (
@ -56,10 +52,10 @@ func (s Storage) S3Config() (S3Config, error) {
func (c S3Config) validate() error {
check := map[string]string{
AWS_ACCESS_KEY_ID: c.AccessKey,
AWS_SECRET_ACCESS_KEY: c.SecretKey,
AWS_SESSION_TOKEN: c.SessionToken,
"bucket": c.Bucket,
credentials.AWS_ACCESS_KEY_ID: c.AccessKey,
credentials.AWS_SECRET_ACCESS_KEY: c.SecretKey,
credentials.AWS_SESSION_TOKEN: c.SessionToken,
"bucket": c.Bucket,
}
for k, v := range check {
if len(v) == 0 {

33
src/pkg/storage/s3_test.go
View File

@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/storage"
)
@ -17,7 +18,16 @@ func TestS3CfgSuite(t *testing.T) {
suite.Run(t, new(S3CfgSuite))
}
var goodS3Config = storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"}
var goodS3Config = storage.S3Config{
AWS: credentials.AWS{
AccessKey: "ak",
SecretKey: "sk",
SessionToken: "tkn",
},
Bucket: "bkt",
Endpoint: "end",
Prefix: "pre",
}
func (suite *S3CfgSuite) TestS3Config_Config() {
s3 := goodS3Config
@ -57,16 +67,29 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
assert.Equal(t, in.SessionToken, out.SessionToken)
}
func makeTestS3Cfg(ak, bkt, end, pre, sk, tkn string) storage.S3Config {
return storage.S3Config{
AWS: credentials.AWS{
AccessKey: ak,
SecretKey: sk,
SessionToken: tkn,
},
Bucket: bkt,
Endpoint: end,
Prefix: pre,
}
}
func (suite *S3CfgSuite) TestStorage_S3Config_InvalidCases() {
// missing required properties
table := []struct {
name string
cfg storage.S3Config
}{
{"missing access key", storage.S3Config{"", "bkt", "end", "pre", "sk", "tkn"}},
{"missing bucket", storage.S3Config{"ak", "", "end", "pre", "sk", "tkn"}},
{"missing secret key", storage.S3Config{"ak", "bkt", "end", "pre", "", "tkn"}},
{"missing session token", storage.S3Config{"ak", "bkt", "end", "pre", "sk", ""}},
{"missing access key", makeTestS3Cfg("", "bkt", "end", "pre", "sk", "tkn")},
{"missing bucket", makeTestS3Cfg("ak", "", "end", "pre", "sk", "tkn")},
{"missing secret key", makeTestS3Cfg("ak", "bkt", "end", "pre", "", "tkn")},
{"missing session token", makeTestS3Cfg("ak", "bkt", "end", "pre", "sk", "")},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {