larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add common config and encryption passwd (#74) (#99)

Browse Source 
* add common config and encryption passwd

Adds a provider-independent configuration handler, and the
the encryption password config property.  The password is used
to encrypt and decrypt the kopia repository properties file.

* fix corso_password in ci.yml

* actually use the corso password in testing

* replace passwd in ci.yml with a secret

* ci.yml secret typo fix
This commit is contained in:
Keepers 2022-06-01 16:00:41 -06:00 committed by GitHub
parent f368c596b5
commit 535cb9e1f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 221 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/ci.yml vendored
View File

@ -49,4 +49,5 @@ jobs:
- name: Deployment Tests
env:
INTEGRATION_TESTING: true
CORSO_PASSWORD: ${{ secrets.INTEGRATION_TEST_CORSO_PASSWORD }}
run: go test ./...

12
src/cli/repo/repo.go
View File

@ -1,6 +1,7 @@
package repo
import (
"errors"
"os"
"github.com/spf13/cobra"
@ -85,3 +86,14 @@ func getM365Vars() m365Vars {
tenantID: "todo:tenantID",
}
}
// validates the existence of the properties in the map.
// expects a map[propName]propVal.
func requireProps(props map[string]string) error {
for name, val := range props {
if len(val) == 0 {
return errors.New(name + " is required to perform this command")
}
}
return nil
}

36
src/cli/repo/repo_test.go Normal file
View File

@ -0,0 +1,36 @@
package repo
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type CliRepoSuite struct {
suite.Suite
}
func TestCliRepoSuite(t *testing.T) {
suite.Run(t, new(CliRepoSuite))
}
func (suite *CliRepoSuite) TestRequireProps() {
table := []struct {
name string
props map[string]string
errCheck assert.ErrorAssertionFunc
}{
{
props: map[string]string{"exists": "I have seen the fnords!"},
errCheck: assert.NoError,
},
{
props: map[string]string{"not-exists": ""},
errCheck: assert.Error,
},
}
for _, test := range table {
test.errCheck(suite.T(), requireProps(test.props))
}
}

48
src/cli/repo/s3.go
View File

@ -49,7 +49,12 @@ var s3InitCmd = &cobra.Command{
// initializes a s3 repo.
func initS3Cmd(cmd *cobra.Command, args []string) {
mv := getM365Vars()
s3Cfg := makeS3Config()
s3Cfg, commonCfg, err := makeS3Config()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf(
"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
cmd.CommandPath(),
@ -64,7 +69,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) {
ClientID: mv.clientID,
ClientSecret: mv.clientSecret,
}
s := storage.NewStorage(storage.ProviderS3, s3Cfg)
s := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if _, err := repository.Initialize(cmd.Context(), a, s); err != nil {
fmt.Printf("Failed to initialize a new S3 repository: %v", err)
@ -86,7 +91,12 @@ var s3ConnectCmd = &cobra.Command{
// connects to an existing s3 repo.
func connectS3Cmd(cmd *cobra.Command, args []string) {
mv := getM365Vars()
s3Cfg := makeS3Config()
s3Cfg, commonCfg, err := makeS3Config()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf(
"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
cmd.CommandPath(),
@ -101,7 +111,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
ClientID: mv.clientID,
ClientSecret: mv.clientSecret,
}
s := storage.NewStorage(storage.ProviderS3, s3Cfg)
s := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if _, err := repository.Connect(cmd.Context(), a, s); err != nil {
fmt.Printf("Failed to connect to the S3 repository: %v", err)
@ -112,17 +122,31 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
}
// helper for aggregating aws connection details.
func makeS3Config() storage.S3Config {
func makeS3Config() (storage.S3Config, storage.CommonConfig, error) {
ak := os.Getenv(storage.AWS_ACCESS_KEY_ID)
if len(accessKey) > 0 {
ak = accessKey
}
secretKey := os.Getenv(storage.AWS_SECRET_ACCESS_KEY)
sessToken := os.Getenv(storage.AWS_SESSION_TOKEN)
corsoPasswd := os.Getenv(storage.CORSO_PASSWORD)
return storage.S3Config{
AccessKey: ak,
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
SecretKey: os.Getenv(storage.AWS_SECRET_ACCESS_KEY),
SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN),
}
AccessKey: ak,
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
SecretKey: secretKey,
SessionToken: sessToken,
},
storage.CommonConfig{
CorsoPassword: corsoPasswd,
},
requireProps(map[string]string{
storage.AWS_ACCESS_KEY_ID: ak,
"bucket": bucket,
storage.AWS_SECRET_ACCESS_KEY: secretKey,
storage.AWS_SESSION_TOKEN: sessToken,
storage.CORSO_PASSWORD: corsoPasswd,
})
}

1
src/go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1
github.com/zeebo/assert v1.1.0
)
require (

22
src/internal/kopia/kopia.go
View File

@ -12,12 +12,12 @@ import (
const (
defaultKopiaConfigFilePath = "/tmp/repository.config"
defaultKopiaConfigPasswd = "todo:passwd"
)
var (
errInit = errors.New("initializing repo")
errConnect = errors.New("connecting repo")
errInit = errors.New("initializing repo")
errConnect = errors.New("connecting repo")
errRequriesPassword = errors.New("corso password required")
)
type kopiaWrapper struct {
@ -35,8 +35,13 @@ func (kw kopiaWrapper) Initialize(ctx context.Context) error {
}
defer bst.Close(ctx)
cfg := kw.storage.CommonConfig()
if len(cfg.CorsoPassword) == 0 {
return errRequriesPassword
}
// todo - issue #75: nil here should be a storage.NewRepoOptions()
if err = repo.Initialize(ctx, bst, nil, defaultKopiaConfigPasswd); err != nil {
if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassword); err != nil {
return errors.Wrap(err, errInit.Error())
}
@ -45,7 +50,7 @@ func (kw kopiaWrapper) Initialize(ctx context.Context) error {
ctx,
defaultKopiaConfigFilePath,
bst,
defaultKopiaConfigPasswd,
cfg.CorsoPassword,
nil,
); err != nil {
return errors.Wrap(err, errConnect.Error())
@ -61,12 +66,17 @@ func (kw kopiaWrapper) Connect(ctx context.Context) error {
}
defer bst.Close(ctx)
cfg := kw.storage.CommonConfig()
if len(cfg.CorsoPassword) == 0 {
return errRequriesPassword
}
// todo - issue #75: nil here should be storage.ConnectOptions()
if err := repo.Connect(
ctx,
defaultKopiaConfigFilePath,
bst,
defaultKopiaConfigPasswd,
cfg.CorsoPassword,
nil,
); err != nil {
return errors.Wrap(err, errConnect.Error())

3
src/pkg/repository/repository_test.go
View File

@ -122,6 +122,9 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
SecretKey: os.Getenv(storage.AWS_SECRET_ACCESS_KEY),
SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN),
},
storage.CommonConfig{
CorsoPassword: os.Getenv(storage.CORSO_PASSWORD),
},
),
errCheck: assert.NoError,
},

30
src/pkg/storage/common.go Normal file
View File

@ -0,0 +1,30 @@
package storage
type CommonConfig struct {
CorsoPassword string
}
// envvar consts
const (
CORSO_PASSWORD = "CORSO_PASSWORD"
)
// config key consts
const (
keyCommonCorsoPassword = "common_corsoPassword"
)
func (c CommonConfig) Config() config {
return config{
keyCommonCorsoPassword: c.CorsoPassword,
}
}
// CommonConfig retrieves the CommonConfig details from the Storage config.
func (s Storage) CommonConfig() CommonConfig {
c := CommonConfig{}
if len(s.Config) > 0 {
c.CorsoPassword = orEmptyString(s.Config[keyCommonCorsoPassword])
}
return c
}

41
src/pkg/storage/common_test.go Normal file
View File

@ -0,0 +1,41 @@
package storage_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/zeebo/assert"
"github.com/alcionai/corso/pkg/storage"
)
type CommonCfgSuite struct {
suite.Suite
}
func TestCommonCfgSuite(t *testing.T) {
suite.Run(t, new(CommonCfgSuite))
}
func (suite *CommonCfgSuite) TestCommonConfig_Config() {
cfg := storage.CommonConfig{"passwd"}
c := cfg.Config()
table := []struct {
key string
expect string
}{
{"common_corsoPassword", cfg.CorsoPassword},
}
for _, test := range table {
suite.T().Run(test.key, func(t *testing.T) {
assert.Equal(t, c[test.key], test.expect)
})
}
}
func (suite *CommonCfgSuite) TestStorage_CommonConfig() {
in := storage.CommonConfig{"passwd"}
out := storage.NewStorage(storage.ProviderUnknown, in).CommonConfig()
t := suite.T()
assert.Equal(t, in.CorsoPassword, out.CorsoPassword)
}

12
src/pkg/storage/s3.go
View File

@ -41,12 +41,12 @@ func (c S3Config) Config() config {
func (s Storage) S3Config() S3Config {
c := S3Config{}
if len(s.Config) > 0 {
c.AccessKey = s.Config[keyS3AccessKey].(string)
c.Bucket = s.Config[keyS3Bucket].(string)
c.Endpoint = s.Config[keyS3Endpoint].(string)
c.Prefix = s.Config[keyS3Prefix].(string)
c.SecretKey = s.Config[keyS3SecretKey].(string)
c.SessionToken = s.Config[keyS3SessionToken].(string)
c.AccessKey = orEmptyString(s.Config[keyS3AccessKey])
c.Bucket = orEmptyString(s.Config[keyS3Bucket])
c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
c.Prefix = orEmptyString(s.Config[keyS3Prefix])
c.SecretKey = orEmptyString(s.Config[keyS3SecretKey])
c.SessionToken = orEmptyString(s.Config[keyS3SessionToken])
}
return c
}

49
src/pkg/storage/s3_test.go
View File

@ -3,10 +3,21 @@ package storage_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/zeebo/assert"
"github.com/alcionai/corso/pkg/storage"
)
func TestS3Config_Config(t *testing.T) {
type S3CfgSuite struct {
suite.Suite
}
func TestS3CfgSuite(t *testing.T) {
suite.Run(t, new(S3CfgSuite))
}
func (suite *S3CfgSuite) TestS3Config_Config() {
s3 := storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"}
c := s3.Config()
table := []struct {
@ -21,34 +32,18 @@ func TestS3Config_Config(t *testing.T) {
{"s3_sessionToken", s3.SessionToken},
}
for _, test := range table {
key := test.key
expect := test.expect
if c[key] != expect {
t.Errorf("expected config key [%s] to hold value [%s], got [%s]", key, expect, c[key])
}
assert.Equal(suite.T(), c[test.key], test.expect)
}
}
func TestStorage_S3Config(t *testing.T) {
func (suite *S3CfgSuite) TestStorage_S3Config() {
in := storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"}
s := storage.NewStorage(storage.ProviderS3, in)
out := s.S3Config()
if in.Bucket != out.Bucket {
t.Errorf("expected S3Config.Bucket to be [%s], got [%s]", in.Bucket, out.Bucket)
}
if in.AccessKey != out.AccessKey {
t.Errorf("expected S3Config.AccessKey to be [%s], got [%s]", in.AccessKey, out.AccessKey)
}
if in.Endpoint != out.Endpoint {
t.Errorf("expected S3Config.Endpoint to be [%s], got [%s]", in.Endpoint, out.Endpoint)
}
if in.Prefix != out.Prefix {
t.Errorf("expected S3Config.Prefix to be [%s], got [%s]", in.Prefix, out.Prefix)
}
if in.SecretKey != out.SecretKey {
t.Errorf("expected S3Config.SecretKey to be [%s], got [%s]", in.SecretKey, out.SecretKey)
}
if in.SessionToken != out.SessionToken {
t.Errorf("expected S3Config.SessionToken to be [%s], got [%s]", in.SessionToken, out.SessionToken)
}
out := storage.NewStorage(storage.ProviderS3, in).S3Config()
t := suite.T()
assert.Equal(t, in.Bucket, out.Bucket)
assert.Equal(t, in.AccessKey, out.AccessKey)
assert.Equal(t, in.Endpoint, out.Endpoint)
assert.Equal(t, in.Prefix, out.Prefix)
assert.Equal(t, in.SecretKey, out.SecretKey)
assert.Equal(t, in.SessionToken, out.SessionToken)
}

17
src/pkg/storage/storage.go
View File

@ -1,5 +1,7 @@
package storage
import "fmt"
type storageProvider int
//go:generate stringer -type=storageProvider -linecomment
@ -39,3 +41,18 @@ func unionConfigs(cfgs ...configurer) config {
}
return c
}
// Helper for parsing the values in a config object.
// If the value is nil or not a string, returns an empty string.
func orEmptyString(v any) string {
defer func() {
r := recover()
if r != nil {
fmt.Printf("panic recovery casting %v to string\n", v)
}
}()
if v == nil {
return ""
}
return v.(string)
}