add common config and encryption passwd (#74) (#99)

* add common config and encryption passwd

Adds a provider-independent configuration handler, and the
the encryption password config property.  The password is used
to encrypt and decrypt the kopia repository properties file.

* fix corso_password in ci.yml

* actually use the corso password in testing

* replace passwd in ci.yml with a secret

* ci.yml secret typo fix

.github/workflows/ci.yml

@@ -49,4 +49,5 @@ jobs:
       - name: Deployment Tests
         env:
           INTEGRATION_TESTING: true 
+          CORSO_PASSWORD: ${{ secrets.INTEGRATION_TEST_CORSO_PASSWORD }}
         run: go test ./...

src/cli/repo/repo.go

@@ -1,6 +1,7 @@
 package repo
 
 import (
+	"errors"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -85,3 +86,14 @@ func getM365Vars() m365Vars {
 		tenantID:     "todo:tenantID",
 	}
 }
+
+// validates the existence of the properties in the map.
+// expects a map[propName]propVal.
+func requireProps(props map[string]string) error {
+	for name, val := range props {
+		if len(val) == 0 {
+			return errors.New(name + " is required to perform this command")
+		}
+	}
+	return nil
+}

src/cli/repo/repo_test.go

@@ -0,0 +1,36 @@
+package repo
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type CliRepoSuite struct {
+	suite.Suite
+}
+
+func TestCliRepoSuite(t *testing.T) {
+	suite.Run(t, new(CliRepoSuite))
+}
+
+func (suite *CliRepoSuite) TestRequireProps() {
+	table := []struct {
+		name     string
+		props    map[string]string
+		errCheck assert.ErrorAssertionFunc
+	}{
+		{
+			props:    map[string]string{"exists": "I have seen the fnords!"},
+			errCheck: assert.NoError,
+		},
+		{
+			props:    map[string]string{"not-exists": ""},
+			errCheck: assert.Error,
+		},
+	}
+	for _, test := range table {
+		test.errCheck(suite.T(), requireProps(test.props))
+	}
+}

src/cli/repo/s3.go

@@ -49,7 +49,12 @@ var s3InitCmd = &cobra.Command{
 // initializes a s3 repo.
 func initS3Cmd(cmd *cobra.Command, args []string) {
 	mv := getM365Vars()
-	s3Cfg := makeS3Config()
+	s3Cfg, commonCfg, err := makeS3Config()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
 	fmt.Printf(
 		"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
 		cmd.CommandPath(),
@@ -64,7 +69,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) {
 		ClientID:     mv.clientID,
 		ClientSecret: mv.clientSecret,
 	}
-	s := storage.NewStorage(storage.ProviderS3, s3Cfg)
+	s := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
 
 	if _, err := repository.Initialize(cmd.Context(), a, s); err != nil {
 		fmt.Printf("Failed to initialize a new S3 repository: %v", err)
@@ -86,7 +91,12 @@ var s3ConnectCmd = &cobra.Command{
 // connects to an existing s3 repo.
 func connectS3Cmd(cmd *cobra.Command, args []string) {
 	mv := getM365Vars()
-	s3Cfg := makeS3Config()
+	s3Cfg, commonCfg, err := makeS3Config()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
 	fmt.Printf(
 		"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
 		cmd.CommandPath(),
@@ -101,7 +111,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
 		ClientID:     mv.clientID,
 		ClientSecret: mv.clientSecret,
 	}
-	s := storage.NewStorage(storage.ProviderS3, s3Cfg)
+	s := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
 
 	if _, err := repository.Connect(cmd.Context(), a, s); err != nil {
 		fmt.Printf("Failed to connect to the S3 repository: %v", err)
@@ -112,17 +122,31 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
 }
 
 // helper for aggregating aws connection details.
-func makeS3Config() storage.S3Config {
+func makeS3Config() (storage.S3Config, storage.CommonConfig, error) {
 	ak := os.Getenv(storage.AWS_ACCESS_KEY_ID)
 	if len(accessKey) > 0 {
 		ak = accessKey
 	}
+	secretKey := os.Getenv(storage.AWS_SECRET_ACCESS_KEY)
+	sessToken := os.Getenv(storage.AWS_SESSION_TOKEN)
+	corsoPasswd := os.Getenv(storage.CORSO_PASSWORD)
+
 	return storage.S3Config{
-		AccessKey:    ak,
+			AccessKey:    ak,
-		Bucket:       bucket,
+			Bucket:       bucket,
-		Endpoint:     endpoint,
+			Endpoint:     endpoint,
-		Prefix:       prefix,
+			Prefix:       prefix,
-		SecretKey:    os.Getenv(storage.AWS_SECRET_ACCESS_KEY),
+			SecretKey:    secretKey,
-		SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN),
+			SessionToken: sessToken,
-	}
+		},
+		storage.CommonConfig{
+			CorsoPassword: corsoPasswd,
+		},
+		requireProps(map[string]string{
+			storage.AWS_ACCESS_KEY_ID:     ak,
+			"bucket":                      bucket,
+			storage.AWS_SECRET_ACCESS_KEY: secretKey,
+			storage.AWS_SESSION_TOKEN:     sessToken,
+			storage.CORSO_PASSWORD:        corsoPasswd,
+		})
 }

src/go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
+	github.com/zeebo/assert v1.1.0
 )
 
 require (

src/internal/kopia/kopia.go

@@ -12,12 +12,12 @@ import (
 
 const (
 	defaultKopiaConfigFilePath = "/tmp/repository.config"
-	defaultKopiaConfigPasswd   = "todo:passwd"
 )
 
 var (
-	errInit    = errors.New("initializing repo")
+	errInit             = errors.New("initializing repo")
-	errConnect = errors.New("connecting repo")
+	errConnect          = errors.New("connecting repo")
+	errRequriesPassword = errors.New("corso password required")
 )
 
 type kopiaWrapper struct {
@@ -35,8 +35,13 @@ func (kw kopiaWrapper) Initialize(ctx context.Context) error {
 	}
 	defer bst.Close(ctx)
 
+	cfg := kw.storage.CommonConfig()
+	if len(cfg.CorsoPassword) == 0 {
+		return errRequriesPassword
+	}
+
 	// todo - issue #75: nil here should be a storage.NewRepoOptions()
-	if err = repo.Initialize(ctx, bst, nil, defaultKopiaConfigPasswd); err != nil {
+	if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassword); err != nil {
 		return errors.Wrap(err, errInit.Error())
 	}
 
@@ -45,7 +50,7 @@ func (kw kopiaWrapper) Initialize(ctx context.Context) error {
 		ctx,
 		defaultKopiaConfigFilePath,
 		bst,
-		defaultKopiaConfigPasswd,
+		cfg.CorsoPassword,
 		nil,
 	); err != nil {
 		return errors.Wrap(err, errConnect.Error())
@@ -61,12 +66,17 @@ func (kw kopiaWrapper) Connect(ctx context.Context) error {
 	}
 	defer bst.Close(ctx)
 
+	cfg := kw.storage.CommonConfig()
+	if len(cfg.CorsoPassword) == 0 {
+		return errRequriesPassword
+	}
+
 	// todo - issue #75: nil here should be storage.ConnectOptions()
 	if err := repo.Connect(
 		ctx,
 		defaultKopiaConfigFilePath,
 		bst,
-		defaultKopiaConfigPasswd,
+		cfg.CorsoPassword,
 		nil,
 	); err != nil {
 		return errors.Wrap(err, errConnect.Error())

src/pkg/repository/repository_test.go

@@ -122,6 +122,9 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
 					SecretKey:    os.Getenv(storage.AWS_SECRET_ACCESS_KEY),
 					SessionToken: os.Getenv(storage.AWS_SESSION_TOKEN),
 				},
+				storage.CommonConfig{
+					CorsoPassword: os.Getenv(storage.CORSO_PASSWORD),
+				},
 			),
 			errCheck: assert.NoError,
 		},

src/pkg/storage/common.go

@@ -0,0 +1,30 @@
+package storage
+
+type CommonConfig struct {
+	CorsoPassword string
+}
+
+// envvar consts
+const (
+	CORSO_PASSWORD = "CORSO_PASSWORD"
+)
+
+// config key consts
+const (
+	keyCommonCorsoPassword = "common_corsoPassword"
+)
+
+func (c CommonConfig) Config() config {
+	return config{
+		keyCommonCorsoPassword: c.CorsoPassword,
+	}
+}
+
+// CommonConfig retrieves the CommonConfig details from the Storage config.
+func (s Storage) CommonConfig() CommonConfig {
+	c := CommonConfig{}
+	if len(s.Config) > 0 {
+		c.CorsoPassword = orEmptyString(s.Config[keyCommonCorsoPassword])
+	}
+	return c
+}

src/pkg/storage/common_test.go

@@ -0,0 +1,41 @@
+package storage_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/zeebo/assert"
+
+	"github.com/alcionai/corso/pkg/storage"
+)
+
+type CommonCfgSuite struct {
+	suite.Suite
+}
+
+func TestCommonCfgSuite(t *testing.T) {
+	suite.Run(t, new(CommonCfgSuite))
+}
+
+func (suite *CommonCfgSuite) TestCommonConfig_Config() {
+	cfg := storage.CommonConfig{"passwd"}
+	c := cfg.Config()
+	table := []struct {
+		key    string
+		expect string
+	}{
+		{"common_corsoPassword", cfg.CorsoPassword},
+	}
+	for _, test := range table {
+		suite.T().Run(test.key, func(t *testing.T) {
+			assert.Equal(t, c[test.key], test.expect)
+		})
+	}
+}
+
+func (suite *CommonCfgSuite) TestStorage_CommonConfig() {
+	in := storage.CommonConfig{"passwd"}
+	out := storage.NewStorage(storage.ProviderUnknown, in).CommonConfig()
+	t := suite.T()
+	assert.Equal(t, in.CorsoPassword, out.CorsoPassword)
+}

src/pkg/storage/s3.go

@@ -41,12 +41,12 @@ func (c S3Config) Config() config {
 func (s Storage) S3Config() S3Config {
 	c := S3Config{}
 	if len(s.Config) > 0 {
-		c.AccessKey = s.Config[keyS3AccessKey].(string)
+		c.AccessKey = orEmptyString(s.Config[keyS3AccessKey])
-		c.Bucket = s.Config[keyS3Bucket].(string)
+		c.Bucket = orEmptyString(s.Config[keyS3Bucket])
-		c.Endpoint = s.Config[keyS3Endpoint].(string)
+		c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
-		c.Prefix = s.Config[keyS3Prefix].(string)
+		c.Prefix = orEmptyString(s.Config[keyS3Prefix])
-		c.SecretKey = s.Config[keyS3SecretKey].(string)
+		c.SecretKey = orEmptyString(s.Config[keyS3SecretKey])
-		c.SessionToken = s.Config[keyS3SessionToken].(string)
+		c.SessionToken = orEmptyString(s.Config[keyS3SessionToken])
 	}
 	return c
 }

src/pkg/storage/s3_test.go

@@ -3,10 +3,21 @@ package storage_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/suite"
+	"github.com/zeebo/assert"
+
 	"github.com/alcionai/corso/pkg/storage"
 )
 
-func TestS3Config_Config(t *testing.T) {
+type S3CfgSuite struct {
+	suite.Suite
+}
+
+func TestS3CfgSuite(t *testing.T) {
+	suite.Run(t, new(S3CfgSuite))
+}
+
+func (suite *S3CfgSuite) TestS3Config_Config() {
 	s3 := storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"}
 	c := s3.Config()
 	table := []struct {
@@ -21,34 +32,18 @@ func TestS3Config_Config(t *testing.T) {
 		{"s3_sessionToken", s3.SessionToken},
 	}
 	for _, test := range table {
-		key := test.key
+		assert.Equal(suite.T(), c[test.key], test.expect)
-		expect := test.expect
-		if c[key] != expect {
-			t.Errorf("expected config key [%s] to hold value [%s], got [%s]", key, expect, c[key])
-		}
 	}
 }
 
-func TestStorage_S3Config(t *testing.T) {
+func (suite *S3CfgSuite) TestStorage_S3Config() {
 	in := storage.S3Config{"ak", "bkt", "end", "pre", "sk", "tkn"}
-	s := storage.NewStorage(storage.ProviderS3, in)
+	out := storage.NewStorage(storage.ProviderS3, in).S3Config()
-	out := s.S3Config()
+	t := suite.T()
-	if in.Bucket != out.Bucket {
+	assert.Equal(t, in.Bucket, out.Bucket)
-		t.Errorf("expected S3Config.Bucket to be [%s], got [%s]", in.Bucket, out.Bucket)
+	assert.Equal(t, in.AccessKey, out.AccessKey)
-	}
+	assert.Equal(t, in.Endpoint, out.Endpoint)
-	if in.AccessKey != out.AccessKey {
+	assert.Equal(t, in.Prefix, out.Prefix)
-		t.Errorf("expected S3Config.AccessKey to be [%s], got [%s]", in.AccessKey, out.AccessKey)
+	assert.Equal(t, in.SecretKey, out.SecretKey)
-	}
+	assert.Equal(t, in.SessionToken, out.SessionToken)
-	if in.Endpoint != out.Endpoint {
-		t.Errorf("expected S3Config.Endpoint to be [%s], got [%s]", in.Endpoint, out.Endpoint)
-	}
-	if in.Prefix != out.Prefix {
-		t.Errorf("expected S3Config.Prefix to be [%s], got [%s]", in.Prefix, out.Prefix)
-	}
-	if in.SecretKey != out.SecretKey {
-		t.Errorf("expected S3Config.SecretKey to be [%s], got [%s]", in.SecretKey, out.SecretKey)
-	}
-	if in.SessionToken != out.SessionToken {
-		t.Errorf("expected S3Config.SessionToken to be [%s], got [%s]", in.SessionToken, out.SessionToken)
-	}
 }

src/pkg/storage/storage.go

@@ -1,5 +1,7 @@
 package storage
 
+import "fmt"
+
 type storageProvider int
 
 //go:generate stringer -type=storageProvider -linecomment
@@ -39,3 +41,18 @@ func unionConfigs(cfgs ...configurer) config {
 	}
 	return c
 }
+
+// Helper for parsing the values in a config object.
+// If the value is nil or not a string, returns an empty string.
+func orEmptyString(v any) string {
+	defer func() {
+		r := recover()
+		if r != nil {
+			fmt.Printf("panic recovery casting %v to string\n", v)
+		}
+	}()
+	if v == nil {
+		return ""
+	}
+	return v.(string)
+}