use config file for backup create (#171)

* use config file for backup create

Now that corso stores its prior connection details in a config
file, that file should get used to retrieve storage details as needed.

src/cli/backup/exchange.go

@@ -6,10 +6,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
+	"github.com/alcionai/corso/cli/config"
 	"github.com/alcionai/corso/cli/utils"
 	"github.com/alcionai/corso/pkg/credentials"
 	"github.com/alcionai/corso/pkg/repository"
-	"github.com/alcionai/corso/pkg/storage"
 )
 
 // exchange bucket info from flags
@@ -40,7 +40,21 @@ var exchangeCreateCmd = &cobra.Command{
 
 // initializes a s3 repo.
 func createExchangeCmd(cmd *cobra.Command, args []string) error {
+	s, cfgTenantID, err := config.MakeS3Config(true, nil)
+	if err != nil {
+		return err
+	}
+
 	m365 := credentials.GetM365()
+	a := repository.Account{
+		TenantID:     m365.TenantID,
+		ClientID:     m365.ClientID,
+		ClientSecret: m365.ClientSecret,
+	}
+	if len(cfgTenantID) > 0 {
+		a.TenantID = cfgTenantID
+	}
+
 	fmt.Printf(
 		"Called - %s\n\t365TenantID:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n",
 		cmd.CommandPath(),
@@ -48,17 +62,6 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
 		m365.ClientID,
 		len(m365.ClientSecret) > 0)
 
-	a := repository.Account{
-		TenantID:     m365.TenantID,
-		ClientID:     m365.ClientID,
-		ClientSecret: m365.ClientSecret,
-	}
-	// todo (rkeepers) - retrieve storage details from corso config
-	s, err := storage.NewStorage(storage.ProviderUnknown)
-	if err != nil {
-		return errors.Wrap(err, "Failed to configure storage provider")
-	}
-
 	r, err := repository.Connect(cmd.Context(), a, s)
 	if err != nil {
 		return errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider)

src/cli/config/config.go

@@ -5,6 +5,8 @@ import (
 	"path"
 	"strings"
 
+	"github.com/alcionai/corso/cli/utils"
+	"github.com/alcionai/corso/pkg/credentials"
 	"github.com/alcionai/corso/pkg/repository"
 	"github.com/alcionai/corso/pkg/storage"
 	"github.com/pkg/errors"
@@ -92,3 +94,70 @@ func ReadRepoConfig() (s3Config storage.S3Config, account repository.Account, er
 
 	return s3Config, account, nil
 }
+
+// MakeS3Config creates a storage instance by mediating all the possible
+// data sources (config file, env vars, flag overrides) in the config.
+func MakeS3Config(readFromFile bool, overrides map[string]string) (storage.Storage, string, error) {
+	var (
+		s3Cfg   storage.S3Config
+		account repository.Account
+		err     error
+	)
+
+	// possibly read the prior config from a .corso file
+	if readFromFile {
+		s3Cfg, account, err = ReadRepoConfig()
+		if err != nil {
+			return storage.Storage{}, "", errors.Wrap(err, "reading corso config file")
+		}
+	}
+
+	// compose the s3 storage config and credentials
+	aws := credentials.GetAWS(overrides)
+	if err := aws.Validate(); err != nil {
+		return storage.Storage{}, "", errors.Wrap(err, "validating aws credentials")
+	}
+	s3Cfg = storage.S3Config{
+		AWS:      aws,
+		Bucket:   first(overrides[storage.Bucket], s3Cfg.Bucket),
+		Endpoint: first(overrides[storage.Endpoint], s3Cfg.Endpoint),
+		Prefix:   first(overrides[storage.Prefix], s3Cfg.Prefix),
+	}
+
+	// compose the common config and credentials
+	corso := credentials.GetCorso()
+	if err := corso.Validate(); err != nil {
+		return storage.Storage{}, "", errors.Wrap(err, "validating corso credentials")
+	}
+	cCfg := storage.CommonConfig{
+		Corso: corso,
+	}
+
+	// ensure requried properties are present
+	if err := utils.RequireProps(map[string]string{
+		credentials.AWSAccessKeyID:     aws.AccessKey,
+		storage.Bucket:                 s3Cfg.Bucket,
+		credentials.AWSSecretAccessKey: aws.SecretKey,
+		credentials.AWSSessionToken:    aws.SessionToken,
+		credentials.CorsoPassword:      corso.CorsoPassword,
+	}); err != nil {
+		return storage.Storage{}, "", err
+	}
+
+	// return a complete storage
+	s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg)
+	if err != nil {
+		return storage.Storage{}, "", errors.Wrap(err, "configuring repository storage")
+	}
+	return s, account.TenantID, nil
+}
+
+// returns the first non-zero valued string
+func first(vs ...string) string {
+	for _, v := range vs {
+		if len(v) > 0 {
+			return v
+		}
+	}
+	return ""
+}

src/cli/config/config_test.go

@@ -6,12 +6,13 @@ import (
 	"path"
 	"testing"
 
-	"github.com/alcionai/corso/cli/config"
-	"github.com/alcionai/corso/pkg/repository"
-	"github.com/alcionai/corso/pkg/storage"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/cli/config"
+	"github.com/alcionai/corso/pkg/repository"
+	"github.com/alcionai/corso/pkg/storage"
 )
 
 const (
@@ -34,7 +35,7 @@ func TestConfigSuite(t *testing.T) {
 
 func (suite *ConfigSuite) TestReadRepoConfigBasic() {
 	// Generate test config file
-	b := "test-bucket"
+	b := "read-repo-config-basic-bucket"
 	tID := "6f34ac30-8196-469b-bf8f-d83deadbbbba"
 	testConfigData := fmt.Sprintf(configFileTemplate, b, tID)
 	testConfigFilePath := path.Join(suite.T().TempDir(), "corso.toml")
@@ -58,7 +59,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	err := config.InitConfig(testConfigFilePath)
 	assert.NoError(suite.T(), err)
 
-	s3Cfg := storage.S3Config{Bucket: "bucket"}
+	s3Cfg := storage.S3Config{Bucket: "write-read-config-bucket"}
 	account := repository.Account{TenantID: "6f34ac30-8196-469b-bf8f-d83deadbbbbd"}
 	err = config.WriteRepoConfig(s3Cfg, account)
 	assert.NoError(suite.T(), err)

src/cli/repo/s3.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 
 	"github.com/alcionai/corso/cli/config"
 	"github.com/alcionai/corso/cli/utils"
@@ -55,11 +54,30 @@ var s3InitCmd = &cobra.Command{
 func initS3Cmd(cmd *cobra.Command, args []string) error {
 	log := logger.Ctx(cmd.Context())
 
-	m365 := credentials.GetM365()
+	overrides := map[string]string{
-	s3Cfg, commonCfg, err := makeS3Config()
+		credentials.AWSAccessKeyID: accessKey,
+		storage.Bucket:             bucket,
+		storage.Endpoint:           endpoint,
+		storage.Prefix:             prefix,
+	}
+	s, cfgTenantID, err := config.MakeS3Config(false, overrides)
 	if err != nil {
 		return err
 	}
+	s3Cfg, err := s.S3Config()
+	if err != nil {
+		return errors.Wrap(err, "Retrieving s3 configuration")
+	}
+
+	m365 := credentials.GetM365()
+	a := repository.Account{
+		TenantID:     m365.TenantID,
+		ClientID:     m365.ClientID,
+		ClientSecret: m365.ClientSecret,
+	}
+	if len(cfgTenantID) > 0 {
+		a.TenantID = cfgTenantID
+	}
 
 	log.Debugw(
 		"Called - "+cmd.CommandPath(),
@@ -69,16 +87,6 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
 		"accessKey", s3Cfg.AccessKey,
 		"hasSecretKey", len(s3Cfg.SecretKey) > 0)
 
-	a := repository.Account{
-		TenantID:     m365.TenantID,
-		ClientID:     m365.ClientID,
-		ClientSecret: m365.ClientSecret,
-	}
-	s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
-	if err != nil {
-		return errors.Wrap(err, "Failed to configure storage provider")
-	}
-
 	r, err := repository.Initialize(cmd.Context(), a, s)
 	if err != nil {
 		return errors.Wrap(err, "Failed to initialize a new S3 repository")
@@ -106,11 +114,30 @@ var s3ConnectCmd = &cobra.Command{
 func connectS3Cmd(cmd *cobra.Command, args []string) error {
 	log := logger.Ctx(cmd.Context())
 
-	m365 := credentials.GetM365()
+	overrides := map[string]string{
-	s3Cfg, commonCfg, err := makeS3Config()
+		credentials.AWSAccessKeyID: accessKey,
+		storage.Bucket:             bucket,
+		storage.Endpoint:           endpoint,
+		storage.Prefix:             prefix,
+	}
+	s, cfgTenantID, err := config.MakeS3Config(true, overrides)
 	if err != nil {
 		return err
 	}
+	s3Cfg, err := s.S3Config()
+	if err != nil {
+		return errors.Wrap(err, "Retrieving s3 configuration")
+	}
+
+	m365 := credentials.GetM365()
+	a := repository.Account{
+		TenantID:     m365.TenantID,
+		ClientID:     m365.ClientID,
+		ClientSecret: m365.ClientSecret,
+	}
+	if len(cfgTenantID) > 0 {
+		a.TenantID = cfgTenantID
+	}
 
 	log.Debugw(
 		"Called - "+cmd.CommandPath(),
@@ -120,23 +147,6 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 		"accessKey", s3Cfg.AccessKey,
 		"hasSecretKey", len(s3Cfg.SecretKey) > 0)
 
-	// TODO: Merge/Validate any local configuration here to make sure there are no conflicts
-	// For now - just reading/logging the local config here (a successful repo connect will overwrite)
-	localS3Cfg, localAccount, err := config.ReadRepoConfig()
-	if err == nil {
-		fmt.Printf("ConfigFile - %s\n\tbucket:\t%s\n\ttenantID:\t%s\n", viper.ConfigFileUsed(), localS3Cfg.Bucket, localAccount.TenantID)
-	}
-
-	a := repository.Account{
-		TenantID:     m365.TenantID,
-		ClientID:     m365.ClientID,
-		ClientSecret: m365.ClientSecret,
-	}
-	s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
-	if err != nil {
-		return errors.Wrap(err, "Failed to configure storage provider")
-	}
-
 	r, err := repository.Connect(cmd.Context(), a, s)
 	if err != nil {
 		return errors.Wrap(err, "Failed to connect to the S3 repository")
@@ -150,25 +160,3 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 	}
 	return nil
 }
-
-// helper for aggregating aws connection details.
-func makeS3Config() (storage.S3Config, storage.CommonConfig, error) {
-	aws := credentials.GetAWS(map[string]string{credentials.AWSAccessKeyID: accessKey})
-	corso := credentials.GetCorso()
-	return storage.S3Config{
-			AWS:      aws,
-			Bucket:   bucket,
-			Endpoint: endpoint,
-			Prefix:   prefix,
-		},
-		storage.CommonConfig{
-			Corso: corso,
-		},
-		utils.RequireProps(map[string]string{
-			credentials.AWSAccessKeyID:     aws.AccessKey,
-			"bucket":                       bucket,
-			credentials.AWSSecretAccessKey: aws.SecretKey,
-			credentials.AWSSessionToken:    aws.SessionToken,
-			credentials.CorsoPassword:      corso.CorsoPassword,
-		})
-}

src/internal/testing/integration_runners.go

@@ -12,8 +12,8 @@ import (
 const (
 	CorsoCITests             = "CORSO_CI_TESTS"
 	CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
-	CorsoRepositoryTests     = "CORSO_REPOSITORY_TESTS"
 	CorsoKopiaWrapperTests   = "CORSO_KOPIA_WRAPPER_TESTS"
+	CorsoRepositoryTests     = "CORSO_REPOSITORY_TESTS"
 )
 
 // RunOnAny takes in a list of env variable names and returns

src/pkg/credentials/aws.go

@@ -2,6 +2,8 @@ package credentials
 
 import (
 	"os"
+
+	"github.com/pkg/errors"
 )
 
 // envvar consts
@@ -35,3 +37,17 @@ func GetAWS(override map[string]string) AWS {
 		SessionToken: sessToken,
 	}
 }
+
+func (c AWS) Validate() error {
+	check := map[string]string{
+		AWSAccessKeyID:     c.AccessKey,
+		AWSSecretAccessKey: c.SecretKey,
+		AWSSessionToken:    c.SessionToken,
+	}
+	for k, v := range check {
+		if len(v) == 0 {
+			return errors.Wrap(errMissingRequired, k)
+		}
+	}
+	return nil
+}

src/pkg/credentials/corso.go

@@ -1,6 +1,10 @@
 package credentials
 
-import "os"
+import (
+	"os"
+
+	"github.com/pkg/errors"
+)
 
 // envvar consts
 const (
@@ -21,3 +25,15 @@ func GetCorso() Corso {
 		CorsoPassword: corsoPasswd,
 	}
 }
+
+func (c Corso) Validate() error {
+	check := map[string]string{
+		CorsoPassword: c.CorsoPassword,
+	}
+	for k, v := range check {
+		if len(v) == 0 {
+			return errors.Wrap(errMissingRequired, k)
+		}
+	}
+	return nil
+}

src/pkg/storage/s3.go

@@ -24,6 +24,13 @@ const (
 	keyS3SessionToken = "s3_sessionToken"
 )
 
+// config exported name consts
+const (
+	Bucket   = "bucket"
+	Endpoint = "endpoint"
+	Prefix   = "prefix"
+)
+
 func (c S3Config) Config() (config, error) {
 	cfg := config{
 		keyS3AccessKey:    c.AccessKey,
@@ -55,7 +62,7 @@ func (c S3Config) validate() error {
 		credentials.AWSAccessKeyID:     c.AccessKey,
 		credentials.AWSSecretAccessKey: c.SecretKey,
 		credentials.AWSSessionToken:    c.SessionToken,
-		"bucket":                       c.Bucket,
+		Bucket:                         c.Bucket,
 	}
 	for k, v := range check {
 		if len(v) == 0 {