Move config code around, introduce interface for storageconfiguration

src/cli/config/config.go

@@ -203,14 +203,14 @@ func Read(ctx context.Context) error {
 // It does not check for conflicts or existing data.
 func WriteRepoConfig(
 	ctx context.Context,
-	s3Config storage.S3Config,
+	storageConfig StorageConfigurer,
 	m365Config account.M365Config,
 	repoOpts repository.Options,
 	repoID string,
 ) error {
 	return writeRepoConfigWithViper(
 		GetViper(ctx),
-		s3Config,
+		storageConfig,
 		m365Config,
 		repoOpts,
 		repoID)
@@ -220,20 +220,16 @@ func WriteRepoConfig(
 // struct for testing.
 func writeRepoConfigWithViper(
 	vpr *viper.Viper,
-	s3Config storage.S3Config,
+	storageConfig StorageConfigurer,
 	m365Config account.M365Config,
 	repoOpts repository.Options,
 	repoID string,
 ) error {
-	s3Config = s3Config.Normalize()
+	storageConfig.WriteConfigToViper(vpr)
+
 	// Rudimentary support for persisting repo config
 	// TODO: Handle conflicts, support other config types
-	vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String())
+
-	vpr.Set(BucketNameKey, s3Config.Bucket)
-	vpr.Set(EndpointKey, s3Config.Endpoint)
-	vpr.Set(PrefixKey, s3Config.Prefix)
-	vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS)
-	vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
 	vpr.Set(RepoID, repoID)
 
 	// Need if-checks as Viper will write empty values otherwise.
@@ -335,12 +331,13 @@ func getUserHost(vpr *viper.Viper, readConfigFromViper bool) (string, string) {
 // Helper funcs
 // ---------------------------------------------------------------------------
 
+// TODO: This is not really needed?
 var constToTomlKeyMap = map[string]string{
 	account.AzureTenantID:  AzureTenantIDKey,
 	AccountProviderTypeKey: AccountProviderTypeKey,
-	storage.Bucket:         BucketNameKey,
+	Bucket:                 BucketNameKey,
-	storage.Endpoint:       EndpointKey,
+	Endpoint:               EndpointKey,
-	storage.Prefix:         PrefixKey,
+	Prefix:                 PrefixKey,
 	StorageProviderTypeKey: StorageProviderTypeKey,
 }
 

src/cli/config/s3config.go

@@ -0,0 +1,263 @@
+package config
+
+import (
+	"strconv"
+
+	"github.com/alcionai/clues"
+	"github.com/aws/aws-sdk-go/aws/defaults"
+	"github.com/spf13/viper"
+
+	"github.com/alcionai/corso/src/internal/common"
+	"github.com/alcionai/corso/src/internal/common/str"
+	"github.com/alcionai/corso/src/pkg/credentials"
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
+func s3ConfigsFromViper(vpr *viper.Viper) (S3Config, error) {
+	var s3Config S3Config
+
+	s3Config.Bucket = vpr.GetString(BucketNameKey)
+	s3Config.Endpoint = vpr.GetString(EndpointKey)
+	s3Config.Prefix = vpr.GetString(PrefixKey)
+	s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
+	s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
+
+	return s3Config, nil
+}
+
+// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
+func s3CredsFromViper(vpr *viper.Viper, s3Config S3Config) (S3Config, error) {
+	s3Config.AccessKey = vpr.GetString(AccessKey)
+	s3Config.SecretKey = vpr.GetString(SecretAccessKey)
+	s3Config.SessionToken = vpr.GetString(SessionToken)
+
+	return s3Config, nil
+}
+
+func s3Overrides(in map[string]string) map[string]string {
+	return map[string]string{
+		Bucket:                 in[Bucket],
+		Endpoint:               in[Endpoint],
+		Prefix:                 in[Prefix],
+		DoNotUseTLS:            in[DoNotUseTLS],
+		DoNotVerifyTLS:         in[DoNotVerifyTLS],
+		StorageProviderTypeKey: in[StorageProviderTypeKey],
+	}
+}
+
+type StorageConfigurer interface {
+	common.StringConfigurer
+	//Normalize() StorageConfigurer
+	WriteConfigToViper(vpr *viper.Viper)
+}
+
+// func NewStorageConfig(
+// 	provider storage.StorageProvider,
+// ) (StorageConfigurer, error) {
+// 	switch provider {
+// 	case storage.ProviderS3:
+// 		return S3Config{}, nil
+// 	default:
+// 		return nil, clues.New("unsupported storage type")
+// 	}
+// }
+
+// Hydrate from a config map
+func NewStorageConfigFrom(s storage.Storage) (StorageConfigurer, error) {
+	switch s.Provider {
+	case storage.ProviderS3:
+		return makeS3Config(s.Config)
+	default:
+		return nil, clues.New("unsupported storage type")
+	}
+}
+
+type S3Config struct {
+	credentials.AWS
+	Bucket         string // required
+	Endpoint       string
+	Prefix         string
+	DoNotUseTLS    bool
+	DoNotVerifyTLS bool
+}
+
+// MakeS3Config retrieves the S3Config details from the Storage config.
+func makeS3Config(config map[string]string) (StorageConfigurer, error) {
+	c := S3Config{}
+
+	if len(config) > 0 {
+		c.AccessKey = orEmptyString(config[keyS3AccessKey])
+		c.SecretKey = orEmptyString(config[keyS3SecretKey])
+		c.SessionToken = orEmptyString(config[keyS3SessionToken])
+
+		c.Bucket = orEmptyString(config[keyS3Bucket])
+		c.Endpoint = orEmptyString(config[keyS3Endpoint])
+		c.Prefix = orEmptyString(config[keyS3Prefix])
+		c.DoNotUseTLS = str.ParseBool(config[keyS3DoNotUseTLS])
+		c.DoNotVerifyTLS = str.ParseBool(config[keyS3DoNotVerifyTLS])
+	}
+
+	return c, c.validate()
+}
+
+// TODO: Remove storageconfigurer return value.
+// Have it contained in current s3config.
+func fetchS3ConfigFromViper(
+	vpr *viper.Viper,
+	readConfigFromViper bool,
+	matchFromConfig bool,
+	overrides map[string]string,
+) (StorageConfigurer, error) {
+	var (
+		s3Cfg S3Config
+		err   error
+	)
+
+	if readConfigFromViper {
+		if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil {
+			clues.Wrap(err, "reading s3 configs from corso config file")
+		}
+
+		if b, ok := overrides[Bucket]; ok {
+			overrides[Bucket] = common.NormalizeBucket(b)
+		}
+
+		if p, ok := overrides[Prefix]; ok {
+			overrides[Prefix] = common.NormalizePrefix(p)
+		}
+
+		if matchFromConfig {
+			providerType := vpr.GetString(StorageProviderTypeKey)
+			if providerType != storage.ProviderS3.String() {
+				return nil, clues.New("unsupported storage provider" + providerType)
+			}
+
+			if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
+				return nil, clues.Wrap(err, "verifying s3 configs in corso config file")
+			}
+		}
+	}
+
+	if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
+		return nil, clues.Wrap(err, "reading s3 configs from corso config file")
+	}
+
+	s3Overrides(overrides)
+	aws := credentials.GetAWS(overrides)
+
+	if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
+		_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
+		if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
+			aws = credentials.AWS{
+				AccessKey:    s3Cfg.AccessKey,
+				SecretKey:    s3Cfg.SecretKey,
+				SessionToken: s3Cfg.SessionToken,
+			}
+			err = nil
+		}
+
+		if err != nil {
+			return nil, clues.Wrap(err, "validating aws credentials")
+		}
+	}
+
+	s3Cfg = S3Config{
+		AWS:      aws,
+		Bucket:   str.First(overrides[Bucket], s3Cfg.Bucket),
+		Endpoint: str.First(overrides[Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"),
+		Prefix:   str.First(overrides[Prefix], s3Cfg.Prefix),
+		DoNotUseTLS: str.ParseBool(str.First(
+			overrides[DoNotUseTLS],
+			strconv.FormatBool(s3Cfg.DoNotUseTLS),
+			"false",
+		)),
+		DoNotVerifyTLS: str.ParseBool(str.First(
+			overrides[DoNotVerifyTLS],
+			strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
+			"false",
+		)),
+	}
+
+	return s3Cfg, s3Cfg.validate()
+}
+
+// config key consts
+const (
+	keyS3AccessKey      = "s3_access_key"
+	keyS3Bucket         = "s3_bucket"
+	keyS3Endpoint       = "s3_endpoint"
+	keyS3Prefix         = "s3_prefix"
+	keyS3SecretKey      = "s3_secret_key"
+	keyS3SessionToken   = "s3_session_token"
+	keyS3DoNotUseTLS    = "s3_donotusetls"
+	keyS3DoNotVerifyTLS = "s3_donotverifytls"
+)
+
+// config exported name consts
+// TODO: Move these to storage?
+const (
+	Bucket         = "bucket"
+	Endpoint       = "endpoint"
+	Prefix         = "prefix"
+	DoNotUseTLS    = "donotusetls"
+	DoNotVerifyTLS = "donotverifytls"
+)
+
+func normalize(c S3Config) S3Config {
+	return S3Config{
+		Bucket:         common.NormalizeBucket(c.Bucket),
+		Endpoint:       c.Endpoint,
+		Prefix:         common.NormalizePrefix(c.Prefix),
+		DoNotUseTLS:    c.DoNotUseTLS,
+		DoNotVerifyTLS: c.DoNotVerifyTLS,
+	}
+}
+
+// StringConfig transforms a s3Config struct into a plain
+// map[string]string.  All values in the original struct which
+// serialize into the map are expected to be strings.
+func (c S3Config) StringConfig() (map[string]string, error) {
+	cn := normalize(c)
+	cfg := map[string]string{
+		keyS3AccessKey:      c.AccessKey,
+		keyS3Bucket:         cn.Bucket,
+		keyS3Endpoint:       cn.Endpoint,
+		keyS3Prefix:         cn.Prefix,
+		keyS3SecretKey:      c.SecretKey,
+		keyS3SessionToken:   c.SessionToken,
+		keyS3DoNotUseTLS:    strconv.FormatBool(cn.DoNotUseTLS),
+		keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
+	}
+
+	return cfg, c.validate()
+}
+
+func (c S3Config) WriteConfigToViper(vpr *viper.Viper) {
+	cn := normalize(c)
+
+	vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String())
+	vpr.Set(BucketNameKey, cn.Bucket)
+	vpr.Set(EndpointKey, cn.Endpoint)
+	vpr.Set(PrefixKey, cn.Prefix)
+	vpr.Set(DisableTLSKey, cn.DoNotUseTLS)
+	vpr.Set(DisableTLSVerificationKey, cn.DoNotVerifyTLS)
+}
+
+// storage parsing errors
+var (
+	errMissingRequired = clues.New("missing required storage configuration")
+)
+
+func (c S3Config) validate() error {
+	check := map[string]string{
+		Bucket: c.Bucket,
+	}
+	for k, v := range check {
+		if len(v) == 0 {
+			return clues.Stack(errMissingRequired, clues.New(k))
+		}
+	}
+
+	return nil
+}

src/cli/config/storage.go

@@ -1,54 +1,19 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
-	"strconv"
 
 	"github.com/alcionai/clues"
-	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/spf13/viper"
 
 	"github.com/alcionai/corso/src/cli/flags"
-	"github.com/alcionai/corso/src/internal/common"
 	"github.com/alcionai/corso/src/internal/common/str"
 	"github.com/alcionai/corso/src/pkg/credentials"
 	"github.com/alcionai/corso/src/pkg/storage"
 )
 
-// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
-func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
-	var s3Config storage.S3Config
-
-	s3Config.Bucket = vpr.GetString(BucketNameKey)
-	s3Config.Endpoint = vpr.GetString(EndpointKey)
-	s3Config.Prefix = vpr.GetString(PrefixKey)
-	s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
-	s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
-
-	return s3Config, nil
-}
-
-// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
-func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Config, error) {
-	s3Config.AccessKey = vpr.GetString(AccessKey)
-	s3Config.SecretKey = vpr.GetString(SecretAccessKey)
-	s3Config.SessionToken = vpr.GetString(SessionToken)
-
-	return s3Config, nil
-}
-
-func s3Overrides(in map[string]string) map[string]string {
-	return map[string]string{
-		storage.Bucket:         in[storage.Bucket],
-		storage.Endpoint:       in[storage.Endpoint],
-		storage.Prefix:         in[storage.Prefix],
-		storage.DoNotUseTLS:    in[storage.DoNotUseTLS],
-		storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS],
-		StorageProviderTypeKey: in[StorageProviderTypeKey],
-	}
-}
-
 // configureStorage builds a complete storage configuration from a mix of
 // viper properties and manual overrides.
 func configureStorage(
@@ -58,74 +23,19 @@ func configureStorage(
 	overrides map[string]string,
 ) (storage.Storage, error) {
 	var (
-		s3Cfg storage.S3Config
 		store    storage.Storage
 		err      error
+		provider = storage.ProviderS3 // temporary
 	)
 
-	if readConfigFromViper {
+	storageCfg, err := fetchStorageConfigFromViper(
-		if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil {
+		vpr,
-			return store, clues.Wrap(err, "reading s3 configs from corso config file")
+		provider, // Make it generic
-		}
+		readConfigFromViper,
-
+		matchFromConfig,
-		if b, ok := overrides[storage.Bucket]; ok {
+		overrides)
-			overrides[storage.Bucket] = common.NormalizeBucket(b)
-		}
-
-		if p, ok := overrides[storage.Prefix]; ok {
-			overrides[storage.Prefix] = common.NormalizePrefix(p)
-		}
-
-		if matchFromConfig {
-			providerType := vpr.GetString(StorageProviderTypeKey)
-			if providerType != storage.ProviderS3.String() {
-				return store, clues.New("unsupported storage provider: " + providerType)
-			}
-
-			if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
-				return store, clues.Wrap(err, "verifying s3 configs in corso config file")
-			}
-		}
-	}
-
-	if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
-		return store, clues.Wrap(err, "reading s3 configs from corso config file")
-	}
-
-	s3Overrides(overrides)
-	aws := credentials.GetAWS(overrides)
-
-	if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
-		_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
-		if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
-			aws = credentials.AWS{
-				AccessKey:    s3Cfg.AccessKey,
-				SecretKey:    s3Cfg.SecretKey,
-				SessionToken: s3Cfg.SessionToken,
-			}
-			err = nil
-		}
-
 	if err != nil {
-			return store, clues.Wrap(err, "validating aws credentials")
+		return store, err
-		}
-	}
-
-	s3Cfg = storage.S3Config{
-		AWS:      aws,
-		Bucket:   str.First(overrides[storage.Bucket], s3Cfg.Bucket),
-		Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"),
-		Prefix:   str.First(overrides[storage.Prefix], s3Cfg.Prefix),
-		DoNotUseTLS: str.ParseBool(str.First(
-			overrides[storage.DoNotUseTLS],
-			strconv.FormatBool(s3Cfg.DoNotUseTLS),
-			"false",
-		)),
-		DoNotVerifyTLS: str.ParseBool(str.First(
-			overrides[storage.DoNotVerifyTLS],
-			strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
-			"false",
-		)),
 	}
 
 	// compose the common config and credentials
@@ -147,14 +57,13 @@ func configureStorage(
 
 	// ensure required properties are present
 	if err := requireProps(map[string]string{
-		storage.Bucket:              s3Cfg.Bucket,
 		credentials.CorsoPassphrase: corso.CorsoPassphrase,
 	}); err != nil {
 		return storage.Storage{}, err
 	}
 
 	// build the storage
-	store, err = storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg)
+	store, err = storage.NewStorage(provider, storageCfg, cCfg)
 	if err != nil {
 		return store, clues.Wrap(err, "configuring repository storage")
 	}
@@ -162,6 +71,25 @@ func configureStorage(
 	return store, nil
 }
 
+func fetchStorageConfigFromViper(
+	vpr *viper.Viper,
+	p storage.StorageProvider,
+	readConfigFromViper bool,
+	matchFromConfig bool,
+	overrides map[string]string,
+) (StorageConfigurer, error) {
+	switch p {
+	case storage.ProviderS3:
+		return fetchS3ConfigFromViper(
+			vpr,
+			readConfigFromViper,
+			matchFromConfig,
+			overrides)
+	}
+
+	return nil, clues.New("unsupported storage provider")
+}
+
 // GetCorso is a helper for aggregating Corso secrets and credentials.
 func GetAndInsertCorso(passphase string) credentials.Corso {
 	// fetch data from flag, env var or func param giving priority to func param
@@ -172,3 +100,20 @@ func GetAndInsertCorso(passphase string) credentials.Corso {
 		CorsoPassphrase: corsoPassph,
 	}
 }
+
+// Helper for parsing the values in a config object.
+// If the value is nil or not a string, returns an empty string.
+func orEmptyString(v any) string {
+	defer func() {
+		r := recover()
+		if r != nil {
+			fmt.Printf("panic recovery casting %v to string\n", v)
+		}
+	}()
+
+	if v == nil {
+		return ""
+	}
+
+	return v.(string)
+}

src/cli/repo/s3.go

@@ -128,6 +128,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
 	// s3 values from flags
 	s3Override := S3Overrides(cmd)
 
+	// Need to send provider here
 	cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
 	if err != nil {
 		return Only(ctx, err)
@@ -149,11 +150,17 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
 		cfg.Account.ID(),
 		opt)
 
-	s3Cfg, err := cfg.Storage.S3Config()
+	storageCfg, err := config.NewStorageConfigFrom(cfg.Storage)
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
 	}
 
+	s3Cfg, ok := storageCfg.(config.S3Config)
+	if !ok {
+		return Only(ctx, clues.New("Casting storage config to S3Config"))
+	}
+
+	// TODO: move this to cfg validate
 	if strings.HasPrefix(s3Cfg.Endpoint, "http://") || strings.HasPrefix(s3Cfg.Endpoint, "https://") {
 		invalidEndpointErr := "endpoint doesn't support specifying protocol. " +
 			"pass --disable-tls flag to use http:// instead of default https://"
@@ -214,6 +221,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 	// s3 values from flags
 	s3Override := S3Overrides(cmd)
 
+	// Send provider here
 	cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
 	if err != nil {
 		return Only(ctx, err)
@@ -224,11 +232,16 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 		repoID = events.RepoIDNotFound
 	}
 
-	s3Cfg, err := cfg.Storage.S3Config()
+	storageCfg, err := config.NewStorageConfigFrom(cfg.Storage)
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
 	}
 
+	s3Cfg, ok := storageCfg.(config.S3Config)
+	if !ok {
+		return Only(ctx, clues.New("Casting storage config to S3Config"))
+	}
+
 	m365, err := cfg.Account.M365Config()
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
@@ -287,23 +300,23 @@ func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string {
 	}
 
 	if _, ok := flagset[bucketFN]; ok {
-		s3Overrides[storage.Bucket] = bucket
+		s3Overrides[config.Bucket] = bucket
 	}
 
 	if _, ok := flagset[prefixFN]; ok {
-		s3Overrides[storage.Prefix] = prefix
+		s3Overrides[config.Prefix] = prefix
 	}
 
 	if _, ok := flagset[doNotUseTLSFN]; ok {
-		s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS)
+		s3Overrides[config.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS)
 	}
 
 	if _, ok := flagset[doNotVerifyTLSFN]; ok {
-		s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS)
+		s3Overrides[config.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS)
 	}
 
 	if _, ok := flagset[endpointFN]; ok {
-		s3Overrides[storage.Endpoint] = endpoint
+		s3Overrides[config.Endpoint] = endpoint
 	}
 
 	return s3Overrides

src/cli/utils/utils.go

@@ -63,7 +63,7 @@ func AccountConnectAndWriteRepoConfig(
 		return nil, nil, err
 	}
 
-	s3Config, err := stg.S3Config()
+	storageConfig, err := config.NewStorageConfigFrom(*stg)
 	if err != nil {
 		logger.CtxErr(ctx, err).Info("getting storage configuration")
 		return nil, nil, err
@@ -77,7 +77,7 @@ func AccountConnectAndWriteRepoConfig(
 
 	// repo config gets set during repo connect and init.
 	// This call confirms we have the correct values.
-	err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID())
+	err = config.WriteRepoConfig(ctx, storageConfig, m365Config, opts.Repo, r.GetID())
 	if err != nil {
 		logger.CtxErr(ctx, err).Info("writing to repository configuration")
 		return nil, nil, err

src/internal/kopia/s3.go

@@ -7,6 +7,7 @@ import (
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/blob/s3"
 
+	"github.com/alcionai/corso/src/cli/config"
 	"github.com/alcionai/corso/src/pkg/control/repository"
 	"github.com/alcionai/corso/src/pkg/storage"
 )
@@ -20,11 +21,16 @@ func s3BlobStorage(
 	repoOpts repository.Options,
 	s storage.Storage,
 ) (blob.Storage, error) {
-	cfg, err := s.S3Config()
+	sCfg, err := config.NewStorageConfigFrom(s)
 	if err != nil {
 		return nil, clues.Stack(err).WithClues(ctx)
 	}
 
+	cfg, ok := sCfg.(config.S3Config)
+	if !ok {
+		return nil, clues.New("casting storage config to S3Config")
+	}
+
 	endpoint := defaultS3Endpoint
 	if len(cfg.Endpoint) > 0 {
 		endpoint = cfg.Endpoint

src/pkg/storage/common.go

@@ -1,11 +1,14 @@
 package storage
 
 import (
+	"fmt"
+
 	"github.com/alcionai/clues"
 
 	"github.com/alcionai/corso/src/pkg/credentials"
 )
 
+// Move this to config
 type CommonConfig struct {
 	credentials.Corso // requires: CorsoPassphrase
 
@@ -42,6 +45,11 @@ func (s Storage) CommonConfig() (CommonConfig, error) {
 	return c, c.validate()
 }
 
+// storage parsing errors
+var (
+	errMissingRequired = clues.New("missing required storage configuration")
+)
+
 // ensures all required properties are present
 func (c CommonConfig) validate() error {
 	if len(c.CorsoPassphrase) == 0 {
@@ -51,3 +59,20 @@ func (c CommonConfig) validate() error {
 	// kopiaCfgFilePath is not required
 	return nil
 }
+
+// Helper for parsing the values in a config object.
+// If the value is nil or not a string, returns an empty string.
+func orEmptyString(v any) string {
+	defer func() {
+		r := recover()
+		if r != nil {
+			fmt.Printf("panic recovery casting %v to string\n", v)
+		}
+	}()
+
+	if v == nil {
+		return ""
+	}
+
+	return v.(string)
+}

src/pkg/storage/s3.go

@@ -1,102 +0,0 @@
-package storage
-
-import (
-	"strconv"
-
-	"github.com/alcionai/clues"
-
-	"github.com/alcionai/corso/src/internal/common"
-	"github.com/alcionai/corso/src/internal/common/str"
-	"github.com/alcionai/corso/src/pkg/credentials"
-)
-
-type S3Config struct {
-	credentials.AWS
-	Bucket         string // required
-	Endpoint       string
-	Prefix         string
-	DoNotUseTLS    bool
-	DoNotVerifyTLS bool
-}
-
-// config key consts
-const (
-	keyS3AccessKey      = "s3_access_key"
-	keyS3Bucket         = "s3_bucket"
-	keyS3Endpoint       = "s3_endpoint"
-	keyS3Prefix         = "s3_prefix"
-	keyS3SecretKey      = "s3_secret_key"
-	keyS3SessionToken   = "s3_session_token"
-	keyS3DoNotUseTLS    = "s3_donotusetls"
-	keyS3DoNotVerifyTLS = "s3_donotverifytls"
-)
-
-// config exported name consts
-const (
-	Bucket         = "bucket"
-	Endpoint       = "endpoint"
-	Prefix         = "prefix"
-	DoNotUseTLS    = "donotusetls"
-	DoNotVerifyTLS = "donotverifytls"
-)
-
-func (c S3Config) Normalize() S3Config {
-	return S3Config{
-		Bucket:         common.NormalizeBucket(c.Bucket),
-		Endpoint:       c.Endpoint,
-		Prefix:         common.NormalizePrefix(c.Prefix),
-		DoNotUseTLS:    c.DoNotUseTLS,
-		DoNotVerifyTLS: c.DoNotVerifyTLS,
-	}
-}
-
-// StringConfig transforms a s3Config struct into a plain
-// map[string]string.  All values in the original struct which
-// serialize into the map are expected to be strings.
-func (c S3Config) StringConfig() (map[string]string, error) {
-	cn := c.Normalize()
-	cfg := map[string]string{
-		keyS3AccessKey:      c.AccessKey,
-		keyS3Bucket:         cn.Bucket,
-		keyS3Endpoint:       cn.Endpoint,
-		keyS3Prefix:         cn.Prefix,
-		keyS3SecretKey:      c.SecretKey,
-		keyS3SessionToken:   c.SessionToken,
-		keyS3DoNotUseTLS:    strconv.FormatBool(cn.DoNotUseTLS),
-		keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
-	}
-
-	return cfg, c.validate()
-}
-
-// S3Config retrieves the S3Config details from the Storage config.
-func (s Storage) S3Config() (S3Config, error) {
-	c := S3Config{}
-
-	if len(s.Config) > 0 {
-		c.AccessKey = orEmptyString(s.Config[keyS3AccessKey])
-		c.SecretKey = orEmptyString(s.Config[keyS3SecretKey])
-		c.SessionToken = orEmptyString(s.Config[keyS3SessionToken])
-
-		c.Bucket = orEmptyString(s.Config[keyS3Bucket])
-		c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
-		c.Prefix = orEmptyString(s.Config[keyS3Prefix])
-		c.DoNotUseTLS = str.ParseBool(s.Config[keyS3DoNotUseTLS])
-		c.DoNotVerifyTLS = str.ParseBool(s.Config[keyS3DoNotVerifyTLS])
-	}
-
-	return c, c.validate()
-}
-
-func (c S3Config) validate() error {
-	check := map[string]string{
-		Bucket: c.Bucket,
-	}
-	for k, v := range check {
-		if len(v) == 0 {
-			return clues.Stack(errMissingRequired, clues.New(k))
-		}
-	}
-
-	return nil
-}

src/pkg/storage/storage.go

@@ -1,30 +1,21 @@
 package storage
 
 import (
-	"fmt"
-
-	"github.com/alcionai/clues"
-
 	"github.com/alcionai/corso/src/internal/common"
 )
 
-type storageProvider int
+type StorageProvider int
 
-//go:generate stringer -type=storageProvider -linecomment
+//go:generate stringer -type=StorageProvider -linecomment
 const (
-	ProviderUnknown storageProvider = 0 // Unknown Provider
+	ProviderUnknown StorageProvider = 0 // Unknown Provider
-	ProviderS3      storageProvider = 1 // S3
+	ProviderS3      StorageProvider = 1 // S3
-)
-
-// storage parsing errors
-var (
-	errMissingRequired = clues.New("missing required storage configuration")
 )
 
 // Storage defines a storage provider, along with any configuration
 // required to set up or communicate with that provider.
 type Storage struct {
-	Provider storageProvider
+	Provider StorageProvider
 	Config   map[string]string
 	// TODO: These are AWS S3 specific -> move these out
 	SessionTags     map[string]string
@@ -34,7 +25,7 @@ type Storage struct {
 }
 
 // NewStorage aggregates all the supplied configurations into a single configuration.
-func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, error) {
+func NewStorage(p StorageProvider, cfgs ...common.StringConfigurer) (Storage, error) {
 	cs, err := common.UnionStringConfigs(cfgs...)
 
 	return Storage{
@@ -46,7 +37,7 @@ func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, er
 // NewStorageUsingRole supports specifying an AWS IAM role the storage provider
 // should assume.
 func NewStorageUsingRole(
-	p storageProvider,
+	p StorageProvider,
 	roleARN string,
 	sessionName string,
 	sessionTags map[string]string,
@@ -64,20 +55,3 @@ func NewStorageUsingRole(
 		SessionDuration: duration,
 	}, err
 }
-
-// Helper for parsing the values in a config object.
-// If the value is nil or not a string, returns an empty string.
-func orEmptyString(v any) string {
-	defer func() {
-		r := recover()
-		if r != nil {
-			fmt.Printf("panic recovery casting %v to string\n", v)
-		}
-	}()
-
-	if v == nil {
-		return ""
-	}
-
-	return v.(string)
-}

src/pkg/storage/storageprovider_string.go

@@ -1,4 +1,4 @@
-// Code generated by "stringer -type=storageProvider -linecomment"; DO NOT EDIT.
+// Code generated by "stringer -type=StorageProvider -linecomment"; DO NOT EDIT.
 
 package storage
 
@@ -12,13 +12,13 @@ func _() {
 	_ = x[ProviderS3-1]
 }
 
-const _storageProvider_name = "Unknown ProviderS3"
+const _StorageProvider_name = "Unknown ProviderS3"
 
-var _storageProvider_index = [...]uint8{0, 16, 18}
+var _StorageProvider_index = [...]uint8{0, 16, 18}
 
-func (i storageProvider) String() string {
+func (i StorageProvider) String() string {
-	if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) {
+	if i < 0 || i >= StorageProvider(len(_StorageProvider_index)-1) {
-		return "storageProvider(" + strconv.FormatInt(int64(i), 10) + ")"
+		return "StorageProvider(" + strconv.FormatInt(int64(i), 10) + ")"
 	}
-	return _storageProvider_name[_storageProvider_index[i]:_storageProvider_index[i+1]]
+	return _StorageProvider_name[_StorageProvider_index[i]:_StorageProvider_index[i+1]]
 }