diff --git a/src/cli/config/config.go b/src/cli/config/config.go index 39a34eb9c..1e8d166ac 100644 --- a/src/cli/config/config.go +++ b/src/cli/config/config.go @@ -203,14 +203,14 @@ func Read(ctx context.Context) error { // It does not check for conflicts or existing data. func WriteRepoConfig( ctx context.Context, - s3Config storage.S3Config, + storageConfig StorageConfigurer, m365Config account.M365Config, repoOpts repository.Options, repoID string, ) error { return writeRepoConfigWithViper( GetViper(ctx), - s3Config, + storageConfig, m365Config, repoOpts, repoID) @@ -220,20 +220,16 @@ func WriteRepoConfig( // struct for testing. func writeRepoConfigWithViper( vpr *viper.Viper, - s3Config storage.S3Config, + storageConfig StorageConfigurer, m365Config account.M365Config, repoOpts repository.Options, repoID string, ) error { - s3Config = s3Config.Normalize() + storageConfig.WriteConfigToViper(vpr) + // Rudimentary support for persisting repo config // TODO: Handle conflicts, support other config types - vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String()) - vpr.Set(BucketNameKey, s3Config.Bucket) - vpr.Set(EndpointKey, s3Config.Endpoint) - vpr.Set(PrefixKey, s3Config.Prefix) - vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS) - vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS) + vpr.Set(RepoID, repoID) // Need if-checks as Viper will write empty values otherwise. @@ -335,12 +331,13 @@ func getUserHost(vpr *viper.Viper, readConfigFromViper bool) (string, string) { // Helper funcs // --------------------------------------------------------------------------- +// TODO: This is not really needed? var constToTomlKeyMap = map[string]string{ account.AzureTenantID: AzureTenantIDKey, AccountProviderTypeKey: AccountProviderTypeKey, - storage.Bucket: BucketNameKey, - storage.Endpoint: EndpointKey, - storage.Prefix: PrefixKey, + Bucket: BucketNameKey, + Endpoint: EndpointKey, + Prefix: PrefixKey, StorageProviderTypeKey: StorageProviderTypeKey, } diff --git a/src/cli/config/s3config.go b/src/cli/config/s3config.go new file mode 100644 index 000000000..bc3efa546 --- /dev/null +++ b/src/cli/config/s3config.go @@ -0,0 +1,263 @@ +package config + +import ( + "strconv" + + "github.com/alcionai/clues" + "github.com/aws/aws-sdk-go/aws/defaults" + "github.com/spf13/viper" + + "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/common/str" + "github.com/alcionai/corso/src/pkg/credentials" + "github.com/alcionai/corso/src/pkg/storage" +) + +// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values. +func s3ConfigsFromViper(vpr *viper.Viper) (S3Config, error) { + var s3Config S3Config + + s3Config.Bucket = vpr.GetString(BucketNameKey) + s3Config.Endpoint = vpr.GetString(EndpointKey) + s3Config.Prefix = vpr.GetString(PrefixKey) + s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey) + s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey) + + return s3Config, nil +} + +// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values. +func s3CredsFromViper(vpr *viper.Viper, s3Config S3Config) (S3Config, error) { + s3Config.AccessKey = vpr.GetString(AccessKey) + s3Config.SecretKey = vpr.GetString(SecretAccessKey) + s3Config.SessionToken = vpr.GetString(SessionToken) + + return s3Config, nil +} + +func s3Overrides(in map[string]string) map[string]string { + return map[string]string{ + Bucket: in[Bucket], + Endpoint: in[Endpoint], + Prefix: in[Prefix], + DoNotUseTLS: in[DoNotUseTLS], + DoNotVerifyTLS: in[DoNotVerifyTLS], + StorageProviderTypeKey: in[StorageProviderTypeKey], + } +} + +type StorageConfigurer interface { + common.StringConfigurer + //Normalize() StorageConfigurer + WriteConfigToViper(vpr *viper.Viper) +} + +// func NewStorageConfig( +// provider storage.StorageProvider, +// ) (StorageConfigurer, error) { +// switch provider { +// case storage.ProviderS3: +// return S3Config{}, nil +// default: +// return nil, clues.New("unsupported storage type") +// } +// } + +// Hydrate from a config map +func NewStorageConfigFrom(s storage.Storage) (StorageConfigurer, error) { + switch s.Provider { + case storage.ProviderS3: + return makeS3Config(s.Config) + default: + return nil, clues.New("unsupported storage type") + } +} + +type S3Config struct { + credentials.AWS + Bucket string // required + Endpoint string + Prefix string + DoNotUseTLS bool + DoNotVerifyTLS bool +} + +// MakeS3Config retrieves the S3Config details from the Storage config. +func makeS3Config(config map[string]string) (StorageConfigurer, error) { + c := S3Config{} + + if len(config) > 0 { + c.AccessKey = orEmptyString(config[keyS3AccessKey]) + c.SecretKey = orEmptyString(config[keyS3SecretKey]) + c.SessionToken = orEmptyString(config[keyS3SessionToken]) + + c.Bucket = orEmptyString(config[keyS3Bucket]) + c.Endpoint = orEmptyString(config[keyS3Endpoint]) + c.Prefix = orEmptyString(config[keyS3Prefix]) + c.DoNotUseTLS = str.ParseBool(config[keyS3DoNotUseTLS]) + c.DoNotVerifyTLS = str.ParseBool(config[keyS3DoNotVerifyTLS]) + } + + return c, c.validate() +} + +// TODO: Remove storageconfigurer return value. +// Have it contained in current s3config. +func fetchS3ConfigFromViper( + vpr *viper.Viper, + readConfigFromViper bool, + matchFromConfig bool, + overrides map[string]string, +) (StorageConfigurer, error) { + var ( + s3Cfg S3Config + err error + ) + + if readConfigFromViper { + if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil { + clues.Wrap(err, "reading s3 configs from corso config file") + } + + if b, ok := overrides[Bucket]; ok { + overrides[Bucket] = common.NormalizeBucket(b) + } + + if p, ok := overrides[Prefix]; ok { + overrides[Prefix] = common.NormalizePrefix(p) + } + + if matchFromConfig { + providerType := vpr.GetString(StorageProviderTypeKey) + if providerType != storage.ProviderS3.String() { + return nil, clues.New("unsupported storage provider" + providerType) + } + + if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil { + return nil, clues.Wrap(err, "verifying s3 configs in corso config file") + } + } + } + + if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil { + return nil, clues.Wrap(err, "reading s3 configs from corso config file") + } + + s3Overrides(overrides) + aws := credentials.GetAWS(overrides) + + if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 { + _, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get() + if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) { + aws = credentials.AWS{ + AccessKey: s3Cfg.AccessKey, + SecretKey: s3Cfg.SecretKey, + SessionToken: s3Cfg.SessionToken, + } + err = nil + } + + if err != nil { + return nil, clues.Wrap(err, "validating aws credentials") + } + } + + s3Cfg = S3Config{ + AWS: aws, + Bucket: str.First(overrides[Bucket], s3Cfg.Bucket), + Endpoint: str.First(overrides[Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"), + Prefix: str.First(overrides[Prefix], s3Cfg.Prefix), + DoNotUseTLS: str.ParseBool(str.First( + overrides[DoNotUseTLS], + strconv.FormatBool(s3Cfg.DoNotUseTLS), + "false", + )), + DoNotVerifyTLS: str.ParseBool(str.First( + overrides[DoNotVerifyTLS], + strconv.FormatBool(s3Cfg.DoNotVerifyTLS), + "false", + )), + } + + return s3Cfg, s3Cfg.validate() +} + +// config key consts +const ( + keyS3AccessKey = "s3_access_key" + keyS3Bucket = "s3_bucket" + keyS3Endpoint = "s3_endpoint" + keyS3Prefix = "s3_prefix" + keyS3SecretKey = "s3_secret_key" + keyS3SessionToken = "s3_session_token" + keyS3DoNotUseTLS = "s3_donotusetls" + keyS3DoNotVerifyTLS = "s3_donotverifytls" +) + +// config exported name consts +// TODO: Move these to storage? +const ( + Bucket = "bucket" + Endpoint = "endpoint" + Prefix = "prefix" + DoNotUseTLS = "donotusetls" + DoNotVerifyTLS = "donotverifytls" +) + +func normalize(c S3Config) S3Config { + return S3Config{ + Bucket: common.NormalizeBucket(c.Bucket), + Endpoint: c.Endpoint, + Prefix: common.NormalizePrefix(c.Prefix), + DoNotUseTLS: c.DoNotUseTLS, + DoNotVerifyTLS: c.DoNotVerifyTLS, + } +} + +// StringConfig transforms a s3Config struct into a plain +// map[string]string. All values in the original struct which +// serialize into the map are expected to be strings. +func (c S3Config) StringConfig() (map[string]string, error) { + cn := normalize(c) + cfg := map[string]string{ + keyS3AccessKey: c.AccessKey, + keyS3Bucket: cn.Bucket, + keyS3Endpoint: cn.Endpoint, + keyS3Prefix: cn.Prefix, + keyS3SecretKey: c.SecretKey, + keyS3SessionToken: c.SessionToken, + keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS), + keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS), + } + + return cfg, c.validate() +} + +func (c S3Config) WriteConfigToViper(vpr *viper.Viper) { + cn := normalize(c) + + vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String()) + vpr.Set(BucketNameKey, cn.Bucket) + vpr.Set(EndpointKey, cn.Endpoint) + vpr.Set(PrefixKey, cn.Prefix) + vpr.Set(DisableTLSKey, cn.DoNotUseTLS) + vpr.Set(DisableTLSVerificationKey, cn.DoNotVerifyTLS) +} + +// storage parsing errors +var ( + errMissingRequired = clues.New("missing required storage configuration") +) + +func (c S3Config) validate() error { + check := map[string]string{ + Bucket: c.Bucket, + } + for k, v := range check { + if len(v) == 0 { + return clues.Stack(errMissingRequired, clues.New(k)) + } + } + + return nil +} diff --git a/src/cli/config/storage.go b/src/cli/config/storage.go index d948e6af3..41151c584 100644 --- a/src/cli/config/storage.go +++ b/src/cli/config/storage.go @@ -1,54 +1,19 @@ package config import ( + "fmt" "os" "path/filepath" - "strconv" "github.com/alcionai/clues" - "github.com/aws/aws-sdk-go/aws/defaults" "github.com/spf13/viper" "github.com/alcionai/corso/src/cli/flags" - "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/storage" ) -// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values. -func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) { - var s3Config storage.S3Config - - s3Config.Bucket = vpr.GetString(BucketNameKey) - s3Config.Endpoint = vpr.GetString(EndpointKey) - s3Config.Prefix = vpr.GetString(PrefixKey) - s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey) - s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey) - - return s3Config, nil -} - -// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values. -func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Config, error) { - s3Config.AccessKey = vpr.GetString(AccessKey) - s3Config.SecretKey = vpr.GetString(SecretAccessKey) - s3Config.SessionToken = vpr.GetString(SessionToken) - - return s3Config, nil -} - -func s3Overrides(in map[string]string) map[string]string { - return map[string]string{ - storage.Bucket: in[storage.Bucket], - storage.Endpoint: in[storage.Endpoint], - storage.Prefix: in[storage.Prefix], - storage.DoNotUseTLS: in[storage.DoNotUseTLS], - storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS], - StorageProviderTypeKey: in[StorageProviderTypeKey], - } -} - // configureStorage builds a complete storage configuration from a mix of // viper properties and manual overrides. func configureStorage( @@ -58,74 +23,19 @@ func configureStorage( overrides map[string]string, ) (storage.Storage, error) { var ( - s3Cfg storage.S3Config - store storage.Storage - err error + store storage.Storage + err error + provider = storage.ProviderS3 // temporary ) - if readConfigFromViper { - if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil { - return store, clues.Wrap(err, "reading s3 configs from corso config file") - } - - if b, ok := overrides[storage.Bucket]; ok { - overrides[storage.Bucket] = common.NormalizeBucket(b) - } - - if p, ok := overrides[storage.Prefix]; ok { - overrides[storage.Prefix] = common.NormalizePrefix(p) - } - - if matchFromConfig { - providerType := vpr.GetString(StorageProviderTypeKey) - if providerType != storage.ProviderS3.String() { - return store, clues.New("unsupported storage provider: " + providerType) - } - - if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil { - return store, clues.Wrap(err, "verifying s3 configs in corso config file") - } - } - } - - if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil { - return store, clues.Wrap(err, "reading s3 configs from corso config file") - } - - s3Overrides(overrides) - aws := credentials.GetAWS(overrides) - - if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 { - _, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get() - if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) { - aws = credentials.AWS{ - AccessKey: s3Cfg.AccessKey, - SecretKey: s3Cfg.SecretKey, - SessionToken: s3Cfg.SessionToken, - } - err = nil - } - - if err != nil { - return store, clues.Wrap(err, "validating aws credentials") - } - } - - s3Cfg = storage.S3Config{ - AWS: aws, - Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket), - Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"), - Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix), - DoNotUseTLS: str.ParseBool(str.First( - overrides[storage.DoNotUseTLS], - strconv.FormatBool(s3Cfg.DoNotUseTLS), - "false", - )), - DoNotVerifyTLS: str.ParseBool(str.First( - overrides[storage.DoNotVerifyTLS], - strconv.FormatBool(s3Cfg.DoNotVerifyTLS), - "false", - )), + storageCfg, err := fetchStorageConfigFromViper( + vpr, + provider, // Make it generic + readConfigFromViper, + matchFromConfig, + overrides) + if err != nil { + return store, err } // compose the common config and credentials @@ -147,14 +57,13 @@ func configureStorage( // ensure required properties are present if err := requireProps(map[string]string{ - storage.Bucket: s3Cfg.Bucket, credentials.CorsoPassphrase: corso.CorsoPassphrase, }); err != nil { return storage.Storage{}, err } // build the storage - store, err = storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg) + store, err = storage.NewStorage(provider, storageCfg, cCfg) if err != nil { return store, clues.Wrap(err, "configuring repository storage") } @@ -162,6 +71,25 @@ func configureStorage( return store, nil } +func fetchStorageConfigFromViper( + vpr *viper.Viper, + p storage.StorageProvider, + readConfigFromViper bool, + matchFromConfig bool, + overrides map[string]string, +) (StorageConfigurer, error) { + switch p { + case storage.ProviderS3: + return fetchS3ConfigFromViper( + vpr, + readConfigFromViper, + matchFromConfig, + overrides) + } + + return nil, clues.New("unsupported storage provider") +} + // GetCorso is a helper for aggregating Corso secrets and credentials. func GetAndInsertCorso(passphase string) credentials.Corso { // fetch data from flag, env var or func param giving priority to func param @@ -172,3 +100,20 @@ func GetAndInsertCorso(passphase string) credentials.Corso { CorsoPassphrase: corsoPassph, } } + +// Helper for parsing the values in a config object. +// If the value is nil or not a string, returns an empty string. +func orEmptyString(v any) string { + defer func() { + r := recover() + if r != nil { + fmt.Printf("panic recovery casting %v to string\n", v) + } + }() + + if v == nil { + return "" + } + + return v.(string) +} diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 972701502..a3037428b 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -128,6 +128,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { // s3 values from flags s3Override := S3Overrides(cmd) + // Need to send provider here cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override) if err != nil { return Only(ctx, err) @@ -149,11 +150,17 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { cfg.Account.ID(), opt) - s3Cfg, err := cfg.Storage.S3Config() + storageCfg, err := config.NewStorageConfigFrom(cfg.Storage) if err != nil { return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration")) } + s3Cfg, ok := storageCfg.(config.S3Config) + if !ok { + return Only(ctx, clues.New("Casting storage config to S3Config")) + } + + // TODO: move this to cfg validate if strings.HasPrefix(s3Cfg.Endpoint, "http://") || strings.HasPrefix(s3Cfg.Endpoint, "https://") { invalidEndpointErr := "endpoint doesn't support specifying protocol. " + "pass --disable-tls flag to use http:// instead of default https://" @@ -214,6 +221,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { // s3 values from flags s3Override := S3Overrides(cmd) + // Send provider here cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override) if err != nil { return Only(ctx, err) @@ -224,11 +232,16 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { repoID = events.RepoIDNotFound } - s3Cfg, err := cfg.Storage.S3Config() + storageCfg, err := config.NewStorageConfigFrom(cfg.Storage) if err != nil { return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration")) } + s3Cfg, ok := storageCfg.(config.S3Config) + if !ok { + return Only(ctx, clues.New("Casting storage config to S3Config")) + } + m365, err := cfg.Account.M365Config() if err != nil { return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) @@ -287,23 +300,23 @@ func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string { } if _, ok := flagset[bucketFN]; ok { - s3Overrides[storage.Bucket] = bucket + s3Overrides[config.Bucket] = bucket } if _, ok := flagset[prefixFN]; ok { - s3Overrides[storage.Prefix] = prefix + s3Overrides[config.Prefix] = prefix } if _, ok := flagset[doNotUseTLSFN]; ok { - s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS) + s3Overrides[config.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS) } if _, ok := flagset[doNotVerifyTLSFN]; ok { - s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS) + s3Overrides[config.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS) } if _, ok := flagset[endpointFN]; ok { - s3Overrides[storage.Endpoint] = endpoint + s3Overrides[config.Endpoint] = endpoint } return s3Overrides diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 5a639474a..06308bd7a 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -63,7 +63,7 @@ func AccountConnectAndWriteRepoConfig( return nil, nil, err } - s3Config, err := stg.S3Config() + storageConfig, err := config.NewStorageConfigFrom(*stg) if err != nil { logger.CtxErr(ctx, err).Info("getting storage configuration") return nil, nil, err @@ -77,7 +77,7 @@ func AccountConnectAndWriteRepoConfig( // repo config gets set during repo connect and init. // This call confirms we have the correct values. - err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID()) + err = config.WriteRepoConfig(ctx, storageConfig, m365Config, opts.Repo, r.GetID()) if err != nil { logger.CtxErr(ctx, err).Info("writing to repository configuration") return nil, nil, err diff --git a/src/internal/kopia/s3.go b/src/internal/kopia/s3.go index adad4330e..9b3a1a1c9 100644 --- a/src/internal/kopia/s3.go +++ b/src/internal/kopia/s3.go @@ -7,6 +7,7 @@ import ( "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/blob/s3" + "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" ) @@ -20,11 +21,16 @@ func s3BlobStorage( repoOpts repository.Options, s storage.Storage, ) (blob.Storage, error) { - cfg, err := s.S3Config() + sCfg, err := config.NewStorageConfigFrom(s) if err != nil { return nil, clues.Stack(err).WithClues(ctx) } + cfg, ok := sCfg.(config.S3Config) + if !ok { + return nil, clues.New("casting storage config to S3Config") + } + endpoint := defaultS3Endpoint if len(cfg.Endpoint) > 0 { endpoint = cfg.Endpoint diff --git a/src/pkg/storage/common.go b/src/pkg/storage/common.go index 06c082b57..574c7b2f1 100644 --- a/src/pkg/storage/common.go +++ b/src/pkg/storage/common.go @@ -1,11 +1,14 @@ package storage import ( + "fmt" + "github.com/alcionai/clues" "github.com/alcionai/corso/src/pkg/credentials" ) +// Move this to config type CommonConfig struct { credentials.Corso // requires: CorsoPassphrase @@ -42,6 +45,11 @@ func (s Storage) CommonConfig() (CommonConfig, error) { return c, c.validate() } +// storage parsing errors +var ( + errMissingRequired = clues.New("missing required storage configuration") +) + // ensures all required properties are present func (c CommonConfig) validate() error { if len(c.CorsoPassphrase) == 0 { @@ -51,3 +59,20 @@ func (c CommonConfig) validate() error { // kopiaCfgFilePath is not required return nil } + +// Helper for parsing the values in a config object. +// If the value is nil or not a string, returns an empty string. +func orEmptyString(v any) string { + defer func() { + r := recover() + if r != nil { + fmt.Printf("panic recovery casting %v to string\n", v) + } + }() + + if v == nil { + return "" + } + + return v.(string) +} diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go deleted file mode 100644 index a332326e8..000000000 --- a/src/pkg/storage/s3.go +++ /dev/null @@ -1,102 +0,0 @@ -package storage - -import ( - "strconv" - - "github.com/alcionai/clues" - - "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/pkg/credentials" -) - -type S3Config struct { - credentials.AWS - Bucket string // required - Endpoint string - Prefix string - DoNotUseTLS bool - DoNotVerifyTLS bool -} - -// config key consts -const ( - keyS3AccessKey = "s3_access_key" - keyS3Bucket = "s3_bucket" - keyS3Endpoint = "s3_endpoint" - keyS3Prefix = "s3_prefix" - keyS3SecretKey = "s3_secret_key" - keyS3SessionToken = "s3_session_token" - keyS3DoNotUseTLS = "s3_donotusetls" - keyS3DoNotVerifyTLS = "s3_donotverifytls" -) - -// config exported name consts -const ( - Bucket = "bucket" - Endpoint = "endpoint" - Prefix = "prefix" - DoNotUseTLS = "donotusetls" - DoNotVerifyTLS = "donotverifytls" -) - -func (c S3Config) Normalize() S3Config { - return S3Config{ - Bucket: common.NormalizeBucket(c.Bucket), - Endpoint: c.Endpoint, - Prefix: common.NormalizePrefix(c.Prefix), - DoNotUseTLS: c.DoNotUseTLS, - DoNotVerifyTLS: c.DoNotVerifyTLS, - } -} - -// StringConfig transforms a s3Config struct into a plain -// map[string]string. All values in the original struct which -// serialize into the map are expected to be strings. -func (c S3Config) StringConfig() (map[string]string, error) { - cn := c.Normalize() - cfg := map[string]string{ - keyS3AccessKey: c.AccessKey, - keyS3Bucket: cn.Bucket, - keyS3Endpoint: cn.Endpoint, - keyS3Prefix: cn.Prefix, - keyS3SecretKey: c.SecretKey, - keyS3SessionToken: c.SessionToken, - keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS), - keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS), - } - - return cfg, c.validate() -} - -// S3Config retrieves the S3Config details from the Storage config. -func (s Storage) S3Config() (S3Config, error) { - c := S3Config{} - - if len(s.Config) > 0 { - c.AccessKey = orEmptyString(s.Config[keyS3AccessKey]) - c.SecretKey = orEmptyString(s.Config[keyS3SecretKey]) - c.SessionToken = orEmptyString(s.Config[keyS3SessionToken]) - - c.Bucket = orEmptyString(s.Config[keyS3Bucket]) - c.Endpoint = orEmptyString(s.Config[keyS3Endpoint]) - c.Prefix = orEmptyString(s.Config[keyS3Prefix]) - c.DoNotUseTLS = str.ParseBool(s.Config[keyS3DoNotUseTLS]) - c.DoNotVerifyTLS = str.ParseBool(s.Config[keyS3DoNotVerifyTLS]) - } - - return c, c.validate() -} - -func (c S3Config) validate() error { - check := map[string]string{ - Bucket: c.Bucket, - } - for k, v := range check { - if len(v) == 0 { - return clues.Stack(errMissingRequired, clues.New(k)) - } - } - - return nil -} diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index e197f4081..2c8c502a3 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -1,30 +1,21 @@ package storage import ( - "fmt" - - "github.com/alcionai/clues" - "github.com/alcionai/corso/src/internal/common" ) -type storageProvider int +type StorageProvider int -//go:generate stringer -type=storageProvider -linecomment +//go:generate stringer -type=StorageProvider -linecomment const ( - ProviderUnknown storageProvider = 0 // Unknown Provider - ProviderS3 storageProvider = 1 // S3 -) - -// storage parsing errors -var ( - errMissingRequired = clues.New("missing required storage configuration") + ProviderUnknown StorageProvider = 0 // Unknown Provider + ProviderS3 StorageProvider = 1 // S3 ) // Storage defines a storage provider, along with any configuration // required to set up or communicate with that provider. type Storage struct { - Provider storageProvider + Provider StorageProvider Config map[string]string // TODO: These are AWS S3 specific -> move these out SessionTags map[string]string @@ -34,7 +25,7 @@ type Storage struct { } // NewStorage aggregates all the supplied configurations into a single configuration. -func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, error) { +func NewStorage(p StorageProvider, cfgs ...common.StringConfigurer) (Storage, error) { cs, err := common.UnionStringConfigs(cfgs...) return Storage{ @@ -46,7 +37,7 @@ func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, er // NewStorageUsingRole supports specifying an AWS IAM role the storage provider // should assume. func NewStorageUsingRole( - p storageProvider, + p StorageProvider, roleARN string, sessionName string, sessionTags map[string]string, @@ -64,20 +55,3 @@ func NewStorageUsingRole( SessionDuration: duration, }, err } - -// Helper for parsing the values in a config object. -// If the value is nil or not a string, returns an empty string. -func orEmptyString(v any) string { - defer func() { - r := recover() - if r != nil { - fmt.Printf("panic recovery casting %v to string\n", v) - } - }() - - if v == nil { - return "" - } - - return v.(string) -} diff --git a/src/pkg/storage/storageprovider_string.go b/src/pkg/storage/storageprovider_string.go index 389dee37a..75a29b22e 100644 --- a/src/pkg/storage/storageprovider_string.go +++ b/src/pkg/storage/storageprovider_string.go @@ -1,4 +1,4 @@ -// Code generated by "stringer -type=storageProvider -linecomment"; DO NOT EDIT. +// Code generated by "stringer -type=StorageProvider -linecomment"; DO NOT EDIT. package storage @@ -12,13 +12,13 @@ func _() { _ = x[ProviderS3-1] } -const _storageProvider_name = "Unknown ProviderS3" +const _StorageProvider_name = "Unknown ProviderS3" -var _storageProvider_index = [...]uint8{0, 16, 18} +var _StorageProvider_index = [...]uint8{0, 16, 18} -func (i storageProvider) String() string { - if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) { - return "storageProvider(" + strconv.FormatInt(int64(i), 10) + ")" +func (i StorageProvider) String() string { + if i < 0 || i >= StorageProvider(len(_StorageProvider_index)-1) { + return "StorageProvider(" + strconv.FormatInt(int64(i), 10) + ")" } - return _storageProvider_name[_storageProvider_index[i]:_storageProvider_index[i+1]] + return _StorageProvider_name[_StorageProvider_index[i]:_StorageProvider_index[i+1]] }