larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move config code around, introduce interface for storageconfiguration

Browse Source
This commit is contained in:
Abhishek Pandey 2023-08-26 16:05:26 +05:30
parent c1c4218994
commit 711245684d
10 changed files with 390 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/cli/config/config.go
View File

@ -203,14 +203,14 @@ func Read(ctx context.Context) error {
// It does not check for conflicts or existing data.
func WriteRepoConfig(
ctx context.Context,
s3Config storage.S3Config,
storageConfig StorageConfigurer,
m365Config account.M365Config,
repoOpts repository.Options,
repoID string,
) error {
return writeRepoConfigWithViper(
GetViper(ctx),
s3Config,
storageConfig,
m365Config,
repoOpts,
repoID)
@ -220,20 +220,16 @@ func WriteRepoConfig(
// struct for testing.
func writeRepoConfigWithViper(
vpr *viper.Viper,
s3Config storage.S3Config,
storageConfig StorageConfigurer,
m365Config account.M365Config,
repoOpts repository.Options,
repoID string,
) error {
s3Config = s3Config.Normalize()
storageConfig.WriteConfigToViper(vpr)
// Rudimentary support for persisting repo config
// TODO: Handle conflicts, support other config types
vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String())
vpr.Set(BucketNameKey, s3Config.Bucket)
vpr.Set(EndpointKey, s3Config.Endpoint)
vpr.Set(PrefixKey, s3Config.Prefix)
vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS)
vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
vpr.Set(RepoID, repoID)
// Need if-checks as Viper will write empty values otherwise.
@ -335,12 +331,13 @@ func getUserHost(vpr *viper.Viper, readConfigFromViper bool) (string, string) {
// Helper funcs
// ---------------------------------------------------------------------------
// TODO: This is not really needed?
var constToTomlKeyMap = map[string]string{
account.AzureTenantID: AzureTenantIDKey,
AccountProviderTypeKey: AccountProviderTypeKey,
storage.Bucket: BucketNameKey,
storage.Endpoint: EndpointKey,
storage.Prefix: PrefixKey,
Bucket: BucketNameKey,
Endpoint: EndpointKey,
Prefix: PrefixKey,
StorageProviderTypeKey: StorageProviderTypeKey,
}

263
src/cli/config/s3config.go Normal file
View File

@ -0,0 +1,263 @@
package config
import (
"strconv"
"github.com/alcionai/clues"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/spf13/viper"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
func s3ConfigsFromViper(vpr *viper.Viper) (S3Config, error) {
var s3Config S3Config
s3Config.Bucket = vpr.GetString(BucketNameKey)
s3Config.Endpoint = vpr.GetString(EndpointKey)
s3Config.Prefix = vpr.GetString(PrefixKey)
s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
return s3Config, nil
}
// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
func s3CredsFromViper(vpr *viper.Viper, s3Config S3Config) (S3Config, error) {
s3Config.AccessKey = vpr.GetString(AccessKey)
s3Config.SecretKey = vpr.GetString(SecretAccessKey)
s3Config.SessionToken = vpr.GetString(SessionToken)
return s3Config, nil
}
func s3Overrides(in map[string]string) map[string]string {
return map[string]string{
Bucket: in[Bucket],
Endpoint: in[Endpoint],
Prefix: in[Prefix],
DoNotUseTLS: in[DoNotUseTLS],
DoNotVerifyTLS: in[DoNotVerifyTLS],
StorageProviderTypeKey: in[StorageProviderTypeKey],
}
}
type StorageConfigurer interface {
common.StringConfigurer
//Normalize() StorageConfigurer
WriteConfigToViper(vpr *viper.Viper)
}
// func NewStorageConfig(
// provider storage.StorageProvider,
// ) (StorageConfigurer, error) {
// switch provider {
// case storage.ProviderS3:
// return S3Config{}, nil
// default:
// return nil, clues.New("unsupported storage type")
// }
// }
// Hydrate from a config map
func NewStorageConfigFrom(s storage.Storage) (StorageConfigurer, error) {
switch s.Provider {
case storage.ProviderS3:
return makeS3Config(s.Config)
default:
return nil, clues.New("unsupported storage type")
}
}
type S3Config struct {
credentials.AWS
Bucket string // required
Endpoint string
Prefix string
DoNotUseTLS bool
DoNotVerifyTLS bool
}
// MakeS3Config retrieves the S3Config details from the Storage config.
func makeS3Config(config map[string]string) (StorageConfigurer, error) {
c := S3Config{}
if len(config) > 0 {
c.AccessKey = orEmptyString(config[keyS3AccessKey])
c.SecretKey = orEmptyString(config[keyS3SecretKey])
c.SessionToken = orEmptyString(config[keyS3SessionToken])
c.Bucket = orEmptyString(config[keyS3Bucket])
c.Endpoint = orEmptyString(config[keyS3Endpoint])
c.Prefix = orEmptyString(config[keyS3Prefix])
c.DoNotUseTLS = str.ParseBool(config[keyS3DoNotUseTLS])
c.DoNotVerifyTLS = str.ParseBool(config[keyS3DoNotVerifyTLS])
}
return c, c.validate()
}
// TODO: Remove storageconfigurer return value.
// Have it contained in current s3config.
func fetchS3ConfigFromViper(
vpr *viper.Viper,
readConfigFromViper bool,
matchFromConfig bool,
overrides map[string]string,
) (StorageConfigurer, error) {
var (
s3Cfg S3Config
err error
)
if readConfigFromViper {
if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil {
clues.Wrap(err, "reading s3 configs from corso config file")
}
if b, ok := overrides[Bucket]; ok {
overrides[Bucket] = common.NormalizeBucket(b)
}
if p, ok := overrides[Prefix]; ok {
overrides[Prefix] = common.NormalizePrefix(p)
}
if matchFromConfig {
providerType := vpr.GetString(StorageProviderTypeKey)
if providerType != storage.ProviderS3.String() {
return nil, clues.New("unsupported storage provider" + providerType)
}
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
return nil, clues.Wrap(err, "verifying s3 configs in corso config file")
}
}
}
if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
return nil, clues.Wrap(err, "reading s3 configs from corso config file")
}
s3Overrides(overrides)
aws := credentials.GetAWS(overrides)
if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
aws = credentials.AWS{
AccessKey: s3Cfg.AccessKey,
SecretKey: s3Cfg.SecretKey,
SessionToken: s3Cfg.SessionToken,
}
err = nil
}
if err != nil {
return nil, clues.Wrap(err, "validating aws credentials")
}
}
s3Cfg = S3Config{
AWS: aws,
Bucket: str.First(overrides[Bucket], s3Cfg.Bucket),
Endpoint: str.First(overrides[Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"),
Prefix: str.First(overrides[Prefix], s3Cfg.Prefix),
DoNotUseTLS: str.ParseBool(str.First(
overrides[DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS),
"false",
)),
DoNotVerifyTLS: str.ParseBool(str.First(
overrides[DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
"false",
)),
}
return s3Cfg, s3Cfg.validate()
}
// config key consts
const (
keyS3AccessKey = "s3_access_key"
keyS3Bucket = "s3_bucket"
keyS3Endpoint = "s3_endpoint"
keyS3Prefix = "s3_prefix"
keyS3SecretKey = "s3_secret_key"
keyS3SessionToken = "s3_session_token"
keyS3DoNotUseTLS = "s3_donotusetls"
keyS3DoNotVerifyTLS = "s3_donotverifytls"
)
// config exported name consts
// TODO: Move these to storage?
const (
Bucket = "bucket"
Endpoint = "endpoint"
Prefix = "prefix"
DoNotUseTLS = "donotusetls"
DoNotVerifyTLS = "donotverifytls"
)
func normalize(c S3Config) S3Config {
return S3Config{
Bucket: common.NormalizeBucket(c.Bucket),
Endpoint: c.Endpoint,
Prefix: common.NormalizePrefix(c.Prefix),
DoNotUseTLS: c.DoNotUseTLS,
DoNotVerifyTLS: c.DoNotVerifyTLS,
}
}
// StringConfig transforms a s3Config struct into a plain
// map[string]string. All values in the original struct which
// serialize into the map are expected to be strings.
func (c S3Config) StringConfig() (map[string]string, error) {
cn := normalize(c)
cfg := map[string]string{
keyS3AccessKey: c.AccessKey,
keyS3Bucket: cn.Bucket,
keyS3Endpoint: cn.Endpoint,
keyS3Prefix: cn.Prefix,
keyS3SecretKey: c.SecretKey,
keyS3SessionToken: c.SessionToken,
keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS),
keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
}
return cfg, c.validate()
}
func (c S3Config) WriteConfigToViper(vpr *viper.Viper) {
cn := normalize(c)
vpr.Set(StorageProviderTypeKey, storage.ProviderS3.String())
vpr.Set(BucketNameKey, cn.Bucket)
vpr.Set(EndpointKey, cn.Endpoint)
vpr.Set(PrefixKey, cn.Prefix)
vpr.Set(DisableTLSKey, cn.DoNotUseTLS)
vpr.Set(DisableTLSVerificationKey, cn.DoNotVerifyTLS)
}
// storage parsing errors
var (
errMissingRequired = clues.New("missing required storage configuration")
)
func (c S3Config) validate() error {
check := map[string]string{
Bucket: c.Bucket,
}
for k, v := range check {
if len(v) == 0 {
return clues.Stack(errMissingRequired, clues.New(k))
}
}
return nil
}

153
src/cli/config/storage.go
View File

@ -1,54 +1,19 @@
package config
import (
"fmt"
"os"
"path/filepath"
"strconv"
"github.com/alcionai/clues"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/spf13/viper"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
var s3Config storage.S3Config
s3Config.Bucket = vpr.GetString(BucketNameKey)
s3Config.Endpoint = vpr.GetString(EndpointKey)
s3Config.Prefix = vpr.GetString(PrefixKey)
s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
return s3Config, nil
}
// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Config, error) {
s3Config.AccessKey = vpr.GetString(AccessKey)
s3Config.SecretKey = vpr.GetString(SecretAccessKey)
s3Config.SessionToken = vpr.GetString(SessionToken)
return s3Config, nil
}
func s3Overrides(in map[string]string) map[string]string {
return map[string]string{
storage.Bucket: in[storage.Bucket],
storage.Endpoint: in[storage.Endpoint],
storage.Prefix: in[storage.Prefix],
storage.DoNotUseTLS: in[storage.DoNotUseTLS],
storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS],
StorageProviderTypeKey: in[StorageProviderTypeKey],
}
}
// configureStorage builds a complete storage configuration from a mix of
// viper properties and manual overrides.
func configureStorage(
@ -58,74 +23,19 @@ func configureStorage(
overrides map[string]string,
) (storage.Storage, error) {
var (
s3Cfg storage.S3Config
store storage.Storage
err error
store storage.Storage
err error
provider = storage.ProviderS3 // temporary
)
if readConfigFromViper {
if s3Cfg, err = s3ConfigsFromViper(vpr); err != nil {
return store, clues.Wrap(err, "reading s3 configs from corso config file")
}
if b, ok := overrides[storage.Bucket]; ok {
overrides[storage.Bucket] = common.NormalizeBucket(b)
}
if p, ok := overrides[storage.Prefix]; ok {
overrides[storage.Prefix] = common.NormalizePrefix(p)
}
if matchFromConfig {
providerType := vpr.GetString(StorageProviderTypeKey)
if providerType != storage.ProviderS3.String() {
return store, clues.New("unsupported storage provider: " + providerType)
}
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
return store, clues.Wrap(err, "verifying s3 configs in corso config file")
}
}
}
if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
return store, clues.Wrap(err, "reading s3 configs from corso config file")
}
s3Overrides(overrides)
aws := credentials.GetAWS(overrides)
if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
aws = credentials.AWS{
AccessKey: s3Cfg.AccessKey,
SecretKey: s3Cfg.SecretKey,
SessionToken: s3Cfg.SessionToken,
}
err = nil
}
if err != nil {
return store, clues.Wrap(err, "validating aws credentials")
}
}
s3Cfg = storage.S3Config{
AWS: aws,
Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket),
Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"),
Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix),
DoNotUseTLS: str.ParseBool(str.First(
overrides[storage.DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS),
"false",
)),
DoNotVerifyTLS: str.ParseBool(str.First(
overrides[storage.DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
"false",
)),
storageCfg, err := fetchStorageConfigFromViper(
vpr,
provider, // Make it generic
readConfigFromViper,
matchFromConfig,
overrides)
if err != nil {
return store, err
}
// compose the common config and credentials
@ -147,14 +57,13 @@ func configureStorage(
// ensure required properties are present
if err := requireProps(map[string]string{
storage.Bucket: s3Cfg.Bucket,
credentials.CorsoPassphrase: corso.CorsoPassphrase,
}); err != nil {
return storage.Storage{}, err
}
// build the storage
store, err = storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg)
store, err = storage.NewStorage(provider, storageCfg, cCfg)
if err != nil {
return store, clues.Wrap(err, "configuring repository storage")
}
@ -162,6 +71,25 @@ func configureStorage(
return store, nil
}
func fetchStorageConfigFromViper(
vpr *viper.Viper,
p storage.StorageProvider,
readConfigFromViper bool,
matchFromConfig bool,
overrides map[string]string,
) (StorageConfigurer, error) {
switch p {
case storage.ProviderS3:
return fetchS3ConfigFromViper(
vpr,
readConfigFromViper,
matchFromConfig,
overrides)
}
return nil, clues.New("unsupported storage provider")
}
// GetCorso is a helper for aggregating Corso secrets and credentials.
func GetAndInsertCorso(passphase string) credentials.Corso {
// fetch data from flag, env var or func param giving priority to func param
@ -172,3 +100,20 @@ func GetAndInsertCorso(passphase string) credentials.Corso {
CorsoPassphrase: corsoPassph,
}
}
// Helper for parsing the values in a config object.
// If the value is nil or not a string, returns an empty string.
func orEmptyString(v any) string {
defer func() {
r := recover()
if r != nil {
fmt.Printf("panic recovery casting %v to string\n", v)
}
}()
if v == nil {
return ""
}
return v.(string)
}

27
src/cli/repo/s3.go
View File

@ -128,6 +128,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
// s3 values from flags
s3Override := S3Overrides(cmd)
// Need to send provider here
cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
if err != nil {
return Only(ctx, err)
@ -149,11 +150,17 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
cfg.Account.ID(),
opt)
s3Cfg, err := cfg.Storage.S3Config()
storageCfg, err := config.NewStorageConfigFrom(cfg.Storage)
if err != nil {
return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
}
s3Cfg, ok := storageCfg.(config.S3Config)
if !ok {
return Only(ctx, clues.New("Casting storage config to S3Config"))
}
// TODO: move this to cfg validate
if strings.HasPrefix(s3Cfg.Endpoint, "http://") || strings.HasPrefix(s3Cfg.Endpoint, "https://") {
invalidEndpointErr := "endpoint doesn't support specifying protocol. " +
"pass --disable-tls flag to use http:// instead of default https://"
@ -214,6 +221,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
// s3 values from flags
s3Override := S3Overrides(cmd)
// Send provider here
cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
if err != nil {
return Only(ctx, err)
@ -224,11 +232,16 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
repoID = events.RepoIDNotFound
}
s3Cfg, err := cfg.Storage.S3Config()
storageCfg, err := config.NewStorageConfigFrom(cfg.Storage)
if err != nil {
return Only(ctx, clues.Wrap(err, "Retrieving s3 configuration"))
}
s3Cfg, ok := storageCfg.(config.S3Config)
if !ok {
return Only(ctx, clues.New("Casting storage config to S3Config"))
}
m365, err := cfg.Account.M365Config()
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
@ -287,23 +300,23 @@ func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string {
}
if _, ok := flagset[bucketFN]; ok {
s3Overrides[storage.Bucket] = bucket
s3Overrides[config.Bucket] = bucket
}
if _, ok := flagset[prefixFN]; ok {
s3Overrides[storage.Prefix] = prefix
s3Overrides[config.Prefix] = prefix
}
if _, ok := flagset[doNotUseTLSFN]; ok {
s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS)
s3Overrides[config.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS)
}
if _, ok := flagset[doNotVerifyTLSFN]; ok {
s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS)
s3Overrides[config.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS)
}
if _, ok := flagset[endpointFN]; ok {
s3Overrides[storage.Endpoint] = endpoint
s3Overrides[config.Endpoint] = endpoint
}
return s3Overrides

4
src/cli/utils/utils.go
View File

@ -63,7 +63,7 @@ func AccountConnectAndWriteRepoConfig(
return nil, nil, err
}
s3Config, err := stg.S3Config()
storageConfig, err := config.NewStorageConfigFrom(*stg)
if err != nil {
logger.CtxErr(ctx, err).Info("getting storage configuration")
return nil, nil, err
@ -77,7 +77,7 @@ func AccountConnectAndWriteRepoConfig(
// repo config gets set during repo connect and init.
// This call confirms we have the correct values.
err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID())
err = config.WriteRepoConfig(ctx, storageConfig, m365Config, opts.Repo, r.GetID())
if err != nil {
logger.CtxErr(ctx, err).Info("writing to repository configuration")
return nil, nil, err

8
src/internal/kopia/s3.go
View File

@ -7,6 +7,7 @@ import (
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/s3"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -20,11 +21,16 @@ func s3BlobStorage(
repoOpts repository.Options,
s storage.Storage,
) (blob.Storage, error) {
cfg, err := s.S3Config()
sCfg, err := config.NewStorageConfigFrom(s)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx)
}
cfg, ok := sCfg.(config.S3Config)
if !ok {
return nil, clues.New("casting storage config to S3Config")
}
endpoint := defaultS3Endpoint
if len(cfg.Endpoint) > 0 {
endpoint = cfg.Endpoint

25
src/pkg/storage/common.go
View File

@ -1,11 +1,14 @@
package storage
import (
"fmt"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/credentials"
)
// Move this to config
type CommonConfig struct {
credentials.Corso // requires: CorsoPassphrase
@ -42,6 +45,11 @@ func (s Storage) CommonConfig() (CommonConfig, error) {
return c, c.validate()
}
// storage parsing errors
var (
errMissingRequired = clues.New("missing required storage configuration")
)
// ensures all required properties are present
func (c CommonConfig) validate() error {
if len(c.CorsoPassphrase) == 0 {
@ -51,3 +59,20 @@ func (c CommonConfig) validate() error {
// kopiaCfgFilePath is not required
return nil
}
// Helper for parsing the values in a config object.
// If the value is nil or not a string, returns an empty string.
func orEmptyString(v any) string {
defer func() {
r := recover()
if r != nil {
fmt.Printf("panic recovery casting %v to string\n", v)
}
}()
if v == nil {
return ""
}
return v.(string)
}

102
src/pkg/storage/s3.go
View File

@ -1,102 +0,0 @@
package storage
import (
"strconv"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
)
type S3Config struct {
credentials.AWS
Bucket string // required
Endpoint string
Prefix string
DoNotUseTLS bool
DoNotVerifyTLS bool
}
// config key consts
const (
keyS3AccessKey = "s3_access_key"
keyS3Bucket = "s3_bucket"
keyS3Endpoint = "s3_endpoint"
keyS3Prefix = "s3_prefix"
keyS3SecretKey = "s3_secret_key"
keyS3SessionToken = "s3_session_token"
keyS3DoNotUseTLS = "s3_donotusetls"
keyS3DoNotVerifyTLS = "s3_donotverifytls"
)
// config exported name consts
const (
Bucket = "bucket"
Endpoint = "endpoint"
Prefix = "prefix"
DoNotUseTLS = "donotusetls"
DoNotVerifyTLS = "donotverifytls"
)
func (c S3Config) Normalize() S3Config {
return S3Config{
Bucket: common.NormalizeBucket(c.Bucket),
Endpoint: c.Endpoint,
Prefix: common.NormalizePrefix(c.Prefix),
DoNotUseTLS: c.DoNotUseTLS,
DoNotVerifyTLS: c.DoNotVerifyTLS,
}
}
// StringConfig transforms a s3Config struct into a plain
// map[string]string. All values in the original struct which
// serialize into the map are expected to be strings.
func (c S3Config) StringConfig() (map[string]string, error) {
cn := c.Normalize()
cfg := map[string]string{
keyS3AccessKey: c.AccessKey,
keyS3Bucket: cn.Bucket,
keyS3Endpoint: cn.Endpoint,
keyS3Prefix: cn.Prefix,
keyS3SecretKey: c.SecretKey,
keyS3SessionToken: c.SessionToken,
keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS),
keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
}
return cfg, c.validate()
}
// S3Config retrieves the S3Config details from the Storage config.
func (s Storage) S3Config() (S3Config, error) {
c := S3Config{}
if len(s.Config) > 0 {
c.AccessKey = orEmptyString(s.Config[keyS3AccessKey])
c.SecretKey = orEmptyString(s.Config[keyS3SecretKey])
c.SessionToken = orEmptyString(s.Config[keyS3SessionToken])
c.Bucket = orEmptyString(s.Config[keyS3Bucket])
c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
c.Prefix = orEmptyString(s.Config[keyS3Prefix])
c.DoNotUseTLS = str.ParseBool(s.Config[keyS3DoNotUseTLS])
c.DoNotVerifyTLS = str.ParseBool(s.Config[keyS3DoNotVerifyTLS])
}
return c, c.validate()
}
func (c S3Config) validate() error {
check := map[string]string{
Bucket: c.Bucket,
}
for k, v := range check {
if len(v) == 0 {
return clues.Stack(errMissingRequired, clues.New(k))
}
}
return nil
}

40
src/pkg/storage/storage.go
View File

@ -1,30 +1,21 @@
package storage
import (
"fmt"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
)
type storageProvider int
type StorageProvider int
//go:generate stringer -type=storageProvider -linecomment
//go:generate stringer -type=StorageProvider -linecomment
const (
ProviderUnknown storageProvider = 0 // Unknown Provider
ProviderS3 storageProvider = 1 // S3
)
// storage parsing errors
var (
errMissingRequired = clues.New("missing required storage configuration")
ProviderUnknown StorageProvider = 0 // Unknown Provider
ProviderS3 StorageProvider = 1 // S3
)
// Storage defines a storage provider, along with any configuration
// required to set up or communicate with that provider.
type Storage struct {
Provider storageProvider
Provider StorageProvider
Config map[string]string
// TODO: These are AWS S3 specific -> move these out
SessionTags map[string]string
@ -34,7 +25,7 @@ type Storage struct {
}
// NewStorage aggregates all the supplied configurations into a single configuration.
func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, error) {
func NewStorage(p StorageProvider, cfgs ...common.StringConfigurer) (Storage, error) {
cs, err := common.UnionStringConfigs(cfgs...)
return Storage{
@ -46,7 +37,7 @@ func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, er
// NewStorageUsingRole supports specifying an AWS IAM role the storage provider
// should assume.
func NewStorageUsingRole(
p storageProvider,
p StorageProvider,
roleARN string,
sessionName string,
sessionTags map[string]string,
@ -64,20 +55,3 @@ func NewStorageUsingRole(
SessionDuration: duration,
}, err
}
// Helper for parsing the values in a config object.
// If the value is nil or not a string, returns an empty string.
func orEmptyString(v any) string {
defer func() {
r := recover()
if r != nil {
fmt.Printf("panic recovery casting %v to string\n", v)
}
}()
if v == nil {
return ""
}
return v.(string)
}

14
src/pkg/storage/storageprovider_string.go
View File

@ -1,4 +1,4 @@
// Code generated by "stringer -type=storageProvider -linecomment"; DO NOT EDIT.
// Code generated by "stringer -type=StorageProvider -linecomment"; DO NOT EDIT.
package storage
@ -12,13 +12,13 @@ func _() {
_ = x[ProviderS3-1]
}
const _storageProvider_name = "Unknown ProviderS3"
const _StorageProvider_name = "Unknown ProviderS3"
var _storageProvider_index = [...]uint8{0, 16, 18}
var _StorageProvider_index = [...]uint8{0, 16, 18}
func (i storageProvider) String() string {
if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) {
return "storageProvider(" + strconv.FormatInt(int64(i), 10) + ")"
func (i StorageProvider) String() string {
if i < 0 || i >= StorageProvider(len(_StorageProvider_index)-1) {
return "StorageProvider(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _storageProvider_name[_storageProvider_index[i]:_storageProvider_index[i+1]]
return _StorageProvider_name[_StorageProvider_index[i]:_StorageProvider_index[i+1]]
}