diff --git a/src/cli/config/config.go b/src/cli/config/config.go index a7a141255..297b7afc3 100644 --- a/src/cli/config/config.go +++ b/src/cli/config/config.go @@ -18,10 +18,12 @@ import ( const ( // S3 config - StorageProviderTypeKey = "provider" - BucketNameKey = "bucket" - EndpointKey = "endpoint" - PrefixKey = "prefix" + StorageProviderTypeKey = "provider" + BucketNameKey = "bucket" + EndpointKey = "endpoint" + PrefixKey = "prefix" + DisableTLSKey = "disable_tls" + DisableTLSVerificationKey = "disable_tls_verification" // M365 config AccountProviderTypeKey = "account_provider" @@ -32,7 +34,6 @@ var ( configFilePath string configFilePathFlag string configDir string - defaultDir string displayDefaultFP = filepath.Join("$HOME", ".corso.toml") ) @@ -54,8 +55,6 @@ func init() { Infof(context.Background(), "cannot stat user's $HOME directory: %v", err) } - defaultDir = homeDir - if len(configDir) == 0 { configDir = homeDir configFilePath = filepath.Join(configDir, ".corso.toml") @@ -195,6 +194,8 @@ func writeRepoConfigWithViper(vpr *viper.Viper, s3Config storage.S3Config, m365C vpr.Set(BucketNameKey, s3Config.Bucket) vpr.Set(EndpointKey, s3Config.Endpoint) vpr.Set(PrefixKey, s3Config.Prefix) + vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS) + vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS) vpr.Set(AccountProviderTypeKey, account.ProviderM365.String()) vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID) diff --git a/src/cli/config/config_test.go b/src/cli/config/config_test.go index d8f5371f4..b2ffaf761 100644 --- a/src/cli/config/config_test.go +++ b/src/cli/config/config_test.go @@ -26,6 +26,8 @@ const ( ` + StorageProviderTypeKey + ` = 'S3' ` + AccountProviderTypeKey + ` = 'M365' ` + AzureTenantIDKey + ` = '%s' +` + DisableTLSKey + ` = 'false' +` + DisableTLSVerificationKey + ` = 'false' ` ) @@ -84,7 +86,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() { testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml") require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config") - s3Cfg := storage.S3Config{Bucket: bkt} + s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true} m365 := account.M365Config{AzureTenantID: tid} require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config") @@ -93,6 +95,8 @@ func (suite *ConfigSuite) TestWriteReadConfig() { readS3Cfg, err := s3ConfigsFromViper(vpr) require.NoError(t, err) assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket) + assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS) + assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS) readM365, err := m365ConfigsFromViper(vpr) require.NoError(t, err) @@ -217,9 +221,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() { require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config") s3Cfg := storage.S3Config{ - Bucket: bkt, - Endpoint: end, - Prefix: pfx, + Bucket: bkt, + Endpoint: end, + Prefix: pfx, + DoNotVerifyTLS: true, + DoNotUseTLS: true, } m365 := account.M365Config{AzureTenantID: tid} @@ -234,6 +240,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() { assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket) assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint) assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix) + assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS) + assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS) common, err := st.CommonConfig() require.NoError(t, err, "reading common config from storage") @@ -257,12 +265,6 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1" ) - // Configure viper to read test config file - s3Cfg := storage.S3Config{ - Bucket: bkt, - Endpoint: end, - Prefix: pfx, - } m365 := account.M365Config{AzureTenantID: tid} overrides := map[string]string{ @@ -271,6 +273,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride storage.Bucket: bkt, storage.Endpoint: end, storage.Prefix: pfx, + storage.DoNotUseTLS: "true", + storage.DoNotVerifyTLS: "true", StorageProviderTypeKey: storage.ProviderS3.String(), } @@ -279,9 +283,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride readS3Cfg, err := st.S3Config() require.NoError(t, err, "reading s3 config from storage") - assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket) - assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint) - assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix) + assert.Equal(t, readS3Cfg.Bucket, bkt) + assert.Equal(t, readS3Cfg.Endpoint, end) + assert.Equal(t, readS3Cfg.Prefix, pfx) + assert.True(t, readS3Cfg.DoNotUseTLS) + assert.True(t, readS3Cfg.DoNotVerifyTLS) common, err := st.CommonConfig() require.NoError(t, err, "reading common config from storage") diff --git a/src/cli/config/storage.go b/src/cli/config/storage.go index ab3a2807b..5049d5a07 100644 --- a/src/cli/config/storage.go +++ b/src/cli/config/storage.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strconv" "github.com/aws/aws-sdk-go/aws/defaults" "github.com/pkg/errors" @@ -26,6 +27,8 @@ func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) { s3Config.Bucket = vpr.GetString(BucketNameKey) s3Config.Endpoint = vpr.GetString(EndpointKey) s3Config.Prefix = vpr.GetString(PrefixKey) + s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey) + s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey) return s3Config, nil } @@ -35,6 +38,8 @@ func s3Overrides(in map[string]string) map[string]string { storage.Bucket: in[storage.Bucket], storage.Endpoint: in[storage.Endpoint], storage.Prefix: in[storage.Prefix], + storage.DoNotUseTLS: in[storage.DoNotUseTLS], + storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS], StorageProviderTypeKey: in[StorageProviderTypeKey], } } @@ -79,6 +84,14 @@ func configureStorage( Bucket: common.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)), Endpoint: common.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)), Prefix: common.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)), + DoNotUseTLS: common.ParseBool(common.First( + overrides[storage.DoNotUseTLS], + strconv.FormatBool(s3Cfg.DoNotUseTLS), + os.Getenv(storage.PrefixKey))), + DoNotVerifyTLS: common.ParseBool(common.First( + overrides[storage.DoNotVerifyTLS], + strconv.FormatBool(s3Cfg.DoNotVerifyTLS), + os.Getenv(storage.PrefixKey))), } // compose the common config and credentials diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index e906c921e..6cbe44c13 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -1,6 +1,8 @@ package repo import ( + "strconv" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -19,6 +21,8 @@ var ( bucket string endpoint string prefix string + doNotUseTLS bool + doNotVerifyTLS bool succeedIfExists bool ) @@ -45,6 +49,8 @@ func addS3Commands(parent *cobra.Command) *cobra.Command { cobra.CheckErr(c.MarkFlagRequired("bucket")) fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.") fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.") + fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)") + fs.BoolVar(&doNotVerifyTLS, "disable-tls-verification", false, "Disable TLS (HTTPS) certificate verification.") // In general, we don't want to expose this flag to users and have them mistake it // for a broad-scale idempotency solution. We can un-hide it later the need arises. @@ -200,5 +206,7 @@ func s3Overrides() map[string]string { storage.Bucket: bucket, storage.Endpoint: endpoint, storage.Prefix: prefix, + storage.DoNotUseTLS: strconv.FormatBool(doNotUseTLS), + storage.DoNotVerifyTLS: strconv.FormatBool(doNotVerifyTLS), } } diff --git a/src/internal/common/slices.go b/src/internal/common/slices.go index 7600400e9..2a9e02003 100644 --- a/src/internal/common/slices.go +++ b/src/internal/common/slices.go @@ -1,5 +1,7 @@ package common +import "strconv" + func ContainsString(super []string, sub string) bool { for _, s := range super { if s == sub { @@ -20,3 +22,14 @@ func First(vs ...string) string { return "" } + +// parseBool returns the bool value represented by the string +// or false on error +func ParseBool(v string) bool { + s, err := strconv.ParseBool(v) + if err != nil { + return false + } + + return s +} diff --git a/src/internal/kopia/s3.go b/src/internal/kopia/s3.go index 833fa0da9..d25dec610 100644 --- a/src/internal/kopia/s3.go +++ b/src/internal/kopia/s3.go @@ -25,9 +25,11 @@ func s3BlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) } opts := s3.Options{ - BucketName: cfg.Bucket, - Endpoint: endpoint, - Prefix: cfg.Prefix, + BucketName: cfg.Bucket, + Endpoint: endpoint, + Prefix: cfg.Prefix, + DoNotUseTLS: cfg.DoNotUseTLS, + DoNotVerifyTLS: cfg.DoNotVerifyTLS, } return s3.New(ctx, &opts) diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index a6a5674c1..67711f421 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -1,36 +1,46 @@ package storage import ( + "strconv" + "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common" ) type S3Config struct { - Bucket string // required - Endpoint string - Prefix string + Bucket string // required + Endpoint string + Prefix string + DoNotUseTLS bool + DoNotVerifyTLS bool } // config key consts const ( - keyS3Bucket = "s3_bucket" - keyS3Endpoint = "s3_endpoint" - keyS3Prefix = "s3_prefix" + keyS3Bucket = "s3_bucket" + keyS3Endpoint = "s3_endpoint" + keyS3Prefix = "s3_prefix" + keyS3DoNotUseTLS = "s3_donotusetls" + keyS3DoNotVerifyTLS = "s3_donotverifytls" ) // config exported name consts const ( - Bucket = "bucket" - Endpoint = "endpoint" - Prefix = "prefix" + Bucket = "bucket" + Endpoint = "endpoint" + Prefix = "prefix" + DoNotUseTLS = "donotusetls" + DoNotVerifyTLS = "donotverifytls" ) func (c S3Config) Normalize() S3Config { return S3Config{ - Bucket: common.NormalizeBucket(c.Bucket), - Endpoint: c.Endpoint, - Prefix: common.NormalizePrefix(c.Prefix), + Bucket: common.NormalizeBucket(c.Bucket), + Endpoint: c.Endpoint, + Prefix: common.NormalizePrefix(c.Prefix), + DoNotUseTLS: c.DoNotUseTLS, + DoNotVerifyTLS: c.DoNotVerifyTLS, } } @@ -40,9 +50,11 @@ func (c S3Config) Normalize() S3Config { func (c S3Config) StringConfig() (map[string]string, error) { cn := c.Normalize() cfg := map[string]string{ - keyS3Bucket: cn.Bucket, - keyS3Endpoint: cn.Endpoint, - keyS3Prefix: cn.Prefix, + keyS3Bucket: cn.Bucket, + keyS3Endpoint: cn.Endpoint, + keyS3Prefix: cn.Prefix, + keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS), + keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS), } return cfg, c.validate() @@ -56,6 +68,8 @@ func (s Storage) S3Config() (S3Config, error) { c.Bucket = orEmptyString(s.Config[keyS3Bucket]) c.Endpoint = orEmptyString(s.Config[keyS3Endpoint]) c.Prefix = orEmptyString(s.Config[keyS3Prefix]) + c.DoNotUseTLS = common.ParseBool(s.Config[keyS3DoNotUseTLS]) + c.DoNotVerifyTLS = common.ParseBool(s.Config[keyS3DoNotVerifyTLS]) } return c, c.validate() diff --git a/src/pkg/storage/s3_test.go b/src/pkg/storage/s3_test.go index 429196298..1e95cf72b 100644 --- a/src/pkg/storage/s3_test.go +++ b/src/pkg/storage/s3_test.go @@ -1,4 +1,4 @@ -package storage_test +package storage import ( "testing" @@ -6,8 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/pkg/storage" ) type S3CfgSuite struct { @@ -19,16 +17,20 @@ func TestS3CfgSuite(t *testing.T) { } var ( - goodS3Config = storage.S3Config{ - Bucket: "bkt", - Endpoint: "end", - Prefix: "pre/", + goodS3Config = S3Config{ + Bucket: "bkt", + Endpoint: "end", + Prefix: "pre/", + DoNotUseTLS: false, + DoNotVerifyTLS: false, } goodS3Map = map[string]string{ - "s3_bucket": "bkt", - "s3_endpoint": "end", - "s3_prefix": "pre/", + keyS3Bucket: "bkt", + keyS3Endpoint: "end", + keyS3Prefix: "pre/", + keyS3DoNotUseTLS: "false", + keyS3DoNotVerifyTLS: "false", } ) @@ -54,7 +56,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config() { t := suite.T() in := goodS3Config - s, err := storage.NewStorage(storage.ProviderS3, in) + s, err := NewStorage(ProviderS3, in) assert.NoError(t, err) out, err := s.S3Config() assert.NoError(t, err) @@ -64,8 +66,8 @@ func (suite *S3CfgSuite) TestStorage_S3Config() { assert.Equal(t, in.Prefix, out.Prefix) } -func makeTestS3Cfg(bkt, end, pre string) storage.S3Config { - return storage.S3Config{ +func makeTestS3Cfg(bkt, end, pre string) S3Config { + return S3Config{ Bucket: bkt, Endpoint: end, Prefix: pre, @@ -76,13 +78,13 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() { // missing required properties table := []struct { name string - cfg storage.S3Config + cfg S3Config }{ {"missing bucket", makeTestS3Cfg("", "end", "pre/")}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - _, err := storage.NewStorage(storage.ProviderUnknown, test.cfg) + _, err := NewStorage(ProviderUnknown, test.cfg) assert.Error(t, err) }) } @@ -90,18 +92,18 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() { // required property not populated in storage table2 := []struct { name string - amend func(storage.Storage) + amend func(Storage) }{ { "missing bucket", - func(s storage.Storage) { + func(s Storage) { s.Config["s3_bucket"] = "" }, }, } for _, test := range table2 { suite.T().Run(test.name, func(t *testing.T) { - st, err := storage.NewStorage(storage.ProviderUnknown, goodS3Config) + st, err := NewStorage(ProviderUnknown, goodS3Config) assert.NoError(t, err) test.amend(st) _, err = st.S3Config() @@ -113,7 +115,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() { func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() { table := []struct { name string - input storage.S3Config + input S3Config expect map[string]string }{ { @@ -126,6 +128,23 @@ func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() { input: makeTestS3Cfg("s3://"+goodS3Config.Bucket, goodS3Config.Endpoint, goodS3Config.Prefix), expect: goodS3Map, }, + { + name: "disabletls", + input: S3Config{ + Bucket: "bkt", + Endpoint: "end", + Prefix: "pre/", + DoNotUseTLS: true, + DoNotVerifyTLS: true, + }, + expect: map[string]string{ + keyS3Bucket: "bkt", + keyS3Endpoint: "end", + keyS3Prefix: "pre/", + keyS3DoNotUseTLS: "true", + keyS3DoNotVerifyTLS: "true", + }, + }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { @@ -142,7 +161,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_Normalize() { normalBkt = "bkt" ) - st := storage.S3Config{ + st := S3Config{ Bucket: prefixedBkt, } diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index 62436633f..029e29596 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -23,9 +23,11 @@ var ( // envvar consts // TODO: Remove these and leverage Viper AutomaticEnv() instead const ( - BucketKey = "BUCKET" - EndpointKey = "ENDPOINT" - PrefixKey = "PREFIX" + BucketKey = "BUCKET" + EndpointKey = "ENDPOINT" + PrefixKey = "PREFIX" + DisableTLSKey = "DISABLE_TLS" + DisableTLSVerificationKey = "DISABLE_TLS_VERIFICATION" ) // Storage defines a storage provider, along with any configuration