Allow disabling TLS and TLS verification (#1417)

## Description

Introduces config options (`--disable-tls` and `--disable-tls-verification`) to turn off TLS and TLS verification

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #1415 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

src/cli/config/config.go

@@ -22,6 +22,8 @@ const (
 	BucketNameKey             = "bucket"
 	EndpointKey               = "endpoint"
 	PrefixKey                 = "prefix"
+	DisableTLSKey             = "disable_tls"
+	DisableTLSVerificationKey = "disable_tls_verification"
 
 	// M365 config
 	AccountProviderTypeKey = "account_provider"
@@ -32,7 +34,6 @@ var (
 	configFilePath     string
 	configFilePathFlag string
 	configDir          string
-	defaultDir         string
 	displayDefaultFP   = filepath.Join("$HOME", ".corso.toml")
 )
 
@@ -54,8 +55,6 @@ func init() {
 		Infof(context.Background(), "cannot stat user's $HOME directory: %v", err)
 	}
 
-	defaultDir = homeDir
-
 	if len(configDir) == 0 {
 		configDir = homeDir
 		configFilePath = filepath.Join(configDir, ".corso.toml")
@@ -195,6 +194,8 @@ func writeRepoConfigWithViper(vpr *viper.Viper, s3Config storage.S3Config, m365C
 	vpr.Set(BucketNameKey, s3Config.Bucket)
 	vpr.Set(EndpointKey, s3Config.Endpoint)
 	vpr.Set(PrefixKey, s3Config.Prefix)
+	vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS)
+	vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
 
 	vpr.Set(AccountProviderTypeKey, account.ProviderM365.String())
 	vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID)

src/cli/config/config_test.go

@@ -26,6 +26,8 @@ const (
 ` + StorageProviderTypeKey + ` = 'S3'
 ` + AccountProviderTypeKey + ` = 'M365'
 ` + AzureTenantIDKey + ` = '%s'
+` + DisableTLSKey + ` = 'false'
+` + DisableTLSVerificationKey + ` = 'false'
 `
 )
 
@@ -84,7 +86,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
 	require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
 
-	s3Cfg := storage.S3Config{Bucket: bkt}
+	s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
 	m365 := account.M365Config{AzureTenantID: tid}
 
 	require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
@@ -93,6 +95,8 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	readS3Cfg, err := s3ConfigsFromViper(vpr)
 	require.NoError(t, err)
 	assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
+	assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
+	assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
 
 	readM365, err := m365ConfigsFromViper(vpr)
 	require.NoError(t, err)
@@ -220,6 +224,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
 		Bucket:         bkt,
 		Endpoint:       end,
 		Prefix:         pfx,
+		DoNotVerifyTLS: true,
+		DoNotUseTLS:    true,
 	}
 	m365 := account.M365Config{AzureTenantID: tid}
 
@@ -234,6 +240,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
 	assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
 	assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
 	assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
+	assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
+	assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
 
 	common, err := st.CommonConfig()
 	require.NoError(t, err, "reading common config from storage")
@@ -257,12 +265,6 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
 		tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1"
 	)
 
-	// Configure viper to read test config file
-	s3Cfg := storage.S3Config{
-		Bucket:   bkt,
-		Endpoint: end,
-		Prefix:   pfx,
-	}
 	m365 := account.M365Config{AzureTenantID: tid}
 
 	overrides := map[string]string{
@@ -271,6 +273,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
 		storage.Bucket:         bkt,
 		storage.Endpoint:       end,
 		storage.Prefix:         pfx,
+		storage.DoNotUseTLS:    "true",
+		storage.DoNotVerifyTLS: "true",
 		StorageProviderTypeKey: storage.ProviderS3.String(),
 	}
 
@@ -279,9 +283,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
 
 	readS3Cfg, err := st.S3Config()
 	require.NoError(t, err, "reading s3 config from storage")
-	assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
+	assert.Equal(t, readS3Cfg.Bucket, bkt)
-	assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
+	assert.Equal(t, readS3Cfg.Endpoint, end)
-	assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
+	assert.Equal(t, readS3Cfg.Prefix, pfx)
+	assert.True(t, readS3Cfg.DoNotUseTLS)
+	assert.True(t, readS3Cfg.DoNotVerifyTLS)
 
 	common, err := st.CommonConfig()
 	require.NoError(t, err, "reading common config from storage")

src/cli/config/storage.go

@@ -3,6 +3,7 @@ package config
 import (
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/pkg/errors"
@@ -26,6 +27,8 @@ func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
 	s3Config.Bucket = vpr.GetString(BucketNameKey)
 	s3Config.Endpoint = vpr.GetString(EndpointKey)
 	s3Config.Prefix = vpr.GetString(PrefixKey)
+	s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
+	s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
 
 	return s3Config, nil
 }
@@ -35,6 +38,8 @@ func s3Overrides(in map[string]string) map[string]string {
 		storage.Bucket:         in[storage.Bucket],
 		storage.Endpoint:       in[storage.Endpoint],
 		storage.Prefix:         in[storage.Prefix],
+		storage.DoNotUseTLS:    in[storage.DoNotUseTLS],
+		storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS],
 		StorageProviderTypeKey: in[StorageProviderTypeKey],
 	}
 }
@@ -79,6 +84,14 @@ func configureStorage(
 		Bucket:   common.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)),
 		Endpoint: common.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)),
 		Prefix:   common.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)),
+		DoNotUseTLS: common.ParseBool(common.First(
+			overrides[storage.DoNotUseTLS],
+			strconv.FormatBool(s3Cfg.DoNotUseTLS),
+			os.Getenv(storage.PrefixKey))),
+		DoNotVerifyTLS: common.ParseBool(common.First(
+			overrides[storage.DoNotVerifyTLS],
+			strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
+			os.Getenv(storage.PrefixKey))),
 	}
 
 	// compose the common config and credentials

src/cli/repo/s3.go

@@ -1,6 +1,8 @@
 package repo
 
 import (
+	"strconv"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -19,6 +21,8 @@ var (
 	bucket          string
 	endpoint        string
 	prefix          string
+	doNotUseTLS     bool
+	doNotVerifyTLS  bool
 	succeedIfExists bool
 )
 
@@ -45,6 +49,8 @@ func addS3Commands(parent *cobra.Command) *cobra.Command {
 	cobra.CheckErr(c.MarkFlagRequired("bucket"))
 	fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.")
 	fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.")
+	fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)")
+	fs.BoolVar(&doNotVerifyTLS, "disable-tls-verification", false, "Disable TLS (HTTPS) certificate verification.")
 
 	// In general, we don't want to expose this flag to users and have them mistake it
 	// for a broad-scale idempotency solution.  We can un-hide it later the need arises.
@@ -200,5 +206,7 @@ func s3Overrides() map[string]string {
 		storage.Bucket:                bucket,
 		storage.Endpoint:              endpoint,
 		storage.Prefix:                prefix,
+		storage.DoNotUseTLS:           strconv.FormatBool(doNotUseTLS),
+		storage.DoNotVerifyTLS:        strconv.FormatBool(doNotVerifyTLS),
 	}
 }

src/internal/common/slices.go

@@ -1,5 +1,7 @@
 package common
 
+import "strconv"
+
 func ContainsString(super []string, sub string) bool {
 	for _, s := range super {
 		if s == sub {
@@ -20,3 +22,14 @@ func First(vs ...string) string {
 
 	return ""
 }
+
+// parseBool returns the bool value represented by the string
+// or false on error
+func ParseBool(v string) bool {
+	s, err := strconv.ParseBool(v)
+	if err != nil {
+		return false
+	}
+
+	return s
+}

src/internal/kopia/s3.go

@@ -28,6 +28,8 @@ func s3BlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error)
 		BucketName:     cfg.Bucket,
 		Endpoint:       endpoint,
 		Prefix:         cfg.Prefix,
+		DoNotUseTLS:    cfg.DoNotUseTLS,
+		DoNotVerifyTLS: cfg.DoNotVerifyTLS,
 	}
 
 	return s3.New(ctx, &opts)

src/pkg/storage/s3.go

@@ -1,6 +1,8 @@
 package storage
 
 import (
+	"strconv"
+
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/src/internal/common"
@@ -10,6 +12,8 @@ type S3Config struct {
 	Bucket         string // required
 	Endpoint       string
 	Prefix         string
+	DoNotUseTLS    bool
+	DoNotVerifyTLS bool
 }
 
 // config key consts
@@ -17,6 +21,8 @@ const (
 	keyS3Bucket         = "s3_bucket"
 	keyS3Endpoint       = "s3_endpoint"
 	keyS3Prefix         = "s3_prefix"
+	keyS3DoNotUseTLS    = "s3_donotusetls"
+	keyS3DoNotVerifyTLS = "s3_donotverifytls"
 )
 
 // config exported name consts
@@ -24,6 +30,8 @@ const (
 	Bucket         = "bucket"
 	Endpoint       = "endpoint"
 	Prefix         = "prefix"
+	DoNotUseTLS    = "donotusetls"
+	DoNotVerifyTLS = "donotverifytls"
 )
 
 func (c S3Config) Normalize() S3Config {
@@ -31,6 +39,8 @@ func (c S3Config) Normalize() S3Config {
 		Bucket:         common.NormalizeBucket(c.Bucket),
 		Endpoint:       c.Endpoint,
 		Prefix:         common.NormalizePrefix(c.Prefix),
+		DoNotUseTLS:    c.DoNotUseTLS,
+		DoNotVerifyTLS: c.DoNotVerifyTLS,
 	}
 }
 
@@ -43,6 +53,8 @@ func (c S3Config) StringConfig() (map[string]string, error) {
 		keyS3Bucket:         cn.Bucket,
 		keyS3Endpoint:       cn.Endpoint,
 		keyS3Prefix:         cn.Prefix,
+		keyS3DoNotUseTLS:    strconv.FormatBool(cn.DoNotUseTLS),
+		keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
 	}
 
 	return cfg, c.validate()
@@ -56,6 +68,8 @@ func (s Storage) S3Config() (S3Config, error) {
 		c.Bucket = orEmptyString(s.Config[keyS3Bucket])
 		c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
 		c.Prefix = orEmptyString(s.Config[keyS3Prefix])
+		c.DoNotUseTLS = common.ParseBool(s.Config[keyS3DoNotUseTLS])
+		c.DoNotVerifyTLS = common.ParseBool(s.Config[keyS3DoNotVerifyTLS])
 	}
 
 	return c, c.validate()

src/pkg/storage/s3_test.go

@@ -1,4 +1,4 @@
-package storage_test
+package storage
 
 import (
 	"testing"
@@ -6,8 +6,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
-
-	"github.com/alcionai/corso/src/pkg/storage"
 )
 
 type S3CfgSuite struct {
@@ -19,16 +17,20 @@ func TestS3CfgSuite(t *testing.T) {
 }
 
 var (
-	goodS3Config = storage.S3Config{
+	goodS3Config = S3Config{
 		Bucket:         "bkt",
 		Endpoint:       "end",
 		Prefix:         "pre/",
+		DoNotUseTLS:    false,
+		DoNotVerifyTLS: false,
 	}
 
 	goodS3Map = map[string]string{
-		"s3_bucket":   "bkt",
+		keyS3Bucket:         "bkt",
-		"s3_endpoint": "end",
+		keyS3Endpoint:       "end",
-		"s3_prefix":   "pre/",
+		keyS3Prefix:         "pre/",
+		keyS3DoNotUseTLS:    "false",
+		keyS3DoNotVerifyTLS: "false",
 	}
 )
 
@@ -54,7 +56,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
 	t := suite.T()
 
 	in := goodS3Config
-	s, err := storage.NewStorage(storage.ProviderS3, in)
+	s, err := NewStorage(ProviderS3, in)
 	assert.NoError(t, err)
 	out, err := s.S3Config()
 	assert.NoError(t, err)
@@ -64,8 +66,8 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
 	assert.Equal(t, in.Prefix, out.Prefix)
 }
 
-func makeTestS3Cfg(bkt, end, pre string) storage.S3Config {
+func makeTestS3Cfg(bkt, end, pre string) S3Config {
-	return storage.S3Config{
+	return S3Config{
 		Bucket:   bkt,
 		Endpoint: end,
 		Prefix:   pre,
@@ -76,13 +78,13 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
 	// missing required properties
 	table := []struct {
 		name string
-		cfg  storage.S3Config
+		cfg  S3Config
 	}{
 		{"missing bucket", makeTestS3Cfg("", "end", "pre/")},
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
-			_, err := storage.NewStorage(storage.ProviderUnknown, test.cfg)
+			_, err := NewStorage(ProviderUnknown, test.cfg)
 			assert.Error(t, err)
 		})
 	}
@@ -90,18 +92,18 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
 	// required property not populated in storage
 	table2 := []struct {
 		name  string
-		amend func(storage.Storage)
+		amend func(Storage)
 	}{
 		{
 			"missing bucket",
-			func(s storage.Storage) {
+			func(s Storage) {
 				s.Config["s3_bucket"] = ""
 			},
 		},
 	}
 	for _, test := range table2 {
 		suite.T().Run(test.name, func(t *testing.T) {
-			st, err := storage.NewStorage(storage.ProviderUnknown, goodS3Config)
+			st, err := NewStorage(ProviderUnknown, goodS3Config)
 			assert.NoError(t, err)
 			test.amend(st)
 			_, err = st.S3Config()
@@ -113,7 +115,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
 func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
 	table := []struct {
 		name   string
-		input  storage.S3Config
+		input  S3Config
 		expect map[string]string
 	}{
 		{
@@ -126,6 +128,23 @@ func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
 			input:  makeTestS3Cfg("s3://"+goodS3Config.Bucket, goodS3Config.Endpoint, goodS3Config.Prefix),
 			expect: goodS3Map,
 		},
+		{
+			name: "disabletls",
+			input: S3Config{
+				Bucket:         "bkt",
+				Endpoint:       "end",
+				Prefix:         "pre/",
+				DoNotUseTLS:    true,
+				DoNotVerifyTLS: true,
+			},
+			expect: map[string]string{
+				keyS3Bucket:         "bkt",
+				keyS3Endpoint:       "end",
+				keyS3Prefix:         "pre/",
+				keyS3DoNotUseTLS:    "true",
+				keyS3DoNotVerifyTLS: "true",
+			},
+		},
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
@@ -142,7 +161,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_Normalize() {
 		normalBkt   = "bkt"
 	)
 
-	st := storage.S3Config{
+	st := S3Config{
 		Bucket: prefixedBkt,
 	}
 

src/pkg/storage/storage.go

@@ -26,6 +26,8 @@ const (
 	BucketKey                 = "BUCKET"
 	EndpointKey               = "ENDPOINT"
 	PrefixKey                 = "PREFIX"
+	DisableTLSKey             = "DISABLE_TLS"
+	DisableTLSVerificationKey = "DISABLE_TLS_VERIFICATION"
 )
 
 // Storage defines a storage provider, along with any configuration