larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow disabling TLS and TLS verification (#1417)

Browse Source 
## Description

Introduces config options (`--disable-tls` and `--disable-tls-verification`) to turn off TLS and TLS verification

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #1415 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Vaibhav Kamra 2022-11-02 21:47:19 -07:00 committed by GitHub
parent 524c3ba853
commit 09261d5474
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 139 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/cli/config/config.go
View File

@ -18,10 +18,12 @@ import (
const (
// S3 config
StorageProviderTypeKey = "provider"
BucketNameKey = "bucket"
EndpointKey = "endpoint"
PrefixKey = "prefix"
StorageProviderTypeKey = "provider"
BucketNameKey = "bucket"
EndpointKey = "endpoint"
PrefixKey = "prefix"
DisableTLSKey = "disable_tls"
DisableTLSVerificationKey = "disable_tls_verification"
// M365 config
AccountProviderTypeKey = "account_provider"
@ -32,7 +34,6 @@ var (
configFilePath string
configFilePathFlag string
configDir string
defaultDir string
displayDefaultFP = filepath.Join("$HOME", ".corso.toml")
)
@ -54,8 +55,6 @@ func init() {
Infof(context.Background(), "cannot stat user's $HOME directory: %v", err)
}
defaultDir = homeDir
if len(configDir) == 0 {
configDir = homeDir
configFilePath = filepath.Join(configDir, ".corso.toml")
@ -195,6 +194,8 @@ func writeRepoConfigWithViper(vpr *viper.Viper, s3Config storage.S3Config, m365C
vpr.Set(BucketNameKey, s3Config.Bucket)
vpr.Set(EndpointKey, s3Config.Endpoint)
vpr.Set(PrefixKey, s3Config.Prefix)
vpr.Set(DisableTLSKey, s3Config.DoNotUseTLS)
vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
vpr.Set(AccountProviderTypeKey, account.ProviderM365.String())
vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID)

32
src/cli/config/config_test.go
View File

@ -26,6 +26,8 @@ const (
` + StorageProviderTypeKey + ` = 'S3'
` + AccountProviderTypeKey + ` = 'M365'
` + AzureTenantIDKey + ` = '%s'
` + DisableTLSKey + ` = 'false'
` + DisableTLSVerificationKey + ` = 'false'
`
)
@ -84,7 +86,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{Bucket: bkt}
s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
m365 := account.M365Config{AzureTenantID: tid}
require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
@ -93,6 +95,8 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
readS3Cfg, err := s3ConfigsFromViper(vpr)
require.NoError(t, err)
assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
readM365, err := m365ConfigsFromViper(vpr)
require.NoError(t, err)
@ -217,9 +221,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{
Bucket: bkt,
Endpoint: end,
Prefix: pfx,
Bucket: bkt,
Endpoint: end,
Prefix: pfx,
DoNotVerifyTLS: true,
DoNotUseTLS: true,
}
m365 := account.M365Config{AzureTenantID: tid}
@ -234,6 +240,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
common, err := st.CommonConfig()
require.NoError(t, err, "reading common config from storage")
@ -257,12 +265,6 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1"
)
// Configure viper to read test config file
s3Cfg := storage.S3Config{
Bucket: bkt,
Endpoint: end,
Prefix: pfx,
}
m365 := account.M365Config{AzureTenantID: tid}
overrides := map[string]string{
@ -271,6 +273,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
storage.Bucket: bkt,
storage.Endpoint: end,
storage.Prefix: pfx,
storage.DoNotUseTLS: "true",
storage.DoNotVerifyTLS: "true",
StorageProviderTypeKey: storage.ProviderS3.String(),
}
@ -279,9 +283,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
readS3Cfg, err := st.S3Config()
require.NoError(t, err, "reading s3 config from storage")
assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
assert.Equal(t, readS3Cfg.Bucket, bkt)
assert.Equal(t, readS3Cfg.Endpoint, end)
assert.Equal(t, readS3Cfg.Prefix, pfx)
assert.True(t, readS3Cfg.DoNotUseTLS)
assert.True(t, readS3Cfg.DoNotVerifyTLS)
common, err := st.CommonConfig()
require.NoError(t, err, "reading common config from storage")

13
src/cli/config/storage.go
View File

@ -3,6 +3,7 @@ package config
import (
"os"
"path/filepath"
"strconv"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/pkg/errors"
@ -26,6 +27,8 @@ func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
s3Config.Bucket = vpr.GetString(BucketNameKey)
s3Config.Endpoint = vpr.GetString(EndpointKey)
s3Config.Prefix = vpr.GetString(PrefixKey)
s3Config.DoNotUseTLS = vpr.GetBool(DisableTLSKey)
s3Config.DoNotVerifyTLS = vpr.GetBool(DisableTLSVerificationKey)
return s3Config, nil
}
@ -35,6 +38,8 @@ func s3Overrides(in map[string]string) map[string]string {
storage.Bucket: in[storage.Bucket],
storage.Endpoint: in[storage.Endpoint],
storage.Prefix: in[storage.Prefix],
storage.DoNotUseTLS: in[storage.DoNotUseTLS],
storage.DoNotVerifyTLS: in[storage.DoNotVerifyTLS],
StorageProviderTypeKey: in[StorageProviderTypeKey],
}
}
@ -79,6 +84,14 @@ func configureStorage(
Bucket: common.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)),
Endpoint: common.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)),
Prefix: common.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)),
DoNotUseTLS: common.ParseBool(common.First(
overrides[storage.DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS),
os.Getenv(storage.PrefixKey))),
DoNotVerifyTLS: common.ParseBool(common.First(
overrides[storage.DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
os.Getenv(storage.PrefixKey))),
}
// compose the common config and credentials

8
src/cli/repo/s3.go
View File

@ -1,6 +1,8 @@
package repo
import (
"strconv"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -19,6 +21,8 @@ var (
bucket string
endpoint string
prefix string
doNotUseTLS bool
doNotVerifyTLS bool
succeedIfExists bool
)
@ -45,6 +49,8 @@ func addS3Commands(parent *cobra.Command) *cobra.Command {
cobra.CheckErr(c.MarkFlagRequired("bucket"))
fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.")
fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.")
fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)")
fs.BoolVar(&doNotVerifyTLS, "disable-tls-verification", false, "Disable TLS (HTTPS) certificate verification.")
// In general, we don't want to expose this flag to users and have them mistake it
// for a broad-scale idempotency solution. We can un-hide it later the need arises.
@ -200,5 +206,7 @@ func s3Overrides() map[string]string {
storage.Bucket: bucket,
storage.Endpoint: endpoint,
storage.Prefix: prefix,
storage.DoNotUseTLS: strconv.FormatBool(doNotUseTLS),
storage.DoNotVerifyTLS: strconv.FormatBool(doNotVerifyTLS),
}
}

13
src/internal/common/slices.go
View File

@ -1,5 +1,7 @@
package common
import "strconv"
func ContainsString(super []string, sub string) bool {
for _, s := range super {
if s == sub {
@ -20,3 +22,14 @@ func First(vs ...string) string {
return ""
}
// parseBool returns the bool value represented by the string
// or false on error
func ParseBool(v string) bool {
s, err := strconv.ParseBool(v)
if err != nil {
return false
}
return s
}

8
src/internal/kopia/s3.go
View File

@ -25,9 +25,11 @@ func s3BlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error)
}
opts := s3.Options{
BucketName: cfg.Bucket,
Endpoint: endpoint,
Prefix: cfg.Prefix,
BucketName: cfg.Bucket,
Endpoint: endpoint,
Prefix: cfg.Prefix,
DoNotUseTLS: cfg.DoNotUseTLS,
DoNotVerifyTLS: cfg.DoNotVerifyTLS,
}
return s3.New(ctx, &opts)

44
src/pkg/storage/s3.go
View File

@ -1,36 +1,46 @@
package storage
import (
"strconv"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
)
type S3Config struct {
Bucket string // required
Endpoint string
Prefix string
Bucket string // required
Endpoint string
Prefix string
DoNotUseTLS bool
DoNotVerifyTLS bool
}
// config key consts
const (
keyS3Bucket = "s3_bucket"
keyS3Endpoint = "s3_endpoint"
keyS3Prefix = "s3_prefix"
keyS3Bucket = "s3_bucket"
keyS3Endpoint = "s3_endpoint"
keyS3Prefix = "s3_prefix"
keyS3DoNotUseTLS = "s3_donotusetls"
keyS3DoNotVerifyTLS = "s3_donotverifytls"
)
// config exported name consts
const (
Bucket = "bucket"
Endpoint = "endpoint"
Prefix = "prefix"
Bucket = "bucket"
Endpoint = "endpoint"
Prefix = "prefix"
DoNotUseTLS = "donotusetls"
DoNotVerifyTLS = "donotverifytls"
)
func (c S3Config) Normalize() S3Config {
return S3Config{
Bucket: common.NormalizeBucket(c.Bucket),
Endpoint: c.Endpoint,
Prefix: common.NormalizePrefix(c.Prefix),
Bucket: common.NormalizeBucket(c.Bucket),
Endpoint: c.Endpoint,
Prefix: common.NormalizePrefix(c.Prefix),
DoNotUseTLS: c.DoNotUseTLS,
DoNotVerifyTLS: c.DoNotVerifyTLS,
}
}
@ -40,9 +50,11 @@ func (c S3Config) Normalize() S3Config {
func (c S3Config) StringConfig() (map[string]string, error) {
cn := c.Normalize()
cfg := map[string]string{
keyS3Bucket: cn.Bucket,
keyS3Endpoint: cn.Endpoint,
keyS3Prefix: cn.Prefix,
keyS3Bucket: cn.Bucket,
keyS3Endpoint: cn.Endpoint,
keyS3Prefix: cn.Prefix,
keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS),
keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
}
return cfg, c.validate()
@ -56,6 +68,8 @@ func (s Storage) S3Config() (S3Config, error) {
c.Bucket = orEmptyString(s.Config[keyS3Bucket])
c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
c.Prefix = orEmptyString(s.Config[keyS3Prefix])
c.DoNotUseTLS = common.ParseBool(s.Config[keyS3DoNotUseTLS])
c.DoNotVerifyTLS = common.ParseBool(s.Config[keyS3DoNotVerifyTLS])
}
return c, c.validate()

59
src/pkg/storage/s3_test.go
View File

@ -1,4 +1,4 @@
package storage_test
package storage
import (
"testing"
@ -6,8 +6,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/pkg/storage"
)
type S3CfgSuite struct {
@ -19,16 +17,20 @@ func TestS3CfgSuite(t *testing.T) {
}
var (
goodS3Config = storage.S3Config{
Bucket: "bkt",
Endpoint: "end",
Prefix: "pre/",
goodS3Config = S3Config{
Bucket: "bkt",
Endpoint: "end",
Prefix: "pre/",
DoNotUseTLS: false,
DoNotVerifyTLS: false,
}
goodS3Map = map[string]string{
"s3_bucket": "bkt",
"s3_endpoint": "end",
"s3_prefix": "pre/",
keyS3Bucket: "bkt",
keyS3Endpoint: "end",
keyS3Prefix: "pre/",
keyS3DoNotUseTLS: "false",
keyS3DoNotVerifyTLS: "false",
}
)
@ -54,7 +56,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
t := suite.T()
in := goodS3Config
s, err := storage.NewStorage(storage.ProviderS3, in)
s, err := NewStorage(ProviderS3, in)
assert.NoError(t, err)
out, err := s.S3Config()
assert.NoError(t, err)
@ -64,8 +66,8 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
assert.Equal(t, in.Prefix, out.Prefix)
}
func makeTestS3Cfg(bkt, end, pre string) storage.S3Config {
return storage.S3Config{
func makeTestS3Cfg(bkt, end, pre string) S3Config {
return S3Config{
Bucket: bkt,
Endpoint: end,
Prefix: pre,
@ -76,13 +78,13 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
// missing required properties
table := []struct {
name string
cfg storage.S3Config
cfg S3Config
}{
{"missing bucket", makeTestS3Cfg("", "end", "pre/")},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
_, err := storage.NewStorage(storage.ProviderUnknown, test.cfg)
_, err := NewStorage(ProviderUnknown, test.cfg)
assert.Error(t, err)
})
}
@ -90,18 +92,18 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
// required property not populated in storage
table2 := []struct {
name string
amend func(storage.Storage)
amend func(Storage)
}{
{
"missing bucket",
func(s storage.Storage) {
func(s Storage) {
s.Config["s3_bucket"] = ""
},
},
}
for _, test := range table2 {
suite.T().Run(test.name, func(t *testing.T) {
st, err := storage.NewStorage(storage.ProviderUnknown, goodS3Config)
st, err := NewStorage(ProviderUnknown, goodS3Config)
assert.NoError(t, err)
test.amend(st)
_, err = st.S3Config()
@ -113,7 +115,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
table := []struct {
name string
input storage.S3Config
input S3Config
expect map[string]string
}{
{
@ -126,6 +128,23 @@ func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
input: makeTestS3Cfg("s3://"+goodS3Config.Bucket, goodS3Config.Endpoint, goodS3Config.Prefix),
expect: goodS3Map,
},
{
name: "disabletls",
input: S3Config{
Bucket: "bkt",
Endpoint: "end",
Prefix: "pre/",
DoNotUseTLS: true,
DoNotVerifyTLS: true,
},
expect: map[string]string{
keyS3Bucket: "bkt",
keyS3Endpoint: "end",
keyS3Prefix: "pre/",
keyS3DoNotUseTLS: "true",
keyS3DoNotVerifyTLS: "true",
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
@ -142,7 +161,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_Normalize() {
normalBkt = "bkt"
)
st := storage.S3Config{
st := S3Config{
Bucket: prefixedBkt,
}

8
src/pkg/storage/storage.go
View File

@ -23,9 +23,11 @@ var (
// envvar consts
// TODO: Remove these and leverage Viper AutomaticEnv() instead
const (
BucketKey = "BUCKET"
EndpointKey = "ENDPOINT"
PrefixKey = "PREFIX"
BucketKey = "BUCKET"
EndpointKey = "ENDPOINT"
PrefixKey = "PREFIX"
DisableTLSKey = "DISABLE_TLS"
DisableTLSVerificationKey = "DISABLE_TLS_VERIFICATION"
)
// Storage defines a storage provider, along with any configuration