From f03432e30388e58f9403af75c462f895d1efce1a Mon Sep 17 00:00:00 2001 From: Keepers <104464746+ryanfkeepers@users.noreply.github.com> Date: Fri, 10 Jun 2022 13:21:03 -0600 Subject: [PATCH] read test config from local file (#179) * read test config from local file Allows local configuration of the test environment by reading from a .toml config file. If no file exists, the file read is a no-op. Value prioritization is specified in the readTestConfig() func. --- src/go.mod | 3 +- src/go.sum | 2 - .../connector/graph_connector_test.go | 6 +- src/internal/kopia/kopia_test.go | 3 +- src/internal/operations/backup_test.go | 8 +- src/internal/testing/config.go | 83 +++++++++++++++++++ src/internal/testing/integration_runners.go | 2 +- src/internal/testing/storage.go | 32 +++---- src/pkg/repository/repository_test.go | 3 +- 9 files changed, 107 insertions(+), 35 deletions(-) create mode 100644 src/internal/testing/config.go diff --git a/src/go.mod b/src/go.mod index 593b11737..2ee0efdfe 100644 --- a/src/go.mod +++ b/src/go.mod @@ -17,6 +17,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.7.1 + go.uber.org/zap v1.21.0 ) require ( @@ -77,14 +78,12 @@ require ( github.com/rs/xid v1.4.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.2.0 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opentelemetry.io/otel v1.7.0 // indirect go.opentelemetry.io/otel/trace v1.7.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/net v0.0.0-20220531201128-c960675eff93 // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect diff --git a/src/go.sum b/src/go.sum index f4415ca54..2c6b78df7 100644 --- a/src/go.sum +++ b/src/go.sum @@ -336,8 +336,6 @@ github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 8766ee112..8e43f8175 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ctesting "github.com/alcionai/corso/internal/testing" @@ -28,9 +29,8 @@ func TestGraphConnectorSuite(t *testing.T) { func (suite *GraphConnectorIntegrationSuite) SetupSuite() { evs, err := ctesting.GetRequiredEnvVars(credentials.TenantID, credentials.ClientID, credentials.ClientSecret) - if err != nil { - suite.T().Fatal(err) - } + require.NoError(suite.T(), err) + suite.connector, err = NewGraphConnector( evs[credentials.TenantID], evs[credentials.ClientID], diff --git a/src/internal/kopia/kopia_test.go b/src/internal/kopia/kopia_test.go index 90514fafb..fcf69d3bc 100644 --- a/src/internal/kopia/kopia_test.go +++ b/src/internal/kopia/kopia_test.go @@ -230,7 +230,8 @@ func TestKopiaIntegrationSuite(t *testing.T) { } func (suite *KopiaIntegrationSuite) SetupSuite() { - require.NoError(suite.T(), ctesting.CheckS3EnvVars()) + _, err := ctesting.GetRequiredEnvVars(ctesting.AWSCredentialEnvs...) + require.NoError(suite.T(), err) } func (suite *KopiaIntegrationSuite) TestCloseTwiceDoesNotCrash() { diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index c2f03eb03..ebc927e88 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/internal/kopia" @@ -25,13 +26,12 @@ func TestBackupOpIntegrationSuite(t *testing.T) { } func (suite *BackupOpIntegrationSuite) SetupSuite() { - if _, err := ctesting.GetRequiredEnvVars( + _, err := ctesting.GetRequiredEnvVars( credentials.TenantID, credentials.ClientID, credentials.ClientSecret, - ); err != nil { - suite.T().Fatal(err) - } + ) + require.NoError(suite.T(), err) } func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { diff --git a/src/internal/testing/config.go b/src/internal/testing/config.go new file mode 100644 index 000000000..068cff079 --- /dev/null +++ b/src/internal/testing/config.go @@ -0,0 +1,83 @@ +package testing + +import ( + "os" + "path" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/viper" +) + +const ( + // S3 config + testCfgBucket = "bucket" + testCfgEndpoint = "endpoint" + testCfgPrefix = "prefix" + + // M365 config + testCfgTenantID = "tenantid" +) + +func newTestViper() (*viper.Viper, error) { + vpr := viper.New() + + configFilePath := os.Getenv("CORSO_TEST_CONFIG_FILE") + if len(configFilePath) == 0 { + return vpr, nil + } + + // Or use a custom file location + fileName := path.Base(configFilePath) + ext := path.Ext(configFilePath) + if len(ext) == 0 { + return nil, errors.New("corso_test requires an extension") + } + + vpr.SetConfigFile(configFilePath) + vpr.AddConfigPath(path.Dir(configFilePath)) + vpr.SetConfigType(ext[1:]) + fileName = strings.TrimSuffix(fileName, ext) + vpr.SetConfigName(fileName) + + return vpr, nil +} + +// reads a corso configuration file with values specific to +// local integration test controls. Populates values with +// defaults where standard. +func readTestConfig() (map[string]string, error) { + vpr, err := newTestViper() + if err != nil { + return nil, err + } + + // only error if reading an existing file failed. No problem if we're missing files. + if err = vpr.ReadInConfig(); err != nil { + _, ok := err.(viper.ConfigFileNotFoundError) + if !ok { + return nil, errors.Wrap(err, "reading config file: "+viper.ConfigFileUsed()) + } + } + + testEnv := map[string]string{} + fallbackTo(testEnv, testCfgBucket, vpr.GetString(testCfgBucket), "test-corso-repo-init") + fallbackTo(testEnv, testCfgEndpoint, vpr.GetString(testCfgEndpoint), "s3.amazonaws.com") + fallbackTo(testEnv, testCfgPrefix, vpr.GetString(testCfgPrefix)) + fallbackTo(testEnv, testCfgTenantID, vpr.GetString(testCfgTenantID)) + + return testEnv, nil +} + +// writes the first non-zero valued string to the map at the key. +// fallback priority should match viper ordering (manually handled +// here since viper fails to provide fallbacks on fileNotFoundErr): +// manual overrides > flags > env vars > config file > default value +func fallbackTo(m map[string]string, key string, fallbacks ...string) { + for _, fb := range fallbacks { + if len(fb) > 0 { + m[key] = fb + return + } + } +} diff --git a/src/internal/testing/integration_runners.go b/src/internal/testing/integration_runners.go index 6538cf44b..0711eb0ee 100644 --- a/src/internal/testing/integration_runners.go +++ b/src/internal/testing/integration_runners.go @@ -35,7 +35,7 @@ func RunOnAny(tests ...string) error { // LogTimeOfTest logs the test name and the time that it was run. func LogTimeOfTest(t *testing.T) string { - now := time.Now().UTC().Format("2016-01-02T15:04:05") + now := time.Now().UTC().Format("2016-01-02T15:04:05.0000") pc, _, _, ok := runtime.Caller(1) details := runtime.FuncForPC(pc) if !ok || details != nil { diff --git a/src/internal/testing/storage.go b/src/internal/testing/storage.go index 1219ff667..4730e85b3 100644 --- a/src/internal/testing/storage.go +++ b/src/internal/testing/storage.go @@ -1,40 +1,30 @@ package testing import ( - "os" - - "github.com/pkg/errors" - "github.com/alcionai/corso/pkg/credentials" "github.com/alcionai/corso/pkg/storage" + "github.com/pkg/errors" ) -// CheckS3EnvVars returns as error if any of the environment variables required for -// integration tests using S3 is empty. It does not check the validity of the -// variables with S3. -func CheckS3EnvVars() error { - s3Envs := []string{ - credentials.AWSAccessKeyID, - credentials.AWSSecretAccessKey, - credentials.AWSSessionToken, - } - for _, env := range s3Envs { - if os.Getenv(env) == "" { - return errors.Errorf("env var [%s] must be populated for integration testing", env) - } - } - - return nil +var AWSCredentialEnvs = []string{ + credentials.AWSAccessKeyID, + credentials.AWSSecretAccessKey, + credentials.AWSSessionToken, } // NewS3Storage returns a storage.Storage object initialized with environment // variables used for integration tests that use S3. func NewS3Storage(prefix string) (storage.Storage, error) { + cfg, err := readTestConfig() + if err != nil { + return storage.Storage{}, errors.Wrap(err, "configuring storage from test file") + } + return storage.NewStorage( storage.ProviderS3, storage.S3Config{ AWS: credentials.GetAWS(nil), - Bucket: "test-corso-repo-init", + Bucket: cfg[testCfgBucket], Prefix: prefix, }, storage.CommonConfig{ diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 7a8065598..df51d3d9b 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -99,7 +99,8 @@ func TestRepositoryIntegrationSuite(t *testing.T) { // ensure all required env values are populated func (suite *RepositoryIntegrationSuite) SetupSuite() { - require.NoError(suite.T(), ctesting.CheckS3EnvVars()) + _, err := ctesting.GetRequiredEnvVars(ctesting.AWSCredentialEnvs...) + require.NoError(suite.T(), err) } func (suite *RepositoryIntegrationSuite) TestInitialize() {