read test config from local file (#179)

* read test config from local file

Allows local configuration of the test environment by reading from
a .toml config file.  If no file exists, the file read is a no-op.  Value
prioritization is specified in the readTestConfig() func.

src/go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.7.1
+	go.uber.org/zap v1.21.0
 )
 
 require (
@@ -77,14 +78,12 @@ require (
 	github.com/rs/xid v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.opentelemetry.io/otel v1.7.0 // indirect
 	go.opentelemetry.io/otel/trace v1.7.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.21.0 // indirect
 	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
 	golang.org/x/net v0.0.0-20220531201128-c960675eff93 // indirect
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect

src/go.sum

@@ -336,8 +336,6 @@ github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ=
 github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=

src/internal/connector/graph_connector_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	ctesting "github.com/alcionai/corso/internal/testing"
@@ -28,9 +29,8 @@ func TestGraphConnectorSuite(t *testing.T) {
 
 func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
 	evs, err := ctesting.GetRequiredEnvVars(credentials.TenantID, credentials.ClientID, credentials.ClientSecret)
-	if err != nil {
+	require.NoError(suite.T(), err)
-		suite.T().Fatal(err)
+
-	}
 	suite.connector, err = NewGraphConnector(
 		evs[credentials.TenantID],
 		evs[credentials.ClientID],

src/internal/kopia/kopia_test.go

@@ -230,7 +230,8 @@ func TestKopiaIntegrationSuite(t *testing.T) {
 }
 
 func (suite *KopiaIntegrationSuite) SetupSuite() {
-	require.NoError(suite.T(), ctesting.CheckS3EnvVars())
+	_, err := ctesting.GetRequiredEnvVars(ctesting.AWSCredentialEnvs...)
+	require.NoError(suite.T(), err)
 }
 
 func (suite *KopiaIntegrationSuite) TestCloseTwiceDoesNotCrash() {

src/internal/operations/backup_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/internal/kopia"
@@ -25,13 +26,12 @@ func TestBackupOpIntegrationSuite(t *testing.T) {
 }
 
 func (suite *BackupOpIntegrationSuite) SetupSuite() {
-	if _, err := ctesting.GetRequiredEnvVars(
+	_, err := ctesting.GetRequiredEnvVars(
 		credentials.TenantID,
 		credentials.ClientID,
 		credentials.ClientSecret,
-	); err != nil {
+	)
-		suite.T().Fatal(err)
+	require.NoError(suite.T(), err)
-	}
 }
 
 func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {

src/internal/testing/config.go

@@ -0,0 +1,83 @@
+package testing
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+)
+
+const (
+	// S3 config
+	testCfgBucket   = "bucket"
+	testCfgEndpoint = "endpoint"
+	testCfgPrefix   = "prefix"
+
+	// M365 config
+	testCfgTenantID = "tenantid"
+)
+
+func newTestViper() (*viper.Viper, error) {
+	vpr := viper.New()
+
+	configFilePath := os.Getenv("CORSO_TEST_CONFIG_FILE")
+	if len(configFilePath) == 0 {
+		return vpr, nil
+	}
+
+	// Or use a custom file location
+	fileName := path.Base(configFilePath)
+	ext := path.Ext(configFilePath)
+	if len(ext) == 0 {
+		return nil, errors.New("corso_test requires an extension")
+	}
+
+	vpr.SetConfigFile(configFilePath)
+	vpr.AddConfigPath(path.Dir(configFilePath))
+	vpr.SetConfigType(ext[1:])
+	fileName = strings.TrimSuffix(fileName, ext)
+	vpr.SetConfigName(fileName)
+
+	return vpr, nil
+}
+
+// reads a corso configuration file with values specific to
+// local integration test controls.  Populates values with
+// defaults where standard.
+func readTestConfig() (map[string]string, error) {
+	vpr, err := newTestViper()
+	if err != nil {
+		return nil, err
+	}
+
+	// only error if reading an existing file failed.  No problem if we're missing files.
+	if err = vpr.ReadInConfig(); err != nil {
+		_, ok := err.(viper.ConfigFileNotFoundError)
+		if !ok {
+			return nil, errors.Wrap(err, "reading config file: "+viper.ConfigFileUsed())
+		}
+	}
+
+	testEnv := map[string]string{}
+	fallbackTo(testEnv, testCfgBucket, vpr.GetString(testCfgBucket), "test-corso-repo-init")
+	fallbackTo(testEnv, testCfgEndpoint, vpr.GetString(testCfgEndpoint), "s3.amazonaws.com")
+	fallbackTo(testEnv, testCfgPrefix, vpr.GetString(testCfgPrefix))
+	fallbackTo(testEnv, testCfgTenantID, vpr.GetString(testCfgTenantID))
+
+	return testEnv, nil
+}
+
+// writes the first non-zero valued string to the map at the key.
+// fallback priority should match viper ordering (manually handled
+// here since viper fails to provide fallbacks on fileNotFoundErr):
+// manual overrides > flags > env vars > config file > default value
+func fallbackTo(m map[string]string, key string, fallbacks ...string) {
+	for _, fb := range fallbacks {
+		if len(fb) > 0 {
+			m[key] = fb
+			return
+		}
+	}
+}

src/internal/testing/integration_runners.go

@@ -35,7 +35,7 @@ func RunOnAny(tests ...string) error {
 
 // LogTimeOfTest logs the test name and the time that it was run.
 func LogTimeOfTest(t *testing.T) string {
-	now := time.Now().UTC().Format("2016-01-02T15:04:05")
+	now := time.Now().UTC().Format("2016-01-02T15:04:05.0000")
 	pc, _, _, ok := runtime.Caller(1)
 	details := runtime.FuncForPC(pc)
 	if !ok || details != nil {

src/internal/testing/storage.go

@@ -1,40 +1,30 @@
 package testing
 
 import (
-	"os"
-
-	"github.com/pkg/errors"
-
 	"github.com/alcionai/corso/pkg/credentials"
 	"github.com/alcionai/corso/pkg/storage"
+	"github.com/pkg/errors"
 )
 
-// CheckS3EnvVars returns as error if any of the environment variables required for
+var AWSCredentialEnvs = []string{
-// integration tests using S3 is empty. It does not check the validity of the
+	credentials.AWSAccessKeyID,
-// variables with S3.
+	credentials.AWSSecretAccessKey,
-func CheckS3EnvVars() error {
+	credentials.AWSSessionToken,
-	s3Envs := []string{
-		credentials.AWSAccessKeyID,
-		credentials.AWSSecretAccessKey,
-		credentials.AWSSessionToken,
-	}
-	for _, env := range s3Envs {
-		if os.Getenv(env) == "" {
-			return errors.Errorf("env var [%s] must be populated for integration testing", env)
-		}
-	}
-
-	return nil
 }
 
 // NewS3Storage returns a storage.Storage object initialized with environment
 // variables used for integration tests that use S3.
 func NewS3Storage(prefix string) (storage.Storage, error) {
+	cfg, err := readTestConfig()
+	if err != nil {
+		return storage.Storage{}, errors.Wrap(err, "configuring storage from test file")
+	}
+
 	return storage.NewStorage(
 		storage.ProviderS3,
 		storage.S3Config{
 			AWS:    credentials.GetAWS(nil),
-			Bucket: "test-corso-repo-init",
+			Bucket: cfg[testCfgBucket],
 			Prefix: prefix,
 		},
 		storage.CommonConfig{

src/pkg/repository/repository_test.go

@@ -99,7 +99,8 @@ func TestRepositoryIntegrationSuite(t *testing.T) {
 
 // ensure all required env values are populated
 func (suite *RepositoryIntegrationSuite) SetupSuite() {
-	require.NoError(suite.T(), ctesting.CheckS3EnvVars())
+	_, err := ctesting.GetRequiredEnvVars(ctesting.AWSCredentialEnvs...)
+	require.NoError(suite.T(), err)
 }
 
 func (suite *RepositoryIntegrationSuite) TestInitialize() {