package config import ( "fmt" "os" "path/filepath" "testing" "github.com/alcionai/clues" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/storage" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) const ( configFileTemplate = ` ` + storage.BucketNameKey + ` = '%s' ` + storage.EndpointKey + ` = 's3.amazonaws.com' ` + storage.PrefixKey + ` = 'test-prefix/' ` + storage.StorageProviderTypeKey + ` = 'S3' ` + account.AccountProviderTypeKey + ` = 'M365' ` + account.AzureTenantIDKey + ` = '%s' ` + storage.AccessKey + ` = '%s' ` + storage.SecretAccessKey + ` = '%s' ` + storage.SessionToken + ` = '%s' ` + CorsoPassphrase + ` = '%s' ` + account.AzureClientID + ` = '%s' ` + account.AzureSecret + ` = '%s' ` + storage.DisableTLSKey + ` = '%s' ` + storage.DisableTLSVerificationKey + ` = '%s' ` ) type ConfigSuite struct { tester.Suite } func TestConfigSuite(t *testing.T) { suite.Run(t, &ConfigSuite{Suite: tester.NewUnitSuite(t)}) } func (suite *ConfigSuite) TestRequireProps() { table := []struct { name string props map[string]string errCheck assert.ErrorAssertionFunc }{ { props: map[string]string{"exists": "I have seen the fnords!"}, errCheck: assert.NoError, }, { props: map[string]string{"not-exists": ""}, errCheck: assert.Error, }, } for _, test := range table { err := requireProps(test.props) test.errCheck(suite.T(), err, clues.ToCore(err)) } } func (suite *ConfigSuite) TestReadRepoConfigBasic() { var ( t = suite.T() vpr = viper.New() ) const ( b = "read-repo-config-basic-bucket" tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba" accKey = "aws-test-access-key" secret = "aws-test-secret-key" token = "aws-test-session-token" passphrase = "passphrase-test" azureClientID = "azure-client-id-test" azureSecret = "azure-secret-test" endpoint = "s3-test" disableTLS = "true" disableTLSVerification = "true" ) // Generate test config file testConfigData := fmt.Sprintf(configFileTemplate, b, tID, accKey, secret, token, passphrase, azureClientID, azureSecret, disableTLS, disableTLSVerification) testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml") assert.NotEmpty(t, testConfigData) assert.NotEmpty(t, testConfigFilePath) _ = vpr // err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700) // require.NoError(t, err, clues.ToCore(err)) // // Configure viper to read test config file // vpr.SetConfigFile(testConfigFilePath) // // Read and validate config // err = vpr.ReadInConfig() // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // sc, err := storage.NewStorageConfig(storage.ProviderS3) // require.NoError(t, err, clues.ToCore(err)) // err = sc.ApplyConfigOverrides(vpr, true, true, nil) // require.NoError(t, err, clues.ToCore(err)) // s3Cfg := sc.(*storage.S3Config) // assert.Equal(t, b, s3Cfg.Bucket) // assert.Equal(t, "test-prefix/", s3Cfg.Prefix) // assert.Equal(t, disableTLS, strconv.FormatBool(s3Cfg.DoNotUseTLS)) // assert.Equal(t, disableTLSVerification, strconv.FormatBool(s3Cfg.DoNotVerifyTLS)) // // Config file may or may not be the source of truth for below values. These may be // // overridden by env vars (and flags but not relevant for this test). // // // // Other alternatives are: // // 1) unset env vars temporarily so that we can test against config file values. But that // // may be problematic if we decide to parallelize tests in future. // // 2) assert against env var values instead of config file values. This can cause issues // // if CI/local env have different config override mechanisms. // // 3) Skip asserts for these keys. They will be validated in other tests. Choosing this // // option. // // assert.Equal(t, accKey, s3Cfg.AWS.AccessKey) // // assert.Equal(t, secret, s3Cfg.AWS.SecretKey) // // assert.Equal(t, token, s3Cfg.AWS.SessionToken) // m365, err := m365ConfigsFromViper(vpr) // require.NoError(t, err, clues.ToCore(err)) // assert.Equal(t, azureClientID, m365.AzureClientID) // assert.Equal(t, azureSecret, m365.AzureClientSecret) // assert.Equal(t, tID, m365.AzureTenantID) } func (suite *ConfigSuite) TestWriteReadConfig() { var ( t = suite.T() vpr = viper.New() // Configure viper to read test config file testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml") ) ctx, flush := tester.NewContext(t) defer flush() const ( bkt = "write-read-config-bucket" tid = "3c0748d2-470e-444c-9064-1268e52609d5" repoID = "repoid" user = "a-user" host = "some-host" ) err := initWithViper(ctx, vpr, testConfigFilePath) require.NoError(t, err, "initializing repo config", clues.ToCore(err)) s3Cfg := &storage.S3Config{ Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true, } m365 := account.M365Config{AzureTenantID: tid} rOpts := repository.Options{ User: user, Host: host, } assert.NotEmpty(t, s3Cfg) assert.NotEmpty(t, m365) assert.NotEmpty(t, rOpts) // err = writeRepoConfigWithViper(vpr, s3Cfg, m365, rOpts, repoID) // require.NoError(t, err, "writing repo config", clues.ToCore(err)) // err = vpr.ReadInConfig() // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // sc, err := storage.NewStorageConfig(storage.ProviderS3) // require.NoError(t, err, clues.ToCore(err)) // err = sc.ApplyConfigOverrides(vpr, true, true, nil) // require.NoError(t, err, clues.ToCore(err)) // readS3Cfg := sc.(*storage.S3Config) // assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket) // assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS) // assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS) // readM365, err := m365ConfigsFromViper(vpr) // require.NoError(t, err, clues.ToCore(err)) // assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID) // gotUser, gotHost := getUserHost(vpr, true) // assert.Equal(t, user, gotUser) // assert.Equal(t, host, gotHost) } func (suite *ConfigSuite) TestMustMatchConfig() { var ( t = suite.T() vpr = viper.New() // Configure viper to read test config file testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml") ) ctx, flush := tester.NewContext(t) defer flush() const ( bkt = "must-match-config-bucket" tid = "dfb12063-7598-458b-85ab-42352c5c25e2" ) err := initWithViper(ctx, vpr, testConfigFilePath) require.NoError(t, err, "initializing repo config") s3Cfg := &storage.S3Config{Bucket: bkt} m365 := account.M365Config{AzureTenantID: tid} m365PathKeys := []string{} assert.NotEmpty(t, s3Cfg) assert.NotEmpty(t, m365) _ = m365PathKeys // err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid") // require.NoError(t, err, "writing repo config", clues.ToCore(err)) // err = vpr.ReadInConfig() // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // table := []struct { // name string // input map[string]string // errCheck assert.ErrorAssertionFunc // }{ // { // name: "full match", // input: map[string]string{ // storage.Bucket: bkt, // account.AzureTenantID: tid, // }, // errCheck: assert.NoError, // }, // { // name: "empty values", // input: map[string]string{ // storage.Bucket: "", // account.AzureTenantID: "", // }, // errCheck: assert.NoError, // }, // { // name: "no overrides", // input: map[string]string{}, // errCheck: assert.NoError, // }, // { // name: "nil map", // input: nil, // errCheck: assert.NoError, // }, // { // name: "no recognized keys", // input: map[string]string{ // "fnords": "smurfs", // "nonsense": "", // }, // errCheck: assert.NoError, // }, // { // name: "mismatch", // input: map[string]string{ // storage.Bucket: tid, // account.AzureTenantID: bkt, // }, // errCheck: assert.Error, // }, // } // for _, test := range table { // suite.Run(test.name, func() { // test.errCheck(suite.T(), mustMatchConfig(vpr, test.input, m365PathKeys), clues.ToCore(err)) // }) // } } func (suite *ConfigSuite) TestReadFromFlags() { var ( t = suite.T() vpr = viper.New() ) ctx, flush := tester.NewContext(t) defer flush() const ( b = "read-repo-config-basic-bucket" tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba" accKey = "aws-test-access-key" secret = "aws-test-secret-key" token = "aws-test-session-token" passphrase = "passphrase-test" azureClientID = "azure-client-id-test" azureSecret = "azure-secret-test" prefix = "prefix-test" disableTLS = "true" disableTLSVerification = "true" ) t.Cleanup(func() { // reset values flags.AzureClientTenantFV = "" flags.AzureClientIDFV = "" flags.AzureClientSecretFV = "" flags.AWSAccessKeyFV = "" flags.AWSSecretAccessKeyFV = "" flags.AWSSessionTokenFV = "" flags.PassphraseFV = "" }) // Generate test config file testConfigData := fmt.Sprintf(configFileTemplate, b, tID, accKey, secret, token, passphrase, azureClientID, azureSecret, disableTLS, disableTLSVerification) testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml") assert.NotEmpty(t, testConfigData) assert.NotEmpty(t, testConfigFilePath) _ = vpr _ = ctx // err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700) // require.NoError(t, err, clues.ToCore(err)) // // Configure viper to read test config file // vpr.SetConfigFile(testConfigFilePath) // // Read and validate config // err = vpr.ReadInConfig() // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // overrides := map[string]string{} // flags.AzureClientTenantFV = "6f34ac30-8196-469b-bf8f-d83deadbbbba" // flags.AzureClientIDFV = "azure-id-flag-value" // flags.AzureClientSecretFV = "azure-secret-flag-value" // flags.AWSAccessKeyFV = "aws-access-key" // flags.AWSSecretAccessKeyFV = "aws-access-secret-flag-value" // flags.AWSSessionTokenFV = "aws-access-session-flag-value" // overrides[storage.Bucket] = "flag-bucket" // overrides[storage.Endpoint] = "flag-endpoint" // overrides[storage.Prefix] = "flag-prefix" // overrides[storage.DoNotUseTLS] = "true" // overrides[storage.DoNotVerifyTLS] = "true" // overrides[credentials.AWSAccessKeyID] = flags.AWSAccessKeyFV // overrides[credentials.AWSSecretAccessKey] = flags.AWSSecretAccessKeyFV // overrides[credentials.AWSSessionToken] = flags.AWSSessionTokenFV // flags.PassphraseFV = "passphrase-flags" // repoDetails, err := getStorageAndAccountWithViper( // ctx, // vpr, // storage.ProviderS3, // true, // false, // overrides) // require.NoError(t, err, "getting storage and account from config", clues.ToCore(err)) // m365Config, _ := repoDetails.Account.M365Config() // s3Cfg, err := repoDetails.Storage.ToS3Config() // require.NoError(t, err, "reading s3 config from storage", clues.ToCore(err)) // commonConfig, _ := repoDetails.Storage.CommonConfig() // pass := commonConfig.Corso.CorsoPassphrase // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // assert.Equal(t, flags.AWSAccessKeyFV, s3Cfg.AWS.AccessKey) // assert.Equal(t, flags.AWSSecretAccessKeyFV, s3Cfg.AWS.SecretKey) // assert.Equal(t, flags.AWSSessionTokenFV, s3Cfg.AWS.SessionToken) // assert.Equal(t, overrides[storage.Bucket], s3Cfg.Bucket) // assert.Equal(t, overrides[storage.Endpoint], s3Cfg.Endpoint) // assert.Equal(t, overrides[storage.Prefix], s3Cfg.Prefix) // assert.Equal(t, str.ParseBool(overrides[storage.DoNotUseTLS]), s3Cfg.DoNotUseTLS) // assert.Equal(t, str.ParseBool(overrides[storage.DoNotVerifyTLS]), s3Cfg.DoNotVerifyTLS) // assert.Equal(t, flags.AzureClientIDFV, m365Config.AzureClientID) // assert.Equal(t, flags.AzureClientSecretFV, m365Config.AzureClientSecret) // assert.Equal(t, flags.AzureClientTenantFV, m365Config.AzureTenantID) // assert.Equal(t, flags.PassphraseFV, pass) } // ------------------------------------------------------------ // integration tests // ------------------------------------------------------------ type ConfigIntegrationSuite struct { tester.Suite } func TestConfigIntegrationSuite(t *testing.T) { suite.Run(t, &ConfigIntegrationSuite{Suite: tester.NewIntegrationSuite( t, [][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})}) } func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() { t := suite.T() vpr := viper.New() ctx, flush := tester.NewContext(t) defer flush() const ( bkt = "get-storage-and-account-bucket" end = "https://get-storage-and-account.com" pfx = "get-storage-and-account-prefix/" tid = "3a2faa4e-a882-445c-9d27-f552ef189381" ) // Configure viper to read test config file testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml") err := initWithViper(ctx, vpr, testConfigFilePath) require.NoError(t, err, "initializing repo config", clues.ToCore(err)) s3Cfg := &storage.S3Config{ Bucket: bkt, Endpoint: end, Prefix: pfx, DoNotVerifyTLS: true, DoNotUseTLS: true, } m365 := account.M365Config{AzureTenantID: tid} assert.NotEmpty(t, s3Cfg) assert.NotEmpty(t, m365) // err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid") // require.NoError(t, err, "writing repo config", clues.ToCore(err)) // require.Equal( // t, // account.ProviderM365.String(), // vpr.GetString(account.AccountProviderTypeKey), // "viper should have m365 as the account provider") // err = vpr.ReadInConfig() // require.NoError(t, err, "reading repo config", clues.ToCore(err)) // cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, true, true, nil) // require.NoError(t, err, "getting storage and account from config", clues.ToCore(err)) // readS3Cfg, err := cfg.Storage.ToS3Config() // require.NoError(t, err, "reading s3 config from storage", clues.ToCore(err)) // assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket) // assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint) // assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix) // assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS) // assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS) // assert.Equal(t, cfg.RepoID, "repoid") // common, err := cfg.Storage.CommonConfig() // require.NoError(t, err, "reading common config from storage", clues.ToCore(err)) // assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase)) // readM365, err := cfg.Account.M365Config() // require.NoError(t, err, "reading m365 config from account", clues.ToCore(err)) // // Env var gets preference here. Where to get env tenantID from // // assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID) // assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID)) // assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret)) } func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverrides() { t := suite.T() vpr := viper.New() ctx, flush := tester.NewContext(t) defer flush() const ( bkt = "get-storage-and-account-no-file-bucket" end = "https://get-storage-and-account.com/no-file" pfx = "get-storage-and-account-no-file-prefix/" tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1" ) m365 := account.M365Config{AzureTenantID: tid} overrides := map[string]string{ account.AzureTenantID: tid, account.AccountProviderTypeKey: account.ProviderM365.String(), storage.Bucket: bkt, storage.Endpoint: end, storage.Prefix: pfx, storage.DoNotUseTLS: "true", storage.DoNotVerifyTLS: "true", storage.StorageProviderTypeKey: storage.ProviderS3.String(), } cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, false, true, overrides) require.NoError(t, err, "getting storage and account from config", clues.ToCore(err)) readS3Cfg, err := cfg.Storage.ToS3Config() require.NoError(t, err, "reading s3 config from storage", clues.ToCore(err)) assert.Equal(t, readS3Cfg.Bucket, bkt) assert.Equal(t, cfg.RepoID, "") assert.Equal(t, readS3Cfg.Endpoint, end) assert.Equal(t, readS3Cfg.Prefix, pfx) assert.True(t, readS3Cfg.DoNotUseTLS) assert.True(t, readS3Cfg.DoNotVerifyTLS) common, err := cfg.Storage.CommonConfig() require.NoError(t, err, "reading common config from storage", clues.ToCore(err)) assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase)) readM365, err := cfg.Account.M365Config() require.NoError(t, err, "reading m365 config from account", clues.ToCore(err)) assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID) assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID)) assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret)) }