larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use config file for backup create (#171)

Browse Source 
* use config file for backup create

Now that corso stores its prior connection details in a config
file, that file should get used to retrieve storage details as needed.
This commit is contained in:
Keepers 2022-06-09 14:28:26 -06:00 committed by GitHub
parent 9abfb486d8
commit 340d875579
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 174 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/cli/backup/exchange.go
View File

@ -6,10 +6,10 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/alcionai/corso/cli/config"
"github.com/alcionai/corso/cli/utils"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
)
// exchange bucket info from flags
@ -40,7 +40,21 @@ var exchangeCreateCmd = &cobra.Command{
// initializes a s3 repo.
func createExchangeCmd(cmd *cobra.Command, args []string) error {
s, cfgTenantID, err := config.MakeS3Config(true, nil)
if err != nil {
return err
}
m365 := credentials.GetM365()
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
if len(cfgTenantID) > 0 {
a.TenantID = cfgTenantID
}
fmt.Printf(
"Called - %s\n\t365TenantID:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n",
cmd.CommandPath(),
@ -48,17 +62,6 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
m365.ClientID,
len(m365.ClientSecret) > 0)
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
// todo (rkeepers) - retrieve storage details from corso config
s, err := storage.NewStorage(storage.ProviderUnknown)
if err != nil {
return errors.Wrap(err, "Failed to configure storage provider")
}
r, err := repository.Connect(cmd.Context(), a, s)
if err != nil {
return errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider)

69
src/cli/config/config.go
View File

@ -5,6 +5,8 @@ import (
"path"
"strings"
"github.com/alcionai/corso/cli/utils"
"github.com/alcionai/corso/pkg/credentials"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
"github.com/pkg/errors"
@ -92,3 +94,70 @@ func ReadRepoConfig() (s3Config storage.S3Config, account repository.Account, er
return s3Config, account, nil
}
// MakeS3Config creates a storage instance by mediating all the possible
// data sources (config file, env vars, flag overrides) in the config.
func MakeS3Config(readFromFile bool, overrides map[string]string) (storage.Storage, string, error) {
var (
s3Cfg storage.S3Config
account repository.Account
err error
)
// possibly read the prior config from a .corso file
if readFromFile {
s3Cfg, account, err = ReadRepoConfig()
if err != nil {
return storage.Storage{}, "", errors.Wrap(err, "reading corso config file")
}
}
// compose the s3 storage config and credentials
aws := credentials.GetAWS(overrides)
if err := aws.Validate(); err != nil {
return storage.Storage{}, "", errors.Wrap(err, "validating aws credentials")
}
s3Cfg = storage.S3Config{
AWS: aws,
Bucket: first(overrides[storage.Bucket], s3Cfg.Bucket),
Endpoint: first(overrides[storage.Endpoint], s3Cfg.Endpoint),
Prefix: first(overrides[storage.Prefix], s3Cfg.Prefix),
}
// compose the common config and credentials
corso := credentials.GetCorso()
if err := corso.Validate(); err != nil {
return storage.Storage{}, "", errors.Wrap(err, "validating corso credentials")
}
cCfg := storage.CommonConfig{
Corso: corso,
}
// ensure requried properties are present
if err := utils.RequireProps(map[string]string{
credentials.AWSAccessKeyID: aws.AccessKey,
storage.Bucket: s3Cfg.Bucket,
credentials.AWSSecretAccessKey: aws.SecretKey,
credentials.AWSSessionToken: aws.SessionToken,
credentials.CorsoPassword: corso.CorsoPassword,
}); err != nil {
return storage.Storage{}, "", err
}
// return a complete storage
s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, cCfg)
if err != nil {
return storage.Storage{}, "", errors.Wrap(err, "configuring repository storage")
}
return s, account.TenantID, nil
}
// returns the first non-zero valued string
func first(vs ...string) string {
for _, v := range vs {
if len(v) > 0 {
return v
}
}
return ""
}

11
src/cli/config/config_test.go
View File

@ -6,12 +6,13 @@ import (
"path"
"testing"
"github.com/alcionai/corso/cli/config"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/cli/config"
"github.com/alcionai/corso/pkg/repository"
"github.com/alcionai/corso/pkg/storage"
)
const (
@ -34,7 +35,7 @@ func TestConfigSuite(t *testing.T) {
func (suite *ConfigSuite) TestReadRepoConfigBasic() {
// Generate test config file
b := "test-bucket"
b := "read-repo-config-basic-bucket"
tID := "6f34ac30-8196-469b-bf8f-d83deadbbbba"
testConfigData := fmt.Sprintf(configFileTemplate, b, tID)
testConfigFilePath := path.Join(suite.T().TempDir(), "corso.toml")
@ -58,7 +59,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
err := config.InitConfig(testConfigFilePath)
assert.NoError(suite.T(), err)
s3Cfg := storage.S3Config{Bucket: "bucket"}
s3Cfg := storage.S3Config{Bucket: "write-read-config-bucket"}
account := repository.Account{TenantID: "6f34ac30-8196-469b-bf8f-d83deadbbbbd"}
err = config.WriteRepoConfig(s3Cfg, account)
assert.NoError(suite.T(), err)

96
src/cli/repo/s3.go
View File

@ -5,7 +5,6 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/alcionai/corso/cli/config"
"github.com/alcionai/corso/cli/utils"
@ -55,11 +54,30 @@ var s3InitCmd = &cobra.Command{
func initS3Cmd(cmd *cobra.Command, args []string) error {
log := logger.Ctx(cmd.Context())
m365 := credentials.GetM365()
s3Cfg, commonCfg, err := makeS3Config()
overrides := map[string]string{
credentials.AWSAccessKeyID: accessKey,
storage.Bucket: bucket,
storage.Endpoint: endpoint,
storage.Prefix: prefix,
}
s, cfgTenantID, err := config.MakeS3Config(false, overrides)
if err != nil {
return err
}
s3Cfg, err := s.S3Config()
if err != nil {
return errors.Wrap(err, "Retrieving s3 configuration")
}
m365 := credentials.GetM365()
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
if len(cfgTenantID) > 0 {
a.TenantID = cfgTenantID
}
log.Debugw(
"Called - "+cmd.CommandPath(),
@ -69,16 +87,6 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
"accessKey", s3Cfg.AccessKey,
"hasSecretKey", len(s3Cfg.SecretKey) > 0)
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if err != nil {
return errors.Wrap(err, "Failed to configure storage provider")
}
r, err := repository.Initialize(cmd.Context(), a, s)
if err != nil {
return errors.Wrap(err, "Failed to initialize a new S3 repository")
@ -106,11 +114,30 @@ var s3ConnectCmd = &cobra.Command{
func connectS3Cmd(cmd *cobra.Command, args []string) error {
log := logger.Ctx(cmd.Context())
m365 := credentials.GetM365()
s3Cfg, commonCfg, err := makeS3Config()
overrides := map[string]string{
credentials.AWSAccessKeyID: accessKey,
storage.Bucket: bucket,
storage.Endpoint: endpoint,
storage.Prefix: prefix,
}
s, cfgTenantID, err := config.MakeS3Config(true, overrides)
if err != nil {
return err
}
s3Cfg, err := s.S3Config()
if err != nil {
return errors.Wrap(err, "Retrieving s3 configuration")
}
m365 := credentials.GetM365()
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
if len(cfgTenantID) > 0 {
a.TenantID = cfgTenantID
}
log.Debugw(
"Called - "+cmd.CommandPath(),
@ -120,23 +147,6 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
"accessKey", s3Cfg.AccessKey,
"hasSecretKey", len(s3Cfg.SecretKey) > 0)
// TODO: Merge/Validate any local configuration here to make sure there are no conflicts
// For now - just reading/logging the local config here (a successful repo connect will overwrite)
localS3Cfg, localAccount, err := config.ReadRepoConfig()
if err == nil {
fmt.Printf("ConfigFile - %s\n\tbucket:\t%s\n\ttenantID:\t%s\n", viper.ConfigFileUsed(), localS3Cfg.Bucket, localAccount.TenantID)
}
a := repository.Account{
TenantID: m365.TenantID,
ClientID: m365.ClientID,
ClientSecret: m365.ClientSecret,
}
s, err := storage.NewStorage(storage.ProviderS3, s3Cfg, commonCfg)
if err != nil {
return errors.Wrap(err, "Failed to configure storage provider")
}
r, err := repository.Connect(cmd.Context(), a, s)
if err != nil {
return errors.Wrap(err, "Failed to connect to the S3 repository")
@ -150,25 +160,3 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
}
return nil
}
// helper for aggregating aws connection details.
func makeS3Config() (storage.S3Config, storage.CommonConfig, error) {
aws := credentials.GetAWS(map[string]string{credentials.AWSAccessKeyID: accessKey})
corso := credentials.GetCorso()
return storage.S3Config{
AWS: aws,
Bucket: bucket,
Endpoint: endpoint,
Prefix: prefix,
},
storage.CommonConfig{
Corso: corso,
},
utils.RequireProps(map[string]string{
credentials.AWSAccessKeyID: aws.AccessKey,
"bucket": bucket,
credentials.AWSSecretAccessKey: aws.SecretKey,
credentials.AWSSessionToken: aws.SessionToken,
credentials.CorsoPassword: corso.CorsoPassword,
})
}

2
src/internal/testing/integration_runners.go
View File

@ -12,8 +12,8 @@ import (
const (
CorsoCITests = "CORSO_CI_TESTS"
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS"
CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS"
CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS"
)
// RunOnAny takes in a list of env variable names and returns

16
src/pkg/credentials/aws.go
View File

@ -2,6 +2,8 @@ package credentials
import (
"os"
"github.com/pkg/errors"
)
// envvar consts
@ -35,3 +37,17 @@ func GetAWS(override map[string]string) AWS {
SessionToken: sessToken,
}
}
func (c AWS) Validate() error {
check := map[string]string{
AWSAccessKeyID: c.AccessKey,
AWSSecretAccessKey: c.SecretKey,
AWSSessionToken: c.SessionToken,
}
for k, v := range check {
if len(v) == 0 {
return errors.Wrap(errMissingRequired, k)
}
}
return nil
}

18
src/pkg/credentials/corso.go
View File

@ -1,6 +1,10 @@
package credentials
import "os"
import (
"os"
"github.com/pkg/errors"
)
// envvar consts
const (
@ -21,3 +25,15 @@ func GetCorso() Corso {
CorsoPassword: corsoPasswd,
}
}
func (c Corso) Validate() error {
check := map[string]string{
CorsoPassword: c.CorsoPassword,
}
for k, v := range check {
if len(v) == 0 {
return errors.Wrap(errMissingRequired, k)
}
}
return nil
}

9
src/pkg/storage/s3.go
View File

@ -24,6 +24,13 @@ const (
keyS3SessionToken = "s3_sessionToken"
)
// config exported name consts
const (
Bucket = "bucket"
Endpoint = "endpoint"
Prefix = "prefix"
)
func (c S3Config) Config() (config, error) {
cfg := config{
keyS3AccessKey: c.AccessKey,
@ -55,7 +62,7 @@ func (c S3Config) validate() error {
credentials.AWSAccessKeyID: c.AccessKey,
credentials.AWSSecretAccessKey: c.SecretKey,
credentials.AWSSessionToken: c.SessionToken,
"bucket": c.Bucket,
Bucket: c.Bucket,
}
for k, v := range check {
if len(v) == 0 {