larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correct default value for s3 endpoint (#3710)

Browse Source 
<!-- PR description-->

Use default values from endpoint only if its not present in config file

#### Does this PR need a docs update or release note?

- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🐛 Bugfix

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
This commit is contained in:
neha_gupta 2023-06-30 18:47:31 +05:30 committed by GitHub
parent 9d801efa03
commit ead13b0b1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 79 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli/backup/backup.go
View File

@ -271,7 +271,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st
ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
@ -292,7 +292,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st
func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType, args []string) error {
ctx := cmd.Context()
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

4
src/cli/backup/exchange.go
View File

@ -166,7 +166,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
@ -275,7 +275,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

4
src/cli/backup/onedrive.go
View File

@ -148,7 +148,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
@ -234,7 +234,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

4
src/cli/backup/sharepoint.go
View File

@ -163,7 +163,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
@ -325,7 +325,7 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeSharePointOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

2
src/cli/cli.go
View File

@ -69,7 +69,7 @@ func preRun(cc *cobra.Command, args []string) error {
}
if !slices.Contains(avoidTheseDescription, cc.Short) {
overrides := repo.S3Overrides()
overrides := repo.S3Overrides(cc)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, overrides)
if err != nil {

4
src/cli/config/storage.go
View File

@ -114,15 +114,17 @@ func configureStorage(
s3Cfg = storage.S3Config{
AWS: aws,
Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket),
Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint),
Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, "s3.amazonaws.com"),
Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix),
DoNotUseTLS: str.ParseBool(str.First(
overrides[storage.DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS),
"false",
)),
DoNotVerifyTLS: str.ParseBool(str.First(
overrides[storage.DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
"false",
)),
}

2
src/cli/repo/repo.go
View File

@ -122,7 +122,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, _, err := utils.GetAccountAndConnect(ctx, S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, S3Overrides(cmd))
if err != nil {
return print.Only(ctx, err)
}

86
src/cli/repo/s3.go
View File

@ -32,6 +32,16 @@ var (
doNotVerifyTLS bool
)
// s3 bucket flags
const (
succeedIfExistsFN = "succeedIfExists"
bucketFN = "bucket"
endpointFN = "endpoint"
prefixFN = "prefix"
doNotUseTLSFN = "disable-tls"
doNotVerifyTLSFN = "disable-tls-verification"
)
// called by repo.go to map subcommands to provider-specific handling.
func addS3Commands(cmd *cobra.Command) *cobra.Command {
var (
@ -55,11 +65,11 @@ func addS3Commands(cmd *cobra.Command) *cobra.Command {
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic and more frequently used flags take precedence.
fs.StringVar(&bucket, "bucket", "", "Name of S3 bucket for repo. (required)")
fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.")
fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.")
fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)")
fs.BoolVar(&doNotVerifyTLS, "disable-tls-verification", false, "Disable TLS (HTTPS) certificate verification.")
fs.StringVar(&bucket, bucketFN, "", "Name of S3 bucket for repo. (required)")
fs.StringVar(&prefix, prefixFN, "", "Repo prefix within bucket.")
fs.StringVar(&endpoint, endpointFN, "", "S3 service endpoint.")
fs.BoolVar(&doNotUseTLS, doNotUseTLSFN, false, "Disable TLS (HTTPS)")
fs.BoolVar(&doNotVerifyTLS, doNotVerifyTLSFN, false, "Disable TLS (HTTPS) certificate verification.")
// In general, we don't want to expose this flag to users and have them mistake it
// for a broad-scale idempotency solution. We can un-hide it later the need arises.
@ -115,9 +125,9 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// s3 values from flags
s3Override := S3Overrides()
s3Override := S3Overrides(cmd)
// s3 values from envs
s3Override = S3UpdateFromEnvVar(s3Override)
s3Override = S3UpdateFromEnvVar(cmd, s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
if err != nil {
@ -191,9 +201,9 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// s3 values from flags
s3Override := S3Overrides()
s3Override := S3Overrides(cmd)
// s3 values from envs
s3Override = S3UpdateFromEnvVar(s3Override)
s3Override = S3UpdateFromEnvVar(cmd, s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
if err != nil {
@ -238,22 +248,52 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return nil
}
func S3Overrides() map[string]string {
return map[string]string{
config.AccountProviderTypeKey: account.ProviderM365.String(),
config.StorageProviderTypeKey: storage.ProviderS3.String(),
credentials.AWSAccessKeyID: flags.AWSAccessKeyFV,
credentials.AWSSecretAccessKey: flags.AWSSecretAccessKeyFV,
credentials.AWSSessionToken: flags.AWSSessionTokenFV,
storage.Bucket: bucket,
storage.Endpoint: endpoint,
storage.Prefix: prefix,
storage.DoNotUseTLS: strconv.FormatBool(doNotUseTLS),
storage.DoNotVerifyTLS: strconv.FormatBool(doNotVerifyTLS),
}
func S3Overrides(cmd *cobra.Command) map[string]string {
fs := flags.GetPopulatedFlags(cmd)
return PopulateS3Flags(fs)
}
func S3UpdateFromEnvVar(s3Flag map[string]string) map[string]string {
func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string {
s3Overrides := make(map[string]string)
s3Overrides[config.AccountProviderTypeKey] = account.ProviderM365.String()
s3Overrides[config.StorageProviderTypeKey] = storage.ProviderS3.String()
if _, ok := flagset[flags.AWSAccessKeyFN]; ok {
s3Overrides[credentials.AWSAccessKeyID] = flags.AWSAccessKeyFV
}
if _, ok := flagset[flags.AWSSecretAccessKeyFN]; ok {
s3Overrides[credentials.AWSSecretAccessKey] = flags.AWSSecretAccessKeyFV
}
if _, ok := flagset[flags.AWSSessionTokenFN]; ok {
s3Overrides[credentials.AWSSessionToken] = flags.AWSSessionTokenFV
}
if _, ok := flagset[bucketFN]; ok {
s3Overrides[storage.Bucket] = bucket
}
if _, ok := flagset[prefixFN]; ok {
s3Overrides[storage.Prefix] = prefix
}
if _, ok := flagset[doNotUseTLSFN]; ok {
s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(doNotUseTLS)
}
if _, ok := flagset[doNotVerifyTLSFN]; ok {
s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(doNotVerifyTLS)
}
if _, ok := flagset[endpointFN]; ok {
s3Overrides[storage.Endpoint] = endpoint
}
return s3Overrides
}
func S3UpdateFromEnvVar(cmd *cobra.Command, s3Flag map[string]string) map[string]string {
s3Flag[storage.Bucket] = str.First(s3Flag[storage.Bucket], os.Getenv(storage.BucketKey))
s3Flag[storage.Endpoint] = str.First(s3Flag[storage.Endpoint], os.Getenv(storage.EndpointKey))
s3Flag[storage.Prefix] = str.First(s3Flag[storage.Prefix], os.Getenv(storage.PrefixKey))

2
src/cli/restore/exchange.go
View File

@ -93,7 +93,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

2
src/cli/restore/onedrive.go
View File

@ -92,7 +92,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

2
src/cli/restore/sharepoint.go
View File

@ -99,7 +99,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}