larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change how we pass flag overrides to config

Browse Source
This commit is contained in:
Abhishek Pandey 2023-09-11 17:18:26 +05:30
parent d740c6a633
commit 57ee18c847
24 changed files with 89 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cli/backup/backup.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
@ -291,7 +290,7 @@ func genericDeleteCommand(
ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
r, _, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, pst, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -317,7 +316,7 @@ func genericListCommand(
) error {
ctx := cmd.Context()
r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

5
src/cli/backup/exchange.go
View File

@ -10,7 +10,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
@ -168,7 +167,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.ExchangeService, repo.S3Overrides(cmd))
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.ExchangeService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -277,7 +276,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd)
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

5
src/cli/backup/groups.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
@ -152,7 +151,7 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, repo.S3Overrides(cmd))
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -224,7 +223,7 @@ func detailsGroupsCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeGroupsOpts(cmd)
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.GroupsService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.GroupsService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

5
src/cli/backup/onedrive.go
View File

@ -10,7 +10,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
@ -149,7 +148,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.OneDriveService, repo.S3Overrides(cmd))
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.OneDriveService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -235,7 +234,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd)
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

5
src/cli/backup/sharepoint.go
View File

@ -11,7 +11,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
@ -159,7 +158,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.SharePointService, repo.S3Overrides(cmd))
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.SharePointService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -319,7 +318,7 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeSharePointOpts(cmd)
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.SharePointService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

3
src/cli/backup/teams.go
View File

@ -11,7 +11,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/fault"
@ -148,7 +147,7 @@ func createTeamsCmd(cmd *cobra.Command, args []string) error {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, repo.S3Overrides(cmd))
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, cmd.Flags())
if err != nil {
return Only(ctx, err)
}

6
src/cli/cli.go
View File

@ -45,7 +45,7 @@ func preRun(cc *cobra.Command, args []string) error {
ctx := cc.Context()
log := logger.Ctx(ctx)
fs := flags.GetPopulatedFlags(cc)
fs := flags.GetPopulatedFlags(cc.Flags())
flagSl := make([]string, 0, len(fs))
// currently only tracking flag names to avoid pii leakage.
@ -70,9 +70,7 @@ func preRun(cc *cobra.Command, args []string) error {
}
if !slices.Contains(avoidTheseDescription, cc.Short) {
overrides := repo.S3Overrides(cc)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, overrides)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, cc.Flags())
if err != nil {
log.Error("Error while getting config info to run command: ", cc.Use)
return err

8
src/cli/config/config.go
View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/alcionai/corso/src/cli/flags"
@ -265,12 +266,12 @@ func GetConfigRepoDetails(
ctx context.Context,
readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string,
pfs *pflag.FlagSet,
) (
RepoDetails,
error,
) {
config, err := getStorageAndAccountWithViper(GetViper(ctx), readFromFile, mustMatchFromConfig, overrides)
config, err := getStorageAndAccountWithViper(GetViper(ctx), readFromFile, mustMatchFromConfig, pfs)
return config, err
}
@ -280,7 +281,7 @@ func getStorageAndAccountWithViper(
vpr *viper.Viper,
readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string,
pfs *pflag.FlagSet,
) (
RepoDetails,
error,
@ -290,6 +291,7 @@ func getStorageAndAccountWithViper(
err error
)
overrides := S3Overrides(pfs)
readConfigFromViper := readFromFile
// possibly read the prior config from a .corso file

49
src/cli/config/storage.go
View File

@ -7,11 +7,13 @@ import (
"github.com/alcionai/clues"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/spf13/pflag"
"github.com/spf13/viper"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -38,6 +40,7 @@ func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Co
return s3Config, nil
}
// Rename this
func s3Overrides(in map[string]string) map[string]string {
return map[string]string{
storage.Bucket: in[storage.Bucket],
@ -49,6 +52,52 @@ func s3Overrides(in map[string]string) map[string]string {
}
}
// Make it local
func S3Overrides(pfs *pflag.FlagSet) map[string]string {
fs := flags.GetPopulatedFlags(pfs)
return PopulateS3Flags(fs)
}
func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string {
s3Overrides := make(map[string]string)
s3Overrides[AccountProviderTypeKey] = account.ProviderM365.String()
s3Overrides[StorageProviderTypeKey] = storage.ProviderS3.String()
if _, ok := flagset[flags.AWSAccessKeyFN]; ok {
s3Overrides[credentials.AWSAccessKeyID] = flags.AWSAccessKeyFV
}
if _, ok := flagset[flags.AWSSecretAccessKeyFN]; ok {
s3Overrides[credentials.AWSSecretAccessKey] = flags.AWSSecretAccessKeyFV
}
if _, ok := flagset[flags.AWSSessionTokenFN]; ok {
s3Overrides[credentials.AWSSessionToken] = flags.AWSSessionTokenFV
}
if _, ok := flagset[flags.BucketFN]; ok {
s3Overrides[storage.Bucket] = flags.BucketFV
}
if _, ok := flagset[flags.PrefixFN]; ok {
s3Overrides[storage.Prefix] = flags.PrefixFV
}
if _, ok := flagset[flags.DoNotUseTLSFN]; ok {
s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(flags.DoNotUseTLSFV)
}
if _, ok := flagset[flags.DoNotVerifyTLSFN]; ok {
s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(flags.DoNotVerifyTLSFV)
}
if _, ok := flagset[flags.EndpointFN]; ok {
s3Overrides[storage.Endpoint] = flags.EndpointFV
}
return s3Overrides
}
// configureStorage builds a complete storage configuration from a mix of
// viper properties and manual overrides.
func configureStorage(

3
src/cli/export/export.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/data"
@ -71,7 +70,7 @@ func runExport(
sel selectors.Selector,
backupID, serviceName string,
) error {
r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), cmd.Flags())
if err != nil {
return Only(ctx, err)
}

9
src/cli/flags/flags.go
View File

@ -1,7 +1,6 @@
package flags
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
@ -22,15 +21,15 @@ func (fs PopulatedFlags) populate(pf *pflag.Flag) {
// GetPopulatedFlags returns a map of flags that have been
// populated by the user. Entry keys match the flag's long
// name. Values are empty.
func GetPopulatedFlags(cmd *cobra.Command) PopulatedFlags {
func GetPopulatedFlags(pfs *pflag.FlagSet) PopulatedFlags {
pop := PopulatedFlags{}
fs := cmd.Flags()
if fs == nil {
// fs := cmd.Flags()
if pfs == nil {
return pop
}
fs.VisitAll(pop.populate)
pfs.VisitAll(pop.populate)
return pop
}

2
src/cli/repo/repo.go
View File

@ -121,7 +121,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.UnknownService, S3Overrides(cmd))
r, _, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.UnknownService, cmd.Flags())
if err != nil {
return print.Only(ctx, err)
}

59
src/cli/repo/s3.go
View File

@ -1,7 +1,6 @@
package repo
import (
"strconv"
"strings"
"github.com/alcionai/clues"
@ -13,10 +12,7 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
)
// called by repo.go to map subcommands to provider-specific handling.
@ -89,10 +85,7 @@ func s3InitCmd() *cobra.Command {
func initS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// s3 values from flags
s3Override := S3Overrides(cmd)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, true, false, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -175,10 +168,7 @@ func s3ConnectCmd() *cobra.Command {
func connectS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// s3 values from flags
s3Override := S3Overrides(cmd)
cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, true, true, cmd.Flags())
if err != nil {
return Only(ctx, err)
}
@ -227,48 +217,3 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return nil
}
func S3Overrides(cmd *cobra.Command) map[string]string {
fs := flags.GetPopulatedFlags(cmd)
return PopulateS3Flags(fs)
}
func PopulateS3Flags(flagset flags.PopulatedFlags) map[string]string {
s3Overrides := make(map[string]string)
s3Overrides[config.AccountProviderTypeKey] = account.ProviderM365.String()
s3Overrides[config.StorageProviderTypeKey] = storage.ProviderS3.String()
if _, ok := flagset[flags.AWSAccessKeyFN]; ok {
s3Overrides[credentials.AWSAccessKeyID] = flags.AWSAccessKeyFV
}
if _, ok := flagset[flags.AWSSecretAccessKeyFN]; ok {
s3Overrides[credentials.AWSSecretAccessKey] = flags.AWSSecretAccessKeyFV
}
if _, ok := flagset[flags.AWSSessionTokenFN]; ok {
s3Overrides[credentials.AWSSessionToken] = flags.AWSSessionTokenFV
}
if _, ok := flagset[flags.BucketFN]; ok {
s3Overrides[storage.Bucket] = flags.BucketFV
}
if _, ok := flagset[flags.PrefixFN]; ok {
s3Overrides[storage.Prefix] = flags.PrefixFV
}
if _, ok := flagset[flags.DoNotUseTLSFN]; ok {
s3Overrides[storage.DoNotUseTLS] = strconv.FormatBool(flags.DoNotUseTLSFV)
}
if _, ok := flagset[flags.DoNotVerifyTLSFN]; ok {
s3Overrides[storage.DoNotVerifyTLS] = strconv.FormatBool(flags.DoNotVerifyTLSFV)
}
if _, ok := flagset[flags.EndpointFN]; ok {
s3Overrides[storage.Endpoint] = flags.EndpointFV
}
return s3Overrides
}

3
src/cli/restore/restore.go
View File

@ -10,7 +10,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/count"
@ -104,7 +103,7 @@ func runRestore(
sel selectors.Selector,
backupID, serviceName string,
) error {
r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), cmd.Flags())
if err != nil {
return Only(ctx, err)
}

2
src/cli/utils/exchange.go
View File

@ -64,7 +64,7 @@ func MakeExchangeOpts(cmd *cobra.Command) ExchangeOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/export_config.go
View File

@ -22,7 +22,7 @@ func makeExportCfgOpts(cmd *cobra.Command) ExportCfgOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/groups.go
View File

@ -53,7 +53,7 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/onedrive.go
View File

@ -41,7 +41,7 @@ func MakeOneDriveOpts(cmd *cobra.Command) OneDriveOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/restore_config.go
View File

@ -36,7 +36,7 @@ func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/retention.go
View File

@ -28,7 +28,7 @@ func makeRetentionCfgOpts(cmd *cobra.Command) retentionCfgOpts {
// Populated contains the list of flags that appear in the command,
// according to pflags. Use this to differentiate between an "empty" and a
// "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/sharepoint.go
View File

@ -62,7 +62,7 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

2
src/cli/utils/teams.go
View File

@ -25,6 +25,6 @@ func MakeTeamsOpts(cmd *cobra.Command) TeamsOpts {
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
Populated: flags.GetPopulatedFlags(cmd.Flags()),
}
}

8
src/cli/utils/utils.go
View File

@ -24,9 +24,9 @@ var ErrNotYetImplemented = clues.New("not yet implemented")
func GetAccountAndConnect(
ctx context.Context,
pst path.ServiceType,
overrides map[string]string,
pfs *pflag.FlagSet,
) (repository.Repository, *storage.Storage, *account.Account, *control.Options, error) {
cfg, err := config.GetConfigRepoDetails(ctx, true, true, overrides)
cfg, err := config.GetConfigRepoDetails(ctx, true, true, pfs)
if err != nil {
return nil, nil, nil, nil, err
}
@ -55,9 +55,9 @@ func GetAccountAndConnect(
func AccountConnectAndWriteRepoConfig(
ctx context.Context,
pst path.ServiceType,
overrides map[string]string,
pfs *pflag.FlagSet,
) (repository.Repository, *account.Account, error) {
r, stg, acc, opts, err := GetAccountAndConnect(ctx, pst, overrides)
r, stg, acc, opts, err := GetAccountAndConnect(ctx, pst, pfs)
if err != nil {
logger.CtxErr(ctx, err).Info("getting and connecting account")
return nil, nil, err

9
src/cmd/s3checker/s3checker.go
View File

@ -16,7 +16,6 @@ import (
"github.com/alcionai/corso/src/cmd/s3checker/pkg/s3"
"github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/storage"
)
// Matches other definitions of this const.
@ -181,13 +180,7 @@ func handleCheckerCommand(cmd *cobra.Command, args []string, f flags) error {
ctx := cmd.Context()
// Scavenged from src/internal/kopia/s3/s3.go.
overrides := map[string]string{
storage.Bucket: f.bucket,
storage.Prefix: f.bucketPrefix,
}
repoDetails, err := config.GetConfigRepoDetails(ctx, false, false, overrides)
repoDetails, err := config.GetConfigRepoDetails(ctx, false, false, cmd.Flags())
if err != nil {
return clues.Wrap(err, "getting storage config")
}