add flags for azure and aws (#3590)

<!-- PR description-->
Flags for all configs-

Azure cred flags- (azure-tenant-id, azure-client-id, azure-client-secret) present in -
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

AWS cred flags - (aws-access-key, aws-secret-access-key, aws-session-token) present in- 
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

Passphrase flag- (--passphrase) present in- 
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

S3 flags- 
--endpoint, --prefix, --bucket, --disable-tls, --disable-tls-verification - flags is for repo init and connect commands
all the S3 env var will also work only in case of repo init and connect command. For all other commands  user first connects to repo. Which will store the config values in config file. And then user can use that config file for other commands.

No cred configs are save in the config file by Corso. 

Config file values added- 
Azure cred - 
- azure_client_id 
- azure_secret 
- azure_tenantid

AWS cred -
- aws_access_key_id
- aws_secret_access_key
- aws_session_token

Passphrase -
- passphrase

**NOTE:** 
- in case of AWS creds all the three values should be provided from same method. Either put all values in env, config file and so on.
- all the S3 env var will also work only in case of repo init and connect command. For all other commands  user first connects to repo. Which will store the config values in config file. And then user can use that config file for other commands.



---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature


#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/issues/3522

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x]  Unit test
- [ ] 💚 E2E
This commit is contained in:
neha_gupta 2023-06-29 11:00:15 +05:30 committed by GitHub
parent 78f698636d
commit 8c661164ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 689 additions and 104 deletions

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -270,7 +271,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st
ctx := clues.Add(cmd.Context(), "delete_backup_id", bID) ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -291,7 +292,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st
func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType, args []string) error { func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
@ -84,6 +85,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
// More generic (ex: --user) and more frequently used flags take precedence. // More generic (ex: --user) and more frequently used flags take precedence.
flags.AddMailBoxFlag(c) flags.AddMailBoxFlag(c)
flags.AddDataFlag(c, []string{dataEmail, dataContacts, dataEvents}, false) flags.AddDataFlag(c, []string{dataEmail, dataContacts, dataEvents}, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddFetchParallelismFlag(c) flags.AddFetchParallelismFlag(c)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddDisableIncrementalsFlag(c) flags.AddDisableIncrementalsFlag(c)
@ -96,6 +100,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false fs.SortFlags = false
flags.AddBackupIDFlag(c, false) flags.AddBackupIDFlag(c, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
addFailedItemsFN(c) addFailedItemsFN(c)
addSkippedItemsFN(c) addSkippedItemsFN(c)
addRecoveredErrorsFN(c) addRecoveredErrorsFN(c)
@ -112,6 +119,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
// Flags addition ordering should follow the order we want them to appear in help and docs: // Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence. // More generic (ex: --user) and more frequently used flags take precedence.
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddExchangeDetailsAndRestoreFlags(c) flags.AddExchangeDetailsAndRestoreFlags(c)
case deleteCommand: case deleteCommand:
@ -122,6 +132,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
c.Example = exchangeServiceCommandDeleteExamples c.Example = exchangeServiceCommandDeleteExamples
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -153,7 +166,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx) r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -262,7 +275,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd) opts := utils.MakeExchangeOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -35,6 +35,184 @@ var (
events = path.EventsCategory events = path.EventsCategory
) )
// ---------------------------------------------------------------------------
// tests with azure flags in exchange create
// ---------------------------------------------------------------------------
type ExchangeCMDWithFlagsE2ESuite struct {
tester.Suite
acct account.Account
st storage.Storage
vpr *viper.Viper
cfgFP string
repo repository.Repository
m365UserID string
recorder strings.Builder
}
func TestExchangeCMDWithFlagsE2ESuite(t *testing.T) {
suite.Run(t, &ExchangeCMDWithFlagsE2ESuite{Suite: tester.NewE2ESuite(
t,
[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
)})
}
func (suite *ExchangeCMDWithFlagsE2ESuite) SetupSuite() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)
suite.acct = acct
suite.st = st
suite.repo = repo
suite.vpr = vpr
suite.recorder = recorder
suite.cfgFP = cfgFilePath
suite.m365UserID = tester.M365UserID(t)
}
func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_badAzureClientID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--azure-client-id", "invalid-value",
)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_azureIDFromConfigFile() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--config-file", suite.cfgFP)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := suite.recorder.String()
t.Log("backup results", result)
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}
func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupValueFromEnvCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := suite.recorder.String()
t.Log("backup results", result)
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}
// AWS flags
func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupInvalidAWSClientIDCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--aws-access-key", "invalid-value",
"--aws-secret-access-key", "some-invalid-value",
)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
// since invalid aws creds are explicitly set, should see a failure
require.Error(t, err, clues.ToCore(err))
}
func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupAWSValueFromEnvCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := suite.recorder.String()
t.Log("backup results", result)
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// tests with no backups // tests with no backups
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
@ -71,6 +72,10 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
c.Example = oneDriveServiceCommandCreateExamples c.Example = oneDriveServiceCommandCreateExamples
flags.AddUserFlag(c) flags.AddUserFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddDisableIncrementalsFlag(c) flags.AddDisableIncrementalsFlag(c)
@ -79,6 +84,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false fs.SortFlags = false
flags.AddBackupIDFlag(c, false) flags.AddBackupIDFlag(c, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
addFailedItemsFN(c) addFailedItemsFN(c)
addSkippedItemsFN(c) addSkippedItemsFN(c)
addRecoveredErrorsFN(c) addRecoveredErrorsFN(c)
@ -92,6 +100,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSkipReduceFlag(c) flags.AddSkipReduceFlag(c)
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddOneDriveDetailsAndRestoreFlags(c) flags.AddOneDriveDetailsAndRestoreFlags(c)
case deleteCommand: case deleteCommand:
@ -102,6 +113,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
c.Example = oneDriveServiceCommandDeleteExamples c.Example = oneDriveServiceCommandDeleteExamples
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -134,7 +148,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx) r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -220,7 +234,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd) opts := utils.MakeOneDriveOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -86,6 +87,9 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSiteFlag(c) flags.AddSiteFlag(c)
flags.AddSiteIDFlag(c) flags.AddSiteIDFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddDataFlag(c, []string{dataLibraries}, true) flags.AddDataFlag(c, []string{dataLibraries}, true)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddDisableIncrementalsFlag(c) flags.AddDisableIncrementalsFlag(c)
@ -95,6 +99,9 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false fs.SortFlags = false
flags.AddBackupIDFlag(c, false) flags.AddBackupIDFlag(c, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
addFailedItemsFN(c) addFailedItemsFN(c)
addSkippedItemsFN(c) addSkippedItemsFN(c)
addRecoveredErrorsFN(c) addRecoveredErrorsFN(c)
@ -108,6 +115,9 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSkipReduceFlag(c) flags.AddSkipReduceFlag(c)
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddSharePointDetailsAndRestoreFlags(c) flags.AddSharePointDetailsAndRestoreFlags(c)
case deleteCommand: case deleteCommand:
@ -118,6 +128,9 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
c.Example = sharePointServiceCommandDeleteExamples c.Example = sharePointServiceCommandDeleteExamples
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -150,7 +163,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx) r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -312,7 +325,7 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
opts := utils.MakeSharePointOpts(cmd) opts := utils.MakeSharePointOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -65,17 +65,13 @@ func preRun(cc *cobra.Command, args []string) error {
"Initialize a S3 repository", "Initialize a S3 repository",
"Help about any command", "Help about any command",
"Free, Secure, Open-Source Backup for M365.", "Free, Secure, Open-Source Backup for M365.",
"env var guide",
} }
if !slices.Contains(avoidTheseDescription, cc.Short) { if !slices.Contains(avoidTheseDescription, cc.Short) {
overrides := map[string]string{} overrides := repo.S3Overrides()
if cc.Short == "Connect to a S3 repository" {
// Get s3 overrides for connect. Ideally we also need this
// for init, but we don't reach this block for init.
overrides = repo.S3Overrides()
}
cfg, err := config.GetConfigRepoDetails(ctx, true, overrides) cfg, err := config.GetConfigRepoDetails(ctx, true, false, overrides)
if err != nil { if err != nil {
log.Error("Error while getting config info to run command: ", cc.Use) log.Error("Error while getting config info to run command: ", cc.Use)
return err return err

View File

@ -6,6 +6,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
@ -20,6 +21,8 @@ func m365ConfigsFromViper(vpr *viper.Viper) (account.M365Config, error) {
return m365, clues.New("unsupported account provider: " + providerType) return m365, clues.New("unsupported account provider: " + providerType)
} }
m365.AzureClientID = vpr.GetString(AzureClientID)
m365.AzureClientSecret = vpr.GetString(AzureSecret)
m365.AzureTenantID = vpr.GetString(AzureTenantIDKey) m365.AzureTenantID = vpr.GetString(AzureTenantIDKey)
return m365, nil return m365, nil
@ -41,6 +44,7 @@ func configureAccount(
) (account.Account, error) { ) (account.Account, error) {
var ( var (
m365Cfg account.M365Config m365Cfg account.M365Config
m365 credentials.M365
acct account.Account acct account.Account
err error err error
) )
@ -57,7 +61,7 @@ func configureAccount(
} }
// compose the m365 config and credentials // compose the m365 config and credentials
m365 := credentials.GetM365() m365 = GetM365(m365Cfg)
if err := m365.Validate(); err != nil { if err := m365.Validate(); err != nil {
return acct, clues.Wrap(err, "validating m365 credentials") return acct, clues.Wrap(err, "validating m365 credentials")
} }
@ -66,14 +70,15 @@ func configureAccount(
M365: m365, M365: m365,
AzureTenantID: str.First( AzureTenantID: str.First(
overrides[account.AzureTenantID], overrides[account.AzureTenantID],
m365Cfg.AzureTenantID, flags.AzureClientTenantFV,
os.Getenv(account.AzureTenantID)), os.Getenv(account.AzureTenantID),
m365Cfg.AzureTenantID),
} }
// ensure required properties are present // ensure required properties are present
if err := requireProps(map[string]string{ if err := requireProps(map[string]string{
credentials.AzureClientID: m365Cfg.AzureClientID, credentials.AzureClientID: m365Cfg.M365.AzureClientID,
credentials.AzureClientSecret: m365Cfg.AzureClientSecret, credentials.AzureClientSecret: m365Cfg.M365.AzureClientSecret,
account.AzureTenantID: m365Cfg.AzureTenantID, account.AzureTenantID: m365Cfg.AzureTenantID,
}); err != nil { }); err != nil {
return acct, err return acct, err
@ -87,3 +92,20 @@ func configureAccount(
return acct, nil return acct, nil
} }
// M365 is a helper for aggregating m365 secrets and credentials.
func GetM365(m365Cfg account.M365Config) credentials.M365 {
AzureClientID := str.First(
flags.AzureClientIDFV,
os.Getenv(credentials.AzureClientID),
m365Cfg.AzureClientID)
AzureClientSecret := str.First(
flags.AzureClientSecretFV,
os.Getenv(credentials.AzureClientSecret),
m365Cfg.AzureClientSecret)
return credentials.M365{
AzureClientID: AzureClientID,
AzureClientSecret: AzureClientSecret,
}
}

View File

@ -26,9 +26,18 @@ const (
DisableTLSVerificationKey = "disable_tls_verification" DisableTLSVerificationKey = "disable_tls_verification"
RepoID = "repo_id" RepoID = "repo_id"
AccessKey = "aws_access_key_id"
SecretAccessKey = "aws_secret_access_key"
SessionToken = "aws_session_token"
// M365 config // M365 config
AccountProviderTypeKey = "account_provider" AccountProviderTypeKey = "account_provider"
AzureTenantIDKey = "azure_tenantid" AzureTenantIDKey = "azure_tenantid"
AzureClientID = "azure_client_id"
AzureSecret = "azure_secret"
// Corso passphrase in config
CorsoPassphrase = "passphrase"
) )
var ( var (
@ -232,12 +241,13 @@ func writeRepoConfigWithViper(
func GetConfigRepoDetails( func GetConfigRepoDetails(
ctx context.Context, ctx context.Context,
readFromFile bool, readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string, overrides map[string]string,
) ( ) (
RepoDetails, RepoDetails,
error, error,
) { ) {
config, err := getStorageAndAccountWithViper(GetViper(ctx), readFromFile, overrides) config, err := getStorageAndAccountWithViper(GetViper(ctx), readFromFile, mustMatchFromConfig, overrides)
return config, err return config, err
} }
@ -246,6 +256,7 @@ func GetConfigRepoDetails(
func getStorageAndAccountWithViper( func getStorageAndAccountWithViper(
vpr *viper.Viper, vpr *viper.Viper,
readFromFile bool, readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string, overrides map[string]string,
) ( ) (
RepoDetails, RepoDetails,
@ -278,7 +289,7 @@ func getStorageAndAccountWithViper(
return config, clues.Wrap(err, "retrieving account configuration details") return config, clues.Wrap(err, "retrieving account configuration details")
} }
config.Storage, err = configureStorage(vpr, readConfigFromViper, overrides) config.Storage, err = configureStorage(vpr, readConfigFromViper, mustMatchFromConfig, overrides)
if err != nil { if err != nil {
return config, clues.Wrap(err, "retrieving storage provider details") return config, clues.Wrap(err, "retrieving storage provider details")
} }

View File

@ -28,6 +28,10 @@ const (
` + AzureTenantIDKey + ` = '%s' ` + AzureTenantIDKey + ` = '%s'
` + DisableTLSKey + ` = 'false' ` + DisableTLSKey + ` = 'false'
` + DisableTLSVerificationKey + ` = 'false' ` + DisableTLSVerificationKey + ` = 'false'
` + AccessKey + ` = '%s'
` + SecretAccessKey + ` = '%s'
` + SessionToken + ` = '%s'
` + CorsoPassphrase + ` = '%s'
` `
) )
@ -67,12 +71,16 @@ func (suite *ConfigSuite) TestReadRepoConfigBasic() {
) )
const ( const (
b = "read-repo-config-basic-bucket" b = "read-repo-config-basic-bucket"
tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba" tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba"
accKey = "aws-test-access-key"
secret = "aws-test-secret-key"
token = "aws-test-session-token"
passphrase = "passphrase-test"
) )
// Generate test config file // Generate test config file
testConfigData := fmt.Sprintf(configFileTemplate, b, tID) testConfigData := fmt.Sprintf(configFileTemplate, b, tID, accKey, secret, token, passphrase)
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml") testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700) err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -88,6 +96,12 @@ func (suite *ConfigSuite) TestReadRepoConfigBasic() {
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, b, s3Cfg.Bucket) assert.Equal(t, b, s3Cfg.Bucket)
s3Cfg, err = s3CredsFromViper(vpr, s3Cfg)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, accKey, s3Cfg.AWS.AccessKey)
assert.Equal(t, secret, s3Cfg.AWS.SecretKey)
assert.Equal(t, token, s3Cfg.AWS.SessionToken)
m365, err := m365ConfigsFromViper(vpr) m365, err := m365ConfigsFromViper(vpr)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, tID, m365.AzureTenantID) assert.Equal(t, tID, m365.AzureTenantID)
@ -256,7 +270,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
err = vpr.ReadInConfig() err = vpr.ReadInConfig()
require.NoError(t, err, "reading repo config", clues.ToCore(err)) require.NoError(t, err, "reading repo config", clues.ToCore(err))
config, err := getStorageAndAccountWithViper(vpr, true, nil) config, err := getStorageAndAccountWithViper(vpr, true, false, nil)
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err)) require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
readS3Cfg, err := config.Storage.S3Config() readS3Cfg, err := config.Storage.S3Config()
@ -274,7 +288,8 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
readM365, err := config.Account.M365Config() readM365, err := config.Account.M365Config()
require.NoError(t, err, "reading m365 config from account", clues.ToCore(err)) require.NoError(t, err, "reading m365 config from account", clues.ToCore(err))
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID) // Env var gets preference here. Where to get env tenantID from
// assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID)) assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret)) assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
} }
@ -303,7 +318,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
StorageProviderTypeKey: storage.ProviderS3.String(), StorageProviderTypeKey: storage.ProviderS3.String(),
} }
config, err := getStorageAndAccountWithViper(vpr, false, overrides) config, err := getStorageAndAccountWithViper(vpr, false, false, overrides)
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err)) require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
readS3Cfg, err := config.Storage.S3Config() readS3Cfg, err := config.Storage.S3Config()

View File

@ -9,6 +9,7 @@ import (
"github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/defaults"
"github.com/spf13/viper" "github.com/spf13/viper"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
@ -33,6 +34,15 @@ func s3ConfigsFromViper(vpr *viper.Viper) (storage.S3Config, error) {
return s3Config, nil return s3Config, nil
} }
// prerequisite: readRepoConfig must have been run prior to this to populate the global viper values.
func s3CredsFromViper(vpr *viper.Viper, s3Config storage.S3Config) (storage.S3Config, error) {
s3Config.AccessKey = vpr.GetString(AccessKey)
s3Config.SecretKey = vpr.GetString(SecretAccessKey)
s3Config.SessionToken = vpr.GetString(SessionToken)
return s3Config, nil
}
func s3Overrides(in map[string]string) map[string]string { func s3Overrides(in map[string]string) map[string]string {
return map[string]string{ return map[string]string{
storage.Bucket: in[storage.Bucket], storage.Bucket: in[storage.Bucket],
@ -49,6 +59,7 @@ func s3Overrides(in map[string]string) map[string]string {
func configureStorage( func configureStorage(
vpr *viper.Viper, vpr *viper.Viper,
readConfigFromViper bool, readConfigFromViper bool,
matchFromConfig bool,
overrides map[string]string, overrides map[string]string,
) (storage.Storage, error) { ) (storage.Storage, error) {
var ( var (
@ -69,33 +80,54 @@ func configureStorage(
if p, ok := overrides[storage.Prefix]; ok { if p, ok := overrides[storage.Prefix]; ok {
overrides[storage.Prefix] = common.NormalizePrefix(p) overrides[storage.Prefix] = common.NormalizePrefix(p)
} }
}
if matchFromConfig {
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil { if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
return store, clues.Wrap(err, "verifying s3 configs in corso config file") return store, clues.Wrap(err, "verifying s3 configs in corso config file")
} }
} }
_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get() if s3Cfg, err = s3CredsFromViper(vpr, s3Cfg); err != nil {
if err != nil { return store, clues.Wrap(err, "reading s3 configs from corso config file")
return store, clues.Wrap(err, "validating aws credentials") }
s3Overrides(overrides)
aws := credentials.GetAWS(overrides)
if len(aws.AccessKey) <= 0 || len(aws.SecretKey) <= 0 {
_, err = defaults.CredChain(defaults.Config().WithCredentialsChainVerboseErrors(true), defaults.Handlers()).Get()
if err != nil && (len(s3Cfg.AccessKey) > 0 || len(s3Cfg.SecretKey) > 0) {
aws = credentials.AWS{
AccessKey: s3Cfg.AccessKey,
SecretKey: s3Cfg.SecretKey,
SessionToken: s3Cfg.SessionToken,
}
err = nil
}
if err != nil {
return store, clues.Wrap(err, "validating aws credentials")
}
} }
s3Cfg = storage.S3Config{ s3Cfg = storage.S3Config{
Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket, os.Getenv(storage.BucketKey)), AWS: aws,
Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint, os.Getenv(storage.EndpointKey)), Bucket: str.First(overrides[storage.Bucket], s3Cfg.Bucket),
Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix, os.Getenv(storage.PrefixKey)), Endpoint: str.First(overrides[storage.Endpoint], s3Cfg.Endpoint),
Prefix: str.First(overrides[storage.Prefix], s3Cfg.Prefix),
DoNotUseTLS: str.ParseBool(str.First( DoNotUseTLS: str.ParseBool(str.First(
overrides[storage.DoNotUseTLS], overrides[storage.DoNotUseTLS],
strconv.FormatBool(s3Cfg.DoNotUseTLS), strconv.FormatBool(s3Cfg.DoNotUseTLS),
os.Getenv(storage.PrefixKey))), )),
DoNotVerifyTLS: str.ParseBool(str.First( DoNotVerifyTLS: str.ParseBool(str.First(
overrides[storage.DoNotVerifyTLS], overrides[storage.DoNotVerifyTLS],
strconv.FormatBool(s3Cfg.DoNotVerifyTLS), strconv.FormatBool(s3Cfg.DoNotVerifyTLS),
os.Getenv(storage.PrefixKey))), )),
} }
// compose the common config and credentials // compose the common config and credentials
corso := credentials.GetCorso() corso := GetAndInsertCorso(vpr.GetString(CorsoPassphrase))
if err := corso.Validate(); err != nil { if err := corso.Validate(); err != nil {
return store, clues.Wrap(err, "validating corso credentials") return store, clues.Wrap(err, "validating corso credentials")
} }
@ -127,3 +159,14 @@ func configureStorage(
return store, nil return store, nil
} }
// GetCorso is a helper for aggregating Corso secrets and credentials.
func GetAndInsertCorso(passphase string) credentials.Corso {
// fetch data from flag, env var or func param giving priority to func param
// Func param generally will be value fetched from config file using viper.
corsoPassph := str.First(flags.CorsoPassphraseFV, os.Getenv(credentials.CorsoPassphrase), passphase)
return credentials.Corso{
CorsoPassphrase: corsoPassph,
}
}

View File

@ -7,11 +7,19 @@ import (
) )
const ( const (
UserFN = "user" UserFN = "user"
MailBoxFN = "mailbox" MailBoxFN = "mailbox"
AzureClientTenantFN = "azure-tenant-id"
AzureClientIDFN = "azure-client-id"
AzureClientSecretFN = "azure-client-secret"
) )
var UserFV []string var (
UserFV []string
AzureClientTenantFV string
AzureClientIDFV string
AzureClientSecretFV string
)
// AddUserFlag adds the --user flag. // AddUserFlag adds the --user flag.
func AddUserFlag(cmd *cobra.Command) { func AddUserFlag(cmd *cobra.Command) {
@ -38,3 +46,11 @@ func AddMailBoxFlag(cmd *cobra.Command) {
MailBoxFN, nil, MailBoxFN, nil,
"Backup a specific mailbox's data; accepts '"+Wildcard+"' to select all mailbox.") "Backup a specific mailbox's data; accepts '"+Wildcard+"' to select all mailbox.")
} }
// AddAzureCredsFlags adds M365 cred flags
func AddAzureCredsFlags(cmd *cobra.Command) {
fs := cmd.Flags()
fs.StringVar(&AzureClientTenantFV, AzureClientTenantFN, "", "Azure tenant ID")
fs.StringVar(&AzureClientIDFV, AzureClientIDFN, "", "Azure app client ID")
fs.StringVar(&AzureClientSecretFV, AzureClientSecretFN, "", "Azure app client secret")
}

View File

@ -4,9 +4,23 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
const BackupFN = "backup" const (
BackupFN = "backup"
AWSAccessKeyFN = "aws-access-key"
AWSSecretAccessKeyFN = "aws-secret-access-key"
AWSSessionTokenFN = "aws-session-token"
var BackupIDFV string // Corso Flags
CorsoPassphraseFN = "passphrase"
)
var (
BackupIDFV string
AWSAccessKeyFV string
AWSSecretAccessKeyFV string
AWSSessionTokenFV string
CorsoPassphraseFV string
)
// AddBackupIDFlag adds the --backup flag. // AddBackupIDFlag adds the --backup flag.
func AddBackupIDFlag(cmd *cobra.Command, require bool) { func AddBackupIDFlag(cmd *cobra.Command, require bool) {
@ -16,3 +30,19 @@ func AddBackupIDFlag(cmd *cobra.Command, require bool) {
cobra.CheckErr(cmd.MarkFlagRequired(BackupFN)) cobra.CheckErr(cmd.MarkFlagRequired(BackupFN))
} }
} }
func AddAWSCredsFlags(cmd *cobra.Command) {
fs := cmd.Flags()
fs.StringVar(&AWSAccessKeyFV, AWSAccessKeyFN, "", "S3 access key")
fs.StringVar(&AWSSecretAccessKeyFV, AWSSecretAccessKeyFN, "", "S3 access secret")
fs.StringVar(&AWSSessionTokenFV, AWSSessionTokenFN, "", "S3 session token")
}
// M365 flags
func AddCorsoPassphaseFlags(cmd *cobra.Command) {
fs := cmd.Flags()
fs.StringVar(&CorsoPassphraseFV,
CorsoPassphraseFN,
"",
"Passphrase to protect encrypted repository contents")
}

View File

@ -122,7 +122,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, S3Overrides())
if err != nil { if err != nil {
return print.Only(ctx, err) return print.Only(ctx, err)
} }

View File

@ -1,6 +1,7 @@
package repo package repo
import ( import (
"os"
"strconv" "strconv"
"strings" "strings"
@ -10,22 +11,25 @@ import (
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage"
) )
// s3 bucket info from flags // s3 bucket info from flags
var ( var (
succeedIfExists bool
bucket string bucket string
endpoint string endpoint string
prefix string prefix string
doNotUseTLS bool doNotUseTLS bool
doNotVerifyTLS bool doNotVerifyTLS bool
succeedIfExists bool
) )
// called by repo.go to map subcommands to provider-specific handling. // called by repo.go to map subcommands to provider-specific handling.
@ -45,10 +49,13 @@ func addS3Commands(cmd *cobra.Command) *cobra.Command {
c.Use = c.Use + " " + s3ProviderCommandUseSuffix c.Use = c.Use + " " + s3ProviderCommandUseSuffix
c.SetUsageTemplate(cmd.UsageTemplate()) c.SetUsageTemplate(cmd.UsageTemplate())
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddCorsoPassphaseFlags(c)
// Flags addition ordering should follow the order we want them to appear in help and docs: // Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic and more frequently used flags take precedence. // More generic and more frequently used flags take precedence.
fs.StringVar(&bucket, "bucket", "", "Name of S3 bucket for repo. (required)") fs.StringVar(&bucket, "bucket", "", "Name of S3 bucket for repo. (required)")
cobra.CheckErr(c.MarkFlagRequired("bucket"))
fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.") fs.StringVar(&prefix, "prefix", "", "Repo prefix within bucket.")
fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.") fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "S3 service endpoint.")
fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)") fs.BoolVar(&doNotUseTLS, "disable-tls", false, "Disable TLS (HTTPS)")
@ -107,11 +114,12 @@ func s3InitCmd() *cobra.Command {
func initS3Cmd(cmd *cobra.Command, args []string) error { func initS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) { // s3 values from flags
return nil s3Override := S3Overrides()
} // s3 values from envs
s3Override = S3UpdateFromEnvVar(s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, false, S3Overrides()) cfg, err := config.GetConfigRepoDetails(ctx, true, false, s3Override)
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -182,11 +190,12 @@ func s3ConnectCmd() *cobra.Command {
func connectS3Cmd(cmd *cobra.Command, args []string) error { func connectS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context() ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) { // s3 values from flags
return nil s3Override := S3Overrides()
} // s3 values from envs
s3Override = S3UpdateFromEnvVar(s3Override)
cfg, err := config.GetConfigRepoDetails(ctx, true, S3Overrides()) cfg, err := config.GetConfigRepoDetails(ctx, true, true, s3Override)
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }
@ -231,12 +240,27 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
func S3Overrides() map[string]string { func S3Overrides() map[string]string {
return map[string]string{ return map[string]string{
config.AccountProviderTypeKey: account.ProviderM365.String(), config.AccountProviderTypeKey: account.ProviderM365.String(),
config.StorageProviderTypeKey: storage.ProviderS3.String(), config.StorageProviderTypeKey: storage.ProviderS3.String(),
storage.Bucket: bucket, credentials.AWSAccessKeyID: flags.AWSAccessKeyFV,
storage.Endpoint: endpoint, credentials.AWSSecretAccessKey: flags.AWSSecretAccessKeyFV,
storage.Prefix: prefix, credentials.AWSSessionToken: flags.AWSSessionTokenFV,
storage.DoNotUseTLS: strconv.FormatBool(doNotUseTLS), storage.Bucket: bucket,
storage.DoNotVerifyTLS: strconv.FormatBool(doNotVerifyTLS), storage.Endpoint: endpoint,
storage.Prefix: prefix,
storage.DoNotUseTLS: strconv.FormatBool(doNotUseTLS),
storage.DoNotVerifyTLS: strconv.FormatBool(doNotVerifyTLS),
} }
} }
func S3UpdateFromEnvVar(s3Flag map[string]string) map[string]string {
s3Flag[storage.Bucket] = str.First(s3Flag[storage.Bucket], os.Getenv(storage.BucketKey))
s3Flag[storage.Endpoint] = str.First(s3Flag[storage.Endpoint], os.Getenv(storage.EndpointKey))
s3Flag[storage.Prefix] = str.First(s3Flag[storage.Prefix], os.Getenv(storage.PrefixKey))
s3Flag[storage.DoNotUseTLS] = str.First(s3Flag[storage.DoNotUseTLS], os.Getenv(storage.DisableTLSKey))
s3Flag[storage.DoNotVerifyTLS] = str.First(
s3Flag[storage.DoNotVerifyTLS],
os.Getenv(storage.DisableTLSVerificationKey))
return s3Flag
}

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -35,6 +36,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
flags.AddBackupIDFlag(c, true) flags.AddBackupIDFlag(c, true)
flags.AddExchangeDetailsAndRestoreFlags(c) flags.AddExchangeDetailsAndRestoreFlags(c)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -89,7 +93,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -78,6 +78,15 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
"--" + flags.EventStartsAfterFN, testdata.EventStartsAfterInput, "--" + flags.EventStartsAfterFN, testdata.EventStartsAfterInput,
"--" + flags.EventStartsBeforeFN, testdata.EventStartsBeforeInput, "--" + flags.EventStartsBeforeFN, testdata.EventStartsBeforeInput,
"--" + flags.EventSubjectFN, testdata.EventSubjectInput, "--" + flags.EventSubjectFN, testdata.EventSubjectInput,
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
"--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken,
"--" + flags.AzureClientIDFN, testdata.AzureClientID,
"--" + flags.AzureClientTenantFN, testdata.AzureTenantID,
"--" + flags.AzureClientSecretFN, testdata.AzureClientSecret,
"--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase,
}) })
cmd.SetOut(new(bytes.Buffer)) // drop output cmd.SetOut(new(bytes.Buffer)) // drop output
@ -106,6 +115,16 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
assert.Equal(t, testdata.EventStartsAfterInput, opts.EventStartsAfter) assert.Equal(t, testdata.EventStartsAfterInput, opts.EventStartsAfter)
assert.Equal(t, testdata.EventStartsBeforeInput, opts.EventStartsBefore) assert.Equal(t, testdata.EventStartsBeforeInput, opts.EventStartsBefore)
assert.Equal(t, testdata.EventSubjectInput, opts.EventSubject) assert.Equal(t, testdata.EventSubjectInput, opts.EventSubject)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV)
assert.Equal(t, testdata.AzureClientID, flags.AzureClientIDFV)
assert.Equal(t, testdata.AzureTenantID, flags.AzureClientTenantFV)
assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV)
assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV)
}) })
} }
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -35,6 +36,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
flags.AddOneDriveDetailsAndRestoreFlags(c) flags.AddOneDriveDetailsAndRestoreFlags(c)
flags.AddRestorePermissionsFlag(c) flags.AddRestorePermissionsFlag(c)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -88,7 +92,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -67,6 +67,15 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
"--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput,
"--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput,
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
"--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken,
"--" + flags.AzureClientIDFN, testdata.AzureClientID,
"--" + flags.AzureClientTenantFN, testdata.AzureTenantID,
"--" + flags.AzureClientSecretFN, testdata.AzureClientSecret,
"--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase,
}) })
cmd.SetOut(new(bytes.Buffer)) // drop output cmd.SetOut(new(bytes.Buffer)) // drop output
@ -83,6 +92,16 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore)
assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV)
assert.Equal(t, testdata.AzureClientID, flags.AzureClientIDFV)
assert.Equal(t, testdata.AzureTenantID, flags.AzureClientTenantFV)
assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV)
assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV)
}) })
} }
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print" . "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -35,6 +36,10 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSharePointDetailsAndRestoreFlags(c) flags.AddSharePointDetailsAndRestoreFlags(c)
flags.AddRestorePermissionsFlag(c) flags.AddRestorePermissionsFlag(c)
flags.AddFailFastFlag(c) flags.AddFailFastFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
} }
return c return c
@ -94,7 +99,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error {
return err return err
} }
r, _, _, err := utils.GetAccountAndConnect(ctx) r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil { if err != nil {
return Only(ctx, err) return Only(ctx, err)
} }

View File

@ -72,6 +72,15 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput), "--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput),
"--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput), "--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput),
"--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput), "--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput),
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
"--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken,
"--" + flags.AzureClientIDFN, testdata.AzureClientID,
"--" + flags.AzureClientTenantFN, testdata.AzureTenantID,
"--" + flags.AzureClientSecretFN, testdata.AzureClientSecret,
"--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase,
}) })
cmd.SetOut(new(bytes.Buffer)) // drop output cmd.SetOut(new(bytes.Buffer)) // drop output
@ -95,6 +104,16 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.ElementsMatch(t, testdata.PageInput, opts.Page) assert.ElementsMatch(t, testdata.PageInput, opts.Page)
assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder) assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV)
assert.Equal(t, testdata.AzureClientID, flags.AzureClientIDFV)
assert.Equal(t, testdata.AzureTenantID, flags.AzureClientTenantFV)
assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV)
assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV)
}) })
} }
} }

View File

@ -0,0 +1,93 @@
package utils
import (
"testing"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/tester"
)
type FlagUnitSuite struct {
tester.Suite
}
func TestFlagUnitSuite(t *testing.T) {
suite.Run(t, &FlagUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *FlagUnitSuite) TestAddAzureCredsFlags() {
t := suite.T()
cmd := &cobra.Command{
Use: "test",
Run: func(cmd *cobra.Command, args []string) {
assert.Equal(t, "tenantID", flags.AzureClientTenantFV, flags.AzureClientTenantFN)
assert.Equal(t, "clientID", flags.AzureClientIDFV, flags.AzureClientIDFN)
assert.Equal(t, "secret", flags.AzureClientSecretFV, flags.AzureClientSecretFN)
},
}
flags.AddAzureCredsFlags(cmd)
// Test arg parsing for few args
cmd.SetArgs([]string{
"test",
"--" + flags.AzureClientIDFN, "clientID",
"--" + flags.AzureClientTenantFN, "tenantID",
"--" + flags.AzureClientSecretFN, "secret",
})
err := cmd.Execute()
require.NoError(t, err, clues.ToCore(err))
}
func (suite *FlagUnitSuite) TestAddAWSCredsFlags() {
t := suite.T()
cmd := &cobra.Command{
Use: "test",
Run: func(cmd *cobra.Command, args []string) {
assert.Equal(t, "accesskey", flags.AWSAccessKeyFV, flags.AWSAccessKeyFN)
assert.Equal(t, "secretkey", flags.AWSSecretAccessKeyFV, flags.AWSSecretAccessKeyFN)
assert.Equal(t, "token", flags.AWSSessionTokenFV, flags.AWSSessionTokenFN)
},
}
flags.AddAWSCredsFlags(cmd)
// Test arg parsing for few args
cmd.SetArgs([]string{
"test",
"--" + flags.AWSAccessKeyFN, "accesskey",
"--" + flags.AWSSecretAccessKeyFN, "secretkey",
"--" + flags.AWSSessionTokenFN, "token",
})
err := cmd.Execute()
require.NoError(t, err, clues.ToCore(err))
}
func (suite *FlagUnitSuite) TestAddCorsoPassphraseFlags() {
t := suite.T()
cmd := &cobra.Command{
Use: "test",
Run: func(cmd *cobra.Command, args []string) {
assert.Equal(t, "passphrase", flags.CorsoPassphraseFV, flags.CorsoPassphraseFN)
},
}
flags.AddCorsoPassphaseFlags(cmd)
// Test arg parsing for few args
cmd.SetArgs([]string{
"test",
"--" + flags.CorsoPassphraseFN, "passphrase",
})
err := cmd.Execute()
require.NoError(t, err, clues.ToCore(err))
}

View File

@ -45,4 +45,14 @@ var (
PageInput = []string{"page1", "page2"} PageInput = []string{"page1", "page2"}
RestorePermissions = true RestorePermissions = true
AzureClientID = "testAzureClientId"
AzureTenantID = "testAzureTenantId"
AzureClientSecret = "testAzureClientSecret"
AWSAccessKeyID = "testAWSAccessKeyID"
AWSSecretAccessKey = "testAWSSecretAccessKey"
AWSSessionToken = "testAWSSessionToken"
CorsoPassphrase = "testCorsoPassphrase"
) )

View File

@ -19,8 +19,11 @@ import (
"github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage"
) )
func GetAccountAndConnect(ctx context.Context) (repository.Repository, *storage.Storage, *account.Account, error) { func GetAccountAndConnect(
cfg, err := config.GetConfigRepoDetails(ctx, true, nil) ctx context.Context,
overrides map[string]string,
) (repository.Repository, *storage.Storage, *account.Account, error) {
cfg, err := config.GetConfigRepoDetails(ctx, true, true, overrides)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
@ -38,8 +41,11 @@ func GetAccountAndConnect(ctx context.Context) (repository.Repository, *storage.
return r, &cfg.Storage, &cfg.Account, nil return r, &cfg.Storage, &cfg.Account, nil
} }
func AccountConnectAndWriteRepoConfig(ctx context.Context) (repository.Repository, *account.Account, error) { func AccountConnectAndWriteRepoConfig(
r, stg, acc, err := GetAccountAndConnect(ctx) ctx context.Context,
overrides map[string]string,
) (repository.Repository, *account.Account, error) {
r, stg, acc, err := GetAccountAndConnect(ctx, overrides)
if err != nil { if err != nil {
logger.CtxErr(ctx, err).Info("getting and connecting account") logger.CtxErr(ctx, err).Info("getting and connecting account")
return nil, nil, err return nil, nil, err

View File

@ -71,6 +71,7 @@ github.com/aws/aws-sdk-go v1.44.291/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8
github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo=
github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@ -123,6 +124,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@ -225,6 +227,7 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@ -232,6 +235,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
@ -306,6 +310,7 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
@ -438,6 +443,7 @@ go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAf
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
@ -788,6 +794,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -40,6 +40,9 @@ func s3BlobStorage(
SessionName: s.SessionName, SessionName: s.SessionName,
RoleARN: s.Role, RoleARN: s.Role,
RoleDuration: s.SessionDuration, RoleDuration: s.SessionDuration,
AccessKeyID: cfg.AccessKey,
SecretAccessKey: cfg.SecretKey,
SessionToken: cfg.SessionToken,
TLSHandshakeTimeout: 60, TLSHandshakeTimeout: 60,
PointInTime: repoOpts.ViewTimestamp, PointInTime: repoOpts.ViewTimestamp,
} }

View File

@ -1,11 +1,14 @@
package tester package tester
import ( import (
"os"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage"
) )
@ -40,7 +43,7 @@ func NewPrefixedS3Storage(t *testing.T) storage.Storage {
Prefix: prefix, Prefix: prefix,
}, },
storage.CommonConfig{ storage.CommonConfig{
Corso: credentials.GetCorso(), Corso: GetAndInsertCorso(""),
KopiaCfgDir: t.TempDir(), KopiaCfgDir: t.TempDir(),
}, },
) )
@ -48,3 +51,14 @@ func NewPrefixedS3Storage(t *testing.T) storage.Storage {
return st return st
} }
// GetCorso is a helper for aggregating Corso secrets and credentials.
func GetAndInsertCorso(passphase string) credentials.Corso {
// fetch data from flag, env var or func param giving priority to func param
// Func param generally will be value fetched from config file using viper.
corsoPassph := str.First(flags.CorsoPassphraseFV, os.Getenv(credentials.CorsoPassphrase), passphase)
return credentials.Corso{
CorsoPassphrase: corsoPassph,
}
}

View File

@ -1,8 +1,6 @@
package credentials package credentials
import ( import (
"os"
"github.com/alcionai/clues" "github.com/alcionai/clues"
) )
@ -22,20 +20,10 @@ type AWS struct {
// GetAWS is a helper for aggregating aws secrets and credentials. // GetAWS is a helper for aggregating aws secrets and credentials.
func GetAWS(override map[string]string) AWS { func GetAWS(override map[string]string) AWS {
accessKey := os.Getenv(AWSAccessKeyID)
if ovr, ok := override[AWSAccessKeyID]; ok && ovr != "" {
accessKey = ovr
}
secretKey := os.Getenv(AWSSecretAccessKey)
sessToken := os.Getenv(AWSSessionToken)
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
return AWS{ return AWS{
AccessKey: accessKey, AccessKey: override[AWSAccessKeyID],
SecretKey: secretKey, SecretKey: override[AWSSecretAccessKey],
SessionToken: sessToken, SessionToken: override[AWSSessionToken],
} }
} }

View File

@ -1,8 +1,6 @@
package credentials package credentials
import ( import (
"os"
"github.com/alcionai/clues" "github.com/alcionai/clues"
) )
@ -16,17 +14,6 @@ type Corso struct {
CorsoPassphrase string // required CorsoPassphrase string // required
} }
// GetCorso is a helper for aggregating Corso secrets and credentials.
func GetCorso() Corso {
// todo (rkeeprs): read from either corso config file or env vars.
// https://github.com/alcionai/corso/issues/120
corsoPassph := os.Getenv(CorsoPassphrase)
return Corso{
CorsoPassphrase: corsoPassph,
}
}
func (c Corso) Validate() error { func (c Corso) Validate() error {
check := map[string]string{ check := map[string]string{
CorsoPassphrase: c.CorsoPassphrase, CorsoPassphrase: c.CorsoPassphrase,

View File

@ -20,11 +20,14 @@ type M365 struct {
// M365 is a helper for aggregating m365 secrets and credentials. // M365 is a helper for aggregating m365 secrets and credentials.
func GetM365() M365 { func GetM365() M365 {
// todo (rkeeprs): read from either corso config file or env vars. // check env and overide is flags found
// https://github.com/alcionai/corso/issues/120 // var AzureClientID, AzureClientSecret string
AzureClientID := os.Getenv(AzureClientID)
AzureClientSecret := os.Getenv(AzureClientSecret)
return M365{ return M365{
AzureClientID: os.Getenv(AzureClientID), AzureClientID: AzureClientID,
AzureClientSecret: os.Getenv(AzureClientSecret), AzureClientSecret: AzureClientSecret,
} }
} }

View File

@ -7,9 +7,11 @@ import (
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/credentials"
) )
type S3Config struct { type S3Config struct {
credentials.AWS
Bucket string // required Bucket string // required
Endpoint string Endpoint string
Prefix string Prefix string
@ -19,9 +21,12 @@ type S3Config struct {
// config key consts // config key consts
const ( const (
keyS3AccessKey = "s3_access_key"
keyS3Bucket = "s3_bucket" keyS3Bucket = "s3_bucket"
keyS3Endpoint = "s3_endpoint" keyS3Endpoint = "s3_endpoint"
keyS3Prefix = "s3_prefix" keyS3Prefix = "s3_prefix"
keyS3SecretKey = "s3_secret_key"
keyS3SessionToken = "s3_session_token"
keyS3DoNotUseTLS = "s3_donotusetls" keyS3DoNotUseTLS = "s3_donotusetls"
keyS3DoNotVerifyTLS = "s3_donotverifytls" keyS3DoNotVerifyTLS = "s3_donotverifytls"
) )
@ -51,9 +56,12 @@ func (c S3Config) Normalize() S3Config {
func (c S3Config) StringConfig() (map[string]string, error) { func (c S3Config) StringConfig() (map[string]string, error) {
cn := c.Normalize() cn := c.Normalize()
cfg := map[string]string{ cfg := map[string]string{
keyS3AccessKey: c.AccessKey,
keyS3Bucket: cn.Bucket, keyS3Bucket: cn.Bucket,
keyS3Endpoint: cn.Endpoint, keyS3Endpoint: cn.Endpoint,
keyS3Prefix: cn.Prefix, keyS3Prefix: cn.Prefix,
keyS3SecretKey: c.SecretKey,
keyS3SessionToken: c.SessionToken,
keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS), keyS3DoNotUseTLS: strconv.FormatBool(cn.DoNotUseTLS),
keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS), keyS3DoNotVerifyTLS: strconv.FormatBool(cn.DoNotVerifyTLS),
} }
@ -66,6 +74,10 @@ func (s Storage) S3Config() (S3Config, error) {
c := S3Config{} c := S3Config{}
if len(s.Config) > 0 { if len(s.Config) > 0 {
c.AccessKey = orEmptyString(s.Config[keyS3AccessKey])
c.SecretKey = orEmptyString(s.Config[keyS3SecretKey])
c.SessionToken = orEmptyString(s.Config[keyS3SessionToken])
c.Bucket = orEmptyString(s.Config[keyS3Bucket]) c.Bucket = orEmptyString(s.Config[keyS3Bucket])
c.Endpoint = orEmptyString(s.Config[keyS3Endpoint]) c.Endpoint = orEmptyString(s.Config[keyS3Endpoint])
c.Prefix = orEmptyString(s.Config[keyS3Prefix]) c.Prefix = orEmptyString(s.Config[keyS3Prefix])

View File

@ -7,6 +7,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/pkg/credentials"
) )
type S3CfgSuite struct { type S3CfgSuite struct {
@ -24,6 +26,7 @@ var (
Prefix: "pre/", Prefix: "pre/",
DoNotUseTLS: false, DoNotUseTLS: false,
DoNotVerifyTLS: false, DoNotVerifyTLS: false,
AWS: credentials.AWS{AccessKey: "access", SecretKey: "secret", SessionToken: "token"},
} }
goodS3Map = map[string]string{ goodS3Map = map[string]string{
@ -32,6 +35,9 @@ var (
keyS3Prefix: "pre/", keyS3Prefix: "pre/",
keyS3DoNotUseTLS: "false", keyS3DoNotUseTLS: "false",
keyS3DoNotVerifyTLS: "false", keyS3DoNotVerifyTLS: "false",
keyS3AccessKey: "access",
keyS3SecretKey: "secret",
keyS3SessionToken: "token",
} }
) )
@ -68,11 +74,12 @@ func (suite *S3CfgSuite) TestStorage_S3Config() {
assert.Equal(t, in.Prefix, out.Prefix) assert.Equal(t, in.Prefix, out.Prefix)
} }
func makeTestS3Cfg(bkt, end, pre string) S3Config { func makeTestS3Cfg(bkt, end, pre, access, secret, session string) S3Config {
return S3Config{ return S3Config{
Bucket: bkt, Bucket: bkt,
Endpoint: end, Endpoint: end,
Prefix: pre, Prefix: pre,
AWS: credentials.AWS{AccessKey: access, SecretKey: secret, SessionToken: session},
} }
} }
@ -82,7 +89,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() {
name string name string
cfg S3Config cfg S3Config
}{ }{
{"missing bucket", makeTestS3Cfg("", "end", "pre/")}, {"missing bucket", makeTestS3Cfg("", "end", "pre/", "", "", "")},
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
@ -128,8 +135,14 @@ func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
expect: goodS3Map, expect: goodS3Map,
}, },
{ {
name: "normalized bucket name", name: "normalized bucket name",
input: makeTestS3Cfg("s3://"+goodS3Config.Bucket, goodS3Config.Endpoint, goodS3Config.Prefix), input: makeTestS3Cfg(
"s3://"+goodS3Config.Bucket,
goodS3Config.Endpoint,
goodS3Config.Prefix,
goodS3Config.AccessKey,
goodS3Config.SecretKey,
goodS3Config.SessionToken),
expect: goodS3Map, expect: goodS3Map,
}, },
{ {
@ -147,6 +160,9 @@ func (suite *S3CfgSuite) TestStorage_S3Config_StringConfig() {
keyS3Prefix: "pre/", keyS3Prefix: "pre/",
keyS3DoNotUseTLS: "true", keyS3DoNotUseTLS: "true",
keyS3DoNotVerifyTLS: "true", keyS3DoNotVerifyTLS: "true",
keyS3AccessKey: "",
keyS3SecretKey: "",
keyS3SessionToken: "",
}, },
}, },
} }