larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

set up CLI for integration testing (#478)

Browse Source 
Makes the necessary changes, including adding helper
funcs, to bring the CLI up to an integration-testable
state.  The changes made in this commit should be
sufficient for most other CLI tests.  Includes a single
test as verification.
This commit is contained in:
Keepers 2022-08-04 11:15:13 -06:00 committed by GitHub
parent d920589507
commit 342dd2e9f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 330 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cli/backup/exchange.go
View File

@ -126,7 +126,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
s, acct, err := config.GetStorageAndAccount(true, nil)
s, acct, err := config.GetStorageAndAccount(ctx, true, nil)
if err != nil {
return Only(err)
}
@ -224,7 +224,7 @@ var exchangeListCmd = &cobra.Command{
func listExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
s, acct, err := config.GetStorageAndAccount(true, nil)
s, acct, err := config.GetStorageAndAccount(ctx, true, nil)
if err != nil {
return Only(err)
}
@ -285,7 +285,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
s, acct, err := config.GetStorageAndAccount(true, nil)
s, acct, err := config.GetStorageAndAccount(ctx, true, nil)
if err != nil {
return Only(err)
}

30
src/cli/cli.go
View File

@ -5,7 +5,6 @@ import (
"os"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/alcionai/corso/cli/backup"
"github.com/alcionai/corso/cli/config"
@ -18,31 +17,18 @@ import (
// The root-level command.
// `corso <command> [<subcommand>] [<service>] [<flag>...]`
var corsoCmd = &cobra.Command{
Use: "corso",
Short: "Protect your Microsoft 365 data.",
Long: `Reliable, secure, and efficient data protection for Microsoft 365.`,
RunE: handleCorsoCmd,
Use: "corso",
Short: "Protect your Microsoft 365 data.",
Long: `Reliable, secure, and efficient data protection for Microsoft 365.`,
RunE: handleCorsoCmd,
PersistentPreRunE: config.InitFunc(),
}
// the root-level flags
var (
cfgFile string
version bool
)
func init() {
cobra.OnInitialize(initConfig)
}
func initConfig() {
err := config.InitConfig(cfgFile)
cobra.CheckErr(err)
if err := viper.ReadInConfig(); err == nil {
print.Info("Using config file:", viper.ConfigFileUsed())
}
}
// Handler for flat calls to `corso`.
// Produces the same output as `corso --help`.
func handleCorsoCmd(cmd *cobra.Command, args []string) error {
@ -55,8 +41,10 @@ func handleCorsoCmd(cmd *cobra.Command, args []string) error {
// Handle builds and executes the cli processor.
func Handle() {
ctx := config.Seed(context.Background())
corsoCmd.Flags().BoolP("version", "v", version, "current version info")
corsoCmd.PersistentFlags().StringVar(&cfgFile, "config-file", "", "config file (default is $HOME/.corso)")
config.AddConfigFileFlag(corsoCmd)
print.SetRootCommand(corsoCmd)
print.AddOutputFlag(corsoCmd)
@ -66,7 +54,7 @@ func Handle() {
backup.AddCommands(corsoCmd)
restore.AddCommands(corsoCmd)
ctx, log := logger.Seed(context.Background())
ctx, log := logger.Seed(ctx)
defer func() {
_ = log.Sync() // flush all logs in the buffer
}()

105
src/cli/config/config.go
View File

@ -1,13 +1,17 @@
package config
import (
"context"
"os"
"path"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
. "github.com/alcionai/corso/cli/print"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/storage"
)
@ -24,15 +28,43 @@ const (
TenantIDKey = "tenantid"
)
func InitConfig(configFilePath string) error {
return initConfigWithViper(viper.GetViper(), configFilePath)
var configFilePath string
// adds the persistent flag --config-file to the provided command.
func AddConfigFileFlag(cmd *cobra.Command) {
fs := cmd.PersistentFlags()
homeDir, err := os.UserHomeDir()
if err != nil {
Err("finding $HOME directory (default) for config file")
}
fs.StringVar(
&configFilePath,
"config-file",
filepath.Join(homeDir, ".corso.toml"),
"config file (default is $HOME/.corso)")
}
// initConfigWithViper implements InitConfig, but takes in a viper
// ---------------------------------------------------------------------------------------------------------
// Initialization & Storage
// ---------------------------------------------------------------------------------------------------------
// InitFunc provides a func that lazily initializes viper and
// verifies that the configuration was able to read a file.
func InitFunc() func(*cobra.Command, []string) error {
return func(cmd *cobra.Command, args []string) error {
err := initWithViper(GetViper(cmd.Context()), configFilePath)
if err != nil {
return err
}
return Read(cmd.Context())
}
}
// initWithViper implements InitConfig, but takes in a viper
// struct for testing.
func initConfigWithViper(vpr *viper.Viper, configFilePath string) error {
func initWithViper(vpr *viper.Viper, configFP string) error {
// Configure default config file location
if configFilePath == "" {
if configFP == "" {
// Find home directory.
home, err := os.UserHomeDir()
if err != nil {
@ -46,14 +78,14 @@ func initConfigWithViper(vpr *viper.Viper, configFilePath string) error {
return nil
}
vpr.SetConfigFile(configFilePath)
vpr.SetConfigFile(configFP)
// We also configure the path, type and filename
// because `vpr.SafeWriteConfig` needs these set to
// work correctly (it does not use the configured file)
vpr.AddConfigPath(path.Dir(configFilePath))
vpr.AddConfigPath(path.Dir(configFP))
fileName := path.Base(configFilePath)
ext := path.Ext(configFilePath)
fileName := path.Base(configFP)
ext := path.Ext(configFP)
if len(ext) == 0 {
return errors.New("config file requires an extension e.g. `toml`")
}
@ -64,10 +96,53 @@ func initConfigWithViper(vpr *viper.Viper, configFilePath string) error {
return nil
}
type viperCtx struct{}
// Seed embeds a viper instance in the context.
func Seed(ctx context.Context) context.Context {
return SetViper(ctx, nil)
}
// Adds a viper instance to the context.
// If vpr is nil, sets the default (global) viper.
func SetViper(ctx context.Context, vpr *viper.Viper) context.Context {
if vpr == nil {
vpr = viper.GetViper()
}
return context.WithValue(ctx, viperCtx{}, vpr)
}
// Gets a viper instance from the context.
// If no viper instance is found, returns the default
// (global) viper instance.
func GetViper(ctx context.Context) *viper.Viper {
vprIface := ctx.Value(viperCtx{})
vpr, ok := vprIface.(*viper.Viper)
if vpr == nil || !ok {
return viper.GetViper()
}
return vpr
}
// ---------------------------------------------------------------------------------------------------------
// Reading & Writing the config
// ---------------------------------------------------------------------------------------------------------
// Read reads the config from the viper instance in the context.
// Primarily used as a test-check to ensure the instance was
// set up properly.
func Read(ctx context.Context) error {
if err := viper.ReadInConfig(); err == nil {
Info("Using config file:", viper.ConfigFileUsed())
return err
}
return nil
}
// WriteRepoConfig currently just persists corso config to the config file
// It does not check for conflicts or existing data.
func WriteRepoConfig(s3Config storage.S3Config, m365Config account.M365Config) error {
return writeRepoConfigWithViper(viper.GetViper(), s3Config, m365Config)
func WriteRepoConfig(ctx context.Context, s3Config storage.S3Config, m365Config account.M365Config) error {
return writeRepoConfigWithViper(GetViper(ctx), s3Config, m365Config)
}
// writeRepoConfigWithViper implements WriteRepoConfig, but takes in a viper
@ -94,8 +169,12 @@ func writeRepoConfigWithViper(vpr *viper.Viper, s3Config storage.S3Config, m365C
// GetStorageAndAccount creates a storage and account instance by mediating all the possible
// data sources (config file, env vars, flag overrides) and the config file.
func GetStorageAndAccount(readFromFile bool, overrides map[string]string) (storage.Storage, account.Account, error) {
return getStorageAndAccountWithViper(viper.GetViper(), readFromFile, overrides)
func GetStorageAndAccount(
ctx context.Context,
readFromFile bool,
overrides map[string]string,
) (storage.Storage, account.Account, error) {
return getStorageAndAccountWithViper(GetViper(ctx), readFromFile, overrides)
}
// getSorageAndAccountWithViper implements GetSorageAndAccount, but takes in a viper

6
src/cli/config/config_test.go
View File

@ -82,7 +82,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
// Configure viper to read test config file
testConfigFilePath := path.Join(t.TempDir(), "corso.toml")
require.NoError(t, initConfigWithViper(vpr, testConfigFilePath), "initializing repo config")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{Bucket: bkt}
m365 := account.M365Config{TenantID: tid}
@ -112,7 +112,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
// Configure viper to read test config file
testConfigFilePath := path.Join(t.TempDir(), "corso.toml")
require.NoError(t, initConfigWithViper(vpr, testConfigFilePath), "initializing repo config")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{Bucket: bkt}
m365 := account.M365Config{TenantID: tid}
@ -216,7 +216,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
// Configure viper to read test config file
testConfigFilePath := path.Join(t.TempDir(), "corso.toml")
require.NoError(t, initConfigWithViper(vpr, testConfigFilePath), "initializing repo config")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{
Bucket: bkt,

20
src/cli/print/print.go
View File

@ -24,7 +24,7 @@ func SetRootCommand(root *cobra.Command) {
rootCmd = root
}
// adds the --output flag to the provided command.
// adds the persistent flag --output to the provided command.
func AddOutputFlag(parent *cobra.Command) {
fs := parent.PersistentFlags()
fs.BoolVar(&outputAsJSON, "json", false, "output data in JSON format")
@ -43,7 +43,23 @@ func Only(e error) error {
return e
}
// Info prints the strings to cobra's error writer (stdErr by default)
// Err prints the params to cobra's error writer (stdErr by default)
// if s is nil, prints nothing.
// Prepends the message with "Error: "
func Err(s ...any) {
err(rootCmd.ErrOrStderr(), s...)
}
// err is the testable core of Err()
func err(w io.Writer, s ...any) {
if len(s) == 0 {
return
}
msg := append([]any{"Error: "}, s...)
fmt.Fprint(w, msg...)
}
// Info prints the params to cobra's error writer (stdErr by default)
// if s is nil, prints nothing.
func Info(s ...any) {
info(rootCmd.ErrOrStderr(), s...)

9
src/cli/print/print_test.go
View File

@ -27,6 +27,15 @@ func (suite *PrintUnitSuite) TestOnly() {
assert.True(t, c.SilenceUsage)
}
func (suite *PrintUnitSuite) TestErr() {
t := suite.T()
var b bytes.Buffer
msg := "I have seen the fnords!"
err(&b, msg)
assert.Contains(t, b.String(), "Error: ")
assert.Contains(t, b.String(), msg)
}
func (suite *PrintUnitSuite) TestInfo() {
t := suite.T()
var b bytes.Buffer

49
src/cli/repo/s3.go
View File

@ -33,10 +33,11 @@ func addS3Commands(parent *cobra.Command) *cobra.Command {
)
switch parent.Use {
case initCommand:
c, fs = utils.AddCommand(parent, s3InitCmd)
c, fs = utils.AddCommand(parent, s3InitCmd())
case connectCommand:
c, fs = utils.AddCommand(parent, s3ConnectCmd)
c, fs = utils.AddCommand(parent, s3ConnectCmd())
}
fs.StringVar(&accessKey, "access-key", "", "Access key ID (replaces the AWS_ACCESS_KEY_ID env variable).")
fs.StringVar(&bucket, "bucket", "", "Name of the S3 bucket (required).")
cobra.CheckErr(c.MarkFlagRequired("bucket"))
@ -51,13 +52,19 @@ func addS3Commands(parent *cobra.Command) *cobra.Command {
const s3ProviderCommand = "s3"
// ---------------------------------------------------------------------------------------------------------
// Init
// ---------------------------------------------------------------------------------------------------------
// `corso repo init s3 [<flag>...]`
var s3InitCmd = &cobra.Command{
Use: s3ProviderCommand,
Short: "Initialize a S3 repository",
Long: `Bootstraps a new S3 repository and connects it to your m356 account.`,
RunE: initS3Cmd,
Args: cobra.NoArgs,
func s3InitCmd() *cobra.Command {
return &cobra.Command{
Use: s3ProviderCommand,
Short: "Initialize a S3 repository",
Long: `Bootstraps a new S3 repository and connects it to your m356 account.`,
RunE: initS3Cmd,
Args: cobra.NoArgs,
}
}
// initializes a s3 repo.
@ -69,7 +76,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
return nil
}
s, a, err := config.GetStorageAndAccount(false, s3Overrides())
s, a, err := config.GetStorageAndAccount(ctx, false, s3Overrides())
if err != nil {
return Only(err)
}
@ -102,19 +109,25 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
Infof("Initialized a S3 repository within bucket %s.\n", s3Cfg.Bucket)
if err = config.WriteRepoConfig(s3Cfg, m365); err != nil {
if err = config.WriteRepoConfig(ctx, s3Cfg, m365); err != nil {
return Only(errors.Wrap(err, "Failed to write repository configuration"))
}
return nil
}
// ---------------------------------------------------------------------------------------------------------
// Connect
// ---------------------------------------------------------------------------------------------------------
// `corso repo connect s3 [<flag>...]`
var s3ConnectCmd = &cobra.Command{
Use: s3ProviderCommand,
Short: "Connect to a S3 repository",
Long: `Ensures a connection to an existing S3 repository.`,
RunE: connectS3Cmd,
Args: cobra.NoArgs,
func s3ConnectCmd() *cobra.Command {
return &cobra.Command{
Use: s3ProviderCommand,
Short: "Connect to a S3 repository",
Long: `Ensures a connection to an existing S3 repository.`,
RunE: connectS3Cmd,
Args: cobra.NoArgs,
}
}
// connects to an existing s3 repo.
@ -126,7 +139,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return nil
}
s, a, err := config.GetStorageAndAccount(true, s3Overrides())
s, a, err := config.GetStorageAndAccount(ctx, true, s3Overrides())
if err != nil {
return Only(err)
}
@ -155,7 +168,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
Infof("Connected to S3 bucket %s.\n", s3Cfg.Bucket)
if err = config.WriteRepoConfig(s3Cfg, m365); err != nil {
if err = config.WriteRepoConfig(ctx, s3Cfg, m365); err != nil {
return Only(errors.Wrap(err, "Failed to write repository configuration"))
}
return nil

76
src/cli/repo/s3_integration_test.go Normal file
View File

@ -0,0 +1,76 @@
package repo_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/cli/backup"
"github.com/alcionai/corso/cli/config"
"github.com/alcionai/corso/cli/print"
"github.com/alcionai/corso/cli/repo"
"github.com/alcionai/corso/cli/restore"
"github.com/alcionai/corso/internal/tester"
)
// ---------------------------------------------------------------------------------------------------------
// Integration
// ---------------------------------------------------------------------------------------------------------
type S3IntegrationSuite struct {
suite.Suite
}
func TestS3IntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoCLIRepoTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(S3IntegrationSuite))
}
func (suite *S3IntegrationSuite) SetupSuite() {
_, err := tester.GetRequiredEnvVars(
append(
tester.AWSStorageCredEnvs,
tester.M365AcctCredEnvs...,
)...,
)
require.NoError(suite.T(), err)
}
func (suite *S3IntegrationSuite) TestInitS3Cmd() {
ctx := tester.NewContext()
t := suite.T()
st, err := tester.NewPrefixedS3Storage(t)
require.NoError(t, err)
cfg, err := st.S3Config()
require.NoError(t, err)
vpr, configFP, err := tester.MakeTempTestConfigClone(t)
require.NoError(t, err)
ctx = config.SetViper(ctx, vpr)
require.NoError(t, err)
cmd := tester.StubRootCmd(
"repo", "init", "s3",
"--config-file", configFP,
"--bucket", cfg.Bucket,
"--prefix", cfg.Prefix)
cmd.PersistentPostRunE = config.InitFunc()
// TODO: replace with Build() from in-flight PR #453
config.AddConfigFileFlag(cmd)
print.AddOutputFlag(cmd)
repo.AddCommands(cmd)
backup.AddCommands(cmd)
restore.AddCommands(cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
}

4
src/cli/repo/s3_test.go
View File

@ -28,8 +28,8 @@ func (suite *S3Suite) TestAddS3Commands() {
expectShort string
expectRunE func(*cobra.Command, []string) error
}{
{"init s3", initCommand, expectUse, s3InitCmd.Short, initS3Cmd},
{"connect s3", connectCommand, expectUse, s3ConnectCmd.Short, connectS3Cmd},
{"init s3", initCommand, expectUse, s3InitCmd().Short, initS3Cmd},
{"connect s3", connectCommand, expectUse, s3ConnectCmd().Short, connectS3Cmd},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {

2
src/cli/restore/exchange.go
View File

@ -105,7 +105,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}
s, a, err := config.GetStorageAndAccount(true, nil)
s, a, err := config.GetStorageAndAccount(ctx, true, nil)
if err != nil {
return Only(err)
}

4
src/cli/utils/utils.go
View File

@ -46,8 +46,8 @@ func HasNoFlagsAndShownHelp(cmd *cobra.Command) bool {
return false
}
// AddCommand adds the subCommand to the parent, and returns
// both the subCommand and its pflags.
// AddCommand adds a clone of the subCommand to the parent,
// and returns both the clone and its pflags.
func AddCommand(parent, c *cobra.Command) (*cobra.Command, *pflag.FlagSet) {
parent.AddCommand(c)
return c, c.Flags()

36
src/internal/tester/cli.go Normal file
View File

@ -0,0 +1,36 @@
package tester
import (
"context"
"fmt"
"time"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/alcionai/corso/internal/common"
)
// StubRootCmd builds a stub cobra command to be used as
// the root command for integration testing on the CLI
func StubRootCmd(args ...string) *cobra.Command {
id := uuid.NewString()
now := common.FormatTime(time.Now())
cmdArg := "testing-corso"
c := &cobra.Command{
Use: cmdArg,
Short: id,
Long: id + " - " + now,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Fprintf(cmd.OutOrStdout(), "test command args: %+v", args)
return nil
},
}
c.SetArgs(args)
return c
}
func NewContext() context.Context {
type stub struct{}
return context.WithValue(context.Background(), stub{}, stub{})
}

51
src/internal/tester/config.go
View File

@ -4,6 +4,7 @@ import (
"os"
"path"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/spf13/viper"
@ -24,7 +25,8 @@ const (
// test specific env vars
const (
EnvCorsoM365TestUserID = "CORSO_M356_TEST_USER_ID"
EnvCorsoM365TestUserID = "CORSO_M356_TEST_USER_ID"
EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE"
)
// global to hold the test config results.
@ -42,10 +44,10 @@ func cloneTestConfig() map[string]string {
return clone
}
func newTestViper() (*viper.Viper, error) {
func NewTestViper() (*viper.Viper, error) {
vpr := viper.New()
configFilePath := os.Getenv("CORSO_TEST_CONFIG_FILE")
configFilePath := os.Getenv(EnvCorsoTestConfigFilePath)
if len(configFilePath) == 0 {
return vpr, nil
}
@ -74,7 +76,7 @@ func readTestConfig() (map[string]string, error) {
return cloneTestConfig(), nil
}
vpr, err := newTestViper()
vpr, err := NewTestViper()
if err != nil {
return nil, err
}
@ -93,11 +95,52 @@ func readTestConfig() (map[string]string, error) {
fallbackTo(testEnv, testCfgPrefix, vpr.GetString(testCfgPrefix))
fallbackTo(testEnv, testCfgTenantID, os.Getenv(account.TenantID), vpr.GetString(testCfgTenantID))
fallbackTo(testEnv, testCfgUserID, os.Getenv(EnvCorsoM365TestUserID), vpr.GetString(testCfgTenantID), "lidiah@8qzvrj.onmicrosoft.com")
testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
testConfig = testEnv
return cloneTestConfig(), nil
}
// MakeTempTestConfigClone makes a copy of the test config file in a temp directory.
// This allows tests which interface with reading and writing to a config file
// (such as the CLI) to safely manipulate file contents without amending the user's
// original file.
//
// Returns a filepath string pointing to the location of the temp file.
func MakeTempTestConfigClone(t *testing.T) (*viper.Viper, string, error) {
cfg, err := readTestConfig()
if err != nil {
return nil, "", err
}
fName := path.Base(os.Getenv(EnvCorsoTestConfigFilePath))
if len(fName) == 0 || fName == "." || fName == "/" {
fName = ".corso_test.toml"
}
tDir := t.TempDir()
tDirFp := path.Join(tDir, fName)
if _, err := os.Create(tDirFp); err != nil {
return nil, "", err
}
vpr := viper.New()
vpr.SetConfigFile(tDirFp)
vpr.AddConfigPath(tDir)
vpr.SetConfigType(path.Ext(fName))
vpr.SetConfigName(fName)
for k, v := range cfg {
vpr.Set(k, v)
}
if err := vpr.WriteConfig(); err != nil {
return nil, "", err
}
return vpr, tDirFp, nil
}
// writes the first non-zero valued string to the map at the key.
// fallback priority should match viper ordering (manually handled
// here since viper fails to provide fallbacks on fileNotFoundErr):

1
src/internal/tester/integration_runners.go
View File

@ -11,6 +11,7 @@ import (
const (
CorsoCITests = "CORSO_CI_TESTS"
CorsoCLIConfigTests = "CORSO_CLI_CONFIG_TESTS"
CorsoCLIRepoTests = "CORSO_CLI_REPO_TESTS"
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS"
CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS"