larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

allow users to opt-out of metric gathering (#871)

Browse Source 
## Description

Adds a flag and options control for disabling metrics.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #741

## Test Plan

- [x] 💪 Manual
This commit is contained in:
Keepers 2022-09-15 18:56:56 -06:00 committed by GitHub
parent 16d384a232
commit 1f4a490c33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 79 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/cli/backup/exchange.go
View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -199,7 +200,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
r, err := repository.Connect(ctx, acct, s)
r, err := repository.Connect(ctx, acct, s, control.Options{})
if err != nil {
return Only(ctx, errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider))
}
@ -208,7 +209,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
sel := exchangeBackupCreateSelectors(exchangeAll, user, exchangeData)
bo, err := r.NewBackup(ctx, sel, options.Control())
bo, err := r.NewBackup(ctx, sel)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange backup"))
}
@ -297,7 +298,7 @@ func listExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
r, err := repository.Connect(ctx, acct, s)
r, err := repository.Connect(ctx, acct, s, options.Control())
if err != nil {
return Only(ctx, errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider))
}
@ -345,7 +346,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
r, err := repository.Connect(ctx, acct, s)
r, err := repository.Connect(ctx, acct, s, options.Control())
if err != nil {
return Only(ctx, errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider))
}
@ -423,7 +424,7 @@ func deleteExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
r, err := repository.Connect(ctx, acct, s)
r, err := repository.Connect(ctx, acct, s, options.Control())
if err != nil {
return Only(ctx, errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider))
}

16
src/cli/backup/exchange_integration_test.go
View File

@ -85,7 +85,7 @@ func (suite *BackupExchangeIntegrationSuite) SetupSuite() {
suite.m365UserID = tester.M365UserID(t)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st)
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
}
@ -174,7 +174,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
suite.m365UserID = tester.M365UserID(t)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st)
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
suite.backupOps = make(map[path.CategoryType]operations.BackupOperation)
@ -198,10 +198,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(
ctx,
sel.Selector,
control.NewOptions(false))
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, bop.Run(ctx))
require.NoError(t, err)
@ -333,7 +330,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
ctx := config.SetViper(tester.NewContext(), suite.vpr)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st)
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
m365UserID := tester.M365UserID(t)
@ -342,10 +339,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{m365UserID}, []string{"Inbox"}))
suite.backupOp, err = suite.repo.NewBackup(
ctx,
sel.Selector,
control.NewOptions(false))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))
require.NoError(t, err)
}

2
src/cli/cli.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/cli/backup"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/help"
"github.com/alcionai/corso/src/cli/options"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/restore"
@ -62,6 +63,7 @@ func BuildCommandTree(cmd *cobra.Command) {
config.AddConfigFlags(cmd)
print.AddOutputFlag(cmd)
logger.AddLogLevelFlag(cmd)
options.AddGlobalOperationFlags(cmd)
cmd.CompletionOptions.DisableDefaultCmd = true

25
src/cli/options/options.go
View File

@ -6,9 +6,12 @@ import (
"github.com/alcionai/corso/src/pkg/control"
)
var fastFail bool
var (
fastFail bool
noStats bool
)
// AddFlags adds the operation option flags
// AddOperationFlags adds command-local operation flags
func AddOperationFlags(parent *cobra.Command) {
fs := parent.Flags()
fs.BoolVar(&fastFail, "fast-fail", false, "stop processing immediately if any error occurs")
@ -16,7 +19,23 @@ func AddOperationFlags(parent *cobra.Command) {
cobra.CheckErr(fs.MarkHidden("fast-fail"))
}
// AddGlobalOperationFlags adds the global operations flag set.
func AddGlobalOperationFlags(parent *cobra.Command) {
fs := parent.PersistentFlags()
fs.BoolVar(&noStats, "no-stats", false, "disable anonymous usage statistics gathering")
}
// Control produces the control options based on the user's flags.
func Control() control.Options {
return control.NewOptions(fastFail)
opt := control.Defaults()
if fastFail {
opt.FailFast = true
}
if noStats {
opt.DisableMetrics = true
}
return opt
}

5
src/cli/repo/s3.go
View File

@ -6,6 +6,7 @@ import (
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/options"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/kopia"
@ -88,7 +89,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, errors.Wrap(err, "Failed to parse m365 account config"))
}
r, err := repository.Initialize(ctx, a, s)
r, err := repository.Initialize(ctx, a, s, options.Control())
if err != nil {
if succeedIfExists && kopia.IsRepoAlreadyExistsError(err) {
return nil
@ -146,7 +147,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, errors.Wrap(err, "Failed to parse m365 account config"))
}
r, err := repository.Connect(ctx, a, s)
r, err := repository.Connect(ctx, a, s, options.Control())
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to connect to the S3 repository"))
}

3
src/cli/repo/s3_integration_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
)
@ -102,7 +103,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd() {
ctx = config.SetViper(ctx, vpr)
// init the repo first
_, err = repository.Initialize(ctx, account.Account{}, st)
_, err = repository.Initialize(ctx, account.Account{}, st, control.Options{})
require.NoError(t, err)
// then test it

4
src/cli/restore/exchange.go
View File

@ -156,7 +156,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
r, err := repository.Connect(ctx, a, s)
r, err := repository.Connect(ctx, a, s, options.Control())
if err != nil {
return Only(ctx, errors.Wrapf(err, "Failed to connect to the %s repository", s.Provider))
}
@ -191,7 +191,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
sel.Include(sel.Users(selectors.Any()))
}
ro, err := r.NewRestore(ctx, backupID, sel.Selector, options.Control())
ro, err := r.NewRestore(ctx, backupID, sel.Selector)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore"))
}

7
src/cli/restore/exchange_integration_test.go
View File

@ -83,7 +83,7 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
suite.m365UserID = tester.M365UserID(t)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st)
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
suite.backupOps = make(map[path.CategoryType]operations.BackupOperation)
@ -107,10 +107,7 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(
ctx,
sel.Selector,
control.NewOptions(false))
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, bop.Run(ctx))
require.NoError(t, err)

7
src/internal/events/events.go
View File

@ -10,6 +10,7 @@ import (
analytics "github.com/rudderlabs/analytics-go"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -53,7 +54,11 @@ var (
DataPlaneURL string
)
func NewBus(s storage.Storage, a account.Account) Bus {
func NewBus(s storage.Storage, a account.Account, opts control.Options) Bus {
if opts.DisableMetrics {
return Bus{}
}
hash := repoHash(s, a)
envWK := os.Getenv("RUDDERSTACK_CORSO_WRITE_KEY")

10
src/internal/events/events_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -18,7 +19,7 @@ type EventsIntegrationSuite struct {
}
func TestMetricsIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(tester.CorsoCITests, "floob"); err != nil {
if err := tester.RunOnAny(tester.CorsoCITests); err != nil {
t.Skip(err)
}
@ -49,8 +50,11 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
)
require.NoError(t, err)
b := events.NewBus(s, a)
b := events.NewBus(s, a, control.Options{})
require.NotEmpty(t, b)
require.NoError(t, b.Close())
b2 := events.NewBus(s, a, control.Options{DisableMetrics: true})
require.Empty(t, b2)
require.NoError(t, b2.Close())
}

10
src/pkg/control/options.go
View File

@ -13,12 +13,14 @@ const (
// Options holds the optional configurations for a process
type Options struct {
FailFast bool `json:"failFast"`
Collision CollisionPolicy `json:"-"`
Collision CollisionPolicy `json:"-"`
DisableMetrics bool `json:"disableMetrics"`
FailFast bool `json:"failFast"`
}
func NewOptions(failFast bool) Options {
// Defaults provides an Options with the default values set.
func Defaults() Options {
return Options{
FailFast: failFast,
FailFast: true,
}
}

28
src/pkg/control/options_test.go
View File

@ -1,28 +0,0 @@
package control_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/pkg/control"
)
type OptionsSuite struct {
suite.Suite
}
func TestOptionsSuite(t *testing.T) {
suite.Run(t, new(OptionsSuite))
}
func (suite *OptionsSuite) TestNewOptions() {
t := suite.T()
o1 := control.NewOptions(true)
assert.True(t, o1.FailFast, "failFast")
o2 := control.NewOptions(false)
assert.False(t, o2.FailFast, "failFast")
}

13
src/pkg/repository/repository.go
View File

@ -28,6 +28,7 @@ type Repository struct {
Account account.Account // the user's m365 account connection details
Storage storage.Storage // the storage provider details and configuration
Opts control.Options
Bus events.Bus
dataLayer *kopia.Wrapper
@ -46,6 +47,7 @@ func Initialize(
ctx context.Context,
acct account.Account,
s storage.Storage,
opts control.Options,
) (*Repository, error) {
kopiaRef := kopia.NewConn(s)
if err := kopiaRef.Initialize(ctx); err != nil {
@ -70,7 +72,7 @@ func Initialize(
Version: "v1",
Account: acct,
Storage: s,
Bus: events.NewBus(s, acct),
Bus: events.NewBus(s, acct, opts),
dataLayer: w,
modelStore: ms,
}
@ -89,6 +91,7 @@ func Connect(
ctx context.Context,
acct account.Account,
s storage.Storage,
opts control.Options,
) (*Repository, error) {
kopiaRef := kopia.NewConn(s)
if err := kopiaRef.Connect(ctx); err != nil {
@ -113,7 +116,7 @@ func Connect(
Version: "v1",
Account: acct,
Storage: s,
Bus: events.NewBus(s, acct),
Bus: events.NewBus(s, acct, opts),
dataLayer: w,
modelStore: ms,
}
@ -149,11 +152,10 @@ func (r *Repository) Close(ctx context.Context) error {
func (r Repository) NewBackup(
ctx context.Context,
selector selectors.Selector,
opts control.Options,
) (operations.BackupOperation, error) {
return operations.NewBackupOperation(
ctx,
opts,
r.Opts,
r.dataLayer,
store.NewKopiaStore(r.modelStore),
r.Account,
@ -166,11 +168,10 @@ func (r Repository) NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
opts control.Options,
) (operations.RestoreOperation, error) {
return operations.NewRestoreOperation(
ctx,
opts,
r.Opts,
r.dataLayer,
store.NewKopiaStore(r.modelStore),
r.Account,

18
src/pkg/repository/repository_test.go
View File

@ -48,7 +48,7 @@ func (suite *RepositorySuite) TestInitialize() {
suite.T().Run(test.name, func(t *testing.T) {
st, err := test.storage()
assert.NoError(t, err)
_, err = repository.Initialize(context.Background(), test.account, st)
_, err = repository.Initialize(context.Background(), test.account, st, control.Options{})
test.errCheck(t, err, "")
})
}
@ -76,7 +76,7 @@ func (suite *RepositorySuite) TestConnect() {
suite.T().Run(test.name, func(t *testing.T) {
st, err := test.storage()
assert.NoError(t, err)
_, err = repository.Connect(context.Background(), test.account, st)
_, err = repository.Connect(context.Background(), test.account, st, control.Options{})
test.errCheck(t, err)
})
}
@ -127,7 +127,7 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
st := test.storage(t)
r, err := repository.Initialize(ctx, test.account, st)
r, err := repository.Initialize(ctx, test.account, st, control.Options{})
if err == nil {
defer func() {
assert.NoError(t, r.Close(ctx))
@ -146,11 +146,11 @@ func (suite *RepositoryIntegrationSuite) TestConnect() {
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
_, err := repository.Initialize(ctx, account.Account{}, st)
_, err := repository.Initialize(ctx, account.Account{}, st, control.Options{})
require.NoError(t, err)
// now re-connect
_, err = repository.Connect(ctx, account.Account{}, st)
_, err = repository.Connect(ctx, account.Account{}, st, control.Options{})
assert.NoError(t, err)
}
@ -163,10 +163,10 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
r, err := repository.Initialize(ctx, acct, st)
r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err)
bo, err := r.NewBackup(ctx, selectors.Selector{}, control.Options{})
bo, err := r.NewBackup(ctx, selectors.Selector{})
require.NoError(t, err)
require.NotNil(t, bo)
}
@ -180,10 +180,10 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
r, err := repository.Initialize(ctx, acct, st)
r, err := repository.Initialize(ctx, acct, st, control.Options{})
require.NoError(t, err)
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{}, control.Options{})
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{})
require.NoError(t, err)
require.NotNil(t, ro)
}