From 6ac8c9f331fb25a054d920d70b9ddf1d133a82ed Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 27 Apr 2023 15:47:35 -0700 Subject: [PATCH] Allow overriding the default username/hostname in kopia (#3223) This allows setting the username/hostname kopia will use for maintenance. This information is recorded in the kopia config file during connect and reused during open If no values are given, kopia pulls the values from the OS --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3077 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/kopia/conn.go | 24 ++++-- src/internal/kopia/conn_test.go | 76 ++++++++++++++++--- src/internal/kopia/model_store_test.go | 5 +- .../operations/backup_integration_test.go | 2 +- src/internal/operations/restore_test.go | 2 +- src/internal/streamstore/collectables_test.go | 3 +- src/pkg/control/options.go | 7 ++ src/pkg/repository/repository.go | 4 +- .../repository/repository_unexported_test.go | 9 ++- 9 files changed, 103 insertions(+), 29 deletions(-) diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index cabb1a555..c04875f75 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -18,6 +18,7 @@ import ( "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/pkg/errors" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/storage" ) @@ -69,7 +70,7 @@ func NewConn(s storage.Storage) *conn { } } -func (w *conn) Initialize(ctx context.Context) error { +func (w *conn) Initialize(ctx context.Context, opts control.RepoOptions) error { bst, err := blobStoreByProvider(ctx, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -92,6 +93,7 @@ func (w *conn) Initialize(ctx context.Context) error { err = w.commonConnect( ctx, + opts, cfg.KopiaCfgDir, bst, cfg.CorsoPassphrase, @@ -108,7 +110,7 @@ func (w *conn) Initialize(ctx context.Context) error { return nil } -func (w *conn) Connect(ctx context.Context) error { +func (w *conn) Connect(ctx context.Context, opts control.RepoOptions) error { bst, err := blobStoreByProvider(ctx, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -122,6 +124,7 @@ func (w *conn) Connect(ctx context.Context) error { return w.commonConnect( ctx, + opts, cfg.KopiaCfgDir, bst, cfg.CorsoPassphrase, @@ -131,16 +134,21 @@ func (w *conn) Connect(ctx context.Context) error { func (w *conn) commonConnect( ctx context.Context, + opts control.RepoOptions, configDir string, bst blob.Storage, password, compressor string, ) error { - var opts *repo.ConnectOptions + kopiaOpts := &repo.ConnectOptions{ + ClientOptions: repo.ClientOptions{ + Username: opts.User, + Hostname: opts.Host, + }, + } + if len(configDir) > 0 { - opts = &repo.ConnectOptions{ - CachingOptions: content.CachingOptions{ - CacheDirectory: configDir, - }, + kopiaOpts.CachingOptions = content.CachingOptions{ + CacheDirectory: configDir, } } else { configDir = defaultKopiaConfigDir @@ -154,7 +162,7 @@ func (w *conn) commonConnect( cfgFile, bst, password, - opts, + kopiaOpts, ); err != nil { return clues.Wrap(err, "connecting to repo").WithClues(ctx) } diff --git a/src/internal/kopia/conn_test.go b/src/internal/kopia/conn_test.go index 16d2bd943..2a8d06d49 100644 --- a/src/internal/kopia/conn_test.go +++ b/src/internal/kopia/conn_test.go @@ -14,16 +14,18 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/storage" ) -//revive:disable:context-as-argument -func openKopiaRepo(t *testing.T, ctx context.Context) (*conn, error) { - //revive:enable:context-as-argument +func openKopiaRepo( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument +) (*conn, error) { st := tester.NewPrefixedS3Storage(t) k := NewConn(st) - if err := k.Initialize(ctx); err != nil { + if err := k.Initialize(ctx, control.RepoOptions{}); err != nil { return nil, err } @@ -77,13 +79,13 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() { st := tester.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx) + err := k.Initialize(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Initialize(ctx) + err = k.Initialize(ctx, control.RepoOptions{}) assert.Error(t, err, clues.ToCore(err)) assert.ErrorIs(t, err, ErrorRepoAlreadyExists) } @@ -97,7 +99,7 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { st.Provider = storage.ProviderUnknown k := NewConn(st) - err := k.Initialize(ctx) + err := k.Initialize(ctx, control.RepoOptions{}) assert.Error(t, err, clues.ToCore(err)) } @@ -109,7 +111,7 @@ func (suite *WrapperIntegrationSuite) TestConnectWithoutInitErrors() { st := tester.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Connect(ctx) + err := k.Connect(ctx, control.RepoOptions{}) assert.Error(t, err, clues.ToCore(err)) } @@ -356,7 +358,7 @@ func (suite *WrapperIntegrationSuite) TestConfigDefaultsSetOnInitAndNotOnConnect err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Connect(ctx) + err = k.Connect(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) defer func() { @@ -384,9 +386,63 @@ func (suite *WrapperIntegrationSuite) TestInitAndConnWithTempDirectory() { require.NoError(t, err, clues.ToCore(err)) // Re-open with Connect. - err = k.Connect(ctx) + err = k.Connect(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) assert.NoError(t, err, clues.ToCore(err)) } + +func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { + ctx, flush := tester.NewContext() + defer flush() + + opts := control.RepoOptions{ + User: "foo", + Host: "bar", + } + + t := suite.T() + st := tester.NewPrefixedS3Storage(t) + k := NewConn(st) + + err := k.Initialize(ctx, opts) + require.NoError(t, err, clues.ToCore(err)) + + kopiaOpts := k.ClientOptions() + require.Equal(t, opts.User, kopiaOpts.Username) + require.Equal(t, opts.Host, kopiaOpts.Hostname) + + err = k.Close(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // Re-open with Connect and a different user/hostname. + opts.User = "hello" + opts.Host = "world" + + err = k.Connect(ctx, opts) + require.NoError(t, err, clues.ToCore(err)) + + kopiaOpts = k.ClientOptions() + require.Equal(t, opts.User, kopiaOpts.Username) + require.Equal(t, opts.Host, kopiaOpts.Hostname) + + err = k.Close(ctx) + require.NoError(t, err, clues.ToCore(err)) + + // Make sure not setting the values uses the kopia defaults. + opts.User = "" + opts.Host = "" + + err = k.Connect(ctx, opts) + require.NoError(t, err, clues.ToCore(err)) + + kopiaOpts = k.ClientOptions() + assert.NotEmpty(t, kopiaOpts.Username) + assert.NotEqual(t, "hello", kopiaOpts.Username) + assert.NotEmpty(t, kopiaOpts.Hostname) + assert.NotEqual(t, "world", kopiaOpts.Hostname) + + err = k.Close(ctx) + assert.NoError(t, err, clues.ToCore(err)) +} diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index 5a34c56ff..4922dbe95 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/control" ) type fooModel struct { @@ -803,7 +804,7 @@ func openConnAndModelStore( st := tester.NewPrefixedS3Storage(t) c := NewConn(st) - err := c.Initialize(ctx) + err := c.Initialize(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) defer func() { @@ -822,7 +823,7 @@ func reconnectToModelStore( ctx context.Context, //revive:disable-line:context-as-argument c *conn, ) *ModelStore { - err := c.Connect(ctx) + err := c.Connect(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 2ad542fad..5fdda1f71 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -83,7 +83,7 @@ func prepNewTestBackupOp( k = kopia.NewConn(st) ) - err := k.Initialize(ctx) + err := k.Initialize(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 6aa8fc370..5eb06dabb 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -175,7 +175,7 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { suite.acct = tester.NewM365Account(t) - err := k.Initialize(ctx) + err := k.Initialize(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index 8c146e3a6..640aa6406 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -41,7 +42,7 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() { st := tester.NewPrefixedS3Storage(t) k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx)) + require.NoError(t, k.Initialize(ctx, control.RepoOptions{})) suite.kcloser = func() { k.Close(ctx) } diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index de6e76efc..94b2e316c 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -13,6 +13,7 @@ type Options struct { SkipReduce bool `json:"skipReduce"` ToggleFeatures Toggles `json:"toggleFeatures"` Parallelism Parallelism `json:"parallelism"` + Repo RepoOptions `json:"repo"` } type FailureBehavior string @@ -33,6 +34,12 @@ const ( BestEffort FailureBehavior = "best-effort" ) +// Repo represents options that are specific to the repo storing backed up data. +type RepoOptions struct { + User string `json:"user"` + Host string `json:"host"` +} + // Defaults provides an Options with the default values set. func Defaults() Options { return Options{ diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index fdf07174b..a374d400b 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -120,7 +120,7 @@ func Initialize( }() kopiaRef := kopia.NewConn(s) - if err := kopiaRef.Initialize(ctx); err != nil { + if err := kopiaRef.Initialize(ctx, opts.Repo); err != nil { // replace common internal errors so that sdk users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return nil, clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -202,7 +202,7 @@ func Connect( defer close(complete) kopiaRef := kopia.NewConn(s) - if err := kopiaRef.Connect(ctx); err != nil { + if err := kopiaRef.Connect(ctx, opts.Repo); err != nil { return nil, clues.Wrap(err, "connecting kopia client") } // kopiaRef comes with a count of 1 and NewWrapper/NewModelStore bumps it again so safe diff --git a/src/pkg/repository/repository_unexported_test.go b/src/pkg/repository/repository_unexported_test.go index 29a359166..6d05a7b48 100644 --- a/src/pkg/repository/repository_unexported_test.go +++ b/src/pkg/repository/repository_unexported_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -236,10 +237,10 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() { require.NotNil(t, k) - err = k.Initialize(ctx) + err = k.Initialize(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) - err = k.Connect(ctx) + err = k.Connect(ctx, control.RepoOptions{}) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { @@ -286,8 +287,8 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { k = kopia.NewConn(s) ) - require.NoError(t, k.Initialize(ctx)) - require.NoError(t, k.Connect(ctx)) + require.NoError(t, k.Initialize(ctx, control.RepoOptions{})) + require.NoError(t, k.Connect(ctx, control.RepoOptions{})) defer k.Close(ctx)