diff --git a/CHANGELOG.md b/CHANGELOG.md index c80136a75..54ef6cde3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +## [v0.11.0] (beta) - 2023-07-18 + ### Added - Drive items backup and restore link shares - Restore commands now accept an optional top-level restore destination with the `--destination` flag. Setting the destination to '/' will restore items back into their original location. @@ -322,7 +324,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Miscellaneous - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35)) -[Unreleased]: https://github.com/alcionai/corso/compare/v0.10.0...HEAD +[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD +[v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0 [v0.8.0]: https://github.com/alcionai/corso/compare/v0.7.1...v0.8.0 diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 30819eed1..abc44c991 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -277,7 +277,7 @@ func genericDeleteCommand( ctx := clues.Add(cmd.Context(), "delete_backup_id", bID) - r, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd)) + r, _, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } @@ -303,7 +303,7 @@ func genericListCommand( ) error { ctx := cmd.Context() - r, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd)) + r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 16429fb8f..99bb0ff78 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -275,15 +275,13 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.MakeExchangeOpts(cmd) - r, _, _, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd)) + r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } defer utils.CloseRepo(ctx, r) - ctrlOpts := utils.Control() - ds, err := runDetailsExchangeCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 7249bde5d..62ce242d4 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -234,15 +234,13 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.MakeOneDriveOpts(cmd) - r, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) + r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } defer utils.CloseRepo(ctx, r) - ctrlOpts := utils.Control() - ds, err := runDetailsOneDriveCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index 3922bfa3d..7fcf58163 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -325,15 +325,13 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.MakeSharePointOpts(cmd) - r, _, _, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd)) + r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } defer utils.CloseRepo(ctx, r) - ctrlOpts := utils.Control() - ds, err := runDetailsSharePointCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) diff --git a/src/cli/config/config.go b/src/cli/config/config.go index 44dcbf1e7..39a34eb9c 100644 --- a/src/cli/config/config.go +++ b/src/cli/config/config.go @@ -14,6 +14,7 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/storage" ) @@ -204,9 +205,15 @@ func WriteRepoConfig( ctx context.Context, s3Config storage.S3Config, m365Config account.M365Config, + repoOpts repository.Options, repoID string, ) error { - return writeRepoConfigWithViper(GetViper(ctx), s3Config, m365Config, repoID) + return writeRepoConfigWithViper( + GetViper(ctx), + s3Config, + m365Config, + repoOpts, + repoID) } // writeRepoConfigWithViper implements WriteRepoConfig, but takes in a viper @@ -215,6 +222,7 @@ func writeRepoConfigWithViper( vpr *viper.Viper, s3Config storage.S3Config, m365Config account.M365Config, + repoOpts repository.Options, repoID string, ) error { s3Config = s3Config.Normalize() @@ -228,6 +236,15 @@ func writeRepoConfigWithViper( vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS) vpr.Set(RepoID, repoID) + // Need if-checks as Viper will write empty values otherwise. + if len(repoOpts.User) > 0 { + vpr.Set(CorsoUser, repoOpts.User) + } + + if len(repoOpts.Host) > 0 { + vpr.Set(CorsoHost, repoOpts.Host) + } + vpr.Set(AccountProviderTypeKey, account.ProviderM365.String()) vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID) diff --git a/src/cli/config/config_test.go b/src/cli/config/config_test.go index 14f790736..e58c3a99c 100644 --- a/src/cli/config/config_test.go +++ b/src/cli/config/config_test.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/storage" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -135,8 +136,11 @@ func (suite *ConfigSuite) TestWriteReadConfig() { ) const ( - bkt = "write-read-config-bucket" - tid = "3c0748d2-470e-444c-9064-1268e52609d5" + bkt = "write-read-config-bucket" + tid = "3c0748d2-470e-444c-9064-1268e52609d5" + repoID = "repoid" + user = "a-user" + host = "some-host" ) err := initWithViper(vpr, testConfigFilePath) @@ -145,7 +149,12 @@ func (suite *ConfigSuite) TestWriteReadConfig() { s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true} m365 := account.M365Config{AzureTenantID: tid} - err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid") + rOpts := repository.Options{ + User: user, + Host: host, + } + + err = writeRepoConfigWithViper(vpr, s3Cfg, m365, rOpts, repoID) require.NoError(t, err, "writing repo config", clues.ToCore(err)) err = vpr.ReadInConfig() @@ -160,6 +169,10 @@ func (suite *ConfigSuite) TestWriteReadConfig() { readM365, err := m365ConfigsFromViper(vpr) require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID) + + gotUser, gotHost := getUserHost(vpr, true) + assert.Equal(t, user, gotUser) + assert.Equal(t, host, gotHost) } func (suite *ConfigSuite) TestMustMatchConfig() { @@ -181,7 +194,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() { s3Cfg := storage.S3Config{Bucket: bkt} m365 := account.M365Config{AzureTenantID: tid} - err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid") + err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid") require.NoError(t, err, "writing repo config", clues.ToCore(err)) err = vpr.ReadInConfig() @@ -383,7 +396,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() { } m365 := account.M365Config{AzureTenantID: tid} - err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid") + err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid") require.NoError(t, err, "writing repo config", clues.ToCore(err)) err = vpr.ReadInConfig() diff --git a/src/cli/repo/repo.go b/src/cli/repo/repo.go index 21ff2d11e..378cce16a 100644 --- a/src/cli/repo/repo.go +++ b/src/cli/repo/repo.go @@ -121,7 +121,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error { return err } - r, _, _, err := utils.GetAccountAndConnect(ctx, path.UnknownService, S3Overrides(cmd)) + r, _, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.UnknownService, S3Overrides(cmd)) if err != nil { return print.Only(ctx, err) } diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 86a241748..58f04764c 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -171,7 +171,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { Infof(ctx, "Initialized a S3 repository within bucket %s.", s3Cfg.Bucket) - if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil { + if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opt.Repo, r.GetID()); err != nil { return Only(ctx, clues.Wrap(err, "Failed to write repository configuration")) } @@ -228,12 +228,14 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.New(invalidEndpointErr)) } + opts := utils.ControlWithConfig(cfg) + r, err := repository.ConnectAndSendConnectEvent( ctx, cfg.Account, cfg.Storage, repoID, - utils.ControlWithConfig(cfg)) + opts) if err != nil { return Only(ctx, clues.Wrap(err, "Failed to connect to the S3 repository")) } @@ -242,7 +244,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { Infof(ctx, "Connected to S3 bucket %s.", s3Cfg.Bucket) - if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil { + if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opts.Repo, r.GetID()); err != nil { return Only(ctx, clues.Wrap(err, "Failed to write repository configuration")) } diff --git a/src/cli/restore/restore.go b/src/cli/restore/restore.go index a3e3dce85..2ca3ed0eb 100644 --- a/src/cli/restore/restore.go +++ b/src/cli/restore/restore.go @@ -94,7 +94,7 @@ func runRestore( sel selectors.Selector, backupID, serviceName string, ) error { - r, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) + r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) if err != nil { return Only(ctx, err) } diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 3b0e8d86b..a542d55f3 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -23,10 +23,10 @@ func GetAccountAndConnect( ctx context.Context, pst path.ServiceType, overrides map[string]string, -) (repository.Repository, *storage.Storage, *account.Account, error) { +) (repository.Repository, *storage.Storage, *account.Account, *control.Options, error) { cfg, err := config.GetConfigRepoDetails(ctx, true, true, overrides) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } repoID := cfg.RepoID @@ -34,18 +34,20 @@ func GetAccountAndConnect( repoID = events.RepoIDNotFound } - r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, ControlWithConfig(cfg)) + opts := ControlWithConfig(cfg) + + r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, opts) if err != nil { - return nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository") + return nil, nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository") } // this initializes our graph api client configurations, // including control options such as concurency limitations. if _, err := r.ConnectToM365(ctx, pst); err != nil { - return nil, nil, nil, clues.Wrap(err, "connecting to m365") + return nil, nil, nil, nil, clues.Wrap(err, "connecting to m365") } - return r, &cfg.Storage, &cfg.Account, nil + return r, &cfg.Storage, &cfg.Account, &opts, nil } func AccountConnectAndWriteRepoConfig( @@ -53,7 +55,7 @@ func AccountConnectAndWriteRepoConfig( pst path.ServiceType, overrides map[string]string, ) (repository.Repository, *account.Account, error) { - r, stg, acc, err := GetAccountAndConnect(ctx, pst, overrides) + r, stg, acc, opts, err := GetAccountAndConnect(ctx, pst, overrides) if err != nil { logger.CtxErr(ctx, err).Info("getting and connecting account") return nil, nil, err @@ -73,7 +75,7 @@ func AccountConnectAndWriteRepoConfig( // repo config gets set during repo connect and init. // This call confirms we have the correct values. - err = config.WriteRepoConfig(ctx, s3Config, m365Config, r.GetID()) + err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID()) if err != nil { logger.CtxErr(ctx, err).Info("writing to repository configuration") return nil, nil, err diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index 37e01ae02..33de060f8 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -39,7 +39,7 @@ func main() { fatal(cc.Context(), "unknown service", nil) } - r, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil) + r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil) if err != nil { fatal(cc.Context(), "unable to connect account", err) } diff --git a/src/go.mod b/src/go.mod index 63883e489..d2de6dbec 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.300 + github.com/aws/aws-sdk-go v1.44.301 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index e8d9e6bea..2e9794b7f 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY= -github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.301 h1:VofuXktwHFTBUvoPiHxQis/3uKgu0RtgUwLtNujd3Zs= +github.com/aws/aws-sdk-go v1.44.301/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/website/docs/setup/maintenance.md b/website/docs/setup/maintenance.md index b3263f186..a2ec6b1ed 100644 --- a/website/docs/setup/maintenance.md +++ b/website/docs/setup/maintenance.md @@ -4,37 +4,41 @@ description: "Repository maintenance." # Repository maintenance -Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user. +Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user. Maintenance can also free up space by removing data no longer referenced by any backups from the repository. -It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe -to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository +It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe +to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository owner to try to detect concurrent maintenance operations. ## Repository owner The repository owner is set to the user and hostname of the machine that runs maintenance on the repo the first time. -If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user` +If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user` and `--host` flags. The `--force` flag updates the repository owner and runs maintenance. -The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation -but doesn't update repo owner info. +The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation but +doesn't update repo owner info. -*If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same -repository. Concurrent maintenance operations a repository may result in data loss.* +:::danger + +If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same +repository. Concurrent maintenance operations a repository may result in data loss. + +::: ## Maintenance types Corso allows for two different types of maintenance: `metadata` and `complete`. -Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data -in backups and removes unreferenced data from the repository. +Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data in +backups and removes unreferenced data from the repository. -As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a -backup may not result in a reduction of objects in the storage service Corso is backing up to. +As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a backup +may not result in a reduction of objects in the storage service Corso is backing up to. Deletion of old objects in the storage service depends on both wall-clock time and running maintenance. diff --git a/website/docs/setup/restore-options.md b/website/docs/setup/restore-options.md index f1c3a8de0..9072b2d3f 100644 --- a/website/docs/setup/restore-options.md +++ b/website/docs/setup/restore-options.md @@ -1,4 +1,4 @@ -# Restore Options +# Restore options import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; diff --git a/website/sidebars.js b/website/sidebars.js index d15428e14..aecffc02a 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -42,14 +42,15 @@ const sidebars = { items: [ { type: 'category', - label: 'Setup', + label: 'Setup and maintenance', link: { slug: 'cli/setup', - description: 'Documentation for commonly-used Corso setup CLI commands', + description: 'Documentation for Corso setup and maintenance commands', }, items: [ 'cli/corso-repo-init-s3', 'cli/corso-repo-connect-s3', + 'cli/corso-repo-maintenance', 'cli/corso-env'] }, {