Merge branch 'main' into teamsCLI

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased] (beta)
 
+## [v0.11.0] (beta) - 2023-07-18
+
 ### Added
 - Drive items backup and restore link shares
 - Restore commands now accept an optional top-level restore destination with the `--destination` flag.  Setting the destination to '/' will restore items back into their original location.  
@@ -322,7 +324,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Miscellaneous
   - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35))
 
-[Unreleased]: https://github.com/alcionai/corso/compare/v0.10.0...HEAD
+[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD
+[v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0
 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0
 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0
 [v0.8.0]: https://github.com/alcionai/corso/compare/v0.7.1...v0.8.0

src/cli/backup/backup.go

@@ -277,7 +277,7 @@ func genericDeleteCommand(
 
 	ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd))
+	r, _, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
@@ -303,7 +303,7 @@ func genericListCommand(
 ) error {
 	ctx := cmd.Context()
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd))
+	r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}

src/cli/backup/exchange.go

@@ -275,15 +275,13 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	opts := utils.MakeExchangeOpts(cmd)
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd))
+	r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
-	ctrlOpts := utils.Control()
-
 	ds, err := runDetailsExchangeCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
 	if err != nil {
 		return Only(ctx, err)

src/cli/backup/onedrive.go

@@ -234,15 +234,13 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	opts := utils.MakeOneDriveOpts(cmd)
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd))
+	r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
-	ctrlOpts := utils.Control()
-
 	ds, err := runDetailsOneDriveCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
 	if err != nil {
 		return Only(ctx, err)

src/cli/backup/sharepoint.go

@@ -325,15 +325,13 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
 	ctx := cmd.Context()
 	opts := utils.MakeSharePointOpts(cmd)
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd))
+	r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}
 
 	defer utils.CloseRepo(ctx, r)
 
-	ctrlOpts := utils.Control()
-
 	ds, err := runDetailsSharePointCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
 	if err != nil {
 		return Only(ctx, err)

src/cli/config/config.go

@@ -14,6 +14,7 @@ import (
 	. "github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/internal/common/str"
 	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/control/repository"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/storage"
 )
@@ -204,9 +205,15 @@ func WriteRepoConfig(
 	ctx context.Context,
 	s3Config storage.S3Config,
 	m365Config account.M365Config,
+	repoOpts repository.Options,
 	repoID string,
 ) error {
-	return writeRepoConfigWithViper(GetViper(ctx), s3Config, m365Config, repoID)
+	return writeRepoConfigWithViper(
+		GetViper(ctx),
+		s3Config,
+		m365Config,
+		repoOpts,
+		repoID)
 }
 
 // writeRepoConfigWithViper implements WriteRepoConfig, but takes in a viper
@@ -215,6 +222,7 @@ func writeRepoConfigWithViper(
 	vpr *viper.Viper,
 	s3Config storage.S3Config,
 	m365Config account.M365Config,
+	repoOpts repository.Options,
 	repoID string,
 ) error {
 	s3Config = s3Config.Normalize()
@@ -228,6 +236,15 @@ func writeRepoConfigWithViper(
 	vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
 	vpr.Set(RepoID, repoID)
 
+	// Need if-checks as Viper will write empty values otherwise.
+	if len(repoOpts.User) > 0 {
+		vpr.Set(CorsoUser, repoOpts.User)
+	}
+
+	if len(repoOpts.Host) > 0 {
+		vpr.Set(CorsoHost, repoOpts.Host)
+	}
+
 	vpr.Set(AccountProviderTypeKey, account.ProviderM365.String())
 	vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID)
 

src/cli/config/config_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/internal/tester/tconfig"
 	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/control/repository"
 	"github.com/alcionai/corso/src/pkg/credentials"
 	"github.com/alcionai/corso/src/pkg/storage"
 	storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
@@ -135,8 +136,11 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	)
 
 	const (
-		bkt = "write-read-config-bucket"
+		bkt    = "write-read-config-bucket"
-		tid = "3c0748d2-470e-444c-9064-1268e52609d5"
+		tid    = "3c0748d2-470e-444c-9064-1268e52609d5"
+		repoID = "repoid"
+		user   = "a-user"
+		host   = "some-host"
 	)
 
 	err := initWithViper(vpr, testConfigFilePath)
@@ -145,7 +149,12 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
 	m365 := account.M365Config{AzureTenantID: tid}
 
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
+	rOpts := repository.Options{
+		User: user,
+		Host: host,
+	}
+
+	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, rOpts, repoID)
 	require.NoError(t, err, "writing repo config", clues.ToCore(err))
 
 	err = vpr.ReadInConfig()
@@ -160,6 +169,10 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
 	readM365, err := m365ConfigsFromViper(vpr)
 	require.NoError(t, err, clues.ToCore(err))
 	assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
+
+	gotUser, gotHost := getUserHost(vpr, true)
+	assert.Equal(t, user, gotUser)
+	assert.Equal(t, host, gotHost)
 }
 
 func (suite *ConfigSuite) TestMustMatchConfig() {
@@ -181,7 +194,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
 	s3Cfg := storage.S3Config{Bucket: bkt}
 	m365 := account.M365Config{AzureTenantID: tid}
 
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
+	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
 	require.NoError(t, err, "writing repo config", clues.ToCore(err))
 
 	err = vpr.ReadInConfig()
@@ -383,7 +396,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
 	}
 	m365 := account.M365Config{AzureTenantID: tid}
 
-	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
+	err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
 	require.NoError(t, err, "writing repo config", clues.ToCore(err))
 
 	err = vpr.ReadInConfig()

src/cli/repo/repo.go

@@ -121,7 +121,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	r, _, _, err := utils.GetAccountAndConnect(ctx, path.UnknownService, S3Overrides(cmd))
+	r, _, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.UnknownService, S3Overrides(cmd))
 	if err != nil {
 		return print.Only(ctx, err)
 	}

src/cli/repo/s3.go

@@ -171,7 +171,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
 
 	Infof(ctx, "Initialized a S3 repository within bucket %s.", s3Cfg.Bucket)
 
-	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
+	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opt.Repo, r.GetID()); err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
 	}
 
@@ -228,12 +228,14 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 		return Only(ctx, clues.New(invalidEndpointErr))
 	}
 
+	opts := utils.ControlWithConfig(cfg)
+
 	r, err := repository.ConnectAndSendConnectEvent(
 		ctx,
 		cfg.Account,
 		cfg.Storage,
 		repoID,
-		utils.ControlWithConfig(cfg))
+		opts)
 	if err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to connect to the S3 repository"))
 	}
@@ -242,7 +244,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
 
 	Infof(ctx, "Connected to S3 bucket %s.", s3Cfg.Bucket)
 
-	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
+	if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opts.Repo, r.GetID()); err != nil {
 		return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
 	}
 

src/cli/restore/restore.go

@@ -94,7 +94,7 @@ func runRestore(
 	sel selectors.Selector,
 	backupID, serviceName string,
 ) error {
-	r, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
+	r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
 	if err != nil {
 		return Only(ctx, err)
 	}

src/cli/utils/utils.go

@@ -23,10 +23,10 @@ func GetAccountAndConnect(
 	ctx context.Context,
 	pst path.ServiceType,
 	overrides map[string]string,
-) (repository.Repository, *storage.Storage, *account.Account, error) {
+) (repository.Repository, *storage.Storage, *account.Account, *control.Options, error) {
 	cfg, err := config.GetConfigRepoDetails(ctx, true, true, overrides)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, nil, nil, err
 	}
 
 	repoID := cfg.RepoID
@@ -34,18 +34,20 @@ func GetAccountAndConnect(
 		repoID = events.RepoIDNotFound
 	}
 
-	r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, ControlWithConfig(cfg))
+	opts := ControlWithConfig(cfg)
+
+	r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, opts)
 	if err != nil {
-		return nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository")
+		return nil, nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository")
 	}
 
 	// this initializes our graph api client configurations,
 	// including control options such as concurency limitations.
 	if _, err := r.ConnectToM365(ctx, pst); err != nil {
-		return nil, nil, nil, clues.Wrap(err, "connecting to m365")
+		return nil, nil, nil, nil, clues.Wrap(err, "connecting to m365")
 	}
 
-	return r, &cfg.Storage, &cfg.Account, nil
+	return r, &cfg.Storage, &cfg.Account, &opts, nil
 }
 
 func AccountConnectAndWriteRepoConfig(
@@ -53,7 +55,7 @@ func AccountConnectAndWriteRepoConfig(
 	pst path.ServiceType,
 	overrides map[string]string,
 ) (repository.Repository, *account.Account, error) {
-	r, stg, acc, err := GetAccountAndConnect(ctx, pst, overrides)
+	r, stg, acc, opts, err := GetAccountAndConnect(ctx, pst, overrides)
 	if err != nil {
 		logger.CtxErr(ctx, err).Info("getting and connecting account")
 		return nil, nil, err
@@ -73,7 +75,7 @@ func AccountConnectAndWriteRepoConfig(
 
 	// repo config gets set during repo connect and init.
 	// This call confirms we have the correct values.
-	err = config.WriteRepoConfig(ctx, s3Config, m365Config, r.GetID())
+	err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID())
 	if err != nil {
 		logger.CtxErr(ctx, err).Info("writing to repository configuration")
 		return nil, nil, err

src/cmd/longevity_test/longevity.go

@@ -39,7 +39,7 @@ func main() {
 		fatal(cc.Context(), "unknown service", nil)
 	}
 
-	r, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil)
+	r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil)
 	if err != nil {
 		fatal(cc.Context(), "unable to connect account", err)
 	}

src/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
 	github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go v1.44.300
+	github.com/aws/aws-sdk-go v1.44.301
 	github.com/aws/aws-xray-sdk-go v1.8.1
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/google/uuid v1.3.0

src/go.sum

@@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
-github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY=
+github.com/aws/aws-sdk-go v1.44.301 h1:VofuXktwHFTBUvoPiHxQis/3uKgu0RtgUwLtNujd3Zs=
-github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.301/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo=
 github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

website/docs/setup/maintenance.md

@@ -4,37 +4,41 @@ description: "Repository maintenance."
 
 # Repository maintenance
 
-Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user.  
+Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user.
 Maintenance can also free up space by removing data no longer referenced by any backups from the repository.
 
-It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe  
+It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe
-to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository  
+to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository
 owner to try to detect concurrent maintenance operations.
 
 ## Repository owner
 
 The repository owner is set to the user and hostname of the machine that runs maintenance on the repo the first time.  
 
-If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user`  
+If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user`
 and `--host` flags.
 
 The `--force` flag updates the repository owner and runs maintenance.
 
-The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation  
+The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation but
-but doesn't update repo owner info.
+doesn't update repo owner info.
 
-*If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same  
+:::danger
-repository. Concurrent maintenance operations a repository may result in data loss.*
+
+If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same
+repository. Concurrent maintenance operations a repository may result in data loss.
+
+:::
 
 ## Maintenance types
 
 Corso allows for two different types of maintenance: `metadata` and `complete`.
 
-Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data  
+Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data in
-in backups and removes unreferenced data from the repository.
+backups and removes unreferenced data from the repository.
 
-As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a  
+As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a backup
-backup may not result in a reduction of objects in the storage service Corso is backing up to.  
+may not result in a reduction of objects in the storage service Corso is backing up to.
 
 Deletion of old objects in the storage service depends on both wall-clock time and running maintenance.
 

website/docs/setup/restore-options.md

@@ -1,4 +1,4 @@
-# Restore Options
+# Restore options
 
 import CodeBlock from '@theme/CodeBlock';
 import Tabs from '@theme/Tabs';

website/sidebars.js

@@ -42,14 +42,15 @@ const sidebars = {
       items: [
         {
           type: 'category',
-          label: 'Setup',
+          label: 'Setup and maintenance',
           link: {
             slug: 'cli/setup',
-            description: 'Documentation for commonly-used Corso setup CLI commands',
+            description: 'Documentation for Corso setup and maintenance commands',
           },
           items: [
             'cli/corso-repo-init-s3',
             'cli/corso-repo-connect-s3',
+            'cli/corso-repo-maintenance',
             'cli/corso-env']
         },
         {