larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into teamsCLI

Browse Source
This commit is contained in:
neha_gupta 2023-07-19 10:24:23 +05:30 committed by GitHub
commit 7ceb4809a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 86 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] (beta)
## [v0.11.0] (beta) - 2023-07-18
### Added
- Drive items backup and restore link shares
- Restore commands now accept an optional top-level restore destination with the `--destination` flag. Setting the destination to '/' will restore items back into their original location.
@ -322,7 +324,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Miscellaneous
- Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35))
[Unreleased]: https://github.com/alcionai/corso/compare/v0.10.0...HEAD
[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD
[v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0
[v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0
[v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0
[v0.8.0]: https://github.com/alcionai/corso/compare/v0.7.1...v0.8.0

4
src/cli/backup/backup.go
View File

@ -277,7 +277,7 @@ func genericDeleteCommand(
ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)
r, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, pst, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
@ -303,7 +303,7 @@ func genericListCommand(
) error {
ctx := cmd.Context()
r, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

4
src/cli/backup/exchange.go
View File

@ -275,15 +275,13 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.ExchangeService, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ctrlOpts := utils.Control()
ds, err := runDetailsExchangeCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
if err != nil {
return Only(ctx, err)

4
src/cli/backup/onedrive.go
View File

@ -234,15 +234,13 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ctrlOpts := utils.Control()
ds, err := runDetailsOneDriveCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
if err != nil {
return Only(ctx, err)

4
src/cli/backup/sharepoint.go
View File

@ -325,15 +325,13 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeSharePointOpts(cmd)
r, _, _, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd))
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.SharePointService, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ctrlOpts := utils.Control()
ds, err := runDetailsSharePointCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
if err != nil {
return Only(ctx, err)

19
src/cli/config/config.go
View File

@ -14,6 +14,7 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -204,9 +205,15 @@ func WriteRepoConfig(
ctx context.Context,
s3Config storage.S3Config,
m365Config account.M365Config,
repoOpts repository.Options,
repoID string,
) error {
return writeRepoConfigWithViper(GetViper(ctx), s3Config, m365Config, repoID)
return writeRepoConfigWithViper(
GetViper(ctx),
s3Config,
m365Config,
repoOpts,
repoID)
}
// writeRepoConfigWithViper implements WriteRepoConfig, but takes in a viper
@ -215,6 +222,7 @@ func writeRepoConfigWithViper(
vpr *viper.Viper,
s3Config storage.S3Config,
m365Config account.M365Config,
repoOpts repository.Options,
repoID string,
) error {
s3Config = s3Config.Normalize()
@ -228,6 +236,15 @@ func writeRepoConfigWithViper(
vpr.Set(DisableTLSVerificationKey, s3Config.DoNotVerifyTLS)
vpr.Set(RepoID, repoID)
// Need if-checks as Viper will write empty values otherwise.
if len(repoOpts.User) > 0 {
vpr.Set(CorsoUser, repoOpts.User)
}
if len(repoOpts.Host) > 0 {
vpr.Set(CorsoHost, repoOpts.Host)
}
vpr.Set(AccountProviderTypeKey, account.ProviderM365.String())
vpr.Set(AzureTenantIDKey, m365Config.AzureTenantID)

23
src/cli/config/config_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
@ -135,8 +136,11 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
)
const (
bkt = "write-read-config-bucket"
tid = "3c0748d2-470e-444c-9064-1268e52609d5"
bkt = "write-read-config-bucket"
tid = "3c0748d2-470e-444c-9064-1268e52609d5"
repoID = "repoid"
user = "a-user"
host = "some-host"
)
err := initWithViper(vpr, testConfigFilePath)
@ -145,7 +149,12 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
m365 := account.M365Config{AzureTenantID: tid}
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
rOpts := repository.Options{
User: user,
Host: host,
}
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, rOpts, repoID)
require.NoError(t, err, "writing repo config", clues.ToCore(err))
err = vpr.ReadInConfig()
@ -160,6 +169,10 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
readM365, err := m365ConfigsFromViper(vpr)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
gotUser, gotHost := getUserHost(vpr, true)
assert.Equal(t, user, gotUser)
assert.Equal(t, host, gotHost)
}
func (suite *ConfigSuite) TestMustMatchConfig() {
@ -181,7 +194,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
s3Cfg := storage.S3Config{Bucket: bkt}
m365 := account.M365Config{AzureTenantID: tid}
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
require.NoError(t, err, "writing repo config", clues.ToCore(err))
err = vpr.ReadInConfig()
@ -383,7 +396,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
}
m365 := account.M365Config{AzureTenantID: tid}
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, "repoid")
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
require.NoError(t, err, "writing repo config", clues.ToCore(err))
err = vpr.ReadInConfig()

2
src/cli/repo/repo.go
View File

@ -121,7 +121,7 @@ func handleMaintenanceCmd(cmd *cobra.Command, args []string) error {
return err
}
r, _, _, err := utils.GetAccountAndConnect(ctx, path.UnknownService, S3Overrides(cmd))
r, _, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.UnknownService, S3Overrides(cmd))
if err != nil {
return print.Only(ctx, err)
}

8
src/cli/repo/s3.go
View File

@ -171,7 +171,7 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
Infof(ctx, "Initialized a S3 repository within bucket %s.", s3Cfg.Bucket)
if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opt.Repo, r.GetID()); err != nil {
return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
}
@ -228,12 +228,14 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.New(invalidEndpointErr))
}
opts := utils.ControlWithConfig(cfg)
r, err := repository.ConnectAndSendConnectEvent(
ctx,
cfg.Account,
cfg.Storage,
repoID,
utils.ControlWithConfig(cfg))
opts)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to connect to the S3 repository"))
}
@ -242,7 +244,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
Infof(ctx, "Connected to S3 bucket %s.", s3Cfg.Bucket)
if err = config.WriteRepoConfig(ctx, s3Cfg, m365, r.GetID()); err != nil {
if err = config.WriteRepoConfig(ctx, s3Cfg, m365, opts.Repo, r.GetID()); err != nil {
return Only(ctx, clues.Wrap(err, "Failed to write repository configuration"))
}

2
src/cli/restore/restore.go
View File

@ -94,7 +94,7 @@ func runRestore(
sel selectors.Selector,
backupID, serviceName string,
) error {
r, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}

18
src/cli/utils/utils.go
View File

@ -23,10 +23,10 @@ func GetAccountAndConnect(
ctx context.Context,
pst path.ServiceType,
overrides map[string]string,
) (repository.Repository, *storage.Storage, *account.Account, error) {
) (repository.Repository, *storage.Storage, *account.Account, *control.Options, error) {
cfg, err := config.GetConfigRepoDetails(ctx, true, true, overrides)
if err != nil {
return nil, nil, nil, err
return nil, nil, nil, nil, err
}
repoID := cfg.RepoID
@ -34,18 +34,20 @@ func GetAccountAndConnect(
repoID = events.RepoIDNotFound
}
r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, ControlWithConfig(cfg))
opts := ControlWithConfig(cfg)
r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, repoID, opts)
if err != nil {
return nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository")
return nil, nil, nil, nil, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository")
}
// this initializes our graph api client configurations,
// including control options such as concurency limitations.
if _, err := r.ConnectToM365(ctx, pst); err != nil {
return nil, nil, nil, clues.Wrap(err, "connecting to m365")
return nil, nil, nil, nil, clues.Wrap(err, "connecting to m365")
}
return r, &cfg.Storage, &cfg.Account, nil
return r, &cfg.Storage, &cfg.Account, &opts, nil
}
func AccountConnectAndWriteRepoConfig(
@ -53,7 +55,7 @@ func AccountConnectAndWriteRepoConfig(
pst path.ServiceType,
overrides map[string]string,
) (repository.Repository, *account.Account, error) {
r, stg, acc, err := GetAccountAndConnect(ctx, pst, overrides)
r, stg, acc, opts, err := GetAccountAndConnect(ctx, pst, overrides)
if err != nil {
logger.CtxErr(ctx, err).Info("getting and connecting account")
return nil, nil, err
@ -73,7 +75,7 @@ func AccountConnectAndWriteRepoConfig(
// repo config gets set during repo connect and init.
// This call confirms we have the correct values.
err = config.WriteRepoConfig(ctx, s3Config, m365Config, r.GetID())
err = config.WriteRepoConfig(ctx, s3Config, m365Config, opts.Repo, r.GetID())
if err != nil {
logger.CtxErr(ctx, err).Info("writing to repository configuration")
return nil, nil, err

2
src/cmd/longevity_test/longevity.go
View File

@ -39,7 +39,7 @@ func main() {
fatal(cc.Context(), "unknown service", nil)
}
r, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil)
r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil)
if err != nil {
fatal(cc.Context(), "unable to connect account", err)
}

2
src/go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go v1.44.300
github.com/aws/aws-sdk-go v1.44.301
github.com/aws/aws-xray-sdk-go v1.8.1
github.com/cenkalti/backoff/v4 v4.2.1
github.com/google/uuid v1.3.0

4
src/go.sum
View File

@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY=
github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go v1.44.301 h1:VofuXktwHFTBUvoPiHxQis/3uKgu0RtgUwLtNujd3Zs=
github.com/aws/aws-sdk-go v1.44.301/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo=
github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

28
website/docs/setup/maintenance.md
View File

@ -4,37 +4,41 @@ description: "Repository maintenance."
# Repository maintenance
Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user.
Repository maintenance helps optimize the Corso repository as backups are created and possibly deleted by the user.
Maintenance can also free up space by removing data no longer referenced by any backups from the repository.
It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe
to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository
It's safe to run maintenance concurrently with backup, restore, and backup deletion operations. However, it's not safe
to run maintenance operations concurrently on the same repository. Corso uses file locks and the idea of a repository
owner to try to detect concurrent maintenance operations.
## Repository owner
The repository owner is set to the user and hostname of the machine that runs maintenance on the repo the first time.
If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user`
If the user and hostname of the machine running maintenance can change, use either the `--force` flag or the `--user`
and `--host` flags.
The `--force` flag updates the repository owner and runs maintenance.
The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation
but doesn't update repo owner info.
The `--user` and `--host` flags act as if the given user/hostname owns the repository for the maintenance operation but
doesn't update repo owner info.
*If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same
repository. Concurrent maintenance operations a repository may result in data loss.*
:::danger
If any of these flags are passed the user must make sure no concurrent maintenance operations run on the same
repository. Concurrent maintenance operations a repository may result in data loss.
:::
## Maintenance types
Corso allows for two different types of maintenance: `metadata` and `complete`.
Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data
in backups and removes unreferenced data from the repository.
Metadata maintenance runs quickly and optimizes indexing data. Complete maintenance takes more time but compacts data in
backups and removes unreferenced data from the repository.
As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a
backup may not result in a reduction of objects in the storage service Corso is backing up to.
As Corso allows concurrent backups during maintenance, running complete maintenance immediately after deleting a backup
may not result in a reduction of objects in the storage service Corso is backing up to.
Deletion of old objects in the storage service depends on both wall-clock time and running maintenance.

2
website/docs/setup/restore-options.md
View File

@ -1,4 +1,4 @@
# Restore Options
# Restore options
import CodeBlock from '@theme/CodeBlock';
import Tabs from '@theme/Tabs';

5
website/sidebars.js
View File

@ -42,14 +42,15 @@ const sidebars = {
items: [
{
type: 'category',
label: 'Setup',
label: 'Setup and maintenance',
link: {
slug: 'cli/setup',
description: 'Documentation for commonly-used Corso setup CLI commands',
description: 'Documentation for Corso setup and maintenance commands',
},
items: [
'cli/corso-repo-init-s3',
'cli/corso-repo-connect-s3',
'cli/corso-repo-maintenance',
'cli/corso-env']
},
{