From ef5178668f3a20c112f5093579d3cf4f38cec5bd Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Tue, 21 Mar 2023 09:21:16 +0530 Subject: [PATCH] Log user name when enumerating drive items (#2872) This mimics the behavior for exchange where we print out which user and what content is currently being enumerated. While this might not be useful when we are only backing up a single user, it can come in handy when we run backups for multiple users. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * fixes https://github.com/alcionai/corso/issues/2871 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 1 + .../connector/exchange/data_collections.go | 7 ++--- .../exchange/exchange_data_collection.go | 1 - .../connector/exchange/service_restore.go | 1 - .../connector/onedrive/collections.go | 4 +++ .../connector/sharepoint/collection.go | 1 - .../connector/sharepoint/data_collections.go | 7 ++--- src/internal/observe/display.go | 7 ----- src/internal/observe/observe.go | 28 +++++++++++++------ src/internal/observe/observe_test.go | 4 +-- src/internal/operations/backup.go | 2 ++ src/internal/operations/restore.go | 2 ++ 12 files changed, 37 insertions(+), 28 deletions(-) delete mode 100644 src/internal/observe/display.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 905f44514..17ac2a3dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - OneDrive item downloads that return 404 during backup (normally due to external deletion while Corso processes) are now skipped instead of quietly dropped. These items will appear in the skipped list alongside other skipped cases such as malware detection. - Listing a single backup by id will also list the skipped and failed items that occurred during the backup. These can be filtered out with the flags `--failed-items hide`, `--skipped-items hide`, and `--recovered-errors hide`. - Enable incremental backups for OneDrive if permissions aren't being backed up. +- Show progressbar while files for user are enumerated - Hidden flag to control parallelism for fetching Exchange items (`--fetch-parallelism`). May help reduce `ApplicationThrottled` errors but will slow down backup. ### Fixed diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index 1eb29fce6..24037db7b 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -259,10 +259,9 @@ func createCollections( Credentials: creds, } - foldersComplete, closer := observe.MessageWithCompletion(ctx, observe.Bulletf( - "%s - %s", - observe.Safe(qp.Category.String()), - observe.PII(user))) + foldersComplete, closer := observe.MessageWithCompletion( + ctx, + observe.Bulletf("%s", observe.Safe(qp.Category.String()))) defer closer() defer close(foldersComplete) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 9aeba1d2f..1b935cfca 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -186,7 +186,6 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { colProgress, closer = observe.CollectionProgress( ctx, col.fullPath.Category().String(), - observe.PII(user), observe.PII(col.fullPath.Folder(false))) go closer() diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 8f49a01b6..d2e778bed 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -391,7 +391,6 @@ func restoreCollection( colProgress, closer := observe.CollectionProgress( ctx, category.String(), - observe.PII(user), observe.PII(directory.Folder(false))) defer closer() defer close(colProgress) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 044b3f0ad..e3cc6a8d8 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -263,6 +263,10 @@ func (c *Collections) Get( return nil, nil, err } + driveComplete, closer := observe.MessageWithCompletion(ctx, observe.Bulletf("files")) + defer closer() + defer close(driveComplete) + // Enumerate drives for the specified resourceOwner pager, err := c.drivePagerFunc(c.source, c.service, c.resourceOwner, nil) if err != nil { diff --git a/src/internal/connector/sharepoint/collection.go b/src/internal/connector/sharepoint/collection.go index 53e0c9f5e..60a897739 100644 --- a/src/internal/connector/sharepoint/collection.go +++ b/src/internal/connector/sharepoint/collection.go @@ -186,7 +186,6 @@ func (sc *Collection) runPopulate(ctx context.Context, errs *fault.Bus) (support colProgress, closer := observe.CollectionProgress( ctx, sc.fullPath.Category().String(), - observe.Safe("name"), observe.PII(sc.fullPath.Folder(false))) go closer() diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index 35afe01e8..83fa9ede3 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -54,10 +54,9 @@ func DataCollections( break } - foldersComplete, closer := observe.MessageWithCompletion(ctx, observe.Bulletf( - "%s - %s", - observe.Safe(scope.Category().PathType().String()), - observe.PII(site))) + foldersComplete, closer := observe.MessageWithCompletion( + ctx, + observe.Bulletf("%s", observe.Safe(scope.Category().PathType().String()))) defer closer() defer close(foldersComplete) diff --git a/src/internal/observe/display.go b/src/internal/observe/display.go deleted file mode 100644 index 2cf6d9d6f..000000000 --- a/src/internal/observe/display.go +++ /dev/null @@ -1,7 +0,0 @@ -package observe - -// Display holds display-only configuration. Primarily for passing along -// aliased values when we have a pair of more-and-less user friendly -type Display struct { - ResourceOwner string -} diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index 3ff76627c..d49713e12 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "strings" "sync" "github.com/dustin/go-humanize" @@ -23,7 +24,9 @@ const ( ) // styling -const Bullet = "∙" +const bullet = "∙" + +const Bullet = Safe(bullet) var ( wg sync.WaitGroup @@ -141,9 +144,19 @@ const ( // Progress Updates // Message is used to display a progress message -func Message(ctx context.Context, msg cleanable) { - logger.Ctx(ctx).Info(msg.clean()) - message := msg.String() +func Message(ctx context.Context, msgs ...cleanable) { + var ( + cleaned = make([]string, len(msgs)) + msg = make([]string, len(msgs)) + ) + + for i := range msgs { + cleaned[i] = msgs[i].clean() + msg[i] = msgs[i].String() + } + + logger.Ctx(ctx).Info(strings.Join(cleaned, " ")) + message := strings.Join(msg, " ") if cfg.hidden() { return @@ -356,13 +369,12 @@ func makeSpinFrames(barWidth int) { func CollectionProgress( ctx context.Context, category string, - user, dirName cleanable, + dirName cleanable, ) (chan<- struct{}, func()) { var ( counted int ch = make(chan struct{}) log = logger.Ctx(ctx).With( - "user", user.clean(), "category", category, "dir", dirName.clean()) message = "Collecting Directory" @@ -378,7 +390,7 @@ func CollectionProgress( } } - if cfg.hidden() || len(user.String()) == 0 || len(dirName.String()) == 0 { + if cfg.hidden() || len(dirName.String()) == 0 { go listen(ctx, ch, nop, incCount) return ch, func() { log.Infow("done - "+message, "count", counted) } } @@ -389,7 +401,7 @@ func CollectionProgress( mpb.PrependDecorators(decor.Name(string(category))), mpb.AppendDecorators( decor.CurrentNoUnit("%d - ", decor.WCSyncSpace), - decor.Name(fmt.Sprintf("%s - %s", user, dirName)), + decor.Name(dirName.String()), ), mpb.BarFillerOnComplete(spinFrames[0]), } diff --git a/src/internal/observe/observe_test.go b/src/internal/observe/observe_test.go index 023fe8aef..9db84a348 100644 --- a/src/internal/observe/observe_test.go +++ b/src/internal/observe/observe_test.go @@ -105,7 +105,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnCtxCancel SeedWriter(context.Background(), nil, nil) }() - progCh, closer := CollectionProgress(ctx, "test", testcat, testertons) + progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons) require.NotNil(t, progCh) require.NotNil(t, closer) @@ -140,7 +140,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnChannelCl SeedWriter(context.Background(), nil, nil) }() - progCh, closer := CollectionProgress(ctx, "test", testcat, testertons) + progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons) require.NotNil(t, progCh) require.NotNil(t, closer) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index c3d2e73c2..2b885ccf2 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -148,6 +148,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { // Execution // ----- + observe.Message(ctx, observe.Safe("Backing Up"), observe.Bullet, observe.PII(op.ResourceOwner)) + deets, err := op.do( ctx, &opStats, diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index c956d7514..45c7a7300 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -186,6 +186,8 @@ func (op *RestoreOperation) do( return nil, errors.Wrap(err, "getting backup and details") } + observe.Message(ctx, observe.Safe("Restoring"), observe.Bullet, observe.PII(bup.Selector.DiscreteOwner)) + paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) if err != nil { return nil, errors.Wrap(err, "formatting paths from details")