From 68256cf59f31cc4c1cabe0347eb5f85d18d4790f Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 16 Nov 2023 18:13:27 +0530 Subject: [PATCH] Change printing of backup stats to single line (#4526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we were printing a table for a single item which, for one was unnecessary, but this also make the output of a backup more compact. Before: ``` Logging to file: /home/meain/.cache/corso/logs/2023-10-20T10-21-23Z.log Connecting to M365 done Connecting to repository 1s done Backing Up ∙ Teams Testing (meain) Discovering items to backup 27s done Libraries (TeamsTestingmeain) done (found 0 items) Libraries (TeamsTestingmeain-Shared0) done (found 0 items) Libraries (TeamsTestingmeain-Private2) done (found 0 items) Messages 7s done (found 6 channels) Backing up data 7s done Backup complete ∙ 7b40dd40-f808-4d57-8e39-b4553e48dc5d ID Bytes Uploaded Items Uploaded Items Skipped Errors 7b40dd40-f808-4d57-8e39-b4553e48dc5d 0 B 0 0 0 Completed Backups: ID Started At Duration Status Resource Owner 7b40dd40-f808-4d57-8e39-b4553e48dc5d 2023-10-20T10:21:32Z 36.632747912s Completed Teams Testing (meain) ``` After: ``` Connecting to M365 done Connecting to repository 1s done Backing Up ∙ Teams Testing (meain) Discovering items to backup 31s done Libraries (TeamsTestingmeain) done (found 0 items) Libraries (TeamsTestingmeain-Shared0) done (found 0 items) Libraries (TeamsTestingmeain-Private2) done (found 0 items) Messages 9s done (found 6 channels) Backing up data 7s done Backup complete ∙ ffb2f619-1cb7-4a11-b3e2-7300aa513c6a Bytes Uploaded: 0 B | Items Uploaded: 0 | Items Skipped: 0 | Errors: 0 Completed Backups: ID ID Started At Duration Status Resource Owner ffb2f619-1cb7-4a11-b3e2-7300aa513c6a 2023-10-20T10:23:35Z 40.096203016s Completed Teams Testing (meain) ``` --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 3 +- src/cli/backup/backup.go | 5 +-- src/cli/help/env.go | 5 ++- src/cli/print/print.go | 51 +++++++++++++++++++++-- src/pkg/backup/backup.go | 56 ++++++++++++++++++++------ src/pkg/backup/backup_test.go | 31 ++++++++++---- src/pkg/backup/details/details_test.go | 9 ++++- src/pkg/backup/details/entry.go | 40 ++++++++++-------- src/pkg/fault/alert.go | 5 ++- src/pkg/fault/alert_test.go | 7 +++- src/pkg/fault/fault.go | 5 ++- src/pkg/fault/item.go | 5 ++- src/pkg/fault/item_test.go | 7 +++- src/pkg/fault/skipped.go | 5 ++- src/pkg/fault/skipped_test.go | 7 +++- 15 files changed, 180 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c57ba59..83f3d3ee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added export support for emails in exchange backups as `.eml` files +- More colorful and informational cli output ### Changed - Change file extension of messages export to json to match the content ### Fixed - Handle OneDrive folders being deleted and recreated midway through a backup -- Automatically re-run a full delta query on incrmental if the prior backup is found to have malformed prior-state information. +- Automatically re-run a full delta query on incremental if the prior backup is found to have malformed prior-state information. - Retry drive item permission downloads during long-running backups after the jwt token expires and refreshes. ## [v0.15.0] (beta) - 2023-10-31 diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 29631c130..9debea9d6 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -247,7 +247,7 @@ func genericCreateCommand( } if len(bups) > 0 { - Info(ctx, "Completed Backups:") + Info(ctx, "\nCompleted Backups:") backup.PrintAll(ctx, bups) } @@ -420,6 +420,5 @@ func printBackupStats(ctx context.Context, r repository.Repositoryer, bid string logger.CtxErr(ctx, err).Error("finding backup immediately after backup operation completion") } - b.ToPrintable().Stats.Print(ctx) - Info(ctx, " ") + b.ToPrintable().Stats.PrintProperties(ctx) } diff --git a/src/cli/help/env.go b/src/cli/help/env.go index 674db14bb..b8fd92e77 100644 --- a/src/cli/help/env.go +++ b/src/cli/help/env.go @@ -46,11 +46,12 @@ func (ev envVar) MinimumPrintable() any { return ev } -func (ev envVar) Headers() []string { +func (ev envVar) Headers(bool) []string { + // NOTE: skipID does not make sense in this context return []string{ev.category, " "} } -func (ev envVar) Values() []string { +func (ev envVar) Values(bool) []string { return []string{ev.name, ev.description} } diff --git a/src/cli/print/print.go b/src/cli/print/print.go index b17b57551..5204fd294 100644 --- a/src/cli/print/print.go +++ b/src/cli/print/print.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "github.com/spf13/cobra" "github.com/tidwall/pretty" @@ -178,11 +179,11 @@ func outf(ctx context.Context, w io.Writer, t string, s ...any) { type Printable interface { minimumPrintabler // should list the property names of the values surfaced in Values() - Headers() []string + Headers(skipID bool) []string // list of values for tabular or csv formatting // if the backing data is nil or otherwise missing, // values should provide an empty string as opposed to skipping entries - Values() []string + Values(skipID bool) []string } type minimumPrintabler interface { @@ -206,6 +207,23 @@ func printItem(w io.Writer, p Printable) { outputTable(w, []Printable{p}) } +// ItemProperties prints the printable either as in a single line or a json +// The difference between this and Item is that this one does not print the ID +func ItemProperties(ctx context.Context, p Printable) { + printItemProperties(getRootCmd(ctx).OutOrStdout(), p) +} + +// print prints the printable items, +// according to the caller's requested format. +func printItemProperties(w io.Writer, p Printable) { + if outputAsJSON || outputAsJSONDebug { + outputJSON(w, p, outputAsJSONDebug) + return + } + + outputOneLine(w, []Printable{p}) +} + // All prints the slice of printable items, // according to the caller's requested format. func All(ctx context.Context, ps ...Printable) { @@ -240,12 +258,12 @@ func Table(ctx context.Context, ps []Printable) { // output to stdout the list of printable structs in a table func outputTable(w io.Writer, ps []Printable) { t := table.Table{ - Headers: ps[0].Headers(), + Headers: ps[0].Headers(false), Rows: [][]string{}, } for _, p := range ps { - t.Rows = append(t.Rows, p.Values()) + t.Rows = append(t.Rows, p.Values(false)) } _ = t.WriteTable( @@ -309,3 +327,28 @@ func printPrettyJSON(w io.Writer, a any) { fmt.Fprintln(w, string(pretty.Pretty(bs))) } + +// ------------------------------------------------------------------------------------------- +// One line +// ------------------------------------------------------------------------------------------- + +// Output in the following format: +// Bytes Uploaded: 401 kB | Items Uploaded: 59 | Items Skipped: 0 | Errors: 0 +func outputOneLine(w io.Writer, ps []Printable) { + headers := ps[0].Headers(true) + rows := [][]string{} + + for _, p := range ps { + rows = append(rows, p.Values(true)) + } + + printables := []string{} + + for _, row := range rows { + for i, col := range row { + printables = append(printables, fmt.Sprintf("%s: %s", headers[i], col)) + } + } + + fmt.Fprintln(w, strings.Join(printables, " | ")) +} diff --git a/src/pkg/backup/backup.go b/src/pkg/backup/backup.go index 4a6674d7b..146df9834 100644 --- a/src/pkg/backup/backup.go +++ b/src/pkg/backup/backup.go @@ -173,6 +173,12 @@ func (b Backup) Print(ctx context.Context) { print.Item(ctx, b) } +// PrintProperties writes the Backup to StdOut, in the format requested by the caller. +// Unlike Print, it skips the ID of the Backup +func (b Backup) PrintProperties(ctx context.Context) { + print.ItemProperties(ctx, b) +} + // PrintAll writes the slice of Backups to StdOut, in the format requested by the caller. func PrintAll(ctx context.Context, bs []*Backup) { if len(bs) == 0 { @@ -218,19 +224,24 @@ func (b Backup) MinimumPrintable() any { // Headers returns the human-readable names of properties in a Backup // for printing out to a terminal in a columnar display. -func (b Backup) Headers() []string { - return []string{ - "ID", +func (b Backup) Headers(skipID bool) []string { + headers := []string{ "Started At", "Duration", "Status", "Resource Owner", } + + if skipID { + return headers + } + + return append([]string{"ID"}, headers...) } // Values returns the values matching the Headers list for printing // out to a terminal in a columnar display. -func (b Backup) Values() []string { +func (b Backup) Values(skipID bool) []string { var ( status = b.Status errCount = b.ErrorCount @@ -281,13 +292,18 @@ func (b Backup) Values() []string { bs := b.toStats() - return []string{ - string(b.ID), + values := []string{ dttm.FormatToTabularDisplay(b.StartedAt), bs.EndedAt.Sub(bs.StartedAt).String(), status, name, } + + if skipID { + return values + } + + return append([]string{string(b.ID)}, values...) } // ----- print backup stats @@ -326,6 +342,12 @@ func (bs backupStats) Print(ctx context.Context) { print.Item(ctx, bs) } +// PrintProperties writes the Backup to StdOut, in the format requested by the caller. +// Unlike Print, it skips the ID of backupStats +func (bs backupStats) PrintProperties(ctx context.Context) { + print.ItemProperties(ctx, bs) +} + // MinimumPrintable reduces the Backup to its minimally printable details. func (bs backupStats) MinimumPrintable() any { return bs @@ -333,24 +355,34 @@ func (bs backupStats) MinimumPrintable() any { // Headers returns the human-readable names of properties in a Backup // for printing out to a terminal in a columnar display. -func (bs backupStats) Headers() []string { - return []string{ - "ID", +func (bs backupStats) Headers(skipID bool) []string { + headers := []string{ "Bytes Uploaded", "Items Uploaded", "Items Skipped", "Errors", } + + if skipID { + return headers + } + + return append([]string{"ID"}, headers...) } // Values returns the values matching the Headers list for printing // out to a terminal in a columnar display. -func (bs backupStats) Values() []string { - return []string{ - bs.ID, +func (bs backupStats) Values(skipID bool) []string { + values := []string{ humanize.Bytes(uint64(bs.BytesUploaded)), strconv.Itoa(bs.ItemsWritten), strconv.Itoa(bs.ItemsSkipped), strconv.Itoa(bs.ErrorCount), } + + if skipID { + return values + } + + return append([]string{bs.ID}, values...) } diff --git a/src/pkg/backup/backup_test.go b/src/pkg/backup/backup_test.go index f9de49e70..01a1bcf3e 100644 --- a/src/pkg/backup/backup_test.go +++ b/src/pkg/backup/backup_test.go @@ -173,11 +173,17 @@ func (suite *BackupUnitSuite) TestBackup_HeadersValues() { b.StartAndEndTime.CompletedAt = later // single skipped malware - hs := b.Headers() + hs := b.Headers(false) assert.Equal(t, expectHs, hs) - vs := b.Values() + vs := b.Values(false) assert.Equal(t, expectVs, vs) + + hs = b.Headers(true) + assert.Equal(t, expectHs[1:], hs) + + vs = b.Values(true) + assert.Equal(t, expectVs[1:], vs) } func (suite *BackupUnitSuite) TestBackup_HeadersValues_onlyResourceOwners() { @@ -209,11 +215,17 @@ func (suite *BackupUnitSuite) TestBackup_HeadersValues_onlyResourceOwners() { b.StartAndEndTime.CompletedAt = later // single skipped malware - hs := b.Headers() + hs := b.Headers(false) assert.Equal(t, expectHs, hs) - vs := b.Values() + vs := b.Values(false) assert.Equal(t, expectVs, vs) + + hs = b.Headers(true) + assert.Equal(t, expectHs[1:], hs) + + vs = b.Values(true) + assert.Equal(t, expectVs[1:], vs) } func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() { @@ -297,8 +309,11 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() { } for _, test := range table { suite.Run(test.name, func() { - result := test.bup.Values() + result := test.bup.Values(false) assert.Equal(suite.T(), test.expect, result[3], "status value") + + result = test.bup.Values(true) + assert.Equal(suite.T(), test.expect, result[2], "status value") }) } } @@ -355,7 +370,8 @@ func (suite *BackupUnitSuite) TestStats_headersValues() { "Errors", } - assert.Equal(t, expectHeaders, s.Headers()) + assert.Equal(t, expectHeaders, s.Headers(false)) + assert.Equal(t, expectHeaders[1:], s.Headers(true)) expectValues := []string{ "id", @@ -365,5 +381,6 @@ func (suite *BackupUnitSuite) TestStats_headersValues() { strconv.Itoa(b.ErrorCount), } - assert.Equal(t, expectValues, s.Values()) + assert.Equal(t, expectValues, s.Values(false)) + assert.Equal(t, expectValues[1:], s.Values(true)) } diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 591510f70..73a5d1672 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -174,10 +174,15 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { suite.Run(test.name, func() { t := suite.T() - hs := test.entry.Headers() + hs := test.entry.Headers(false) assert.Equal(t, test.expectHs, hs) - vs := test.entry.Values() + vs := test.entry.Values(false) assert.Equal(t, test.expectVs, vs) + + hs = test.entry.Headers(true) + assert.Equal(t, test.expectHs[1:], hs) + vs = test.entry.Values(true) + assert.Equal(t, test.expectVs[1:], vs) }) } } diff --git a/src/pkg/backup/details/entry.go b/src/pkg/backup/details/entry.go index 07530c620..0d040eb67 100644 --- a/src/pkg/backup/details/entry.go +++ b/src/pkg/backup/details/entry.go @@ -140,55 +140,63 @@ func (de Entry) MinimumPrintable() any { // Headers returns the human-readable names of properties in a DetailsEntry // for printing out to a terminal in a columnar display. -func (de Entry) Headers() []string { - hs := []string{"ID"} +func (de Entry) Headers(skipID bool) []string { + hs := []string{} if de.ItemInfo.Folder != nil { - hs = append(hs, de.ItemInfo.Folder.Headers()...) + hs = de.ItemInfo.Folder.Headers() } if de.ItemInfo.Exchange != nil { - hs = append(hs, de.ItemInfo.Exchange.Headers()...) + hs = de.ItemInfo.Exchange.Headers() } if de.ItemInfo.SharePoint != nil { - hs = append(hs, de.ItemInfo.SharePoint.Headers()...) + hs = de.ItemInfo.SharePoint.Headers() } if de.ItemInfo.OneDrive != nil { - hs = append(hs, de.ItemInfo.OneDrive.Headers()...) + hs = de.ItemInfo.OneDrive.Headers() } if de.ItemInfo.Groups != nil { - hs = append(hs, de.ItemInfo.Groups.Headers()...) + hs = de.ItemInfo.Groups.Headers() } - return hs + if skipID { + return hs + } + + return append([]string{"ID"}, hs...) } // Values returns the values matching the Headers list. -func (de Entry) Values() []string { - vs := []string{de.ShortRef} +func (de Entry) Values(skipID bool) []string { + vs := []string{} if de.ItemInfo.Folder != nil { - vs = append(vs, de.ItemInfo.Folder.Values()...) + vs = de.ItemInfo.Folder.Values() } if de.ItemInfo.Exchange != nil { - vs = append(vs, de.ItemInfo.Exchange.Values()...) + vs = de.ItemInfo.Exchange.Values() } if de.ItemInfo.SharePoint != nil { - vs = append(vs, de.ItemInfo.SharePoint.Values()...) + vs = de.ItemInfo.SharePoint.Values() } if de.ItemInfo.OneDrive != nil { - vs = append(vs, de.ItemInfo.OneDrive.Values()...) + vs = de.ItemInfo.OneDrive.Values() } if de.ItemInfo.Groups != nil { - vs = append(vs, de.ItemInfo.Groups.Values()...) + vs = de.ItemInfo.Groups.Values() } - return vs + if skipID { + return vs + } + + return append([]string{de.ShortRef}, vs...) } diff --git a/src/pkg/fault/alert.go b/src/pkg/fault/alert.go index d7b207f8f..c5599d413 100644 --- a/src/pkg/fault/alert.go +++ b/src/pkg/fault/alert.go @@ -42,12 +42,13 @@ func (a Alert) MinimumPrintable() any { // Headers returns the human-readable names of properties of a skipped Item // for printing out to a terminal. -func (a Alert) Headers() []string { +func (a Alert) Headers(bool) []string { + // NOTE: skipID does not make sense in this context and is skipped return []string{"Action", "Message", "Container", "Name", "ID"} } // Values populates the printable values matching the Headers list. -func (a Alert) Values() []string { +func (a Alert) Values(bool) []string { var cn string acn, ok := a.Item.Additional[AddtlContainerName] diff --git a/src/pkg/fault/alert_test.go b/src/pkg/fault/alert_test.go index c45ec2e70..da5b4f55f 100644 --- a/src/pkg/fault/alert_test.go +++ b/src/pkg/fault/alert_test.go @@ -81,8 +81,11 @@ func (suite *AlertUnitSuite) TestAlert_HeadersValues() { suite.Run(test.name, func() { t := suite.T() - assert.Equal(t, []string{"Action", "Message", "Container", "Name", "ID"}, test.alert.Headers()) - assert.Equal(t, test.expect, test.alert.Values()) + assert.Equal(t, []string{"Action", "Message", "Container", "Name", "ID"}, test.alert.Headers(false)) + assert.Equal(t, test.expect, test.alert.Values(false)) + + assert.Equal(t, []string{"Action", "Message", "Container", "Name", "ID"}, test.alert.Headers(true)) + assert.Equal(t, test.expect, test.alert.Values(true)) }) } } diff --git a/src/pkg/fault/fault.go b/src/pkg/fault/fault.go index 2beb55f6f..ff326b4fb 100644 --- a/src/pkg/fault/fault.go +++ b/src/pkg/fault/fault.go @@ -491,11 +491,12 @@ func (pec printableErrCore) MinimumPrintable() any { return pec } -func (pec printableErrCore) Headers() []string { +func (pec printableErrCore) Headers(bool) []string { + // NOTE: skipID does not make sense in this context return []string{"Error"} } -func (pec printableErrCore) Values() []string { +func (pec printableErrCore) Values(bool) []string { if pec.ErrCore == nil { return []string{""} } diff --git a/src/pkg/fault/item.go b/src/pkg/fault/item.go index e43070ebe..d8f1b4c3e 100644 --- a/src/pkg/fault/item.go +++ b/src/pkg/fault/item.go @@ -103,12 +103,13 @@ func (i Item) MinimumPrintable() any { // Headers returns the human-readable names of properties of an Item // for printing out to a terminal. -func (i Item) Headers() []string { +func (i Item) Headers(bool) []string { + // NOTE: skipID does not make sense in this context return []string{"Action", "Type", "Name", "Container", "Cause"} } // Values populates the printable values matching the Headers list. -func (i Item) Values() []string { +func (i Item) Values(bool) []string { var cn string acn, ok := i.Additional[AddtlContainerName] diff --git a/src/pkg/fault/item_test.go b/src/pkg/fault/item_test.go index bdb2ca482..78d836bea 100644 --- a/src/pkg/fault/item_test.go +++ b/src/pkg/fault/item_test.go @@ -149,8 +149,11 @@ func (suite *ItemUnitSuite) TestItem_HeadersValues() { suite.Run(test.name, func() { t := suite.T() - assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.item.Headers()) - assert.Equal(t, test.expect, test.item.Values()) + assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.item.Headers(false)) + assert.Equal(t, test.expect, test.item.Values(false)) + + assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.item.Headers(true)) + assert.Equal(t, test.expect, test.item.Values(true)) }) } } diff --git a/src/pkg/fault/skipped.go b/src/pkg/fault/skipped.go index 6aead57b1..836f079c7 100644 --- a/src/pkg/fault/skipped.go +++ b/src/pkg/fault/skipped.go @@ -76,12 +76,13 @@ func (s Skipped) MinimumPrintable() any { // Headers returns the human-readable names of properties of a skipped Item // for printing out to a terminal. -func (s Skipped) Headers() []string { +func (s Skipped) Headers(bool) []string { + // NOTE: skipID does not make sense in this context return []string{"Action", "Type", "Name", "Container", "Cause"} } // Values populates the printable values matching the Headers list. -func (s Skipped) Values() []string { +func (s Skipped) Values(bool) []string { var cn string acn, ok := s.Item.Additional[AddtlContainerName] diff --git a/src/pkg/fault/skipped_test.go b/src/pkg/fault/skipped_test.go index 22d8cddf4..859a2fe76 100644 --- a/src/pkg/fault/skipped_test.go +++ b/src/pkg/fault/skipped_test.go @@ -120,8 +120,11 @@ func (suite *SkippedUnitSuite) TestSkipped_HeadersValues() { suite.Run(test.name, func() { t := suite.T() - assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.skip.Headers()) - assert.Equal(t, test.expect, test.skip.Values()) + assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.skip.Headers(false)) + assert.Equal(t, test.expect, test.skip.Values(false)) + + assert.Equal(t, []string{"Action", "Type", "Name", "Container", "Cause"}, test.skip.Headers(true)) + assert.Equal(t, test.expect, test.skip.Values(true)) }) } }