diff --git a/src/go.mod b/src/go.mod index 9e5c73ac4..e51d46bc5 100644 --- a/src/go.mod +++ b/src/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836 + github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0 github.com/armon/go-metrics v0.4.0 github.com/aws/aws-sdk-go v1.44.220 github.com/aws/aws-xray-sdk-go v1.8.1 diff --git a/src/go.sum b/src/go.sum index 138a3768b..4dc26b435 100644 --- a/src/go.sum +++ b/src/go.sum @@ -53,8 +53,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836 h1:239Dcnoe7y4kLeWS6XbdtvFwYOKT9Q28wqSZpwwqtbY= -github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8= +github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0 h1:2Fv5zc02wURwUv3Gjo3oqGybSj5tNaXyNIijlrR8SI0= +github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index 3391b96f4..87a0842b8 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -279,7 +279,7 @@ func createCollections( foldersComplete, closer := observe.MessageWithCompletion( ctx, - observe.Bulletf("%s", observe.Safe(qp.Category.String()))) + observe.Bulletf("%s", qp.Category)) defer closer() defer close(foldersComplete) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 1b935cfca..3a0d9c1a8 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -186,7 +186,8 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { colProgress, closer = observe.CollectionProgress( ctx, col.fullPath.Category().String(), - observe.PII(col.fullPath.Folder(false))) + // TODO(keepers): conceal compliance in path, drop Hide() + clues.Hide(col.fullPath.Folder(false))) go closer() diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 3a0cdd68e..c72b1cbf0 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -390,7 +390,7 @@ func restoreCollection( colProgress, closer := observe.CollectionProgress( ctx, category.String(), - observe.PII(directory.Folder(false))) + clues.Hide(directory.Folder(false))) defer closer() defer close(colProgress) diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index 25df599fc..ba8b715fa 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -419,7 +419,8 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { folderProgress, colCloser := observe.ProgressWithCount( ctx, observe.ItemQueueMsg, - observe.PII(queuedPath), + // TODO(keepers): conceal compliance in path, drop Hide() + clues.Hide(queuedPath), int64(len(oc.driveItems))) defer colCloser() defer close(folderProgress) @@ -517,7 +518,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { ctx, itemData, observe.ItemBackupMsg, - observe.PII(itemID+dataSuffix), + clues.Hide(itemID+dataSuffix), itemSize) go closer() @@ -533,8 +534,11 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) { progReader, closer := observe.ItemProgress( - ctx, itemMeta, observe.ItemBackupMsg, - observe.PII(metaFileName+metaSuffix), int64(itemMetaSize)) + ctx, + itemMeta, + observe.ItemBackupMsg, + clues.Hide(metaFileName+metaSuffix), + int64(itemMetaSize)) go closer() return progReader, nil }) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 70ad1be57..4ab890552 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -432,7 +432,7 @@ func (c *Collections) Get( } } - observe.Message(ctx, observe.Safe(fmt.Sprintf("Discovered %d items to backup", c.NumItems))) + observe.Message(ctx, fmt.Sprintf("Discovered %d items to backup", c.NumItems)) // Add an extra for the metadata collection. collections := []data.BackupCollection{} diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index dc1c870a3..3505f6c2b 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -598,7 +598,12 @@ func restoreData( } iReader := itemData.ToReader() - progReader, closer := observe.ItemProgress(ctx, iReader, observe.ItemRestoreMsg, observe.PII(itemName), ss.Size()) + progReader, closer := observe.ItemProgress( + ctx, + iReader, + observe.ItemRestoreMsg, + clues.Hide(itemName), + ss.Size()) go closer() diff --git a/src/internal/connector/sharepoint/collection.go b/src/internal/connector/sharepoint/collection.go index 1db542e9b..0350601e7 100644 --- a/src/internal/connector/sharepoint/collection.go +++ b/src/internal/connector/sharepoint/collection.go @@ -186,7 +186,8 @@ func (sc *Collection) runPopulate(ctx context.Context, errs *fault.Bus) (support colProgress, closer := observe.CollectionProgress( ctx, sc.fullPath.Category().String(), - observe.PII(sc.fullPath.Folder(false))) + // TODO(keepers): conceal compliance in path, drop Hide() + clues.Hide(sc.fullPath.Folder(false))) go closer() defer func() { diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index 02da46e3b..4c3895bc3 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -56,7 +56,7 @@ func DataCollections( foldersComplete, closer := observe.MessageWithCompletion( ctx, - observe.Bulletf("%s", observe.Safe(scope.Category().PathType().String()))) + observe.Bulletf("%s", scope.Category().PathType())) defer closer() defer close(foldersComplete) diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index 50e7411b1..67db8b7a2 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/alcionai/clues" "github.com/dustin/go-humanize" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -23,11 +24,6 @@ const ( progressBarWidth = 32 ) -// styling -const bullet = "∙" - -const Bullet = Safe(bullet) - var ( wg sync.WaitGroup // TODO: Revisit this being a global nd make it a parameter to the progress methods @@ -143,19 +139,19 @@ const ( // Progress Updates // Message is used to display a progress message -func Message(ctx context.Context, msgs ...cleanable) { - var ( - cleaned = make([]string, len(msgs)) - msg = make([]string, len(msgs)) - ) +func Message(ctx context.Context, msgs ...any) { + plainSl := make([]string, 0, len(msgs)) + loggableSl := make([]string, 0, len(msgs)) - for i := range msgs { - cleaned[i] = msgs[i].clean() - msg[i] = msgs[i].String() + for _, m := range msgs { + plainSl = append(plainSl, plainString(m)) + loggableSl = append(loggableSl, fmt.Sprintf("%v", m)) } - logger.Ctx(ctx).Info(strings.Join(cleaned, " ")) - message := strings.Join(msg, " ") + plain := strings.Join(plainSl, " ") + loggable := strings.Join(loggableSl, " ") + + logger.Ctx(ctx).Info(loggable) if cfg.hidden() { return @@ -167,9 +163,9 @@ func Message(ctx context.Context, msgs ...cleanable) { -1, mpb.NopStyle(), mpb.PrependDecorators(decor.Name( - message, + plain, decor.WC{ - W: len(message) + 1, + W: len(plain) + 1, C: decor.DidentRight, }))) @@ -183,19 +179,19 @@ func Message(ctx context.Context, msgs ...cleanable) { // that switches to "done" when the completion channel is signalled func MessageWithCompletion( ctx context.Context, - msg cleanable, + msg any, ) (chan<- struct{}, func()) { var ( - clean = msg.clean() - message = msg.String() - log = logger.Ctx(ctx) - ch = make(chan struct{}, 1) + plain = plainString(msg) + loggable = fmt.Sprintf("%v", msg) + log = logger.Ctx(ctx) + ch = make(chan struct{}, 1) ) - log.Info(clean) + log.Info(loggable) if cfg.hidden() { - return ch, func() { log.Info("done - " + clean) } + return ch, func() { log.Info("done - " + loggable) } } wg.Add(1) @@ -206,7 +202,7 @@ func MessageWithCompletion( -1, mpb.SpinnerStyle(frames...).PositionLeft(), mpb.PrependDecorators( - decor.Name(message+":"), + decor.Name(plain+":"), decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 8})), mpb.BarFillerOnComplete("done")) @@ -224,7 +220,7 @@ func MessageWithCompletion( }) wacb := waitAndCloseBar(bar, func() { - log.Info("done - " + clean) + log.Info("done - " + loggable) }) return ch, wacb @@ -241,11 +237,12 @@ func ItemProgress( ctx context.Context, rc io.ReadCloser, header string, - iname cleanable, + iname any, totalBytes int64, ) (io.ReadCloser, func()) { + plain := plainString(iname) log := logger.Ctx(ctx).With( - "item", iname.clean(), + "item", iname, "size", humanize.Bytes(uint64(totalBytes))) log.Debug(header) @@ -258,7 +255,7 @@ func ItemProgress( barOpts := []mpb.BarOption{ mpb.PrependDecorators( decor.Name(header, decor.WCSyncSpaceR), - decor.Name(iname.String(), decor.WCSyncSpaceR), + decor.Name(plain, decor.WCSyncSpaceR), decor.CountersKibiByte(" %.1f/%.1f ", decor.WC{W: 8}), decor.NewPercentage("%d ", decor.WC{W: 4})), } @@ -284,20 +281,21 @@ func ItemProgress( func ProgressWithCount( ctx context.Context, header string, - message cleanable, + msg any, count int64, ) (chan<- struct{}, func()) { var ( - log = logger.Ctx(ctx) - lmsg = fmt.Sprintf("%s %s - %d", header, message.clean(), count) - ch = make(chan struct{}) + plain = plainString(msg) + loggable = fmt.Sprintf("%s %v - %d", header, msg, count) + log = logger.Ctx(ctx) + ch = make(chan struct{}) ) - log.Info(lmsg) + log.Info(loggable) if cfg.hidden() { go listen(ctx, ch, nop, nop) - return ch, func() { log.Info("done - " + lmsg) } + return ch, func() { log.Info("done - " + loggable) } } wg.Add(1) @@ -305,7 +303,7 @@ func ProgressWithCount( barOpts := []mpb.BarOption{ mpb.PrependDecorators( decor.Name(header, decor.WCSyncSpaceR), - decor.Name(message.String()), + decor.Name(plain), decor.Counters(0, " %d/%d ")), } @@ -322,7 +320,7 @@ func ProgressWithCount( bar.Increment) wacb := waitAndCloseBar(bar, func() { - log.Info("done - " + lmsg) + log.Info("done - " + loggable) }) return ch, wacb @@ -366,14 +364,15 @@ func makeSpinFrames(barWidth int) { func CollectionProgress( ctx context.Context, category string, - dirName cleanable, + dirName any, ) (chan<- struct{}, func()) { var ( counted int + plain = plainString(dirName) ch = make(chan struct{}) log = logger.Ctx(ctx).With( "category", category, - "dir", dirName.clean()) + "dir", dirName) message = "Collecting Directory" ) @@ -387,7 +386,7 @@ func CollectionProgress( } } - if cfg.hidden() || len(dirName.String()) == 0 { + if cfg.hidden() || len(plain) == 0 { go listen(ctx, ch, nop, incCount) return ch, func() { log.Infow("done - "+message, "count", counted) } } @@ -398,7 +397,7 @@ func CollectionProgress( mpb.PrependDecorators(decor.Name(string(category))), mpb.AppendDecorators( decor.CurrentNoUnit("%d - ", decor.WCSyncSpace), - decor.Name(dirName.String()), + decor.Name(plain), ), mpb.BarFillerOnComplete(spinFrames[0]), } @@ -466,62 +465,45 @@ func listen(ctx context.Context, ch <-chan struct{}, onEnd, onInc func()) { } // --------------------------------------------------------------------------- -// PII redaction +// Styling // --------------------------------------------------------------------------- -type cleanable interface { - clean() string - String() string -} +const Bullet = "∙" -type PII string - -func (p PII) clean() string { - return "***" -} - -func (p PII) String() string { - return string(p) -} - -type Safe string - -func (s Safe) clean() string { - return string(s) -} - -func (s Safe) String() string { - return string(s) -} - -type bulletPII struct { +type bulletf struct { tmpl string - vars []cleanable + vs []any } -func Bulletf(template string, vs ...cleanable) bulletPII { - return bulletPII{ - tmpl: "∙ " + template, - vars: vs, - } +func Bulletf(template string, vs ...any) bulletf { + return bulletf{template, vs} } -func (b bulletPII) clean() string { - vs := make([]any, 0, len(b.vars)) - - for _, v := range b.vars { - vs = append(vs, v.clean()) +func (b bulletf) PlainString() string { + ps := make([]any, 0, len(b.vs)) + for _, v := range b.vs { + ps = append(ps, plainString(v)) } - return fmt.Sprintf(b.tmpl, vs...) + return fmt.Sprintf("∙ "+b.tmpl, ps...) } -func (b bulletPII) String() string { - vs := make([]any, 0, len(b.vars)) +func (b bulletf) String() string { + return fmt.Sprintf("∙ "+b.tmpl, b.vs...) +} - for _, v := range b.vars { - vs = append(vs, v.String()) +// plainString attempts to cast v to a PlainStringer +// interface, and retrieve the un-altered value. If +// v is not compliant with PlainStringer, returns the +// %v fmt of v. +// +// This should only be used to display the value in the +// observe progress bar. Logged values should only use +// the fmt %v to ensure Concealers hide PII. +func plainString(v any) string { + if ps, ok := v.(clues.PlainStringer); ok { + return ps.PlainString() } - return fmt.Sprintf(b.tmpl, vs...) + return fmt.Sprintf("%v", v) } diff --git a/src/internal/observe/observe_test.go b/src/internal/observe/observe_test.go index 31c818e05..89361304e 100644 --- a/src/internal/observe/observe_test.go +++ b/src/internal/observe/observe_test.go @@ -29,9 +29,9 @@ func TestObserveProgressUnitSuite(t *testing.T) { } var ( - tst = Safe("test") - testcat = Safe("testcat") - testertons = Safe("testertons") + tst = "test" + testcat = "testcat" + testertons = "testertons" ) func (suite *ObserveProgressUnitSuite) TestItemProgress() { @@ -105,7 +105,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnCtxCancel SeedWriter(context.Background(), nil, nil) }() - progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons) + progCh, closer := CollectionProgress(ctx, testcat, testertons) require.NotNil(t, progCh) require.NotNil(t, closer) @@ -140,7 +140,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnChannelCl SeedWriter(context.Background(), nil, nil) }() - progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons) + progCh, closer := CollectionProgress(ctx, testcat, testertons) require.NotNil(t, progCh) require.NotNil(t, closer) @@ -172,7 +172,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgress() { message := "Test Message" - Message(ctx, Safe(message)) + Message(ctx, message) Complete() require.NotEmpty(suite.T(), recorder.String()) require.Contains(suite.T(), recorder.String(), message) @@ -193,7 +193,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCompletion() { message := "Test Message" - ch, closer := MessageWithCompletion(ctx, Safe(message)) + ch, closer := MessageWithCompletion(ctx, message) // Trigger completion ch <- struct{}{} @@ -223,7 +223,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithChannelClosed() { message := "Test Message" - ch, closer := MessageWithCompletion(ctx, Safe(message)) + ch, closer := MessageWithCompletion(ctx, message) // Close channel without completing close(ch) @@ -255,7 +255,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithContextCancelled() message := "Test Message" - _, closer := MessageWithCompletion(ctx, Safe(message)) + _, closer := MessageWithCompletion(ctx, message) // cancel context cancel() @@ -286,7 +286,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCount() { message := "Test Message" count := 3 - ch, closer := ProgressWithCount(ctx, header, Safe(message), int64(count)) + ch, closer := ProgressWithCount(ctx, header, message, int64(count)) for i := 0; i < count; i++ { ch <- struct{}{} @@ -319,7 +319,7 @@ func (suite *ObserveProgressUnitSuite) TestrogressWithCountChannelClosed() { message := "Test Message" count := 3 - ch, closer := ProgressWithCount(ctx, header, Safe(message), int64(count)) + ch, closer := ProgressWithCount(ctx, header, message, int64(count)) close(ch) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index c97d7a1ec..8139d04ad 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -165,7 +165,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { // Execution // ----- - observe.Message(ctx, observe.Safe("Backing Up"), observe.Bullet, observe.PII(op.ResourceOwner)) + observe.Message(ctx, "Backing Up", observe.Bullet, clues.Hide(op.ResourceOwner)) deets, err := op.do( ctx, @@ -332,7 +332,7 @@ func produceBackupDataCollections( ctrlOpts control.Options, errs *fault.Bus, ) ([]data.BackupCollection, map[string]map[string]struct{}, error) { - complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Discovering items to backup")) + complete, closer := observe.MessageWithCompletion(ctx, "Discovering items to backup") defer func() { complete <- struct{}{} close(complete) @@ -403,7 +403,7 @@ func consumeBackupCollections( isIncremental bool, errs *fault.Bus, ) (*kopia.BackupStats, *details.Builder, map[string]kopia.PrevRefs, error) { - complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Backing up data")) + complete, closer := observe.MessageWithCompletion(ctx, "Backing up data") defer func() { complete <- struct{}{} close(complete) diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 1dac69098..4e45aad4c 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -185,7 +185,7 @@ func (op *RestoreOperation) do( return nil, clues.Wrap(err, "getting backup and details") } - observe.Message(ctx, observe.Safe("Restoring"), observe.Bullet, observe.PII(bup.Selector.DiscreteOwner)) + observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) if err != nil { @@ -210,10 +210,10 @@ func (op *RestoreOperation) do( events.RestoreID: opStats.restoreID, }) - observe.Message(ctx, observe.Safe(fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID))) + observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID)) logger.Ctx(ctx).With("selectors", op.Selectors).Info("restoring selection") - kopiaComplete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Enumerating items in repository")) + kopiaComplete, closer := observe.MessageWithCompletion(ctx, "Enumerating items in repository") defer closer() defer close(kopiaComplete) @@ -317,7 +317,7 @@ func consumeRestoreCollections( dcs []data.RestoreCollection, errs *fault.Bus, ) (*details.Details, error) { - complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Restoring data")) + complete, closer := observe.MessageWithCompletion(ctx, "Restoring data") defer func() { complete <- struct{}{} close(complete) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 10a910bf3..45eef688a 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -188,7 +188,7 @@ func Connect( // their output getting clobbered (#1720) defer observe.Complete() - complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Connecting to repository")) + complete, closer := observe.MessageWithCompletion(ctx, "Connecting to repository") defer closer() defer close(complete) @@ -570,7 +570,7 @@ func connectToM365( acct account.Account, errs *fault.Bus, ) (*connector.GraphConnector, error) { - complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Connecting to M365")) + complete, closer := observe.MessageWithCompletion(ctx, "Connecting to M365") defer func() { complete <- struct{}{} close(complete)