add clues.Hide to replace observe.PII (#2993)

The observe PII handlers are getting replaced by
the more standardized clues secrets.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #2024

#### Test Plan

- [x] 💪 Manual
- [x] 💚 E2E

src/go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
-	github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836
+	github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0
 	github.com/armon/go-metrics v0.4.0
 	github.com/aws/aws-sdk-go v1.44.220
 	github.com/aws/aws-xray-sdk-go v1.8.1

src/go.sum

@@ -53,8 +53,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
-github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836 h1:239Dcnoe7y4kLeWS6XbdtvFwYOKT9Q28wqSZpwwqtbY=
+github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0 h1:2Fv5zc02wURwUv3Gjo3oqGybSj5tNaXyNIijlrR8SI0=
-github.com/alcionai/clues v0.0.0-20230327232656-5b9b43a79836/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8=
+github.com/alcionai/clues v0.0.0-20230330224331-77c1b3be97e0/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

src/internal/connector/exchange/data_collections.go

@@ -279,7 +279,7 @@ func createCollections(
 
 	foldersComplete, closer := observe.MessageWithCompletion(
 		ctx,
-		observe.Bulletf("%s", observe.Safe(qp.Category.String())))
+		observe.Bulletf("%s", qp.Category))
 	defer closer()
 	defer close(foldersComplete)
 

src/internal/connector/exchange/exchange_data_collection.go

@@ -186,7 +186,8 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 		colProgress, closer = observe.CollectionProgress(
 			ctx,
 			col.fullPath.Category().String(),
-			observe.PII(col.fullPath.Folder(false)))
+			// TODO(keepers): conceal compliance in path, drop Hide()
+			clues.Hide(col.fullPath.Folder(false)))
 
 		go closer()
 

src/internal/connector/exchange/service_restore.go

@@ -390,7 +390,7 @@ func restoreCollection(
 	colProgress, closer := observe.CollectionProgress(
 		ctx,
 		category.String(),
-		observe.PII(directory.Folder(false)))
+		clues.Hide(directory.Folder(false)))
 	defer closer()
 	defer close(colProgress)
 

src/internal/connector/onedrive/collection.go

@@ -419,7 +419,8 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
 	folderProgress, colCloser := observe.ProgressWithCount(
 		ctx,
 		observe.ItemQueueMsg,
-		observe.PII(queuedPath),
+		// TODO(keepers): conceal compliance in path, drop Hide()
+		clues.Hide(queuedPath),
 		int64(len(oc.driveItems)))
 	defer colCloser()
 	defer close(folderProgress)
@@ -517,7 +518,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
 						ctx,
 						itemData,
 						observe.ItemBackupMsg,
-						observe.PII(itemID+dataSuffix),
+						clues.Hide(itemID+dataSuffix),
 						itemSize)
 					go closer()
 
@@ -533,8 +534,11 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
 
 			metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) {
 				progReader, closer := observe.ItemProgress(
-					ctx, itemMeta, observe.ItemBackupMsg,
+					ctx,
-					observe.PII(metaFileName+metaSuffix), int64(itemMetaSize))
+					itemMeta,
+					observe.ItemBackupMsg,
+					clues.Hide(metaFileName+metaSuffix),
+					int64(itemMetaSize))
 				go closer()
 				return progReader, nil
 			})

src/internal/connector/onedrive/collections.go

@@ -432,7 +432,7 @@ func (c *Collections) Get(
 		}
 	}
 
-	observe.Message(ctx, observe.Safe(fmt.Sprintf("Discovered %d items to backup", c.NumItems)))
+	observe.Message(ctx, fmt.Sprintf("Discovered %d items to backup", c.NumItems))
 
 	// Add an extra for the metadata collection.
 	collections := []data.BackupCollection{}

src/internal/connector/onedrive/restore.go

@@ -598,7 +598,12 @@ func restoreData(
 	}
 
 	iReader := itemData.ToReader()
-	progReader, closer := observe.ItemProgress(ctx, iReader, observe.ItemRestoreMsg, observe.PII(itemName), ss.Size())
+	progReader, closer := observe.ItemProgress(
+		ctx,
+		iReader,
+		observe.ItemRestoreMsg,
+		clues.Hide(itemName),
+		ss.Size())
 
 	go closer()
 

src/internal/connector/sharepoint/collection.go

@@ -186,7 +186,8 @@ func (sc *Collection) runPopulate(ctx context.Context, errs *fault.Bus) (support
 	colProgress, closer := observe.CollectionProgress(
 		ctx,
 		sc.fullPath.Category().String(),
-		observe.PII(sc.fullPath.Folder(false)))
+		// TODO(keepers): conceal compliance in path, drop Hide()
+		clues.Hide(sc.fullPath.Folder(false)))
 	go closer()
 
 	defer func() {

src/internal/connector/sharepoint/data_collections.go

@@ -56,7 +56,7 @@ func DataCollections(
 
 		foldersComplete, closer := observe.MessageWithCompletion(
 			ctx,
-			observe.Bulletf("%s", observe.Safe(scope.Category().PathType().String())))
+			observe.Bulletf("%s", scope.Category().PathType()))
 		defer closer()
 		defer close(foldersComplete)
 

src/internal/observe/observe.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/alcionai/clues"
 	"github.com/dustin/go-humanize"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -23,11 +24,6 @@ const (
 	progressBarWidth     = 32
 )
 
-// styling
-const bullet = "∙"
-
-const Bullet = Safe(bullet)
-
 var (
 	wg sync.WaitGroup
 	// TODO: Revisit this being a global nd make it a parameter to the progress methods
@@ -143,19 +139,19 @@ const (
 // Progress Updates
 
 // Message is used to display a progress message
-func Message(ctx context.Context, msgs ...cleanable) {
+func Message(ctx context.Context, msgs ...any) {
-	var (
+	plainSl := make([]string, 0, len(msgs))
-		cleaned = make([]string, len(msgs))
+	loggableSl := make([]string, 0, len(msgs))
-		msg     = make([]string, len(msgs))
-	)
 
-	for i := range msgs {
+	for _, m := range msgs {
-		cleaned[i] = msgs[i].clean()
+		plainSl = append(plainSl, plainString(m))
-		msg[i] = msgs[i].String()
+		loggableSl = append(loggableSl, fmt.Sprintf("%v", m))
 	}
 
-	logger.Ctx(ctx).Info(strings.Join(cleaned, " "))
+	plain := strings.Join(plainSl, " ")
-	message := strings.Join(msg, " ")
+	loggable := strings.Join(loggableSl, " ")
+
+	logger.Ctx(ctx).Info(loggable)
 
 	if cfg.hidden() {
 		return
@@ -167,9 +163,9 @@ func Message(ctx context.Context, msgs ...cleanable) {
 		-1,
 		mpb.NopStyle(),
 		mpb.PrependDecorators(decor.Name(
-			message,
+			plain,
 			decor.WC{
-				W: len(message) + 1,
+				W: len(plain) + 1,
 				C: decor.DidentRight,
 			})))
 
@@ -183,19 +179,19 @@ func Message(ctx context.Context, msgs ...cleanable) {
 // that switches to "done" when the completion channel is signalled
 func MessageWithCompletion(
 	ctx context.Context,
-	msg cleanable,
+	msg any,
 ) (chan<- struct{}, func()) {
 	var (
-		clean   = msg.clean()
+		plain    = plainString(msg)
-		message = msg.String()
+		loggable = fmt.Sprintf("%v", msg)
-		log     = logger.Ctx(ctx)
+		log      = logger.Ctx(ctx)
-		ch      = make(chan struct{}, 1)
+		ch       = make(chan struct{}, 1)
 	)
 
-	log.Info(clean)
+	log.Info(loggable)
 
 	if cfg.hidden() {
-		return ch, func() { log.Info("done - " + clean) }
+		return ch, func() { log.Info("done - " + loggable) }
 	}
 
 	wg.Add(1)
@@ -206,7 +202,7 @@ func MessageWithCompletion(
 		-1,
 		mpb.SpinnerStyle(frames...).PositionLeft(),
 		mpb.PrependDecorators(
-			decor.Name(message+":"),
+			decor.Name(plain+":"),
 			decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 8})),
 		mpb.BarFillerOnComplete("done"))
 
@@ -224,7 +220,7 @@ func MessageWithCompletion(
 		})
 
 	wacb := waitAndCloseBar(bar, func() {
-		log.Info("done - " + clean)
+		log.Info("done - " + loggable)
 	})
 
 	return ch, wacb
@@ -241,11 +237,12 @@ func ItemProgress(
 	ctx context.Context,
 	rc io.ReadCloser,
 	header string,
-	iname cleanable,
+	iname any,
 	totalBytes int64,
 ) (io.ReadCloser, func()) {
+	plain := plainString(iname)
 	log := logger.Ctx(ctx).With(
-		"item", iname.clean(),
+		"item", iname,
 		"size", humanize.Bytes(uint64(totalBytes)))
 	log.Debug(header)
 
@@ -258,7 +255,7 @@ func ItemProgress(
 	barOpts := []mpb.BarOption{
 		mpb.PrependDecorators(
 			decor.Name(header, decor.WCSyncSpaceR),
-			decor.Name(iname.String(), decor.WCSyncSpaceR),
+			decor.Name(plain, decor.WCSyncSpaceR),
 			decor.CountersKibiByte(" %.1f/%.1f ", decor.WC{W: 8}),
 			decor.NewPercentage("%d ", decor.WC{W: 4})),
 	}
@@ -284,20 +281,21 @@ func ItemProgress(
 func ProgressWithCount(
 	ctx context.Context,
 	header string,
-	message cleanable,
+	msg any,
 	count int64,
 ) (chan<- struct{}, func()) {
 	var (
-		log  = logger.Ctx(ctx)
+		plain    = plainString(msg)
-		lmsg = fmt.Sprintf("%s %s - %d", header, message.clean(), count)
+		loggable = fmt.Sprintf("%s %v - %d", header, msg, count)
-		ch   = make(chan struct{})
+		log      = logger.Ctx(ctx)
+		ch       = make(chan struct{})
 	)
 
-	log.Info(lmsg)
+	log.Info(loggable)
 
 	if cfg.hidden() {
 		go listen(ctx, ch, nop, nop)
-		return ch, func() { log.Info("done - " + lmsg) }
+		return ch, func() { log.Info("done - " + loggable) }
 	}
 
 	wg.Add(1)
@@ -305,7 +303,7 @@ func ProgressWithCount(
 	barOpts := []mpb.BarOption{
 		mpb.PrependDecorators(
 			decor.Name(header, decor.WCSyncSpaceR),
-			decor.Name(message.String()),
+			decor.Name(plain),
 			decor.Counters(0, " %d/%d ")),
 	}
 
@@ -322,7 +320,7 @@ func ProgressWithCount(
 		bar.Increment)
 
 	wacb := waitAndCloseBar(bar, func() {
-		log.Info("done - " + lmsg)
+		log.Info("done - " + loggable)
 	})
 
 	return ch, wacb
@@ -366,14 +364,15 @@ func makeSpinFrames(barWidth int) {
 func CollectionProgress(
 	ctx context.Context,
 	category string,
-	dirName cleanable,
+	dirName any,
 ) (chan<- struct{}, func()) {
 	var (
 		counted int
+		plain   = plainString(dirName)
 		ch      = make(chan struct{})
 		log     = logger.Ctx(ctx).With(
 			"category", category,
-			"dir", dirName.clean())
+			"dir", dirName)
 		message = "Collecting Directory"
 	)
 
@@ -387,7 +386,7 @@ func CollectionProgress(
 		}
 	}
 
-	if cfg.hidden() || len(dirName.String()) == 0 {
+	if cfg.hidden() || len(plain) == 0 {
 		go listen(ctx, ch, nop, incCount)
 		return ch, func() { log.Infow("done - "+message, "count", counted) }
 	}
@@ -398,7 +397,7 @@ func CollectionProgress(
 		mpb.PrependDecorators(decor.Name(string(category))),
 		mpb.AppendDecorators(
 			decor.CurrentNoUnit("%d - ", decor.WCSyncSpace),
-			decor.Name(dirName.String()),
+			decor.Name(plain),
 		),
 		mpb.BarFillerOnComplete(spinFrames[0]),
 	}
@@ -466,62 +465,45 @@ func listen(ctx context.Context, ch <-chan struct{}, onEnd, onInc func()) {
 }
 
 // ---------------------------------------------------------------------------
-// PII redaction
+// Styling
 // ---------------------------------------------------------------------------
 
-type cleanable interface {
+const Bullet = "∙"
-	clean() string
-	String() string
-}
 
-type PII string
+type bulletf struct {
-
-func (p PII) clean() string {
-	return "***"
-}
-
-func (p PII) String() string {
-	return string(p)
-}
-
-type Safe string
-
-func (s Safe) clean() string {
-	return string(s)
-}
-
-func (s Safe) String() string {
-	return string(s)
-}
-
-type bulletPII struct {
 	tmpl string
-	vars []cleanable
+	vs   []any
 }
 
-func Bulletf(template string, vs ...cleanable) bulletPII {
+func Bulletf(template string, vs ...any) bulletf {
-	return bulletPII{
+	return bulletf{template, vs}
-		tmpl: "∙ " + template,
-		vars: vs,
-	}
 }
 
-func (b bulletPII) clean() string {
+func (b bulletf) PlainString() string {
-	vs := make([]any, 0, len(b.vars))
+	ps := make([]any, 0, len(b.vs))
-
+	for _, v := range b.vs {
-	for _, v := range b.vars {
+		ps = append(ps, plainString(v))
-		vs = append(vs, v.clean())
 	}
 
-	return fmt.Sprintf(b.tmpl, vs...)
+	return fmt.Sprintf("∙ "+b.tmpl, ps...)
 }
 
-func (b bulletPII) String() string {
+func (b bulletf) String() string {
-	vs := make([]any, 0, len(b.vars))
+	return fmt.Sprintf("∙ "+b.tmpl, b.vs...)
+}
 
-	for _, v := range b.vars {
+// plainString attempts to cast v to a PlainStringer
-		vs = append(vs, v.String())
+// interface, and retrieve the un-altered value.  If
+// v is not compliant with PlainStringer, returns the
+// %v fmt of v.
+//
+// This should only be used to display the value in the
+// observe progress bar.  Logged values should only use
+// the fmt %v to ensure Concealers hide PII.
+func plainString(v any) string {
+	if ps, ok := v.(clues.PlainStringer); ok {
+		return ps.PlainString()
 	}
 
-	return fmt.Sprintf(b.tmpl, vs...)
+	return fmt.Sprintf("%v", v)
 }

src/internal/observe/observe_test.go

@@ -29,9 +29,9 @@ func TestObserveProgressUnitSuite(t *testing.T) {
 }
 
 var (
-	tst        = Safe("test")
+	tst        = "test"
-	testcat    = Safe("testcat")
+	testcat    = "testcat"
-	testertons = Safe("testertons")
+	testertons = "testertons"
 )
 
 func (suite *ObserveProgressUnitSuite) TestItemProgress() {
@@ -105,7 +105,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnCtxCancel
 		SeedWriter(context.Background(), nil, nil)
 	}()
 
-	progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons)
+	progCh, closer := CollectionProgress(ctx, testcat, testertons)
 	require.NotNil(t, progCh)
 	require.NotNil(t, closer)
 
@@ -140,7 +140,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnChannelCl
 		SeedWriter(context.Background(), nil, nil)
 	}()
 
-	progCh, closer := CollectionProgress(ctx, testcat.clean(), testertons)
+	progCh, closer := CollectionProgress(ctx, testcat, testertons)
 	require.NotNil(t, progCh)
 	require.NotNil(t, closer)
 
@@ -172,7 +172,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgress() {
 
 	message := "Test Message"
 
-	Message(ctx, Safe(message))
+	Message(ctx, message)
 	Complete()
 	require.NotEmpty(suite.T(), recorder.String())
 	require.Contains(suite.T(), recorder.String(), message)
@@ -193,7 +193,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCompletion() {
 
 	message := "Test Message"
 
-	ch, closer := MessageWithCompletion(ctx, Safe(message))
+	ch, closer := MessageWithCompletion(ctx, message)
 
 	// Trigger completion
 	ch <- struct{}{}
@@ -223,7 +223,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithChannelClosed() {
 
 	message := "Test Message"
 
-	ch, closer := MessageWithCompletion(ctx, Safe(message))
+	ch, closer := MessageWithCompletion(ctx, message)
 
 	// Close channel without completing
 	close(ch)
@@ -255,7 +255,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithContextCancelled()
 
 	message := "Test Message"
 
-	_, closer := MessageWithCompletion(ctx, Safe(message))
+	_, closer := MessageWithCompletion(ctx, message)
 
 	// cancel context
 	cancel()
@@ -286,7 +286,7 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCount() {
 	message := "Test Message"
 	count := 3
 
-	ch, closer := ProgressWithCount(ctx, header, Safe(message), int64(count))
+	ch, closer := ProgressWithCount(ctx, header, message, int64(count))
 
 	for i := 0; i < count; i++ {
 		ch <- struct{}{}
@@ -319,7 +319,7 @@ func (suite *ObserveProgressUnitSuite) TestrogressWithCountChannelClosed() {
 	message := "Test Message"
 	count := 3
 
-	ch, closer := ProgressWithCount(ctx, header, Safe(message), int64(count))
+	ch, closer := ProgressWithCount(ctx, header, message, int64(count))
 
 	close(ch)
 

src/internal/operations/backup.go

@@ -165,7 +165,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 	// Execution
 	// -----
 
-	observe.Message(ctx, observe.Safe("Backing Up"), observe.Bullet, observe.PII(op.ResourceOwner))
+	observe.Message(ctx, "Backing Up", observe.Bullet, clues.Hide(op.ResourceOwner))
 
 	deets, err := op.do(
 		ctx,
@@ -332,7 +332,7 @@ func produceBackupDataCollections(
 	ctrlOpts control.Options,
 	errs *fault.Bus,
 ) ([]data.BackupCollection, map[string]map[string]struct{}, error) {
-	complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Discovering items to backup"))
+	complete, closer := observe.MessageWithCompletion(ctx, "Discovering items to backup")
 	defer func() {
 		complete <- struct{}{}
 		close(complete)
@@ -403,7 +403,7 @@ func consumeBackupCollections(
 	isIncremental bool,
 	errs *fault.Bus,
 ) (*kopia.BackupStats, *details.Builder, map[string]kopia.PrevRefs, error) {
-	complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Backing up data"))
+	complete, closer := observe.MessageWithCompletion(ctx, "Backing up data")
 	defer func() {
 		complete <- struct{}{}
 		close(complete)

src/internal/operations/restore.go

@@ -185,7 +185,7 @@ func (op *RestoreOperation) do(
 		return nil, clues.Wrap(err, "getting backup and details")
 	}
 
-	observe.Message(ctx, observe.Safe("Restoring"), observe.Bullet, observe.PII(bup.Selector.DiscreteOwner))
+	observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner))
 
 	paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors)
 	if err != nil {
@@ -210,10 +210,10 @@ func (op *RestoreOperation) do(
 			events.RestoreID:        opStats.restoreID,
 		})
 
-	observe.Message(ctx, observe.Safe(fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID)))
+	observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID))
 	logger.Ctx(ctx).With("selectors", op.Selectors).Info("restoring selection")
 
-	kopiaComplete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Enumerating items in repository"))
+	kopiaComplete, closer := observe.MessageWithCompletion(ctx, "Enumerating items in repository")
 	defer closer()
 	defer close(kopiaComplete)
 
@@ -317,7 +317,7 @@ func consumeRestoreCollections(
 	dcs []data.RestoreCollection,
 	errs *fault.Bus,
 ) (*details.Details, error) {
-	complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Restoring data"))
+	complete, closer := observe.MessageWithCompletion(ctx, "Restoring data")
 	defer func() {
 		complete <- struct{}{}
 		close(complete)

src/pkg/repository/repository.go

@@ -188,7 +188,7 @@ func Connect(
 	// their output getting clobbered (#1720)
 	defer observe.Complete()
 
-	complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Connecting to repository"))
+	complete, closer := observe.MessageWithCompletion(ctx, "Connecting to repository")
 	defer closer()
 	defer close(complete)
 
@@ -570,7 +570,7 @@ func connectToM365(
 	acct account.Account,
 	errs *fault.Bus,
 ) (*connector.GraphConnector, error) {
-	complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Connecting to M365"))
+	complete, closer := observe.MessageWithCompletion(ctx, "Connecting to M365")
 	defer func() {
 		complete <- struct{}{}
 		close(complete)