From f2f76d932debf12994ee37fab7f551e81c60004a Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 22:35:53 -0600 Subject: [PATCH] release the sensitive-info flag (#3369) Not 100% happy with the flag name, and am open to suggestions. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included #### Type of change - [x] :sunflower: Feature - [x] :world_map: Documentation #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- CHANGELOG.md | 1 + src/pkg/logger/logger.go | 41 ++++++++++++++-------------- src/pkg/logger/logger_test.go | 8 +++--- website/docs/setup/configuration.md | 6 ++++ website/styles/Vocab/Base/accept.txt | 3 +- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39811f5cf..dcfecc3ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) ### Added +- Released the --mask-sensitive-data flag, which will automatically obscure private data in logs. ### Fixed - Graph requests now automatically retry in case of a Bad Gateway or Gateway Timeout. diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index fde379430..cc632b422 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -34,20 +34,20 @@ const ( // flag names const ( - DebugAPIFN = "debug-api-calls" - LogFileFN = "log-file" - LogLevelFN = "log-level" - ReadableLogsFN = "readable-logs" - SensitiveInfoFN = "sensitive-info" + DebugAPIFN = "debug-api-calls" + LogFileFN = "log-file" + LogLevelFN = "log-level" + ReadableLogsFN = "readable-logs" + MaskSensitiveDataFN = "mask-sensitive-data" ) // flag values var ( - DebugAPIFV bool - logFileFV = "" - LogLevelFV = "info" - ReadableLogsFV bool - SensitiveInfoFV = PIIPlainText + DebugAPIFV bool + logFileFV = "" + LogLevelFV = "info" + ReadableLogsFV bool + MaskSensitiveDataFV bool LogFile string // logFileFV after processing ) @@ -83,9 +83,6 @@ func AddLoggingFlags(cmd *cobra.Command) { //nolint:errcheck fs.MarkHidden(ReadableLogsFN) - // TODO(keepers): unhide when we have sufficient/complete coverage of PII handling - //nolint:errcheck - fs.MarkHidden(SensitiveInfoFN) } // internal deduplication for adding flags @@ -106,11 +103,11 @@ func addFlags(fs *pflag.FlagSet, defaultFile string) { false, "minimizes log output for console readability: removes the file and date, colors the level") - fs.StringVar( - &SensitiveInfoFV, - SensitiveInfoFN, - PIIPlainText, - fmt.Sprintf("set the format for sensitive info in logs to %s|%s|%s", PIIHash, PIIMask, PIIPlainText)) + fs.BoolVar( + &MaskSensitiveDataFV, + MaskSensitiveDataFN, + false, + "anonymize personal data in log output") } // Settings records the user's preferred logging settings. @@ -136,7 +133,7 @@ func PreloadLoggingFlags(args []string) Settings { ls := Settings{ File: "", Level: LogLevelFV, - PIIHandling: SensitiveInfoFV, + PIIHandling: PIIPlainText, } // parse the os args list to find the log level flag @@ -144,6 +141,10 @@ func PreloadLoggingFlags(args []string) Settings { return ls } + if MaskSensitiveDataFV { + ls.PIIHandling = PIIHash + } + // retrieve the user's preferred log level // automatically defaults to "info" levelString, err := fs.GetString(LogLevelFN) @@ -165,7 +166,7 @@ func PreloadLoggingFlags(args []string) Settings { // retrieve the user's preferred PII handling algorithm // automatically defaults to default log location - pii, err := fs.GetString(SensitiveInfoFN) + pii, err := fs.GetString(MaskSensitiveDataFN) if err != nil { return ls } diff --git a/src/pkg/logger/logger_test.go b/src/pkg/logger/logger_test.go index 7cb7926fa..644c23aa0 100644 --- a/src/pkg/logger/logger_test.go +++ b/src/pkg/logger/logger_test.go @@ -33,7 +33,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { assert.True(t, logger.DebugAPIFV, logger.DebugAPIFN) assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, logger.LLError, logger.LogLevelFV, logger.LogLevelFN) - assert.Equal(t, logger.PIIMask, logger.SensitiveInfoFV, logger.SensitiveInfoFN) + assert.True(t, logger.MaskSensitiveDataFV, logger.MaskSensitiveDataFN) // empty assertion here, instead of matching "log-file", because the LogFile // var isn't updated by running the command (this is expected and correct), // while the logFileFV remains unexported. @@ -50,7 +50,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, }) err := cmd.Execute() @@ -68,7 +68,7 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, } settings := logger.PreloadLoggingFlags(args) @@ -77,5 +77,5 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, "log-file", settings.File, "settings.File") assert.Equal(t, logger.LLError, settings.Level, "settings.Level") - assert.Equal(t, logger.PIIMask, settings.PIIHandling, "settings.PIIHandling") + assert.Equal(t, logger.PIIHash, settings.PIIHandling, "settings.PIIHandling") } diff --git a/website/docs/setup/configuration.md b/website/docs/setup/configuration.md index d9255f6b7..65c04e99b 100644 --- a/website/docs/setup/configuration.md +++ b/website/docs/setup/configuration.md @@ -132,7 +132,13 @@ directory within the container. Corso generates a unique log file named with its timestamp for every invocation. The default location of Corso's log file is shown below but the location can be overridden by using the `--log-file` flag. The log file will be appended to if multiple Corso invocations are pointed to the same file. + You can also use `stdout` or `stderr` as the `--log-file` location to redirect the logs to "stdout" and "stderr" respectively. +This setting can cause logs to compete with progress bar displays in the terminal. +We suggest using the `--hide-progress` option if you plan to log to stdout or stderr. + +Log entries, by default, include user names and file names. The `--mask-sensitive-data` option can be +used to replace this information with anonymized hashes. diff --git a/website/styles/Vocab/Base/accept.txt b/website/styles/Vocab/Base/accept.txt index 7f8d159c7..b915b5010 100644 --- a/website/styles/Vocab/Base/accept.txt +++ b/website/styles/Vocab/Base/accept.txt @@ -54,4 +54,5 @@ Demetrius Malbrough lockdowns exfiltrate -deduplicating \ No newline at end of file +deduplicating +anonymized