From 1939482eee8beadd232c15749d281f742543e885 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 3 Aug 2023 19:50:12 -0600 Subject: [PATCH] sweeping package update (#3880) run `go get -u` to update all packages. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup --- src/cli/cli.go | 8 +- src/cli/print/print.go | 24 +- src/cli/print/print_test.go | 12 +- src/go.mod | 39 ++-- src/go.sum | 84 ++++--- src/internal/kopia/merge_collection.go | 11 +- src/internal/kopia/upload.go | 6 +- src/internal/m365/exchange/collection.go | 5 +- src/internal/m365/onedrive/collections.go | 4 +- src/internal/m365/onedrive/restore.go | 14 +- src/internal/m365/sharepoint/backup.go | 5 +- src/internal/m365/sharepoint/collection.go | 5 +- src/internal/observe/observe.go | 247 +++++++++++++-------- src/internal/observe/observe_test.go | 153 ++++--------- src/internal/operations/backup.go | 16 +- src/internal/operations/export.go | 2 - src/internal/operations/restore.go | 15 +- src/pkg/repository/repository.go | 23 +- 18 files changed, 328 insertions(+), 345 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 68f454407..230433479 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -163,15 +163,15 @@ func BuildCommandTree(cmd *cobra.Command) { func Handle() { //nolint:forbidigo ctx := config.Seed(context.Background()) + ctx, log := logger.Seed(ctx, logger.PreloadLoggingFlags(os.Args[1:])) ctx = print.SetRootCmd(ctx, corsoCmd) - - observe.SeedWriter(ctx, print.StderrWriter(ctx), observe.PreloadFlags()) + ctx = observe.SeedObserver(ctx, print.StderrWriter(ctx), observe.PreloadFlags()) BuildCommandTree(corsoCmd) - ctx, log := logger.Seed(ctx, logger.PreloadLoggingFlags(os.Args[1:])) - defer func() { + observe.Flush(ctx) // flush the progress bars + _ = log.Sync() // flush all logs in the buffer }() diff --git a/src/cli/print/print.go b/src/cli/print/print.go index 3c4cceb05..9c456b65d 100644 --- a/src/cli/print/print.go +++ b/src/cli/print/print.go @@ -9,6 +9,8 @@ import ( "github.com/spf13/cobra" "github.com/tidwall/pretty" "github.com/tomlazar/table" + + "github.com/alcionai/corso/src/internal/observe" ) var ( @@ -83,38 +85,38 @@ func Only(ctx context.Context, e error) error { // if s is nil, prints nothing. // Prepends the message with "Error: " func Err(ctx context.Context, s ...any) { - out(getRootCmd(ctx).ErrOrStderr(), s...) + out(ctx, getRootCmd(ctx).ErrOrStderr(), s...) } // Errf prints the params to cobra's error writer (stdErr by default) // if s is nil, prints nothing. // Prepends the message with "Error: " func Errf(ctx context.Context, tmpl string, s ...any) { - outf(getRootCmd(ctx).ErrOrStderr(), "\nError: \n\t"+tmpl+"\n", s...) + outf(ctx, getRootCmd(ctx).ErrOrStderr(), "\nError: \n\t"+tmpl+"\n", s...) } // Out prints the params to cobra's output writer (stdOut by default) // if s is nil, prints nothing. func Out(ctx context.Context, s ...any) { - out(getRootCmd(ctx).OutOrStdout(), s...) + out(ctx, getRootCmd(ctx).OutOrStdout(), s...) } // Out prints the formatted strings to cobra's output writer (stdOut by default) // if t is empty, prints nothing. func Outf(ctx context.Context, t string, s ...any) { - outf(getRootCmd(ctx).OutOrStdout(), t, s...) + outf(ctx, getRootCmd(ctx).OutOrStdout(), t, s...) } // Info prints the params to cobra's error writer (stdErr by default) // if s is nil, prints nothing. func Info(ctx context.Context, s ...any) { - out(getRootCmd(ctx).ErrOrStderr(), s...) + out(ctx, getRootCmd(ctx).ErrOrStderr(), s...) } // Info prints the formatted strings to cobra's error writer (stdErr by default) // if t is empty, prints nothing. func Infof(ctx context.Context, t string, s ...any) { - outf(getRootCmd(ctx).ErrOrStderr(), t, s...) + outf(ctx, getRootCmd(ctx).ErrOrStderr(), t, s...) } // PrettyJSON prettifies and prints the value. @@ -127,21 +129,27 @@ func PrettyJSON(ctx context.Context, p minimumPrintabler) { } // out is the testable core of exported print funcs -func out(w io.Writer, s ...any) { +func out(ctx context.Context, w io.Writer, s ...any) { if len(s) == 0 { return } + // observe bars needs to be flushed before printing + observe.Flush(ctx) + fmt.Fprint(w, s...) fmt.Fprintf(w, "\n") } // outf is the testable core of exported print funcs -func outf(w io.Writer, t string, s ...any) { +func outf(ctx context.Context, w io.Writer, t string, s ...any) { if len(t) == 0 { return } + // observe bars needs to be flushed before printing + observe.Flush(ctx) + fmt.Fprintf(w, t, s...) fmt.Fprintf(w, "\n") } diff --git a/src/cli/print/print_test.go b/src/cli/print/print_test.go index 3edafb098..9fcadfa57 100644 --- a/src/cli/print/print_test.go +++ b/src/cli/print/print_test.go @@ -36,20 +36,28 @@ func (suite *PrintUnitSuite) TestOnly() { func (suite *PrintUnitSuite) TestOut() { t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + b := bytes.Buffer{} msg := "I have seen the fnords!" - out(&b, msg) + out(ctx, &b, msg) assert.Contains(t, b.String(), msg) } func (suite *PrintUnitSuite) TestOutf() { t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + b := bytes.Buffer{} msg := "I have seen the fnords!" msg2 := "smarf" - outf(&b, msg, msg2) + outf(ctx, &b, msg, msg2) bs := b.String() assert.Contains(t, bs, msg) assert.Contains(t, bs, msg2) diff --git a/src/go.mod b/src/go.mod index e0de1a44a..f5d797f39 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,12 +8,12 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.313 + github.com/aws/aws-sdk-go v1.44.316 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 - github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 + github.com/kopia/kopia v0.13.0 github.com/microsoft/kiota-abstractions-go v1.2.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 github.com/microsoft/kiota-http-go v1.0.1 @@ -32,8 +32,8 @@ require ( github.com/tidwall/pretty v1.2.1 github.com/tomlazar/table v0.1.2 github.com/vbauerster/mpb/v8 v8.1.6 - go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + go.uber.org/zap v1.25.0 + golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b golang.org/x/time v0.3.0 golang.org/x/tools v0.11.1 ) @@ -41,29 +41,30 @@ require ( require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.34.0 // indirect + github.com/valyala/fasthttp v1.48.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect ) require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -74,25 +75,24 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/microsoft/kiota-serialization-text-go v1.0.0 github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.59 + github.com/minio/minio-go/v7 v7.0.61 github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -103,27 +103,26 @@ require ( github.com/prometheus/client_golang v1.16.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/rivo/uniseg v0.4.3 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/xid v1.5.0 // indirect - github.com/segmentio/backo-go v1.0.0 // indirect + github.com/segmentio/backo-go v1.0.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/tidwall/gjson v1.14.3 // indirect + github.com/tidwall/gjson v1.15.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 + golang.org/x/net v0.13.0 golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - google.golang.org/grpc v1.56.2 // indirect + google.golang.org/grpc v1.57.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/src/go.sum b/src/go.sum index a2bd77b23..57519b404 100644 --- a/src/go.sum +++ b/src/go.sum @@ -42,8 +42,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 h1:HCc0+LpPfpCKs6LGGLAhwBARt9632unrVcI6i8s/8os= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -62,15 +62,15 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.313 h1:u6EuNQqgAmi09GEZ5g/XGHLF0XV31WcdU5rnHyIBHBc= -github.com/aws/aws-sdk-go v1.44.313/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.316 h1:UC3alCEyzj2XU13ZFGIOHW3yjCNLGTIGVauyetl9fwE= +github.com/aws/aws-sdk-go v1.44.316/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -133,7 +133,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -232,9 +233,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= @@ -264,8 +264,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -289,8 +289,8 @@ github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8w github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw= -github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/minio-go/v7 v7.0.61 h1:87c+x8J3jxQ5VUGimV9oHdpjsAvy3fhneEBKuoKEVUI= +github.com/minio/minio-go/v7 v7.0.61/go.mod h1:BTu8FcrEw+HidY0zd/0eny43QnVNkXRPXrLXFuQBHXg= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -309,8 +309,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -340,13 +340,13 @@ github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/puzpuzpuz/xsync/v2 v2.4.1 h1:aGdE1C/HaR/QC6YAFdtZXi60Df8/qBIrs8PKrzkItcM= github.com/puzpuzpuz/xsync/v2 v2.4.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= -github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -354,8 +354,8 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= -github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= +github.com/segmentio/backo-go v1.0.1 h1:68RQccglxZeyURy93ASB/2kc9QudzgIDexJ927N++y4= +github.com/segmentio/backo-go v1.0.1/go.mod h1:9/Rh6yILuLysoQnZ2oNooD2g7aBnvM7r/fNVxRNWfBc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -385,14 +385,13 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tg123/go-htpasswd v1.2.1 h1:i4wfsX1KvvkyoMiHZzjS0VzbAPWfxzI8INcZAKtutoU= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw= +github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -403,9 +402,8 @@ github.com/tomlazar/table v0.1.2/go.mod h1:IecZnpep9f/BatHacfh+++ftE+lFONN8BVPi9 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= -github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= +github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/vbauerster/mpb/v8 v8.1.6 h1:EswHDkAsy4OQ7QBAmU1MUPz4vHzl6KlINjlh7vJoxvY= github.com/vbauerster/mpb/v8 v8.1.6/go.mod h1:O9/Wl8X9dUbR63tZ41MLIAxrtNfwlpwUhGkeYugUPW8= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= @@ -436,13 +434,11 @@ go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26 go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -451,7 +447,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= @@ -465,8 +460,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= +golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -527,11 +522,10 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -594,8 +588,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -742,8 +734,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -760,8 +752,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/src/internal/kopia/merge_collection.go b/src/internal/kopia/merge_collection.go index 25897fd58..346affa3c 100644 --- a/src/internal/kopia/merge_collection.go +++ b/src/internal/kopia/merge_collection.go @@ -43,8 +43,15 @@ func (mc *mergeCollection) addCollection( // Keep a stable sorting of this merged collection set so we can say there's // some deterministic behavior when Fetch is called. We don't expect to have // to merge many collections. - slices.SortStableFunc(mc.cols, func(a, b col) bool { - return a.storagePath < b.storagePath + slices.SortStableFunc(mc.cols, func(a, b col) int { + switch true { + case a.storagePath < b.storagePath: + return -1 + case a.storagePath > b.storagePath: + return 1 + default: + return 0 + } }) return nil diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 9df1327f9..b0dbd40e5 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -74,16 +74,16 @@ func (rw *backupStreamReader) Close() error { rw.combined = nil - var outerErr error + var errs *clues.Err for _, r := range rw.readers { err := r.Close() if err != nil { - outerErr = clues.Stack(err, clues.New("closing reader")) + errs = clues.Stack(clues.Wrap(err, "closing reader"), errs) } } - return outerErr + return errs.OrNil() } // restoreStreamReader is a wrapper around the io.Reader that kopia returns when diff --git a/src/internal/m365/exchange/collection.go b/src/internal/m365/exchange/collection.go index 9bb320f18..8b10b555d 100644 --- a/src/internal/m365/exchange/collection.go +++ b/src/internal/m365/exchange/collection.go @@ -167,10 +167,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { ctx, col.fullPath.Category().String(), col.LocationPath().Elements()) - - defer func() { - close(colProgress) - }() + defer close(colProgress) } semaphoreCh := make(chan struct{}, col.ctrl.Parallelism.ItemFetch) diff --git a/src/internal/m365/onedrive/collections.go b/src/internal/m365/onedrive/collections.go index fa91a0a11..a73d46fec 100644 --- a/src/internal/m365/onedrive/collections.go +++ b/src/internal/m365/onedrive/collections.go @@ -242,8 +242,8 @@ func (c *Collections) Get( driveTombstones[driveID] = struct{}{} } - driveComplete := observe.MessageWithCompletion(ctx, observe.Bulletf("files")) - defer close(driveComplete) + progressBar := observe.MessageWithCompletion(ctx, observe.Bulletf("files")) + defer close(progressBar) // Enumerate drives for the specified resourceOwner pager := c.handler.NewDrivePager(c.resourceOwner, nil) diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 7c920e6ff..a8d70c943 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -863,7 +863,11 @@ func restoreFile( return "", details.ItemInfo{}, clues.Wrap(err, "get item upload session") } - var written int64 + var ( + written int64 + progReader io.ReadCloser + closeProgressBar func() + ) // This is just to retry file upload, the uploadSession creation is // not retried here We need extra logic to retry file upload as we @@ -889,7 +893,7 @@ func restoreFile( iReader = itemData.ToReader() } - progReader, abort := observe.ItemProgress( + progReader, closeProgressBar = observe.ItemProgress( ctx, iReader, observe.ItemRestoreMsg, @@ -902,8 +906,8 @@ func restoreFile( break } - // clear out the bar if err - abort() + // clear out the progress bar immediately on error + closeProgressBar() // refresh the io.Writer to restart the upload // TODO: @vkamra verify if var session is the desired input @@ -914,6 +918,8 @@ func restoreFile( return "", details.ItemInfo{}, clues.Wrap(err, "uploading file") } + defer closeProgressBar() + dii := ir.AugmentItemInfo(details.ItemInfo{}, newItem, written, nil) if shouldDeleteOriginal { diff --git a/src/internal/m365/sharepoint/backup.go b/src/internal/m365/sharepoint/backup.go index f2658032e..61acac7d2 100644 --- a/src/internal/m365/sharepoint/backup.go +++ b/src/internal/m365/sharepoint/backup.go @@ -58,10 +58,10 @@ func ProduceBackupCollections( break } - foldersComplete := observe.MessageWithCompletion( + progressBar := observe.MessageWithCompletion( ctx, observe.Bulletf("%s", scope.Category().PathType())) - defer close(foldersComplete) + defer close(progressBar) var spcs []data.BackupCollection @@ -117,7 +117,6 @@ func ProduceBackupCollections( } collections = append(collections, spcs...) - foldersComplete <- struct{}{} categories[scope.Category().PathType()] = struct{}{} } diff --git a/src/internal/m365/sharepoint/collection.go b/src/internal/m365/sharepoint/collection.go index 687a2ebb8..23a7cb447 100644 --- a/src/internal/m365/sharepoint/collection.go +++ b/src/internal/m365/sharepoint/collection.go @@ -190,10 +190,7 @@ func (sc *Collection) runPopulate( ctx, sc.fullPath.Category().String(), sc.fullPath.Folders()) - - defer func() { - close(colProgress) - }() + defer close(colProgress) // Switch retrieval function based on category switch sc.category { diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index 50994eacb..12895fce0 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -24,19 +24,7 @@ const ( progressBarWidth = 32 ) -var ( - wg sync.WaitGroup - // TODO: Revisit this being a global and make it a parameter to the progress methods - // so that each bar can be initialized with different contexts if needed. - contxt context.Context - writer io.Writer - progress *mpb.Progress - cfg *config -) - func init() { - cfg = &config{} - makeSpinFrames(progressBarWidth) } @@ -52,7 +40,7 @@ func AddProgressBarFlags(cmd *cobra.Command) { // Due to races between the lazy evaluation of flags in cobra and the need to init observer // behavior in a ctx, these options get pre-processed manually here using pflags. The canonical // AddProgressBarFlag() ensures the flags are displayed as part of the help/usage output. -func PreloadFlags() *config { +func PreloadFlags() config { fs := pflag.NewFlagSet("seed-observer", pflag.ContinueOnError) fs.ParseErrorsWhitelist.UnknownFlags = true fs.Bool(hideProgressBarsFN, false, "turn off the progress bar displays") @@ -62,25 +50,26 @@ func PreloadFlags() *config { // parse the os args list to find the observer display flags if err := fs.Parse(os.Args[1:]); err != nil { - return nil + return config{} } // retrieve the user's preferred display // automatically defaults to "info" shouldHide, err := fs.GetBool(hideProgressBarsFN) if err != nil { - return nil + return config{} } // retrieve the user's preferred display // automatically defaults to "info" shouldAlwaysShow, err := fs.GetBool(retainProgressBarsFN) if err != nil { - return nil + return config{} } - return &config{ + return config{ doNotDisplay: shouldHide, + displayIsTerminal: true, keepBarsAfterComplete: shouldAlwaysShow, } } @@ -91,43 +80,83 @@ func PreloadFlags() *config { // config handles observer configuration type config struct { + // under certain conditions (ex: testing) we aren't outputting + // to a terminal. When this happens the observe bars need to be + // given a specific optional value or they'll never flush the + // writer. + displayIsTerminal bool doNotDisplay bool keepBarsAfterComplete bool } -func (c config) hidden() bool { - return c.doNotDisplay || writer == nil +type observerKey string + +const ctxKey observerKey = "corsoObserver" + +type observer struct { + cfg config + mp *mpb.Progress + w io.Writer + wg *sync.WaitGroup } -// SeedWriter adds default writer to the observe package. -// Uses a noop writer until seeded. -func SeedWriter(ctx context.Context, w io.Writer, c *config) { - writer = w - contxt = ctx +func (o observer) hidden() bool { + return o.cfg.doNotDisplay || o.w == nil +} - if contxt == nil { - contxt = context.Background() - } - - if c != nil { - cfg = c - } - - progress = mpb.NewWithContext( - contxt, +func (o *observer) resetWriter(ctx context.Context) { + opts := []mpb.ContainerOption{ mpb.WithWidth(progressBarWidth), - mpb.WithWaitGroup(&wg), - mpb.WithOutput(writer)) -} - -// Complete blocks until the progress finishes writing out all data. -// Afterwards, the progress instance is reset. -func Complete() { - if progress != nil { - progress.Wait() + mpb.WithWaitGroup(o.wg), + mpb.WithOutput(o.w), } - SeedWriter(contxt, writer, cfg) + // needed for tests to pass with mpb versions > 8.1.6 + // if !o.cfg.displayIsTerminal { + // opts = append(opts, mpb.WithAutoRefresh()) + // } + + o.mp = mpb.NewWithContext(ctx, opts...) +} + +// SeedObserver adds an observer to the context. Any calls to observe +// funcs will retrieve the observer from the context. If no observer +// is found in the context, the call no-ops. +func SeedObserver(ctx context.Context, w io.Writer, cfg config) context.Context { + obs := &observer{ + w: w, + cfg: cfg, + wg: &sync.WaitGroup{}, + } + + obs.resetWriter(ctx) + + return setObserver(ctx, obs) +} + +func setObserver(ctx context.Context, obs *observer) context.Context { + return context.WithValue(ctx, ctxKey, obs) +} + +func getObserver(ctx context.Context) *observer { + o := ctx.Value(ctxKey) + if o == nil { + return &observer{cfg: config{doNotDisplay: true}} + } + + return o.(*observer) +} + +// Flush blocks until the progress finishes writing out all data. +// Afterwards, the progress instance is reset. +func Flush(ctx context.Context) { + obs := getObserver(ctx) + + if obs.mp != nil { + obs.mp.Wait() + } + + obs.resetWriter(ctx) } const ( @@ -137,12 +166,17 @@ const ( ItemQueueMsg = "Queuing items" ) +// --------------------------------------------------------------------------- // Progress Updates +// --------------------------------------------------------------------------- // Message is used to display a progress message func Message(ctx context.Context, msgs ...any) { - plainSl := make([]string, 0, len(msgs)) - loggableSl := make([]string, 0, len(msgs)) + var ( + obs = getObserver(ctx) + plainSl = make([]string, 0, len(msgs)) + loggableSl = make([]string, 0, len(msgs)) + ) for _, m := range msgs { plainSl = append(plainSl, plainString(m)) @@ -154,13 +188,13 @@ func Message(ctx context.Context, msgs ...any) { logger.Ctx(ctx).Info(loggable) - if cfg.hidden() { + if obs.hidden() { return } - wg.Add(1) + obs.wg.Add(1) - bar := progress.New( + bar := obs.mp.New( -1, mpb.NopStyle(), mpb.PrependDecorators(decor.Name( @@ -172,8 +206,7 @@ func Message(ctx context.Context, msgs ...any) { // Complete the bar immediately bar.SetTotal(-1, true) - - waitAndCloseBar(bar, func() {})() + waitAndCloseBar(ctx, bar, obs.wg, func() {})() } // MessageWithCompletion is used to display progress with a spinner @@ -183,6 +216,7 @@ func MessageWithCompletion( msg any, ) chan<- struct{} { var ( + obs = getObserver(ctx) plain = plainString(msg) loggable = fmt.Sprintf("%v", msg) log = logger.Ctx(ctx) @@ -191,16 +225,16 @@ func MessageWithCompletion( log.Info(loggable) - if cfg.hidden() { + if obs.hidden() { defer log.Info("done - " + loggable) return ch } - wg.Add(1) + obs.wg.Add(1) frames := []string{"∙∙∙", "●∙∙", "∙●∙", "∙∙●", "∙∙∙"} - bar := progress.New( + bar := obs.mp.New( -1, mpb.SpinnerStyle(frames...).PositionLeft(), mpb.PrependDecorators( @@ -215,13 +249,10 @@ func MessageWithCompletion( bar.SetTotal(-1, true) bar.Abort(true) }, - func() { - // We don't care whether the channel was signalled or closed - // Use either one as an indication that the bar is done - bar.SetTotal(-1, true) - }) + // callers should close the channel + func() {}) - go waitAndCloseBar(bar, func() { + go waitAndCloseBar(ctx, bar, obs.wg, func() { log.Info("done - " + loggable) })() @@ -232,9 +263,30 @@ func MessageWithCompletion( // Progress for Known Quantities // --------------------------------------------------------------------------- +type autoCloser struct { + rc io.ReadCloser + close func() + closed bool +} + +func (ac *autoCloser) Read(p []byte) (n int, err error) { + return ac.rc.Read(p) +} + +func (ac *autoCloser) Close() error { + if !ac.closed { + ac.closed = true + ac.close() + } + + return ac.rc.Close() +} + // ItemProgress tracks the display of an item in a folder by counting the bytes // read through the provided readcloser, up until the byte count matches // the totalBytes. +// The progress bar will close automatically when the reader closes. If an early +// close is needed due to abort or other issue, the returned func can be used. func ItemProgress( ctx context.Context, rc io.ReadCloser, @@ -242,18 +294,22 @@ func ItemProgress( iname any, totalBytes int64, ) (io.ReadCloser, func()) { - plain := plainString(iname) - log := logger.Ctx(ctx).With( - "item", iname, - "size", humanize.Bytes(uint64(totalBytes))) + var ( + obs = getObserver(ctx) + plain = plainString(iname) + log = logger.Ctx(ctx).With( + "item", iname, + "size", humanize.Bytes(uint64(totalBytes))) + ) + log.Debug(header) - if cfg.hidden() || rc == nil || totalBytes == 0 { + if obs.hidden() || rc == nil { defer log.Debug("done - " + header) return rc, func() {} } - wg.Add(1) + obs.wg.Add(1) barOpts := []mpb.BarOption{ mpb.PrependDecorators( @@ -263,23 +319,26 @@ func ItemProgress( decor.NewPercentage("%d ", decor.WC{W: 4})), } - if !cfg.keepBarsAfterComplete { + if !obs.cfg.keepBarsAfterComplete { barOpts = append(barOpts, mpb.BarRemoveOnComplete()) } - bar := progress.New(totalBytes, mpb.NopStyle(), barOpts...) + bar := obs.mp.New(totalBytes, mpb.NopStyle(), barOpts...) - go waitAndCloseBar(bar, func() { + go waitAndCloseBar(ctx, bar, obs.wg, func() { // might be overly chatty, we can remove if needed. log.Debug("done - " + header) })() - abort := func() { + closer := &autoCloser{rc: bar.ProxyReader(rc)} + + closer.close = func() { + closer.closed = true bar.SetTotal(-1, true) bar.Abort(true) } - return bar.ProxyReader(rc), abort + return closer, closer.close } // ItemSpinner is similar to ItemProgress, but for use in cases where @@ -290,16 +349,20 @@ func ItemSpinner( header string, iname any, ) (io.ReadCloser, func()) { - plain := plainString(iname) - log := logger.Ctx(ctx).With("item", iname) + var ( + obs = getObserver(ctx) + plain = plainString(iname) + log = logger.Ctx(ctx).With("item", iname) + ) + log.Debug(header) - if cfg.hidden() || rc == nil { + if obs.hidden() || rc == nil { defer log.Debug("done - " + header) return rc, func() {} } - wg.Add(1) + obs.wg.Add(1) barOpts := []mpb.BarOption{ mpb.PrependDecorators( @@ -308,13 +371,13 @@ func ItemSpinner( decor.CurrentKibiByte(" %.1f", decor.WC{W: 8})), } - if !cfg.keepBarsAfterComplete { + if !obs.cfg.keepBarsAfterComplete { barOpts = append(barOpts, mpb.BarRemoveOnComplete()) } - bar := progress.New(-1, mpb.NopStyle(), barOpts...) + bar := obs.mp.New(-1, mpb.NopStyle(), barOpts...) - go waitAndCloseBar(bar, func() { + go waitAndCloseBar(ctx, bar, obs.wg, func() { // might be overly chatty, we can remove if needed. log.Debug("done - " + header) })() @@ -338,6 +401,7 @@ func ProgressWithCount( count int64, ) chan<- struct{} { var ( + obs = getObserver(ctx) plain = plainString(msg) loggable = fmt.Sprintf("%s %v - %d", header, msg, count) log = logger.Ctx(ctx) @@ -346,7 +410,7 @@ func ProgressWithCount( log.Info(loggable) - if cfg.hidden() { + if obs.hidden() { go listen(ctx, ch, nop, nop) defer log.Info("done - " + loggable) @@ -354,7 +418,7 @@ func ProgressWithCount( return ch } - wg.Add(1) + obs.wg.Add(1) barOpts := []mpb.BarOption{ mpb.PrependDecorators( @@ -363,19 +427,21 @@ func ProgressWithCount( decor.Counters(0, " %d/%d ")), } - if !cfg.keepBarsAfterComplete { + if !obs.cfg.keepBarsAfterComplete { barOpts = append(barOpts, mpb.BarRemoveOnComplete()) } - bar := progress.New(count, mpb.NopStyle(), barOpts...) + bar := obs.mp.New(count, mpb.NopStyle(), barOpts...) go listen( ctx, ch, - func() { bar.Abort(true) }, + func() { + bar.Abort(true) + }, bar.Increment) - go waitAndCloseBar(bar, func() { + go waitAndCloseBar(ctx, bar, obs.wg, func() { log.Info("done - " + loggable) })() @@ -423,6 +489,7 @@ func CollectionProgress( dirName any, ) chan<- struct{} { var ( + obs = getObserver(ctx) counted int plain = plainString(dirName) ch = make(chan struct{}) @@ -442,7 +509,7 @@ func CollectionProgress( } } - if cfg.hidden() || len(plain) == 0 { + if obs.hidden() || len(plain) == 0 { go listen(ctx, ch, nop, incCount) defer log.Infow("done - "+message, "count", counted) @@ -450,7 +517,7 @@ func CollectionProgress( return ch } - wg.Add(1) + obs.wg.Add(1) barOpts := []mpb.BarOption{ mpb.PrependDecorators(decor.Name(string(category))), @@ -461,11 +528,11 @@ func CollectionProgress( mpb.BarFillerOnComplete(spinFrames[0]), } - if !cfg.keepBarsAfterComplete { + if !obs.cfg.keepBarsAfterComplete { barOpts = append(barOpts, mpb.BarRemoveOnComplete()) } - bar := progress.New( + bar := obs.mp.New( -1, // -1 to indicate an unbounded count mpb.SpinnerStyle(spinFrames...), barOpts...) @@ -479,14 +546,14 @@ func CollectionProgress( bar.Increment() }) - go waitAndCloseBar(bar, func() { + go waitAndCloseBar(ctx, bar, obs.wg, func() { log.Infow("done - "+message, "count", counted) })() return ch } -func waitAndCloseBar(bar *mpb.Bar, log func()) func() { +func waitAndCloseBar(ctx context.Context, bar *mpb.Bar, wg *sync.WaitGroup, log func()) func() { return func() { bar.Wait() wg.Done() diff --git a/src/internal/observe/observe_test.go b/src/internal/observe/observe_test.go index 08365462a..063e6d42b 100644 --- a/src/internal/observe/observe_test.go +++ b/src/internal/observe/observe_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/alcionai/clues" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -34,21 +35,14 @@ var ( testertons = "testertons" ) -func (suite *ObserveProgressUnitSuite) TestItemProgress() { +func (suite *ObserveProgressUnitSuite) TestObserve_ItemProgress() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - Complete() - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() + ctx = SeedObserver(ctx, &recorder, config{}) from := make([]byte, 100) prog, abort := ItemProgress( @@ -94,14 +88,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnCtxCancel ctx, cancel := context.WithCancel(ctx) recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - Complete() - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() + ctx = SeedObserver(ctx, &recorder, config{}) progCh := CollectionProgress(ctx, testcat, testertons) require.NotNil(t, progCh) @@ -125,14 +112,7 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnChannelCl defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - Complete() - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() + ctx = SeedObserver(ctx, &recorder, config{}) progCh := CollectionProgress(ctx, testcat, testertons) require.NotNil(t, progCh) @@ -147,88 +127,47 @@ func (suite *ObserveProgressUnitSuite) TestCollectionProgress_unblockOnChannelCl }() } -func (suite *ObserveProgressUnitSuite) TestObserveProgress() { +func (suite *ObserveProgressUnitSuite) TestObserve_message() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) + ctx = SeedObserver(ctx, &recorder, config{}) - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() - - message := "Test Message" + message := uuid.NewString()[:8] Message(ctx, message) - Complete() - require.NotEmpty(t, recorder.String()) - require.Contains(t, recorder.String(), message) + Flush(ctx) + assert.NotEmpty(t, recorder) + assert.Contains(t, recorder.String(), message) } -func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCompletion() { +func (suite *ObserveProgressUnitSuite) TestObserve_progressWithChannelClosed() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) + ctx = SeedObserver(ctx, &recorder, config{}) - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() - - message := "Test Message" - - ch := MessageWithCompletion(ctx, message) - - // Trigger completion - ch <- struct{}{} - - Complete() - - require.NotEmpty(t, recorder.String()) - require.Contains(t, recorder.String(), message) - require.Contains(t, recorder.String(), "done") -} - -func (suite *ObserveProgressUnitSuite) TestObserveProgressWithChannelClosed() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() - - message := "Test Message" + message := uuid.NewString()[:8] ch := MessageWithCompletion(ctx, message) // Close channel without completing close(ch) - Complete() + Flush(ctx) - require.NotEmpty(t, recorder.String()) - require.Contains(t, recorder.String(), message) - require.Contains(t, recorder.String(), "done") + assert.NotEmpty(t, recorder.String()) + assert.Contains(t, recorder.String(), message) + assert.Contains(t, recorder.String(), "done") } -func (suite *ObserveProgressUnitSuite) TestObserveProgressWithContextCancelled() { +func (suite *ObserveProgressUnitSuite) TestObserve_progressWithContextCancelled() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -237,44 +176,32 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithContextCancelled() ctx, cancel := context.WithCancel(ctx) recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) + ctx = SeedObserver(ctx, &recorder, config{}) - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() - - message := "Test Message" + message := uuid.NewString()[:8] _ = MessageWithCompletion(ctx, message) // cancel context cancel() - Complete() + Flush(ctx) require.NotEmpty(t, recorder.String()) require.Contains(t, recorder.String(), message) } -func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCount() { +func (suite *ObserveProgressUnitSuite) TestObserve_progressWithCount() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() + ctx = SeedObserver(ctx, &recorder, config{}) header := "Header" - message := "Test Message" + message := uuid.NewString()[:8] count := 3 ch := ProgressWithCount(ctx, header, message, int64(count)) @@ -283,41 +210,37 @@ func (suite *ObserveProgressUnitSuite) TestObserveProgressWithCount() { ch <- struct{}{} } - Complete() + close(ch) - require.NotEmpty(t, recorder.String()) - require.Contains(t, recorder.String(), message) - require.Contains(t, recorder.String(), fmt.Sprintf("%d/%d", count, count)) + Flush(ctx) + + assert.NotEmpty(t, recorder.String()) + assert.Contains(t, recorder.String(), message) + assert.Contains(t, recorder.String(), fmt.Sprintf("%d/%d", count, count)) } -func (suite *ObserveProgressUnitSuite) TestrogressWithCountChannelClosed() { +func (suite *ObserveProgressUnitSuite) TestObserve_progressWithCountChannelClosed() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() recorder := strings.Builder{} - SeedWriter(ctx, &recorder, nil) - - defer func() { - // don't cross-contaminate other tests. - //nolint:forbidigo - SeedWriter(context.Background(), nil, nil) - }() + ctx = SeedObserver(ctx, &recorder, config{}) header := "Header" - message := "Test Message" + message := uuid.NewString()[:8] count := 3 ch := ProgressWithCount(ctx, header, message, int64(count)) close(ch) - Complete() + Flush(ctx) - require.NotEmpty(t, recorder.String()) - require.Contains(t, recorder.String(), message) - require.Contains(t, recorder.String(), fmt.Sprintf("%d/%d", 0, count)) + assert.NotEmpty(t, recorder.String()) + assert.Contains(t, recorder.String(), message) + assert.Contains(t, recorder.String(), fmt.Sprintf("%d/%d", 0, count)) } func (suite *ObserveProgressUnitSuite) TestListen() { diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index d53f15e08..ed50dd368 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -137,8 +137,6 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { ctx, end := diagnostics.Span(ctx, "operations:backup:run") defer func() { end() - // wait for the progress display to clean up - observe.Complete() }() ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx}) @@ -399,11 +397,8 @@ func produceBackupDataCollections( ctrlOpts control.Options, errs *fault.Bus, ) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) { - complete := observe.MessageWithCompletion(ctx, "Discovering items to backup") - defer func() { - complete <- struct{}{} - close(complete) - }() + progressBar := observe.MessageWithCompletion(ctx, "Discovering items to backup") + defer close(progressBar) bpc := inject.BackupProducerConfig{ LastBackupVersion: lastBackupVersion, @@ -464,11 +459,8 @@ func consumeBackupCollections( ) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) { ctx = clues.Add(ctx, "collection_source", "operations") - complete := observe.MessageWithCompletion(ctx, "Backing up data") - defer func() { - complete <- struct{}{} - close(complete) - }() + progressBar := observe.MessageWithCompletion(ctx, "Backing up data") + defer close(progressBar) tags := map[string]string{ kopia.TagBackupID: string(backupID), diff --git a/src/internal/operations/export.go b/src/internal/operations/export.go index 772ba44f2..09b284b4a 100644 --- a/src/internal/operations/export.go +++ b/src/internal/operations/export.go @@ -130,8 +130,6 @@ func (op *ExportOperation) Run(ctx context.Context) ( ctx, end := diagnostics.Span(ctx, "operations:export:run") defer func() { end() - // wait for the progress display to clean up - observe.Complete() }() ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx}) diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 141300f6a..1e6cc62a5 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -130,8 +130,6 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De ctx, end := diagnostics.Span(ctx, "operations:restore:run") defer func() { end() - // wait for the progress display to clean up - observe.Complete() }() ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx}) @@ -262,8 +260,8 @@ func (op *RestoreOperation) do( observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to restore", len(paths), op.BackupID)) - kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") - defer close(kopiaComplete) + progressBar := observe.MessageWithCompletion(ctx, "Enumerating items in repository") + defer close(progressBar) dcs, err := op.kopia.ProduceRestoreCollections( ctx, @@ -275,8 +273,6 @@ func (op *RestoreOperation) do( return nil, clues.Wrap(err, "producing collections to restore") } - kopiaComplete <- struct{}{} - ctx = clues.Add(ctx, "coll_count", len(dcs)) // should always be 1, since backups are 1:1 with resourceOwners. @@ -372,11 +368,8 @@ func consumeRestoreCollections( errs *fault.Bus, ctr *count.Bus, ) (*details.Details, error) { - complete := observe.MessageWithCompletion(ctx, "Restoring data") - defer func() { - complete <- struct{}{} - close(complete) - }() + progressBar := observe.MessageWithCompletion(ctx, "Restoring data") + defer close(progressBar) rcc := inject.RestoreConsumerConfig{ BackupVersion: backupVersion, diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index d4b0f6eaa..a604a5ac5 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -220,12 +220,8 @@ func Connect( } }() - // Close/Reset the progress bar. This ensures callers don't have to worry about - // their output getting clobbered (#1720) - defer observe.Complete() - - complete := observe.MessageWithCompletion(ctx, "Connecting to repository") - defer close(complete) + progressBar := observe.MessageWithCompletion(ctx, "Connecting to repository") + defer close(progressBar) kopiaRef := kopia.NewConn(s) if err := kopiaRef.Connect(ctx, opts.Repo); err != nil { @@ -264,8 +260,6 @@ func Connect( bus.SetRepoID(repoid) } - complete <- struct{}{} - // todo: ID and CreatedAt should get retrieved from a stored kopia config. return &repository{ ID: repoid, @@ -708,17 +702,20 @@ func newRepoID(s storage.Storage) string { // helpers // --------------------------------------------------------------------------- +var m365nonce bool + func connectToM365( ctx context.Context, pst path.ServiceType, acct account.Account, co control.Options, ) (*m365.Controller, error) { - complete := observe.MessageWithCompletion(ctx, "Connecting to M365") - defer func() { - complete <- struct{}{} - close(complete) - }() + if !m365nonce { + m365nonce = true + + progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365") + defer close(progressBar) + } // retrieve data from the producer rc := resource.Users