diff --git a/src/cli/cli.go b/src/cli/cli.go index 932b2cb72..c34722de2 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -89,7 +89,7 @@ func BuildCommandTree(cmd *cobra.Command) { func Handle() { ctx := config.Seed(context.Background()) ctx = print.SetRootCmd(ctx, corsoCmd) - observe.SeedWriter(print.StderrWriter(ctx)) + observe.SeedWriter(ctx, print.StderrWriter(ctx)) BuildCommandTree(corsoCmd) diff --git a/src/go.mod b/src/go.mod index 68ec4679c..7b96ba6fb 100644 --- a/src/go.mod +++ b/src/go.mod @@ -16,23 +16,24 @@ require ( github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1 github.com/pkg/errors v0.9.1 github.com/rudderlabs/analytics-go v3.3.3+incompatible - github.com/schollz/progressbar/v3 v3.11.0 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 github.com/stretchr/testify v1.8.0 github.com/tidwall/pretty v1.2.0 github.com/tomlazar/table v0.1.2 + github.com/vbauerster/mpb/v8 v8.1.2 go.uber.org/zap v1.23.0 golang.org/x/tools v0.1.12 gopkg.in/resty.v1 v1.12.0 ) require ( + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect @@ -40,7 +41,6 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) @@ -73,7 +73,7 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/microsoft/kiota-serialization-text-go v0.6.0 // indirect diff --git a/src/go.sum b/src/go.sum index 9f006cd20..e72baa54b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -46,6 +46,10 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -205,7 +209,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= @@ -239,8 +242,9 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -266,8 +270,6 @@ github.com/minio/minio-go/v7 v7.0.39 h1:upnbu1jCGOqEvrGSpRauSN9ZG7RCHK7VHxXS8Vmg github.com/minio/minio-go/v7 v7.0.39/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= -github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -330,8 +332,6 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU= -github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -372,6 +372,8 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tomlazar/table v0.1.2 h1:DP8f62FzZAZk8oavepm1v/oyf4ni3/LMHWNlOinmleg= github.com/tomlazar/table v0.1.2/go.mod h1:IecZnpep9f/BatHacfh+++ftE+lFONN8BVPi9nx5U1w= +github.com/vbauerster/mpb/v8 v8.1.2 h1:plrnwcqEj+dNce8q1zz0lU9x6DYnjDIdDhqbSqRU7cc= +github.com/vbauerster/mpb/v8 v8.1.2/go.mod h1:qn8FMJ7ypuwaDVkgA3rR1ooRzGsneXBjdskm6alegYo= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= @@ -562,13 +564,10 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index db758c91d..dd2c974ec 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -139,7 +139,9 @@ func (oc *Collection) populateItems(ctx context.Context) { byteCount += itemInfo.Size itemInfo.ParentPath = parentPathString - progReader := observe.ItemProgress(itemData, itemInfo.ItemName, itemInfo.Size) + progReader, closer := observe.ItemProgress(itemData, itemInfo.ItemName, itemInfo.Size) + + go closer() oc.data <- &Item{ id: itemInfo.ItemName, diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index 5d93fe774..55c504b10 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -1,49 +1,78 @@ package observe import ( + "context" "io" + "sync" - "github.com/schollz/progressbar/v3" + "github.com/vbauerster/mpb/v8" + "github.com/vbauerster/mpb/v8/decor" ) -var writer io.Writer +var ( + wg sync.WaitGroup + con context.Context + writer io.Writer + progress *mpb.Progress +) // SeedWriter adds default writer to the observe package. // Uses a noop writer until seeded. -func SeedWriter(w io.Writer) { +func SeedWriter(ctx context.Context, w io.Writer) { writer = w + con = ctx + + if con == nil { + con = context.Background() + } + + progress = mpb.NewWithContext( + con, + mpb.WithWidth(32), + mpb.WithWaitGroup(&wg), + mpb.WithOutput(writer), + ) +} + +// Complete blocks until the progress finishes writing out all data. +// Afterwards, the progress instance is reset. +func Complete() { + if progress != nil { + progress.Wait() + } + + SeedWriter(con, writer) } // ItemProgress tracks the display of an item by counting the bytes // read through the provided readcloser, up until the byte count matches // the totalBytes. -func ItemProgress(rc io.ReadCloser, iname string, totalBytes int64) io.ReadCloser { - if writer == nil { - return rc +func ItemProgress(rc io.ReadCloser, iname string, totalBytes int64) (io.ReadCloser, func()) { + if writer == nil || rc == nil || totalBytes == 0 { + return rc, func() {} } - opts := progressbar.NewOptions( - int(totalBytes), - progressbar.OptionSetWriter(writer), - progressbar.OptionClearOnFinish(), - progressbar.OptionSetDescription(" | "+iname), - progressbar.OptionShowDescriptionAtLineEnd(), - progressbar.OptionSetRenderBlankState(true), - progressbar.OptionShowCount(), - progressbar.OptionSetPredictTime(false), - progressbar.OptionEnableColorCodes(true), - progressbar.OptionShowBytes(true), - progressbar.OptionSetWidth(20), - progressbar.OptionSetTheme(progressbar.Theme{ - Saucer: "[green]=[reset]", - SaucerHead: "[green]>[reset]", - SaucerPadding: " ", - BarStart: "[", - BarEnd: "]", - }), + wg.Add(1) + + bar := progress.AddBar( + totalBytes, + mpb.BarFillerOnComplete(""), + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators( + decor.OnComplete(decor.NewPercentage("%d", decor.WC{W: 4}), ""), + decor.OnComplete(decor.TotalKiloByte("%.1f", decor.WCSyncSpace), ""), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.Name(iname), ""), + ), ) - pbr := progressbar.NewReader(rc, opts) - - return &pbr + return bar.ProxyReader(rc), waitAndCloseBar(iname, bar) +} + +func waitAndCloseBar(n string, bar *mpb.Bar) func() { + return func() { + bar.Wait() + wg.Done() + } } diff --git a/src/internal/observe/observe_test.go b/src/internal/observe/observe_test.go index c7b7c7960..1d9881fa7 100644 --- a/src/internal/observe/observe_test.go +++ b/src/internal/observe/observe_test.go @@ -2,6 +2,7 @@ package observe_test import ( "bytes" + "context" "errors" "io" "strings" @@ -12,6 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/tester" ) type ObserveProgressUnitSuite struct { @@ -23,17 +25,31 @@ func TestObserveProgressUnitSuite(t *testing.T) { } func (suite *ObserveProgressUnitSuite) TestDoesThings() { + ctx, flush := tester.NewContext() + defer flush() + t := suite.T() recorder := strings.Builder{} - observe.SeedWriter(&recorder) + observe.SeedWriter(ctx, &recorder) + + defer func() { + // don't cross-contaminate other tests. + observe.Complete() + observe.SeedWriter(context.Background(), nil) + }() from := make([]byte, 100) - prog := observe.ItemProgress( + prog, closer := observe.ItemProgress( io.NopCloser(bytes.NewReader(from)), "test", 100) require.NotNil(t, prog) + require.NotNil(t, closer) + + defer closer() + + var i int for { to := make([]byte, 25) @@ -44,11 +60,16 @@ func (suite *ObserveProgressUnitSuite) TestDoesThings() { } assert.NoError(t, err) - assert.Less(t, 0, n) + assert.Equal(t, 25, n) + i++ } - recorded := recorder.String() - assert.Contains(t, recorded, "25%") - assert.Contains(t, recorded, "50%") - assert.Contains(t, recorded, "75%") + // mpb doesn't transmit any written values to the output writer until + // bar completion. Since we clean up after the bars, the recorder + // traces nothing. + // recorded := recorder.String() + // assert.Contains(t, recorded, "25%") + // assert.Contains(t, recorded, "50%") + // assert.Contains(t, recorded, "75%") + assert.Equal(t, 4, i) } diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 118a609a3..0b647e547 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup" @@ -106,6 +107,9 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { // persist operation results to the model store on exit defer func() { + // wait for the progress display to clean up + observe.Complete() + err = op.persistResults(startTime, &opStats) if err != nil { return