diff --git a/src/go.mod b/src/go.mod index ea773dbd1..bfde12c38 100644 --- a/src/go.mod +++ b/src/go.mod @@ -57,7 +57,6 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect diff --git a/src/go.sum b/src/go.sum index e45eb7c80..835728e4e 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,20 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1 h1:tz19qLF65vuu2ibfTqGVJxG/zZAI27NEIIbvAOQwYbw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 h1:8LoU8N2lIUzkmstvwXvVfniMZlFbesfT2AmA1aqvRr8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 h1:Yoicul8bnVdQrhDMTHxdEckRGX01XvwXDHUT9zYZ3k0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -79,8 +71,6 @@ github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dg github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cjlapao/common-go v0.0.25 h1:/a5SdWGtOFzXceM1RnB3v4hgUK8woMo9Ma/XRgCQ87g= -github.com/cjlapao/common-go v0.0.25/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g= github.com/cjlapao/common-go v0.0.27 h1:7k8R1Mz2LAudnPb1kaqQ/l+Ba7uL92FG7Rqp9W67mGM= github.com/cjlapao/common-go v0.0.27/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -129,10 +119,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -192,7 +178,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -268,32 +253,18 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v0.8.2 h1:KxHr72YwlntvVgF0C+8CtNmNQajtIq7fedq2N0+74Uk= -github.com/microsoft/kiota-abstractions-go v0.8.2/go.mod h1:i1wK2rdsLGSjgpnpLHjC60nEa/UdCVS6ulCh1Q+1COw= github.com/microsoft/kiota-abstractions-go v0.10.1 h1:R4vk1l/wn0Dij01yxVX5lD/1UKMEO9Hz5TXkBl6E8do= github.com/microsoft/kiota-abstractions-go v0.10.1/go.mod h1:wOO+hpReDIJa3BxbLNz4qvtZk2llS555mejkMc203bQ= -github.com/microsoft/kiota-authentication-azure-go v0.3.0 h1:iLyy5qldAjBiYMGMk1r/rJkcmARA8cKboiN7/XbRxv4= -github.com/microsoft/kiota-authentication-azure-go v0.3.0/go.mod h1:qyZWSCug2eG1zrRnCSacyFHGsgQa4aSCWn3EOkY9Z1M= github.com/microsoft/kiota-authentication-azure-go v0.4.1 h1:C+n4Vp3oCj8W8LPiKZDyASTNaKyUomQtUiXQ+B2W0TU= github.com/microsoft/kiota-authentication-azure-go v0.4.1/go.mod h1:jIJAhpPh34bDQWNME65kd/yjqY6+CJZi5jus8H9EH4s= -github.com/microsoft/kiota-http-go v0.6.0 h1:pNdkRDLGBMuFryS5XvHUGzCsteAg2a4XMdsG8b7YQJs= -github.com/microsoft/kiota-http-go v0.6.0/go.mod h1:4N7GGz5qCZ5JCsEpMyRmGecRckp2evUYRLetIvPBuYs= github.com/microsoft/kiota-http-go v0.7.2 h1:R40vG0EkIFqGvVz5dZtLe4g1sXGVfBO5HxjdBjpwv8k= github.com/microsoft/kiota-http-go v0.7.2/go.mod h1:QTbXPh25mJsbxE23bFqw64BckCioCGfaE77hF/F3rIQ= -github.com/microsoft/kiota-serialization-json-go v0.5.5 h1:B0iKBKOdi+9NKFlormLRqduQ1+77MPGRsZ7xnd74EqQ= -github.com/microsoft/kiota-serialization-json-go v0.5.5/go.mod h1:GI9vrssO1EvqzDtvMKuhjALn40phZOWkeeaMgtCk6xE= github.com/microsoft/kiota-serialization-json-go v0.6.0 h1:irdhbaY2Vl9t2SLwIC5WyPKJRp3mE52GEBEbAyu7thk= github.com/microsoft/kiota-serialization-json-go v0.6.0/go.mod h1:ceR++Qc8n6McdAR+Ili2UhV4iR8CEx3+RPtANi1UdXc= -github.com/microsoft/kiota-serialization-text-go v0.4.1 h1:6QPH7+geUPCpaSZkKCQw0Scngx2IF0vKodrvvWWiu2A= -github.com/microsoft/kiota-serialization-text-go v0.4.1/go.mod h1:DsriFnVBDCc4D84qxG3j8q/1Sxu16JILfhxMZm3kdfw= github.com/microsoft/kiota-serialization-text-go v0.5.0 h1:TWb9Y6IsIwzsMVcbBBDLFpVg47mRu2FhQJ6i1dqpLOs= github.com/microsoft/kiota-serialization-text-go v0.5.0/go.mod h1:x9h+VE4X4t8njowIZXyTaAzE6bGK8Zr90MLYV6J6S9U= -github.com/microsoftgraph/msgraph-sdk-go v0.34.0 h1:AXPTyCUKaxy4i0qSLBuUbaZTw4thoZTMS2i8KWCItlo= -github.com/microsoftgraph/msgraph-sdk-go v0.34.0/go.mod h1:5KCKGk0dKyEK17M40vCHTnyaayUe/SqIOh9aww0ECpU= github.com/microsoftgraph/msgraph-sdk-go v0.40.0 h1:9AxA3FS+S3c7Him5C+7Lt0I8zaNXoSqXsLDink1Fg40= github.com/microsoftgraph/msgraph-sdk-go v0.40.0/go.mod h1:B8HORKdf1K05Z93FbkpiqJ25dnytjPEyAby6gHhOLiM= -github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0 h1:FqKddh8nTTbNyUtRCCmPKtgxSNGz944Kw8Q+WU/H+lo= -github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0/go.mod h1:kcTY0sEZ/LOJiSj/1OMxcs0T51uodJ/bOeVfWo4lo/s= github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1 h1:gN3pVVvx50KzhQdYwQKVnRSOSOIRaFpXj7pgPfqnNXw= github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1/go.mod h1:BnumnwWU8xUgX7ncgo68novbS1wMlO66Iny9iVhvHuM= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -311,7 +282,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= @@ -322,8 +292,6 @@ github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -452,8 +420,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -530,8 +496,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7 h1:DTGA3sVb/sQX+3poldfq5cO4KiOPwLSRBjn2rtck5RM= golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -610,7 +574,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 93da8cc00..76fcb46cf 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -34,26 +35,38 @@ var ( ) type BackupStats struct { - SnapshotID string + SnapshotID string + + TotalHashedBytes int64 + TotalUploadedBytes int64 + TotalFileCount int - TotalHashedBytes int64 TotalDirectoryCount int IgnoredErrorCount int ErrorCount int - Incomplete bool - IncompleteReason string + + Incomplete bool + IncompleteReason string } -func manifestToStats(man *snapshot.Manifest, progress *corsoProgress) BackupStats { +func manifestToStats( + man *snapshot.Manifest, + progress *corsoProgress, + uploadCount *stats.ByteCounter, +) BackupStats { return BackupStats{ - SnapshotID: string(man.ID), + SnapshotID: string(man.ID), + + TotalHashedBytes: progress.totalBytes, + TotalUploadedBytes: uploadCount.NumBytes, + TotalFileCount: int(man.Stats.TotalFileCount), - TotalHashedBytes: progress.totalBytes, TotalDirectoryCount: int(man.Stats.TotalDirectoryCount), IgnoredErrorCount: int(man.Stats.IgnoredErrorCount), ErrorCount: int(man.Stats.ErrorCount), - Incomplete: man.IncompleteReason != "", - IncompleteReason: man.IncompleteReason, + + Incomplete: man.IncompleteReason != "", + IncompleteReason: man.IncompleteReason, } } @@ -380,12 +393,12 @@ func (w Wrapper) BackupCollections( return nil, nil, errors.Wrap(err, "building kopia directories") } - stats, err := w.makeSnapshotWithRoot(ctx, dirTree, progress) + s, err := w.makeSnapshotWithRoot(ctx, dirTree, progress) if err != nil { return nil, nil, err } - return stats, progress.deets, nil + return s, progress.deets, nil } func (w Wrapper) makeSnapshotWithRoot( @@ -395,6 +408,8 @@ func (w Wrapper) makeSnapshotWithRoot( ) (*BackupStats, error) { var man *snapshot.Manifest + bc := &stats.ByteCounter{} + err := repo.WriteSession( ctx, w.c, @@ -403,6 +418,7 @@ func (w Wrapper) makeSnapshotWithRoot( // Always flush so we don't leak write sessions. Still uses reachability // for consistency. FlushOnFailure: true, + OnUpload: bc.Count, }, func(innerCtx context.Context, rw repo.RepositoryWriter) error { si := snapshot.SourceInfo{ @@ -453,7 +469,7 @@ func (w Wrapper) makeSnapshotWithRoot( return nil, errors.Wrap(err, "kopia backup") } - res := manifestToStats(man, progress) + res := manifestToStats(man, progress, bc) return &res, nil } @@ -481,6 +497,7 @@ func getItemStream( ctx context.Context, itemPath path.Path, snapshotRoot fs.Entry, + bcounter byteCounter, ) (data.Stream, error) { if itemPath == nil { return nil, errors.WithStack(errNoRestorePath) @@ -501,6 +518,10 @@ func getItemStream( return nil, errors.New("requested object is not a file") } + if bcounter != nil { + bcounter.Count(f.Size()) + } + r, err := f.Open(ctx) if err != nil { return nil, errors.Wrap(err, "opening file") @@ -517,6 +538,10 @@ func getItemStream( }, nil } +type byteCounter interface { + Count(numBytes int64) +} + // RestoreMultipleItems looks up all paths- assuming each is an item declaration, // not a directory- in the snapshot with id snapshotID. The path should be the // full path of the item from the root. Returns the results as a slice of single- @@ -528,6 +553,7 @@ func (w Wrapper) RestoreMultipleItems( ctx context.Context, snapshotID string, paths []path.Path, + bcounter byteCounter, ) ([]data.Collection, error) { if len(paths) == 0 { return nil, errors.WithStack(errNoRestorePath) @@ -545,7 +571,7 @@ func (w Wrapper) RestoreMultipleItems( ) for _, itemPath := range paths { - ds, err := getItemStream(ctx, itemPath, snapshotRoot) + ds, err := getItemStream(ctx, itemPath, snapshotRoot, bcounter) if err != nil { errs = multierror.Append(errs, err) continue diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 4965eb14a..42fdc0b99 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -709,7 +709,8 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { []path.Path{ fp1, fp2, - }) + }, + nil) require.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -926,6 +927,14 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TearDownTest() { assert.NoError(suite.T(), suite.w.Close(suite.ctx)) } +type i64counter struct { + i int64 +} + +func (c *i64counter) Count(i int64) { + c.i += i +} + func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { doesntExist, err := path.Builder{}.Append("subdir", "foo").ToDataLayerExchangePathForCategory( testTenant, @@ -1008,14 +1017,17 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { expected[pth.String()] = item.data } + ic := i64counter{} + result, err := suite.w.RestoreMultipleItems( suite.ctx, string(suite.snapshotID), test.inputPaths, - ) + &ic) test.expectedErr(t, err) assert.Len(t, result, test.expectedCollections) + assert.Less(t, int64(0), ic.i) testForFiles(t, expected, result) }) } @@ -1053,7 +1065,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors() suite.ctx, test.snapshotID, test.paths, - ) + nil) assert.Error(t, err) assert.Empty(t, c) }) @@ -1067,13 +1079,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() { // assert the deletion worked itemPath := suite.files[suite.testPath1.String()][0].itemPath + ic := i64counter{} + c, err := suite.w.RestoreMultipleItems( suite.ctx, string(suite.snapshotID), []path.Path{itemPath}, - ) + &ic) assert.Error(t, err, "snapshot should be deleted") assert.Empty(t, c) + assert.Zero(t, ic.i) } func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() { diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index d3d397af7..448198597 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -35,6 +35,7 @@ type BackupOperation struct { // BackupResults aggregate the details of the result of the operation. type BackupResults struct { + stats.Errs stats.ReadWrites stats.StartAndEndTime BackupID model.StableID `json:"backupID"` @@ -167,7 +168,9 @@ func (op *BackupOperation) persistResults( op.Results.ReadErrors = opStats.readErr op.Results.WriteErrors = opStats.writeErr - op.Results.BytesWritten = opStats.k.TotalHashedBytes + + op.Results.BytesRead = opStats.k.TotalHashedBytes + op.Results.BytesUploaded = opStats.k.TotalUploadedBytes op.Results.ItemsRead = opStats.gc.Successful op.Results.ItemsWritten = opStats.k.TotalFileCount op.Results.ResourceOwners = opStats.resourceCount @@ -214,7 +217,7 @@ func (op *BackupOperation) createBackupModels( events.StartTime: op.Results.StartedAt, events.EndTime: op.Results.CompletedAt, events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), - events.DataStored: op.Results.BytesWritten, + events.DataStored: op.Results.BytesUploaded, events.Resources: op.Results.ResourceOwners, // TODO: events.ExchangeDataObserved: , }, diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 4fe152a17..70092b833 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -48,8 +48,9 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { writeErr: assert.AnError, resourceCount: 1, k: &kopia.BackupStats{ - TotalFileCount: 1, - TotalHashedBytes: 1, + TotalFileCount: 1, + TotalHashedBytes: 1, + TotalUploadedBytes: 1, }, gc: &support.ConnectorOperationStatus{ Successful: 1, @@ -73,7 +74,8 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { assert.Equal(t, op.Results.ItemsRead, stats.gc.Successful, "items read") assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors") assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount, "items written") - assert.Equal(t, op.Results.BytesWritten, stats.k.TotalHashedBytes, "bytes written") + assert.Equal(t, stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read") + assert.Equal(t, stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written") assert.Equal(t, op.Results.ResourceOwners, stats.resourceCount, "resource owners") assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors") assert.Equal(t, op.Results.StartedAt, now, "started at") @@ -218,7 +220,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { assert.Equal(t, bo.Status, Completed) assert.Less(t, 0, bo.Results.ItemsRead) assert.Less(t, 0, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesWritten) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners) assert.Zero(t, bo.Results.ReadErrors) assert.Zero(t, bo.Results.WriteErrors) @@ -277,7 +280,8 @@ func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() { require.NotEmpty(t, bo.Results.BackupID) assert.Equal(t, bo.Status, Completed) assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesWritten) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners) assert.NoError(t, bo.Results.ReadErrors) assert.NoError(t, bo.Results.WriteErrors) diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index a3a53332b..fc4281b6d 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -37,6 +37,7 @@ type RestoreOperation struct { // RestoreResults aggregate the details of the results of the operation. type RestoreResults struct { + stats.Errs stats.ReadWrites stats.StartAndEndTime } @@ -77,6 +78,7 @@ func (op RestoreOperation) validate() error { type restoreStats struct { cs []data.Collection gc *support.ConnectorOperationStatus + bytesRead *stats.ByteCounter resourceCount int started bool readErr, writeErr error @@ -87,7 +89,9 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { startTime := time.Now() // persist operation results to the model store on exit - opStats := restoreStats{} + opStats := restoreStats{ + bytesRead: &stats.ByteCounter{}, + } // TODO: persist results? defer func() { @@ -160,7 +164,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { paths[i] = p } - dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths) + dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths, opStats.bytesRead) if err != nil { err = errors.Wrap(err, "retrieving service data") @@ -193,6 +197,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { opStats.started = true opStats.gc = gc.AwaitStatus() + logger.Ctx(ctx).Debug(gc.PrintableStatus()) return nil @@ -220,6 +225,8 @@ func (op *RestoreOperation) persistResults( op.Results.ReadErrors = opStats.readErr op.Results.WriteErrors = opStats.writeErr + + op.Results.BytesRead = opStats.bytesRead.NumBytes op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count op.Results.ItemsWritten = opStats.gc.Successful op.Results.ResourceOwners = opStats.resourceCount @@ -229,16 +236,16 @@ func (op *RestoreOperation) persistResults( events.RestoreEnd, map[string]any{ // TODO: RestoreID - events.BackupID: op.BackupID, - events.Service: op.Selectors.Service.String(), - events.Status: op.Status, - events.StartTime: op.Results.StartedAt, - events.EndTime: op.Results.CompletedAt, - events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), - events.ItemsRead: op.Results.ItemsRead, - events.ItemsWritten: op.Results.ItemsWritten, - events.Resources: op.Results.ResourceOwners, - // TODO: events.ExchangeDataObserved: , + events.BackupID: op.BackupID, + events.Service: op.Selectors.Service.String(), + events.Status: op.Status, + events.StartTime: op.Results.StartedAt, + events.EndTime: op.Results.CompletedAt, + events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), + events.ItemsRead: op.Results.ItemsRead, + events.ItemsWritten: op.Results.ItemsWritten, + events.Resources: op.Results.ResourceOwners, + events.DataRetrieved: op.Results.BytesRead, }, ) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 54d6a3d1c..056996d95 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -17,6 +17,7 @@ import ( evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -43,15 +44,19 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { ctx := context.Background() var ( - kw = &kopia.Wrapper{} - sw = &store.Wrapper{} - acct = account.Account{} - now = time.Now() - stats = restoreStats{ - started: true, - readErr: multierror.Append(nil, assert.AnError), - writeErr: assert.AnError, - cs: []data.Collection{&exchange.Collection{}}, + kw = &kopia.Wrapper{} + sw = &store.Wrapper{} + acct = account.Account{} + now = time.Now() + rs = restoreStats{ + started: true, + readErr: multierror.Append(nil, assert.AnError), + writeErr: assert.AnError, + resourceCount: 1, + bytesRead: &stats.ByteCounter{ + NumBytes: 42, + }, + cs: []data.Collection{&exchange.Collection{}}, gc: &support.ConnectorOperationStatus{ ObjectCount: 1, }, @@ -69,14 +74,15 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { evmock.NewBus()) require.NoError(t, err) - require.NoError(t, op.persistResults(ctx, now, &stats)) + require.NoError(t, op.persistResults(ctx, now, &rs)) assert.Equal(t, op.Status.String(), Completed.String(), "status") - assert.Equal(t, op.Results.ItemsRead, len(stats.cs), "items read") - assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors") - assert.Equal(t, op.Results.ItemsWritten, stats.gc.Successful, "items written") - assert.Equal(t, 0, op.Results.ResourceOwners, "resource owners") - assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors") + assert.Equal(t, op.Results.ItemsRead, len(rs.cs), "items read") + assert.Equal(t, op.Results.ReadErrors, rs.readErr, "read errors") + assert.Equal(t, op.Results.ItemsWritten, rs.gc.Successful, "items written") + assert.Equal(t, rs.bytesRead.NumBytes, op.Results.BytesRead, "resource owners") + assert.Equal(t, rs.resourceCount, op.Results.ResourceOwners, "resource owners") + assert.Equal(t, op.Results.WriteErrors, rs.writeErr, "write errors") assert.Equal(t, op.Results.StartedAt, now, "started at") assert.Less(t, now, op.Results.CompletedAt, "completed at") } @@ -231,9 +237,10 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { require.NoError(t, ro.Run(ctx), "restoreOp.Run()") require.NotEmpty(t, ro.Results, "restoreOp results") assert.Equal(t, ro.Status, Completed, "restoreOp status") - assert.Greater(t, ro.Results.ItemsRead, 0, "restore items read") - assert.Greater(t, ro.Results.ItemsWritten, 0, "restored items written") - assert.Equal(t, 1, ro.Results.ResourceOwners) + assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") + assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written") + assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") + assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners") assert.Zero(t, ro.Results.ReadErrors, "errors while reading restore data") assert.Zero(t, ro.Results.WriteErrors, "errors while writing restore data") assert.Equal(t, suite.numItems, ro.Results.ItemsWritten, "backup and restore wrote the same num of items") @@ -261,7 +268,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() { mb) require.NoError(t, err) require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results") - assert.Equal(t, 0, ro.Results.ResourceOwners) + assert.Zero(t, ro.Results.ResourceOwners, "resource owners") + assert.Zero(t, ro.Results.BytesRead, "bytes read") assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") - assert.Equal(t, 0, mb.TimesCalled[events.RestoreEnd], "restore-end events") + assert.Zero(t, mb.TimesCalled[events.RestoreEnd], "restore-end events") } diff --git a/src/internal/stats/stats.go b/src/internal/stats/stats.go index 3d8eced1c..aacb8c850 100644 --- a/src/internal/stats/stats.go +++ b/src/internal/stats/stats.go @@ -1,22 +1,36 @@ package stats -import "time" +import ( + "sync/atomic" + "time" +) -// ReadWrites tracks the total count of reads and writes, and of -// read and write errors. ItemsRead and ItemsWritten counts are -// assumed to be successful, so the total count of items involved -// would be ItemsRead+ReadErrors. +// ReadWrites tracks the total count of reads and writes. ItemsRead +// and ItemsWritten counts are assumed to be successful reads. type ReadWrites struct { - BytesWritten int64 `json:"bytesWritten,omitempty"` + BytesRead int64 `json:"bytesRead,omitempty"` + BytesUploaded int64 `json:"bytesUploaded,omitempty"` ItemsRead int `json:"itemsRead,omitempty"` ItemsWritten int `json:"itemsWritten,omitempty"` - ReadErrors error `json:"readErrors,omitempty"` - WriteErrors error `json:"writeErrors,omitempty"` ResourceOwners int `json:"resourceOwners,omitempty"` } +// Errs tracks the aggregation of errors that occurred during a process. +type Errs struct { + ReadErrors error `json:"readErrors,omitempty"` + WriteErrors error `json:"writeErrors,omitempty"` +} + // StartAndEndTime tracks a paired starting time and ending time. type StartAndEndTime struct { StartedAt time.Time `json:"startedAt"` CompletedAt time.Time `json:"completedAt"` } + +type ByteCounter struct { + NumBytes int64 +} + +func (bc *ByteCounter) Count(i int64) { + atomic.AddInt64(&bc.NumBytes, i) +} diff --git a/src/pkg/backup/backup.go b/src/pkg/backup/backup.go index 430faf44e..b60c35b4e 100644 --- a/src/pkg/backup/backup.go +++ b/src/pkg/backup/backup.go @@ -32,6 +32,7 @@ type Backup struct { Selectors selectors.Selector `json:"selectors"` // stats are embedded so that the values appear as top-level properties + stats.Errs stats.ReadWrites stats.StartAndEndTime } diff --git a/src/pkg/backup/backup_test.go b/src/pkg/backup/backup_test.go index a295bb299..2edd0cb4d 100644 --- a/src/pkg/backup/backup_test.go +++ b/src/pkg/backup/backup_test.go @@ -37,11 +37,13 @@ func stubBackup(t time.Time) backup.Backup { DetailsID: "details", Status: "status", Selectors: sel.Selector, + Errs: stats.Errs{ + ReadErrors: errors.New("1"), + WriteErrors: errors.New("1"), + }, ReadWrites: stats.ReadWrites{ ItemsRead: 1, ItemsWritten: 1, - ReadErrors: errors.New("1"), - WriteErrors: errors.New("1"), }, StartAndEndTime: stats.StartAndEndTime{ StartedAt: t,