From 38addfaae8668ef3df2972869f8fe16abbd5f2dd Mon Sep 17 00:00:00 2001 From: Keepers Date: Mon, 26 Sep 2022 13:18:42 -0600 Subject: [PATCH] track bytes written, read in kopia (#944) ## Description Track the count of bytes read and written in kopia. For backups, this means the count of bytes fed into kopia (hashed bytes), and the amount written after compression and dedupe (total file bytes). For restore, this is the count of bytes in all files read. ## Type of change - [x] :sunflower: Feature ## Issue(s) * #894 ## Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/go.mod | 1 - src/go.sum | 37 ------------------ src/internal/kopia/wrapper.go | 52 ++++++++++++++++++------- src/internal/kopia/wrapper_test.go | 23 +++++++++-- src/internal/operations/backup.go | 7 +++- src/internal/operations/backup_test.go | 14 ++++--- src/internal/operations/restore.go | 31 +++++++++------ src/internal/operations/restore_test.go | 48 +++++++++++++---------- src/internal/stats/stats.go | 30 ++++++++++---- src/pkg/backup/backup.go | 1 + src/pkg/backup/backup_test.go | 6 ++- 11 files changed, 146 insertions(+), 104 deletions(-) diff --git a/src/go.mod b/src/go.mod index ea773dbd1..bfde12c38 100644 --- a/src/go.mod +++ b/src/go.mod @@ -57,7 +57,6 @@ require ( github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect diff --git a/src/go.sum b/src/go.sum index e45eb7c80..835728e4e 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,20 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1 h1:tz19qLF65vuu2ibfTqGVJxG/zZAI27NEIIbvAOQwYbw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 h1:8LoU8N2lIUzkmstvwXvVfniMZlFbesfT2AmA1aqvRr8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 h1:Yoicul8bnVdQrhDMTHxdEckRGX01XvwXDHUT9zYZ3k0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -79,8 +71,6 @@ github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dg github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cjlapao/common-go v0.0.25 h1:/a5SdWGtOFzXceM1RnB3v4hgUK8woMo9Ma/XRgCQ87g= -github.com/cjlapao/common-go v0.0.25/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g= github.com/cjlapao/common-go v0.0.27 h1:7k8R1Mz2LAudnPb1kaqQ/l+Ba7uL92FG7Rqp9W67mGM= github.com/cjlapao/common-go v0.0.27/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -129,10 +119,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -192,7 +178,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -268,32 +253,18 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v0.8.2 h1:KxHr72YwlntvVgF0C+8CtNmNQajtIq7fedq2N0+74Uk= -github.com/microsoft/kiota-abstractions-go v0.8.2/go.mod h1:i1wK2rdsLGSjgpnpLHjC60nEa/UdCVS6ulCh1Q+1COw= github.com/microsoft/kiota-abstractions-go v0.10.1 h1:R4vk1l/wn0Dij01yxVX5lD/1UKMEO9Hz5TXkBl6E8do= github.com/microsoft/kiota-abstractions-go v0.10.1/go.mod h1:wOO+hpReDIJa3BxbLNz4qvtZk2llS555mejkMc203bQ= -github.com/microsoft/kiota-authentication-azure-go v0.3.0 h1:iLyy5qldAjBiYMGMk1r/rJkcmARA8cKboiN7/XbRxv4= -github.com/microsoft/kiota-authentication-azure-go v0.3.0/go.mod h1:qyZWSCug2eG1zrRnCSacyFHGsgQa4aSCWn3EOkY9Z1M= github.com/microsoft/kiota-authentication-azure-go v0.4.1 h1:C+n4Vp3oCj8W8LPiKZDyASTNaKyUomQtUiXQ+B2W0TU= github.com/microsoft/kiota-authentication-azure-go v0.4.1/go.mod h1:jIJAhpPh34bDQWNME65kd/yjqY6+CJZi5jus8H9EH4s= -github.com/microsoft/kiota-http-go v0.6.0 h1:pNdkRDLGBMuFryS5XvHUGzCsteAg2a4XMdsG8b7YQJs= -github.com/microsoft/kiota-http-go v0.6.0/go.mod h1:4N7GGz5qCZ5JCsEpMyRmGecRckp2evUYRLetIvPBuYs= github.com/microsoft/kiota-http-go v0.7.2 h1:R40vG0EkIFqGvVz5dZtLe4g1sXGVfBO5HxjdBjpwv8k= github.com/microsoft/kiota-http-go v0.7.2/go.mod h1:QTbXPh25mJsbxE23bFqw64BckCioCGfaE77hF/F3rIQ= -github.com/microsoft/kiota-serialization-json-go v0.5.5 h1:B0iKBKOdi+9NKFlormLRqduQ1+77MPGRsZ7xnd74EqQ= -github.com/microsoft/kiota-serialization-json-go v0.5.5/go.mod h1:GI9vrssO1EvqzDtvMKuhjALn40phZOWkeeaMgtCk6xE= github.com/microsoft/kiota-serialization-json-go v0.6.0 h1:irdhbaY2Vl9t2SLwIC5WyPKJRp3mE52GEBEbAyu7thk= github.com/microsoft/kiota-serialization-json-go v0.6.0/go.mod h1:ceR++Qc8n6McdAR+Ili2UhV4iR8CEx3+RPtANi1UdXc= -github.com/microsoft/kiota-serialization-text-go v0.4.1 h1:6QPH7+geUPCpaSZkKCQw0Scngx2IF0vKodrvvWWiu2A= -github.com/microsoft/kiota-serialization-text-go v0.4.1/go.mod h1:DsriFnVBDCc4D84qxG3j8q/1Sxu16JILfhxMZm3kdfw= github.com/microsoft/kiota-serialization-text-go v0.5.0 h1:TWb9Y6IsIwzsMVcbBBDLFpVg47mRu2FhQJ6i1dqpLOs= github.com/microsoft/kiota-serialization-text-go v0.5.0/go.mod h1:x9h+VE4X4t8njowIZXyTaAzE6bGK8Zr90MLYV6J6S9U= -github.com/microsoftgraph/msgraph-sdk-go v0.34.0 h1:AXPTyCUKaxy4i0qSLBuUbaZTw4thoZTMS2i8KWCItlo= -github.com/microsoftgraph/msgraph-sdk-go v0.34.0/go.mod h1:5KCKGk0dKyEK17M40vCHTnyaayUe/SqIOh9aww0ECpU= github.com/microsoftgraph/msgraph-sdk-go v0.40.0 h1:9AxA3FS+S3c7Him5C+7Lt0I8zaNXoSqXsLDink1Fg40= github.com/microsoftgraph/msgraph-sdk-go v0.40.0/go.mod h1:B8HORKdf1K05Z93FbkpiqJ25dnytjPEyAby6gHhOLiM= -github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0 h1:FqKddh8nTTbNyUtRCCmPKtgxSNGz944Kw8Q+WU/H+lo= -github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0/go.mod h1:kcTY0sEZ/LOJiSj/1OMxcs0T51uodJ/bOeVfWo4lo/s= github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1 h1:gN3pVVvx50KzhQdYwQKVnRSOSOIRaFpXj7pgPfqnNXw= github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1/go.mod h1:BnumnwWU8xUgX7ncgo68novbS1wMlO66Iny9iVhvHuM= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -311,7 +282,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= @@ -322,8 +292,6 @@ github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -452,8 +420,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY= golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -530,8 +496,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7 h1:DTGA3sVb/sQX+3poldfq5cO4KiOPwLSRBjn2rtck5RM= golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -610,7 +574,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 93da8cc00..76fcb46cf 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -16,6 +16,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -34,26 +35,38 @@ var ( ) type BackupStats struct { - SnapshotID string + SnapshotID string + + TotalHashedBytes int64 + TotalUploadedBytes int64 + TotalFileCount int - TotalHashedBytes int64 TotalDirectoryCount int IgnoredErrorCount int ErrorCount int - Incomplete bool - IncompleteReason string + + Incomplete bool + IncompleteReason string } -func manifestToStats(man *snapshot.Manifest, progress *corsoProgress) BackupStats { +func manifestToStats( + man *snapshot.Manifest, + progress *corsoProgress, + uploadCount *stats.ByteCounter, +) BackupStats { return BackupStats{ - SnapshotID: string(man.ID), + SnapshotID: string(man.ID), + + TotalHashedBytes: progress.totalBytes, + TotalUploadedBytes: uploadCount.NumBytes, + TotalFileCount: int(man.Stats.TotalFileCount), - TotalHashedBytes: progress.totalBytes, TotalDirectoryCount: int(man.Stats.TotalDirectoryCount), IgnoredErrorCount: int(man.Stats.IgnoredErrorCount), ErrorCount: int(man.Stats.ErrorCount), - Incomplete: man.IncompleteReason != "", - IncompleteReason: man.IncompleteReason, + + Incomplete: man.IncompleteReason != "", + IncompleteReason: man.IncompleteReason, } } @@ -380,12 +393,12 @@ func (w Wrapper) BackupCollections( return nil, nil, errors.Wrap(err, "building kopia directories") } - stats, err := w.makeSnapshotWithRoot(ctx, dirTree, progress) + s, err := w.makeSnapshotWithRoot(ctx, dirTree, progress) if err != nil { return nil, nil, err } - return stats, progress.deets, nil + return s, progress.deets, nil } func (w Wrapper) makeSnapshotWithRoot( @@ -395,6 +408,8 @@ func (w Wrapper) makeSnapshotWithRoot( ) (*BackupStats, error) { var man *snapshot.Manifest + bc := &stats.ByteCounter{} + err := repo.WriteSession( ctx, w.c, @@ -403,6 +418,7 @@ func (w Wrapper) makeSnapshotWithRoot( // Always flush so we don't leak write sessions. Still uses reachability // for consistency. FlushOnFailure: true, + OnUpload: bc.Count, }, func(innerCtx context.Context, rw repo.RepositoryWriter) error { si := snapshot.SourceInfo{ @@ -453,7 +469,7 @@ func (w Wrapper) makeSnapshotWithRoot( return nil, errors.Wrap(err, "kopia backup") } - res := manifestToStats(man, progress) + res := manifestToStats(man, progress, bc) return &res, nil } @@ -481,6 +497,7 @@ func getItemStream( ctx context.Context, itemPath path.Path, snapshotRoot fs.Entry, + bcounter byteCounter, ) (data.Stream, error) { if itemPath == nil { return nil, errors.WithStack(errNoRestorePath) @@ -501,6 +518,10 @@ func getItemStream( return nil, errors.New("requested object is not a file") } + if bcounter != nil { + bcounter.Count(f.Size()) + } + r, err := f.Open(ctx) if err != nil { return nil, errors.Wrap(err, "opening file") @@ -517,6 +538,10 @@ func getItemStream( }, nil } +type byteCounter interface { + Count(numBytes int64) +} + // RestoreMultipleItems looks up all paths- assuming each is an item declaration, // not a directory- in the snapshot with id snapshotID. The path should be the // full path of the item from the root. Returns the results as a slice of single- @@ -528,6 +553,7 @@ func (w Wrapper) RestoreMultipleItems( ctx context.Context, snapshotID string, paths []path.Path, + bcounter byteCounter, ) ([]data.Collection, error) { if len(paths) == 0 { return nil, errors.WithStack(errNoRestorePath) @@ -545,7 +571,7 @@ func (w Wrapper) RestoreMultipleItems( ) for _, itemPath := range paths { - ds, err := getItemStream(ctx, itemPath, snapshotRoot) + ds, err := getItemStream(ctx, itemPath, snapshotRoot, bcounter) if err != nil { errs = multierror.Append(errs, err) continue diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 4965eb14a..42fdc0b99 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -709,7 +709,8 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { []path.Path{ fp1, fp2, - }) + }, + nil) require.NoError(t, err) assert.Equal(t, 2, len(result)) @@ -926,6 +927,14 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TearDownTest() { assert.NoError(suite.T(), suite.w.Close(suite.ctx)) } +type i64counter struct { + i int64 +} + +func (c *i64counter) Count(i int64) { + c.i += i +} + func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { doesntExist, err := path.Builder{}.Append("subdir", "foo").ToDataLayerExchangePathForCategory( testTenant, @@ -1008,14 +1017,17 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() { expected[pth.String()] = item.data } + ic := i64counter{} + result, err := suite.w.RestoreMultipleItems( suite.ctx, string(suite.snapshotID), test.inputPaths, - ) + &ic) test.expectedErr(t, err) assert.Len(t, result, test.expectedCollections) + assert.Less(t, int64(0), ic.i) testForFiles(t, expected, result) }) } @@ -1053,7 +1065,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors() suite.ctx, test.snapshotID, test.paths, - ) + nil) assert.Error(t, err) assert.Empty(t, c) }) @@ -1067,13 +1079,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() { // assert the deletion worked itemPath := suite.files[suite.testPath1.String()][0].itemPath + ic := i64counter{} + c, err := suite.w.RestoreMultipleItems( suite.ctx, string(suite.snapshotID), []path.Path{itemPath}, - ) + &ic) assert.Error(t, err, "snapshot should be deleted") assert.Empty(t, c) + assert.Zero(t, ic.i) } func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() { diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index d3d397af7..448198597 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -35,6 +35,7 @@ type BackupOperation struct { // BackupResults aggregate the details of the result of the operation. type BackupResults struct { + stats.Errs stats.ReadWrites stats.StartAndEndTime BackupID model.StableID `json:"backupID"` @@ -167,7 +168,9 @@ func (op *BackupOperation) persistResults( op.Results.ReadErrors = opStats.readErr op.Results.WriteErrors = opStats.writeErr - op.Results.BytesWritten = opStats.k.TotalHashedBytes + + op.Results.BytesRead = opStats.k.TotalHashedBytes + op.Results.BytesUploaded = opStats.k.TotalUploadedBytes op.Results.ItemsRead = opStats.gc.Successful op.Results.ItemsWritten = opStats.k.TotalFileCount op.Results.ResourceOwners = opStats.resourceCount @@ -214,7 +217,7 @@ func (op *BackupOperation) createBackupModels( events.StartTime: op.Results.StartedAt, events.EndTime: op.Results.CompletedAt, events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), - events.DataStored: op.Results.BytesWritten, + events.DataStored: op.Results.BytesUploaded, events.Resources: op.Results.ResourceOwners, // TODO: events.ExchangeDataObserved: , }, diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 4fe152a17..70092b833 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -48,8 +48,9 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { writeErr: assert.AnError, resourceCount: 1, k: &kopia.BackupStats{ - TotalFileCount: 1, - TotalHashedBytes: 1, + TotalFileCount: 1, + TotalHashedBytes: 1, + TotalUploadedBytes: 1, }, gc: &support.ConnectorOperationStatus{ Successful: 1, @@ -73,7 +74,8 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { assert.Equal(t, op.Results.ItemsRead, stats.gc.Successful, "items read") assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors") assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount, "items written") - assert.Equal(t, op.Results.BytesWritten, stats.k.TotalHashedBytes, "bytes written") + assert.Equal(t, stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read") + assert.Equal(t, stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written") assert.Equal(t, op.Results.ResourceOwners, stats.resourceCount, "resource owners") assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors") assert.Equal(t, op.Results.StartedAt, now, "started at") @@ -218,7 +220,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { assert.Equal(t, bo.Status, Completed) assert.Less(t, 0, bo.Results.ItemsRead) assert.Less(t, 0, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesWritten) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners) assert.Zero(t, bo.Results.ReadErrors) assert.Zero(t, bo.Results.WriteErrors) @@ -277,7 +280,8 @@ func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() { require.NotEmpty(t, bo.Results.BackupID) assert.Equal(t, bo.Status, Completed) assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten) - assert.Less(t, int64(0), bo.Results.BytesWritten) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") assert.Equal(t, 1, bo.Results.ResourceOwners) assert.NoError(t, bo.Results.ReadErrors) assert.NoError(t, bo.Results.WriteErrors) diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index a3a53332b..fc4281b6d 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -37,6 +37,7 @@ type RestoreOperation struct { // RestoreResults aggregate the details of the results of the operation. type RestoreResults struct { + stats.Errs stats.ReadWrites stats.StartAndEndTime } @@ -77,6 +78,7 @@ func (op RestoreOperation) validate() error { type restoreStats struct { cs []data.Collection gc *support.ConnectorOperationStatus + bytesRead *stats.ByteCounter resourceCount int started bool readErr, writeErr error @@ -87,7 +89,9 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { startTime := time.Now() // persist operation results to the model store on exit - opStats := restoreStats{} + opStats := restoreStats{ + bytesRead: &stats.ByteCounter{}, + } // TODO: persist results? defer func() { @@ -160,7 +164,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { paths[i] = p } - dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths) + dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths, opStats.bytesRead) if err != nil { err = errors.Wrap(err, "retrieving service data") @@ -193,6 +197,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { opStats.started = true opStats.gc = gc.AwaitStatus() + logger.Ctx(ctx).Debug(gc.PrintableStatus()) return nil @@ -220,6 +225,8 @@ func (op *RestoreOperation) persistResults( op.Results.ReadErrors = opStats.readErr op.Results.WriteErrors = opStats.writeErr + + op.Results.BytesRead = opStats.bytesRead.NumBytes op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count op.Results.ItemsWritten = opStats.gc.Successful op.Results.ResourceOwners = opStats.resourceCount @@ -229,16 +236,16 @@ func (op *RestoreOperation) persistResults( events.RestoreEnd, map[string]any{ // TODO: RestoreID - events.BackupID: op.BackupID, - events.Service: op.Selectors.Service.String(), - events.Status: op.Status, - events.StartTime: op.Results.StartedAt, - events.EndTime: op.Results.CompletedAt, - events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), - events.ItemsRead: op.Results.ItemsRead, - events.ItemsWritten: op.Results.ItemsWritten, - events.Resources: op.Results.ResourceOwners, - // TODO: events.ExchangeDataObserved: , + events.BackupID: op.BackupID, + events.Service: op.Selectors.Service.String(), + events.Status: op.Status, + events.StartTime: op.Results.StartedAt, + events.EndTime: op.Results.CompletedAt, + events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), + events.ItemsRead: op.Results.ItemsRead, + events.ItemsWritten: op.Results.ItemsWritten, + events.Resources: op.Results.ResourceOwners, + events.DataRetrieved: op.Results.BytesRead, }, ) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 54d6a3d1c..056996d95 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -17,6 +17,7 @@ import ( evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -43,15 +44,19 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { ctx := context.Background() var ( - kw = &kopia.Wrapper{} - sw = &store.Wrapper{} - acct = account.Account{} - now = time.Now() - stats = restoreStats{ - started: true, - readErr: multierror.Append(nil, assert.AnError), - writeErr: assert.AnError, - cs: []data.Collection{&exchange.Collection{}}, + kw = &kopia.Wrapper{} + sw = &store.Wrapper{} + acct = account.Account{} + now = time.Now() + rs = restoreStats{ + started: true, + readErr: multierror.Append(nil, assert.AnError), + writeErr: assert.AnError, + resourceCount: 1, + bytesRead: &stats.ByteCounter{ + NumBytes: 42, + }, + cs: []data.Collection{&exchange.Collection{}}, gc: &support.ConnectorOperationStatus{ ObjectCount: 1, }, @@ -69,14 +74,15 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { evmock.NewBus()) require.NoError(t, err) - require.NoError(t, op.persistResults(ctx, now, &stats)) + require.NoError(t, op.persistResults(ctx, now, &rs)) assert.Equal(t, op.Status.String(), Completed.String(), "status") - assert.Equal(t, op.Results.ItemsRead, len(stats.cs), "items read") - assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors") - assert.Equal(t, op.Results.ItemsWritten, stats.gc.Successful, "items written") - assert.Equal(t, 0, op.Results.ResourceOwners, "resource owners") - assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors") + assert.Equal(t, op.Results.ItemsRead, len(rs.cs), "items read") + assert.Equal(t, op.Results.ReadErrors, rs.readErr, "read errors") + assert.Equal(t, op.Results.ItemsWritten, rs.gc.Successful, "items written") + assert.Equal(t, rs.bytesRead.NumBytes, op.Results.BytesRead, "resource owners") + assert.Equal(t, rs.resourceCount, op.Results.ResourceOwners, "resource owners") + assert.Equal(t, op.Results.WriteErrors, rs.writeErr, "write errors") assert.Equal(t, op.Results.StartedAt, now, "started at") assert.Less(t, now, op.Results.CompletedAt, "completed at") } @@ -231,9 +237,10 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { require.NoError(t, ro.Run(ctx), "restoreOp.Run()") require.NotEmpty(t, ro.Results, "restoreOp results") assert.Equal(t, ro.Status, Completed, "restoreOp status") - assert.Greater(t, ro.Results.ItemsRead, 0, "restore items read") - assert.Greater(t, ro.Results.ItemsWritten, 0, "restored items written") - assert.Equal(t, 1, ro.Results.ResourceOwners) + assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") + assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written") + assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") + assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners") assert.Zero(t, ro.Results.ReadErrors, "errors while reading restore data") assert.Zero(t, ro.Results.WriteErrors, "errors while writing restore data") assert.Equal(t, suite.numItems, ro.Results.ItemsWritten, "backup and restore wrote the same num of items") @@ -261,7 +268,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() { mb) require.NoError(t, err) require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results") - assert.Equal(t, 0, ro.Results.ResourceOwners) + assert.Zero(t, ro.Results.ResourceOwners, "resource owners") + assert.Zero(t, ro.Results.BytesRead, "bytes read") assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") - assert.Equal(t, 0, mb.TimesCalled[events.RestoreEnd], "restore-end events") + assert.Zero(t, mb.TimesCalled[events.RestoreEnd], "restore-end events") } diff --git a/src/internal/stats/stats.go b/src/internal/stats/stats.go index 3d8eced1c..aacb8c850 100644 --- a/src/internal/stats/stats.go +++ b/src/internal/stats/stats.go @@ -1,22 +1,36 @@ package stats -import "time" +import ( + "sync/atomic" + "time" +) -// ReadWrites tracks the total count of reads and writes, and of -// read and write errors. ItemsRead and ItemsWritten counts are -// assumed to be successful, so the total count of items involved -// would be ItemsRead+ReadErrors. +// ReadWrites tracks the total count of reads and writes. ItemsRead +// and ItemsWritten counts are assumed to be successful reads. type ReadWrites struct { - BytesWritten int64 `json:"bytesWritten,omitempty"` + BytesRead int64 `json:"bytesRead,omitempty"` + BytesUploaded int64 `json:"bytesUploaded,omitempty"` ItemsRead int `json:"itemsRead,omitempty"` ItemsWritten int `json:"itemsWritten,omitempty"` - ReadErrors error `json:"readErrors,omitempty"` - WriteErrors error `json:"writeErrors,omitempty"` ResourceOwners int `json:"resourceOwners,omitempty"` } +// Errs tracks the aggregation of errors that occurred during a process. +type Errs struct { + ReadErrors error `json:"readErrors,omitempty"` + WriteErrors error `json:"writeErrors,omitempty"` +} + // StartAndEndTime tracks a paired starting time and ending time. type StartAndEndTime struct { StartedAt time.Time `json:"startedAt"` CompletedAt time.Time `json:"completedAt"` } + +type ByteCounter struct { + NumBytes int64 +} + +func (bc *ByteCounter) Count(i int64) { + atomic.AddInt64(&bc.NumBytes, i) +} diff --git a/src/pkg/backup/backup.go b/src/pkg/backup/backup.go index 430faf44e..b60c35b4e 100644 --- a/src/pkg/backup/backup.go +++ b/src/pkg/backup/backup.go @@ -32,6 +32,7 @@ type Backup struct { Selectors selectors.Selector `json:"selectors"` // stats are embedded so that the values appear as top-level properties + stats.Errs stats.ReadWrites stats.StartAndEndTime } diff --git a/src/pkg/backup/backup_test.go b/src/pkg/backup/backup_test.go index a295bb299..2edd0cb4d 100644 --- a/src/pkg/backup/backup_test.go +++ b/src/pkg/backup/backup_test.go @@ -37,11 +37,13 @@ func stubBackup(t time.Time) backup.Backup { DetailsID: "details", Status: "status", Selectors: sel.Selector, + Errs: stats.Errs{ + ReadErrors: errors.New("1"), + WriteErrors: errors.New("1"), + }, ReadWrites: stats.ReadWrites{ ItemsRead: 1, ItemsWritten: 1, - ReadErrors: errors.New("1"), - WriteErrors: errors.New("1"), }, StartAndEndTime: stats.StartAndEndTime{ StartedAt: t,