track bytes written, read in kopia (#944)

## Description

Track the count of bytes read and written in
kopia.  For backups, this means the count of
bytes fed into kopia (hashed bytes), and the
amount written after compression and dedupe
(total file bytes).  For restore, this is the count of bytes in all files read.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #894

## Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-09-26 13:18:42 -06:00 committed by GitHub
parent ff2db0c553
commit 38addfaae8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 146 additions and 104 deletions

View File

@ -57,7 +57,6 @@ require (
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect

View File

@ -36,20 +36,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1 h1:tz19qLF65vuu2ibfTqGVJxG/zZAI27NEIIbvAOQwYbw=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 h1:8LoU8N2lIUzkmstvwXvVfniMZlFbesfT2AmA1aqvRr8=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0 h1:Yoicul8bnVdQrhDMTHxdEckRGX01XvwXDHUT9zYZ3k0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c=
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -79,8 +71,6 @@ github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dg
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cjlapao/common-go v0.0.25 h1:/a5SdWGtOFzXceM1RnB3v4hgUK8woMo9Ma/XRgCQ87g=
github.com/cjlapao/common-go v0.0.25/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g=
github.com/cjlapao/common-go v0.0.27 h1:7k8R1Mz2LAudnPb1kaqQ/l+Ba7uL92FG7Rqp9W67mGM=
github.com/cjlapao/common-go v0.0.27/go.mod h1:OyTAY388jfEj8uaRzx0uYneFghKDLL5KP+ewSydlQ5g=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@ -129,10 +119,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@ -192,7 +178,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -268,32 +253,18 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/microsoft/kiota-abstractions-go v0.8.2 h1:KxHr72YwlntvVgF0C+8CtNmNQajtIq7fedq2N0+74Uk=
github.com/microsoft/kiota-abstractions-go v0.8.2/go.mod h1:i1wK2rdsLGSjgpnpLHjC60nEa/UdCVS6ulCh1Q+1COw=
github.com/microsoft/kiota-abstractions-go v0.10.1 h1:R4vk1l/wn0Dij01yxVX5lD/1UKMEO9Hz5TXkBl6E8do=
github.com/microsoft/kiota-abstractions-go v0.10.1/go.mod h1:wOO+hpReDIJa3BxbLNz4qvtZk2llS555mejkMc203bQ=
github.com/microsoft/kiota-authentication-azure-go v0.3.0 h1:iLyy5qldAjBiYMGMk1r/rJkcmARA8cKboiN7/XbRxv4=
github.com/microsoft/kiota-authentication-azure-go v0.3.0/go.mod h1:qyZWSCug2eG1zrRnCSacyFHGsgQa4aSCWn3EOkY9Z1M=
github.com/microsoft/kiota-authentication-azure-go v0.4.1 h1:C+n4Vp3oCj8W8LPiKZDyASTNaKyUomQtUiXQ+B2W0TU=
github.com/microsoft/kiota-authentication-azure-go v0.4.1/go.mod h1:jIJAhpPh34bDQWNME65kd/yjqY6+CJZi5jus8H9EH4s=
github.com/microsoft/kiota-http-go v0.6.0 h1:pNdkRDLGBMuFryS5XvHUGzCsteAg2a4XMdsG8b7YQJs=
github.com/microsoft/kiota-http-go v0.6.0/go.mod h1:4N7GGz5qCZ5JCsEpMyRmGecRckp2evUYRLetIvPBuYs=
github.com/microsoft/kiota-http-go v0.7.2 h1:R40vG0EkIFqGvVz5dZtLe4g1sXGVfBO5HxjdBjpwv8k=
github.com/microsoft/kiota-http-go v0.7.2/go.mod h1:QTbXPh25mJsbxE23bFqw64BckCioCGfaE77hF/F3rIQ=
github.com/microsoft/kiota-serialization-json-go v0.5.5 h1:B0iKBKOdi+9NKFlormLRqduQ1+77MPGRsZ7xnd74EqQ=
github.com/microsoft/kiota-serialization-json-go v0.5.5/go.mod h1:GI9vrssO1EvqzDtvMKuhjALn40phZOWkeeaMgtCk6xE=
github.com/microsoft/kiota-serialization-json-go v0.6.0 h1:irdhbaY2Vl9t2SLwIC5WyPKJRp3mE52GEBEbAyu7thk=
github.com/microsoft/kiota-serialization-json-go v0.6.0/go.mod h1:ceR++Qc8n6McdAR+Ili2UhV4iR8CEx3+RPtANi1UdXc=
github.com/microsoft/kiota-serialization-text-go v0.4.1 h1:6QPH7+geUPCpaSZkKCQw0Scngx2IF0vKodrvvWWiu2A=
github.com/microsoft/kiota-serialization-text-go v0.4.1/go.mod h1:DsriFnVBDCc4D84qxG3j8q/1Sxu16JILfhxMZm3kdfw=
github.com/microsoft/kiota-serialization-text-go v0.5.0 h1:TWb9Y6IsIwzsMVcbBBDLFpVg47mRu2FhQJ6i1dqpLOs=
github.com/microsoft/kiota-serialization-text-go v0.5.0/go.mod h1:x9h+VE4X4t8njowIZXyTaAzE6bGK8Zr90MLYV6J6S9U=
github.com/microsoftgraph/msgraph-sdk-go v0.34.0 h1:AXPTyCUKaxy4i0qSLBuUbaZTw4thoZTMS2i8KWCItlo=
github.com/microsoftgraph/msgraph-sdk-go v0.34.0/go.mod h1:5KCKGk0dKyEK17M40vCHTnyaayUe/SqIOh9aww0ECpU=
github.com/microsoftgraph/msgraph-sdk-go v0.40.0 h1:9AxA3FS+S3c7Him5C+7Lt0I8zaNXoSqXsLDink1Fg40=
github.com/microsoftgraph/msgraph-sdk-go v0.40.0/go.mod h1:B8HORKdf1K05Z93FbkpiqJ25dnytjPEyAby6gHhOLiM=
github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0 h1:FqKddh8nTTbNyUtRCCmPKtgxSNGz944Kw8Q+WU/H+lo=
github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0/go.mod h1:kcTY0sEZ/LOJiSj/1OMxcs0T51uodJ/bOeVfWo4lo/s=
github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1 h1:gN3pVVvx50KzhQdYwQKVnRSOSOIRaFpXj7pgPfqnNXw=
github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1/go.mod h1:BnumnwWU8xUgX7ncgo68novbS1wMlO66Iny9iVhvHuM=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
@ -311,7 +282,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
@ -322,8 +292,6 @@ github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -452,8 +420,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0 h1:a5Yg6ylndHHYJqIPrdq0AhvR6KTvDTAvgBtaidhEevY=
golang.org/x/crypto v0.0.0-20220919173607-35f4265a4bc0/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -530,8 +496,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7 h1:DTGA3sVb/sQX+3poldfq5cO4KiOPwLSRBjn2rtck5RM=
golang.org/x/net v0.0.0-20220920191752-2e0b12c274b7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -610,7 +574,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

View File

@ -16,6 +16,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/stats"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -35,23 +36,35 @@ var (
type BackupStats struct {
SnapshotID string
TotalFileCount int
TotalHashedBytes int64
TotalUploadedBytes int64
TotalFileCount int
TotalDirectoryCount int
IgnoredErrorCount int
ErrorCount int
Incomplete bool
IncompleteReason string
}
func manifestToStats(man *snapshot.Manifest, progress *corsoProgress) BackupStats {
func manifestToStats(
man *snapshot.Manifest,
progress *corsoProgress,
uploadCount *stats.ByteCounter,
) BackupStats {
return BackupStats{
SnapshotID: string(man.ID),
TotalFileCount: int(man.Stats.TotalFileCount),
TotalHashedBytes: progress.totalBytes,
TotalUploadedBytes: uploadCount.NumBytes,
TotalFileCount: int(man.Stats.TotalFileCount),
TotalDirectoryCount: int(man.Stats.TotalDirectoryCount),
IgnoredErrorCount: int(man.Stats.IgnoredErrorCount),
ErrorCount: int(man.Stats.ErrorCount),
Incomplete: man.IncompleteReason != "",
IncompleteReason: man.IncompleteReason,
}
@ -380,12 +393,12 @@ func (w Wrapper) BackupCollections(
return nil, nil, errors.Wrap(err, "building kopia directories")
}
stats, err := w.makeSnapshotWithRoot(ctx, dirTree, progress)
s, err := w.makeSnapshotWithRoot(ctx, dirTree, progress)
if err != nil {
return nil, nil, err
}
return stats, progress.deets, nil
return s, progress.deets, nil
}
func (w Wrapper) makeSnapshotWithRoot(
@ -395,6 +408,8 @@ func (w Wrapper) makeSnapshotWithRoot(
) (*BackupStats, error) {
var man *snapshot.Manifest
bc := &stats.ByteCounter{}
err := repo.WriteSession(
ctx,
w.c,
@ -403,6 +418,7 @@ func (w Wrapper) makeSnapshotWithRoot(
// Always flush so we don't leak write sessions. Still uses reachability
// for consistency.
FlushOnFailure: true,
OnUpload: bc.Count,
},
func(innerCtx context.Context, rw repo.RepositoryWriter) error {
si := snapshot.SourceInfo{
@ -453,7 +469,7 @@ func (w Wrapper) makeSnapshotWithRoot(
return nil, errors.Wrap(err, "kopia backup")
}
res := manifestToStats(man, progress)
res := manifestToStats(man, progress, bc)
return &res, nil
}
@ -481,6 +497,7 @@ func getItemStream(
ctx context.Context,
itemPath path.Path,
snapshotRoot fs.Entry,
bcounter byteCounter,
) (data.Stream, error) {
if itemPath == nil {
return nil, errors.WithStack(errNoRestorePath)
@ -501,6 +518,10 @@ func getItemStream(
return nil, errors.New("requested object is not a file")
}
if bcounter != nil {
bcounter.Count(f.Size())
}
r, err := f.Open(ctx)
if err != nil {
return nil, errors.Wrap(err, "opening file")
@ -517,6 +538,10 @@ func getItemStream(
}, nil
}
type byteCounter interface {
Count(numBytes int64)
}
// RestoreMultipleItems looks up all paths- assuming each is an item declaration,
// not a directory- in the snapshot with id snapshotID. The path should be the
// full path of the item from the root. Returns the results as a slice of single-
@ -528,6 +553,7 @@ func (w Wrapper) RestoreMultipleItems(
ctx context.Context,
snapshotID string,
paths []path.Path,
bcounter byteCounter,
) ([]data.Collection, error) {
if len(paths) == 0 {
return nil, errors.WithStack(errNoRestorePath)
@ -545,7 +571,7 @@ func (w Wrapper) RestoreMultipleItems(
)
for _, itemPath := range paths {
ds, err := getItemStream(ctx, itemPath, snapshotRoot)
ds, err := getItemStream(ctx, itemPath, snapshotRoot, bcounter)
if err != nil {
errs = multierror.Append(errs, err)
continue

View File

@ -709,7 +709,8 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
[]path.Path{
fp1,
fp2,
})
},
nil)
require.NoError(t, err)
assert.Equal(t, 2, len(result))
@ -926,6 +927,14 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TearDownTest() {
assert.NoError(suite.T(), suite.w.Close(suite.ctx))
}
type i64counter struct {
i int64
}
func (c *i64counter) Count(i int64) {
c.i += i
}
func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() {
doesntExist, err := path.Builder{}.Append("subdir", "foo").ToDataLayerExchangePathForCategory(
testTenant,
@ -1008,14 +1017,17 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems() {
expected[pth.String()] = item.data
}
ic := i64counter{}
result, err := suite.w.RestoreMultipleItems(
suite.ctx,
string(suite.snapshotID),
test.inputPaths,
)
&ic)
test.expectedErr(t, err)
assert.Len(t, result, test.expectedCollections)
assert.Less(t, int64(0), ic.i)
testForFiles(t, expected, result)
})
}
@ -1053,7 +1065,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestRestoreMultipleItems_Errors()
suite.ctx,
test.snapshotID,
test.paths,
)
nil)
assert.Error(t, err)
assert.Empty(t, c)
})
@ -1067,13 +1079,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() {
// assert the deletion worked
itemPath := suite.files[suite.testPath1.String()][0].itemPath
ic := i64counter{}
c, err := suite.w.RestoreMultipleItems(
suite.ctx,
string(suite.snapshotID),
[]path.Path{itemPath},
)
&ic)
assert.Error(t, err, "snapshot should be deleted")
assert.Empty(t, c)
assert.Zero(t, ic.i)
}
func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot_BadIDs() {

View File

@ -35,6 +35,7 @@ type BackupOperation struct {
// BackupResults aggregate the details of the result of the operation.
type BackupResults struct {
stats.Errs
stats.ReadWrites
stats.StartAndEndTime
BackupID model.StableID `json:"backupID"`
@ -167,7 +168,9 @@ func (op *BackupOperation) persistResults(
op.Results.ReadErrors = opStats.readErr
op.Results.WriteErrors = opStats.writeErr
op.Results.BytesWritten = opStats.k.TotalHashedBytes
op.Results.BytesRead = opStats.k.TotalHashedBytes
op.Results.BytesUploaded = opStats.k.TotalUploadedBytes
op.Results.ItemsRead = opStats.gc.Successful
op.Results.ItemsWritten = opStats.k.TotalFileCount
op.Results.ResourceOwners = opStats.resourceCount
@ -214,7 +217,7 @@ func (op *BackupOperation) createBackupModels(
events.StartTime: op.Results.StartedAt,
events.EndTime: op.Results.CompletedAt,
events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt),
events.DataStored: op.Results.BytesWritten,
events.DataStored: op.Results.BytesUploaded,
events.Resources: op.Results.ResourceOwners,
// TODO: events.ExchangeDataObserved: <amount of data retrieved>,
},

View File

@ -50,6 +50,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
k: &kopia.BackupStats{
TotalFileCount: 1,
TotalHashedBytes: 1,
TotalUploadedBytes: 1,
},
gc: &support.ConnectorOperationStatus{
Successful: 1,
@ -73,7 +74,8 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
assert.Equal(t, op.Results.ItemsRead, stats.gc.Successful, "items read")
assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors")
assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount, "items written")
assert.Equal(t, op.Results.BytesWritten, stats.k.TotalHashedBytes, "bytes written")
assert.Equal(t, stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read")
assert.Equal(t, stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written")
assert.Equal(t, op.Results.ResourceOwners, stats.resourceCount, "resource owners")
assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors")
assert.Equal(t, op.Results.StartedAt, now, "started at")
@ -218,7 +220,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
assert.Equal(t, bo.Status, Completed)
assert.Less(t, 0, bo.Results.ItemsRead)
assert.Less(t, 0, bo.Results.ItemsWritten)
assert.Less(t, int64(0), bo.Results.BytesWritten)
assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read")
assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded")
assert.Equal(t, 1, bo.Results.ResourceOwners)
assert.Zero(t, bo.Results.ReadErrors)
assert.Zero(t, bo.Results.WriteErrors)
@ -277,7 +280,8 @@ func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() {
require.NotEmpty(t, bo.Results.BackupID)
assert.Equal(t, bo.Status, Completed)
assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten)
assert.Less(t, int64(0), bo.Results.BytesWritten)
assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read")
assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded")
assert.Equal(t, 1, bo.Results.ResourceOwners)
assert.NoError(t, bo.Results.ReadErrors)
assert.NoError(t, bo.Results.WriteErrors)

View File

@ -37,6 +37,7 @@ type RestoreOperation struct {
// RestoreResults aggregate the details of the results of the operation.
type RestoreResults struct {
stats.Errs
stats.ReadWrites
stats.StartAndEndTime
}
@ -77,6 +78,7 @@ func (op RestoreOperation) validate() error {
type restoreStats struct {
cs []data.Collection
gc *support.ConnectorOperationStatus
bytesRead *stats.ByteCounter
resourceCount int
started bool
readErr, writeErr error
@ -87,7 +89,9 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
startTime := time.Now()
// persist operation results to the model store on exit
opStats := restoreStats{}
opStats := restoreStats{
bytesRead: &stats.ByteCounter{},
}
// TODO: persist results?
defer func() {
@ -160,7 +164,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
paths[i] = p
}
dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths)
dcs, err := op.kopia.RestoreMultipleItems(ctx, b.SnapshotID, paths, opStats.bytesRead)
if err != nil {
err = errors.Wrap(err, "retrieving service data")
@ -193,6 +197,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
opStats.started = true
opStats.gc = gc.AwaitStatus()
logger.Ctx(ctx).Debug(gc.PrintableStatus())
return nil
@ -220,6 +225,8 @@ func (op *RestoreOperation) persistResults(
op.Results.ReadErrors = opStats.readErr
op.Results.WriteErrors = opStats.writeErr
op.Results.BytesRead = opStats.bytesRead.NumBytes
op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count
op.Results.ItemsWritten = opStats.gc.Successful
op.Results.ResourceOwners = opStats.resourceCount
@ -238,7 +245,7 @@ func (op *RestoreOperation) persistResults(
events.ItemsRead: op.Results.ItemsRead,
events.ItemsWritten: op.Results.ItemsWritten,
events.Resources: op.Results.ResourceOwners,
// TODO: events.ExchangeDataObserved: <amount of data retrieved>,
events.DataRetrieved: op.Results.BytesRead,
},
)

View File

@ -17,6 +17,7 @@ import (
evmock "github.com/alcionai/corso/src/internal/events/mock"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/stats"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
@ -47,10 +48,14 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
sw = &store.Wrapper{}
acct = account.Account{}
now = time.Now()
stats = restoreStats{
rs = restoreStats{
started: true,
readErr: multierror.Append(nil, assert.AnError),
writeErr: assert.AnError,
resourceCount: 1,
bytesRead: &stats.ByteCounter{
NumBytes: 42,
},
cs: []data.Collection{&exchange.Collection{}},
gc: &support.ConnectorOperationStatus{
ObjectCount: 1,
@ -69,14 +74,15 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
evmock.NewBus())
require.NoError(t, err)
require.NoError(t, op.persistResults(ctx, now, &stats))
require.NoError(t, op.persistResults(ctx, now, &rs))
assert.Equal(t, op.Status.String(), Completed.String(), "status")
assert.Equal(t, op.Results.ItemsRead, len(stats.cs), "items read")
assert.Equal(t, op.Results.ReadErrors, stats.readErr, "read errors")
assert.Equal(t, op.Results.ItemsWritten, stats.gc.Successful, "items written")
assert.Equal(t, 0, op.Results.ResourceOwners, "resource owners")
assert.Equal(t, op.Results.WriteErrors, stats.writeErr, "write errors")
assert.Equal(t, op.Results.ItemsRead, len(rs.cs), "items read")
assert.Equal(t, op.Results.ReadErrors, rs.readErr, "read errors")
assert.Equal(t, op.Results.ItemsWritten, rs.gc.Successful, "items written")
assert.Equal(t, rs.bytesRead.NumBytes, op.Results.BytesRead, "resource owners")
assert.Equal(t, rs.resourceCount, op.Results.ResourceOwners, "resource owners")
assert.Equal(t, op.Results.WriteErrors, rs.writeErr, "write errors")
assert.Equal(t, op.Results.StartedAt, now, "started at")
assert.Less(t, now, op.Results.CompletedAt, "completed at")
}
@ -231,9 +237,10 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
require.NoError(t, ro.Run(ctx), "restoreOp.Run()")
require.NotEmpty(t, ro.Results, "restoreOp results")
assert.Equal(t, ro.Status, Completed, "restoreOp status")
assert.Greater(t, ro.Results.ItemsRead, 0, "restore items read")
assert.Greater(t, ro.Results.ItemsWritten, 0, "restored items written")
assert.Equal(t, 1, ro.Results.ResourceOwners)
assert.Less(t, 0, ro.Results.ItemsRead, "restore items read")
assert.Less(t, 0, ro.Results.ItemsWritten, "restored items written")
assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read")
assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners")
assert.Zero(t, ro.Results.ReadErrors, "errors while reading restore data")
assert.Zero(t, ro.Results.WriteErrors, "errors while writing restore data")
assert.Equal(t, suite.numItems, ro.Results.ItemsWritten, "backup and restore wrote the same num of items")
@ -261,7 +268,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() {
mb)
require.NoError(t, err)
require.Error(t, ro.Run(ctx), "restoreOp.Run() should have 0 results")
assert.Equal(t, 0, ro.Results.ResourceOwners)
assert.Zero(t, ro.Results.ResourceOwners, "resource owners")
assert.Zero(t, ro.Results.BytesRead, "bytes read")
assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events")
assert.Equal(t, 0, mb.TimesCalled[events.RestoreEnd], "restore-end events")
assert.Zero(t, mb.TimesCalled[events.RestoreEnd], "restore-end events")
}

View File

@ -1,18 +1,24 @@
package stats
import "time"
import (
"sync/atomic"
"time"
)
// ReadWrites tracks the total count of reads and writes, and of
// read and write errors. ItemsRead and ItemsWritten counts are
// assumed to be successful, so the total count of items involved
// would be ItemsRead+ReadErrors.
// ReadWrites tracks the total count of reads and writes. ItemsRead
// and ItemsWritten counts are assumed to be successful reads.
type ReadWrites struct {
BytesWritten int64 `json:"bytesWritten,omitempty"`
BytesRead int64 `json:"bytesRead,omitempty"`
BytesUploaded int64 `json:"bytesUploaded,omitempty"`
ItemsRead int `json:"itemsRead,omitempty"`
ItemsWritten int `json:"itemsWritten,omitempty"`
ResourceOwners int `json:"resourceOwners,omitempty"`
}
// Errs tracks the aggregation of errors that occurred during a process.
type Errs struct {
ReadErrors error `json:"readErrors,omitempty"`
WriteErrors error `json:"writeErrors,omitempty"`
ResourceOwners int `json:"resourceOwners,omitempty"`
}
// StartAndEndTime tracks a paired starting time and ending time.
@ -20,3 +26,11 @@ type StartAndEndTime struct {
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
}
type ByteCounter struct {
NumBytes int64
}
func (bc *ByteCounter) Count(i int64) {
atomic.AddInt64(&bc.NumBytes, i)
}

View File

@ -32,6 +32,7 @@ type Backup struct {
Selectors selectors.Selector `json:"selectors"`
// stats are embedded so that the values appear as top-level properties
stats.Errs
stats.ReadWrites
stats.StartAndEndTime
}

View File

@ -37,11 +37,13 @@ func stubBackup(t time.Time) backup.Backup {
DetailsID: "details",
Status: "status",
Selectors: sel.Selector,
Errs: stats.Errs{
ReadErrors: errors.New("1"),
WriteErrors: errors.New("1"),
},
ReadWrites: stats.ReadWrites{
ItemsRead: 1,
ItemsWritten: 1,
ReadErrors: errors.New("1"),
WriteErrors: errors.New("1"),
},
StartAndEndTime: stats.StartAndEndTime{
StartedAt: t,