diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce252765d..5d39aa392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,6 +171,7 @@ jobs: CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} run: | set -euo pipefail go test \ @@ -193,6 +194,76 @@ jobs: if-no-files-found: error retention-days: 14 + Retention-Test-Suite-Trusted: + needs: [Precheck, Checkout, SetM365App] + environment: Testing + runs-on: ubuntu-latest + if: (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main') || (needs.precheck.outputs.srcfileschanged == 'true' && github.event.pull_request.head.repo.full_name == github.repository) + defaults: + run: + working-directory: src + env: + # Resolve the name of the secret that contains the Azure client ID/secret + AZURE_CLIENT_ID_NAME: ${{ needs.SetM365App.outputs.client_id_env }} + AZURE_CLIENT_SECRET_NAME: ${{ needs.SetM365App.outputs.client_secret_env }} + CLIENT_APP_SLOT: ${{ needs.SetM365App.outputs.client_app_slot }} + CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-ci-retention.log + LOG_GRAPH_REQUESTS: true + steps: + - uses: actions/checkout@v3 + + - name: Setup Golang with cache + uses: magnetikonline/action-golang-cache@v4 + with: + go-version-file: src/go.mod + + - run: mkdir testlog + + # Install gotestfmt + - name: Set up gotestfmt + run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest + + # AWS creds + - name: Configure AWS credentials from Test account + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + role-session-name: integration-testing + aws-region: us-east-1 + + # run the tests + - name: Retention Tests + env: + AZURE_CLIENT_ID: ${{ secrets[env.AZURE_CLIENT_ID_NAME] }} + AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }} + AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} + CORSO_RETENTION_TESTS: true + CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} + CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} + CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + S3_BUCKET: ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} + run: | + set -euo pipefail + go test \ + -tags testing \ + -json \ + -v \ + -failfast \ + -p 1 \ + -timeout 10m \ + ./... \ + 2>&1 | tee ./testlog/gotest-ci.log | gotestfmt -hide successful-tests + + # Upload the original go test output as an artifact for later review. + - name: Upload test log + if: failure() + uses: actions/upload-artifact@v3 + with: + name: ci-retention-test-log + path: src/testlog/* + if-no-files-found: error + retention-days: 14 + Unit-Test-Suite: needs: [Precheck, Checkout] environment: Testing diff --git a/src/go.mod b/src/go.mod index 6770ec9e4..99581228f 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,7 +2,7 @@ module github.com/alcionai/corso/src go 1.20 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230616023302-6c5412bbf417 +replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 @@ -33,7 +33,7 @@ require ( github.com/tomlazar/table v0.1.2 github.com/vbauerster/mpb/v8 v8.1.6 go.uber.org/zap v1.24.0 - golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/time v0.3.0 golang.org/x/tools v0.11.0 ) @@ -57,10 +57,12 @@ require ( github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -80,41 +82,41 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/microsoft/kiota-serialization-text-go v1.0.0 github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.52 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/minio-go/v7 v7.0.59 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/natefinch/atomic v1.0.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.15.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/rivo/uniseg v0.4.3 // indirect - github.com/rs/xid v1.4.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/segmentio/backo-go v1.0.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/tidwall/gjson v1.14.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.opentelemetry.io/otel v1.15.1 // indirect - go.opentelemetry.io/otel/trace v1.15.1 // indirect - go.uber.org/atomic v1.10.0 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.12.0 // indirect @@ -122,9 +124,8 @@ require ( golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/grpc v1.56.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/src/go.sum b/src/go.sum index 07fa2e51e..030dabe8a 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= @@ -48,15 +48,15 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 h1:KeNholpO2xKjgaaSyd+DyQRrsQjhbSeS7qe4nEw8aQw= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a h1:mtJyeK/FhArTn06M5Lfgxk/GWnu8yqCGNN1BY16vjaA= github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= -github.com/alcionai/kopia v0.12.2-0.20230616023302-6c5412bbf417 h1:s0B7Be1qqZW+XDRStDYvyXZ7ovQAMkg0N1F/ji4TJyc= -github.com/alcionai/kopia v0.12.2-0.20230616023302-6c5412bbf417/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI= +github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 h1:w50/aVU+zRP5lvE86TSSCCYrrEyuXOlJA06R5RdTS8Y= +github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377/go.mod h1:WH725ws0BYpZpTkVh4uqFHHPiiJuirl1Cm73jv5RYyA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -97,7 +97,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -237,16 +237,15 @@ github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= -github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kopia/htmluibuild v0.0.0-20230326183719-f482ef17e2c9 h1:s5Wa89s8RlPjuwqd8K8kuf+T9Kz4+NsbKwR/pJ3PAT0= +github.com/kopia/htmluibuild v0.0.0-20230605144737-e386b860759d h1:qvV3TN5X/RsgmckkxsKh9P7Vtf9GYy6vOPzQY1SY4qM= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -263,8 +262,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -291,10 +290,10 @@ github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8w github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= -github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/minio-go/v7 v7.0.59 h1:lxIXwsTIcQkYoEG25rUJbzpmSB/oWeVDmxFo/uWUUsw= +github.com/minio/minio-go/v7 v7.0.59/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -328,32 +327,32 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/puzpuzpuz/xsync/v2 v2.4.1 h1:aGdE1C/HaR/QC6YAFdtZXi60Df8/qBIrs8PKrzkItcM= github.com/puzpuzpuz/xsync/v2 v2.4.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -361,8 +360,8 @@ github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1 h1:lQ3JvmcVO1/AMFbabvUSJ4YtJRpEAX9Qza73p5j03sw= github.com/spatialcurrent/go-lazy v0.0.0-20211115014721-47315cc003d1/go.mod h1:4aKqcbhASNqjbrG0h9BmkzcWvPJGxbef4B+j0XfFrZo= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -420,7 +419,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4= +github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -433,12 +432,14 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8= -go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc= -go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY= -go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -466,8 +467,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -599,12 +600,13 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -742,8 +744,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -760,8 +762,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -774,8 +776,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 9da85195a..f65827f76 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -4,10 +4,13 @@ import ( "context" "errors" "strings" + "time" "github.com/alcionai/clues" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -16,6 +19,7 @@ import ( "github.com/kopia/kopia/snapshot/snapshotmaintenance" "github.com/alcionai/corso/src/internal/common/prefixmatcher" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/observe" @@ -710,3 +714,210 @@ func (w Wrapper) setMaintenanceParams( return nil } + +// SetRetentionParameters sets configuration values related to immutable backups +// and retention policies on the storage bucket. The minimum retention period +// must be >= 24hrs due to kopia default expectations about full maintenance. +func (w *Wrapper) SetRetentionParameters( + ctx context.Context, + retention repository.Retention, +) error { + if retention.Mode == nil && retention.Duration == nil && retention.Extend == nil { + return nil + } + + // Somewhat confusing case, when we have no retention but a non-zero duration + // it acts like we passed in only the duration and returns an error about + // having to set both. Return a clearer error here instead. Check if mode is + // set so we still allow changing duration if mode is already set. + if m, ok := ptr.ValOK(retention.Mode); ok && m == repository.NoRetention && ptr.Val(retention.Duration) != 0 { + return clues.New("duration must be 0 if retention is disabled").WithClues(ctx) + } + + dr, ok := w.c.Repository.(repo.DirectRepository) + if !ok { + return clues.New("getting handle to repo").WithClues(ctx) + } + + blobCfg, params, err := getRetentionConfigs(ctx, dr) + if err != nil { + return clues.Stack(err) + } + + // Update blob config information. + blobChanged, err := w.setBlobConfigParams(retention.Mode, retention.Duration, blobCfg) + if err != nil { + return clues.Wrap(err, "setting retention mode or duration").WithClues(ctx) + } + + // Update maintenance config information. + var maintenanceChanged bool + + if retention.Extend != nil && params.ExtendObjectLocks != *retention.Extend { + params.ExtendObjectLocks = *retention.Extend + maintenanceChanged = true + } + + // Check the new config is valid. + if blobCfg.IsRetentionEnabled() { + if err := maintenance.CheckExtendRetention(ctx, *blobCfg, params); err != nil { + return clues.Wrap(err, "invalid retention config").WithClues(ctx) + } + } + + return clues.Stack(persistRetentionConfigs( + ctx, + dr, + blobCfg, + blobChanged, + params, + maintenanceChanged, + )).OrNil() +} + +func getRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, +) (*format.BlobStorageConfiguration, *maintenance.Params, error) { + blobCfg, err := dr.FormatManager().BlobCfgBlob() + if err != nil { + return nil, nil, clues.Wrap(err, "getting storage config").WithClues(ctx) + } + + params, err := maintenance.GetParams(ctx, dr) + if err != nil { + return nil, nil, clues.Wrap(err, "getting maintenance config").WithClues(ctx) + } + + return &blobCfg, params, nil +} + +func persistRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, + blobCfg *format.BlobStorageConfiguration, + blobChanged bool, + params *maintenance.Params, + maintenanceChanged bool, +) error { + // Persist changes. + if !blobChanged && !maintenanceChanged { + return nil + } + + mp, err := dr.FormatManager().GetMutableParameters() + if err != nil { + return clues.Wrap(err, "getting mutable parameters") + } + + requiredFeatures, err := dr.FormatManager().RequiredFeatures() + if err != nil { + return clues.Wrap(err, "getting required features").WithClues(ctx) + } + + // Must be the case that only blob changed. + if !maintenanceChanged { + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), + "persisting storage config", + ).WithClues(ctx).OrNil() + } + + // Both blob and maintenance changed. A DirectWriteSession is required to + // update the maintenance config but not the blob config. + err = repo.DirectWriteSession( + ctx, + dr, + repo.WriteSessionOptions{ + Purpose: "Corso immutable backups config", + }, + func(ctx context.Context, dw repo.DirectRepositoryWriter) error { + // Set the maintenance config first as we can bail out of the write + // session later. + if err := maintenance.SetParams(ctx, dw, params); err != nil { + return clues.Wrap(err, "maintenance config"). + WithClues(ctx) + } + + if !blobChanged { + return nil + } + + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), + "storage config", + ).WithClues(ctx).OrNil() + }) + + return clues.Wrap(err, "persisting config changes").WithClues(ctx).OrNil() +} + +func (w Wrapper) setBlobConfigParams( + mode *repository.RetentionMode, + duration *time.Duration, + blobCfg *format.BlobStorageConfiguration, +) (bool, error) { + changed, err := setBlobConfigMode(mode, blobCfg) + if err != nil { + return false, clues.Stack(err) + } + + tmp := setBlobConfigDuration(duration, blobCfg) + changed = changed || tmp + + return changed, nil +} + +func setBlobConfigDuration( + duration *time.Duration, + blobCfg *format.BlobStorageConfiguration, +) bool { + var changed bool + + if duration != nil && blobCfg.RetentionPeriod != *duration { + blobCfg.RetentionPeriod = *duration + changed = true + } + + return changed +} + +func setBlobConfigMode( + mode *repository.RetentionMode, + blobCfg *format.BlobStorageConfiguration, +) (bool, error) { + if mode == nil { + return false, nil + } + + startMode := blobCfg.RetentionMode + + switch *mode { + case repository.NoRetention: + if !blobCfg.IsRetentionEnabled() { + return false, nil + } + + blobCfg.RetentionMode = "" + blobCfg.RetentionPeriod = 0 + + case repository.GovernanceRetention: + blobCfg.RetentionMode = blob.Governance + + case repository.ComplianceRetention: + blobCfg.RetentionMode = blob.Compliance + + default: + return false, clues.New("unknown retention mode"). + With("provided_retention_mode", mode.String()) + } + + // Only check if the retention mode is not empty. IsValid errors out if it's + // empty. + if len(blobCfg.RetentionMode) > 0 && !blobCfg.RetentionMode.IsValid() { + return false, clues.New("invalid retention mode"). + With("retention_mode", blobCfg.RetentionMode) + } + + return startMode != blobCfg.RetentionMode, nil +} diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 0146dc0c0..12857904f 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -6,10 +6,13 @@ import ( "io" stdpath "path" "testing" + "time" "github.com/alcionai/clues" "github.com/google/uuid" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -19,6 +22,7 @@ import ( "golang.org/x/exp/maps" pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" @@ -277,6 +281,342 @@ func (suite *BasicKopiaIntegrationSuite) TestMaintenance_WrongUser_Force_Succeed require.NoError(t, err, clues.ToCore(err)) } +// Test that failing to put the storage blob will skip updating the maintenance +// manifest too. It's still possible to end up halfway updating the repo config +// blobs as there's several of them, but at least this gives us something. +func (suite *BasicKopiaIntegrationSuite) TestSetRetentionParameters_NoChangesOnFailure() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + k, err := openKopiaRepo(t, ctx) + require.NoError(t, err, clues.ToCore(err)) + + w := &Wrapper{k} + + // Enable retention. + err = w.SetRetentionParameters(ctx, repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }) + require.Error(t, err, clues.ToCore(err)) + + checkRetentionParams( + t, + ctx, + k, + blob.RetentionMode(""), + 0, + assert.False) + + // Close and reopen the repo to make sure it's the same. + err = w.Close(ctx) + require.NoError(t, err, clues.ToCore(err)) + + k.Close(ctx) + require.NoError(t, err, clues.ToCore(err)) + + err = k.Connect(ctx, repository.Options{}) + require.NoError(t, err, clues.ToCore(err)) + + defer k.Close(ctx) + + checkRetentionParams( + t, + ctx, + k, + blob.RetentionMode(""), + 0, + assert.False) +} + +// --------------- +// integration tests that require object locking to be enabled on the bucket. +// --------------- +func mustGetBlobConfig(t *testing.T, c *conn) format.BlobStorageConfiguration { + require.Implements(t, (*repo.DirectRepository)(nil), c.Repository) + dr := c.Repository.(repo.DirectRepository) + + blobCfg, err := dr.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting repo config blob") + + return blobCfg +} + +func checkRetentionParams( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + c *conn, + expectMode blob.RetentionMode, + expectDuration time.Duration, + expectExtend assert.BoolAssertionFunc, +) { + blobCfg := mustGetBlobConfig(t, c) + + assert.Equal(t, expectMode, blobCfg.RetentionMode, "retention mode") + // Empty mode isn't considered valid so only check if it's non-empty. + if len(blobCfg.RetentionMode) > 0 { + assert.True(t, blobCfg.RetentionMode.IsValid(), "valid retention mode") + } + + assert.Equal(t, expectDuration, blobCfg.RetentionPeriod, "retention duration") + + params, err := maintenance.GetParams(ctx, c) + require.NoError(t, err, "getting maintenance config") + + expectExtend(t, params.ExtendObjectLocks, "extend object locks") +} + +// mustReopen closes and reopens the connection that w uses. Assumes no other +// structs besides w are holding a reference to the conn that w has. +// TODO(ashmrtn): Remove this when kopia caching is fixed. +// +//revive:disable-next-line:context-as-argument +func mustReopen(t *testing.T, ctx context.Context, w *Wrapper) { + k := w.c + + err := w.Close(ctx) + require.NoError(t, err, "closing wrapper: %v", clues.ToCore(err)) + + err = k.Close(ctx) + require.NoError(t, err, "closing conn: %v", clues.ToCore(err)) + + err = k.Connect(ctx, repository.Options{}) + require.NoError(t, err, "reconnecting conn: %v", clues.ToCore(err)) + + w.c = k +} + +type RetentionIntegrationSuite struct { + tester.Suite +} + +func TestRetentionIntegrationSuite(t *testing.T) { + suite.Run(t, &RetentionIntegrationSuite{ + Suite: tester.NewRetentionSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}, + ), + }) +} + +func (suite *RetentionIntegrationSuite) TestSetRetentionParameters() { + table := []struct { + name string + opts repository.Retention + expectErr assert.ErrorAssertionFunc + expectMode blob.RetentionMode + expectDuration time.Duration + expectExtend assert.BoolAssertionFunc + }{ + { + name: "NoChanges", + opts: repository.Retention{}, + expectErr: assert.NoError, + expectExtend: assert.False, + }, + { + name: "UpdateMode", + opts: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + }, + expectErr: assert.Error, + expectExtend: assert.False, + }, + { + name: "UpdateDuration", + opts: repository.Retention{ + Duration: ptr.To(time.Hour * 48), + }, + expectErr: assert.Error, + expectExtend: assert.False, + }, + { + name: "UpdateExtend", + opts: repository.Retention{ + Extend: ptr.To(true), + }, + expectErr: assert.NoError, + expectExtend: assert.True, + }, + { + name: "UpdateModeAndDuration_Governance", + opts: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + }, + expectErr: assert.NoError, + expectMode: blob.Governance, + expectDuration: time.Hour * 48, + expectExtend: assert.False, + }, + // Skip for now since compliance mode won't let us delete the blobs at all + // until they expire. + //{ + // name: "UpdateModeAndDuration_Compliance", + // opts: repository.Retention{ + // Mode: ptr.To(repository.ComplianceRetention), + // Duration: ptr.To(time.Hour * 48), + // }, + // expectErr: assert.NoError, + // expectMode: blob.Compliance, + // expectDuration: time.Hour * 48, + // expectExtend: assert.False, + //}, + { + name: "UpdateModeAndDuration_Invalid", + opts: repository.Retention{ + Mode: ptr.To(repository.RetentionMode(-1)), + Duration: ptr.To(time.Hour * 48), + }, + expectErr: assert.Error, + expectExtend: assert.False, + }, + { + name: "UpdateMode_NoRetention", + opts: repository.Retention{ + Mode: ptr.To(repository.NoRetention), + }, + expectErr: assert.NoError, + expectExtend: assert.False, + }, + { + name: "UpdateModeAndDuration_NoRetention", + opts: repository.Retention{ + Mode: ptr.To(repository.NoRetention), + Duration: ptr.To(time.Hour * 48), + }, + expectErr: assert.Error, + expectExtend: assert.False, + }, + { + name: "UpdateModeAndDurationAndExtend", + opts: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }, + expectErr: assert.NoError, + expectMode: blob.Governance, + expectDuration: time.Hour * 48, + expectExtend: assert.True, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + k, err := openKopiaRepo(t, ctx) + require.NoError(t, err, clues.ToCore(err)) + + w := &Wrapper{k} + + err = w.SetRetentionParameters(ctx, test.opts) + test.expectErr(t, err, clues.ToCore(err)) + + mustReopen(t, ctx, w) + checkRetentionParams( + t, + ctx, + k, + test.expectMode, + test.expectDuration, + test.expectExtend) + }) + } +} + +func (suite *RetentionIntegrationSuite) TestSetRetentionParameters_And_Maintenance() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + k, err := openKopiaRepo(t, ctx) + require.NoError(t, err, clues.ToCore(err)) + + w := &Wrapper{k} + + mOpts := repository.Maintenance{ + Safety: repository.FullMaintenanceSafety, + Type: repository.MetadataMaintenance, + } + + // This will set common maintenance config parameters. There's some interplay + // between the maintenance schedule and retention period that we want to check + // below. + err = w.RepoMaintenance(ctx, mOpts) + require.NoError(t, err, clues.ToCore(err)) + + // Enable retention. + err = w.SetRetentionParameters(ctx, repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }) + require.NoError(t, err, clues.ToCore(err)) + + mustReopen(t, ctx, w) + checkRetentionParams( + t, + ctx, + k, + blob.Governance, + time.Hour*48, + assert.True) + + // Change retention duration without updating mode. + err = w.SetRetentionParameters(ctx, repository.Retention{ + Duration: ptr.To(time.Hour * 49), + }) + require.NoError(t, err, clues.ToCore(err)) + + mustReopen(t, ctx, w) + checkRetentionParams( + t, + ctx, + k, + blob.Governance, + time.Hour*49, + assert.True) + + // Disable retention. + err = w.SetRetentionParameters(ctx, repository.Retention{ + Mode: ptr.To(repository.NoRetention), + }) + require.NoError(t, err, clues.ToCore(err)) + + mustReopen(t, ctx, w) + checkRetentionParams( + t, + ctx, + k, + blob.RetentionMode(""), + 0, + assert.True) + + // Disable object lock extension. + err = w.SetRetentionParameters(ctx, repository.Retention{ + Extend: ptr.To(false), + }) + require.NoError(t, err, clues.ToCore(err)) + + mustReopen(t, ctx, w) + checkRetentionParams( + t, + ctx, + k, + blob.RetentionMode(""), + 0, + assert.False) +} + // --------------- // integration tests that use kopia and initialize a repo // --------------- diff --git a/src/internal/tester/suite.go b/src/internal/tester/suite.go index f643b70d2..69cca7255 100644 --- a/src/internal/tester/suite.go +++ b/src/internal/tester/suite.go @@ -13,10 +13,11 @@ import ( // Flags for declaring which scope of tests to run. const ( - CorsoCITests = "CORSO_CI_TESTS" - CorsoE2ETests = "CORSO_E2E_TESTS" - CorsoLoadTests = "CORSO_LOAD_TESTS" - CorsoNightlyTests = "CORSO_NIGHTLY_TESTS" + CorsoCITests = "CORSO_CI_TESTS" + CorsoE2ETests = "CORSO_E2E_TESTS" + CorsoLoadTests = "CORSO_LOAD_TESTS" + CorsoNightlyTests = "CORSO_NIGHTLY_TESTS" + CorsoRetentionTests = "CORSO_RETENTION_TESTS" ) type Suite interface { @@ -152,6 +153,35 @@ type nightlySuite struct { suite.Suite } +// --------------------------------------------------------------------------- +// Retention; requires object locking enabled on the S3 bucket. +// --------------------------------------------------------------------------- + +func NewRetentionSuite( + t *testing.T, + envSets [][]string, + runOnAnyEnv ...string, +) *retentionSuite { + // ensure clues does not obscure logging + clues.SetHasher(clues.NoHash()) + + RunOnAny( + t, + append( + []string{CorsoRetentionTests}, + runOnAnyEnv..., + )..., + ) + + MustGetEnvSets(t, envSets...) + + return new(retentionSuite) +} + +type retentionSuite struct { + suite.Suite +} + // --------------------------------------------------------------------------- // Run Condition Checkers // --------------------------------------------------------------------------- diff --git a/src/internal/tester/suite_test.go b/src/internal/tester/suite_test.go index 74a6e095c..38b4afa18 100644 --- a/src/internal/tester/suite_test.go +++ b/src/internal/tester/suite_test.go @@ -93,3 +93,20 @@ func (suite *TesterNightlySuite) SetupSuite() { func (suite *TesterNightlySuite) TestNightlySuite() { require.True(suite.T(), suite.called) } + +type TesterRetentionSuite struct { + tester.Suite + called bool +} + +func TestTesterRetentionSuite(t *testing.T) { + suite.Run(t, &TesterRetentionSuite{Suite: tester.NewRetentionSuite(t, nil)}) +} + +func (suite *TesterRetentionSuite) SetupSuite() { + suite.called = true +} + +func (suite *TesterRetentionSuite) TestRetentionSuite() { + require.True(suite.T(), suite.called) +} diff --git a/src/pkg/control/repository/repo.go b/src/pkg/control/repository/repo.go index b65f8715b..720e35331 100644 --- a/src/pkg/control/repository/repo.go +++ b/src/pkg/control/repository/repo.go @@ -51,3 +51,25 @@ const ( // https://github.com/kopia/kopia/blob/f9de453efc198b6e993af8922f953a7e5322dc5f/repo/maintenance/maintenance_safety.go#L42 NoMaintenanceSafety ) + +type RetentionMode int + +// Can't be reordered as we rely on iota for numbering. +// +//go:generate stringer -type=RetentionMode -linecomment +const ( + UnknownRetention RetentionMode = 0 + NoRetention RetentionMode = 1 + GovernanceRetention RetentionMode = 2 + ComplianceRetention RetentionMode = 3 +) + +// Retention contains various options for configuring the retention mode. Takes +// pointers instead of values so that we can tell the difference between an +// unset value and a set but invalid value. This allows for partial +// (re)configuration of things. +type Retention struct { + Mode *RetentionMode + Duration *time.Duration + Extend *bool +} diff --git a/src/pkg/control/repository/retentionmode_string.go b/src/pkg/control/repository/retentionmode_string.go new file mode 100644 index 000000000..470647ff2 --- /dev/null +++ b/src/pkg/control/repository/retentionmode_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=RetentionMode -linecomment"; DO NOT EDIT. + +package repository + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[UnknownRetention-0] + _ = x[NoRetention-1] + _ = x[GovernanceRetention-2] + _ = x[ComplianceRetention-3] +} + +const _RetentionMode_name = "UnknownRetentionNoRetentionGovernanceRetentionComplianceRetention" + +var _RetentionMode_index = [...]uint8{0, 16, 27, 46, 65} + +func (i RetentionMode) String() string { + if i < 0 || i >= RetentionMode(len(_RetentionMode_index)-1) { + return "RetentionMode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _RetentionMode_name[_RetentionMode_index[i]:_RetentionMode_index[i+1]] +}