From 3b1a71902b40b69093721297db8abe8c1de8c97a Mon Sep 17 00:00:00 2001 From: Georgi Matev Date: Tue, 9 May 2023 12:44:44 -0700 Subject: [PATCH 01/33] Handle special case OD site name after recreating user (#3361) Special case Primary CI user LynneR for OneDrive CI cleanup --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/purge/scripts/onedrivePurge.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 4b72ebe8f..ae1acf328 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -131,6 +131,12 @@ if (![string]::IsNullOrEmpty($User)) { # Works for dev domains where format is @.onmicrosoft.com $domain = $User.Split('@')[1].Split('.')[0] $userNameEscaped = $User.Replace('.', '_').Replace('@', '_') + + # hacky special case because of recreated CI user + if ($userNameEscaped -ilike "lynner*") { + $userNameEscaped += '1' + } + $siteUrl = "https://$domain-my.sharepoint.com/personal/$userNameEscaped/" if ($LibraryNameList.count -eq 0) { From dbb3bd486dae5edb22320d36450a687580f56b18 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 9 May 2023 14:12:08 -0600 Subject: [PATCH 02/33] handle large item client redirects (#3350) adds the khttp redirectMiddleware to the http wrapper used for large item downloads, to ensure that proxy servers and other 3xx class responses appropriately follow their redirection. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included #### Type of change - [x] :bug: Bugfix #### Issue(s) * #3344 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 1 + src/internal/connector/graph/http_wrapper.go | 9 ++- .../connector/graph/http_wrapper_test.go | 69 +++++++++++++++++++ src/internal/connector/graph/service.go | 14 ++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 259289b6a..39811f5cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improve restore time on large restores by optimizing how items are loaded from the remote repository. - Remove exchange item filtering based on m365 item ID via the CLI. - OneDrive backups no longer include a user's non-default drives. +- OneDrive and SharePoint file downloads will properly redirect from 3xx responses. ## [v0.7.0] (beta) - 2023-05-02 diff --git a/src/internal/connector/graph/http_wrapper.go b/src/internal/connector/graph/http_wrapper.go index bc469c5f2..55e9f9556 100644 --- a/src/internal/connector/graph/http_wrapper.go +++ b/src/internal/connector/graph/http_wrapper.go @@ -140,13 +140,20 @@ func defaultTransport() http.RoundTripper { } func internalMiddleware(cc *clientConfig) []khttp.Middleware { - return []khttp.Middleware{ + mw := []khttp.Middleware{ &RetryMiddleware{ MaxRetries: cc.maxRetries, Delay: cc.minDelay, }, + khttp.NewRedirectHandler(), &LoggingMiddleware{}, &ThrottleControlMiddleware{}, &MetricsMiddleware{}, } + + if len(cc.appendMiddleware) > 0 { + mw = append(mw, cc.appendMiddleware...) + } + + return mw } diff --git a/src/internal/connector/graph/http_wrapper_test.go b/src/internal/connector/graph/http_wrapper_test.go index 483a5f0ba..d5edaf27d 100644 --- a/src/internal/connector/graph/http_wrapper_test.go +++ b/src/internal/connector/graph/http_wrapper_test.go @@ -2,9 +2,11 @@ package graph import ( "net/http" + "strings" "testing" "github.com/alcionai/clues" + khttp "github.com/microsoft/kiota-http-go" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -43,3 +45,70 @@ func (suite *HTTPWrapperIntgSuite) TestNewHTTPWrapper() { require.NotNil(t, resp) require.Equal(t, http.StatusOK, resp.StatusCode) } + +type mwForceResp struct { + err error + resp *http.Response + alternate func(*http.Request) (bool, *http.Response, error) +} + +func (mw *mwForceResp) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + ok, r, e := mw.alternate(req) + if ok { + return r, e + } + + return mw.resp, mw.err +} + +type HTTPWrapperUnitSuite struct { + tester.Suite +} + +func TestHTTPWrapperUnitSuite(t *testing.T) { + suite.Run(t, &HTTPWrapperUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() { + ctx, flush := tester.NewContext() + defer flush() + + var ( + t = suite.T() + uri = "https://graph.microsoft.com" + path = "/fnords/beaux/regard" + url = uri + path + ) + + // can't use gock for this, or else it'll short-circuit the transport, + // and thus skip all the middleware + hdr := http.Header{} + hdr.Set("Location", "localhost:99999999/smarfs") + toResp := &http.Response{ + StatusCode: 302, + Header: hdr, + } + mwResp := mwForceResp{ + resp: toResp, + alternate: func(req *http.Request) (bool, *http.Response, error) { + if strings.HasSuffix(req.URL.String(), "smarfs") { + return true, &http.Response{StatusCode: http.StatusOK}, nil + } + + return false, nil, nil + }, + } + + hw := NewHTTPWrapper(appendMiddleware(&mwResp)) + + resp, err := hw.Request(ctx, http.MethodGet, url, nil, nil) + + require.NoError(t, err, clues.ToCore(err)) + require.NotNil(t, resp) + // require.Equal(t, 1, calledCorrectly, "test server was called with expected path") + require.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 9db7fb825..288725831 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -173,6 +173,8 @@ type clientConfig struct { // The minimum delay in seconds between retries minDelay time.Duration overrideRetryCount bool + + appendMiddleware []khttp.Middleware } type Option func(*clientConfig) @@ -225,6 +227,14 @@ func MinimumBackoff(dur time.Duration) Option { } } +func appendMiddleware(mw ...khttp.Middleware) Option { + return func(c *clientConfig) { + if len(mw) > 0 { + c.appendMiddleware = mw + } + } +} + // --------------------------------------------------------------------------- // Middleware Control // --------------------------------------------------------------------------- @@ -257,5 +267,9 @@ func kiotaMiddlewares( &MetricsMiddleware{}, }...) + if len(cc.appendMiddleware) > 0 { + mw = append(mw, cc.appendMiddleware...) + } + return mw } From 88812dc70accddc4017b65f4d1b7cd99f8d05f2d Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 9 May 2023 15:21:37 -0600 Subject: [PATCH 03/33] run cli e2e on the nightly schedule, not CI (#3367) e2e tests are large, slow, and lack configuration control that prevents data restoration explosions. This moves those tests out of the standard CI and into the nightly test suite to be run on a less frequent cadence. The goal is to improve CI test speed and test stability. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Test Plan - [x] :green_heart: E2E --- .github/workflows/load_test.yml | 10 ++++++---- .github/workflows/nightly_test.yml | 5 +++-- src/cli/backup/exchange_e2e_test.go | 4 ---- src/cli/backup/onedrive_e2e_test.go | 8 ++------ src/cli/backup/sharepoint_e2e_test.go | 5 +---- src/cli/repo/s3_e2e_test.go | 1 - src/cli/restore/exchange_e2e_test.go | 4 +--- 7 files changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index 5cc1e3c05..9241b3d8f 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -1,10 +1,8 @@ name: Nightly Load Testing on: schedule: - # every day at 01:59 (01:59am) UTC - # - cron: "59 1 * * *" - # temp, for testing: every 4 hours - - cron: "0 */4 * * *" + # every day at 03:59 GMT (roughly 8pm PST) + - cron: "59 3 * * *" permissions: # required to retrieve AWS credentials @@ -20,6 +18,10 @@ jobs: Load-Tests: environment: Load Testing runs-on: ubuntu-latest + # Skipping load testing for now. They need some love to get up and + # running properly, and it's better to not fight for resources with + # tests that are guaranteed to fail. + if: false defaults: run: working-directory: src diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index ccc93fdce..2ab0b7b8d 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -3,8 +3,8 @@ on: workflow_dispatch: schedule: - # Run every day at 0 minutes and 0 hours (midnight GMT) - - cron: "0 0 * * *" + # Run every day at 04:00 GMT (roughly 8pm PST) + - cron: "0 4 * * *" permissions: # required to retrieve AWS credentials @@ -122,6 +122,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} CORSO_NIGHTLY_TESTS: true + CORSO_E2E_TESTS: true CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} diff --git a/src/cli/backup/exchange_e2e_test.go b/src/cli/backup/exchange_e2e_test.go index d135c8747..e5c60df2b 100644 --- a/src/cli/backup/exchange_e2e_test.go +++ b/src/cli/backup/exchange_e2e_test.go @@ -54,7 +54,6 @@ func TestNoBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &NoBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -120,7 +119,6 @@ func TestBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &BackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -235,7 +233,6 @@ func TestPreparedBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &PreparedBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -490,7 +487,6 @@ func TestBackupDeleteExchangeE2ESuite(t *testing.T) { Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, ), }) } diff --git a/src/cli/backup/onedrive_e2e_test.go b/src/cli/backup/onedrive_e2e_test.go index d41bbc1aa..9e6c134bc 100644 --- a/src/cli/backup/onedrive_e2e_test.go +++ b/src/cli/backup/onedrive_e2e_test.go @@ -44,9 +44,7 @@ func TestNoBackupOneDriveE2ESuite(t *testing.T) { suite.Run(t, &NoBackupOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } @@ -148,9 +146,7 @@ func TestBackupDeleteOneDriveE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/backup/sharepoint_e2e_test.go b/src/cli/backup/sharepoint_e2e_test.go index 4471e9755..09d65d90e 100644 --- a/src/cli/backup/sharepoint_e2e_test.go +++ b/src/cli/backup/sharepoint_e2e_test.go @@ -45,7 +45,6 @@ func TestNoBackupSharePointE2ESuite(t *testing.T) { suite.Run(t, &NoBackupSharePointE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -112,9 +111,7 @@ func TestBackupDeleteSharePointE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteSharePointE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index d5e6c992e..388b687e2 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -25,7 +25,6 @@ func TestS3E2ESuite(t *testing.T) { suite.Run(t, &S3E2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 0d9bf7b58..30114aa4f 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -48,9 +48,7 @@ func TestRestoreExchangeE2ESuite(t *testing.T) { suite.Run(t, &RestoreExchangeE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } From 66103892c59520538d4e90c3190504a54584e376 Mon Sep 17 00:00:00 2001 From: Vaibhav Kamra Date: Tue, 9 May 2023 14:40:10 -0700 Subject: [PATCH 04/33] Allow specifying a role to assume when initializing a storage provider (#3284) Allows caller to specify a IAM role to assume in the Kopia storage provider --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #2106 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- src/go.mod | 6 +++--- src/go.sum | 14 ++++++------- src/internal/kopia/s3.go | 4 ++++ src/internal/tester/config.go | 2 +- src/pkg/repository/repository_test.go | 29 +++++++++++++++++++++++++++ src/pkg/storage/storage.go | 27 +++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/go.mod b/src/go.mod index 526a5151d..44e95351c 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,7 +2,7 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f +replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 @@ -79,7 +79,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/reedsolomon v1.11.7 // indirect @@ -122,7 +122,7 @@ require ( golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.54.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/src/go.sum b/src/go.sum index fd158fd2f..9daad7bb8 100644 --- a/src/go.sum +++ b/src/go.sum @@ -55,8 +55,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c h1:Njdw/Nnq2DN3f8QMaHuZZHdVHTUSxFqPMMxDIInDWB4= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8= -github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f h1:cD7mcWVTEu83qX6Ml3aqgo8DDv+fBZt/7mQQps2TokM= -github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f/go.mod h1:eTgZSDaU2pDzVGC7QRubbKOeohvHzzbRXvhZMH+AGHA= +github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 h1:Wrl99Y7jftZMnNDiOIcRJrjstZO3IEj3+Q/sip27vmI= +github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -203,7 +203,7 @@ github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hanwen/go-fuse/v2 v2.2.0 h1:jo5QZYmBLNcl9ovypWaQ5yXMSSV+Ch68xoC3rtZvvBM= +github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= @@ -234,8 +234,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -741,8 +741,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/src/internal/kopia/s3.go b/src/internal/kopia/s3.go index 5810487dc..6b5c081d7 100644 --- a/src/internal/kopia/s3.go +++ b/src/internal/kopia/s3.go @@ -31,6 +31,10 @@ func s3BlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) Prefix: cfg.Prefix, DoNotUseTLS: cfg.DoNotUseTLS, DoNotVerifyTLS: cfg.DoNotVerifyTLS, + Tags: s.SessionTags, + SessionName: s.SessionName, + RoleARN: s.Role, + RoleDuration: s.SessionDuration, } store, err := s3.New(ctx, &opts, false) diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 8a002fd2c..14a4f54d9 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -106,7 +106,7 @@ func readTestConfig() (map[string]string, error) { testEnv := map[string]string{} fallbackTo(testEnv, TestCfgStorageProvider, vpr.GetString(TestCfgStorageProvider)) fallbackTo(testEnv, TestCfgAccountProvider, vpr.GetString(TestCfgAccountProvider)) - fallbackTo(testEnv, TestCfgBucket, vpr.GetString(TestCfgBucket), "test-corso-repo-init") + fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket), "test-corso-repo-init") fallbackTo(testEnv, TestCfgEndpoint, vpr.GetString(TestCfgEndpoint), "s3.amazonaws.com") fallbackTo(testEnv, TestCfgPrefix, vpr.GetString(TestCfgPrefix)) fallbackTo(testEnv, TestCfgAzureTenantID, os.Getenv(account.AzureTenantID), vpr.GetString(TestCfgAzureTenantID)) diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 649601142..1a80d6793 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -1,7 +1,9 @@ package repository_test import ( + "os" "testing" + "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" @@ -145,6 +147,33 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { } } +const ( + roleARNEnvKey = "CORSO_TEST_S3_ROLE" + roleDuration = time.Minute * 20 +) + +func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { + if _, ok := os.LookupEnv(roleARNEnvKey); !ok { + suite.T().Skip(roleARNEnvKey + " not set") + } + + ctx, flush := tester.NewContext() + defer flush() + + st := tester.NewPrefixedS3Storage(suite.T()) + + st.Role = os.Getenv(roleARNEnvKey) + st.SessionName = "corso-repository-test" + st.SessionDuration = roleDuration.String() + + r, err := repository.Initialize(ctx, account.Account{}, st, control.Options{}) + require.NoError(suite.T(), err) + + defer func() { + r.Close(ctx) + }() +} + func (suite *RepositoryIntegrationSuite) TestConnect() { ctx, flush := tester.NewContext() defer flush() diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index e635f9981..19cc9ddc7 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -36,6 +36,11 @@ const ( type Storage struct { Provider storageProvider Config map[string]string + // TODO: These are AWS S3 specific -> move these out + SessionTags map[string]string + Role string + SessionName string + SessionDuration string } // NewStorage aggregates all the supplied configurations into a single configuration. @@ -48,6 +53,28 @@ func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, er }, err } +// NewStorageUsingRole supports specifying an AWS IAM role the storage provider +// should assume. +func NewStorageUsingRole( + p storageProvider, + roleARN string, + sessionName string, + sessionTags map[string]string, + duration string, + cfgs ...common.StringConfigurer, +) (Storage, error) { + cs, err := common.UnionStringConfigs(cfgs...) + + return Storage{ + Provider: p, + Config: cs, + Role: roleARN, + SessionTags: sessionTags, + SessionName: sessionName, + SessionDuration: duration, + }, err +} + // Helper for parsing the values in a config object. // If the value is nil or not a string, returns an empty string. func orEmptyString(v any) string { From 33e57c0d5a56962828276a8d7bf7b864658f1cb8 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 9 May 2023 16:08:36 -0600 Subject: [PATCH 05/33] introduce per-service rate limiter configurations (#3357) Adds a context passdown that allows GC to define the service being queried at a high level, and the rate limiter to utilize different rate limiters based on that info. Malformed or missing limiter config uses the default limiter. --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #2951 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/connector/data_collections.go | 10 ++- src/internal/connector/graph/middleware.go | 54 +++++++++++++-- .../connector/graph/middleware_test.go | 68 ++++++++++++++++++- 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 9f0f738e5..e66846fef 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -49,6 +49,8 @@ func (gc *GraphConnector) ProduceBackupCollections( diagnostics.Index("service", sels.Service.String())) defer end() + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + // Limit the max number of active requests to graph from this collection. ctrlOpts.Parallelism.ItemFetch = graph.Parallelism(sels.PathService()). ItemOverride(ctx, ctrlOpts.Parallelism.ItemFetch) @@ -194,7 +196,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( ctx context.Context, backupVersion int, acct account.Account, - selector selectors.Selector, + sels selectors.Selector, dest control.RestoreDestination, opts control.Options, dcs []data.RestoreCollection, @@ -203,6 +205,8 @@ func (gc *GraphConnector) ConsumeRestoreCollections( ctx, end := diagnostics.Span(ctx, "connector:restore") defer end() + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + var ( status *support.ConnectorOperationStatus deets = &details.Builder{} @@ -213,7 +217,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( return nil, clues.Wrap(err, "malformed azure credentials") } - switch selector.Service { + switch sels.Service { case selectors.ServiceExchange: status, err = exchange.RestoreExchangeDataCollections(ctx, creds, gc.Service, dest, dcs, deets, errs) case selectors.ServiceOneDrive: @@ -221,7 +225,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( case selectors.ServiceSharePoint: status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets, errs) default: - err = clues.Wrap(clues.New(selector.Service.String()), "service not supported") + err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") } gc.incrementAwaitingMessages() diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index b1d4ad99f..004798cad 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" ) type nexter interface { @@ -369,18 +370,61 @@ func (mw RetryMiddleware) getRetryDelay( // the volume keeps up after that, we'll always stay between 9000 and 9900 out // of 10k. const ( - perSecond = 15 - maxCap = 900 + defaultPerSecond = 15 + defaultMaxCap = 900 + drivePerSecond = 15 + driveMaxCap = 1100 ) -// Single, global rate limiter at this time. Refinements for method (creates, -// versus reads) or service can come later. -var limiter = rate.NewLimiter(perSecond, maxCap) +var ( + driveLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) + // also used as the exchange service limiter + defaultLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) +) + +type LimiterCfg struct { + Service path.ServiceType +} + +type limiterCfgKey string + +const limiterCfgCtxKey limiterCfgKey = "corsoGraphRateLimiterCfg" + +func ctxLimiter(ctx context.Context) *rate.Limiter { + lc, ok := extractRateLimiterConfig(ctx) + if !ok { + return defaultLimiter + } + + switch lc.Service { + case path.OneDriveService, path.SharePointService: + return driveLimiter + default: + return defaultLimiter + } +} + +func BindRateLimiterConfig(ctx context.Context, lc LimiterCfg) context.Context { + return context.WithValue(ctx, limiterCfgCtxKey, lc) +} + +func extractRateLimiterConfig(ctx context.Context) (LimiterCfg, bool) { + l := ctx.Value(limiterCfgCtxKey) + if l == nil { + return LimiterCfg{}, false + } + + lc, ok := l.(LimiterCfg) + + return lc, ok +} // QueueRequest will allow the request to occur immediately if we're under the // 1k-calls-per-minute rate. Otherwise, the call will wait in a queue until // the next token set is available. func QueueRequest(ctx context.Context) { + limiter := ctxLimiter(ctx) + if err := limiter.Wait(ctx); err != nil { logger.CtxErr(ctx, err).Error("graph middleware waiting on the limiter") } diff --git a/src/internal/connector/graph/middleware_test.go b/src/internal/connector/graph/middleware_test.go index 0874a38f6..6ca660231 100644 --- a/src/internal/connector/graph/middleware_test.go +++ b/src/internal/connector/graph/middleware_test.go @@ -17,10 +17,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/time/rate" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/path" ) type mwReturns struct { @@ -132,9 +134,9 @@ func (suite *RetryMWIntgSuite) SetupSuite() { func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() { var ( - uri = "https://graph.microsoft.com" - path = "/v1.0/users/user/messages/foo" - url = uri + path + uri = "https://graph.microsoft.com" + urlPath = "/v1.0/users/user/messages/foo" + url = uri + urlPath ) tests := []struct { @@ -230,3 +232,63 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_RetryRequest_resetBodyAfter50 Post(ctx, body, nil) require.NoError(t, err, clues.ToCore(err)) } + +type MiddlewareUnitSuite struct { + tester.Suite +} + +func TestMiddlewareUnitSuite(t *testing.T) { + suite.Run(t, &MiddlewareUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *MiddlewareUnitSuite) TestBindExtractLimiterConfig() { + ctx, flush := tester.NewContext() + defer flush() + + // an unpopulated ctx should produce the default limiter + assert.Equal(suite.T(), defaultLimiter, ctxLimiter(ctx)) + + table := []struct { + name string + service path.ServiceType + expectOK require.BoolAssertionFunc + expectLimiter *rate.Limiter + }{ + { + name: "exchange", + service: path.ExchangeService, + expectLimiter: defaultLimiter, + }, + { + name: "oneDrive", + service: path.OneDriveService, + expectLimiter: driveLimiter, + }, + { + name: "sharePoint", + service: path.SharePointService, + expectLimiter: driveLimiter, + }, + { + name: "unknownService", + service: path.UnknownService, + expectLimiter: defaultLimiter, + }, + { + name: "badService", + service: path.ServiceType(-1), + expectLimiter: defaultLimiter, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + tctx := BindRateLimiterConfig(ctx, LimiterCfg{Service: test.service}) + lc, ok := extractRateLimiterConfig(tctx) + require.True(t, ok, "found rate limiter in ctx") + assert.Equal(t, test.service, lc.Service) + assert.Equal(t, test.expectLimiter, ctxLimiter(tctx)) + }) + } +} From e5b1291d36841611881f35dc716c8a7d0632ce82 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Tue, 9 May 2023 15:51:30 -0700 Subject: [PATCH 06/33] Fix parse error in sanity tests (#3349) #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/sanity-test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 20f830b82..e011069e0 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -427,7 +427,7 @@ jobs: id: sharepoint-test run: | set -euo pipefail - echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup create sharepoint \ --no-stats \ @@ -450,7 +450,7 @@ jobs: - name: Backup sharepoint list test run: | set -euo pipefail - echo -e "\nBackup List SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup List SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup list sharepoint \ --no-stats \ @@ -467,7 +467,7 @@ jobs: - name: Backup sharepoint list single backup test run: | set -euo pipefail - echo -e "\nBackup List single backup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup List single backup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup list sharepoint \ --no-stats \ @@ -486,7 +486,7 @@ jobs: id: sharepoint-restore-test run: | set -euo pipefail - echo -e "\nRestore SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nRestore SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso restore sharepoint \ --no-stats \ @@ -513,7 +513,7 @@ jobs: id: sharepoint-incremental-test run: | set -euo pipefail - echo -e "\nIncremental Backup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nIncremental Backup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup create sharepoint \ --no-stats \ @@ -537,7 +537,7 @@ jobs: id: sharepoint-incremental-restore-test run: | set -euo pipefail - echo -e "\nIncremental Restore SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nIncremental Restore SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso restore sharepoint \ --no-stats \ From 255c027c94af1b6d6daef32c38310b5cf44df59b Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Tue, 9 May 2023 17:29:14 -0700 Subject: [PATCH 07/33] Lint GitHub actions changes (#3359) Reduce errors when updating actions and workflows and hopefully stop silent failures for things --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/_filechange_checker.yml | 10 +++++- .github/workflows/actions-lint.yml | 39 +++++++++++++++++++++++ .github/workflows/nightly_test.yml | 1 - .github/workflows/website-publish.yml | 3 +- 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/actions-lint.yml diff --git a/.github/workflows/_filechange_checker.yml b/.github/workflows/_filechange_checker.yml index 8d02d1437..96e03c9d8 100644 --- a/.github/workflows/_filechange_checker.yml +++ b/.github/workflows/_filechange_checker.yml @@ -19,6 +19,7 @@ jobs: outputs: srcfileschanged: ${{ steps.srcchecker.outputs.srcfileschanged }} websitefileschanged: ${{ steps.websitechecker.outputs.websitefileschanged }} + actionsfileschanged: ${{ steps.actionschecker.outputs.actionsfileschanged }} steps: - uses: actions/checkout@v3 @@ -49,4 +50,11 @@ jobs: if: steps.dornycheck.outputs.src == 'true' || steps.dornycheck.outputs.website == 'true' || steps.dornycheck.outputs.actions == 'true' run: | echo "website or workflow file changes occurred" - echo websitefileschanged=true >> $GITHUB_OUTPUT \ No newline at end of file + echo websitefileschanged=true >> $GITHUB_OUTPUT + + - name: Check dorny for changes in workflow filepaths + id: actionschecker + if: steps.dornycheck.outputs.actions == 'true' + run: | + echo "workflow file changes occurred" + echo actionsfileschanged=true >> $GITHUB_OUTPUT diff --git a/.github/workflows/actions-lint.yml b/.github/workflows/actions-lint.yml new file mode 100644 index 000000000..95629a134 --- /dev/null +++ b/.github/workflows/actions-lint.yml @@ -0,0 +1,39 @@ +name: Lint GitHub actions +on: + workflow_dispatch: + + pull_request: + + push: + branches: [main] + tags: ["v*.*.*"] + +# cancel currently running jobs if a new version of the branch is pushed +concurrency: + group: actions-lint-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ---------------------------------------------------------------------------------------------------- + # --- Prechecks and Checkouts ------------------------------------------------------------------------ + # ---------------------------------------------------------------------------------------------------- + Precheck: + uses: alcionai/corso/.github/workflows/_filechange_checker.yml@main + + # ---------------------------------------------------------------------------------------------------- + # --- Workflow Action Linting ------------------------------------------------------------------------ + # ---------------------------------------------------------------------------------------------------- + + Actions-Lint: + needs: [Precheck] + environment: Testing + runs-on: ubuntu-latest + if: needs.precheck.outputs.actionsfileschanged == 'true' + steps: + - uses: actions/checkout@v3 + + - name: actionlint + uses: raven-actions/actionlint@v1 + with: + fail-on-error: true + cache: true diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index 2ab0b7b8d..22eddba52 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -50,7 +50,6 @@ jobs: environment: ${{ steps.environment.outputs.environment }} version: ${{ steps.version.outputs.version }} website-bucket: ${{ steps.website-bucket.outputs.website-bucket }} - website-cfid: ${{ steps.website-cfid.outputs.website-cfid }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/website-publish.yml b/.github/workflows/website-publish.yml index b53ed320d..dda3909e9 100644 --- a/.github/workflows/website-publish.yml +++ b/.github/workflows/website-publish.yml @@ -28,8 +28,7 @@ jobs: - name: Get version string id: version run: | - echo "set-output name=version::$(git describe --tags --abbrev=0)" - echo "::set-output name=version::$(git describe --tags --abbrev=0)" + echo version=$(git describe --tags --abbrev=0) | tee -a $GITHUB_OUTPUT # ---------------------------------------------------------------------------------------------------- # --- Website Linting ----------------------------------------------------------------------------------- From a162425c12ce579f952f5bfa45532e77a005fe56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 01:11:39 +0000 Subject: [PATCH 08/33] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.256=20to=201.44.260=20in=20/src=20(#?= =?UTF-8?q?3370)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.256 to 1.44.260.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.260 (2023-05-09)

Service Client Updates

  • service/application-autoscaling: Updates service API, documentation, and examples
  • service/glue: Updates service API and documentation
    • This release adds AmazonRedshift Source and Target nodes in addition to DynamicTransform OutputSchemas
  • service/sagemaker: Updates service API and documentation
    • This release includes support for (1) Provisioned Concurrency for Amazon SageMaker Serverless Inference and (2) UpdateEndpointWeightsAndCapacities API for Serverless endpoints.

Release v1.44.259 (2023-05-08)

Service Client Updates

  • service/glue: Updates service API and documentation
    • Support large worker types G.4x and G.8x for Glue Spark
  • service/guardduty: Updates service API and documentation
    • Add AccessDeniedException 403 Error message code to support 3 Tagging related APIs
  • service/iotsitewise: Updates service API and documentation
  • service/sts: Updates service documentation
    • Documentation updates for AWS Security Token Service.

SDK Bugs

  • restjson: Correct failure to deserialize errors.
    • Deserialize generic error information when no response body is present.

Release v1.44.258 (2023-05-05)

Service Client Updates

  • service/ec2: Updates service API
    • This release adds support the inf2 and trn1n instances. inf2 instances are purpose built for deep learning inference while trn1n instances are powered by AWS Trainium accelerators and they build on the capabilities of Trainium-powered trn1 instances.
  • service/inspector2: Updates service API, documentation, and paginators
  • service/mediatailor: Updates service API and documentation
  • service/sqs: Updates service API, documentation, and paginators
    • Revert previous SQS protocol change.

Release v1.44.257 (2023-05-04)

Service Client Updates

  • service/config: Updates service API
  • service/connect: Updates service API and documentation
  • service/ecs: Updates service API
    • Documentation update for new error type NamespaceNotFoundException for CreateCluster and UpdateCluster
  • service/monitoring: Updates service API and documentation
    • Adds support for filtering by metric names in CloudWatch Metric Streams.
  • service/network-firewall: Updates service API and documentation
  • service/opensearch: Updates service API and documentation
  • service/quicksight: Updates service API, documentation, and paginators

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.256&new-version=1.44.260)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 44e95351c..90d9b61f4 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.256 + github.com/aws/aws-sdk-go v1.44.260 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 9daad7bb8..b86aff848 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= -github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.260 h1:78IJkDpDPXvLXvIkNAKDP/i3z8Vj+3sTAtQYw/v/2o8= +github.com/aws/aws-sdk-go v1.44.260/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From b508ea3b72067337bf7ac93f9ee52cb29202cafa Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Tue, 9 May 2023 18:50:38 -0700 Subject: [PATCH 09/33] Update kopia restore logic to take a (RepoRef, Collection Restore Path) pair (#3337) Begin expanding the restore logic to take a pair of paths, one denoting the precise location of the item in kopia and the other denoting the "restore location" or path the item should be placed at during restore This PR is not expected to change system functionality at all This is the first of 2 PRs to setup all the logic for this. This PR does not handle properly merging together multiple collections that have the same restore location but different RepoRefs due to recent updates to the kopia wrapper restore logic --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3197 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/connector/onedrive/restore.go | 79 ++++- .../connector/onedrive/restore_test.go | 126 +++++++- src/internal/kopia/merge_collection.go | 112 +++++++ src/internal/kopia/merge_collection_test.go | 297 ++++++++++++++++++ src/internal/kopia/wrapper.go | 123 +++++--- src/internal/kopia/wrapper_test.go | 225 ++++++++++++- .../operations/backup_integration_test.go | 17 +- src/internal/operations/backup_test.go | 24 +- src/internal/operations/inject/inject.go | 2 +- src/internal/operations/manifests.go | 11 +- src/internal/operations/restore.go | 17 +- src/internal/streamstore/streamstore.go | 12 +- src/pkg/path/path.go | 7 + 13 files changed, 958 insertions(+), 94 deletions(-) create mode 100644 src/internal/kopia/merge_collection.go create mode 100644 src/internal/kopia/merge_collection_test.go diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 0cff8b465..3f34cc9c4 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -781,17 +781,29 @@ func getMetadata(metar io.ReadCloser) (metadata.Metadata, error) { // Augment restore path to add extra files(meta) needed for restore as // well as do any other ordering operations on the paths -func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, error) { - colPaths := map[string]path.Path{} +// +// Only accepts StoragePath/RestorePath pairs where the RestorePath is +// at least as long as the StoragePath. If the RestorePath is longer than the +// StoragePath then the first few (closest to the root) directories will use +// default permissions during restore. +func AugmentRestorePaths( + backupVersion int, + paths []path.RestorePaths, +) ([]path.RestorePaths, error) { + // Keyed by each value's StoragePath.String() which corresponds to the RepoRef + // of the directory. + colPaths := map[string]path.RestorePaths{} for _, p := range paths { + first := true + for { - np, err := p.Dir() + sp, err := p.StoragePath.Dir() if err != nil { return nil, err } - drivePath, err := path.ToDrivePath(np) + drivePath, err := path.ToDrivePath(sp) if err != nil { return nil, err } @@ -800,8 +812,31 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err break } - colPaths[np.String()] = np - p = np + if len(p.RestorePath.Elements()) < len(sp.Elements()) { + return nil, clues.New("restorePath shorter than storagePath"). + With("restore_path", p.RestorePath, "storage_path", sp) + } + + rp := p.RestorePath + + // Make sure the RestorePath always points to the level of the current + // collection. We need to track if it's the first iteration because the + // RestorePath starts out at the collection level to begin with. + if !first { + rp, err = p.RestorePath.Dir() + if err != nil { + return nil, err + } + } + + paths := path.RestorePaths{ + StoragePath: sp, + RestorePath: rp, + } + + colPaths[sp.String()] = paths + p = paths + first = false } } @@ -814,32 +849,45 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err // As of now look up metadata for parent directories from a // collection. for _, p := range colPaths { - el := p.Elements() + el := p.StoragePath.Elements() if backupVersion >= version.OneDrive6NameInMeta { - mPath, err := p.Append(".dirmeta", true) + mPath, err := p.StoragePath.Append(".dirmeta", true) if err != nil { return nil, err } - paths = append(paths, mPath) + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) } else if backupVersion >= version.OneDrive4DirIncludesPermissions { - mPath, err := p.Append(el[len(el)-1]+".dirmeta", true) + mPath, err := p.StoragePath.Append(el[len(el)-1]+".dirmeta", true) if err != nil { return nil, err } - paths = append(paths, mPath) + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) } else if backupVersion >= version.OneDrive1DataAndMetaFiles { - pp, err := p.Dir() + pp, err := p.StoragePath.Dir() if err != nil { return nil, err } + mPath, err := pp.Append(el[len(el)-1]+".dirmeta", true) if err != nil { return nil, err } - paths = append(paths, mPath) + + prp, err := p.RestorePath.Dir() + if err != nil { + return nil, err + } + + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: prp}) } } @@ -847,8 +895,11 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err // files. This is only a necessity for OneDrive as we are storing // metadata for files/folders in separate meta files and we the // data to be restored before we can restore the metadata. + // + // This sorting assumes stuff in the same StoragePath directory end up in the + // same RestorePath collection. sort.Slice(paths, func(i, j int) bool { - return paths[i].String() < paths[j].String() + return paths[i].StoragePath.String() < paths[j].StoragePath.String() }) return paths, nil diff --git a/src/internal/connector/onedrive/restore_test.go b/src/internal/connector/onedrive/restore_test.go index 56e5d467b..c085d689f 100644 --- a/src/internal/connector/onedrive/restore_test.go +++ b/src/internal/connector/onedrive/restore_test.go @@ -172,20 +172,30 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { base := "id/onedrive/user/files/drives/driveID/root:/" - inPaths := []path.Path{} + inPaths := []path.RestorePaths{} for _, ps := range test.input { p, err := path.FromDataLayerPath(base+ps, true) require.NoError(t, err, "creating path", clues.ToCore(err)) - inPaths = append(inPaths, p) + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) } - outPaths := []path.Path{} + outPaths := []path.RestorePaths{} for _, ps := range test.output { p, err := path.FromDataLayerPath(base+ps, true) require.NoError(t, err, "creating path", clues.ToCore(err)) - outPaths = append(outPaths, p) + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) } actual, err := AugmentRestorePaths(test.version, inPaths) @@ -197,3 +207,111 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { }) } } + +// TestAugmentRestorePaths_DifferentRestorePath tests that RestorePath +// substitution works properly. Since it's only possible for future backup +// versions to need restore path substitution (i.e. due to storing folders by +// ID instead of name) this is only tested against the most recent backup +// version at the moment. +func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { + // Adding a simple test here so that we can be sure that this + // function gets updated whenever we add a new version. + require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") + + type pathPair struct { + storage string + restore string + } + + table := []struct { + name string + version int + input []pathPair + output []pathPair + errCheck assert.ErrorAssertionFunc + }{ + { + name: "nested folders", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "folder"}, + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/.dirmeta", restore: "folder/folder2"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path longer one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "corso_restore/folder"}, + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path shorter one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: ""}, + }, + errCheck: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + _, flush := tester.NewContext() + defer flush() + + base := "id/onedrive/user/files/drives/driveID/root:/" + + inPaths := []path.RestorePaths{} + for _, ps := range test.input { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + outPaths := []path.RestorePaths{} + for _, ps := range test.output { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + actual, err := AugmentRestorePaths(test.version, inPaths) + test.errCheck(t, err, "augmenting paths", clues.ToCore(err)) + + if err != nil { + return + } + + // Ordering of paths matter here as we need dirmeta files + // to show up before file in dir + assert.Equal(t, outPaths, actual, "augmented paths") + }) + } +} diff --git a/src/internal/kopia/merge_collection.go b/src/internal/kopia/merge_collection.go new file mode 100644 index 000000000..ab95dead8 --- /dev/null +++ b/src/internal/kopia/merge_collection.go @@ -0,0 +1,112 @@ +package kopia + +import ( + "context" + "errors" + + "github.com/alcionai/clues" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +var _ data.RestoreCollection = &mergeCollection{} + +type col struct { + storagePath string + data.RestoreCollection +} + +type mergeCollection struct { + cols []col + // Technically don't need to track this but it can help detect errors. + fullPath path.Path +} + +func (mc *mergeCollection) addCollection( + storagePath string, + c data.RestoreCollection, +) error { + if c == nil { + return clues.New("adding nil collection"). + With("current_path", mc.FullPath()) + } else if mc.FullPath().String() != c.FullPath().String() { + return clues.New("attempting to merge collection with different path"). + With("current_path", mc.FullPath(), "new_path", c.FullPath()) + } + + mc.cols = append(mc.cols, col{storagePath: storagePath, RestoreCollection: c}) + + // Keep a stable sorting of this merged collection set so we can say there's + // some deterministic behavior when Fetch is called. We don't expect to have + // to merge many collections. + slices.SortStableFunc(mc.cols, func(a, b col) bool { + return a.storagePath < b.storagePath + }) + + return nil +} + +func (mc mergeCollection) FullPath() path.Path { + return mc.fullPath +} + +func (mc *mergeCollection) Items( + ctx context.Context, + errs *fault.Bus, +) <-chan data.Stream { + res := make(chan data.Stream) + + go func() { + defer close(res) + + logger.Ctx(ctx).Infow( + "getting items for merged collection", + "merged_collection_count", len(mc.cols)) + + for _, c := range mc.cols { + // Unfortunately doesn't seem to be a way right now to see if the + // iteration failed and we should be exiting early. + ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + logger.Ctx(ictx).Debug("sending items from merged collection") + + for item := range c.Items(ictx, errs) { + res <- item + } + } + }() + + return res +} + +// Fetch goes through all the collections in this one and returns the first +// match found or the first error that is not data.ErrNotFound. If multiple +// collections have the requested item, the instance in the collection with the +// lexicographically smallest storage path is returned. +func (mc *mergeCollection) Fetch( + ctx context.Context, + name string, +) (data.Stream, error) { + logger.Ctx(ctx).Infow( + "fetching item in merged collection", + "merged_collection_count", len(mc.cols)) + + for _, c := range mc.cols { + ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + + logger.Ctx(ictx).Debug("looking for item in merged collection") + + s, err := c.Fetch(ictx, name) + if err == nil { + return s, nil + } else if err != nil && !errors.Is(err, data.ErrNotFound) { + return nil, clues.Wrap(err, "fetching from merged collection"). + WithClues(ictx) + } + } + + return nil, clues.Wrap(data.ErrNotFound, "merged collection fetch") +} diff --git a/src/internal/kopia/merge_collection_test.go b/src/internal/kopia/merge_collection_test.go new file mode 100644 index 000000000..e287452dc --- /dev/null +++ b/src/internal/kopia/merge_collection_test.go @@ -0,0 +1,297 @@ +package kopia + +import ( + "bytes" + "io" + "testing" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/connector/exchange/mock" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type MergeCollectionUnitSuite struct { + tester.Suite +} + +func TestMergeCollectionUnitSuite(t *testing.T) { + suite.Run(t, &MergeCollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *MergeCollectionUnitSuite) TestReturnsPath() { + t := suite.T() + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + c := mergeCollection{ + fullPath: pth, + } + + assert.Equal(t, pth, c.FullPath()) +} + +func (suite *MergeCollectionUnitSuite) TestItems() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + storagePaths := []string{ + "tenant-id/exchange/user-id/mail/some/folder/path1", + "tenant-id/exchange/user-id/mail/some/folder/path2", + } + + expectedItemNames := []string{"1", "2"} + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + c1 := mock.NewCollection(pth, nil, 1) + c1.Names[0] = expectedItemNames[0] + + c2 := mock.NewCollection(pth, nil, 1) + c2.Names[0] = expectedItemNames[1] + + // Not testing fetch here so safe to use this wrapper. + cols := []data.RestoreCollection{ + data.NotFoundRestoreCollection{Collection: c1}, + data.NotFoundRestoreCollection{Collection: c2}, + } + + dc := &mergeCollection{fullPath: pth} + + for i, c := range cols { + err := dc.addCollection(storagePaths[i], c) + require.NoError(t, err, "adding collection", clues.ToCore(err)) + } + + gotItemNames := []string{} + + for item := range dc.Items(ctx, fault.New(true)) { + gotItemNames = append(gotItemNames, item.UUID()) + } + + assert.ElementsMatch(t, expectedItemNames, gotItemNames) +} + +func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() { + t := suite.T() + + pth1, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + pth2, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data2") + require.NoError(t, err, clues.ToCore(err)) + + dc := mergeCollection{fullPath: pth1} + + err = dc.addCollection("some/path", &kopiaDataCollection{path: pth2}) + assert.Error(t, err, clues.ToCore(err)) +} + +func (suite *MergeCollectionUnitSuite) TestFetch() { + var ( + fileData1 = []byte("abcdefghijklmnopqrstuvwxyz") + fileData2 = []byte("zyxwvutsrqponmlkjihgfedcba") + fileData3 = []byte("foo bar baz") + + fileName1 = "file1" + fileName2 = "file2" + fileLookupErrName = "errLookup" + fileOpenErrName = "errOpen" + + colPaths = []string{ + "tenant-id/exchange/user-id/mail/some/data/directory1", + "tenant-id/exchange/user-id/mail/some/data/directory2", + } + ) + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(suite.T(), err, clues.ToCore(err)) + + // Needs to be a function so the readers get refreshed each time. + layouts := []func() fs.Directory{ + // Has the following; + // - file1: data[0] + // - errOpen: (error opening file) + func() fs.Directory { + return virtualfs.NewStaticDirectory(encodeAsPath(colPaths[0]), []fs.Entry{ + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName1), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData1)), + ), + size: int64(len(fileData1) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileOpenErrName), + nil, + ), + openErr: assert.AnError, + }, + }) + }, + + // Has the following; + // - file1: data[1] + // - file2: data[0] + // - errOpen: data[2] + func() fs.Directory { + return virtualfs.NewStaticDirectory(encodeAsPath(colPaths[1]), []fs.Entry{ + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName1), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData2)), + ), + size: int64(len(fileData2) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName2), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData1)), + ), + size: int64(len(fileData1) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileOpenErrName), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData3)), + ), + size: int64(len(fileData3) + versionSize), + }, + }) + }, + } + + table := []struct { + name string + fileName string + expectError assert.ErrorAssertionFunc + expectData []byte + notFoundErr bool + }{ + { + name: "Duplicate File, first collection", + fileName: fileName1, + expectError: assert.NoError, + expectData: fileData1, + }, + { + name: "Distinct File, second collection", + fileName: fileName2, + expectError: assert.NoError, + expectData: fileData1, + }, + { + name: "Error opening file", + fileName: fileOpenErrName, + expectError: assert.Error, + }, + { + name: "File not found", + fileName: fileLookupErrName, + expectError: assert.Error, + notFoundErr: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + c := &i64counter{} + + dc := mergeCollection{fullPath: pth} + + for i, layout := range layouts { + col := &kopiaDataCollection{ + path: pth, + dir: layout(), + counter: c, + expectedVersion: serializationVersion, + } + + err := dc.addCollection(colPaths[i], col) + require.NoError(t, err, "adding collection", clues.ToCore(err)) + } + + s, err := dc.Fetch(ctx, test.fileName) + test.expectError(t, err, clues.ToCore(err)) + + if err != nil { + if test.notFoundErr { + assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err)) + } + + return + } + + fileData, err := io.ReadAll(s.ToReader()) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + if err != nil { + return + } + + assert.Equal(t, test.expectData, fileData) + }) + } +} diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index e35d61cb6..e4d73bb4c 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -365,6 +365,11 @@ type ByteCounter interface { Count(numBytes int64) } +type restoreCollection struct { + restorePath path.Path + storageDirs map[string]*dirAndItems +} + type dirAndItems struct { dir path.Path items []string @@ -380,7 +385,7 @@ func loadDirsAndItems( ctx context.Context, snapshotRoot fs.Entry, bcounter ByteCounter, - toLoad map[string]*dirAndItems, + toLoad map[string]*restoreCollection, bus *fault.Bus, ) ([]data.RestoreCollection, error) { var ( @@ -389,50 +394,67 @@ func loadDirsAndItems( loadCount = 0 ) - for _, dirItems := range toLoad { + for _, col := range toLoad { if el.Failure() != nil { return nil, el.Failure() } - ictx := clues.Add(ctx, "directory_path", dirItems.dir) + ictx := clues.Add(ctx, "restore_path", col.restorePath) - dir, err := getDir(ictx, dirItems.dir, snapshotRoot) - if err != nil { - el.AddRecoverable(clues.Wrap(err, "loading directory"). - WithClues(ictx). - Label(fault.LabelForceNoBackupCreation)) + mergeCol := &mergeCollection{fullPath: col.restorePath} + res = append(res, mergeCol) - continue - } - - dc := &kopiaDataCollection{ - path: dirItems.dir, - dir: dir, - counter: bcounter, - expectedVersion: serializationVersion, - } - - res = append(res, dc) - - for _, item := range dirItems.items { + for _, dirItems := range col.storageDirs { if el.Failure() != nil { return nil, el.Failure() } - err := dc.addStream(ictx, item) + ictx = clues.Add(ictx, "storage_directory_path", dirItems.dir) + + dir, err := getDir(ictx, dirItems.dir, snapshotRoot) if err != nil { - el.AddRecoverable(clues.Wrap(err, "loading item"). + el.AddRecoverable(clues.Wrap(err, "loading storage directory"). WithClues(ictx). Label(fault.LabelForceNoBackupCreation)) continue } - loadCount++ - if loadCount%1000 == 0 { - logger.Ctx(ctx).Infow( - "loading items from kopia", - "loaded_items", loadCount) + dc := &kopiaDataCollection{ + path: col.restorePath, + dir: dir, + counter: bcounter, + expectedVersion: serializationVersion, + } + + if err := mergeCol.addCollection(dirItems.dir.String(), dc); err != nil { + el.AddRecoverable(clues.Wrap(err, "adding collection to merge collection"). + WithClues(ctx). + Label(fault.LabelForceNoBackupCreation)) + + continue + } + + for _, item := range dirItems.items { + if el.Failure() != nil { + return nil, el.Failure() + } + + err := dc.addStream(ictx, item) + if err != nil { + el.AddRecoverable(clues.Wrap(err, "loading item"). + WithClues(ictx). + Label(fault.LabelForceNoBackupCreation)) + + continue + } + + loadCount++ + if loadCount%1000 == 0 { + logger.Ctx(ctx).Infow( + "loading items from kopia", + "loaded_items", loadCount) + } } } } @@ -454,7 +476,7 @@ func loadDirsAndItems( func (w Wrapper) ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bcounter ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) { @@ -474,36 +496,53 @@ func (w Wrapper) ProduceRestoreCollections( var ( loadCount int - // Directory path -> set of items to load from the directory. - dirsToItems = map[string]*dirAndItems{} + // RestorePath -> []StoragePath directory -> set of items to load from the + // directory. + dirsToItems = map[string]*restoreCollection{} el = errs.Local() ) - for _, itemPath := range paths { + for _, itemPaths := range paths { if el.Failure() != nil { return nil, el.Failure() } - // Group things by directory so we can load all items from a single - // directory instance lower down. - ictx := clues.Add(ctx, "item_path", itemPath.String()) + // Group things by RestorePath and then StoragePath so we can load multiple + // items from a single directory instance lower down. + ictx := clues.Add( + ctx, + "item_path", itemPaths.StoragePath.String(), + "restore_path", itemPaths.RestorePath.String()) - parentPath, err := itemPath.Dir() + parentStoragePath, err := itemPaths.StoragePath.Dir() if err != nil { - el.AddRecoverable(clues.Wrap(err, "making directory collection"). + el.AddRecoverable(clues.Wrap(err, "getting storage directory path"). WithClues(ictx). Label(fault.LabelForceNoBackupCreation)) continue } - di := dirsToItems[parentPath.ShortRef()] - if di == nil { - dirsToItems[parentPath.ShortRef()] = &dirAndItems{dir: parentPath} - di = dirsToItems[parentPath.ShortRef()] + // Find the location this item is restored to. + rc := dirsToItems[itemPaths.RestorePath.ShortRef()] + if rc == nil { + dirsToItems[itemPaths.RestorePath.ShortRef()] = &restoreCollection{ + restorePath: itemPaths.RestorePath, + storageDirs: map[string]*dirAndItems{}, + } + rc = dirsToItems[itemPaths.RestorePath.ShortRef()] } - di.items = append(di.items, itemPath.Item()) + // Find the collection this item is sourced from. + di := rc.storageDirs[parentStoragePath.ShortRef()] + if di == nil { + rc.storageDirs[parentStoragePath.ShortRef()] = &dirAndItems{ + dir: parentStoragePath, + } + di = rc.storageDirs[parentStoragePath.ShortRef()] + } + + di.items = append(di.items, itemPaths.StoragePath.Item()) loadCount++ if loadCount%1000 == 0 { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 67540aec7..abe96fdc2 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -59,14 +59,12 @@ var ( testFileData6 = testFileData ) -//revive:disable:context-as-argument func testForFiles( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument expected map[string][]byte, collections []data.RestoreCollection, ) { - //revive:enable:context-as-argument t.Helper() count := 0 @@ -107,6 +105,19 @@ func checkSnapshotTags( assert.Equal(t, expectedTags, man.Tags) } +func toRestorePaths(t *testing.T, paths ...path.Path) []path.RestorePaths { + res := make([]path.RestorePaths, 0, len(paths)) + + for _, p := range paths { + dir, err := p.Dir() + require.NoError(t, err, clues.ToCore(err)) + + res = append(res, path.RestorePaths{StoragePath: p, RestorePath: dir}) + } + + return res +} + // --------------- // unit tests // --------------- @@ -705,10 +716,7 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { result, err := w.ProduceRestoreCollections( ctx, string(stats.SnapshotID), - []path.Path{ - fp1, - fp2, - }, + toRestorePaths(t, fp1, fp2), nil, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -838,7 +846,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { _, err = suite.w.ProduceRestoreCollections( suite.ctx, string(stats.SnapshotID), - []path.Path{failedPath}, + toRestorePaths(t, failedPath), &ic, fault.New(true)) // Files that had an error shouldn't make a dir entry in kopia. If they do we @@ -1219,9 +1227,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { _, err = suite.w.ProduceRestoreCollections( suite.ctx, string(stats.SnapshotID), - []path.Path{ - suite.files[suite.testPath1.String()][0].itemPath, - }, + toRestorePaths(t, suite.files[suite.testPath1.String()][0].itemPath), &ic, fault.New(true)) test.restoreCheck(t, err, clues.ToCore(err)) @@ -1322,7 +1328,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { result, err := suite.w.ProduceRestoreCollections( suite.ctx, string(suite.snapshotID), - test.inputPaths, + toRestorePaths(t, test.inputPaths...), &ic, fault.New(true)) test.expectedErr(t, err, clues.ToCore(err)) @@ -1338,6 +1344,193 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { } } +// TestProduceRestoreCollections_PathChanges tests that having different +// Restore and Storage paths works properly. Having the same Restore and Storage +// paths is tested by TestProduceRestoreCollections. +func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_PathChanges() { + rp1, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Inbox") + require.NoError(suite.T(), err) + + rp2, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Archive") + require.NoError(suite.T(), err) + + // Expected items is generated during the test by looking up paths in the + // suite's map of files. + table := []struct { + name string + inputPaths []path.RestorePaths + expectedCollections int + }{ + { + name: "SingleItem", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + { + name: "MultipleItemsSameCollection", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath1.String()][1].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + { + name: "MultipleItemsDifferentCollections", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp2, + }, + }, + expectedCollections: 2, + }, + { + name: "Multiple Items From Different Collections To Same Collection", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + expected := make(map[string][]byte, len(test.inputPaths)) + + for _, pth := range test.inputPaths { + item, ok := suite.filesByPath[pth.StoragePath.String()] + require.True(t, ok, "getting expected file data") + + itemPath, err := pth.RestorePath.Append(pth.StoragePath.Item(), true) + require.NoError(t, err, "getting expected item path") + + expected[itemPath.String()] = item.data + } + + ic := i64counter{} + + result, err := suite.w.ProduceRestoreCollections( + suite.ctx, + string(suite.snapshotID), + test.inputPaths, + &ic, + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + + assert.Len(t, result, test.expectedCollections) + assert.Less(t, int64(0), ic.i) + testForFiles(t, ctx, expected, result) + }) + } +} + +// TestProduceRestoreCollections_Fetch tests that the Fetch function still works +// properly even with different Restore and Storage paths and items from +// different kopia directories. +func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Fetch() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + + rp1, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Inbox") + require.NoError(suite.T(), err) + + inputPaths := []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp1, + }, + } + + // Really only interested in getting the collection so we can call fetch on + // it. + ic := i64counter{} + + result, err := suite.w.ProduceRestoreCollections( + suite.ctx, + string(suite.snapshotID), + inputPaths, + &ic, + fault.New(true)) + require.NoError(t, err, "getting collection", clues.ToCore(err)) + require.Len(t, result, 1) + + // Item from first kopia directory. + f := suite.files[suite.testPath1.String()][0] + + item, err := result[0].Fetch(ctx, f.itemPath.Item()) + require.NoError(t, err, "fetching file", clues.ToCore(err)) + + r := item.ToReader() + + buf, err := io.ReadAll(r) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + assert.Equal(t, f.data, buf) + + // Item from second kopia directory. + f = suite.files[suite.testPath2.String()][0] + + item, err = result[0].Fetch(ctx, f.itemPath.Item()) + require.NoError(t, err, "fetching file", clues.ToCore(err)) + + r = item.ToReader() + + buf, err = io.ReadAll(r) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + assert.Equal(t, f.data, buf) +} + func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Errors() { itemPath, err := suite.testPath1.Append(testFileName, true) require.NoError(suite.T(), err, clues.ToCore(err)) @@ -1345,7 +1538,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Erro table := []struct { name string snapshotID string - paths []path.Path + paths []path.RestorePaths }{ { "NilPaths", @@ -1355,12 +1548,12 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Erro { "EmptyPaths", string(suite.snapshotID), - []path.Path{}, + []path.RestorePaths{}, }, { "NoSnapshot", "foo", - []path.Path{itemPath}, + toRestorePaths(suite.T(), itemPath), }, } @@ -1393,7 +1586,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() { c, err := suite.w.ProduceRestoreCollections( suite.ctx, string(suite.snapshotID), - []path.Path{itemPath}, + toRestorePaths(t, itemPath), &ic, fault.New(true)) assert.Error(t, err, "snapshot should be deleted", clues.ToCore(err)) diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 38a28ac86..fefbb5dde 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -248,10 +248,9 @@ func checkBackupIsInManifests( } } -//revive:disable:context-as-argument func checkMetadataFilesExist( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument backupID model.StableID, kw *kopia.Wrapper, ms *kopia.ModelStore, @@ -259,7 +258,6 @@ func checkMetadataFilesExist( service path.ServiceType, filesByCat map[path.CategoryType][]string, ) { - //revive:enable:context-as-argument for category, files := range filesByCat { t.Run(category.String(), func(t *testing.T) { bup := &backup.Backup{} @@ -269,7 +267,7 @@ func checkMetadataFilesExist( return } - paths := []path.Path{} + paths := []path.RestorePaths{} pathsByRef := map[string][]string{} for _, fName := range files { @@ -285,11 +283,18 @@ func checkMetadataFilesExist( continue } - paths = append(paths, p) + paths = append( + paths, + path.RestorePaths{StoragePath: p, RestorePath: dir}) pathsByRef[dir.ShortRef()] = append(pathsByRef[dir.ShortRef()], fName) } - cols, err := kw.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, nil, fault.New(true)) + cols, err := kw.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + nil, + fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) for _, col := range cols { diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index ea710fcf3..1928dfc66 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -46,16 +46,28 @@ type mockRestoreProducer struct { onRestore restoreFunc } -type restoreFunc func(id string, ps []path.Path) ([]data.RestoreCollection, error) +type restoreFunc func( + id string, + ps []path.RestorePaths, +) ([]data.RestoreCollection, error) func (mr *mockRestoreProducer) buildRestoreFunc( t *testing.T, oid string, ops []path.Path, ) { - mr.onRestore = func(id string, ps []path.Path) ([]data.RestoreCollection, error) { + mr.onRestore = func( + id string, + ps []path.RestorePaths, + ) ([]data.RestoreCollection, error) { + gotPaths := make([]path.Path, 0, len(ps)) + + for _, rp := range ps { + gotPaths = append(gotPaths, rp.StoragePath) + } + assert.Equal(t, oid, id, "manifest id") - checkPaths(t, ops, ps) + checkPaths(t, ops, gotPaths) return mr.colls, mr.err } @@ -64,11 +76,13 @@ func (mr *mockRestoreProducer) buildRestoreFunc( func (mr *mockRestoreProducer) ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bc kopia.ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) { - mr.gotPaths = append(mr.gotPaths, paths...) + for _, ps := range paths { + mr.gotPaths = append(mr.gotPaths, ps.StoragePath) + } if mr.onRestore != nil { return mr.onRestore(snapshotID, paths) diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 41f934692..55c472f7c 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -47,7 +47,7 @@ type ( ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bc kopia.ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index a402808f2..16e2029f9 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -308,7 +308,7 @@ func collectMetadata( tenantID string, errs *fault.Bus, ) ([]data.RestoreCollection, error) { - paths := []path.Path{} + paths := []path.RestorePaths{} for _, fn := range fileNames { for _, reason := range man.Reasons { @@ -326,7 +326,14 @@ func collectMetadata( With("metadata_file", fn, "category", reason.Category) } - paths = append(paths, p) + dir, err := p.Dir() + if err != nil { + return nil, clues. + Wrap(err, "building metadata collection path"). + With("metadata_file", fn, "category", reason.Category) + } + + paths = append(paths, path.RestorePaths{StoragePath: p, RestorePath: dir}) } } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 370869801..2dd5cd40c 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -349,7 +349,7 @@ func formatDetailsForRestoration( sel selectors.Selector, deets *details.Details, errs *fault.Bus, -) ([]path.Path, error) { +) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) if err != nil { return nil, err @@ -357,7 +357,7 @@ func formatDetailsForRestoration( var ( fdsPaths = fds.Paths() - paths = make([]path.Path, len(fdsPaths)) + paths = make([]path.RestorePaths, len(fdsPaths)) shortRefs = make([]string, len(fdsPaths)) el = errs.Local() ) @@ -377,7 +377,18 @@ func formatDetailsForRestoration( continue } - paths[i] = p + dir, err := p.Dir() + if err != nil { + el.AddRecoverable(clues. + Wrap(err, "getting restore directory after reduction"). + WithClues(ctx). + With("path", fdsPaths[i])) + + continue + } + + paths[i].StoragePath = p + paths[i].RestorePath = dir shortRefs[i] = p.ShortRef() } diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index bc86687ef..146f0d1c7 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -262,12 +262,22 @@ func read( return clues.Stack(err).WithClues(ctx) } + pd, err := p.Dir() + if err != nil { + return clues.Stack(err).WithClues(ctx) + } + ctx = clues.Add(ctx, "snapshot_id", snapshotID) cs, err := rer.ProduceRestoreCollections( ctx, snapshotID, - []path.Path{p}, + []path.RestorePaths{ + { + StoragePath: p, + RestorePath: pd, + }, + }, &stats.ByteCounter{}, errs) if err != nil { diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 52daa1e87..79a14ea95 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -130,6 +130,13 @@ var ( _ fmt.Stringer = &Builder{} ) +// RestorePaths denotes the location to find an item in kopia and the path of +// the collection to place the item in for restore. +type RestorePaths struct { + StoragePath Path + RestorePath Path +} + // Builder is a simple path representation that only tracks path elements. It // can join, escape, and unescape elements. Higher-level packages are expected // to wrap this struct to build resource-specific contexts (e.x. an From 211701f9b1557e164ddad175254473cd22cf1d23 Mon Sep 17 00:00:00 2001 From: Georgi Matev Date: Tue, 9 May 2023 19:09:34 -0700 Subject: [PATCH 10/33] Add option to manually trigger CI cleanup workflow (#3366) Can be useful for easy trigger of new changes or on off runs. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/ci_test_cleanup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 35191afdc..65e678e4b 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -1,5 +1,6 @@ name: CI Test Cleanup on: + workflow_dispatch: schedule: # every half hour - cron: "*/30 * * * *" From 6c2b78de07b6b7873aa69728a75753ad74aec352 Mon Sep 17 00:00:00 2001 From: neha_gupta Date: Wed, 10 May 2023 08:04:12 +0530 Subject: [PATCH 11/33] Item size- sum of attachment and email body (#3291) Size of emails will be - sum of - size of attachment and size of email body In case of contacts and events, since mostly everything is data we will check the size as - total serialised bytes #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Issue(s) * #3152 #### Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/connector/exchange/api/mail.go | 31 ++++++++++++++++--- .../connector/exchange/api/mail_test.go | 26 ++++++++++++++-- .../exchange/exchange_data_collection.go | 7 ++++- .../connector/exchange/service_restore.go | 3 +- 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index b011921b5..57a561b8a 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -134,6 +134,10 @@ func (c Mail) GetItem( immutableIDs bool, errs *fault.Bus, ) (serialization.Parsable, *details.ExchangeInfo, error) { + var ( + size int64 + mailBody models.ItemBodyable + ) // Will need adjusted if attachments start allowing paging. headers := buildPreferHeaders(false, immutableIDs) itemOpts := &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{ @@ -145,8 +149,16 @@ func (c Mail) GetItem( return nil, nil, graph.Stack(ctx, err) } - if !ptr.Val(mail.GetHasAttachments()) && !HasAttachments(mail.GetBody()) { - return mail, MailInfo(mail), nil + mailBody = mail.GetBody() + if mailBody != nil { + content := ptr.Val(mailBody.GetContent()) + if len(content) > 0 { + size = int64(len(content)) + } + } + + if !ptr.Val(mail.GetHasAttachments()) && !HasAttachments(mailBody) { + return mail, MailInfo(mail, size), nil } options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{ @@ -163,8 +175,14 @@ func (c Mail) GetItem( Attachments(). Get(ctx, options) if err == nil { + for _, a := range attached.GetValue() { + attachSize := ptr.Val(a.GetSize()) + size = +int64(attachSize) + } + mail.SetAttachments(attached.GetValue()) - return mail, MailInfo(mail), nil + + return mail, MailInfo(mail, size), nil } // A failure can be caused by having a lot of attachments as @@ -214,11 +232,13 @@ func (c Mail) GetItem( } atts = append(atts, att) + attachSize := ptr.Val(a.GetSize()) + size = +int64(attachSize) } mail.SetAttachments(atts) - return mail, MailInfo(mail), nil + return mail, MailInfo(mail, size), nil } // EnumerateContainers iterates through all of the users current @@ -419,7 +439,7 @@ func (c Mail) Serialize( // Helpers // --------------------------------------------------------------------------- -func MailInfo(msg models.Messageable) *details.ExchangeInfo { +func MailInfo(msg models.Messageable, size int64) *details.ExchangeInfo { var ( sender = UnwrapEmailAddress(msg.GetSender()) subject = ptr.Val(msg.GetSubject()) @@ -444,6 +464,7 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo { Recipient: recipients, Subject: subject, Received: received, + Size: size, Created: created, Modified: ptr.OrNow(msg.GetLastModifiedDateTime()), } diff --git a/src/internal/connector/exchange/api/mail_test.go b/src/internal/connector/exchange/api/mail_test.go index 2ce0cd537..f98093cf6 100644 --- a/src/internal/connector/exchange/api/mail_test.go +++ b/src/internal/connector/exchange/api/mail_test.go @@ -152,7 +152,7 @@ func (suite *MailAPIUnitSuite) TestMailInfo() { for _, tt := range tests { suite.Run(tt.name, func() { msg, expected := tt.msgAndRP() - assert.Equal(suite.T(), expected, api.MailInfo(msg)) + assert.Equal(suite.T(), expected, api.MailInfo(msg, 0)) }) } } @@ -213,6 +213,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { name string setupf func() attachmentCount int + size int64 expect assert.ErrorAssertionFunc }{ { @@ -242,6 +243,9 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { atts := models.NewAttachmentCollectionResponse() aitem := models.NewAttachment() + + asize := int32(50) + aitem.SetSize(&asize) atts.SetValue([]models.Attachmentable{aitem}) gock.New("https://graph.microsoft.com"). @@ -250,6 +254,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { JSON(getJSONObject(suite.T(), atts)) }, attachmentCount: 1, + size: 50, expect: assert.NoError, }, { @@ -289,6 +294,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { JSON(getJSONObject(suite.T(), aitem)) }, attachmentCount: 1, + size: 200, expect: assert.NoError, }, { @@ -330,6 +336,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { } }, attachmentCount: 5, + size: 200, expect: assert.NoError, }, } @@ -348,8 +355,23 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { it, ok := item.(models.Messageable) require.True(suite.T(), ok, "convert to messageable") + var size int64 + mailBody := it.GetBody() + if mailBody != nil { + content := ptr.Val(mailBody.GetContent()) + if len(content) > 0 { + size = int64(len(content)) + } + } + + attachments := it.GetAttachments() + for _, attachment := range attachments { + size = +int64(*attachment.GetSize()) + } + assert.Equal(suite.T(), *it.GetId(), mid) - assert.Equal(suite.T(), tt.attachmentCount, len(it.GetAttachments()), "attachment count") + assert.Equal(suite.T(), tt.attachmentCount, len(attachments), "attachment count") + assert.Equal(suite.T(), tt.size, size, "mail size") assert.True(suite.T(), gock.IsDone(), "made all requests") }) } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 4a2760be4..441056ed6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -260,7 +260,12 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { return } - info.Size = int64(len(data)) + // In case of mail the size of data is calc as- size of body content+size of attachment + // in all other case the size is - total item's serialized size + if info.Size <= 0 { + info.Size = int64(len(data)) + } + info.ParentPath = col.locationPath.String() col.data <- &Stream{ diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 4d49e3df9..f88a3f966 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -218,8 +218,7 @@ func RestoreMailMessage( return nil, err } - info := api.MailInfo(clone) - info.Size = int64(len(bits)) + info := api.MailInfo(clone, int64(len(bits))) return info, nil } From 49bef351d974f26302313a9a6e1359b0e317a297 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 02:54:51 +0000 Subject: [PATCH 12/33] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20golang.org/x/?= =?UTF-8?q?tools=20from=200.8.0=20to=200.9.1=20in=20/src=20(#3372)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.8.0 to 0.9.1.
Release notes

Sourced from golang.org/x/tools's releases.

gopls/v0.9.1

This release fixes a regression in the v0.9.0 release: a crash when running the go:generategolang/go#53781

Thank you to all those who filed crash reports, and apologies for the breakage!

gopls@v0.9.0

This release contains significant performance improvements (especially in incremental processing after source changes), bug fixes, and support for the LSP “inlay hints” feature, along with several other minor enhancements.

Performance improvements

Source edits cause gopls to invalidate and recompute information about the workspace, most of which has not changed. Previously, gopls would spend significant CPU copying data structures, sometimes more than 100ms per keystroke in a large workspace. This release includes many optimizations to avoid copying data needlessly, including a new map representation to achieve copying in constant time. Special thanks to @​euroelessar for the design and implementation of this data structure.

As a result of these improvements, gopls should be more responsive while typing in large codebases, though it will still use a lot of memory.

Time to process a change notification in the Kubernetes repo: image

New Features

Inlay hints

Added support for displaying inlay hints of composite literal field names and types, constant values, function parameter names, function type params, and short variable declarations. You can try these out in the vscode-go nightly by enabling inlay hints settings.

image3

Package References

Find references on package foo now lists locations where the given package is imported.

Quick-fix to add field names to struct literals

A new quick fix adds field names to struct literals with unkeyed fields.

image1

Bug fixes

This release includes the following notable bugfixes:

  • Fixes for goimports performance and correctness when using a go.work file (#52784)
  • Fix a crash during renaming in a package that uses generics (#52940)
  • Fix gopls getting confused when moving a file from the foo_test package to foo package (#45317)

A full list of all issues fixed can be found in the gopls/v0.9.0 milestone. To report a new problem, please file a new issue at https://go.dev/issues/new.

Thank you to our contributors!

Thank you for your contribution, @​alandonovan, @​euroelessar, @​findleyr, @​hyangah, @​jamalc, @​jba, @​marwan-at-work, @​suzmue, and @​dle8!

What’s Next?

... (truncated)

Commits
  • 4609d79 cmd/bisect: add -compile and -godebug shorthands
  • ddfa220 internal/fuzzy: improvements to the symbol scoring algorithm
  • 3449242 go/types/objectpath: don't panic when receiver is missing a method
  • 0809ec2 gopls/internal/lsp/source: document {All,Workspace}Metadata
  • 8f7fb01 go/analysis/unitchecker: add test of go vet on std
  • 23e52a3 bisect: diagnose bad targets better
  • d5af889 gopls: set GOWORK=off for loads from debug and safetoken tests
  • c93329a go/analysis/passes/printf: reshorten diagnostic about %s in Println call
  • 6219726 go.mod: update golang.org/x dependencies
  • f4d143e go/ssa: cleanup TestGenericBodies to pickup package name
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/tools&package-manager=go_modules&previous-version=0.8.0&new-version=0.9.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 8 ++++---- src/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/go.mod b/src/go.mod index 90d9b61f4..60e25c606 100644 --- a/src/go.mod +++ b/src/go.mod @@ -34,7 +34,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb golang.org/x/time v0.1.0 - golang.org/x/tools v0.8.0 + golang.org/x/tools v0.9.1 gopkg.in/resty.v1 v1.12.0 ) @@ -118,9 +118,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.8.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.54.0 // indirect diff --git a/src/go.sum b/src/go.sum index b86aff848..57680c065 100644 --- a/src/go.sum +++ b/src/go.sum @@ -530,8 +530,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -552,8 +552,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -604,8 +604,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -673,8 +673,8 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 79bb9321a4cd0592753079f200658aee7aa552fa Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 9 May 2023 22:14:07 -0600 Subject: [PATCH 13/33] fix the unbounded var sanity test site flags (#3376) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Test Plan - [x] :green_heart: E2E --- .github/workflows/sanity-test.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index e011069e0..c6772396d 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -36,6 +36,7 @@ jobs: CORSO_LOG_DIR: testlog CORSO_LOG_FILE: testlog/testlogging.log TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }} + TEST_SITE: ${{ secrets.CORSO_M365_TEST_SITE_URL }} SECONDARY_TEST_USER : ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} TEST_RESULT: test_results @@ -123,7 +124,7 @@ jobs: AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | go run . exchange emails \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --tenant ${{ env.AZURE_TENANT_ID }} \ --destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ --count 4 @@ -268,7 +269,7 @@ jobs: suffix=`date +"%Y-%m-%d_%H-%M"` go run . onedrive files \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ --destination Corso_Restore_st_$suffix \ @@ -364,7 +365,7 @@ jobs: AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | go run . onedrive files \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ --destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ @@ -432,7 +433,7 @@ jobs: ./corso backup create sharepoint \ --no-stats \ --hide-progress \ - --site "${CORSO_M365_TEST_SITE_URL}" \ + --site "${TEST_SITE}" \ --json \ 2>&1 | tee $TEST_RESULT/backup_sharepoint.txt @@ -518,7 +519,7 @@ jobs: ./corso backup create sharepoint \ --no-stats \ --hide-progress \ - --site "${CORSO_M365_TEST_SITE_URL}" \ + --site "${TEST_SITE}" \ --json \ 2>&1 | tee $TEST_RESULT/backup_sharepoint_incremental.txt From 7b378f601380a22cf49c88e8c9dee436e05954dd Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Wed, 10 May 2023 11:03:26 +0530 Subject: [PATCH 14/33] Fix incorrect jq parsing for sanity tests for SharePoint (#3377) --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/sanity-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index c6772396d..fd5333464 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -439,7 +439,7 @@ jobs: resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint.txt ) - if [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then + if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then echo "backup was not successful" exit 1 fi @@ -525,7 +525,7 @@ jobs: resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint_incremental.txt ) - if [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then + if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then echo "backup was not successful" exit 1 fi From 833216c8ae4eb6be81cad3379629cbf71adbad29 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 19:02:30 -0600 Subject: [PATCH 15/33] normalize folders restored in testing (#3363) Normalize all test folders to use a constant prefix: Corso_Test. This cleans up and centralizes all per- test variations on the restore destination. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .github/workflows/sanity-test.yaml | 22 +++++----- src/cli/backup/onedrive_e2e_test.go | 3 +- .../connector/data_collections_test.go | 12 +++--- .../exchange/container_resolver_test.go | 2 +- .../connector/exchange/restore_test.go | 41 +++++++++---------- .../graph_connector_disconnected_test.go | 13 +++--- .../connector/graph_connector_helper_test.go | 2 +- .../connector/graph_connector_test.go | 16 ++++---- src/internal/connector/onedrive/drive_test.go | 10 ++--- src/internal/connector/onedrive/item_test.go | 15 +++---- .../connector/sharepoint/api/pages_test.go | 6 +-- .../connector/sharepoint/collection_test.go | 3 +- src/internal/connector/sharepoint/restore.go | 2 +- .../operations/backup_integration_test.go | 13 +++--- src/internal/operations/restore_test.go | 8 ++-- src/internal/tester/restore_destination.go | 21 ++++++++-- .../loadtest/repository_load_test.go | 7 ++-- src/pkg/repository/repository_test.go | 2 +- src/pkg/selectors/testdata/onedrive.go | 9 ++++ 19 files changed, 112 insertions(+), 95 deletions(-) create mode 100644 src/pkg/selectors/testdata/onedrive.go diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index fd5333464..4f5020e47 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -126,7 +126,7 @@ jobs: go run . exchange emails \ --user ${TEST_USER} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --destination Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ --count 4 - name: Backup exchange test @@ -192,7 +192,7 @@ jobs: echo -e "\nBackup Exchange restore test\n" >> ${CORSO_LOG_FILE} ./corso restore exchange \ --no-stats \ - --email-folder Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --email-folder Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ --hide-progress \ --backup "${{ steps.exchange-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/exchange-restore-test.txt @@ -202,7 +202,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.exchange-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "exchange" - TEST_DATA: Corso_Restore_st_${{ steps.repo-init.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.repo-init.outputs.result }} run: | set -euo pipefail ./sanityTest @@ -239,7 +239,7 @@ jobs: --no-stats \ --hide-progress \ --backup "${{ steps.exchange-incremental-test.outputs.result }}" \ - --email-folder Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --email-folder Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ 2>&1 | tee $TEST_RESULT/exchange-incremantal-restore-test.txt echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/exchange-incremantal-restore-test.txt | sed "s/Restoring to folder//" ) >> $GITHUB_OUTPUT @@ -247,7 +247,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.exchange-incremantal-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "exchange" - TEST_DATA: Corso_Restore_st_${{ steps.repo-init.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.repo-init.outputs.result }} BASE_BACKUP: ${{ steps.exchange-restore-test.outputs.result }} run: | set -euo pipefail @@ -272,7 +272,7 @@ jobs: --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_$suffix \ + --destination Corso_Test_sanity$suffix \ --count 4 echo result="$suffix" >> $GITHUB_OUTPUT @@ -341,7 +341,7 @@ jobs: ./corso restore onedrive \ --no-stats \ --restore-permissions \ - --folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --folder Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --hide-progress \ --backup "${{ steps.onedrive-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt @@ -351,7 +351,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive" - TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} run: | set -euo pipefail ./sanityTest @@ -368,7 +368,7 @@ jobs: --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --destination Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --count 4 # incremental backup @@ -404,7 +404,7 @@ jobs: --no-stats \ --restore-permissions \ --hide-progress \ - --folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --folder Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --backup "${{ steps.onedrive-incremental-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT @@ -413,7 +413,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive" - TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} run: | set -euo pipefail ./sanityTest diff --git a/src/cli/backup/onedrive_e2e_test.go b/src/cli/backup/onedrive_e2e_test.go index 9e6c134bc..73cedd2ca 100644 --- a/src/cli/backup/onedrive_e2e_test.go +++ b/src/cli/backup/onedrive_e2e_test.go @@ -22,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/storage" ) @@ -172,7 +173,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) SetupSuite() { // some tests require an existing backup sel := selectors.NewOneDriveBackup(users) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) backupOp, err := suite.repo.NewBackupWithLookup(ctx, sel.Selector, ins) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index fedc85106..3025a385c 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -20,7 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/selectors/testdata" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------------------- @@ -160,7 +160,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "Invalid onedrive backup user", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup(owners) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, }, @@ -168,7 +168,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "Invalid sharepoint backup site", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup(owners) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) return sel.Selector }, }, @@ -185,7 +185,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "missing onedrive backup user", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup(owners) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "" return sel.Selector }, @@ -194,7 +194,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "missing sharepoint backup site", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup(owners) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.DiscreteOwner = "" return sel.Selector }, @@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { name: "Libraries", getSelector: func() selectors.Selector { sel := selectors.NewSharePointBackup(selSites) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) return sel.Selector }, }, diff --git a/src/internal/connector/exchange/container_resolver_test.go b/src/internal/connector/exchange/container_resolver_test.go index 572162263..de050d25a 100644 --- a/src/internal/connector/exchange/container_resolver_test.go +++ b/src/internal/connector/exchange/container_resolver_test.go @@ -549,7 +549,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { var ( user = tester.M365UserID(suite.T()) directoryCaches = make(map[path.CategoryType]graph.ContainerResolver) - folderName = tester.DefaultTestRestoreDestination().ContainerName + folderName = tester.DefaultTestRestoreDestination("").ContainerName tests = []struct { name string pathFunc1 func(t *testing.T) path.Path diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 1aa2beece..b6ec9168f 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -3,14 +3,12 @@ package exchange import ( "context" "testing" - "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" @@ -67,8 +65,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreContact() { var ( t = suite.T() userID = tester.M365UserID(t) - now = time.Now() - folderName = "TestRestoreContact: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName = tester.DefaultTestRestoreDestination("contact").ContainerName ) aFolder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName) @@ -102,7 +99,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreEvent() { var ( t = suite.T() userID = tester.M365UserID(t) - subject = "TestRestoreEvent: " + dttm.FormatNow(dttm.SafeForTesting) + subject = tester.DefaultTestRestoreDestination("event").ContainerName ) calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, subject) @@ -172,7 +169,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { } userID := tester.M365UserID(suite.T()) - now := time.Now() + tests := []struct { name string bytes []byte @@ -184,7 +181,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageBytes("Restore Exchange Object"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailObject: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailobj").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -196,7 +193,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithDirectAttachment("Restore 1 Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailwattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -208,7 +205,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithItemAttachmentEvent("Event Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreEventItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("eventwattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -220,7 +217,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithItemAttachmentMail("Mail Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailitemattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -235,7 +232,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailBasicItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailbasicattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -250,7 +247,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "ItemMailAttachmentwAttachment " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailnestattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -265,7 +262,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "ItemMailAttachment_Contact " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailcontactattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -277,7 +274,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithNestedItemAttachmentEvent("Nested Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreNestedEventItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("nestedattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -289,7 +286,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithLargeAttachment("Restore Large Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithLargeAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("maillargeattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -301,7 +298,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithTwoAttachments("Restore 2 Attachments"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithAttachments: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailtwoattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -313,7 +310,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithReferenceAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailrefattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -326,7 +323,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.ContactBytes("Test_Omega"), category: path.ContactsCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreContactObject: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("contact").ContainerName folder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -338,8 +335,8 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.EventBytes("Restored Event Object"), category: path.EventsCategory, destination: func(t *testing.T, ctx context.Context) string { - calendarName := "TestRestoreEventObject: " + dttm.FormatTo(now, dttm.SafeForTesting) - calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName) + folderName := tester.DefaultTestRestoreDestination("event").ContainerName + calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) return ptr.Val(calendar.GetId()) @@ -350,8 +347,8 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.EventWithAttachment("Restored Event Attachment"), category: path.EventsCategory, destination: func(t *testing.T, ctx context.Context) string { - calendarName := "TestRestoreEventObject_" + dttm.FormatTo(now, dttm.SafeForTesting) - calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName) + folderName := tester.DefaultTestRestoreDestination("eventobj").ContainerName + calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) return ptr.Val(calendar.GetId()) diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index b95f75335..23a6ab1dc 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------- @@ -82,19 +83,19 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices checkError: assert.NoError, excludes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Exclude(sel.Folders(selectors.Any())) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, filters: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Filter(sel.Folders(selectors.Any())) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, includes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, @@ -104,17 +105,17 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices checkError: assert.NoError, excludes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Exclude(sel.Folders(selectors.Any())) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, filters: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Filter(sel.Folders(selectors.Any())) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, includes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, }, diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 6934162ab..99043e5bc 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -1099,7 +1099,7 @@ func makeSharePointBackupSel( } // backupSelectorForExpected creates a selector that can be used to backup the -// given items in expected based on the item paths. Fails the test if items from +// given dests based on the item paths. Fails the test if items from // multiple services are in expected. func backupSelectorForExpected( t *testing.T, diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 00731b93e..37dc480f3 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -295,7 +295,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() { var ( t = suite.T() acct = tester.NewM365Account(t) - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") sel = selectors.Selector{ Service: selectors.ServiceUnknown, } @@ -323,7 +323,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() { } func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") table := []struct { name string col []data.RestoreCollection @@ -579,7 +579,7 @@ func runRestoreBackupTest( service: test.service, tenant: tenant, resourceOwners: resourceOwners, - dest: tester.DefaultTestRestoreDestination(), + dest: tester.DefaultTestRestoreDestination(""), } totalItems, totalKopiaItems, collections, expectedData := getCollectionsAndExpected( @@ -625,7 +625,7 @@ func runRestoreTestWithVerion( service: test.service, tenant: tenant, resourceOwners: resourceOwners, - dest: tester.DefaultTestRestoreDestination(), + dest: tester.DefaultTestRestoreDestination(""), } totalItems, _, collections, _ := getCollectionsAndExpected( @@ -664,7 +664,7 @@ func runRestoreBackupTestVersions( service: test.service, tenant: tenant, resourceOwners: resourceOwners, - dest: tester.DefaultTestRestoreDestination(), + dest: tester.DefaultTestRestoreDestination(""), } totalItems, _, collections, _ := getCollectionsAndExpected( @@ -1042,7 +1042,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames for i, collection := range test.collections { // Get a dest per collection so they're independent. - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") expectedDests = append(expectedDests, destAndCats{ resourceOwner: suite.user, dest: dest.ContainerName, @@ -1214,9 +1214,7 @@ func (suite *GraphConnectorIntegrationSuite) TestBackup_CreatesPrefixCollections resource: Users, selectorFunc: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{suite.user}) - sel.Include( - sel.Folders([]string{selectors.NoneTgt}), - ) + sel.Include(sel.Folders([]string{selectors.NoneTgt})) return sel.Selector }, diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index d2f1a68b6..2a5d4b5a8 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -279,24 +279,24 @@ func (suite *OneDriveUnitSuite) TestDrives() { // Integration tests -type OneDriveSuite struct { +type OneDriveIntgSuite struct { tester.Suite userID string } func TestOneDriveSuite(t *testing.T) { - suite.Run(t, &OneDriveSuite{ + suite.Run(t, &OneDriveIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tester.M365AcctCredEnvs}), }) } -func (suite *OneDriveSuite) SetupSuite() { +func (suite *OneDriveIntgSuite) SetupSuite() { suite.userID = tester.SecondaryM365UserID(suite.T()) } -func (suite *OneDriveSuite) TestCreateGetDeleteFolder() { +func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { ctx, flush := tester.NewContext() defer flush() @@ -401,7 +401,7 @@ func (fm testFolderMatcher) Matches(p string) bool { return fm.scope.Matches(selectors.OneDriveFolder, p) } -func (suite *OneDriveSuite) TestOneDriveNewCollections() { +func (suite *OneDriveIntgSuite) TestOneDriveNewCollections() { creds, err := tester.NewM365Account(suite.T()).M365Config() require.NoError(suite.T(), err, clues.ToCore(err)) diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 65b69ede7..47feea0ff 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -128,8 +128,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { } // TestItemWriter is an integration test for uploading data to OneDrive -// It creates a new `testfolder_ item and writes data to it +// It creates a new folder with a new item and writes data to it func (suite *ItemIntegrationSuite) TestItemWriter() { table := []struct { name string @@ -155,24 +154,20 @@ func (suite *ItemIntegrationSuite) TestItemWriter() { root, err := srv.Client().DrivesById(test.driveID).Root().Get(ctx, nil) require.NoError(t, err, clues.ToCore(err)) - // Test Requirement 2: "Test Folder" should exist - folder, err := api.GetFolderByName(ctx, srv, test.driveID, ptr.Val(root.GetId()), "Test Folder") - require.NoError(t, err, clues.ToCore(err)) - - newFolderName := "testfolder_" + dttm.FormatNow(dttm.SafeForTesting) - t.Logf("Test will create folder %s", newFolderName) + newFolderName := tester.DefaultTestRestoreDestination("folder").ContainerName + t.Logf("creating folder %s", newFolderName) newFolder, err := CreateItem( ctx, srv, test.driveID, - ptr.Val(folder.GetId()), + ptr.Val(root.GetId()), newItem(newFolderName, true)) require.NoError(t, err, clues.ToCore(err)) require.NotNil(t, newFolder.GetId()) newItemName := "testItem_" + dttm.FormatNow(dttm.SafeForTesting) - t.Logf("Test will create item %s", newItemName) + t.Logf("creating item %s", newItemName) newItem, err := CreateItem( ctx, diff --git a/src/internal/connector/sharepoint/api/pages_test.go b/src/internal/connector/sharepoint/api/pages_test.go index 32d0aa07c..c56c3bc86 100644 --- a/src/internal/connector/sharepoint/api/pages_test.go +++ b/src/internal/connector/sharepoint/api/pages_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock" @@ -81,7 +80,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { t := suite.T() - destName := "Corso_Restore_" + dttm.FormatNow(dttm.SafeForTesting) + destName := tester.DefaultTestRestoreDestination("").ContainerName testName := "MockPage" // Create Test Page @@ -98,8 +97,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { suite.service, pageData, suite.siteID, - destName, - ) + destName) require.NoError(t, err, clues.ToCore(err)) require.NotNil(t, info) diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index 6beb811f3..596b1bb34 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock" @@ -193,7 +192,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { info: sharePointListInfo(listing, int64(len(byteArray))), } - destName := "Corso_Restore_" + dttm.FormatNow(dttm.SafeForTesting) + destName := tester.DefaultTestRestoreDestination("").ContainerName deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName) assert.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 1c06e8ae3..013f2ef79 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -125,7 +125,7 @@ func RestoreCollections( } // restoreListItem utility function restores a List to the siteID. -// The name is changed to to Corso_Restore_{timeStame}_name +// The name is changed to to {DestName}_{name} // API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http // Restored List can be verified within the Site contents. func restoreListItem( diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index fefbb5dde..bc69c1d90 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -46,10 +46,13 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/selectors/testdata" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/store" ) +// Does not use the tester.DefaultTestRestoreDestination syntax as some of these +// items are created directly, not as a result of restoration, and we want to ensure +// they get clearly selected without accidental overlap. const incrementalsDestContainerPrefix = "incrementals_ci_" // --------------------------------------------------------------------------- @@ -1136,7 +1139,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() { osel = selectors.NewOneDriveBackup([]string{m365UserID}) ) - osel.Include(osel.AllData()) + osel.Include(selTD.OneDriveBackupFolderScope(osel)) bo, _, _, _, _, _, closer := prepNewTestBackupOp(t, ctx, mb, osel.Selector, control.Toggles{}, version.Backup) defer closer() @@ -1694,7 +1697,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { uname := ptr.Val(userable.GetUserPrincipalName()) oldsel := selectors.NewOneDriveBackup([]string{uname}) - oldsel.Include(oldsel.Folders([]string{"test"}, selectors.ExactMatch())) + oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel)) bo, _, kw, ms, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, ffs, 0) defer closer() @@ -1716,7 +1719,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { runAndCheckBackup(t, ctx, &bo, mb, false) newsel := selectors.NewOneDriveBackup([]string{uid}) - newsel.Include(newsel.Folders([]string{"test"}, selectors.ExactMatch())) + newsel.Include(selTD.OneDriveBackupFolderScope(newsel)) sel = newsel.SetDiscreteOwnerIDName(uid, uname) var ( @@ -1795,7 +1798,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { sel = selectors.NewSharePointBackup([]string{suite.site}) ) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) bo, _, kw, _, _, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, control.Toggles{}, version.Backup) defer closer() diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 320f2933d..ea42a5c4f 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -55,7 +55,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { gc = &mock.GraphConnector{} acct = account.Account{} now = time.Now() - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") ) table := []struct { @@ -220,7 +220,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { sw = &store.Wrapper{} gc = &mock.GraphConnector{} acct = tester.NewM365Account(suite.T()) - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") opts = control.Defaults() ) @@ -392,7 +392,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { { name: "Exchange_Restore", owner: tester.M365UserID(suite.T()), - dest: tester.DefaultTestRestoreDestination(), + dest: tester.DefaultTestRestoreDestination(""), getSelector: func(t *testing.T, owners []string) selectors.Selector { rsel := selectors.NewExchangeRestore(owners) rsel.Include(rsel.AllData()) @@ -464,7 +464,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoResults() { var ( t = suite.T() - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") mb = evmock.NewBus() ) diff --git a/src/internal/tester/restore_destination.go b/src/internal/tester/restore_destination.go index b22e8593b..af247258d 100644 --- a/src/internal/tester/restore_destination.go +++ b/src/internal/tester/restore_destination.go @@ -1,11 +1,26 @@ package tester import ( + "strings" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/control" ) -func DefaultTestRestoreDestination() control.RestoreDestination { - // Use microsecond granularity to help reduce collisions. - return control.DefaultRestoreDestination(dttm.SafeForTesting) +const RestoreFolderPrefix = "Corso_Test" + +func DefaultTestRestoreDestination(namespace string) control.RestoreDestination { + var ( + dest = control.DefaultRestoreDestination(dttm.SafeForTesting) + sft = dttm.FormatNow(dttm.SafeForTesting) + ) + + parts := []string{RestoreFolderPrefix, namespace, sft} + if len(namespace) == 0 { + parts = []string{RestoreFolderPrefix, sft} + } + + dest.ContainerName = strings.Join(parts, "_") + + return dest } diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index 4d9b718c1..7ef56fdb0 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -24,6 +24,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/storage" ) @@ -150,7 +151,7 @@ func runRestoreLoadTest( t.Skip("restore load test is toggled off") } - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") rst, err := r.NewRestore(ctx, backupID, restSel, dest) require.NoError(t, err, clues.ToCore(err)) @@ -541,7 +542,7 @@ func (suite *LoadOneDriveSuite) TestOneDrive() { defer flush() bsel := selectors.NewOneDriveBackup(suite.usersUnderTest) - bsel.Include(bsel.AllData()) + bsel.Include(selTD.OneDriveBackupFolderScope(bsel)) sel := bsel.Selector runLoadTest( @@ -588,7 +589,7 @@ func (suite *IndividualLoadOneDriveSuite) TestOneDrive() { defer flush() bsel := selectors.NewOneDriveBackup(suite.usersUnderTest) - bsel.Include(bsel.AllData()) + bsel.Include(selTD.OneDriveBackupFolderScope(bsel)) sel := bsel.Selector runLoadTest( diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 1a80d6793..8efe44f31 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -242,7 +242,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { t := suite.T() acct := tester.NewM365Account(t) - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") // need to initialize the repository before we can test connecting to it. st := tester.NewPrefixedS3Storage(t) diff --git a/src/pkg/selectors/testdata/onedrive.go b/src/pkg/selectors/testdata/onedrive.go new file mode 100644 index 000000000..8592d3d80 --- /dev/null +++ b/src/pkg/selectors/testdata/onedrive.go @@ -0,0 +1,9 @@ +package testdata + +import "github.com/alcionai/corso/src/pkg/selectors" + +// OneDriveBackupFolderScope is the standard folder scope that should be used +// in integration backups with onedrive. +func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope { + return sel.Folders([]string{"test"}, selectors.PrefixMatch()) +} From 522d6d2206f14deb1260c348b128a29ddd253ba4 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 19:22:05 -0600 Subject: [PATCH 16/33] fix up per-service token constraints (#3378) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../connector/graph/http_wrapper_test.go | 2 + src/internal/connector/graph/middleware.go | 50 +++++++++++++------ src/internal/events/events.go | 16 ++++-- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/internal/connector/graph/http_wrapper_test.go b/src/internal/connector/graph/http_wrapper_test.go index d5edaf27d..40abea977 100644 --- a/src/internal/connector/graph/http_wrapper_test.go +++ b/src/internal/connector/graph/http_wrapper_test.go @@ -88,10 +88,12 @@ func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() { // and thus skip all the middleware hdr := http.Header{} hdr.Set("Location", "localhost:99999999/smarfs") + toResp := &http.Response{ StatusCode: 302, Header: hdr, } + mwResp := mwForceResp{ resp: toResp, alternate: func(req *http.Request) (bool, *http.Response, error) { diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index 004798cad..988644db7 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -360,24 +360,31 @@ func (mw RetryMiddleware) getRetryDelay( return exponentialBackoff.NextBackOff() } -// We're trying to keep calls below the 10k-per-10-minute threshold. -// 15 tokens every second nets 900 per minute. That's 9000 every 10 minutes, -// which is a bit below the mark. -// But suppose we have a minute-long dry spell followed by a 10 minute tsunami. -// We'll have built up 900 tokens in reserve, so the first 900 calls go through -// immediately. Over the next 10 minutes, we'll partition out the other calls -// at a rate of 900-per-minute, ending at a total of 9900. Theoretically, if -// the volume keeps up after that, we'll always stay between 9000 and 9900 out -// of 10k. const ( - defaultPerSecond = 15 - defaultMaxCap = 900 - drivePerSecond = 15 - driveMaxCap = 1100 + // Default goal is to keep calls below the 10k-per-10-minute threshold. + // 14 tokens every second nets 840 per minute. That's 8400 every 10 minutes, + // which is a bit below the mark. + // But suppose we have a minute-long dry spell followed by a 10 minute tsunami. + // We'll have built up 750 tokens in reserve, so the first 750 calls go through + // immediately. Over the next 10 minutes, we'll partition out the other calls + // at a rate of 840-per-minute, ending at a total of 9150. Theoretically, if + // the volume keeps up after that, we'll always stay between 8400 and 9150 out + // of 10k. Worst case scenario, we have an extra minute of padding to allow + // up to 9990. + defaultPerSecond = 14 // 14 * 60 = 840 + defaultMaxCap = 750 // real cap is 10k-per-10-minutes + // since drive runs on a per-minute, rather than per-10-minute bucket, we have + // to keep the max cap equal to the per-second cap. A large maxCap pool (say, + // 1200, similar to the per-minute cap) would allow us to make a flood of 2400 + // calls in the first minute, putting us over the per-minute limit. Keeping + // the cap at the per-second burst means we only dole out a max of 1240 in one + // minute (20 cap + 1200 per minute + one burst of padding). + drivePerSecond = 20 // 20 * 60 = 1200 + driveMaxCap = 20 // real cap is 1250-per-minute ) var ( - driveLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) + driveLimiter = rate.NewLimiter(drivePerSecond, driveMaxCap) // also used as the exchange service limiter defaultLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) ) @@ -454,6 +461,8 @@ func (mw *ThrottleControlMiddleware) Intercept( // MetricsMiddleware aggregates per-request metrics on the events bus type MetricsMiddleware struct{} +const xmruHeader = "x-ms-resource-unit" + func (mw *MetricsMiddleware) Intercept( pipeline khttp.Pipeline, middlewareIndex int, @@ -474,5 +483,18 @@ func (mw *MetricsMiddleware) Intercept( events.Since(start, events.APICall) events.Since(start, events.APICall, status) + // track the graph "resource cost" for each call (if not provided, assume 1) + + // from msoft throttling documentation: + // x-ms-resource-unit - Indicates the resource unit used for this request. Values are positive integer + xmru := resp.Header.Get(xmruHeader) + xmrui, e := strconv.Atoi(xmru) + + if len(xmru) == 0 || e != nil { + xmrui = 1 + } + + events.IncN(xmrui, events.APICall, xmruHeader) + return resp, err } diff --git a/src/internal/events/events.go b/src/internal/events/events.go index 47a15f5e9..f900c50c4 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -188,10 +188,12 @@ func tenantHash(tenID string) string { // metrics aggregation // --------------------------------------------------------------------------- -type m string +type metricsCategory string // metrics collection bucket -const APICall m = "api_call" +const ( + APICall metricsCategory = "api_call" +) // configurations const ( @@ -256,13 +258,19 @@ func dumpMetrics(ctx context.Context, stop <-chan struct{}, sig *metrics.InmemSi } // Inc increments the given category by 1. -func Inc(cat m, keys ...string) { +func Inc(cat metricsCategory, keys ...string) { cats := append([]string{string(cat)}, keys...) metrics.IncrCounter(cats, 1) } +// IncN increments the given category by N. +func IncN(n int, cat metricsCategory, keys ...string) { + cats := append([]string{string(cat)}, keys...) + metrics.IncrCounter(cats, float32(n)) +} + // Since records the duration between the provided time and now, in millis. -func Since(start time.Time, cat m, keys ...string) { +func Since(start time.Time, cat metricsCategory, keys ...string) { cats := append([]string{string(cat)}, keys...) metrics.MeasureSince(cats, start) } From c0725b9cf9cb12717e3146ee272bbdbd5a7beb2d Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 20:03:31 -0600 Subject: [PATCH 17/33] some quick logging and error naming updates (#3348) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #3344 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cli/backup/backup.go | 7 +- src/internal/connector/graph/middleware.go | 77 ++++++++++++++-------- src/internal/connector/graph/service.go | 5 +- src/internal/operations/backup.go | 15 +++-- 4 files changed, 68 insertions(+), 36 deletions(-) diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index fe2a07a75..0a8b45dc3 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -207,7 +207,7 @@ func runBackups( var ( owner = discSel.DiscreteOwner - ictx = clues.Add(ctx, "resource_owner", owner) + ictx = clues.Add(ctx, "resource_owner_selected", owner) ) bo, err := r.NewBackupWithLookup(ictx, discSel, ins) @@ -218,6 +218,11 @@ func runBackups( continue } + ictx = clues.Add( + ctx, + "resource_owner_id", bo.ResourceOwner.ID(), + "resource_owner_name", bo.ResourceOwner.Name()) + err = bo.Run(ictx) if err != nil { errs = append(errs, clues.Wrap(err, owner).WithClues(ictx)) diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index 988644db7..a20b12ade 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -2,7 +2,6 @@ package graph import ( "context" - "fmt" "io" "net/http" "net/http/httputil" @@ -101,6 +100,9 @@ func LoggableURL(url string) pii.SafeURL { } } +// 1 MB +const logMBLimit = 1 * 1048576 + func (mw *LoggingMiddleware) Intercept( pipeline khttp.Pipeline, middlewareIndex int, @@ -123,42 +125,61 @@ func (mw *LoggingMiddleware) Intercept( return resp, err } - ctx = clues.Add(ctx, "status", resp.Status, "statusCode", resp.StatusCode) - log := logger.Ctx(ctx) + ctx = clues.Add( + ctx, + "status", resp.Status, + "statusCode", resp.StatusCode, + "content_len", resp.ContentLength) - // Return immediately if the response is good (2xx). - // If api logging is toggled, log a body-less dump of the request/resp. - if (resp.StatusCode / 100) == 2 { - if logger.DebugAPIFV || os.Getenv(log2xxGraphRequestsEnvKey) != "" { - log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "")) - } + var ( + log = logger.Ctx(ctx) + respClass = resp.StatusCode / 100 + logExtra = logger.DebugAPIFV || os.Getenv(logGraphRequestsEnvKey) != "" + ) - return resp, err - } - - // Log errors according to api debugging configurations. - // When debugging is toggled, every non-2xx is recorded with a response dump. - // Otherwise, throttling cases and other non-2xx responses are logged - // with a slimmer reference for telemetry/supportability purposes. - if logger.DebugAPIFV || os.Getenv(logGraphRequestsEnvKey) != "" { - log.Errorw("non-2xx graph api response", "response", getRespDump(ctx, resp, true)) - return resp, err - } - - msg := fmt.Sprintf("graph api error: %s", resp.Status) - - // special case for supportability: log all throttling cases. + // special case: always info log 429 responses if resp.StatusCode == http.StatusTooManyRequests { - log = log.With( + log.Infow( + "graph api throttling", "limit", resp.Header.Get(rateLimitHeader), "remaining", resp.Header.Get(rateRemainingHeader), "reset", resp.Header.Get(rateResetHeader), "retry-after", resp.Header.Get(retryAfterHeader)) - } else if resp.StatusCode/100 == 4 || resp.StatusCode == http.StatusServiceUnavailable { - log = log.With("response", getRespDump(ctx, resp, true)) + + return resp, err } - log.Info(msg) + // special case: always dump status-400-bad-request + if resp.StatusCode == http.StatusBadRequest { + log.With("response", getRespDump(ctx, resp, true)). + Error("graph api error: " + resp.Status) + + return resp, err + } + + // Log api calls according to api debugging configurations. + switch respClass { + case 2: + if logExtra { + // only dump the body if it's under a size limit. We don't want to copy gigs into memory for a log. + dump := getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "" && resp.ContentLength < logMBLimit) + log.Infow("2xx graph api resp", "response", dump) + } + case 3: + log.With("redirect_location", LoggableURL(resp.Header.Get(locationHeader))) + + if logExtra { + log.With("response", getRespDump(ctx, resp, false)) + } + + log.Info("graph api redirect: " + resp.Status) + default: + if logExtra { + log.With("response", getRespDump(ctx, resp, true)) + } + + log.Error("graph api error: " + resp.Status) + } return resp, err } diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 288725831..e05838793 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -21,13 +21,14 @@ const ( logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS" log2xxGraphRequestsEnvKey = "LOG_2XX_GRAPH_REQUESTS" log2xxGraphResponseEnvKey = "LOG_2XX_GRAPH_RESPONSES" - retryAttemptHeader = "Retry-Attempt" - retryAfterHeader = "Retry-After" defaultMaxRetries = 3 defaultDelay = 3 * time.Second + locationHeader = "Location" rateLimitHeader = "RateLimit-Limit" rateRemainingHeader = "RateLimit-Remaining" rateResetHeader = "RateLimit-Reset" + retryAfterHeader = "Retry-After" + retryAttemptHeader = "Retry-Attempt" defaultHTTPClientTimeout = 1 * time.Hour ) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 2d926b692..6c6049156 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -512,11 +512,16 @@ func consumeBackupCollections( "kopia_ignored_errors", kopiaStats.IgnoredErrorCount) } - if kopiaStats.ErrorCount > 0 || - (kopiaStats.IgnoredErrorCount > kopiaStats.ExpectedIgnoredErrorCount) { - err = clues.New("building kopia snapshot").With( - "kopia_errors", kopiaStats.ErrorCount, - "kopia_ignored_errors", kopiaStats.IgnoredErrorCount) + ctx = clues.Add( + ctx, + "kopia_errors", kopiaStats.ErrorCount, + "kopia_ignored_errors", kopiaStats.IgnoredErrorCount, + "kopia_expected_ignored_errors", kopiaStats.ExpectedIgnoredErrorCount) + + if kopiaStats.ErrorCount > 0 { + err = clues.New("building kopia snapshot").WithClues(ctx) + } else if kopiaStats.IgnoredErrorCount > kopiaStats.ExpectedIgnoredErrorCount { + err = clues.New("downloading items for persistence").WithClues(ctx) } return kopiaStats, deets, itemsSourcedFromBase, err From c5b388a721b30ee644e43722a67527c793af1209 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 20:28:18 -0600 Subject: [PATCH 18/33] add indeets test helper, implement in exchange op (#3295) Adds a helper for building expected details entries and checking them after a backup. Implements the helper in the exchange backup tests in operations/backup integration. Will follow with a onedrive implementation. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Issue(s) * #3240 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/getM365/onedrive/get_item.go | 2 +- .../connector/exchange/service_restore.go | 5 +- src/internal/connector/onedrive/api/drive.go | 25 +- .../operations/backup_integration_test.go | 472 +++++++++++++----- src/pkg/backup/details/testdata/in_deets.go | 368 ++++++++++++++ .../backup/details/testdata/in_deets_test.go | 445 +++++++++++++++++ src/pkg/path/path.go | 2 +- 7 files changed, 1170 insertions(+), 149 deletions(-) create mode 100644 src/pkg/backup/details/testdata/in_deets.go create mode 100644 src/pkg/backup/details/testdata/in_deets_test.go diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go index 4868ab343..3b338ca74 100644 --- a/src/cmd/getM365/onedrive/get_item.go +++ b/src/cmd/getM365/onedrive/get_item.go @@ -112,7 +112,7 @@ func runDisplayM365JSON( creds account.M365Config, user, itemID string, ) error { - drive, err := api.GetDriveByID(ctx, srv, user) + drive, err := api.GetUsersDrive(ctx, srv, user) if err != nil { return err } diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index f88a3f966..8ac120619 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -441,10 +441,7 @@ func restoreCollection( continue } - locationRef := &path.Builder{} - if category == path.ContactsCategory { - locationRef = locationRef.Append(itemPath.Folders()...) - } + locationRef := path.Builder{}.Append(itemPath.Folders()...) err = deets.Add( itemPath, diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 3b2674553..8d0b1571f 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -336,18 +336,33 @@ func GetItemPermission( return perm, nil } -func GetDriveByID( +func GetUsersDrive( ctx context.Context, srv graph.Servicer, - userID string, + user string, ) (models.Driveable, error) { - //revive:enable:context-as-argument d, err := srv.Client(). - UsersById(userID). + UsersById(user). Drive(). Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting drive") + return nil, graph.Wrap(ctx, err, "getting user's drive") + } + + return d, nil +} + +func GetSitesDefaultDrive( + ctx context.Context, + srv graph.Servicer, + site string, +) (models.Driveable, error) { + d, err := srv.Client(). + SitesById(site). + Drive(). + Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting site's drive") } return d, nil diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index bc69c1d90..0b6283078 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -3,6 +3,7 @@ package operations import ( "context" "fmt" + "strings" "testing" "time" @@ -22,11 +23,12 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/exchange" - "github.com/alcionai/corso/src/internal/connector/exchange/api" + exapi "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" + odapi "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -41,6 +43,7 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" + deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/fault" @@ -62,11 +65,9 @@ const incrementalsDestContainerPrefix = "incrementals_ci_" // prepNewTestBackupOp generates all clients required to run a backup operation, // returning both a backup operation created with those clients, as well as // the clients themselves. -// -//revive:disable:context-as-argument func prepNewTestBackupOp( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument bus events.Eventer, sel selectors.Selector, featureToggles control.Toggles, @@ -76,11 +77,11 @@ func prepNewTestBackupOp( account.Account, *kopia.Wrapper, *kopia.ModelStore, + streamstore.Streamer, *connector.GraphConnector, selectors.Selector, func(), ) { - //revive:enable:context-as-argument var ( acct = tester.NewM365Account(t) // need to initialize the repository before we can test connecting to it. @@ -126,18 +127,18 @@ func prepNewTestBackupOp( gc, sel := GCWithSelector(t, ctx, acct, connectorResource, sel, nil, closer) bo := newTestBackupOp(t, ctx, kw, ms, gc, acct, sel, bus, featureToggles, closer) - return bo, acct, kw, ms, gc, sel, closer + ss := streamstore.NewStreamer(kw, acct.ID(), sel.PathService()) + + return bo, acct, kw, ms, ss, gc, sel, closer } // newTestBackupOp accepts the clients required to compose a backup operation, plus // any other metadata, and uses them to generate a new backup operation. This // allows backup chains to utilize the same temp directory and configuration // details. -// -//revive:disable:context-as-argument func newTestBackupOp( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument kw *kopia.Wrapper, ms *kopia.ModelStore, gc *connector.GraphConnector, @@ -147,7 +148,6 @@ func newTestBackupOp( featureToggles control.Toggles, closer func(), ) BackupOperation { - //revive:enable:context-as-argument var ( sw = store.NewKopiaStore(ms) opts = control.Defaults() @@ -165,15 +165,13 @@ func newTestBackupOp( return bo } -//revive:disable:context-as-argument func runAndCheckBackup( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument bo *BackupOperation, mb *evmock.Bus, acceptNoData bool, ) { - //revive:enable:context-as-argument err := bo.Run(ctx) require.NoError(t, err, clues.ToCore(err)) require.NotEmpty(t, bo.Results, "the backup had non-zero results") @@ -206,17 +204,15 @@ func runAndCheckBackup( bo.Results.BackupID, "backupID pre-declaration") } -//revive:disable:context-as-argument func checkBackupIsInManifests( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument kw *kopia.Wrapper, bo *BackupOperation, sel selectors.Selector, resourceOwner string, categories ...path.CategoryType, ) { - //revive:enable:context-as-argument for _, category := range categories { t.Run(category.String(), func(t *testing.T) { var ( @@ -343,10 +339,9 @@ func checkMetadataFilesExist( // the callback provider can use them, or not, as wanted. type dataBuilderFunc func(id, timeStamp, subject, body string) []byte -//revive:disable:context-as-argument func generateContainerOfItems( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument gc *connector.GraphConnector, service path.ServiceType, acct account.Account, @@ -357,7 +352,6 @@ func generateContainerOfItems( backupVersion int, dbf dataBuilderFunc, ) *details.Details { - //revive:enable:context-as-argument t.Helper() items := make([]incrementalItem, 0, howManyItems) @@ -584,11 +578,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { defer flush() tests := []struct { - name string - selector func() *selectors.ExchangeBackup - category path.CategoryType - metadataFiles []string - runIncremental bool + name string + selector func() *selectors.ExchangeBackup + category path.CategoryType + metadataFiles []string }{ { name: "Mail", @@ -599,9 +592,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { return sel }, - category: path.EmailCategory, - metadataFiles: exchange.MetadataFileNames(path.EmailCategory), - runIncremental: true, + category: path.EmailCategory, + metadataFiles: exchange.MetadataFileNames(path.EmailCategory), }, { name: "Contacts", @@ -610,9 +602,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { sel.Include(sel.ContactFolders([]string{exchange.DefaultContactFolder}, selectors.PrefixMatch())) return sel }, - category: path.ContactsCategory, - metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), - runIncremental: true, + category: path.ContactsCategory, + metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), }, { name: "Calendar Events", @@ -628,13 +619,14 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { for _, test := range tests { suite.Run(test.name, func() { var ( - t = suite.T() - mb = evmock.NewBus() - sel = test.selector().Selector - ffs = control.Toggles{} + t = suite.T() + mb = evmock.NewBus() + sel = test.selector().Selector + ffs = control.Toggles{} + whatSet = deeTD.CategoryFromRepoRef ) - bo, acct, kw, ms, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) + bo, acct, kw, ms, ss, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) defer closer() userID := sel.ID() @@ -656,9 +648,17 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { path.ExchangeService, map[path.CategoryType][]string{test.category: test.metadataFiles}) - if !test.runIncremental { - return - } + _, expectDeets := deeTD.GetDeetsInBackup( + t, + ctx, + bo.Results.BackupID, + acct.ID(), + userID, + path.ExchangeService, + whatSet, + ms, + ss) + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, whatSet, ms, ss, expectDeets, false) // Basic, happy path incremental test. No changes are dictated or expected. // This only tests that an incremental backup is runnable at all, and that it @@ -680,6 +680,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { userID, path.ExchangeService, map[path.CategoryType][]string{test.category: test.metadataFiles}) + deeTD.CheckBackupDetails( + t, + ctx, + incBO.Results.BackupID, + whatSet, + ms, + ss, + expectDeets, + false) // do some additional checks to ensure the incremental dealt with fewer items. assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written") @@ -700,7 +709,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { // TestBackup_Run ensures that Integration Testing works // for the following scopes: Contacts, Events, and Mail -func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalExchange() { ctx, flush := tester.NewContext() defer flush() @@ -712,6 +721,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ffs = control.Toggles{} mb = evmock.NewBus() now = dttm.Now() + service = path.ExchangeService categories = map[path.CategoryType][]string{ path.EmailCategory: exchange.MetadataFileNames(path.EmailCategory), path.ContactsCategory: exchange.MetadataFileNames(path.ContactsCategory), @@ -728,11 +738,12 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // at this point is harmless. containers = []string{container1, container2, container3, containerRename} sel = selectors.NewExchangeBackup([]string{suite.user}) + whatSet = deeTD.CategoryFromRepoRef ) gc, sels := GCWithSelector(t, ctx, acct, connector.Users, sel.Selector, nil, nil) - sel, err := sels.ToExchangeBackup() - require.NoError(t, err, clues.ToCore(err)) + sel.DiscreteOwner = sels.ID() + sel.DiscreteOwnerName = sels.Name() uidn := inMock.NewProvider(sels.ID(), sels.Name()) @@ -743,7 +754,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { m365, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(m365) + ac, err := exapi.NewClient(m365) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -754,7 +765,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // container into another generates a delta for both addition and deletion. type contDeets struct { containerID string - deets *details.Details + locRef string + itemRefs []string // cached for populating expected deets, otherwise not used } mailDBF := func(id, timeStamp, subject, body string) []byte { @@ -812,11 +824,14 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // populate initial test data for category, gen := range dataset { for destName := range gen.dests { + // TODO: the details.Builder returned by restore can contain entries with + // incorrect information. non-representative repo-refs and the like. Until + // that gets fixed, we can't consume that info for testing. deets := generateContainerOfItems( t, ctx, gc, - path.ExchangeService, + service, acct, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, @@ -825,41 +840,103 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { version.Backup, gen.dbf) - dataset[category].dests[destName] = contDeets{"", deets} + itemRefs := []string{} + + for _, ent := range deets.Entries { + if ent.Exchange == nil || ent.Folder != nil { + continue + } + + if len(ent.ItemRef) > 0 { + itemRefs = append(itemRefs, ent.ItemRef) + } + } + + // save the item ids for building expectedDeets later on + cd := dataset[category].dests[destName] + cd.itemRefs = itemRefs + dataset[category].dests[destName] = cd + } + } + + bo, acct, kw, ms, ss, gc, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs, version.Backup) + defer closer() + + // run the initial backup + runAndCheckBackup(t, ctx, &bo, mb, false) + + rrPfx, err := path.ServicePrefix(acct.ID(), uidn.ID(), service, path.EmailCategory) + require.NoError(t, err, clues.ToCore(err)) + + // strip the category from the prefix; we primarily want the tenant and resource owner. + expectDeets := deeTD.NewInDeets(rrPfx.ToBuilder().Dir().String()) + bupDeets, _ := deeTD.GetDeetsInBackup(t, ctx, bo.Results.BackupID, acct.ID(), uidn.ID(), service, whatSet, ms, ss) + + // update the datasets with their location refs + for category, gen := range dataset { + for destName, cd := range gen.dests { + var longestLR string + + for _, ent := range bupDeets.Entries { + // generated destinations should always contain items + if ent.Folder != nil { + continue + } + + p, err := path.FromDataLayerPath(ent.RepoRef, false) + require.NoError(t, err, clues.ToCore(err)) + + // category must match, and the owning folder must be this destination + if p.Category() != category || strings.HasSuffix(ent.LocationRef, destName) { + continue + } + + // emails, due to folder nesting and our design for populating data via restore, + // will duplicate the dest folder as both the restore destination, and the "old parent + // folder". we'll get both a prefix/destName and a prefix/destName/destName folder. + // since we want future comparison to only use the leaf dir, we select for the longest match. + if len(ent.LocationRef) > len(longestLR) { + longestLR = ent.LocationRef + } + } + + require.NotEmptyf(t, longestLR, "must find an expected details entry matching the generated folder: %s", destName) + + cd.locRef = longestLR + + dataset[category].dests[destName] = cd + expectDeets.AddLocation(category.String(), cd.locRef) + + for _, i := range dataset[category].dests[destName].itemRefs { + expectDeets.AddItem(category.String(), cd.locRef, i) + } } } // verify test data was populated, and track it for comparisons + // TODO: this can be swapped out for InDeets checks if we add itemRefs to folder ents. for category, gen := range dataset { qp := graph.QueryParams{ Category: category, ResourceOwner: uidn, Credentials: m365, } + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating container resolver", category, clues.ToCore(err)) for destName, dest := range gen.dests { - p, err := path.FromDataLayerPath(dest.deets.Entries[0].RepoRef, true) - require.NoError(t, err, clues.ToCore(err)) + id, ok := cr.LocationInCache(dest.locRef) + require.True(t, ok, "dir %s found in %s cache", dest.locRef, category) - id, ok := cr.LocationInCache(p.Folder(false)) - require.True(t, ok, "dir %s found in %s cache", p.Folder(false), category) - - d := dataset[category].dests[destName] - d.containerID = id - dataset[category].dests[destName] = d + dest.containerID = id + dataset[category].dests[destName] = dest } } - bo, _, kw, ms, gc, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs, version.Backup) - defer closer() - - sel, err = sels.ToExchangeBackup() - require.NoError(t, err, clues.ToCore(err)) - - // run the initial backup - runAndCheckBackup(t, ctx, &bo, mb, false) + // precheck to ensure the expectedDeets are correct. + // if we fail here, the expectedDeets were populated incorrectly. + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, whatSet, ms, ss, expectDeets, true) // Although established as a table, these tests are no isolated from each other. // Assume that every test's side effects cascade to all following test cases. @@ -881,20 +958,25 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { { name: "move an email folder to a subfolder", updateUserData: func(t *testing.T) { + cat := path.EmailCategory + // contacts and events cannot be sufoldered; this is an email-only change - toContainer := dataset[path.EmailCategory].dests[container1].containerID - fromContainer := dataset[path.EmailCategory].dests[container2].containerID + from := dataset[cat].dests[container2] + to := dataset[cat].dests[container1] body := users.NewItemMailFoldersItemMovePostRequestBody() - body.SetDestinationId(&toContainer) + body.SetDestinationId(ptr.To(to.containerID)) _, err := gc.Service. Client(). UsersById(uidn.ID()). - MailFoldersById(fromContainer). + MailFoldersById(from.containerID). Move(). Post(ctx, body, nil) require.NoError(t, err, clues.ToCore(err)) + + newLoc := expectDeets.MoveLocation(cat.String(), from.locRef, to.locRef) + from.locRef = newLoc }, itemsRead: 0, // zero because we don't count container reads itemsWritten: 2, @@ -916,6 +998,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err := ac.Events().DeleteContainer(ctx, uidn.ID(), containerID) require.NoError(t, err, "deleting a calendar", clues.ToCore(err)) } + + expectDeets.RemoveLocation(category.String(), d.dests[container2].locRef) } }, itemsRead: 0, @@ -929,7 +1013,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { t, ctx, gc, - path.ExchangeService, + service, acct, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, @@ -944,16 +1028,28 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { Credentials: m365, } + expectedLocRef := container3 + if category == path.EmailCategory { + expectedLocRef = path.Builder{}.Append(container3, container3).String() + } + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating container resolver", category, clues.ToCore(err)) - p, err := path.FromDataLayerPath(deets.Entries[0].RepoRef, true) - require.NoError(t, err, clues.ToCore(err)) + id, ok := cr.LocationInCache(expectedLocRef) + require.Truef(t, ok, "dir %s found in %s cache", expectedLocRef, category) - id, ok := cr.LocationInCache(p.Folder(false)) - require.Truef(t, ok, "dir %s found in %s cache", p.Folder(false), category) + dataset[category].dests[container3] = contDeets{ + containerID: id, + locRef: expectedLocRef, + itemRefs: nil, // not needed at this point + } - dataset[category].dests[container3] = contDeets{id, deets} + for _, ent := range deets.Entries { + if ent.Folder == nil { + expectDeets.AddItem(category.String(), expectedLocRef, ent.ItemRef) + } + } } }, itemsRead: 4, @@ -963,17 +1059,24 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { name: "rename a folder", updateUserData: func(t *testing.T) { for category, d := range dataset { - containerID := d.dests[container3].containerID cli := gc.Service.Client().UsersById(uidn.ID()) + containerID := d.dests[container3].containerID + newLoc := containerRename - // copy the container info, since both names should - // reference the same container by id. Though the - // details refs won't line up, so those get deleted. - d.dests[containerRename] = contDeets{ - containerID: d.dests[container3].containerID, - deets: nil, + if category == path.EmailCategory { + newLoc = path.Builder{}.Append(container3, containerRename).String() } + d.dests[containerRename] = contDeets{ + containerID: containerID, + locRef: newLoc, + } + + expectDeets.RenameLocation( + category.String(), + d.dests[container3].containerID, + newLoc) + switch category { case path.EmailCategory: cmf := cli.MailFoldersById(containerID) @@ -1023,24 +1126,39 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { body, err := support.CreateMessageFromBytes(itemData) require.NoError(t, err, "transforming mail bytes to messageable", clues.ToCore(err)) - _, err = cli.MailFoldersById(containerID).Messages().Post(ctx, body, nil) + itm, err := cli.MailFoldersById(containerID).Messages().Post(ctx, body, nil) require.NoError(t, err, "posting email item", clues.ToCore(err)) + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) + case path.ContactsCategory: _, itemData := generateItemData(t, category, uidn.ID(), contactDBF) body, err := support.CreateContactFromBytes(itemData) require.NoError(t, err, "transforming contact bytes to contactable", clues.ToCore(err)) - _, err = cli.ContactFoldersById(containerID).Contacts().Post(ctx, body, nil) + itm, err := cli.ContactFoldersById(containerID).Contacts().Post(ctx, body, nil) require.NoError(t, err, "posting contact item", clues.ToCore(err)) + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) + case path.EventsCategory: _, itemData := generateItemData(t, category, uidn.ID(), eventDBF) body, err := support.CreateEventFromBytes(itemData) require.NoError(t, err, "transforming event bytes to eventable", clues.ToCore(err)) - _, err = cli.CalendarsById(containerID).Events().Post(ctx, body, nil) + itm, err := cli.CalendarsById(containerID).Events().Post(ctx, body, nil) require.NoError(t, err, "posting events item", clues.ToCore(err)) + + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) } } }, @@ -1063,6 +1181,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.MessagesById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting email item", clues.ToCore(err)) + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) + case path.ContactsCategory: ids, _, _, err := ac.Contacts().GetAddedAndRemovedItemIDs(ctx, uidn.ID(), containerID, "", false) require.NoError(t, err, "getting contact ids", clues.ToCore(err)) @@ -1071,6 +1194,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.ContactsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting contact item", clues.ToCore(err)) + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) + case path.EventsCategory: ids, _, _, err := ac.Events().GetAddedAndRemovedItemIDs(ctx, uidn.ID(), containerID, "", false) require.NoError(t, err, "getting event ids", clues.ToCore(err)) @@ -1078,6 +1206,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.CalendarsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting calendar", clues.ToCore(err)) + + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) } } }, @@ -1090,24 +1223,20 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { var ( t = suite.T() incMB = evmock.NewBus() - incBO = newTestBackupOp(t, ctx, kw, ms, gc, acct, sel.Selector, incMB, ffs, closer) + incBO = newTestBackupOp(t, ctx, kw, ms, gc, acct, sels, incMB, ffs, closer) + atid = m365.AzureTenantID ) test.updateUserData(t) err := incBO.Run(ctx) require.NoError(t, err, clues.ToCore(err)) - checkBackupIsInManifests(t, ctx, kw, &incBO, sel.Selector, uidn.ID(), maps.Keys(categories)...) - checkMetadataFilesExist( - t, - ctx, - incBO.Results.BackupID, - kw, - ms, - m365.AzureTenantID, - uidn.ID(), - path.ExchangeService, - categories) + + bupID := incBO.Results.BackupID + + checkBackupIsInManifests(t, ctx, kw, &incBO, sels, uidn.ID(), maps.Keys(categories)...) + checkMetadataFilesExist(t, ctx, bupID, kw, ms, atid, uidn.ID(), service, categories) + deeTD.CheckBackupDetails(t, ctx, bupID, whatSet, ms, ss, expectDeets, true) // do some additional checks to ensure the incremental dealt with fewer items. // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. @@ -1119,7 +1248,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") assert.Equal(t, incMB.CalledWith[events.BackupStart][0][events.BackupID], - incBO.Results.BackupID, "incremental backupID pre-declaration") + bupID, "incremental backupID pre-declaration") }) } } @@ -1133,21 +1262,29 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() { defer flush() var ( - t = suite.T() - mb = evmock.NewBus() - m365UserID = tester.SecondaryM365UserID(t) - osel = selectors.NewOneDriveBackup([]string{m365UserID}) + t = suite.T() + tenID = tester.M365TenantID(t) + mb = evmock.NewBus() + userID = tester.SecondaryM365UserID(t) + osel = selectors.NewOneDriveBackup([]string{userID}) + ws = deeTD.DriveIDFromRepoRef + svc = path.OneDriveService ) osel.Include(selTD.OneDriveBackupFolderScope(osel)) - bo, _, _, _, _, _, closer := prepNewTestBackupOp(t, ctx, mb, osel.Selector, control.Toggles{}, version.Backup) + bo, _, _, ms, ss, _, sel, closer := prepNewTestBackupOp(t, ctx, mb, osel.Selector, control.Toggles{}, version.Backup) defer closer() runAndCheckBackup(t, ctx, &bo, mb, false) + + bID := bo.Results.BackupID + + _, expectDeets := deeTD.GetDeetsInBackup(t, ctx, bID, tenID, sel.ID(), svc, ws, ms, ss) + deeTD.CheckBackupDetails(t, ctx, bID, ws, ms, ss, expectDeets, false) } -func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalOneDrive() { sel := selectors.NewOneDriveRestore([]string{suite.user}) ic := func(cs []string) selectors.Selector { @@ -1158,9 +1295,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { gtdi := func( t *testing.T, ctx context.Context, - svc graph.Servicer, + gs graph.Servicer, ) string { - d, err := svc.Client().UsersById(suite.user).Drive().Get(ctx, nil) + d, err := odapi.GetUsersDrive(ctx, gs, suite.user) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). With("user", suite.user) @@ -1186,7 +1323,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { false) } -func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePointIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalSharePoint() { sel := selectors.NewSharePointRestore([]string{suite.site}) ic := func(cs []string) selectors.Selector { @@ -1197,9 +1334,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePointIncrementals() { gtdi := func( t *testing.T, ctx context.Context, - svc graph.Servicer, + gs graph.Servicer, ) string { - d, err := svc.Client().SitesById(suite.site).Drive().Get(ctx, nil) + d, err := odapi.GetSitesDefaultDrive(ctx, gs, suite.site) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). With("site", suite.site) @@ -1243,6 +1380,7 @@ func runDriveIncrementalTest( acct = tester.NewM365Account(t) ffs = control.Toggles{} mb = evmock.NewBus() + ws = deeTD.DriveIDFromRepoRef // `now` has to be formatted with SimpleDateTimeTesting as // some drives cannot have `:` in file/folder names @@ -1251,9 +1389,10 @@ func runDriveIncrementalTest( categories = map[path.CategoryType][]string{ category: {graph.DeltaURLsFileName, graph.PreviousPathFileName}, } - container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) - container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) - container3 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 3, now) + container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) + container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) + container3 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 3, now) + containerRename = "renamed_folder" genDests = []string{container1, container2} @@ -1269,13 +1408,26 @@ func runDriveIncrementalTest( gc, sel := GCWithSelector(t, ctx, acct, resource, sel, nil, nil) + roidn := inMock.NewProvider(sel.ID(), sel.Name()) + var ( + atid = creds.AzureTenantID driveID = getTestDriveID(t, ctx, gc.Service) fileDBF = func(id, timeStamp, subject, body string) []byte { return []byte(id + subject) } + makeLocRef = func(flds ...string) string { + elems := append([]string{driveID, "root:"}, flds...) + return path.Builder{}.Append(elems...).String() + } ) + rrPfx, err := path.ServicePrefix(atid, roidn.ID(), service, category) + require.NoError(t, err, clues.ToCore(err)) + + // strip the category from the prefix; we primarily want the tenant and resource owner. + expectDeets := deeTD.NewInDeets(rrPfx.ToBuilder().Dir().String()) + // Populate initial test data. // Generate 2 new folders with two items each. Only the first two // folders will be part of the initial backup and @@ -1283,7 +1435,7 @@ func runDriveIncrementalTest( // through the changes. This should be enough to cover most delta // actions. for _, destName := range genDests { - generateContainerOfItems( + deets := generateContainerOfItems( t, ctx, gc, @@ -1291,11 +1443,19 @@ func runDriveIncrementalTest( acct, category, sel, - creds.AzureTenantID, owner, driveID, destName, + atid, roidn.ID(), driveID, destName, 2, // Use an old backup version so we don't need metadata files. 0, fileDBF) + + for _, ent := range deets.Entries { + if ent.Folder != nil { + continue + } + + expectDeets.AddItem(driveID, makeLocRef(destName), ent.ItemRef) + } } containerIDs := map[string]string{} @@ -1313,15 +1473,20 @@ func runDriveIncrementalTest( containerIDs[destName] = ptr.Val(resp.GetId()) } - bo, _, kw, ms, gc, _, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) + bo, _, kw, ms, ss, gc, _, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) defer closer() // run the initial backup runAndCheckBackup(t, ctx, &bo, mb, false) + // precheck to ensure the expectedDeets are correct. + // if we fail here, the expectedDeets were populated incorrectly. + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, ws, ms, ss, expectDeets, true) + var ( newFile models.DriveItemable newFileName = "new_file.txt" + newFileID string permissionIDMappings = map[string]string{} writePerm = metadata.Permission{ @@ -1363,6 +1528,10 @@ func runDriveIncrementalTest( targetContainer, driveItem) require.NoErrorf(t, err, "creating new file %v", clues.ToCore(err)) + + newFileID = ptr.Val(newFile.GetId()) + + expectDeets.AddItem(driveID, makeLocRef(container1), newFileID) }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1382,8 +1551,10 @@ func runDriveIncrementalTest( *newFile.GetId(), []metadata.Permission{writePerm}, []metadata.Permission{}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 1, // .data file for newitem itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) @@ -1403,8 +1574,10 @@ func runDriveIncrementalTest( *newFile.GetId(), []metadata.Permission{}, []metadata.Permission{writePerm}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 1, // .data file for newitem itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) @@ -1425,8 +1598,10 @@ func runDriveIncrementalTest( targetContainer, []metadata.Permission{writePerm}, []metadata.Permission{}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked5tgb }, itemsRead: 0, itemsWritten: 1, // .dirmeta for collection @@ -1447,8 +1622,10 @@ func runDriveIncrementalTest( targetContainer, []metadata.Permission{}, []metadata.Permission{writePerm}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 0, itemsWritten: 1, // .dirmeta for collection @@ -1463,6 +1640,7 @@ func runDriveIncrementalTest( Content(). Put(ctx, []byte("new content"), nil) require.NoErrorf(t, err, "updating file contents: %v", clues.ToCore(err)) + // no expectedDeets: neither file id nor location changed }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1488,11 +1666,12 @@ func runDriveIncrementalTest( }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + // no expectedDeets: neither file id nor location changed }, { name: "move a file between folders", updateFiles: func(t *testing.T) { - dest := containerIDs[container1] + dest := containerIDs[container2] driveItem := models.NewDriveItem() driveItem.SetName(&newFileName) @@ -1506,6 +1685,12 @@ func runDriveIncrementalTest( ItemsById(ptr.Val(newFile.GetId())). Patch(ctx, driveItem, nil) require.NoErrorf(t, err, "moving file between folders %v", clues.ToCore(err)) + + expectDeets.MoveItem( + driveID, + makeLocRef(container1), + makeLocRef(container2), + ptr.Val(newFile.GetId())) }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1521,6 +1706,8 @@ func runDriveIncrementalTest( ItemsById(ptr.Val(newFile.GetId())). Delete(ctx, nil) require.NoErrorf(t, err, "deleting file %v", clues.ToCore(err)) + + expectDeets.RemoveItem(driveID, makeLocRef(container2), ptr.Val(newFile.GetId())) }, itemsRead: 0, itemsWritten: 0, @@ -1528,21 +1715,26 @@ func runDriveIncrementalTest( { name: "move a folder to a subfolder", updateFiles: func(t *testing.T) { - dest := containerIDs[container1] - source := containerIDs[container2] + parent := containerIDs[container1] + child := containerIDs[container2] driveItem := models.NewDriveItem() driveItem.SetName(&container2) parentRef := models.NewItemReference() - parentRef.SetId(&dest) + parentRef.SetId(&parent) driveItem.SetParentReference(parentRef) _, err := gc.Service. Client(). DrivesById(driveID). - ItemsById(source). + ItemsById(child). Patch(ctx, driveItem, nil) require.NoError(t, err, "moving folder", clues.ToCore(err)) + + expectDeets.MoveLocation( + driveID, + makeLocRef(container2), + makeLocRef(container1)) }, itemsRead: 0, itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) @@ -1554,8 +1746,7 @@ func runDriveIncrementalTest( child := containerIDs[container2] driveItem := models.NewDriveItem() - name := "renamed_folder" - driveItem.SetName(&name) + driveItem.SetName(&containerRename) parentRef := models.NewItemReference() parentRef.SetId(&parent) driveItem.SetParentReference(parentRef) @@ -1566,6 +1757,13 @@ func runDriveIncrementalTest( ItemsById(child). Patch(ctx, driveItem, nil) require.NoError(t, err, "renaming folder", clues.ToCore(err)) + + containerIDs[containerRename] = containerIDs[container2] + + expectDeets.RenameLocation( + driveID, + makeLocRef(container1, container2), + makeLocRef(container1, containerRename)) }, itemsRead: 0, itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) @@ -1573,7 +1771,7 @@ func runDriveIncrementalTest( { name: "delete a folder", updateFiles: func(t *testing.T) { - container := containerIDs[container2] + container := containerIDs[containerRename] // deletes require unique http clients // https://github.com/alcionai/corso/issues/2707 err = newDeleteServicer(t). @@ -1582,6 +1780,8 @@ func runDriveIncrementalTest( ItemsById(container). Delete(ctx, nil) require.NoError(t, err, "deleting folder", clues.ToCore(err)) + + expectDeets.RemoveLocation(driveID, makeLocRef(container1, containerRename)) }, itemsRead: 0, itemsWritten: 0, @@ -1597,7 +1797,7 @@ func runDriveIncrementalTest( acct, category, sel, - creds.AzureTenantID, owner, driveID, container3, + atid, roidn.ID(), driveID, container3, 2, 0, fileDBF) @@ -1612,6 +1812,8 @@ func runDriveIncrementalTest( require.NoError(t, err, "getting drive folder ID", "folder name", container3, clues.ToCore(err)) containerIDs[container3] = ptr.Val(resp.GetId()) + + expectDeets.AddLocation(driveID, container3) }, itemsRead: 2, // 2 .data for 2 files itemsWritten: 6, // read items + 2 directory meta @@ -1639,17 +1841,11 @@ func runDriveIncrementalTest( err = incBO.Run(ctx) require.NoError(t, err, clues.ToCore(err)) - checkBackupIsInManifests(t, ctx, kw, &incBO, sel, sel.ID(), maps.Keys(categories)...) - checkMetadataFilesExist( - t, - ctx, - incBO.Results.BackupID, - kw, - ms, - creds.AzureTenantID, - sel.ID(), - service, - categories) + bupID := incBO.Results.BackupID + + checkBackupIsInManifests(t, ctx, kw, &incBO, sel, roidn.ID(), maps.Keys(categories)...) + checkMetadataFilesExist(t, ctx, bupID, kw, ms, atid, roidn.ID(), service, categories) + deeTD.CheckBackupDetails(t, ctx, bupID, ws, ms, ss, expectDeets, true) // do some additional checks to ensure the incremental dealt with fewer items. // +2 on read/writes to account for metadata: 1 delta and 1 path. @@ -1661,7 +1857,7 @@ func runDriveIncrementalTest( assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") assert.Equal(t, incMB.CalledWith[events.BackupStart][0][events.BackupID], - incBO.Results.BackupID, "incremental backupID pre-declaration") + bupID, "incremental backupID pre-declaration") }) } } @@ -1699,7 +1895,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { oldsel := selectors.NewOneDriveBackup([]string{uname}) oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel)) - bo, _, kw, ms, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, ffs, 0) + bo, _, kw, ms, _, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, ffs, 0) defer closer() // ensure the initial owner uses name in both cases @@ -1800,7 +1996,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { sel.Include(selTD.SharePointBackupFolderScope(sel)) - bo, _, kw, _, _, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, control.Toggles{}, version.Backup) + bo, _, kw, _, _, _, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, control.Toggles{}, version.Backup) defer closer() runAndCheckBackup(t, ctx, &bo, mb, false) diff --git a/src/pkg/backup/details/testdata/in_deets.go b/src/pkg/backup/details/testdata/in_deets.go new file mode 100644 index 000000000..b15c50f17 --- /dev/null +++ b/src/pkg/backup/details/testdata/in_deets.go @@ -0,0 +1,368 @@ +package testdata + +import ( + "context" + "strings" + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// location set handling +// --------------------------------------------------------------------------- + +var exists = struct{}{} + +type locSet struct { + // map [locationRef] map [itemRef] {} + // refs may be either the canonical ent refs, or something else, + // so long as they are consistent for the test in question + Locations map[string]map[string]struct{} + Deleted map[string]map[string]struct{} +} + +func newLocSet() *locSet { + return &locSet{ + Locations: map[string]map[string]struct{}{}, + Deleted: map[string]map[string]struct{}{}, + } +} + +func (ls *locSet) AddItem(locationRef, itemRef string) { + ls.AddLocation(locationRef) + + ls.Locations[locationRef][itemRef] = exists + delete(ls.Deleted[locationRef], itemRef) +} + +func (ls *locSet) RemoveItem(locationRef, itemRef string) { + delete(ls.Locations[locationRef], itemRef) + + if _, ok := ls.Deleted[locationRef]; !ok { + ls.Deleted[locationRef] = map[string]struct{}{} + } + + ls.Deleted[locationRef][itemRef] = exists +} + +func (ls *locSet) MoveItem(fromLocation, toLocation, ir string) { + ls.RemoveItem(fromLocation, ir) + ls.AddItem(toLocation, ir) +} + +func (ls *locSet) AddLocation(locationRef string) { + if _, ok := ls.Locations[locationRef]; !ok { + ls.Locations[locationRef] = map[string]struct{}{} + } + // don't purge previously deleted items, or child locations. + // Assumption is that their itemRef is unique, and still deleted. + delete(ls.Deleted, locationRef) +} + +func (ls *locSet) RemoveLocation(locationRef string) { + ss := ls.Subset(locationRef) + + for lr := range ss.Locations { + items := ls.Locations[lr] + + delete(ls.Locations, lr) + + if _, ok := ls.Deleted[lr]; !ok { + ls.Deleted[lr] = map[string]struct{}{} + } + + for ir := range items { + ls.Deleted[lr][ir] = exists + } + } +} + +// MoveLocation takes the LAST elemet in the fromLocation (and all) +// children matching the prefix, and relocates it as a child of toLocation. +// ex: MoveLocation("/a/b/c", "/d") will move all entries with the prefix +// "/a/b/c" into "/d/c". This also deletes all "/a/b/c" entries and children. +// assumes item IDs don't change across the migration. If item IDs do change, +// that difference will need to be handled manually by the caller. +// returns the base folder's new location (ex: /d/c) +func (ls *locSet) MoveLocation(fromLocation, toLocation string) string { + fromBuilder := path.Builder{}.Append(path.Split(fromLocation)...) + toBuilder := path.Builder{}.Append(path.Split(toLocation)...).Append(fromBuilder.LastElem()) + + ls.RenameLocation(fromBuilder.String(), toBuilder.String()) + + return toBuilder.String() +} + +func (ls *locSet) RenameLocation(fromLocation, toLocation string) { + ss := ls.Subset(fromLocation) + fromBuilder := path.Builder{}.Append(path.Split(fromLocation)...) + toBuilder := path.Builder{}.Append(path.Split(toLocation)...) + + for lr, items := range ss.Locations { + lrBuilder := path.Builder{}.Append(path.Split(lr)...) + lrBuilder.UpdateParent(fromBuilder, toBuilder) + + newLoc := lrBuilder.String() + + for ir := range items { + ls.RemoveItem(lr, ir) + ls.AddItem(newLoc, ir) + } + + ls.RemoveLocation(lr) + ls.AddLocation(newLoc) + } +} + +// Subset produces a new locSet containing only Items and Locations +// whose location matches the locationPfx +func (ls *locSet) Subset(locationPfx string) *locSet { + ss := newLocSet() + + for lr, items := range ls.Locations { + if strings.HasPrefix(lr, locationPfx) { + ss.AddLocation(lr) + + for ir := range items { + ss.AddItem(lr, ir) + } + } + } + + return ss +} + +// --------------------------------------------------------------------------- +// The goal of InDeets is to provide a struct and interface which allows +// tests to predict not just the elements within a set of details entries, +// but also their changes (relocation, renaming, etc) in a way that consolidates +// building an "expected set" of details entries that can be compared against +// the details results after a backup. +// --------------------------------------------------------------------------- + +// InDeets is a helper for comparing details state in tests +// across backup instances. +type InDeets struct { + // only: tenantID/service/resourceOwnerID + RRPrefix string + // map of container setting the uniqueness boundary for location + // ref entries (eg, data type like email, contacts, etc, or + // drive id) to the unique entries in that set. + Sets map[string]*locSet +} + +func NewInDeets(repoRefPrefix string) *InDeets { + return &InDeets{ + RRPrefix: repoRefPrefix, + Sets: map[string]*locSet{}, + } +} + +func (id *InDeets) getSet(set string) *locSet { + s, ok := id.Sets[set] + if ok { + return s + } + + return newLocSet() +} + +func (id *InDeets) AddAll(deets details.Details, ws whatSet) { + if id.Sets == nil { + id.Sets = map[string]*locSet{} + } + + for _, ent := range deets.Entries { + set, err := ws(ent) + if err != nil { + set = err.Error() + } + + dir := ent.LocationRef + + if ent.Folder != nil { + dir = dir + ent.Folder.DisplayName + id.AddLocation(set, dir) + } else { + id.AddItem(set, ent.LocationRef, ent.ItemRef) + } + } +} + +func (id *InDeets) AddItem(set, locationRef, itemRef string) { + id.getSet(set).AddItem(locationRef, itemRef) +} + +func (id *InDeets) RemoveItem(set, locationRef, itemRef string) { + id.getSet(set).RemoveItem(locationRef, itemRef) +} + +func (id *InDeets) MoveItem(set, fromLocation, toLocation, ir string) { + id.getSet(set).MoveItem(fromLocation, toLocation, ir) +} + +func (id *InDeets) AddLocation(set, locationRef string) { + id.getSet(set).AddLocation(locationRef) +} + +// RemoveLocation removes the provided location, and all children +// of that location. +func (id *InDeets) RemoveLocation(set, locationRef string) { + id.getSet(set).RemoveLocation(locationRef) +} + +// MoveLocation takes the LAST elemet in the fromLocation (and all) +// children matching the prefix, and relocates it as a child of toLocation. +// ex: MoveLocation("/a/b/c", "/d") will move all entries with the prefix +// "/a/b/c" into "/d/c". This also deletes all "/a/b/c" entries and children. +// assumes item IDs don't change across the migration. If item IDs do change, +// that difference will need to be handled manually by the caller. +// returns the base folder's new location (ex: /d/c) +func (id *InDeets) MoveLocation(set, fromLocation, toLocation string) string { + return id.getSet(set).MoveLocation(fromLocation, toLocation) +} + +func (id *InDeets) RenameLocation(set, fromLocation, toLocation string) { + id.getSet(set).RenameLocation(fromLocation, toLocation) +} + +// Subset produces a new locSet containing only Items and Locations +// whose location matches the locationPfx +func (id *InDeets) Subset(set, locationPfx string) *locSet { + return id.getSet(set).Subset(locationPfx) +} + +// --------------------------------------------------------------------------- +// whatSet helpers for extracting a set identifier from an arbitrary repoRef +// --------------------------------------------------------------------------- + +type whatSet func(details.Entry) (string, error) + +// common whatSet parser that extracts the service category from +// a repoRef. +func CategoryFromRepoRef(ent details.Entry) (string, error) { + p, err := path.FromDataLayerPath(ent.RepoRef, false) + if err != nil { + return "", err + } + + return p.Category().String(), nil +} + +// common whatSet parser that extracts the driveID from a repoRef. +func DriveIDFromRepoRef(ent details.Entry) (string, error) { + p, err := path.FromDataLayerPath(ent.RepoRef, false) + if err != nil { + return "", err + } + + odp, err := path.ToDrivePath(p) + if err != nil { + return "", err + } + + return odp.DriveID, nil +} + +// --------------------------------------------------------------------------- +// helpers and comparators +// --------------------------------------------------------------------------- + +func CheckBackupDetails( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupID model.StableID, + ws whatSet, + ms *kopia.ModelStore, + ssr streamstore.Reader, + expect *InDeets, + // standard check is assert.Subset due to issues of external data cross- + // pollination. This should be true if the backup contains a unique directory + // of data. + mustEqualFolders bool, +) { + deets, result := GetDeetsInBackup(t, ctx, backupID, "", "", path.UnknownService, ws, ms, ssr) + + t.Log("details entries in result") + + for _, ent := range deets.Entries { + if ent.Folder == nil { + t.Log(ent.LocationRef) + t.Log(ent.ItemRef) + } + + assert.Truef( + t, + strings.HasPrefix(ent.RepoRef, expect.RRPrefix), + "all details should begin with the expected prefix\nwant: %s\ngot: %s", + expect.RRPrefix, ent.RepoRef) + } + + for set := range expect.Sets { + check := assert.Subsetf + + if mustEqualFolders { + check = assert.ElementsMatchf + } + + check( + t, + maps.Keys(result.Sets[set].Locations), + maps.Keys(expect.Sets[set].Locations), + "results in %s missing expected location", set) + + for lr, items := range expect.Sets[set].Deleted { + _, ok := result.Sets[set].Locations[lr] + assert.Falsef(t, ok, "deleted location in %s found in result: %s", set, lr) + + for ir := range items { + _, ok := result.Sets[set].Locations[lr][ir] + assert.Falsef(t, ok, "deleted item in %s found in result: %s", set, lr) + } + } + } +} + +func GetDeetsInBackup( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupID model.StableID, + tid, resourceOwner string, + service path.ServiceType, + ws whatSet, + ms *kopia.ModelStore, + ssr streamstore.Reader, +) (details.Details, *InDeets) { + bup := backup.Backup{} + + err := ms.Get(ctx, model.BackupSchema, backupID, &bup) + require.NoError(t, err, clues.ToCore(err)) + + ssid := bup.StreamStoreID + require.NotEmpty(t, ssid, "stream store ID") + + var deets details.Details + err = ssr.Read( + ctx, + ssid, + streamstore.DetailsReader(details.UnmarshalTo(&deets)), + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + + id := NewInDeets(path.Builder{}.Append(tid, service.String(), resourceOwner).String()) + id.AddAll(deets, ws) + + return deets, id +} diff --git a/src/pkg/backup/details/testdata/in_deets_test.go b/src/pkg/backup/details/testdata/in_deets_test.go new file mode 100644 index 000000000..81beb0b0f --- /dev/null +++ b/src/pkg/backup/details/testdata/in_deets_test.go @@ -0,0 +1,445 @@ +package testdata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/tester" +) + +type LocSetUnitSuite struct { + tester.Suite +} + +func TestLocSetUnitSuite(t *testing.T) { + suite.Run(t, &LocSetUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +const ( + l1 = "lr_1" + l2 = "lr_2" + l13 = "lr_1/lr_3" + l14 = "lr_1/lr_4" + i1 = "ir_1" + i2 = "ir_2" + i3 = "ir_3" + i4 = "ir_4" +) + +func (suite *LocSetUnitSuite) TestAdd() { + t := suite.T() + + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddLocation(l2) + + assert.ElementsMatch(t, []string{l1, l2}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) +} + +func (suite *LocSetUnitSuite) TestRemove() { + t := suite.T() + + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[l14])) + + // nop removal + ls.RemoveItem(l2, i1) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + + // item removal + ls.RemoveItem(l1, i2) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + + // nop location removal + ls.RemoveLocation(l2) + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + + // non-cascading location removal + ls.RemoveLocation(l13) + assert.ElementsMatch(t, []string{l1, l14}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[l14])) + + // cascading location removal + ls.RemoveLocation(l1) + assert.Empty(t, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) +} + +func (suite *LocSetUnitSuite) TestSubset() { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + table := []struct { + name string + locPfx string + expect func(*testing.T, *locSet) + }{ + { + name: "nop", + locPfx: l2, + expect: func(t *testing.T, ss *locSet) { + assert.Empty(t, maps.Keys(ss.Locations)) + }, + }, + { + name: "no items", + locPfx: l13, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l13}, maps.Keys(ss.Locations)) + assert.Empty(t, maps.Keys(ss.Locations[l13])) + }, + }, + { + name: "non-cascading", + locPfx: l14, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l14}, maps.Keys(ss.Locations)) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ss.Locations[l14])) + }, + }, + { + name: "cascading", + locPfx: l1, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ss.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ss.Locations[l1])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ss.Locations[l14])) + assert.Empty(t, maps.Keys(ss.Locations[l13])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + test.expect(t, ls.Subset(test.locPfx)) + }) + } +} + +func (suite *LocSetUnitSuite) TestRename() { + t := suite.T() + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + + table := []struct { + name string + from string + to string + expect func(*testing.T, *locSet) + }{ + { + name: "nop", + from: l2, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "no items", + from: l13, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, "foo", l14}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "with items", + from: l14, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, "foo"}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "cascading locations", + from: l1, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{"foo", "foo/lr_3", "foo/lr_4"}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations["foo"])) + assert.Empty(t, maps.Keys(ls.Locations["foo/lr_3"])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations["foo/lr_4"])) + }, + }, + { + name: "to existing location", + from: l14, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i1, i2, i3, i4}, maps.Keys(ls.Locations[l1])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + ls.RenameLocation(test.from, test.to) + test.expect(t, ls) + }) + } +} + +func (suite *LocSetUnitSuite) TestItem() { + t := suite.T() + b4 := "bar/lr_4" + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + ls.AddItem(b4, "fnord") + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + assert.ElementsMatch(t, []string{"fnord"}, maps.Keys(ts.Locations[b4])) + + table := []struct { + name string + item string + from string + to string + expect func(*testing.T, *locSet) + }{ + { + name: "nop item", + item: "floob", + from: l2, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2, "floob"}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + }, + }, + { + name: "nop origin", + item: i1, + from: "smarf", + to: l2, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations["smarf"])) + }, + }, + { + name: "new location", + item: i1, + from: l1, + to: "fnords", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations["fnords"])) + }, + }, + { + name: "existing location", + item: i1, + from: l1, + to: l2, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l2])) + }, + }, + { + name: "same location", + item: i1, + from: l1, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + ls.MoveItem(test.from, test.to, test.item) + test.expect(t, ls) + }) + } +} + +func (suite *LocSetUnitSuite) TestMoveLocation() { + t := suite.T() + b4 := "bar/lr_4" + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + ls.AddItem(b4, "fnord") + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + assert.ElementsMatch(t, []string{"fnord"}, maps.Keys(ts.Locations[b4])) + + table := []struct { + name string + from string + to string + expect func(*testing.T, *locSet) + expectNewLoc string + }{ + { + name: "nop root", + from: l2, + to: "", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + }, + expectNewLoc: l2, + }, + { + name: "nop child", + from: l2, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + assert.Empty(t, maps.Keys(ls.Locations["foo/"+l2])) + }, + expectNewLoc: "foo/" + l2, + }, + { + name: "no items", + from: l13, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + newLoc := "foo/lr_3" + assert.ElementsMatch(t, []string{l1, newLoc, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations[newLoc])) + }, + expectNewLoc: "foo/lr_3", + }, + { + name: "with items", + from: l14, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + newLoc := "foo/lr_4" + assert.ElementsMatch(t, []string{l1, l13, newLoc, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[newLoc])) + }, + expectNewLoc: "foo/lr_4", + }, + { + name: "cascading locations", + from: l1, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + pfx := "foo/" + assert.ElementsMatch(t, []string{pfx + l1, pfx + l13, pfx + l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[pfx+l1])) + assert.Empty(t, maps.Keys(ls.Locations[pfx+l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[pfx+l14])) + }, + expectNewLoc: "foo/" + l1, + }, + { + name: "to existing location", + from: l14, + to: "bar", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations["bar"])) + assert.ElementsMatch(t, []string{"fnord", i3, i4}, maps.Keys(ls.Locations[b4])) + }, + expectNewLoc: b4, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + newLoc := ls.MoveLocation(test.from, test.to) + test.expect(t, ls) + assert.Equal(t, test.expectNewLoc, newLoc) + }) + } +} diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 79a14ea95..33fae1763 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -85,7 +85,7 @@ type Path interface { Category() CategoryType Tenant() string ResourceOwner() string - Folder(bool) string + Folder(escaped bool) string Folders() Elements Item() string // UpdateParent updates parent from old to new if the item/folder was From 2e4fc71310e5942148571adb71bb5ed0c4a3d9df Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 10 May 2023 19:49:32 -0700 Subject: [PATCH 19/33] Add restore path generation code (#3362) In preparation for switching to folder IDs, add logic to generate the restore path based on prefix information from the RepoRef and LocationRef of items Contains fallback code (and tests) to handle older details versions that may not have had LocationRef Manually tested restore from old backup that didn't have any LocationRef information Manually tested restore checking that calendar names are shown instead of IDs in progress bar --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3197 * fixes #3218 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- .../restore_path_transformer.go | 180 ++++++++++ .../restore_path_transformer_test.go | 340 ++++++++++++++++++ src/internal/operations/restore.go | 43 +-- src/pkg/backup/details/testdata/testdata.go | 101 +++--- src/pkg/path/drive.go | 10 + src/pkg/path/drive_test.go | 47 +++ src/pkg/selectors/selectors_reduce_test.go | 12 - 7 files changed, 632 insertions(+), 101 deletions(-) create mode 100644 src/internal/operations/pathtransformer/restore_path_transformer.go create mode 100644 src/internal/operations/pathtransformer/restore_path_transformer_test.go diff --git a/src/internal/operations/pathtransformer/restore_path_transformer.go b/src/internal/operations/pathtransformer/restore_path_transformer.go new file mode 100644 index 000000000..db8b2befd --- /dev/null +++ b/src/internal/operations/pathtransformer/restore_path_transformer.go @@ -0,0 +1,180 @@ +package pathtransformer + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +func locationRef( + ent *details.Entry, + repoRef path.Path, + backupVersion int, +) (*path.Builder, error) { + loc := ent.LocationRef + + // At this backup version all data types should populate LocationRef. + if len(loc) > 0 || backupVersion >= version.OneDrive7LocationRef { + return path.Builder{}.SplitUnescapeAppend(loc) + } + + // We could get an empty LocationRef either because it wasn't populated or it + // was in the root of the data type. + elems := repoRef.Folders() + + if ent.OneDrive != nil || ent.SharePoint != nil { + dp, err := path.ToDrivePath(repoRef) + if err != nil { + return nil, clues.Wrap(err, "fallback for LocationRef") + } + + elems = append([]string{dp.Root}, dp.Folders...) + } + + return path.Builder{}.Append(elems...), nil +} + +func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, error) { + if len(locRef.Elements()) == 0 { + res, err := path.ServicePrefix( + repoRef.Tenant(), + repoRef.ResourceOwner(), + repoRef.Service(), + repoRef.Category()) + if err != nil { + return nil, clues.Wrap(err, "getting prefix for empty location") + } + + return res, nil + } + + return locRef.ToDataLayerPath( + repoRef.Tenant(), + repoRef.ResourceOwner(), + repoRef.Service(), + repoRef.Category(), + false) +} + +func drivePathMerge( + ent *details.Entry, + repoRef path.Path, + locRef *path.Builder, +) (path.Path, error) { + // Try getting the drive ID from the item. Not all details versions had it + // though. + var driveID string + + if ent.SharePoint != nil { + driveID = ent.SharePoint.DriveID + } else if ent.OneDrive != nil { + driveID = ent.OneDrive.DriveID + } + + // Fallback to trying to get from RepoRef. + if len(driveID) == 0 { + odp, err := path.ToDrivePath(repoRef) + if err != nil { + return nil, clues.Wrap(err, "fallback getting DriveID") + } + + driveID = odp.DriveID + } + + return basicLocationPath( + repoRef, + path.BuildDriveLocation(driveID, locRef.Elements()...)) +} + +func makeRestorePathsForEntry( + ctx context.Context, + backupVersion int, + ent *details.Entry, +) (path.RestorePaths, error) { + res := path.RestorePaths{} + + repoRef, err := path.FromDataLayerPath(ent.RepoRef, true) + if err != nil { + err = clues.Wrap(err, "parsing RepoRef"). + WithClues(ctx). + With("repo_ref", clues.Hide(ent.RepoRef), "location_ref", clues.Hide(ent.LocationRef)) + + return res, err + } + + res.StoragePath = repoRef + ctx = clues.Add(ctx, "repo_ref", repoRef) + + // Get the LocationRef so we can munge it onto our path. + locRef, err := locationRef(ent, repoRef, backupVersion) + if err != nil { + err = clues.Wrap(err, "parsing LocationRef after reduction"). + WithClues(ctx). + With("location_ref", clues.Hide(ent.LocationRef)) + + return res, err + } + + ctx = clues.Add(ctx, "location_ref", locRef) + + // Now figure out what type of ent it is and munge the path accordingly. + // Eventually we're going to need munging for: + // * Exchange Calendars (different folder handling) + // * Exchange Email/Contacts + // * OneDrive/SharePoint (needs drive information) + if ent.Exchange != nil { + // TODO(ashmrtn): Eventually make Events have it's own function to handle + // setting the restore destination properly. + res.RestorePath, err = basicLocationPath(repoRef, locRef) + } else if ent.OneDrive != nil || + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.SharePointLibrary) || + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.OneDriveItem) { + res.RestorePath, err = drivePathMerge(ent, repoRef, locRef) + } else { + return res, clues.New("unknown entry type").WithClues(ctx) + } + + if err != nil { + return res, clues.Wrap(err, "generating RestorePath").WithClues(ctx) + } + + return res, nil +} + +// GetPaths takes a set of filtered details entries and returns a set of +// RestorePaths for the entries. +func GetPaths( + ctx context.Context, + backupVersion int, + items []*details.Entry, + errs *fault.Bus, +) ([]path.RestorePaths, error) { + var ( + paths = make([]path.RestorePaths, len(items)) + el = errs.Local() + ) + + for i, ent := range items { + if el.Failure() != nil { + break + } + + restorePaths, err := makeRestorePathsForEntry(ctx, backupVersion, ent) + if err != nil { + el.AddRecoverable(clues.Wrap(err, "getting restore paths")) + continue + } + + paths[i] = restorePaths + } + + logger.Ctx(ctx).Infof("found %d details entries to restore", len(paths)) + + return paths, el.Failure() +} diff --git a/src/internal/operations/pathtransformer/restore_path_transformer_test.go b/src/internal/operations/pathtransformer/restore_path_transformer_test.go new file mode 100644 index 000000000..57381c3cf --- /dev/null +++ b/src/internal/operations/pathtransformer/restore_path_transformer_test.go @@ -0,0 +1,340 @@ +package pathtransformer_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/operations/pathtransformer" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/details/testdata" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type RestorePathTransformerUnitSuite struct { + tester.Suite +} + +func TestRestorePathTransformerUnitSuite(t *testing.T) { + suite.Run(t, &RestorePathTransformerUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { + type expectPaths struct { + storage string + restore string + isRestorePrefix bool + } + + toRestore := func( + repoRef path.Path, + unescapedFolders ...string, + ) string { + return path.Builder{}. + Append( + repoRef.Tenant(), + repoRef.Service().String(), + repoRef.ResourceOwner(), + repoRef.Category().String()). + Append(unescapedFolders...). + String() + } + + var ( + driveID = "some-drive-id" + extraItemName = "some-item" + SharePointRootItemPath = testdata.SharePointRootPath.MustAppend(extraItemName, true) + ) + + table := []struct { + name string + backupVersion int + input []*details.Entry + expectErr assert.ErrorAssertionFunc + expected []expectPaths + }{ + { + name: "SharePoint List Errors", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointList, + }, + }, + }, + }, + expectErr: assert.Error, + }, + { + name: "SharePoint Page Errors", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointPage, + }, + }, + }, + }, + expectErr: assert.Error, + }, + { + name: "SharePoint old format, item in root", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.OneDriveItem, + DriveID: driveID, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: SharePointRootItemPath.RR.String(), + restore: toRestore( + SharePointRootItemPath.RR, + append( + []string{"drives", driveID}, + SharePointRootItemPath.Loc.Elements()...)...), + }, + }, + }, + { + name: "SharePoint, no LocationRef, no DriveID, item in root", + backupVersion: version.OneDrive6NameInMeta, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointLibrary, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: SharePointRootItemPath.RR.String(), + restore: toRestore( + SharePointRootItemPath.RR, + append( + []string{"drives"}, + // testdata path has '.d' on the drives folder we need to remove. + SharePointRootItemPath.RR.Folders()[1:]...)...), + }, + }, + }, + { + name: "OneDrive, nested item", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.OneDriveItemPath2.RR.String(), + LocationRef: testdata.OneDriveItemPath2.Loc.String(), + ItemInfo: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + ItemType: details.OneDriveItem, + DriveID: driveID, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.OneDriveItemPath2.RR.String(), + restore: toRestore( + testdata.OneDriveItemPath2.RR, + append( + []string{"drives", driveID}, + testdata.OneDriveItemPath2.Loc.Elements()...)...), + }, + }, + }, + { + name: "Exchange Email, extra / in path", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Email, no LocationRef, extra / in path", + backupVersion: version.OneDrive7LocationRef, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Contact", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeContactsItemPath1.RR.String(), + LocationRef: testdata.ExchangeContactsItemPath1.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeContact, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeContactsItemPath1.RR.String(), + restore: toRestore( + testdata.ExchangeContactsItemPath1.RR, + testdata.ExchangeContactsItemPath1.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Contact, root dir", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeContactsItemPath1.RR.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeContact, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeContactsItemPath1.RR.String(), + restore: toRestore(testdata.ExchangeContactsItemPath1.RR, "tmp"), + isRestorePrefix: true, + }, + }, + }, + { + name: "Exchange Event", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + + paths, err := pathtransformer.GetPaths( + ctx, + test.backupVersion, + test.input, + fault.New(true)) + test.expectErr(t, err, clues.ToCore(err)) + + if err != nil { + return + } + + expected := make([]path.RestorePaths, 0, len(test.expected)) + + for _, e := range test.expected { + tmp := path.RestorePaths{} + p, err := path.FromDataLayerPath(e.storage, true) + require.NoError(t, err, "parsing expected storage path", clues.ToCore(err)) + + tmp.StoragePath = p + + p, err = path.FromDataLayerPath(e.restore, false) + require.NoError(t, err, "parsing expected restore path", clues.ToCore(err)) + + if e.isRestorePrefix { + p, err = p.Dir() + require.NoError(t, err, "getting service prefix", clues.ToCore(err)) + } + + tmp.RestorePath = p + + expected = append(expected, tmp) + } + + assert.ElementsMatch(t, expected, paths) + }) + } +} diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 2dd5cd40c..28dbb5e1a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/operations/pathtransformer" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/pkg/account" @@ -355,41 +356,9 @@ func formatDetailsForRestoration( return nil, err } - var ( - fdsPaths = fds.Paths() - paths = make([]path.RestorePaths, len(fdsPaths)) - shortRefs = make([]string, len(fdsPaths)) - el = errs.Local() - ) - - for i := range fdsPaths { - if el.Failure() != nil { - break - } - - p, err := path.FromDataLayerPath(fdsPaths[i], true) - if err != nil { - el.AddRecoverable(clues. - Wrap(err, "parsing details path after reduction"). - WithMap(clues.In(ctx)). - With("path", fdsPaths[i])) - - continue - } - - dir, err := p.Dir() - if err != nil { - el.AddRecoverable(clues. - Wrap(err, "getting restore directory after reduction"). - WithClues(ctx). - With("path", fdsPaths[i])) - - continue - } - - paths[i].StoragePath = p - paths[i].RestorePath = dir - shortRefs[i] = p.ShortRef() + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) + if err != nil { + return nil, clues.Wrap(err, "getting restore paths") } if sel.Service == selectors.ServiceOneDrive { @@ -399,7 +368,5 @@ func formatDetailsForRestoration( } } - logger.Ctx(ctx).With("short_refs", shortRefs).Infof("found %d details entries to restore", len(shortRefs)) - - return paths, el.Failure() + return paths, nil } diff --git a/src/pkg/backup/details/testdata/testdata.go b/src/pkg/backup/details/testdata/testdata.go index 0b770c050..a406d838a 100644 --- a/src/pkg/backup/details/testdata/testdata.go +++ b/src/pkg/backup/details/testdata/testdata.go @@ -54,10 +54,10 @@ func locFromRepo(rr path.Path, isItem bool) *path.Builder { type repoRefAndLocRef struct { RR path.Path - loc *path.Builder + Loc *path.Builder } -func (p repoRefAndLocRef) mustAppend(newElement string, isItem bool) repoRefAndLocRef { +func (p repoRefAndLocRef) MustAppend(newElement string, isItem bool) repoRefAndLocRef { e := newElement + folderSuffix if isItem { @@ -68,7 +68,7 @@ func (p repoRefAndLocRef) mustAppend(newElement string, isItem bool) repoRefAndL RR: mustAppendPath(p.RR, e, isItem), } - res.loc = locFromRepo(res.RR, isItem) + res.Loc = locFromRepo(res.RR, isItem) return res } @@ -85,7 +85,7 @@ func (p repoRefAndLocRef) FolderLocation() string { lastElem = f[len(f)-2] } - return p.loc.Append(strings.TrimSuffix(lastElem, folderSuffix)).String() + return p.Loc.Append(strings.TrimSuffix(lastElem, folderSuffix)).String() } func mustPathRep(ref string, isItem bool) repoRefAndLocRef { @@ -115,7 +115,7 @@ func mustPathRep(ref string, isItem bool) repoRefAndLocRef { } res.RR = rr - res.loc = locFromRepo(rr, isItem) + res.Loc = locFromRepo(rr, isItem) return res } @@ -138,12 +138,12 @@ var ( Time4 = time.Date(2023, 10, 21, 10, 0, 0, 0, time.UTC) ExchangeEmailInboxPath = mustPathRep("tenant-id/exchange/user-id/email/Inbox", false) - ExchangeEmailBasePath = ExchangeEmailInboxPath.mustAppend("subfolder", false) - ExchangeEmailBasePath2 = ExchangeEmailInboxPath.mustAppend("othersubfolder/", false) - ExchangeEmailBasePath3 = ExchangeEmailBasePath2.mustAppend("subsubfolder", false) - ExchangeEmailItemPath1 = ExchangeEmailBasePath.mustAppend(ItemName1, true) - ExchangeEmailItemPath2 = ExchangeEmailBasePath2.mustAppend(ItemName2, true) - ExchangeEmailItemPath3 = ExchangeEmailBasePath3.mustAppend(ItemName3, true) + ExchangeEmailBasePath = ExchangeEmailInboxPath.MustAppend("subfolder", false) + ExchangeEmailBasePath2 = ExchangeEmailInboxPath.MustAppend("othersubfolder/", false) + ExchangeEmailBasePath3 = ExchangeEmailBasePath2.MustAppend("subsubfolder", false) + ExchangeEmailItemPath1 = ExchangeEmailBasePath.MustAppend(ItemName1, true) + ExchangeEmailItemPath2 = ExchangeEmailBasePath2.MustAppend(ItemName2, true) + ExchangeEmailItemPath3 = ExchangeEmailBasePath3.MustAppend(ItemName3, true) ExchangeEmailItems = []details.Entry{ { @@ -151,7 +151,7 @@ var ( ShortRef: ExchangeEmailItemPath1.RR.ShortRef(), ParentRef: ExchangeEmailItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath1.ItemLocation(), - LocationRef: ExchangeEmailItemPath1.loc.String(), + LocationRef: ExchangeEmailItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -166,7 +166,7 @@ var ( ShortRef: ExchangeEmailItemPath2.RR.ShortRef(), ParentRef: ExchangeEmailItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath2.ItemLocation(), - LocationRef: ExchangeEmailItemPath2.loc.String(), + LocationRef: ExchangeEmailItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -181,7 +181,7 @@ var ( ShortRef: ExchangeEmailItemPath3.RR.ShortRef(), ParentRef: ExchangeEmailItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath3.ItemLocation(), - LocationRef: ExchangeEmailItemPath3.loc.String(), + LocationRef: ExchangeEmailItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -194,10 +194,10 @@ var ( } ExchangeContactsRootPath = mustPathRep("tenant-id/exchange/user-id/contacts/contacts", false) - ExchangeContactsBasePath = ExchangeContactsRootPath.mustAppend("contacts", false) - ExchangeContactsBasePath2 = ExchangeContactsRootPath.mustAppend("morecontacts", false) - ExchangeContactsItemPath1 = ExchangeContactsBasePath.mustAppend(ItemName1, true) - ExchangeContactsItemPath2 = ExchangeContactsBasePath2.mustAppend(ItemName2, true) + ExchangeContactsBasePath = ExchangeContactsRootPath.MustAppend("contacts", false) + ExchangeContactsBasePath2 = ExchangeContactsRootPath.MustAppend("morecontacts", false) + ExchangeContactsItemPath1 = ExchangeContactsBasePath.MustAppend(ItemName1, true) + ExchangeContactsItemPath2 = ExchangeContactsBasePath2.MustAppend(ItemName2, true) ExchangeContactsItems = []details.Entry{ { @@ -205,7 +205,7 @@ var ( ShortRef: ExchangeContactsItemPath1.RR.ShortRef(), ParentRef: ExchangeContactsItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeContactsItemPath1.ItemLocation(), - LocationRef: ExchangeContactsItemPath1.loc.String(), + LocationRef: ExchangeContactsItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -218,7 +218,7 @@ var ( ShortRef: ExchangeContactsItemPath2.RR.ShortRef(), ParentRef: ExchangeContactsItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeContactsItemPath2.ItemLocation(), - LocationRef: ExchangeContactsItemPath2.loc.String(), + LocationRef: ExchangeContactsItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -228,11 +228,10 @@ var ( }, } - ExchangeEventsRootPath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false) - ExchangeEventsBasePath = ExchangeEventsRootPath.mustAppend("holidays", false) - ExchangeEventsBasePath2 = ExchangeEventsRootPath.mustAppend("moreholidays", false) - ExchangeEventsItemPath1 = ExchangeEventsBasePath.mustAppend(ItemName1, true) - ExchangeEventsItemPath2 = ExchangeEventsBasePath2.mustAppend(ItemName2, true) + ExchangeEventsBasePath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false) + ExchangeEventsBasePath2 = mustPathRep("tenant-id/exchange/user-id/events/moreholidays", false) + ExchangeEventsItemPath1 = ExchangeEventsBasePath.MustAppend(ItemName1, true) + ExchangeEventsItemPath2 = ExchangeEventsBasePath2.MustAppend(ItemName2, true) ExchangeEventsItems = []details.Entry{ { @@ -240,7 +239,7 @@ var ( ShortRef: ExchangeEventsItemPath1.RR.ShortRef(), ParentRef: ExchangeEventsItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEventsItemPath1.ItemLocation(), - LocationRef: ExchangeEventsItemPath1.loc.String(), + LocationRef: ExchangeEventsItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -256,7 +255,7 @@ var ( ShortRef: ExchangeEventsItemPath2.RR.ShortRef(), ParentRef: ExchangeEventsItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEventsItemPath2.ItemLocation(), - LocationRef: ExchangeEventsItemPath2.loc.String(), + LocationRef: ExchangeEventsItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -270,17 +269,17 @@ var ( } OneDriveRootPath = mustPathRep("tenant-id/onedrive/user-id/files/drives/foo/root:", false) - OneDriveFolderPath = OneDriveRootPath.mustAppend("folder", false) - OneDriveBasePath1 = OneDriveFolderPath.mustAppend("a", false) - OneDriveBasePath2 = OneDriveFolderPath.mustAppend("b", false) + OneDriveFolderPath = OneDriveRootPath.MustAppend("folder", false) + OneDriveBasePath1 = OneDriveFolderPath.MustAppend("a", false) + OneDriveBasePath2 = OneDriveFolderPath.MustAppend("b", false) - OneDriveItemPath1 = OneDriveFolderPath.mustAppend(ItemName1, true) - OneDriveItemPath2 = OneDriveBasePath1.mustAppend(ItemName2, true) - OneDriveItemPath3 = OneDriveBasePath2.mustAppend(ItemName3, true) + OneDriveItemPath1 = OneDriveFolderPath.MustAppend(ItemName1, true) + OneDriveItemPath2 = OneDriveBasePath1.MustAppend(ItemName2, true) + OneDriveItemPath3 = OneDriveBasePath2.MustAppend(ItemName3, true) - OneDriveFolderFolder = OneDriveFolderPath.loc.PopFront().String() - OneDriveParentFolder1 = OneDriveBasePath1.loc.PopFront().String() - OneDriveParentFolder2 = OneDriveBasePath2.loc.PopFront().String() + OneDriveFolderFolder = OneDriveFolderPath.Loc.PopFront().String() + OneDriveParentFolder1 = OneDriveBasePath1.Loc.PopFront().String() + OneDriveParentFolder2 = OneDriveBasePath2.Loc.PopFront().String() OneDriveItems = []details.Entry{ { @@ -288,7 +287,7 @@ var ( ShortRef: OneDriveItemPath1.RR.ShortRef(), ParentRef: OneDriveItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath1.ItemLocation(), - LocationRef: OneDriveItemPath1.loc.String(), + LocationRef: OneDriveItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -306,7 +305,7 @@ var ( ShortRef: OneDriveItemPath2.RR.ShortRef(), ParentRef: OneDriveItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath2.ItemLocation(), - LocationRef: OneDriveItemPath2.loc.String(), + LocationRef: OneDriveItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -324,7 +323,7 @@ var ( ShortRef: OneDriveItemPath3.RR.ShortRef(), ParentRef: OneDriveItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath3.ItemLocation(), - LocationRef: OneDriveItemPath3.loc.String(), + LocationRef: OneDriveItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -340,17 +339,17 @@ var ( } SharePointRootPath = mustPathRep("tenant-id/sharepoint/site-id/libraries/drives/foo/root:", false) - SharePointLibraryPath = SharePointRootPath.mustAppend("library", false) - SharePointBasePath1 = SharePointLibraryPath.mustAppend("a", false) - SharePointBasePath2 = SharePointLibraryPath.mustAppend("b", false) + SharePointLibraryPath = SharePointRootPath.MustAppend("library", false) + SharePointBasePath1 = SharePointLibraryPath.MustAppend("a", false) + SharePointBasePath2 = SharePointLibraryPath.MustAppend("b", false) - SharePointLibraryItemPath1 = SharePointLibraryPath.mustAppend(ItemName1, true) - SharePointLibraryItemPath2 = SharePointBasePath1.mustAppend(ItemName2, true) - SharePointLibraryItemPath3 = SharePointBasePath2.mustAppend(ItemName3, true) + SharePointLibraryItemPath1 = SharePointLibraryPath.MustAppend(ItemName1, true) + SharePointLibraryItemPath2 = SharePointBasePath1.MustAppend(ItemName2, true) + SharePointLibraryItemPath3 = SharePointBasePath2.MustAppend(ItemName3, true) - SharePointLibraryFolder = SharePointLibraryPath.loc.PopFront().String() - SharePointParentLibrary1 = SharePointBasePath1.loc.PopFront().String() - SharePointParentLibrary2 = SharePointBasePath2.loc.PopFront().String() + SharePointLibraryFolder = SharePointLibraryPath.Loc.PopFront().String() + SharePointParentLibrary1 = SharePointBasePath1.Loc.PopFront().String() + SharePointParentLibrary2 = SharePointBasePath2.Loc.PopFront().String() SharePointLibraryItems = []details.Entry{ { @@ -358,7 +357,7 @@ var ( ShortRef: SharePointLibraryItemPath1.RR.ShortRef(), ParentRef: SharePointLibraryItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath1.ItemLocation(), - LocationRef: SharePointLibraryItemPath1.loc.String(), + LocationRef: SharePointLibraryItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -376,7 +375,7 @@ var ( ShortRef: SharePointLibraryItemPath2.RR.ShortRef(), ParentRef: SharePointLibraryItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath2.ItemLocation(), - LocationRef: SharePointLibraryItemPath2.loc.String(), + LocationRef: SharePointLibraryItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -394,7 +393,7 @@ var ( ShortRef: SharePointLibraryItemPath3.RR.ShortRef(), ParentRef: SharePointLibraryItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath3.ItemLocation(), - LocationRef: SharePointLibraryItemPath3.loc.String(), + LocationRef: SharePointLibraryItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, diff --git a/src/pkg/path/drive.go b/src/pkg/path/drive.go index b073ff125..033f9934b 100644 --- a/src/pkg/path/drive.go +++ b/src/pkg/path/drive.go @@ -38,3 +38,13 @@ func GetDriveFolderPath(p Path) (string, error) { return Builder{}.Append(drivePath.Folders...).String(), nil } + +// BuildDriveLocation takes a driveID and a set of unescaped element names, +// including the root folder, and returns a *path.Builder containing the +// canonical path representation for the drive path. +func BuildDriveLocation( + driveID string, + unescapedElements ...string, +) *Builder { + return Builder{}.Append("drives", driveID).Append(unescapedElements...) +} diff --git a/src/pkg/path/drive_test.go b/src/pkg/path/drive_test.go index cddd050bf..bdbf09d9c 100644 --- a/src/pkg/path/drive_test.go +++ b/src/pkg/path/drive_test.go @@ -1,6 +1,7 @@ package path_test import ( + "strings" "testing" "github.com/alcionai/clues" @@ -63,3 +64,49 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }) } } + +func (suite *OneDrivePathSuite) TestFormatDriveFolders() { + const ( + driveID = "some-drive-id" + drivePrefix = "drives/" + driveID + ) + + table := []struct { + name string + input []string + expected string + }{ + { + name: "normal", + input: []string{ + "root:", + "foo", + "bar", + }, + expected: strings.Join( + append([]string{drivePrefix}, "root:", "foo", "bar"), + "/"), + }, + { + name: "has character that would be escaped", + input: []string{ + "root:", + "foo/", + "bar", + }, + // Element "foo/" should end up escaped in the string output. + expected: strings.Join( + append([]string{drivePrefix}, "root:", `foo\/`, "bar"), + "/"), + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + assert.Equal( + suite.T(), + test.expected, + path.BuildDriveLocation(driveID, test.input...).String()) + }) + } +} diff --git a/src/pkg/selectors/selectors_reduce_test.go b/src/pkg/selectors/selectors_reduce_test.go index b72a4c65e..c57cde409 100644 --- a/src/pkg/selectors/selectors_reduce_test.go +++ b/src/pkg/selectors/selectors_reduce_test.go @@ -249,18 +249,6 @@ func (suite *SelectorReduceSuite) TestReduce() { }, expected: []details.Entry{testdata.ExchangeEventsItems[0]}, }, - { - name: "ExchangeEventsByFolderRoot", - selFunc: func() selectors.Reducer { - sel := selectors.NewExchangeRestore(selectors.Any()) - sel.Include(sel.EventCalendars( - []string{testdata.ExchangeEventsRootPath.FolderLocation()}, - )) - - return sel - }, - expected: testdata.ExchangeEventsItems, - }, } for _, test := range table { From e7d2aeac5dc96346c4a87901185e1bef68a929f1 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 21:20:28 -0600 Subject: [PATCH 20/33] add drive-level tombstone for deleted drives (#3381) adds a tombstone collection for any drive that has been completely deleted (or that surfaced from a prior backup, but does not exist in the current) from the driveish account. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included #### Type of change - [x] :bug: Bugfix #### Issue(s) * #3379 #### Test Plan - [x] :zap: Unit test --- src/internal/connector/onedrive/collection.go | 14 +-- .../connector/onedrive/collections.go | 63 ++++++++--- .../connector/onedrive/collections_test.go | 106 +++++++++++++----- src/internal/connector/onedrive/drive.go | 20 ++++ src/pkg/path/drive_test.go | 6 +- src/pkg/selectors/onedrive_test.go | 2 +- src/pkg/selectors/sharepoint_test.go | 4 +- 7 files changed, 163 insertions(+), 52 deletions(-) diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index a4caafae2..26fd41283 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -131,7 +131,7 @@ func pathToLocation(p path.Path) (*path.Builder, error) { // NewCollection creates a Collection func NewCollection( itemClient graph.Requester, - folderPath path.Path, + currPath path.Path, prevPath path.Path, driveID string, service graph.Servicer, @@ -145,9 +145,9 @@ func NewCollection( // to be changed as we won't be able to extract path information from the // storage path. In that case, we'll need to start storing the location paths // like we do the previous path. - locPath, err := pathToLocation(folderPath) + locPath, err := pathToLocation(currPath) if err != nil { - return nil, clues.Wrap(err, "getting location").With("folder_path", folderPath.String()) + return nil, clues.Wrap(err, "getting location").With("curr_path", currPath.String()) } prevLocPath, err := pathToLocation(prevPath) @@ -157,7 +157,7 @@ func NewCollection( c := newColl( itemClient, - folderPath, + currPath, prevPath, driveID, service, @@ -175,7 +175,7 @@ func NewCollection( func newColl( gr graph.Requester, - folderPath path.Path, + currPath path.Path, prevPath path.Path, driveID string, service graph.Servicer, @@ -188,7 +188,7 @@ func newColl( c := &Collection{ itemClient: gr, itemGetter: api.GetDriveItem, - folderPath: folderPath, + folderPath: currPath, prevPath: prevPath, driveItems: map[string]models.DriveItemable{}, driveID: driveID, @@ -197,7 +197,7 @@ func newColl( data: make(chan data.Stream, graph.Parallelism(path.OneDriveMetadataService).CollectionBufferSize()), statusUpdater: statusUpdater, ctrl: ctrlOpts, - state: data.StateOf(prevPath, folderPath), + state: data.StateOf(prevPath, currPath), scope: colScope, doNotMergeItems: doNotMergeItems, } diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 8594e4a6f..4cb1944f7 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -101,6 +101,7 @@ type Collections struct { servicer graph.Servicer, driveID, link string, ) itemPager + servicePathPfxFunc pathPrefixerFunc // Track stats from drive enumeration. Represents the items backed up. NumItems int @@ -119,17 +120,18 @@ func NewCollections( ctrlOpts control.Options, ) *Collections { return &Collections{ - itemClient: itemClient, - tenant: tenant, - resourceOwner: resourceOwner, - source: source, - matcher: matcher, - CollectionMap: map[string]map[string]*Collection{}, - drivePagerFunc: PagerForSource, - itemPagerFunc: defaultItemPager, - service: service, - statusUpdater: statusUpdater, - ctrl: ctrlOpts, + itemClient: itemClient, + tenant: tenant, + resourceOwner: resourceOwner, + source: source, + matcher: matcher, + CollectionMap: map[string]map[string]*Collection{}, + drivePagerFunc: PagerForSource, + itemPagerFunc: defaultItemPager, + servicePathPfxFunc: pathPrefixerForSource(tenant, resourceOwner, source), + service: service, + statusUpdater: statusUpdater, + ctrl: ctrlOpts, } } @@ -280,6 +282,12 @@ func (c *Collections) Get( return nil, err } + driveTombstones := map[string]struct{}{} + + for driveID := range oldPathsByDriveID { + driveTombstones[driveID] = struct{}{} + } + driveComplete, closer := observe.MessageWithCompletion(ctx, observe.Bulletf("files")) defer closer() defer close(driveComplete) @@ -312,6 +320,8 @@ func (c *Collections) Get( ictx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) ) + delete(driveTombstones, driveID) + if _, ok := c.CollectionMap[driveID]; !ok { c.CollectionMap[driveID] = map[string]*Collection{} } @@ -408,7 +418,7 @@ func (c *Collections) Get( col, err := NewCollection( c.itemClient, - nil, + nil, // delete the folder prevPath, driveID, c.service, @@ -427,15 +437,41 @@ func (c *Collections) Get( observe.Message(ctx, fmt.Sprintf("Discovered %d items to backup", c.NumItems)) - // Add an extra for the metadata collection. collections := []data.BackupCollection{} + // add all the drives we found for _, driveColls := range c.CollectionMap { for _, coll := range driveColls { collections = append(collections, coll) } } + // generate tombstones for drives that were removed. + for driveID := range driveTombstones { + prevDrivePath, err := c.servicePathPfxFunc(driveID) + if err != nil { + return nil, clues.Wrap(err, "making drive tombstone previous path").WithClues(ctx) + } + + coll, err := NewCollection( + c.itemClient, + nil, // delete the drive + prevDrivePath, + driveID, + c.service, + c.statusUpdater, + c.source, + c.ctrl, + CollectionScopeUnknown, + true) + if err != nil { + return nil, clues.Wrap(err, "making drive tombstone").WithClues(ctx) + } + + collections = append(collections, coll) + } + + // add metadata collections service, category := c.source.toPathServiceCat() md, err := graph.MakeMetadataCollection( c.tenant, @@ -457,7 +493,6 @@ func (c *Collections) Get( collections = append(collections, md) } - // TODO(ashmrtn): Track and return the set of items to exclude. return collections, nil } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 1baaed521..e55bf2db8 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -1246,16 +1246,15 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { user, path.OneDriveService, path.FilesCategory, - false, - ) + false) require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err)) - driveID1 := uuid.NewString() + driveID1 := "drive-1-" + uuid.NewString() drive1 := models.NewDrive() drive1.SetId(&driveID1) drive1.SetName(&driveID1) - driveID2 := uuid.NewString() + driveID2 := "drive-2-" + uuid.NewString() drive2 := models.NewDrive() drive2.SetId(&driveID2) drive2.SetName(&driveID2) @@ -1287,7 +1286,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedFolderPaths map[string]map[string]string expectedDelList *pmMock.PrefixMap expectedSkippedCount int - doNotMergeItems bool + // map full or previous path (prefers full) -> bool + doNotMergeItems map[string]bool }{ { name: "OneDrive_OneItemPage_DelFileOnly_NoFolders_NoErrors", @@ -1321,7 +1321,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }), }, { - name: "OneDrive_OneItemPage_NoFolders_NoErrors", + name: "OneDrive_OneItemPage_NoFolderDeltas_NoErrors", drives: []models.Driveable{drive1}, items: map[string][]deltaPagerResult{ driveID1: { @@ -1699,7 +1699,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + }, }, { name: "OneDrive_TwoItemPage_DeltaError", @@ -1741,7 +1743,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "OneDrive_TwoItemPage_NoDeltaError", @@ -1785,7 +1790,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ rootFolderPath1: getDelList("file", "file2"), }), - doNotMergeItems: false, + doNotMergeItems: map[string]bool{}, }, { name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder", @@ -1827,7 +1832,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + expectedPath1("/folder2"): true, + }, }, { name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation", @@ -1873,7 +1882,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "OneDrive Two Item Pages with Malware", @@ -1973,7 +1985,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + expectedPath1("/folder2"): true, + }, }, { name: "One Drive Delta Error Random Folder Delete", @@ -2012,7 +2028,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "One Drive Delta Error Random Item Delete", @@ -2049,7 +2068,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + }, }, { name: "One Drive Folder Made And Deleted", @@ -2200,6 +2221,37 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { rootFolderPath1: getDelList("file"), }), }, + { + name: "TwoPriorDrives_OneTombstoned", + drives: []models.Driveable{drive1}, + items: map[string][]deltaPagerResult{ + driveID1: { + { + items: []models.DriveItemable{ + driveRootItem("root"), // will be present + }, + deltaLink: &delta, + }, + }, + }, + errCheck: assert.NoError, + prevFolderPaths: map[string]map[string]string{ + driveID1: {"root": rootFolderPath1}, + driveID2: {"root": rootFolderPath2}, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + rootFolderPath1: {data.NotMovedState: {}}, + rootFolderPath2: {data.DeletedState: {}}, + }, + expectedDeltaURLs: map[string]string{driveID1: delta}, + expectedFolderPaths: map[string]map[string]string{ + driveID1: {"root": rootFolderPath1}, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + rootFolderPath2: true, + }, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -2257,12 +2309,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { map[string]string{ driveID1: prevDelta, driveID2: prevDelta, - }, - ), + }), graph.NewMetadataEntry( graph.PreviousPathFileName, - test.prevFolderPaths, - ), + test.prevFolderPaths), }, func(*support.ConnectorOperationStatus) {}, ) @@ -2329,18 +2379,24 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { "state: %d, path: %s", baseCol.State(), folderPath) - assert.Equal(t, test.doNotMergeItems, baseCol.DoNotMergeItems(), "DoNotMergeItems") + + p := baseCol.FullPath() + if p == nil { + p = baseCol.PreviousPath() + } + + assert.Equalf( + t, + test.doNotMergeItems[p.String()], + baseCol.DoNotMergeItems(), + "DoNotMergeItems in collection: %s", p) } expectedCollectionCount := 0 - for c := range test.expectedCollections { - for range test.expectedCollections[c] { - expectedCollectionCount++ - } + for _, ec := range test.expectedCollections { + expectedCollectionCount += len(ec) } - // This check is necessary to make sure we are all the - // collections we expect it to assert.Equal(t, expectedCollectionCount, collectionCount, "number of collections") test.expectedDelList.AssertEqual(t, delList) diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 99487c66b..b34f860da 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" ) const ( @@ -55,6 +56,25 @@ func PagerForSource( } } +type pathPrefixerFunc func(driveID string) (path.Path, error) + +func pathPrefixerForSource( + tenantID, resourceOwner string, + source driveSource, +) pathPrefixerFunc { + cat := path.FilesCategory + serv := path.OneDriveService + + if source == SharePointSource { + cat = path.LibrariesCategory + serv = path.SharePointService + } + + return func(driveID string) (path.Path, error) { + return path.Build(tenantID, resourceOwner, serv, cat, false, "drives", driveID, "root:") + } +} + // itemCollector functions collect the items found in a drive type itemCollector func( ctx context.Context, diff --git a/src/pkg/path/drive_test.go b/src/pkg/path/drive_test.go index bdbf09d9c..459548db5 100644 --- a/src/pkg/path/drive_test.go +++ b/src/pkg/path/drive_test.go @@ -32,18 +32,18 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }{ { name: "Not enough path elements", - pathElements: []string{"drive", "driveID"}, + pathElements: []string{"drives", "driveID"}, errCheck: assert.Error, }, { name: "Root path", - pathElements: []string{"drive", "driveID", root}, + pathElements: []string{"drives", "driveID", root}, expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{}}, errCheck: assert.NoError, }, { name: "Deeper path", - pathElements: []string{"drive", "driveID", root, "folder1", "folder2"}, + pathElements: []string{"drives", "driveID", root, "folder1", "folder2"}, expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{"folder1", "folder2"}}, errCheck: assert.NoError, }, diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index c91f27b04..f8fe4297d 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -315,7 +315,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { fileName := "file" fileID := fileName + "-id" shortRef := "short" - elems := []string{"drive", "driveID", "root:", "dir1.d", "dir2.d", fileID} + elems := []string{"drives", "driveID", "root:", "dir1.d", "dir2.d", fileID} filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index b2c9e2344..63ec7e8ec 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -223,7 +223,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( prefixElems = []string{ - "drive", + "drives", "drive!id", "root:", } @@ -415,7 +415,7 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { itemName = "item" itemID = "item-id" shortRef = "short" - driveElems = []string{"drive", "drive!id", "root:.d", "dir1.d", "dir2.d", itemID} + driveElems = []string{"drives", "drive!id", "root:.d", "dir1.d", "dir2.d", itemID} elems = []string{"dir1", "dir2", itemID} ) From 3be3b72d0aa8397a69b9c2ad07e25de881989692 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 10 May 2023 20:54:49 -0700 Subject: [PATCH 21/33] Combine collection structs (#3375) They both implement the same underlying functionality, just in slightly different ways. Combine them so there's less code duplication. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/connector/graph/collections.go | 84 +++++++-------------- 1 file changed, 29 insertions(+), 55 deletions(-) diff --git a/src/internal/connector/graph/collections.go b/src/internal/connector/graph/collections.go index ce93aa6c9..ee941f81c 100644 --- a/src/internal/connector/graph/collections.go +++ b/src/internal/connector/graph/collections.go @@ -11,14 +11,19 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -var _ data.BackupCollection = emptyCollection{} +var _ data.BackupCollection = prefixCollection{} -type emptyCollection struct { - p path.Path - su support.StatusUpdater +// TODO: move this out of graph. /data would be a much better owner +// for a generic struct like this. However, support.StatusUpdater makes +// it difficult to extract from this package in a generic way. +type prefixCollection struct { + full path.Path + prev path.Path + su support.StatusUpdater + state data.CollectionState } -func (c emptyCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { +func (c prefixCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { res := make(chan data.Stream) close(res) @@ -28,21 +33,19 @@ func (c emptyCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.St return res } -func (c emptyCollection) FullPath() path.Path { - return c.p +func (c prefixCollection) FullPath() path.Path { + return c.full } -func (c emptyCollection) PreviousPath() path.Path { - return c.p +func (c prefixCollection) PreviousPath() path.Path { + return c.prev } -func (c emptyCollection) State() data.CollectionState { - // This assumes we won't change the prefix path. Could probably use MovedState - // as well if we do need to change things around. - return data.NotMovedState +func (c prefixCollection) State() data.CollectionState { + return c.state } -func (c emptyCollection) DoNotMergeItems() bool { +func (c prefixCollection) DoNotMergeItems() bool { return false } @@ -76,7 +79,7 @@ func BaseCollections( for cat := range categories { ictx := clues.Add(ctx, "base_service", service, "base_category", cat) - p, err := path.ServicePrefix(tenant, rOwner, service, cat) + full, err := path.ServicePrefix(tenant, rOwner, service, cat) if err != nil { // Shouldn't happen. err = clues.Wrap(err, "making path").WithClues(ictx) @@ -87,8 +90,13 @@ func BaseCollections( } // only add this collection if it doesn't already exist in the set. - if _, ok := collKeys[p.String()]; !ok { - res = append(res, emptyCollection{p: p, su: su}) + if _, ok := collKeys[full.String()]; !ok { + res = append(res, &prefixCollection{ + prev: full, + full: full, + su: su, + state: data.StateOf(full, full), + }) } } @@ -99,45 +107,11 @@ func BaseCollections( // prefix migration // --------------------------------------------------------------------------- -var _ data.BackupCollection = prefixCollection{} - -// TODO: move this out of graph. /data would be a much better owner -// for a generic struct like this. However, support.StatusUpdater makes -// it difficult to extract from this package in a generic way. -type prefixCollection struct { - full, prev path.Path - su support.StatusUpdater - state data.CollectionState -} - -func (c prefixCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { - res := make(chan data.Stream) - close(res) - - s := support.CreateStatus(ctx, support.Backup, 0, support.CollectionMetrics{}, "") - c.su(s) - - return res -} - -func (c prefixCollection) FullPath() path.Path { - return c.full -} - -func (c prefixCollection) PreviousPath() path.Path { - return c.prev -} - -func (c prefixCollection) State() data.CollectionState { - return c.state -} - -func (c prefixCollection) DoNotMergeItems() bool { - return false -} - // Creates a new collection that only handles prefix pathing. -func NewPrefixCollection(prev, full path.Path, su support.StatusUpdater) (*prefixCollection, error) { +func NewPrefixCollection( + prev, full path.Path, + su support.StatusUpdater, +) (*prefixCollection, error) { if prev != nil { if len(prev.Item()) > 0 { return nil, clues.New("prefix collection previous path contains an item") From f2f76d932debf12994ee37fab7f551e81c60004a Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 10 May 2023 22:35:53 -0600 Subject: [PATCH 22/33] release the sensitive-info flag (#3369) Not 100% happy with the flag name, and am open to suggestions. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included #### Type of change - [x] :sunflower: Feature - [x] :world_map: Documentation #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- CHANGELOG.md | 1 + src/pkg/logger/logger.go | 41 ++++++++++++++-------------- src/pkg/logger/logger_test.go | 8 +++--- website/docs/setup/configuration.md | 6 ++++ website/styles/Vocab/Base/accept.txt | 3 +- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39811f5cf..dcfecc3ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) ### Added +- Released the --mask-sensitive-data flag, which will automatically obscure private data in logs. ### Fixed - Graph requests now automatically retry in case of a Bad Gateway or Gateway Timeout. diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index fde379430..cc632b422 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -34,20 +34,20 @@ const ( // flag names const ( - DebugAPIFN = "debug-api-calls" - LogFileFN = "log-file" - LogLevelFN = "log-level" - ReadableLogsFN = "readable-logs" - SensitiveInfoFN = "sensitive-info" + DebugAPIFN = "debug-api-calls" + LogFileFN = "log-file" + LogLevelFN = "log-level" + ReadableLogsFN = "readable-logs" + MaskSensitiveDataFN = "mask-sensitive-data" ) // flag values var ( - DebugAPIFV bool - logFileFV = "" - LogLevelFV = "info" - ReadableLogsFV bool - SensitiveInfoFV = PIIPlainText + DebugAPIFV bool + logFileFV = "" + LogLevelFV = "info" + ReadableLogsFV bool + MaskSensitiveDataFV bool LogFile string // logFileFV after processing ) @@ -83,9 +83,6 @@ func AddLoggingFlags(cmd *cobra.Command) { //nolint:errcheck fs.MarkHidden(ReadableLogsFN) - // TODO(keepers): unhide when we have sufficient/complete coverage of PII handling - //nolint:errcheck - fs.MarkHidden(SensitiveInfoFN) } // internal deduplication for adding flags @@ -106,11 +103,11 @@ func addFlags(fs *pflag.FlagSet, defaultFile string) { false, "minimizes log output for console readability: removes the file and date, colors the level") - fs.StringVar( - &SensitiveInfoFV, - SensitiveInfoFN, - PIIPlainText, - fmt.Sprintf("set the format for sensitive info in logs to %s|%s|%s", PIIHash, PIIMask, PIIPlainText)) + fs.BoolVar( + &MaskSensitiveDataFV, + MaskSensitiveDataFN, + false, + "anonymize personal data in log output") } // Settings records the user's preferred logging settings. @@ -136,7 +133,7 @@ func PreloadLoggingFlags(args []string) Settings { ls := Settings{ File: "", Level: LogLevelFV, - PIIHandling: SensitiveInfoFV, + PIIHandling: PIIPlainText, } // parse the os args list to find the log level flag @@ -144,6 +141,10 @@ func PreloadLoggingFlags(args []string) Settings { return ls } + if MaskSensitiveDataFV { + ls.PIIHandling = PIIHash + } + // retrieve the user's preferred log level // automatically defaults to "info" levelString, err := fs.GetString(LogLevelFN) @@ -165,7 +166,7 @@ func PreloadLoggingFlags(args []string) Settings { // retrieve the user's preferred PII handling algorithm // automatically defaults to default log location - pii, err := fs.GetString(SensitiveInfoFN) + pii, err := fs.GetString(MaskSensitiveDataFN) if err != nil { return ls } diff --git a/src/pkg/logger/logger_test.go b/src/pkg/logger/logger_test.go index 7cb7926fa..644c23aa0 100644 --- a/src/pkg/logger/logger_test.go +++ b/src/pkg/logger/logger_test.go @@ -33,7 +33,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { assert.True(t, logger.DebugAPIFV, logger.DebugAPIFN) assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, logger.LLError, logger.LogLevelFV, logger.LogLevelFN) - assert.Equal(t, logger.PIIMask, logger.SensitiveInfoFV, logger.SensitiveInfoFN) + assert.True(t, logger.MaskSensitiveDataFV, logger.MaskSensitiveDataFN) // empty assertion here, instead of matching "log-file", because the LogFile // var isn't updated by running the command (this is expected and correct), // while the logFileFV remains unexported. @@ -50,7 +50,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, }) err := cmd.Execute() @@ -68,7 +68,7 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, } settings := logger.PreloadLoggingFlags(args) @@ -77,5 +77,5 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, "log-file", settings.File, "settings.File") assert.Equal(t, logger.LLError, settings.Level, "settings.Level") - assert.Equal(t, logger.PIIMask, settings.PIIHandling, "settings.PIIHandling") + assert.Equal(t, logger.PIIHash, settings.PIIHandling, "settings.PIIHandling") } diff --git a/website/docs/setup/configuration.md b/website/docs/setup/configuration.md index d9255f6b7..65c04e99b 100644 --- a/website/docs/setup/configuration.md +++ b/website/docs/setup/configuration.md @@ -132,7 +132,13 @@ directory within the container. Corso generates a unique log file named with its timestamp for every invocation. The default location of Corso's log file is shown below but the location can be overridden by using the `--log-file` flag. The log file will be appended to if multiple Corso invocations are pointed to the same file. + You can also use `stdout` or `stderr` as the `--log-file` location to redirect the logs to "stdout" and "stderr" respectively. +This setting can cause logs to compete with progress bar displays in the terminal. +We suggest using the `--hide-progress` option if you plan to log to stdout or stderr. + +Log entries, by default, include user names and file names. The `--mask-sensitive-data` option can be +used to replace this information with anonymized hashes. diff --git a/website/styles/Vocab/Base/accept.txt b/website/styles/Vocab/Base/accept.txt index 7f8d159c7..b915b5010 100644 --- a/website/styles/Vocab/Base/accept.txt +++ b/website/styles/Vocab/Base/accept.txt @@ -54,4 +54,5 @@ Demetrius Malbrough lockdowns exfiltrate -deduplicating \ No newline at end of file +deduplicating +anonymized From 245d3ee089e974c392e74c2374719effc1988246 Mon Sep 17 00:00:00 2001 From: neha_gupta Date: Thu, 11 May 2023 11:04:13 +0530 Subject: [PATCH 23/33] treat / as root for restore of onedrive and sharepoint (#3328) passing '/' will select anything for backup details and restore for onedrive and sharepoint #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #https://github.com/alcionai/corso/issues/3252 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/utils/flags.go | 6 ++++++ src/cli/utils/onedrive_test.go | 11 +++++++++++ src/cli/utils/sharepoint_test.go | 9 +++++++++ src/cli/utils/testdata/opts.go | 14 ++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/src/cli/utils/flags.go b/src/cli/utils/flags.go index b03fe2e06..3ca50d93e 100644 --- a/src/cli/utils/flags.go +++ b/src/cli/utils/flags.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" ) // common flag vars (eg: FV) @@ -215,6 +216,11 @@ func trimFolderSlash(folders []string) []string { res := make([]string, 0, len(folders)) for _, p := range folders { + if p == string(path.PathSeparator) { + res = selectors.Any() + break + } + // Use path package because it has logic to handle escaping already. res = append(res, path.TrimTrailingSlash(p)) } diff --git a/src/cli/utils/onedrive_test.go b/src/cli/utils/onedrive_test.go index 43c0507c0..61653045f 100644 --- a/src/cli/utils/onedrive_test.go +++ b/src/cli/utils/onedrive_test.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/path" ) type OneDriveUtilsSuite struct { @@ -26,6 +27,7 @@ func (suite *OneDriveUtilsSuite) TestIncludeOneDriveRestoreDataSelectors() { containsOnly = []string{"contains"} prefixOnly = []string{"/prefix"} containsAndPrefix = []string{"contains", "/prefix"} + onlySlash = []string{string(path.PathSeparator)} ) table := []struct { @@ -87,6 +89,15 @@ func (suite *OneDriveUtilsSuite) TestIncludeOneDriveRestoreDataSelectors() { }, expectIncludeLen: 2, }, + { + name: "folder with just /", + opts: utils.OneDriveOpts{ + Users: empty, + FileName: empty, + FolderPath: onlySlash, + }, + expectIncludeLen: 1, + }, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/sharepoint_test.go b/src/cli/utils/sharepoint_test.go index 41bb87e10..0201ab29e 100644 --- a/src/cli/utils/sharepoint_test.go +++ b/src/cli/utils/sharepoint_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -30,6 +31,7 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() { containsOnly = []string{"contains"} prefixOnly = []string{"/prefix"} containsAndPrefix = []string{"contains", "/prefix"} + onlySlash = []string{string(path.PathSeparator)} ) table := []struct { @@ -182,6 +184,13 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() { }, expectIncludeLen: 2, }, + { + name: "folder with just /", + opts: utils.SharePointOpts{ + FolderPath: onlySlash, + }, + expectIncludeLen: 1, + }, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index cde3023fd..8bbb35a58 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -356,6 +356,13 @@ var ( FolderPath: selectors.Any(), }, }, + { + Name: "FilesWithSingleSlash", + Expected: testdata.OneDriveItems, + Opts: utils.OneDriveOpts{ + FolderPath: []string{"/"}, + }, + }, { Name: "FolderPrefixMatch", Expected: testdata.OneDriveItems, @@ -482,6 +489,13 @@ var ( FolderPath: selectors.Any(), }, }, + { + Name: "LibraryItemsWithSingleSlash", + Expected: testdata.SharePointLibraryItems, + Opts: utils.SharePointOpts{ + FolderPath: []string{"/"}, + }, + }, { Name: "FolderPrefixMatch", Expected: testdata.SharePointLibraryItems, From 0202207f3ead9ff450a658fe948dec9524bbe5cd Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 11 May 2023 10:38:13 -0600 Subject: [PATCH 24/33] add nil pointer guard to resp.Headers in middleware (#3393) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Test Plan - [x] :green_heart: E2E --- src/internal/connector/graph/middleware.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index a20b12ade..8730067e7 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -506,6 +506,11 @@ func (mw *MetricsMiddleware) Intercept( // track the graph "resource cost" for each call (if not provided, assume 1) + // nil-pointer guard + if len(resp.Header) == 0 { + resp.Header = http.Header{} + } + // from msoft throttling documentation: // x-ms-resource-unit - Indicates the resource unit used for this request. Values are positive integer xmru := resp.Header.Get(xmruHeader) From ebbf8aef754176adc3ddb034d30864469d062f44 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 11 May 2023 10:17:54 -0700 Subject: [PATCH 25/33] More workflow linting changes (#3380) Pickup a few missed things to get linting running on github actions changes Lint job is re-added in #3391 which will merge after this one does so we can verify it's working --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issues * #3389 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/_filechange_checker.yml | 7 ++-- .github/workflows/actions-lint.yml | 39 ----------------------- 2 files changed, 5 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/actions-lint.yml diff --git a/.github/workflows/_filechange_checker.yml b/.github/workflows/_filechange_checker.yml index 96e03c9d8..92201d961 100644 --- a/.github/workflows/_filechange_checker.yml +++ b/.github/workflows/_filechange_checker.yml @@ -9,6 +9,9 @@ on: websitefileschanged: description: "'true' if websites/** or .github/workflows/** files have changed in the branch" value: ${{ jobs.file-change-check.outputs.websitefileschanged }} + actionsfileschanged: + description: "'true' if .github/actions/** or .github/workflows/** files have changed in the branch" + value: ${{ jobs.file-change-check.outputs.actionsfileschanged }} jobs: file-change-check: @@ -52,9 +55,9 @@ jobs: echo "website or workflow file changes occurred" echo websitefileschanged=true >> $GITHUB_OUTPUT - - name: Check dorny for changes in workflow filepaths + - name: Check dorny for changes in actions filepaths id: actionschecker if: steps.dornycheck.outputs.actions == 'true' run: | - echo "workflow file changes occurred" + echo "actions file changes occurred" echo actionsfileschanged=true >> $GITHUB_OUTPUT diff --git a/.github/workflows/actions-lint.yml b/.github/workflows/actions-lint.yml deleted file mode 100644 index 95629a134..000000000 --- a/.github/workflows/actions-lint.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Lint GitHub actions -on: - workflow_dispatch: - - pull_request: - - push: - branches: [main] - tags: ["v*.*.*"] - -# cancel currently running jobs if a new version of the branch is pushed -concurrency: - group: actions-lint-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # ---------------------------------------------------------------------------------------------------- - # --- Prechecks and Checkouts ------------------------------------------------------------------------ - # ---------------------------------------------------------------------------------------------------- - Precheck: - uses: alcionai/corso/.github/workflows/_filechange_checker.yml@main - - # ---------------------------------------------------------------------------------------------------- - # --- Workflow Action Linting ------------------------------------------------------------------------ - # ---------------------------------------------------------------------------------------------------- - - Actions-Lint: - needs: [Precheck] - environment: Testing - runs-on: ubuntu-latest - if: needs.precheck.outputs.actionsfileschanged == 'true' - steps: - - uses: actions/checkout@v3 - - - name: actionlint - uses: raven-actions/actionlint@v1 - with: - fail-on-error: true - cache: true From aae991686de10b44ec24cdf9b23db154c1b02c54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 May 2023 17:36:02 +0000 Subject: [PATCH 26/33] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.260=20to=201.44.261=20in=20/src=20(#?= =?UTF-8?q?3387)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.260 to 1.44.261.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.261 (2023-05-10)

Service Client Updates

  • service/elasticmapreduce: Updates service API and documentation
    • EMR Studio now supports programmatically executing a Notebooks on an EMR on EKS cluster. In addition, notebooks can now be executed by specifying its location in S3.
  • service/rds: Updates service API, documentation, waiters, paginators, and examples
    • Amazon Relational Database Service (RDS) updates for the new Aurora I/O-Optimized storage type for Amazon Aurora DB clusters
  • service/swf: Updates service API and documentation
    • This release adds a new API parameter to exclude old history events from decision tasks.

SDK Bugs

  • service/sms: Remove deprecated services (SMS) integration tests.
    • SMS integration tests will fail because SMS deprecated their service.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.260&new-version=1.44.261)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 60e25c606..af189d6cf 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.260 + github.com/aws/aws-sdk-go v1.44.261 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index 57680c065..ecbc814cc 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.260 h1:78IJkDpDPXvLXvIkNAKDP/i3z8Vj+3sTAtQYw/v/2o8= -github.com/aws/aws-sdk-go v1.44.260/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.261 h1:PcTMX/QVk+P3yh2n34UzuXDF5FS2z5Lse2bt+r3IpU4= +github.com/aws/aws-sdk-go v1.44.261/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From cf2aa9013adaeac7c0d7299f6d7eaf157efa838d Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 11 May 2023 12:23:14 -0600 Subject: [PATCH 27/33] refine rate limiter: per-query token consumption (#3358) Now that the rate limiter is split by service, we can further split by per-query token consumption. Two primary service cases exist: all exchange queries assume to cost a single token (for now). Drive-service queries are split between permissions (5), default cost (2), and single-item or delta gets (1). --- #### Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR #### Type of change - [x] :sunflower: Feature #### Issue(s) * #2951 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 1 + .../connector/graph/concurrency_limiter.go | 53 ----- .../connector/graph/concurrency_middleware.go | 202 ++++++++++++++++++ ...test.go => concurrency_middleware_test.go} | 0 src/internal/connector/graph/http_wrapper.go | 2 +- src/internal/connector/graph/middleware.go | 96 --------- .../connector/graph/middleware_test.go | 44 ++++ src/internal/connector/graph/service.go | 2 +- src/internal/connector/onedrive/api/drive.go | 5 +- src/internal/connector/onedrive/drive.go | 3 +- src/internal/connector/onedrive/item.go | 6 +- src/internal/connector/onedrive/permission.go | 8 +- src/internal/connector/sharepoint/restore.go | 6 +- 13 files changed, 271 insertions(+), 157 deletions(-) delete mode 100644 src/internal/connector/graph/concurrency_limiter.go create mode 100644 src/internal/connector/graph/concurrency_middleware.go rename src/internal/connector/graph/{concurrency_limiter_test.go => concurrency_middleware_test.go} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcfecc3ec..3d49a12c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Remove exchange item filtering based on m365 item ID via the CLI. - OneDrive backups no longer include a user's non-default drives. - OneDrive and SharePoint file downloads will properly redirect from 3xx responses. +- Refined oneDrive rate limiter controls to reduce throttling errors. ## [v0.7.0] (beta) - 2023-05-02 diff --git a/src/internal/connector/graph/concurrency_limiter.go b/src/internal/connector/graph/concurrency_limiter.go deleted file mode 100644 index 6fe1ea0cd..000000000 --- a/src/internal/connector/graph/concurrency_limiter.go +++ /dev/null @@ -1,53 +0,0 @@ -package graph - -import ( - "net/http" - "sync" - - "github.com/alcionai/clues" - khttp "github.com/microsoft/kiota-http-go" -) - -// concurrencyLimiter middleware limits the number of concurrent requests to graph API -type concurrencyLimiter struct { - semaphore chan struct{} -} - -var ( - once sync.Once - concurrencyLim *concurrencyLimiter - maxConcurrentRequests = 4 -) - -func generateConcurrencyLimiter(capacity int) *concurrencyLimiter { - if capacity < 1 || capacity > maxConcurrentRequests { - capacity = maxConcurrentRequests - } - - return &concurrencyLimiter{ - semaphore: make(chan struct{}, capacity), - } -} - -func InitializeConcurrencyLimiter(capacity int) { - once.Do(func() { - concurrencyLim = generateConcurrencyLimiter(capacity) - }) -} - -func (cl *concurrencyLimiter) Intercept( - pipeline khttp.Pipeline, - middlewareIndex int, - req *http.Request, -) (*http.Response, error) { - if cl == nil || cl.semaphore == nil { - return nil, clues.New("nil concurrency limiter") - } - - cl.semaphore <- struct{}{} - defer func() { - <-cl.semaphore - }() - - return pipeline.Next(req, middlewareIndex) -} diff --git a/src/internal/connector/graph/concurrency_middleware.go b/src/internal/connector/graph/concurrency_middleware.go new file mode 100644 index 000000000..2756a60c6 --- /dev/null +++ b/src/internal/connector/graph/concurrency_middleware.go @@ -0,0 +1,202 @@ +package graph + +import ( + "context" + "net/http" + "sync" + + "github.com/alcionai/clues" + khttp "github.com/microsoft/kiota-http-go" + "golang.org/x/time/rate" + + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// Concurrency Limiter +// "how many calls at one time" +// --------------------------------------------------------------------------- + +// concurrencyLimiter middleware limits the number of concurrent requests to graph API +type concurrencyLimiter struct { + semaphore chan struct{} +} + +var ( + once sync.Once + concurrencyLim *concurrencyLimiter + maxConcurrentRequests = 4 +) + +func generateConcurrencyLimiter(capacity int) *concurrencyLimiter { + if capacity < 1 || capacity > maxConcurrentRequests { + capacity = maxConcurrentRequests + } + + return &concurrencyLimiter{ + semaphore: make(chan struct{}, capacity), + } +} + +func InitializeConcurrencyLimiter(capacity int) { + once.Do(func() { + concurrencyLim = generateConcurrencyLimiter(capacity) + }) +} + +func (cl *concurrencyLimiter) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + if cl == nil || cl.semaphore == nil { + return nil, clues.New("nil concurrency limiter") + } + + cl.semaphore <- struct{}{} + defer func() { + <-cl.semaphore + }() + + return pipeline.Next(req, middlewareIndex) +} + +//nolint:lll +// --------------------------------------------------------------------------- +// Rate Limiter +// "how many calls in a minute" +// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online +// --------------------------------------------------------------------------- + +const ( + // Default goal is to keep calls below the 10k-per-10-minute threshold. + // 14 tokens every second nets 840 per minute. That's 8400 every 10 minutes, + // which is a bit below the mark. + // But suppose we have a minute-long dry spell followed by a 10 minute tsunami. + // We'll have built up 750 tokens in reserve, so the first 750 calls go through + // immediately. Over the next 10 minutes, we'll partition out the other calls + // at a rate of 840-per-minute, ending at a total of 9150. Theoretically, if + // the volume keeps up after that, we'll always stay between 8400 and 9150 out + // of 10k. Worst case scenario, we have an extra minute of padding to allow + // up to 9990. + defaultPerSecond = 14 // 14 * 60 = 840 + defaultMaxCap = 750 // real cap is 10k-per-10-minutes + // since drive runs on a per-minute, rather than per-10-minute bucket, we have + // to keep the max cap equal to the per-second cap. A large maxCap pool (say, + // 1200, similar to the per-minute cap) would allow us to make a flood of 2400 + // calls in the first minute, putting us over the per-minute limit. Keeping + // the cap at the per-second burst means we only dole out a max of 1240 in one + // minute (20 cap + 1200 per minute + one burst of padding). + drivePerSecond = 20 // 20 * 60 = 1200 + driveMaxCap = 20 // real cap is 1250-per-minute +) + +var ( + driveLimiter = rate.NewLimiter(drivePerSecond, driveMaxCap) + // also used as the exchange service limiter + defaultLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) +) + +type LimiterCfg struct { + Service path.ServiceType +} + +type limiterCfgKey string + +const limiterCfgCtxKey limiterCfgKey = "corsoGaphRateLimiterCfg" + +func BindRateLimiterConfig(ctx context.Context, lc LimiterCfg) context.Context { + return context.WithValue(ctx, limiterCfgCtxKey, lc) +} + +func ctxLimiter(ctx context.Context) *rate.Limiter { + lc, ok := extractRateLimiterConfig(ctx) + if !ok { + return defaultLimiter + } + + switch lc.Service { + case path.OneDriveService, path.SharePointService: + return driveLimiter + default: + return defaultLimiter + } +} + +func extractRateLimiterConfig(ctx context.Context) (LimiterCfg, bool) { + l := ctx.Value(limiterCfgCtxKey) + if l == nil { + return LimiterCfg{}, false + } + + lc, ok := l.(LimiterCfg) + + return lc, ok +} + +type limiterConsumptionKey string + +const limiterConsumptionCtxKey limiterConsumptionKey = "corsoGraphRateLimiterConsumption" + +const ( + defaultLC = 1 + driveDefaultLC = 2 + // limit consumption rate for single-item GETs requests, + // or delta-based multi-item GETs. + SingleGetOrDeltaLC = 1 + // limit consumption rate for anything permissions related + PermissionsLC = 5 +) + +// ConsumeNTokens ensures any calls using this context will consume +// n rate-limiter tokens. Default is 1, and this value does not need +// to be established in the context to consume the default tokens. +// This should only get used on a per-call basis, to avoid cross-pollination. +func ConsumeNTokens(ctx context.Context, n int) context.Context { + return context.WithValue(ctx, limiterConsumptionCtxKey, n) +} + +func ctxLimiterConsumption(ctx context.Context, defaultConsumption int) int { + l := ctx.Value(limiterConsumptionCtxKey) + if l == nil { + return defaultConsumption + } + + lc, ok := l.(int) + if !ok || lc < 1 { + return defaultConsumption + } + + return lc +} + +// QueueRequest will allow the request to occur immediately if we're under the +// calls-per-minute rate. Otherwise, the call will wait in a queue until +// the next token set is available. +func QueueRequest(ctx context.Context) { + limiter := ctxLimiter(ctx) + defaultConsumed := defaultLC + + if limiter == driveLimiter { + defaultConsumed = driveDefaultLC + } + + consume := ctxLimiterConsumption(ctx, defaultConsumed) + + if err := limiter.WaitN(ctx, consume); err != nil { + logger.CtxErr(ctx, err).Error("graph middleware waiting on the limiter") + } +} + +// RateLimiterMiddleware is used to ensure we don't overstep per-min request limits. +type RateLimiterMiddleware struct{} + +func (mw *RateLimiterMiddleware) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + QueueRequest(req.Context()) + return pipeline.Next(req, middlewareIndex) +} diff --git a/src/internal/connector/graph/concurrency_limiter_test.go b/src/internal/connector/graph/concurrency_middleware_test.go similarity index 100% rename from src/internal/connector/graph/concurrency_limiter_test.go rename to src/internal/connector/graph/concurrency_middleware_test.go diff --git a/src/internal/connector/graph/http_wrapper.go b/src/internal/connector/graph/http_wrapper.go index 55e9f9556..b0bca76e2 100644 --- a/src/internal/connector/graph/http_wrapper.go +++ b/src/internal/connector/graph/http_wrapper.go @@ -147,7 +147,7 @@ func internalMiddleware(cc *clientConfig) []khttp.Middleware { }, khttp.NewRedirectHandler(), &LoggingMiddleware{}, - &ThrottleControlMiddleware{}, + &RateLimiterMiddleware{}, &MetricsMiddleware{}, } diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index 8730067e7..108f03cac 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -14,12 +14,10 @@ import ( backoff "github.com/cenkalti/backoff/v4" khttp "github.com/microsoft/kiota-http-go" "golang.org/x/exp/slices" - "golang.org/x/time/rate" "github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/path" ) type nexter interface { @@ -381,100 +379,6 @@ func (mw RetryMiddleware) getRetryDelay( return exponentialBackoff.NextBackOff() } -const ( - // Default goal is to keep calls below the 10k-per-10-minute threshold. - // 14 tokens every second nets 840 per minute. That's 8400 every 10 minutes, - // which is a bit below the mark. - // But suppose we have a minute-long dry spell followed by a 10 minute tsunami. - // We'll have built up 750 tokens in reserve, so the first 750 calls go through - // immediately. Over the next 10 minutes, we'll partition out the other calls - // at a rate of 840-per-minute, ending at a total of 9150. Theoretically, if - // the volume keeps up after that, we'll always stay between 8400 and 9150 out - // of 10k. Worst case scenario, we have an extra minute of padding to allow - // up to 9990. - defaultPerSecond = 14 // 14 * 60 = 840 - defaultMaxCap = 750 // real cap is 10k-per-10-minutes - // since drive runs on a per-minute, rather than per-10-minute bucket, we have - // to keep the max cap equal to the per-second cap. A large maxCap pool (say, - // 1200, similar to the per-minute cap) would allow us to make a flood of 2400 - // calls in the first minute, putting us over the per-minute limit. Keeping - // the cap at the per-second burst means we only dole out a max of 1240 in one - // minute (20 cap + 1200 per minute + one burst of padding). - drivePerSecond = 20 // 20 * 60 = 1200 - driveMaxCap = 20 // real cap is 1250-per-minute -) - -var ( - driveLimiter = rate.NewLimiter(drivePerSecond, driveMaxCap) - // also used as the exchange service limiter - defaultLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) -) - -type LimiterCfg struct { - Service path.ServiceType -} - -type limiterCfgKey string - -const limiterCfgCtxKey limiterCfgKey = "corsoGraphRateLimiterCfg" - -func ctxLimiter(ctx context.Context) *rate.Limiter { - lc, ok := extractRateLimiterConfig(ctx) - if !ok { - return defaultLimiter - } - - switch lc.Service { - case path.OneDriveService, path.SharePointService: - return driveLimiter - default: - return defaultLimiter - } -} - -func BindRateLimiterConfig(ctx context.Context, lc LimiterCfg) context.Context { - return context.WithValue(ctx, limiterCfgCtxKey, lc) -} - -func extractRateLimiterConfig(ctx context.Context) (LimiterCfg, bool) { - l := ctx.Value(limiterCfgCtxKey) - if l == nil { - return LimiterCfg{}, false - } - - lc, ok := l.(LimiterCfg) - - return lc, ok -} - -// QueueRequest will allow the request to occur immediately if we're under the -// 1k-calls-per-minute rate. Otherwise, the call will wait in a queue until -// the next token set is available. -func QueueRequest(ctx context.Context) { - limiter := ctxLimiter(ctx) - - if err := limiter.Wait(ctx); err != nil { - logger.CtxErr(ctx, err).Error("graph middleware waiting on the limiter") - } -} - -// --------------------------------------------------------------------------- -// Rate Limiting -// --------------------------------------------------------------------------- - -// ThrottleControlMiddleware is used to ensure we don't overstep 10k-per-10-min -// request limits. -type ThrottleControlMiddleware struct{} - -func (mw *ThrottleControlMiddleware) Intercept( - pipeline khttp.Pipeline, - middlewareIndex int, - req *http.Request, -) (*http.Response, error) { - QueueRequest(req.Context()) - return pipeline.Next(req, middlewareIndex) -} - // --------------------------------------------------------------------------- // Metrics // --------------------------------------------------------------------------- diff --git a/src/internal/connector/graph/middleware_test.go b/src/internal/connector/graph/middleware_test.go index 6ca660231..3aa77778c 100644 --- a/src/internal/connector/graph/middleware_test.go +++ b/src/internal/connector/graph/middleware_test.go @@ -292,3 +292,47 @@ func (suite *MiddlewareUnitSuite) TestBindExtractLimiterConfig() { }) } } + +func (suite *MiddlewareUnitSuite) TestLimiterConsumption() { + ctx, flush := tester.NewContext() + defer flush() + + // an unpopulated ctx should produce the default consumption + assert.Equal(suite.T(), defaultLC, ctxLimiterConsumption(ctx, defaultLC)) + + table := []struct { + name string + n int + expect int + }{ + { + name: "matches default", + n: defaultLC, + expect: defaultLC, + }, + { + name: "default+1", + n: defaultLC + 1, + expect: defaultLC + 1, + }, + { + name: "zero", + n: 0, + expect: defaultLC, + }, + { + name: "negative", + n: -1, + expect: defaultLC, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + tctx := ConsumeNTokens(ctx, test.n) + lc := ctxLimiterConsumption(tctx, defaultLC) + assert.Equal(t, test.expect, lc) + }) + } +} diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index e05838793..dc5129ac4 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -264,7 +264,7 @@ func kiotaMiddlewares( khttp.NewParametersNameDecodingHandler(), khttp.NewUserAgentHandler(), &LoggingMiddleware{}, - &ThrottleControlMiddleware{}, + &RateLimiterMiddleware{}, &MetricsMiddleware{}, }...) diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 8d0b1571f..d87546830 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -373,7 +373,10 @@ func GetDriveRoot( srv graph.Servicer, driveID string, ) (models.DriveItemable, error) { - root, err := srv.Client().DrivesById(driveID).Root().Get(ctx, nil) + root, err := srv.Client(). + DrivesById(driveID). + Root(). + Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting drive root") } diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index b34f860da..c499aac05 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -157,7 +157,8 @@ func collectItems( } for { - page, err := pager.GetPage(ctx) + // assume delta urls here, which allows single-token consumption + page, err := pager.GetPage(graph.ConsumeNTokens(ctx, graph.SingleGetOrDeltaLC)) if graph.IsErrInvalidDelta(err) { logger.Ctx(ctx).Infow("Invalid previous delta link", "link", prevDelta) diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index c7cebc8c1..ac992e90a 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -333,7 +333,11 @@ func driveItemWriter( session := drives.NewItemItemsItemCreateUploadSessionPostRequestBody() ctx = clues.Add(ctx, "upload_item_id", itemID) - r, err := service.Client().DrivesById(driveID).ItemsById(itemID).CreateUploadSession().Post(ctx, session, nil) + r, err := service.Client(). + DrivesById(driveID). + ItemsById(itemID). + CreateUploadSession(). + Post(ctx, session, nil) if err != nil { return nil, graph.Wrap(ctx, err, "creating item upload session") } diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index 7cd4b530d..b67973be0 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -161,7 +161,7 @@ func UpdatePermissions( DrivesById(driveID). ItemsById(itemID). PermissionsById(pid). - Delete(ctx, nil) + Delete(graph.ConsumeNTokens(ctx, graph.PermissionsLC), nil) if err != nil { return graph.Wrap(ctx, err, "removing permissions") } @@ -207,7 +207,11 @@ func UpdatePermissions( pbody.SetRecipients([]models.DriveRecipientable{rec}) - np, err := service.Client().DrivesById(driveID).ItemsById(itemID).Invite().Post(ctx, pbody, nil) + np, err := service.Client(). + DrivesById(driveID). + ItemsById(itemID). + Invite(). + Post(graph.ConsumeNTokens(ctx, graph.PermissionsLC), pbody, nil) if err != nil { return graph.Wrap(ctx, err, "setting permissions") } diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 013f2ef79..642e9fd32 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -172,7 +172,11 @@ func restoreListItem( newList.SetItems(contents) // Restore to List base to M365 back store - restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil) + restoredList, err := service. + Client(). + SitesById(siteID). + Lists(). + Post(ctx, newList, nil) if err != nil { return dii, graph.Wrap(ctx, err, "restoring list") } From 97ca68fba1d57beb9ff85159fe6691bac6d634c9 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 11 May 2023 12:05:15 -0700 Subject: [PATCH 28/33] Re-add GitHub Actions linting (#3391) Move linting into the main CI workflow Split into a different PR so that the file checker gets updated and we can actually see if this is working as intended --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * closes #3389 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++---- .github/workflows/sanity-test.yaml | 6 +++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2771a4b95..5a344bf59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -364,7 +364,7 @@ jobs: # --- Source Code Linting ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------- - Linting: + Source-Code-Linting: needs: [Precheck, Checkout] environment: Testing runs-on: ubuntu-latest @@ -404,12 +404,36 @@ jobs: working-directory: src + # ---------------------------------------------------------------------------------------------------- + # --- GitHub Actions Linting ------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------- + + Actions-Lint: + needs: [Precheck] + environment: Testing + runs-on: ubuntu-latest + if: needs.precheck.outputs.actionsfileschanged == 'true' + steps: + - uses: actions/checkout@v3 + + - name: actionlint + uses: raven-actions/actionlint@v1 + with: + fail-on-error: true + cache: true + # Ignore + # * combining commands into a subshell and using single output + # redirect + # * various variable quoting patterns + # * possible ineffective echo commands + flags: "-ignore SC2129 -ignore SC2086 -ignore SC2046 -ignore 2116" + # ---------------------------------------------------------------------------------------------------- # --- Publish steps ---------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------- Publish-Binary: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' @@ -426,7 +450,7 @@ jobs: rudderstack_data_plane_url: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }} Publish-Image: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') @@ -568,7 +592,7 @@ jobs: ./corso.exe --version 2>&1 | grep -E "version: ${{ env.CORSO_VERSION }}$" Publish-Website-Test: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 4f5020e47..c2dcc4aaa 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -66,7 +66,7 @@ jobs: - name: Version Test run: | set -euo pipefail - if [ $( ./corso --version | grep 'Corso version:' | wc -l) -ne 1 ] + if [ $( ./corso --version | grep -c 'Corso version:' ) -ne 1 ] then echo "valid version not found" exit 1 @@ -78,7 +78,7 @@ jobs: TEST_RESULT: "test_results" run: | set -euo pipefail - prefix=`date +"%Y-%m-%d-%T"` + prefix=$(date +"%Y-%m-%d-%T") echo -e "\nRepo init test\n" >> ${CORSO_LOG_FILE} ./corso repo init s3 \ --no-stats \ @@ -266,7 +266,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | - suffix=`date +"%Y-%m-%d_%H-%M"` + suffix=$(date +"%Y-%m-%d_%H-%M") go run . onedrive files \ --user ${TEST_USER} \ From caea3ab6da06a8f0aa2d8ec279aae9f9d99d1e5b Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 11 May 2023 14:38:33 -0600 Subject: [PATCH 29/33] make the linter happy (#3394) #### Type of change - [x] :broom: Tech Debt/Cleanup --- .../pathtransformer/restore_path_transformer.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/internal/operations/pathtransformer/restore_path_transformer.go b/src/internal/operations/pathtransformer/restore_path_transformer.go index db8b2befd..8993328f3 100644 --- a/src/internal/operations/pathtransformer/restore_path_transformer.go +++ b/src/internal/operations/pathtransformer/restore_path_transformer.go @@ -128,15 +128,16 @@ func makeRestorePathsForEntry( // * Exchange Calendars (different folder handling) // * Exchange Email/Contacts // * OneDrive/SharePoint (needs drive information) - if ent.Exchange != nil { + switch true { + case ent.Exchange != nil: // TODO(ashmrtn): Eventually make Events have it's own function to handle // setting the restore destination properly. res.RestorePath, err = basicLocationPath(repoRef, locRef) - } else if ent.OneDrive != nil || + case ent.OneDrive != nil || (ent.SharePoint != nil && ent.SharePoint.ItemType == details.SharePointLibrary) || - (ent.SharePoint != nil && ent.SharePoint.ItemType == details.OneDriveItem) { + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.OneDriveItem): res.RestorePath, err = drivePathMerge(ent, repoRef, locRef) - } else { + default: return res, clues.New("unknown entry type").WithClues(ctx) } From 6c410c298c2756c424017c0fcbbaf79f821b27b6 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 11 May 2023 16:46:51 -0600 Subject: [PATCH 30/33] consts for drives, root: (#3385) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../graph_connector_onedrive_test.go | 84 +++++++++---------- .../connector/onedrive/consts/consts.go | 10 +++ src/internal/connector/onedrive/drive.go | 3 +- .../operations/backup_integration_test.go | 3 +- src/internal/operations/backup_test.go | 13 +-- src/pkg/backup/details/details_test.go | 15 ++-- src/pkg/path/drive_test.go | 25 ++++-- src/pkg/selectors/onedrive_test.go | 3 +- src/pkg/selectors/sharepoint_test.go | 22 +++-- 9 files changed, 104 insertions(+), 74 deletions(-) create mode 100644 src/internal/connector/onedrive/consts/consts.go diff --git a/src/internal/connector/graph_connector_onedrive_test.go b/src/internal/connector/graph_connector_onedrive_test.go index 0c4c40a47..98ade5372 100644 --- a/src/internal/connector/graph_connector_onedrive_test.go +++ b/src/internal/connector/graph_connector_onedrive_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -109,7 +110,6 @@ var ( folderAName = "folder-a" folderBName = "b" folderNamedFolder = "folder" - rootFolder = "root:" fileAData = []byte(strings.Repeat("a", 33)) fileBData = []byte(strings.Repeat("b", 65)) @@ -255,7 +255,7 @@ func (c *onedriveCollection) withPermissions(perm permData) *onedriveCollection metaName = "" } - if name == rootFolder { + if name == odConsts.RootPathDir { return c } @@ -631,35 +631,35 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( suite.BackupResourceOwner()) rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } subfolderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, } subfolderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, folderAName, } folderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderBName, } @@ -776,34 +776,34 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { folderCName := "folder-c" rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } folderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderBName, } // For skipped test // subfolderAPath := []string{ - // "drives", + // odConsts.DrivesPathDir, // driveID, - // rootFolder, + // odConsts.RootPathDir, // folderBName, // folderAName, // } folderCPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderCName, } @@ -987,9 +987,9 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { inputCols := []onedriveColInfo{ { pathElements: []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, }, files: []itemData{ { @@ -1008,9 +1008,9 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { expectedCols := []onedriveColInfo{ { pathElements: []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, }, files: []itemData{ { @@ -1073,34 +1073,34 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio folderCName := "empty" rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } subfolderAAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderAName, } subfolderABPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, } subfolderACPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderCName, } @@ -1246,20 +1246,20 @@ func testRestoreFolderNamedFolderRegression( suite.BackupResourceOwner()) rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderFolderPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderNamedFolder, } subfolderPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderNamedFolder, folderBName, } diff --git a/src/internal/connector/onedrive/consts/consts.go b/src/internal/connector/onedrive/consts/consts.go new file mode 100644 index 000000000..662354ad6 --- /dev/null +++ b/src/internal/connector/onedrive/consts/consts.go @@ -0,0 +1,10 @@ +package onedrive + +const ( + // const used as the root dir for the drive portion of a path prefix. + // eg: tid/onedrive/ro/files/drives/driveid/... + DrivesPathDir = "drives" + // const used as the root-of-drive dir for the drive portion of a path prefix. + // eg: tid/onedrive/ro/files/drives/driveid/root:/... + RootPathDir = "root:" +) diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index c499aac05..27bf2091c 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -71,7 +72,7 @@ func pathPrefixerForSource( } return func(driveID string) (path.Path, error) { - return path.Build(tenantID, resourceOwner, serv, cat, false, "drives", driveID, "root:") + return path.Build(tenantID, resourceOwner, serv, cat, false, odConsts.DrivesPathDir, driveID, odConsts.RootPathDir) } } diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 0b6283078..310036a64 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -29,6 +29,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" odapi "github.com/alcionai/corso/src/internal/connector/onedrive/api" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -369,7 +370,7 @@ func generateContainerOfItems( switch service { case path.OneDriveService, path.SharePointService: - pathFolders = []string{"drives", driveID, "root:", destFldr} + pathFolders = []string{odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, destFldr} } collections := []incrementalCollection{{ diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 1928dfc66..608f6a20a 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/connector/mock" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/data" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" @@ -657,15 +658,15 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems path.OneDriveService.String(), ro, path.FilesCategory.String(), - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "work", "item1", }, true, ) - locationPath1 = path.Builder{}.Append("root:", "work-display-name") + locationPath1 = path.Builder{}.Append(odConsts.RootPathDir, "work-display-name") itemPath2 = makePath( suite.T(), []string{ @@ -673,15 +674,15 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems path.OneDriveService.String(), ro, path.FilesCategory.String(), - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "personal", "item2", }, true, ) - locationPath2 = path.Builder{}.Append("root:", "personal-display-name") + locationPath2 = path.Builder{}.Append(odConsts.RootPathDir, "personal-display-name") itemPath3 = makePath( suite.T(), []string{ diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 7c6466d3c..d6aae6bbc 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -242,9 +243,9 @@ func oneDriveishEntry(t *testing.T, id string, size int, it ItemType) Entry { "tenant-id", "user-id", []string{ - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "Inbox", "folder1", id, @@ -408,7 +409,7 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_LocationFolders() { { ItemInfo: ItemInfo{ Folder: &FolderInfo{ - DisplayName: "root:", + DisplayName: odConsts.RootPathDir, ItemType: FolderItem, DriveName: "drive-name", DriveID: "drive-id", @@ -416,7 +417,7 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_LocationFolders() { }, }, { - LocationRef: "root:", + LocationRef: odConsts.RootPathDir, ItemInfo: ItemInfo{ Folder: &FolderInfo{ DisplayName: "Inbox", @@ -958,7 +959,7 @@ func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() { "a-user", []string{ "drive-id", - "root:", + odConsts.RootPathDir, "folder", name + "-id", }) @@ -971,7 +972,7 @@ func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() { "a-user", []string{ "drive-id", - "root:", + odConsts.RootPathDir, "folder", name + "-id", name, @@ -1060,7 +1061,7 @@ func (suite *DetailsUnitSuite) TestUpdateItem() { ) newExchangePB := path.Builder{}.Append(folder2) - newOneDrivePB := path.Builder{}.Append("root:", folder2) + newOneDrivePB := path.Builder{}.Append(odConsts.RootPathDir, folder2) table := []struct { name string diff --git a/src/pkg/path/drive_test.go b/src/pkg/path/drive_test.go index 459548db5..5a6853caf 100644 --- a/src/pkg/path/drive_test.go +++ b/src/pkg/path/drive_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) @@ -22,8 +23,6 @@ func TestOneDrivePathSuite(t *testing.T) { } func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { - const root = "root:" - tests := []struct { name string pathElements []string @@ -32,20 +31,28 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }{ { name: "Not enough path elements", - pathElements: []string{"drives", "driveID"}, + pathElements: []string{odConsts.DrivesPathDir, "driveID"}, errCheck: assert.Error, }, { name: "Root path", - pathElements: []string{"drives", "driveID", root}, - expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{}}, - errCheck: assert.NoError, + pathElements: []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir}, + expected: &path.DrivePath{ + DriveID: "driveID", + Root: odConsts.RootPathDir, + Folders: []string{}, + }, + errCheck: assert.NoError, }, { name: "Deeper path", - pathElements: []string{"drives", "driveID", root, "folder1", "folder2"}, - expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{"folder1", "folder2"}}, - errCheck: assert.NoError, + pathElements: []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "folder1", "folder2"}, + expected: &path.DrivePath{ + DriveID: "driveID", + Root: odConsts.RootPathDir, + Folders: []string{"folder1", "folder2"}, + }, + errCheck: assert.NoError, }, } for _, tt := range tests { diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index f8fe4297d..41835875b 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -315,7 +316,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { fileName := "file" fileID := fileName + "-id" shortRef := "short" - elems := []string{"drives", "driveID", "root:", "dir1.d", "dir2.d", fileID} + elems := []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "dir1.d", "dir2.d", fileID} filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 63ec7e8ec..2b8f3edf4 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -223,9 +224,9 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( prefixElems = []string{ - "drives", + odConsts.DrivesPathDir, "drive!id", - "root:", + odConsts.RootPathDir, } itemElems1 = []string{"folderA", "folderB"} itemElems2 = []string{"folderA", "folderC"} @@ -257,7 +258,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { { RepoRef: item, ItemRef: "item", - LocationRef: strings.Join(append([]string{"root:"}, itemElems1...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems1...), "/"), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -268,7 +269,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item2, - LocationRef: strings.Join(append([]string{"root:"}, itemElems2...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems2...), "/"), // ItemRef intentionally blank to test fallback case ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ @@ -281,7 +282,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { { RepoRef: item3, ItemRef: "item3", - LocationRef: strings.Join(append([]string{"root:"}, itemElems3...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems3...), "/"), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -415,8 +416,15 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { itemName = "item" itemID = "item-id" shortRef = "short" - driveElems = []string{"drives", "drive!id", "root:.d", "dir1.d", "dir2.d", itemID} - elems = []string{"dir1", "dir2", itemID} + driveElems = []string{ + odConsts.DrivesPathDir, + "drive!id", + odConsts.RootPathDir + ".d", + "dir1.d", + "dir2.d", + itemID, + } + elems = []string{"dir1", "dir2", itemID} ) table := []struct { From 674d3eec91537cd6cfc8ef2fb28993453efb9348 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 11 May 2023 16:07:38 -0700 Subject: [PATCH 31/33] Switch to folder IDs in Exchange (#3373) Store all folders from Exchange by folder ID in kopia. Remove the old duplicate folder name stop-gap measure as well since it's no longer required Update tests to check LocationPath instead of FullPath since LocationPath still has display name info --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * closes #3197 #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 4 + .../exchange/data_collections_test.go | 24 +- .../connector/exchange/service_functions.go | 10 +- .../connector/exchange/service_iterators.go | 51 --- .../exchange/service_iterators_test.go | 364 +++--------------- .../connector/graph_connector_helper_test.go | 31 +- .../operations/backup_integration_test.go | 6 +- 7 files changed, 114 insertions(+), 376 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d49a12c8..3140e6ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - OneDrive backups no longer include a user's non-default drives. - OneDrive and SharePoint file downloads will properly redirect from 3xx responses. - Refined oneDrive rate limiter controls to reduce throttling errors. +- Fix handling of duplicate folders at the same hierarchy level in Exchange. Duplicate folders will be merged during restore operations. + +### Known Issues +- Restore operations will merge duplicate Exchange folders at the same hierarchy level into a single folder. ## [v0.7.0] (beta) - 2023-05-02 diff --git a/src/internal/connector/exchange/data_collections_test.go b/src/internal/connector/exchange/data_collections_test.go index e2c460cb8..2c23747df 100644 --- a/src/internal/connector/exchange/data_collections_test.go +++ b/src/internal/connector/exchange/data_collections_test.go @@ -282,9 +282,18 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() { } require.NotEmpty(t, c.FullPath().Folder(false)) - folder := c.FullPath().Folder(false) - delete(test.folderNames, folder) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), c) { + continue + } + + loc := c.(data.LocationPather).LocationPath().String() + + require.NotEmpty(t, loc) + + delete(test.folderNames, loc) } assert.Empty(t, test.folderNames) @@ -525,7 +534,16 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression continue } - assert.Equal(t, edc.FullPath().Folder(false), DefaultContactFolder) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), edc) { + continue + } + + assert.Equal( + t, + edc.(data.LocationPather).LocationPath().String(), + DefaultContactFolder) assert.NotZero(t, count) } diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index cad25cdd8..52d46ba42 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -137,21 +137,15 @@ func includeContainer( directory = locPath.Folder(false) } - var ( - ok bool - pathRes path.Path - ) + var ok bool switch category { case path.EmailCategory: ok = scope.Matches(selectors.ExchangeMailFolder, directory) - pathRes = locPath case path.ContactsCategory: ok = scope.Matches(selectors.ExchangeContactFolder, directory) - pathRes = locPath case path.EventsCategory: ok = scope.Matches(selectors.ExchangeEventCalendar, directory) - pathRes = dirPath default: return nil, nil, false } @@ -162,5 +156,5 @@ func includeContainer( "matches_input", directory, ).Debug("backup folder selection filter") - return pathRes, loc, ok + return dirPath, loc, ok } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 34ea37d3f..9f707df21 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -56,10 +56,6 @@ func filterContainersAndFillCollections( // deleted from this map, leaving only the deleted folders behind tombstones = makeTombstones(dps) category = qp.Category - - // Stop-gap: Track folders by LocationPath and if there's duplicates pick - // the one with the lexicographically larger ID. - dupPaths = map[string]string{} ) logger.Ctx(ctx).Infow("filling collections", "len_deltapaths", len(dps)) @@ -108,53 +104,6 @@ func filterContainersAndFillCollections( continue } - // This is a duplicate collection. Either the collection we're examining now - // should be skipped or the collection we previously added should be - // skipped. - // - // Calendars is already using folder IDs so we don't need to pick the - // "newest" folder for that. - if oldCID := dupPaths[locPath.String()]; category != path.EventsCategory && len(oldCID) > 0 { - if cID < oldCID { - logger.Ctx(ictx).Infow( - "skipping duplicate folder with lesser ID", - "previous_folder_id", clues.Hide(oldCID), - "current_folder_id", clues.Hide(cID), - "duplicate_path", locPath) - - // Readd this entry to the tombstone map because we remove it first off. - if oldDP, ok := dps[cID]; ok { - tombstones[cID] = oldDP.path - } - - // Continuing here ensures we don't add anything to the paths map or the - // delta map which is the behavior we want. - continue - } - - logger.Ctx(ictx).Infow( - "switching duplicate folders as newer folder found", - "previous_folder_id", clues.Hide(oldCID), - "current_folder_id", clues.Hide(cID), - "duplicate_path", locPath) - - // Remove the previous collection from the maps. This will make us think - // it's a new item and properly populate it if it ever: - // * moves - // * replaces the current entry (current entry moves/is deleted) - delete(collections, oldCID) - delete(deltaURLs, oldCID) - delete(currPaths, oldCID) - - // Re-add the tombstone entry for the old folder so that it can be marked - // as deleted if need. - if oldDP, ok := dps[oldCID]; ok { - tombstones[oldCID] = oldDP.path - } - } - - dupPaths[locPath.String()] = cID - if len(prevPathStr) > 0 { if prevPath, err = pathFromPrevString(prevPathStr); err != nil { logger.CtxErr(ictx, err).Error("parsing prev path") diff --git a/src/internal/connector/exchange/service_iterators_test.go b/src/internal/connector/exchange/service_iterators_test.go index d7a355122..5b4d11940 100644 --- a/src/internal/connector/exchange/service_iterators_test.go +++ b/src/internal/connector/exchange/service_iterators_test.go @@ -384,6 +384,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli ResourceOwner: inMock.NewProvider("user_id", "user_name"), Credentials: suite.creds, } + statusUpdater = func(*support.ConnectorOperationStatus) {} dataTypes = []scopeCat{ @@ -395,6 +396,10 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli scope: selectors.NewExchangeBackup(nil).ContactFolders(selectors.Any())[0], cat: path.ContactsCategory, }, + { + scope: selectors.NewExchangeBackup(nil).EventCalendars(selectors.Any())[0], + cat: path.EventsCategory, + }, } location = path.Builder{}.Append("foo", "bar") @@ -448,8 +453,20 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli return res } - locPath := func(t *testing.T, cat path.CategoryType) path.Path { - res, err := location.ToDataLayerPath( + idPath1 := func(t *testing.T, cat path.CategoryType) path.Path { + res, err := path.Builder{}.Append("1").ToDataLayerPath( + suite.creds.AzureTenantID, + qp.ResourceOwner.ID(), + path.ExchangeService, + cat, + false) + require.NoError(t, err, clues.ToCore(err)) + + return res + } + + idPath2 := func(t *testing.T, cat path.CategoryType) path.Path { + res, err := path.Builder{}.Append("2").ToDataLayerPath( suite.creds.AzureTenantID, qp.ResourceOwner.ID(), path.ExchangeService, @@ -467,8 +484,6 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli inputMetadata func(t *testing.T, cat path.CategoryType) DeltaPaths expectNewColls int expectDeleted int - expectAdded []string - expectRemoved []string expectMetadata func(t *testing.T, cat path.CategoryType) DeltaPaths }{ { @@ -486,49 +501,19 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli }, "2": DeltaPath{ delta: "old_delta", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { - return DeltaPaths{ - "2": DeltaPath{ - delta: "delta_url2", - path: locPath(t, cat).String(), - }, - } - }, - }, - { - name: "1 moved to duplicate, other order", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container2, container1), - inputMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ "1": DeltaPath{ - delta: "old_delta", - path: oldPath1(t, cat).String(), + delta: "delta_url", + path: idPath1(t, cat).String(), }, - "2": DeltaPath{ - delta: "old_delta", - path: locPath(t, cat).String(), - }, - } - }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, - expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { - return DeltaPaths{ "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -552,14 +537,15 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli }, } }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ + "1": DeltaPath{ + delta: "delta_url", + path: idPath1(t, cat).String(), + }, "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -574,14 +560,16 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli inputMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{} }, - expectNewColls: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, + expectNewColls: 2, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ + "1": DeltaPath{ + delta: "delta_url", + path: idPath1(t, cat).String(), + }, "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -596,19 +584,17 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli return DeltaPaths{ "2": DeltaPath{ delta: "old_delta", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, expectNewColls: 1, expectDeleted: 1, - expectAdded: result1.added, - expectRemoved: result1.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ "1": DeltaPath{ delta: "delta_url", - path: locPath(t, cat).String(), + path: idPath1(t, cat).String(), }, } }, @@ -633,7 +619,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli statusUpdater, test.resolver, sc.scope, - test.inputMetadata(t, sc.cat), + test.inputMetadata(t, qp.Category), control.Options{FailureHandling: control.FailFast}, fault.New(true)) require.NoError(t, err, "getting collections", clues.ToCore(err)) @@ -649,21 +635,30 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli if c.FullPath().Service() == path.ExchangeMetadataService { metadatas++ - checkMetadata(t, ctx, sc.cat, test.expectMetadata(t, sc.cat), c) + checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c) continue } if c.State() == data.NewState { news++ } + } - exColl, ok := c.(*Collection) - require.True(t, ok, "collection is an *exchange.Collection") + assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") + assert.Equal(t, test.expectNewColls, news, "new collections") + assert.Equal(t, 1, metadatas, "metadata collections") - if exColl.LocationPath() != nil { - assert.Equal(t, location.String(), exColl.LocationPath().String()) + // items in collections assertions + for k, expect := range test.getter { + coll := collections[k] + + if coll == nil { + continue } + exColl, ok := coll.(*Collection) + require.True(t, ok, "collection is an *exchange.Collection") + ids := [][]string{ make([]string, 0, len(exColl.added)), make([]string, 0, len(exColl.removed)), @@ -675,268 +670,15 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli } } - assert.ElementsMatch(t, test.expectAdded, ids[0], "added items") - assert.ElementsMatch(t, test.expectRemoved, ids[1], "removed items") + assert.ElementsMatch(t, expect.added, ids[0], "added items") + assert.ElementsMatch(t, expect.removed, ids[1], "removed items") } - - assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") - assert.Equal(t, test.expectNewColls, news, "new collections") - assert.Equal(t, 1, metadatas, "metadata collections") }) } }) } } -func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_DuplicateFolders_Events() { - var ( - qp = graph.QueryParams{ - ResourceOwner: inMock.NewProvider("user_id", "user_name"), - Category: path.EventsCategory, - Credentials: suite.creds, - } - statusUpdater = func(*support.ConnectorOperationStatus) {} - - scope = selectors.NewExchangeBackup(nil).EventCalendars(selectors.Any())[0] - - location = path.Builder{}.Append("foo", "bar") - - result1 = mockGetterResults{ - added: []string{"a1", "a2", "a3"}, - removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, - } - result2 = mockGetterResults{ - added: []string{"a4", "a5", "a6"}, - removed: []string{"r4", "r5", "r6"}, - newDelta: api.DeltaUpdate{URL: "delta_url2"}, - } - - container1 = mockContainer{ - id: strPtr("1"), - displayName: strPtr("bar"), - p: path.Builder{}.Append("1"), - l: location, - } - container2 = mockContainer{ - id: strPtr("2"), - displayName: strPtr("bar"), - p: path.Builder{}.Append("2"), - l: location, - } - ) - - oldPath1, err := location.Append("1").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - oldPath2, err := location.Append("2").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - idPath1, err := path.Builder{}.Append("1").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - idPath2, err := path.Builder{}.Append("2").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - table := []struct { - name string - getter mockGetter - resolver graph.ContainerResolver - inputMetadata DeltaPaths - expectNewColls int - expectDeleted int - expectMetadata DeltaPaths - }{ - { - name: "1 moved to duplicate", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "old_delta", - path: oldPath1.String(), - }, - "2": DeltaPath{ - delta: "old_delta", - path: idPath2.String(), - }, - }, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "both move to duplicate", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "old_delta", - path: oldPath1.String(), - }, - "2": DeltaPath{ - delta: "old_delta", - path: oldPath2.String(), - }, - }, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "both new", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{}, - expectNewColls: 2, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "add 1 remove 2", - getter: map[string]mockGetterResults{ - "1": result1, - }, - resolver: newMockResolver(container1), - inputMetadata: DeltaPaths{ - "2": DeltaPath{ - delta: "old_delta", - path: idPath2.String(), - }, - }, - expectNewColls: 1, - expectDeleted: 1, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - }, - }, - } - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() - - ctx, flush := tester.NewContext() - defer flush() - - collections, err := filterContainersAndFillCollections( - ctx, - qp, - test.getter, - statusUpdater, - test.resolver, - scope, - test.inputMetadata, - control.Options{FailureHandling: control.FailFast}, - fault.New(true)) - require.NoError(t, err, "getting collections", clues.ToCore(err)) - - // collection assertions - - deleteds, news, metadatas := 0, 0, 0 - for _, c := range collections { - if c.State() == data.DeletedState { - deleteds++ - continue - } - - if c.FullPath().Service() == path.ExchangeMetadataService { - metadatas++ - checkMetadata(t, ctx, qp.Category, test.expectMetadata, c) - continue - } - - if c.State() == data.NewState { - news++ - } - } - - assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") - assert.Equal(t, test.expectNewColls, news, "new collections") - assert.Equal(t, 1, metadatas, "metadata collections") - - // items in collections assertions - for k, expect := range test.getter { - coll := collections[k] - - if coll == nil { - continue - } - - exColl, ok := coll.(*Collection) - require.True(t, ok, "collection is an *exchange.Collection") - - ids := [][]string{ - make([]string, 0, len(exColl.added)), - make([]string, 0, len(exColl.removed)), - } - - for i, cIDs := range []map[string]struct{}{exColl.added, exColl.removed} { - for id := range cIDs { - ids[i] = append(ids[i], id) - } - } - - assert.ElementsMatch(t, expect.added, ids[0], "added items") - assert.ElementsMatch(t, expect.removed, ids[1], "removed items") - } - }) - } -} - func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repeatedItems() { newDelta := api.DeltaUpdate{URL: "delta_url"} diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 99043e5bc..3aa309f04 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -918,7 +918,36 @@ func checkHasCollections( } for _, g := range got { - gotNames = append(gotNames, g.FullPath().String()) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), g) { + continue + } + + fp := g.FullPath() + loc := g.(data.LocationPather).LocationPath() + + if fp.Service() == path.OneDriveService || + (fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) { + dp, err := path.ToDrivePath(fp) + if !assert.NoError(t, err, clues.ToCore(err)) { + continue + } + + loc = path.BuildDriveLocation(dp.DriveID, loc.Elements()...) + } + + p, err := loc.ToDataLayerPath( + fp.Tenant(), + fp.ResourceOwner(), + fp.Service(), + fp.Category(), + false) + if !assert.NoError(t, err, clues.ToCore(err)) { + continue + } + + gotNames = append(gotNames, p.String()) } assert.ElementsMatch(t, expectedNames, gotNames, "returned collections") diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 310036a64..ddc59e6ce 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -1111,8 +1111,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalExchange() { } } }, - itemsRead: 0, // containers are not counted as reads - itemsWritten: 4, // two items per category + itemsRead: 0, // containers are not counted as reads + // Renaming a folder doesn't cause kopia changes as the folder ID doesn't + // change. + itemsWritten: 0, }, { name: "add a new item", From 1b417af5bb1e6f250adb7e21216a731e9decb909 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 11 May 2023 19:29:17 -0600 Subject: [PATCH 32/33] minor updates to the path.Path interface (#3384) Swaps path.Append from a single item method to accept a variadic list of string elements. Since 95% of all calls to path.Append were items, also adds a shorthand AppendItem func to the interface for easy clarity. Finally, adds a Last() method to elements for getting the last element in the slice. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- .../connector/exchange/service_restore.go | 2 +- .../connector/onedrive/collections.go | 2 +- src/internal/connector/onedrive/restore.go | 8 ++++---- src/internal/connector/sharepoint/restore.go | 4 ++-- src/internal/kopia/upload.go | 6 +++--- src/internal/kopia/wrapper_test.go | 14 ++++++------- src/internal/operations/manifests_test.go | 12 +++++------ src/pkg/backup/details/testdata/testdata.go | 2 +- src/pkg/path/elements.go | 9 +++++++++ src/pkg/path/path.go | 4 +++- src/pkg/path/path_test.go | 20 +++++++++++++++++++ src/pkg/path/resource_path.go | 8 ++++++-- src/pkg/path/resource_path_test.go | 2 +- 13 files changed, 64 insertions(+), 29 deletions(-) diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 8ac120619..9e293ce5d 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -435,7 +435,7 @@ func restoreCollection( metrics.Bytes += int64(len(byteArray)) metrics.Successes++ - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { errs.AddRecoverable(clues.Wrap(err, "building full path with item").WithClues(ctx)) continue diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 4cb1944f7..52f29f879 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -677,7 +677,7 @@ func (c *Collections) getCollectionPath( return nil, clues.New("folder with empty name") } - collectionPath, err = collectionPath.Append(name, false) + collectionPath, err = collectionPath.Append(false, name) if err != nil { return nil, clues.Wrap(err, "making non-root folder path") } diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 3f34cc9c4..41d037b13 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -231,7 +231,7 @@ func RestoreCollection( return metrics, nil } - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue @@ -852,7 +852,7 @@ func AugmentRestorePaths( el := p.StoragePath.Elements() if backupVersion >= version.OneDrive6NameInMeta { - mPath, err := p.StoragePath.Append(".dirmeta", true) + mPath, err := p.StoragePath.AppendItem(".dirmeta") if err != nil { return nil, err } @@ -861,7 +861,7 @@ func AugmentRestorePaths( paths, path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) } else if backupVersion >= version.OneDrive4DirIncludesPermissions { - mPath, err := p.StoragePath.Append(el[len(el)-1]+".dirmeta", true) + mPath, err := p.StoragePath.AppendItem(el.Last() + ".dirmeta") if err != nil { return nil, err } @@ -875,7 +875,7 @@ func AugmentRestorePaths( return nil, err } - mPath, err := pp.Append(el[len(el)-1]+".dirmeta", true) + mPath, err := pp.AppendItem(el.Last() + ".dirmeta") if err != nil { return nil, err } diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 642e9fd32..2f64454da 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -251,7 +251,7 @@ func RestoreListCollection( metrics.Bytes += itemInfo.SharePoint.Size - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue @@ -339,7 +339,7 @@ func RestorePageCollection( metrics.Bytes += itemInfo.SharePoint.Size - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 6f7f5388c..a1cc0bed2 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -347,7 +347,7 @@ func collectionEntries( seen[encodedName] = struct{}{} // For now assuming that item IDs don't need escaping. - itemPath, err := streamedEnts.FullPath().Append(e.UUID(), true) + itemPath, err := streamedEnts.FullPath().AppendItem(e.UUID()) if err != nil { err = clues.Wrap(err, "getting full item path") progress.errs.AddRecoverable(err) @@ -464,7 +464,7 @@ func streamBaseEntries( } // For now assuming that item IDs don't need escaping. - itemPath, err := curPath.Append(entName, true) + itemPath, err := curPath.AppendItem(entName) if err != nil { return clues.Wrap(err, "getting full item path for base entry") } @@ -473,7 +473,7 @@ func streamBaseEntries( // backup details. If the item moved and we had only the new path, we'd be // unable to find it in the old backup details because we wouldn't know what // to look for. - prevItemPath, err := prevPath.Append(entName, true) + prevItemPath, err := prevPath.AppendItem(entName) if err != nil { return clues.Wrap(err, "getting previous full item path for base entry") } diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index abe96fdc2..48041cd91 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -73,7 +73,7 @@ func testForFiles( for s := range c.Items(ctx, fault.New(true)) { count++ - fullPath, err := c.FullPath().Append(s.UUID(), true) + fullPath, err := c.FullPath().AppendItem(s.UUID()) require.NoError(t, err, clues.ToCore(err)) expected, ok := expected[fullPath.String()] @@ -689,10 +689,10 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1) dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1) - fp1, err := suite.storePath1.Append(dc1.Names[0], true) + fp1, err := suite.storePath1.AppendItem(dc1.Names[0]) require.NoError(t, err, clues.ToCore(err)) - fp2, err := suite.storePath2.Append(dc2.Names[0], true) + fp2, err := suite.storePath2.AppendItem(dc2.Names[0]) require.NoError(t, err, clues.ToCore(err)) stats, _, _, err := w.ConsumeBackupCollections( @@ -838,7 +838,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { // 5 file and 2 folder entries. assert.Len(t, deets.Details().Entries, 5+2) - failedPath, err := suite.storePath2.Append(testFileName4, true) + failedPath, err := suite.storePath2.AppendItem(testFileName4) require.NoError(t, err, clues.ToCore(err)) ic := i64counter{} @@ -987,7 +987,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { } for _, item := range filesInfo { - pth, err := item.parentPath.Append(item.name, true) + pth, err := item.parentPath.AppendItem(item.name) require.NoError(suite.T(), err, clues.ToCore(err)) mapKey := item.parentPath.String() @@ -1439,7 +1439,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Path item, ok := suite.filesByPath[pth.StoragePath.String()] require.True(t, ok, "getting expected file data") - itemPath, err := pth.RestorePath.Append(pth.StoragePath.Item(), true) + itemPath, err := pth.RestorePath.AppendItem(pth.StoragePath.Item()) require.NoError(t, err, "getting expected item path") expected[itemPath.String()] = item.data @@ -1532,7 +1532,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Fetc } func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Errors() { - itemPath, err := suite.testPath1.Append(testFileName, true) + itemPath, err := suite.testPath1.AppendItem(testFileName) require.NoError(suite.T(), err, clues.ToCore(err)) table := []struct { diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index aa481ade7..ccef6e248 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -140,7 +140,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -163,7 +163,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -191,10 +191,10 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) - p, err = contactPath.Append(f, true) + p, err = contactPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -222,10 +222,10 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) - p, err = contactPath.Append(f, true) + p, err = contactPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } diff --git a/src/pkg/backup/details/testdata/testdata.go b/src/pkg/backup/details/testdata/testdata.go index a406d838a..0d98ec7df 100644 --- a/src/pkg/backup/details/testdata/testdata.go +++ b/src/pkg/backup/details/testdata/testdata.go @@ -25,7 +25,7 @@ func mustParsePath(ref string, isItem bool) path.Path { // path with the element appended to it. Panics if the path cannot be parsed. // Useful for simple variable assignments. func mustAppendPath(p path.Path, newElement string, isItem bool) path.Path { - newP, err := p.Append(newElement, isItem) + newP, err := p.Append(isItem, newElement) if err != nil { panic(err) } diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index 0a55bd8e4..a77ea3345 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -86,3 +86,12 @@ func (el Elements) String() string { func (el Elements) PlainString() string { return join(el) } + +// Last returns the last element. Returns "" if empty. +func (el Elements) Last() string { + if len(el) == 0 { + return "" + } + + return el[len(el)-1] +} diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 33fae1763..189e24449 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -106,7 +106,9 @@ type Path interface { // Append returns a new Path object with the given element added to the end of // the old Path if possible. If the old Path is an item Path then Append // returns an error. - Append(element string, isItem bool) (Path, error) + Append(isItem bool, elems ...string) (Path, error) + // AppendItem is a shorthand for Append(true, someItem) + AppendItem(item string) (Path, error) // ShortRef returns a short reference representing this path. The short // reference is guaranteed to be unique. No guarantees are made about whether // a short reference can be converted back into the Path that generated it. diff --git a/src/pkg/path/path_test.go b/src/pkg/path/path_test.go index 21631f7bf..be43d3732 100644 --- a/src/pkg/path/path_test.go +++ b/src/pkg/path/path_test.go @@ -245,6 +245,26 @@ func (suite *PathUnitSuite) TestAppend() { } } +func (suite *PathUnitSuite) TestAppendItem() { + t := suite.T() + + p, err := Build("t", "ro", ExchangeService, EmailCategory, false, "foo", "bar") + require.NoError(t, err, clues.ToCore(err)) + + pb := p.ToBuilder() + assert.Equal(t, pb.String(), p.String()) + + pb = pb.Append("qux") + + p, err = p.AppendItem("qux") + + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, pb.String(), p.String()) + + _, err = p.AppendItem("fnords") + require.Error(t, err, clues.ToCore(err)) +} + func (suite *PathUnitSuite) TestUnescapeAndAppend() { table := append(append([]testData{}, genericCases...), basicEscapedInputs...) for _, test := range table { diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 47d481a46..923d66453 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -253,21 +253,25 @@ func (rp dataLayerResourcePath) Dir() (Path, error) { } func (rp dataLayerResourcePath) Append( - element string, isItem bool, + elems ...string, ) (Path, error) { if rp.hasItem { return nil, clues.New("appending to an item path") } return &dataLayerResourcePath{ - Builder: *rp.Builder.Append(element), + Builder: *rp.Builder.Append(elems...), service: rp.service, category: rp.category, hasItem: isItem, }, nil } +func (rp dataLayerResourcePath) AppendItem(item string) (Path, error) { + return rp.Append(true, item) +} + func (rp dataLayerResourcePath) ToBuilder() *Builder { // Safe to directly return the Builder because Builders are immutable. return &rp.Builder diff --git a/src/pkg/path/resource_path_test.go b/src/pkg/path/resource_path_test.go index 3453737e6..e49f797e2 100644 --- a/src/pkg/path/resource_path_test.go +++ b/src/pkg/path/resource_path_test.go @@ -547,7 +547,7 @@ func (suite *PopulatedDataLayerResourcePath) TestAppend() { suite.Run(test.name, func() { t := suite.T() - newPath, err := suite.paths[m.isItem].Append(newElement, test.hasItem) + newPath, err := suite.paths[m.isItem].Append(test.hasItem, newElement) // Items don't allow appending. if m.isItem { From 4274de2b7315f45f47a211834b2d624848915c60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 10:12:22 +0000 Subject: [PATCH 33/33] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aw?= =?UTF-8?q?s/aws-sdk-go=20from=201.44.261=20to=201.44.262=20in=20/src=20(#?= =?UTF-8?q?3404)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.261 to 1.44.262.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.262 (2023-05-11)

Service Client Updates

  • service/connect: Updates service documentation
  • service/elasticache: Updates service API and documentation
    • Added support to modify the cluster mode configuration for the existing ElastiCache ReplicationGroups. Customers can now modify the configuration from cluster mode disabled to cluster mode enabled.
  • service/es: Updates service API and documentation
    • This release fixes DescribePackages API error with null filter value parameter.
  • service/health: Updates service documentation
    • Add support for regional endpoints
  • service/ivs-realtime: Updates service API, documentation, and paginators
  • service/omics: Updates service API, documentation, and paginators
  • service/opensearch: Updates service API
  • service/route53resolver: Adds new service
  • service/support: Updates service API and documentation
    • This release adds 2 new Support APIs, DescribeCreateCaseOptions and DescribeSupportedLanguages. You can use these new APIs to get available support languages.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.261&new-version=1.44.262)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index af189d6cf..a90058680 100644 --- a/src/go.mod +++ b/src/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.261 + github.com/aws/aws-sdk-go v1.44.262 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 diff --git a/src/go.sum b/src/go.sum index ecbc814cc..f8a86e102 100644 --- a/src/go.sum +++ b/src/go.sum @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.261 h1:PcTMX/QVk+P3yh2n34UzuXDF5FS2z5Lse2bt+r3IpU4= -github.com/aws/aws-sdk-go v1.44.261/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I= +github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=