diff --git a/.github/workflows/_filechange_checker.yml b/.github/workflows/_filechange_checker.yml index 8d02d1437..92201d961 100644 --- a/.github/workflows/_filechange_checker.yml +++ b/.github/workflows/_filechange_checker.yml @@ -9,6 +9,9 @@ on: websitefileschanged: description: "'true' if websites/** or .github/workflows/** files have changed in the branch" value: ${{ jobs.file-change-check.outputs.websitefileschanged }} + actionsfileschanged: + description: "'true' if .github/actions/** or .github/workflows/** files have changed in the branch" + value: ${{ jobs.file-change-check.outputs.actionsfileschanged }} jobs: file-change-check: @@ -19,6 +22,7 @@ jobs: outputs: srcfileschanged: ${{ steps.srcchecker.outputs.srcfileschanged }} websitefileschanged: ${{ steps.websitechecker.outputs.websitefileschanged }} + actionsfileschanged: ${{ steps.actionschecker.outputs.actionsfileschanged }} steps: - uses: actions/checkout@v3 @@ -49,4 +53,11 @@ jobs: if: steps.dornycheck.outputs.src == 'true' || steps.dornycheck.outputs.website == 'true' || steps.dornycheck.outputs.actions == 'true' run: | echo "website or workflow file changes occurred" - echo websitefileschanged=true >> $GITHUB_OUTPUT \ No newline at end of file + echo websitefileschanged=true >> $GITHUB_OUTPUT + + - name: Check dorny for changes in actions filepaths + id: actionschecker + if: steps.dornycheck.outputs.actions == 'true' + run: | + echo "actions file changes occurred" + echo actionsfileschanged=true >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2771a4b95..5a344bf59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -364,7 +364,7 @@ jobs: # --- Source Code Linting ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------- - Linting: + Source-Code-Linting: needs: [Precheck, Checkout] environment: Testing runs-on: ubuntu-latest @@ -404,12 +404,36 @@ jobs: working-directory: src + # ---------------------------------------------------------------------------------------------------- + # --- GitHub Actions Linting ------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------- + + Actions-Lint: + needs: [Precheck] + environment: Testing + runs-on: ubuntu-latest + if: needs.precheck.outputs.actionsfileschanged == 'true' + steps: + - uses: actions/checkout@v3 + + - name: actionlint + uses: raven-actions/actionlint@v1 + with: + fail-on-error: true + cache: true + # Ignore + # * combining commands into a subshell and using single output + # redirect + # * various variable quoting patterns + # * possible ineffective echo commands + flags: "-ignore SC2129 -ignore SC2086 -ignore SC2046 -ignore 2116" + # ---------------------------------------------------------------------------------------------------- # --- Publish steps ---------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------- Publish-Binary: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' @@ -426,7 +450,7 @@ jobs: rudderstack_data_plane_url: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }} Publish-Image: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') @@ -568,7 +592,7 @@ jobs: ./corso.exe --version 2>&1 | grep -E "version: ${{ env.CORSO_VERSION }}$" Publish-Website-Test: - needs: [Test-Suite-Trusted, Linting, Website-Linting, SetEnv] + needs: [Test-Suite-Trusted, Source-Code-Linting, Website-Linting, SetEnv] environment: ${{ needs.SetEnv.outputs.environment }} runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 35191afdc..65e678e4b 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -1,5 +1,6 @@ name: CI Test Cleanup on: + workflow_dispatch: schedule: # every half hour - cron: "*/30 * * * *" diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index 5cc1e3c05..9241b3d8f 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -1,10 +1,8 @@ name: Nightly Load Testing on: schedule: - # every day at 01:59 (01:59am) UTC - # - cron: "59 1 * * *" - # temp, for testing: every 4 hours - - cron: "0 */4 * * *" + # every day at 03:59 GMT (roughly 8pm PST) + - cron: "59 3 * * *" permissions: # required to retrieve AWS credentials @@ -20,6 +18,10 @@ jobs: Load-Tests: environment: Load Testing runs-on: ubuntu-latest + # Skipping load testing for now. They need some love to get up and + # running properly, and it's better to not fight for resources with + # tests that are guaranteed to fail. + if: false defaults: run: working-directory: src diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index ccc93fdce..22eddba52 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -3,8 +3,8 @@ on: workflow_dispatch: schedule: - # Run every day at 0 minutes and 0 hours (midnight GMT) - - cron: "0 0 * * *" + # Run every day at 04:00 GMT (roughly 8pm PST) + - cron: "0 4 * * *" permissions: # required to retrieve AWS credentials @@ -50,7 +50,6 @@ jobs: environment: ${{ steps.environment.outputs.environment }} version: ${{ steps.version.outputs.version }} website-bucket: ${{ steps.website-bucket.outputs.website-bucket }} - website-cfid: ${{ steps.website-cfid.outputs.website-cfid }} steps: - uses: actions/checkout@v3 @@ -122,6 +121,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} CORSO_NIGHTLY_TESTS: true + CORSO_E2E_TESTS: true CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 20f830b82..c2dcc4aaa 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -36,6 +36,7 @@ jobs: CORSO_LOG_DIR: testlog CORSO_LOG_FILE: testlog/testlogging.log TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }} + TEST_SITE: ${{ secrets.CORSO_M365_TEST_SITE_URL }} SECONDARY_TEST_USER : ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} TEST_RESULT: test_results @@ -65,7 +66,7 @@ jobs: - name: Version Test run: | set -euo pipefail - if [ $( ./corso --version | grep 'Corso version:' | wc -l) -ne 1 ] + if [ $( ./corso --version | grep -c 'Corso version:' ) -ne 1 ] then echo "valid version not found" exit 1 @@ -77,7 +78,7 @@ jobs: TEST_RESULT: "test_results" run: | set -euo pipefail - prefix=`date +"%Y-%m-%d-%T"` + prefix=$(date +"%Y-%m-%d-%T") echo -e "\nRepo init test\n" >> ${CORSO_LOG_FILE} ./corso repo init s3 \ --no-stats \ @@ -123,9 +124,9 @@ jobs: AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | go run . exchange emails \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --destination Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ --count 4 - name: Backup exchange test @@ -191,7 +192,7 @@ jobs: echo -e "\nBackup Exchange restore test\n" >> ${CORSO_LOG_FILE} ./corso restore exchange \ --no-stats \ - --email-folder Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --email-folder Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ --hide-progress \ --backup "${{ steps.exchange-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/exchange-restore-test.txt @@ -201,7 +202,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.exchange-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "exchange" - TEST_DATA: Corso_Restore_st_${{ steps.repo-init.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.repo-init.outputs.result }} run: | set -euo pipefail ./sanityTest @@ -238,7 +239,7 @@ jobs: --no-stats \ --hide-progress \ --backup "${{ steps.exchange-incremental-test.outputs.result }}" \ - --email-folder Corso_Restore_st_${{ steps.repo-init.outputs.result }} \ + --email-folder Corso_Test_sanity${{ steps.repo-init.outputs.result }} \ 2>&1 | tee $TEST_RESULT/exchange-incremantal-restore-test.txt echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/exchange-incremantal-restore-test.txt | sed "s/Restoring to folder//" ) >> $GITHUB_OUTPUT @@ -246,7 +247,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.exchange-incremantal-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "exchange" - TEST_DATA: Corso_Restore_st_${{ steps.repo-init.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.repo-init.outputs.result }} BASE_BACKUP: ${{ steps.exchange-restore-test.outputs.result }} run: | set -euo pipefail @@ -265,13 +266,13 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | - suffix=`date +"%Y-%m-%d_%H-%M"` + suffix=$(date +"%Y-%m-%d_%H-%M") go run . onedrive files \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_$suffix \ + --destination Corso_Test_sanity$suffix \ --count 4 echo result="$suffix" >> $GITHUB_OUTPUT @@ -340,7 +341,7 @@ jobs: ./corso restore onedrive \ --no-stats \ --restore-permissions \ - --folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --folder Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --hide-progress \ --backup "${{ steps.onedrive-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/onedrive-restore-test.txt @@ -350,7 +351,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive" - TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} run: | set -euo pipefail ./sanityTest @@ -364,10 +365,10 @@ jobs: AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} run: | go run . onedrive files \ - --user ${{ env.TEST_USER }} \ + --user ${TEST_USER} \ --secondaryuser ${{ env.SECONDARY_TEST_USER }} \ --tenant ${{ env.AZURE_TENANT_ID }} \ - --destination Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --destination Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --count 4 # incremental backup @@ -403,7 +404,7 @@ jobs: --no-stats \ --restore-permissions \ --hide-progress \ - --folder Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} \ + --folder Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} \ --backup "${{ steps.onedrive-incremental-test.outputs.result }}" \ 2>&1 | tee $TEST_RESULT/onedrive-incremental-restore-test.txt echo result=$(grep -i -e 'Restoring to folder ' $TEST_RESULT/onedrive-incremental-restore-test.txt | sed "s/Restoring to folder//") >> $GITHUB_OUTPUT @@ -412,7 +413,7 @@ jobs: env: SANITY_RESTORE_FOLDER: ${{ steps.onedrive-incremental-restore-test.outputs.result }} SANITY_RESTORE_SERVICE: "onedrive" - TEST_DATA: Corso_Restore_st_${{ steps.new-data-creation-onedrive.outputs.result }} + TEST_DATA: Corso_Test_sanity${{ steps.new-data-creation-onedrive.outputs.result }} run: | set -euo pipefail ./sanityTest @@ -427,18 +428,18 @@ jobs: id: sharepoint-test run: | set -euo pipefail - echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup create sharepoint \ --no-stats \ --hide-progress \ - --site "${CORSO_M365_TEST_SITE_URL}" \ + --site "${TEST_SITE}" \ --json \ 2>&1 | tee $TEST_RESULT/backup_sharepoint.txt resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint.txt ) - if [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then + if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then echo "backup was not successful" exit 1 fi @@ -450,7 +451,7 @@ jobs: - name: Backup sharepoint list test run: | set -euo pipefail - echo -e "\nBackup List SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup List SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup list sharepoint \ --no-stats \ @@ -467,7 +468,7 @@ jobs: - name: Backup sharepoint list single backup test run: | set -euo pipefail - echo -e "\nBackup List single backup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nBackup List single backup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup list sharepoint \ --no-stats \ @@ -486,7 +487,7 @@ jobs: id: sharepoint-restore-test run: | set -euo pipefail - echo -e "\nRestore SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nRestore SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso restore sharepoint \ --no-stats \ @@ -513,18 +514,18 @@ jobs: id: sharepoint-incremental-test run: | set -euo pipefail - echo -e "\nIncremental Backup SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nIncremental Backup SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso backup create sharepoint \ --no-stats \ --hide-progress \ - --site "${CORSO_M365_TEST_SITE_URL}" \ + --site "${TEST_SITE}" \ --json \ 2>&1 | tee $TEST_RESULT/backup_sharepoint_incremental.txt resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_sharepoint_incremental.txt ) - if [[ $( echo $resultjson | jq -r '.[0] | .errorCount') -ne 0 ]]; then + if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then echo "backup was not successful" exit 1 fi @@ -537,7 +538,7 @@ jobs: id: sharepoint-incremental-restore-test run: | set -euo pipefail - echo -e "\nIncremental Restore SharePoint test\n" >> ${CORSO_LOG_FILE} + echo -e "\nIncremental Restore SharePoint test\n" >> ${CORSO_LOG_FILE} ./corso restore sharepoint \ --no-stats \ diff --git a/.github/workflows/website-publish.yml b/.github/workflows/website-publish.yml index b53ed320d..dda3909e9 100644 --- a/.github/workflows/website-publish.yml +++ b/.github/workflows/website-publish.yml @@ -28,8 +28,7 @@ jobs: - name: Get version string id: version run: | - echo "set-output name=version::$(git describe --tags --abbrev=0)" - echo "::set-output name=version::$(git describe --tags --abbrev=0)" + echo version=$(git describe --tags --abbrev=0) | tee -a $GITHUB_OUTPUT # ---------------------------------------------------------------------------------------------------- # --- Website Linting ----------------------------------------------------------------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index 259289b6a..3140e6ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) ### Added +- Released the --mask-sensitive-data flag, which will automatically obscure private data in logs. ### Fixed - Graph requests now automatically retry in case of a Bad Gateway or Gateway Timeout. @@ -17,6 +18,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improve restore time on large restores by optimizing how items are loaded from the remote repository. - Remove exchange item filtering based on m365 item ID via the CLI. - OneDrive backups no longer include a user's non-default drives. +- OneDrive and SharePoint file downloads will properly redirect from 3xx responses. +- Refined oneDrive rate limiter controls to reduce throttling errors. +- Fix handling of duplicate folders at the same hierarchy level in Exchange. Duplicate folders will be merged during restore operations. + +### Known Issues +- Restore operations will merge duplicate Exchange folders at the same hierarchy level into a single folder. ## [v0.7.0] (beta) - 2023-05-02 diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index fe2a07a75..0a8b45dc3 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -207,7 +207,7 @@ func runBackups( var ( owner = discSel.DiscreteOwner - ictx = clues.Add(ctx, "resource_owner", owner) + ictx = clues.Add(ctx, "resource_owner_selected", owner) ) bo, err := r.NewBackupWithLookup(ictx, discSel, ins) @@ -218,6 +218,11 @@ func runBackups( continue } + ictx = clues.Add( + ctx, + "resource_owner_id", bo.ResourceOwner.ID(), + "resource_owner_name", bo.ResourceOwner.Name()) + err = bo.Run(ictx) if err != nil { errs = append(errs, clues.Wrap(err, owner).WithClues(ictx)) diff --git a/src/cli/backup/exchange_e2e_test.go b/src/cli/backup/exchange_e2e_test.go index d135c8747..e5c60df2b 100644 --- a/src/cli/backup/exchange_e2e_test.go +++ b/src/cli/backup/exchange_e2e_test.go @@ -54,7 +54,6 @@ func TestNoBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &NoBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -120,7 +119,6 @@ func TestBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &BackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -235,7 +233,6 @@ func TestPreparedBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &PreparedBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -490,7 +487,6 @@ func TestBackupDeleteExchangeE2ESuite(t *testing.T) { Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, ), }) } diff --git a/src/cli/backup/onedrive_e2e_test.go b/src/cli/backup/onedrive_e2e_test.go index d41bbc1aa..73cedd2ca 100644 --- a/src/cli/backup/onedrive_e2e_test.go +++ b/src/cli/backup/onedrive_e2e_test.go @@ -22,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/storage" ) @@ -44,9 +45,7 @@ func TestNoBackupOneDriveE2ESuite(t *testing.T) { suite.Run(t, &NoBackupOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } @@ -148,9 +147,7 @@ func TestBackupDeleteOneDriveE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } @@ -176,7 +173,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) SetupSuite() { // some tests require an existing backup sel := selectors.NewOneDriveBackup(users) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) backupOp, err := suite.repo.NewBackupWithLookup(ctx, sel.Selector, ins) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/cli/backup/sharepoint_e2e_test.go b/src/cli/backup/sharepoint_e2e_test.go index 4471e9755..09d65d90e 100644 --- a/src/cli/backup/sharepoint_e2e_test.go +++ b/src/cli/backup/sharepoint_e2e_test.go @@ -45,7 +45,6 @@ func TestNoBackupSharePointE2ESuite(t *testing.T) { suite.Run(t, &NoBackupSharePointE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -112,9 +111,7 @@ func TestBackupDeleteSharePointE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteSharePointE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index d5e6c992e..388b687e2 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -25,7 +25,6 @@ func TestS3E2ESuite(t *testing.T) { suite.Run(t, &S3E2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 0d9bf7b58..30114aa4f 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -48,9 +48,7 @@ func TestRestoreExchangeE2ESuite(t *testing.T) { suite.Run(t, &RestoreExchangeE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/utils/flags.go b/src/cli/utils/flags.go index b03fe2e06..3ca50d93e 100644 --- a/src/cli/utils/flags.go +++ b/src/cli/utils/flags.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" ) // common flag vars (eg: FV) @@ -215,6 +216,11 @@ func trimFolderSlash(folders []string) []string { res := make([]string, 0, len(folders)) for _, p := range folders { + if p == string(path.PathSeparator) { + res = selectors.Any() + break + } + // Use path package because it has logic to handle escaping already. res = append(res, path.TrimTrailingSlash(p)) } diff --git a/src/cli/utils/onedrive_test.go b/src/cli/utils/onedrive_test.go index 43c0507c0..61653045f 100644 --- a/src/cli/utils/onedrive_test.go +++ b/src/cli/utils/onedrive_test.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/path" ) type OneDriveUtilsSuite struct { @@ -26,6 +27,7 @@ func (suite *OneDriveUtilsSuite) TestIncludeOneDriveRestoreDataSelectors() { containsOnly = []string{"contains"} prefixOnly = []string{"/prefix"} containsAndPrefix = []string{"contains", "/prefix"} + onlySlash = []string{string(path.PathSeparator)} ) table := []struct { @@ -87,6 +89,15 @@ func (suite *OneDriveUtilsSuite) TestIncludeOneDriveRestoreDataSelectors() { }, expectIncludeLen: 2, }, + { + name: "folder with just /", + opts: utils.OneDriveOpts{ + Users: empty, + FileName: empty, + FolderPath: onlySlash, + }, + expectIncludeLen: 1, + }, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/sharepoint_test.go b/src/cli/utils/sharepoint_test.go index 41bb87e10..0201ab29e 100644 --- a/src/cli/utils/sharepoint_test.go +++ b/src/cli/utils/sharepoint_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -30,6 +31,7 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() { containsOnly = []string{"contains"} prefixOnly = []string{"/prefix"} containsAndPrefix = []string{"contains", "/prefix"} + onlySlash = []string{string(path.PathSeparator)} ) table := []struct { @@ -182,6 +184,13 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() { }, expectIncludeLen: 2, }, + { + name: "folder with just /", + opts: utils.SharePointOpts{ + FolderPath: onlySlash, + }, + expectIncludeLen: 1, + }, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index cde3023fd..8bbb35a58 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -356,6 +356,13 @@ var ( FolderPath: selectors.Any(), }, }, + { + Name: "FilesWithSingleSlash", + Expected: testdata.OneDriveItems, + Opts: utils.OneDriveOpts{ + FolderPath: []string{"/"}, + }, + }, { Name: "FolderPrefixMatch", Expected: testdata.OneDriveItems, @@ -482,6 +489,13 @@ var ( FolderPath: selectors.Any(), }, }, + { + Name: "LibraryItemsWithSingleSlash", + Expected: testdata.SharePointLibraryItems, + Opts: utils.SharePointOpts{ + FolderPath: []string{"/"}, + }, + }, { Name: "FolderPrefixMatch", Expected: testdata.SharePointLibraryItems, diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index b07ea4a08..c899e9b21 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -420,7 +420,7 @@ func generateAndRestoreOnedriveItems( Service: service, Tenant: tenantID, ResourceOwners: []string{resourceOwner}, - Dest: tester.DefaultTestRestoreDestination(), + Dest: tester.DefaultTestRestoreDestination(""), } _, _, collections, _ := connector.GetCollectionsAndExpected( diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go index 4868ab343..3b338ca74 100644 --- a/src/cmd/getM365/onedrive/get_item.go +++ b/src/cmd/getM365/onedrive/get_item.go @@ -112,7 +112,7 @@ func runDisplayM365JSON( creds account.M365Config, user, itemID string, ) error { - drive, err := api.GetDriveByID(ctx, srv, user) + drive, err := api.GetUsersDrive(ctx, srv, user) if err != nil { return err } diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 4b72ebe8f..ae1acf328 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -131,6 +131,12 @@ if (![string]::IsNullOrEmpty($User)) { # Works for dev domains where format is @.onmicrosoft.com $domain = $User.Split('@')[1].Split('.')[0] $userNameEscaped = $User.Replace('.', '_').Replace('@', '_') + + # hacky special case because of recreated CI user + if ($userNameEscaped -ilike "lynner*") { + $userNameEscaped += '1' + } + $siteUrl = "https://$domain-my.sharepoint.com/personal/$userNameEscaped/" if ($LibraryNameList.count -eq 0) { diff --git a/src/go.mod b/src/go.mod index 526a5151d..a90058680 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,13 +2,13 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f +replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.256 + github.com/aws/aws-sdk-go v1.44.262 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 @@ -34,7 +34,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb golang.org/x/time v0.1.0 - golang.org/x/tools v0.8.0 + golang.org/x/tools v0.9.1 gopkg.in/resty.v1 v1.12.0 ) @@ -79,7 +79,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.4 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/klauspost/reedsolomon v1.11.7 // indirect @@ -118,11 +118,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.8.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.54.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/src/go.sum b/src/go.sum index fd158fd2f..f8a86e102 100644 --- a/src/go.sum +++ b/src/go.sum @@ -55,8 +55,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c h1:Njdw/Nnq2DN3f8QMaHuZZHdVHTUSxFqPMMxDIInDWB4= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8= -github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f h1:cD7mcWVTEu83qX6Ml3aqgo8DDv+fBZt/7mQQps2TokM= -github.com/alcionai/kopia v0.12.2-0.20230417220734-efdcd8c54f7f/go.mod h1:eTgZSDaU2pDzVGC7QRubbKOeohvHzzbRXvhZMH+AGHA= +github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 h1:Wrl99Y7jftZMnNDiOIcRJrjstZO3IEj3+Q/sip27vmI= +github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -66,8 +66,8 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4= -github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I= +github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -203,7 +203,7 @@ github.com/h2non/gock v1.2.0 h1:K6ol8rfrRkUOefooBC8elXoaNGYkpp7y2qcxGG6BzUE= github.com/h2non/gock v1.2.0/go.mod h1:tNhoxHYW2W42cYkYb1WqzdbYIieALC99kpYr7rH/BQk= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hanwen/go-fuse/v2 v2.2.0 h1:jo5QZYmBLNcl9ovypWaQ5yXMSSV+Ch68xoC3rtZvvBM= +github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= @@ -234,8 +234,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= -github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -530,8 +530,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -552,8 +552,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -604,8 +604,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -673,8 +673,8 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -741,8 +741,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd h1:sLpv7bNL1AsX3fdnWh9WVh7ejIzXdOc1RRHGeAmeStU= -google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 9f0f738e5..e66846fef 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -49,6 +49,8 @@ func (gc *GraphConnector) ProduceBackupCollections( diagnostics.Index("service", sels.Service.String())) defer end() + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + // Limit the max number of active requests to graph from this collection. ctrlOpts.Parallelism.ItemFetch = graph.Parallelism(sels.PathService()). ItemOverride(ctx, ctrlOpts.Parallelism.ItemFetch) @@ -194,7 +196,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( ctx context.Context, backupVersion int, acct account.Account, - selector selectors.Selector, + sels selectors.Selector, dest control.RestoreDestination, opts control.Options, dcs []data.RestoreCollection, @@ -203,6 +205,8 @@ func (gc *GraphConnector) ConsumeRestoreCollections( ctx, end := diagnostics.Span(ctx, "connector:restore") defer end() + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + var ( status *support.ConnectorOperationStatus deets = &details.Builder{} @@ -213,7 +217,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( return nil, clues.Wrap(err, "malformed azure credentials") } - switch selector.Service { + switch sels.Service { case selectors.ServiceExchange: status, err = exchange.RestoreExchangeDataCollections(ctx, creds, gc.Service, dest, dcs, deets, errs) case selectors.ServiceOneDrive: @@ -221,7 +225,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections( case selectors.ServiceSharePoint: status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets, errs) default: - err = clues.Wrap(clues.New(selector.Service.String()), "service not supported") + err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") } gc.incrementAwaitingMessages() diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index fedc85106..3025a385c 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -20,7 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/selectors/testdata" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------------------- @@ -160,7 +160,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "Invalid onedrive backup user", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup(owners) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, }, @@ -168,7 +168,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "Invalid sharepoint backup site", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup(owners) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) return sel.Selector }, }, @@ -185,7 +185,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "missing onedrive backup user", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup(owners) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "" return sel.Selector }, @@ -194,7 +194,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() name: "missing sharepoint backup site", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup(owners) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.DiscreteOwner = "" return sel.Selector }, @@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { name: "Libraries", getSelector: func() selectors.Selector { sel := selectors.NewSharePointBackup(selSites) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) return sel.Selector }, }, diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index b011921b5..57a561b8a 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -134,6 +134,10 @@ func (c Mail) GetItem( immutableIDs bool, errs *fault.Bus, ) (serialization.Parsable, *details.ExchangeInfo, error) { + var ( + size int64 + mailBody models.ItemBodyable + ) // Will need adjusted if attachments start allowing paging. headers := buildPreferHeaders(false, immutableIDs) itemOpts := &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{ @@ -145,8 +149,16 @@ func (c Mail) GetItem( return nil, nil, graph.Stack(ctx, err) } - if !ptr.Val(mail.GetHasAttachments()) && !HasAttachments(mail.GetBody()) { - return mail, MailInfo(mail), nil + mailBody = mail.GetBody() + if mailBody != nil { + content := ptr.Val(mailBody.GetContent()) + if len(content) > 0 { + size = int64(len(content)) + } + } + + if !ptr.Val(mail.GetHasAttachments()) && !HasAttachments(mailBody) { + return mail, MailInfo(mail, size), nil } options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{ @@ -163,8 +175,14 @@ func (c Mail) GetItem( Attachments(). Get(ctx, options) if err == nil { + for _, a := range attached.GetValue() { + attachSize := ptr.Val(a.GetSize()) + size = +int64(attachSize) + } + mail.SetAttachments(attached.GetValue()) - return mail, MailInfo(mail), nil + + return mail, MailInfo(mail, size), nil } // A failure can be caused by having a lot of attachments as @@ -214,11 +232,13 @@ func (c Mail) GetItem( } atts = append(atts, att) + attachSize := ptr.Val(a.GetSize()) + size = +int64(attachSize) } mail.SetAttachments(atts) - return mail, MailInfo(mail), nil + return mail, MailInfo(mail, size), nil } // EnumerateContainers iterates through all of the users current @@ -419,7 +439,7 @@ func (c Mail) Serialize( // Helpers // --------------------------------------------------------------------------- -func MailInfo(msg models.Messageable) *details.ExchangeInfo { +func MailInfo(msg models.Messageable, size int64) *details.ExchangeInfo { var ( sender = UnwrapEmailAddress(msg.GetSender()) subject = ptr.Val(msg.GetSubject()) @@ -444,6 +464,7 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo { Recipient: recipients, Subject: subject, Received: received, + Size: size, Created: created, Modified: ptr.OrNow(msg.GetLastModifiedDateTime()), } diff --git a/src/internal/connector/exchange/api/mail_test.go b/src/internal/connector/exchange/api/mail_test.go index 2ce0cd537..f98093cf6 100644 --- a/src/internal/connector/exchange/api/mail_test.go +++ b/src/internal/connector/exchange/api/mail_test.go @@ -152,7 +152,7 @@ func (suite *MailAPIUnitSuite) TestMailInfo() { for _, tt := range tests { suite.Run(tt.name, func() { msg, expected := tt.msgAndRP() - assert.Equal(suite.T(), expected, api.MailInfo(msg)) + assert.Equal(suite.T(), expected, api.MailInfo(msg, 0)) }) } } @@ -213,6 +213,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { name string setupf func() attachmentCount int + size int64 expect assert.ErrorAssertionFunc }{ { @@ -242,6 +243,9 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { atts := models.NewAttachmentCollectionResponse() aitem := models.NewAttachment() + + asize := int32(50) + aitem.SetSize(&asize) atts.SetValue([]models.Attachmentable{aitem}) gock.New("https://graph.microsoft.com"). @@ -250,6 +254,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { JSON(getJSONObject(suite.T(), atts)) }, attachmentCount: 1, + size: 50, expect: assert.NoError, }, { @@ -289,6 +294,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { JSON(getJSONObject(suite.T(), aitem)) }, attachmentCount: 1, + size: 200, expect: assert.NoError, }, { @@ -330,6 +336,7 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { } }, attachmentCount: 5, + size: 200, expect: assert.NoError, }, } @@ -348,8 +355,23 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() { it, ok := item.(models.Messageable) require.True(suite.T(), ok, "convert to messageable") + var size int64 + mailBody := it.GetBody() + if mailBody != nil { + content := ptr.Val(mailBody.GetContent()) + if len(content) > 0 { + size = int64(len(content)) + } + } + + attachments := it.GetAttachments() + for _, attachment := range attachments { + size = +int64(*attachment.GetSize()) + } + assert.Equal(suite.T(), *it.GetId(), mid) - assert.Equal(suite.T(), tt.attachmentCount, len(it.GetAttachments()), "attachment count") + assert.Equal(suite.T(), tt.attachmentCount, len(attachments), "attachment count") + assert.Equal(suite.T(), tt.size, size, "mail size") assert.True(suite.T(), gock.IsDone(), "made all requests") }) } diff --git a/src/internal/connector/exchange/container_resolver_test.go b/src/internal/connector/exchange/container_resolver_test.go index 572162263..de050d25a 100644 --- a/src/internal/connector/exchange/container_resolver_test.go +++ b/src/internal/connector/exchange/container_resolver_test.go @@ -549,7 +549,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { var ( user = tester.M365UserID(suite.T()) directoryCaches = make(map[path.CategoryType]graph.ContainerResolver) - folderName = tester.DefaultTestRestoreDestination().ContainerName + folderName = tester.DefaultTestRestoreDestination("").ContainerName tests = []struct { name string pathFunc1 func(t *testing.T) path.Path diff --git a/src/internal/connector/exchange/data_collections_test.go b/src/internal/connector/exchange/data_collections_test.go index e2c460cb8..2c23747df 100644 --- a/src/internal/connector/exchange/data_collections_test.go +++ b/src/internal/connector/exchange/data_collections_test.go @@ -282,9 +282,18 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() { } require.NotEmpty(t, c.FullPath().Folder(false)) - folder := c.FullPath().Folder(false) - delete(test.folderNames, folder) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), c) { + continue + } + + loc := c.(data.LocationPather).LocationPath().String() + + require.NotEmpty(t, loc) + + delete(test.folderNames, loc) } assert.Empty(t, test.folderNames) @@ -525,7 +534,16 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression continue } - assert.Equal(t, edc.FullPath().Folder(false), DefaultContactFolder) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), edc) { + continue + } + + assert.Equal( + t, + edc.(data.LocationPather).LocationPath().String(), + DefaultContactFolder) assert.NotZero(t, count) } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 4a2760be4..441056ed6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -260,7 +260,12 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { return } - info.Size = int64(len(data)) + // In case of mail the size of data is calc as- size of body content+size of attachment + // in all other case the size is - total item's serialized size + if info.Size <= 0 { + info.Size = int64(len(data)) + } + info.ParentPath = col.locationPath.String() col.data <- &Stream{ diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 1aa2beece..b6ec9168f 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -3,14 +3,12 @@ package exchange import ( "context" "testing" - "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" @@ -67,8 +65,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreContact() { var ( t = suite.T() userID = tester.M365UserID(t) - now = time.Now() - folderName = "TestRestoreContact: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName = tester.DefaultTestRestoreDestination("contact").ContainerName ) aFolder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName) @@ -102,7 +99,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreEvent() { var ( t = suite.T() userID = tester.M365UserID(t) - subject = "TestRestoreEvent: " + dttm.FormatNow(dttm.SafeForTesting) + subject = tester.DefaultTestRestoreDestination("event").ContainerName ) calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, subject) @@ -172,7 +169,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { } userID := tester.M365UserID(suite.T()) - now := time.Now() + tests := []struct { name string bytes []byte @@ -184,7 +181,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageBytes("Restore Exchange Object"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailObject: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailobj").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -196,7 +193,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithDirectAttachment("Restore 1 Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailwattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -208,7 +205,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithItemAttachmentEvent("Event Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreEventItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("eventwattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -220,7 +217,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithItemAttachmentMail("Mail Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailitemattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -235,7 +232,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailBasicItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailbasicattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -250,7 +247,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "ItemMailAttachmentwAttachment " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailnestattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -265,7 +262,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "ItemMailAttachment_Contact " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailcontactattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -277,7 +274,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithNestedItemAttachmentEvent("Nested Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreNestedEventItemAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("nestedattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -289,7 +286,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithLargeAttachment("Restore Large Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithLargeAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("maillargeattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -301,7 +298,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithTwoAttachments("Restore 2 Attachments"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithAttachments: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailtwoattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -313,7 +310,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.MessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailwithReferenceAttachment: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("mailrefattch").ContainerName folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -326,7 +323,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.ContactBytes("Test_Omega"), category: path.ContactsCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreContactObject: " + dttm.FormatTo(now, dttm.SafeForTesting) + folderName := tester.DefaultTestRestoreDestination("contact").ContainerName folder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) @@ -338,8 +335,8 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.EventBytes("Restored Event Object"), category: path.EventsCategory, destination: func(t *testing.T, ctx context.Context) string { - calendarName := "TestRestoreEventObject: " + dttm.FormatTo(now, dttm.SafeForTesting) - calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName) + folderName := tester.DefaultTestRestoreDestination("event").ContainerName + calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) return ptr.Val(calendar.GetId()) @@ -350,8 +347,8 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { bytes: exchMock.EventWithAttachment("Restored Event Attachment"), category: path.EventsCategory, destination: func(t *testing.T, ctx context.Context) string { - calendarName := "TestRestoreEventObject_" + dttm.FormatTo(now, dttm.SafeForTesting) - calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName) + folderName := tester.DefaultTestRestoreDestination("eventobj").ContainerName + calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, folderName) require.NoError(t, err, clues.ToCore(err)) return ptr.Val(calendar.GetId()) diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index cad25cdd8..52d46ba42 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -137,21 +137,15 @@ func includeContainer( directory = locPath.Folder(false) } - var ( - ok bool - pathRes path.Path - ) + var ok bool switch category { case path.EmailCategory: ok = scope.Matches(selectors.ExchangeMailFolder, directory) - pathRes = locPath case path.ContactsCategory: ok = scope.Matches(selectors.ExchangeContactFolder, directory) - pathRes = locPath case path.EventsCategory: ok = scope.Matches(selectors.ExchangeEventCalendar, directory) - pathRes = dirPath default: return nil, nil, false } @@ -162,5 +156,5 @@ func includeContainer( "matches_input", directory, ).Debug("backup folder selection filter") - return pathRes, loc, ok + return dirPath, loc, ok } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 34ea37d3f..9f707df21 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -56,10 +56,6 @@ func filterContainersAndFillCollections( // deleted from this map, leaving only the deleted folders behind tombstones = makeTombstones(dps) category = qp.Category - - // Stop-gap: Track folders by LocationPath and if there's duplicates pick - // the one with the lexicographically larger ID. - dupPaths = map[string]string{} ) logger.Ctx(ctx).Infow("filling collections", "len_deltapaths", len(dps)) @@ -108,53 +104,6 @@ func filterContainersAndFillCollections( continue } - // This is a duplicate collection. Either the collection we're examining now - // should be skipped or the collection we previously added should be - // skipped. - // - // Calendars is already using folder IDs so we don't need to pick the - // "newest" folder for that. - if oldCID := dupPaths[locPath.String()]; category != path.EventsCategory && len(oldCID) > 0 { - if cID < oldCID { - logger.Ctx(ictx).Infow( - "skipping duplicate folder with lesser ID", - "previous_folder_id", clues.Hide(oldCID), - "current_folder_id", clues.Hide(cID), - "duplicate_path", locPath) - - // Readd this entry to the tombstone map because we remove it first off. - if oldDP, ok := dps[cID]; ok { - tombstones[cID] = oldDP.path - } - - // Continuing here ensures we don't add anything to the paths map or the - // delta map which is the behavior we want. - continue - } - - logger.Ctx(ictx).Infow( - "switching duplicate folders as newer folder found", - "previous_folder_id", clues.Hide(oldCID), - "current_folder_id", clues.Hide(cID), - "duplicate_path", locPath) - - // Remove the previous collection from the maps. This will make us think - // it's a new item and properly populate it if it ever: - // * moves - // * replaces the current entry (current entry moves/is deleted) - delete(collections, oldCID) - delete(deltaURLs, oldCID) - delete(currPaths, oldCID) - - // Re-add the tombstone entry for the old folder so that it can be marked - // as deleted if need. - if oldDP, ok := dps[oldCID]; ok { - tombstones[oldCID] = oldDP.path - } - } - - dupPaths[locPath.String()] = cID - if len(prevPathStr) > 0 { if prevPath, err = pathFromPrevString(prevPathStr); err != nil { logger.CtxErr(ictx, err).Error("parsing prev path") diff --git a/src/internal/connector/exchange/service_iterators_test.go b/src/internal/connector/exchange/service_iterators_test.go index d7a355122..5b4d11940 100644 --- a/src/internal/connector/exchange/service_iterators_test.go +++ b/src/internal/connector/exchange/service_iterators_test.go @@ -384,6 +384,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli ResourceOwner: inMock.NewProvider("user_id", "user_name"), Credentials: suite.creds, } + statusUpdater = func(*support.ConnectorOperationStatus) {} dataTypes = []scopeCat{ @@ -395,6 +396,10 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli scope: selectors.NewExchangeBackup(nil).ContactFolders(selectors.Any())[0], cat: path.ContactsCategory, }, + { + scope: selectors.NewExchangeBackup(nil).EventCalendars(selectors.Any())[0], + cat: path.EventsCategory, + }, } location = path.Builder{}.Append("foo", "bar") @@ -448,8 +453,20 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli return res } - locPath := func(t *testing.T, cat path.CategoryType) path.Path { - res, err := location.ToDataLayerPath( + idPath1 := func(t *testing.T, cat path.CategoryType) path.Path { + res, err := path.Builder{}.Append("1").ToDataLayerPath( + suite.creds.AzureTenantID, + qp.ResourceOwner.ID(), + path.ExchangeService, + cat, + false) + require.NoError(t, err, clues.ToCore(err)) + + return res + } + + idPath2 := func(t *testing.T, cat path.CategoryType) path.Path { + res, err := path.Builder{}.Append("2").ToDataLayerPath( suite.creds.AzureTenantID, qp.ResourceOwner.ID(), path.ExchangeService, @@ -467,8 +484,6 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli inputMetadata func(t *testing.T, cat path.CategoryType) DeltaPaths expectNewColls int expectDeleted int - expectAdded []string - expectRemoved []string expectMetadata func(t *testing.T, cat path.CategoryType) DeltaPaths }{ { @@ -486,49 +501,19 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli }, "2": DeltaPath{ delta: "old_delta", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { - return DeltaPaths{ - "2": DeltaPath{ - delta: "delta_url2", - path: locPath(t, cat).String(), - }, - } - }, - }, - { - name: "1 moved to duplicate, other order", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container2, container1), - inputMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ "1": DeltaPath{ - delta: "old_delta", - path: oldPath1(t, cat).String(), + delta: "delta_url", + path: idPath1(t, cat).String(), }, - "2": DeltaPath{ - delta: "old_delta", - path: locPath(t, cat).String(), - }, - } - }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, - expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { - return DeltaPaths{ "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -552,14 +537,15 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli }, } }, - expectDeleted: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ + "1": DeltaPath{ + delta: "delta_url", + path: idPath1(t, cat).String(), + }, "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -574,14 +560,16 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli inputMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{} }, - expectNewColls: 1, - expectAdded: result2.added, - expectRemoved: result2.removed, + expectNewColls: 2, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ + "1": DeltaPath{ + delta: "delta_url", + path: idPath1(t, cat).String(), + }, "2": DeltaPath{ delta: "delta_url2", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, @@ -596,19 +584,17 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli return DeltaPaths{ "2": DeltaPath{ delta: "old_delta", - path: locPath(t, cat).String(), + path: idPath2(t, cat).String(), }, } }, expectNewColls: 1, expectDeleted: 1, - expectAdded: result1.added, - expectRemoved: result1.removed, expectMetadata: func(t *testing.T, cat path.CategoryType) DeltaPaths { return DeltaPaths{ "1": DeltaPath{ delta: "delta_url", - path: locPath(t, cat).String(), + path: idPath1(t, cat).String(), }, } }, @@ -633,7 +619,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli statusUpdater, test.resolver, sc.scope, - test.inputMetadata(t, sc.cat), + test.inputMetadata(t, qp.Category), control.Options{FailureHandling: control.FailFast}, fault.New(true)) require.NoError(t, err, "getting collections", clues.ToCore(err)) @@ -649,21 +635,30 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli if c.FullPath().Service() == path.ExchangeMetadataService { metadatas++ - checkMetadata(t, ctx, sc.cat, test.expectMetadata(t, sc.cat), c) + checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c) continue } if c.State() == data.NewState { news++ } + } - exColl, ok := c.(*Collection) - require.True(t, ok, "collection is an *exchange.Collection") + assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") + assert.Equal(t, test.expectNewColls, news, "new collections") + assert.Equal(t, 1, metadatas, "metadata collections") - if exColl.LocationPath() != nil { - assert.Equal(t, location.String(), exColl.LocationPath().String()) + // items in collections assertions + for k, expect := range test.getter { + coll := collections[k] + + if coll == nil { + continue } + exColl, ok := coll.(*Collection) + require.True(t, ok, "collection is an *exchange.Collection") + ids := [][]string{ make([]string, 0, len(exColl.added)), make([]string, 0, len(exColl.removed)), @@ -675,268 +670,15 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_Dupli } } - assert.ElementsMatch(t, test.expectAdded, ids[0], "added items") - assert.ElementsMatch(t, test.expectRemoved, ids[1], "removed items") + assert.ElementsMatch(t, expect.added, ids[0], "added items") + assert.ElementsMatch(t, expect.removed, ids[1], "removed items") } - - assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") - assert.Equal(t, test.expectNewColls, news, "new collections") - assert.Equal(t, 1, metadatas, "metadata collections") }) } }) } } -func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_DuplicateFolders_Events() { - var ( - qp = graph.QueryParams{ - ResourceOwner: inMock.NewProvider("user_id", "user_name"), - Category: path.EventsCategory, - Credentials: suite.creds, - } - statusUpdater = func(*support.ConnectorOperationStatus) {} - - scope = selectors.NewExchangeBackup(nil).EventCalendars(selectors.Any())[0] - - location = path.Builder{}.Append("foo", "bar") - - result1 = mockGetterResults{ - added: []string{"a1", "a2", "a3"}, - removed: []string{"r1", "r2", "r3"}, - newDelta: api.DeltaUpdate{URL: "delta_url"}, - } - result2 = mockGetterResults{ - added: []string{"a4", "a5", "a6"}, - removed: []string{"r4", "r5", "r6"}, - newDelta: api.DeltaUpdate{URL: "delta_url2"}, - } - - container1 = mockContainer{ - id: strPtr("1"), - displayName: strPtr("bar"), - p: path.Builder{}.Append("1"), - l: location, - } - container2 = mockContainer{ - id: strPtr("2"), - displayName: strPtr("bar"), - p: path.Builder{}.Append("2"), - l: location, - } - ) - - oldPath1, err := location.Append("1").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - oldPath2, err := location.Append("2").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - idPath1, err := path.Builder{}.Append("1").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - idPath2, err := path.Builder{}.Append("2").ToDataLayerPath( - suite.creds.AzureTenantID, - qp.ResourceOwner.ID(), - path.ExchangeService, - qp.Category, - false) - require.NoError(suite.T(), err, clues.ToCore(err)) - - table := []struct { - name string - getter mockGetter - resolver graph.ContainerResolver - inputMetadata DeltaPaths - expectNewColls int - expectDeleted int - expectMetadata DeltaPaths - }{ - { - name: "1 moved to duplicate", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "old_delta", - path: oldPath1.String(), - }, - "2": DeltaPath{ - delta: "old_delta", - path: idPath2.String(), - }, - }, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "both move to duplicate", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "old_delta", - path: oldPath1.String(), - }, - "2": DeltaPath{ - delta: "old_delta", - path: oldPath2.String(), - }, - }, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "both new", - getter: map[string]mockGetterResults{ - "1": result1, - "2": result2, - }, - resolver: newMockResolver(container1, container2), - inputMetadata: DeltaPaths{}, - expectNewColls: 2, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - "2": DeltaPath{ - delta: "delta_url2", - path: idPath2.String(), - }, - }, - }, - { - name: "add 1 remove 2", - getter: map[string]mockGetterResults{ - "1": result1, - }, - resolver: newMockResolver(container1), - inputMetadata: DeltaPaths{ - "2": DeltaPath{ - delta: "old_delta", - path: idPath2.String(), - }, - }, - expectNewColls: 1, - expectDeleted: 1, - expectMetadata: DeltaPaths{ - "1": DeltaPath{ - delta: "delta_url", - path: idPath1.String(), - }, - }, - }, - } - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() - - ctx, flush := tester.NewContext() - defer flush() - - collections, err := filterContainersAndFillCollections( - ctx, - qp, - test.getter, - statusUpdater, - test.resolver, - scope, - test.inputMetadata, - control.Options{FailureHandling: control.FailFast}, - fault.New(true)) - require.NoError(t, err, "getting collections", clues.ToCore(err)) - - // collection assertions - - deleteds, news, metadatas := 0, 0, 0 - for _, c := range collections { - if c.State() == data.DeletedState { - deleteds++ - continue - } - - if c.FullPath().Service() == path.ExchangeMetadataService { - metadatas++ - checkMetadata(t, ctx, qp.Category, test.expectMetadata, c) - continue - } - - if c.State() == data.NewState { - news++ - } - } - - assert.Equal(t, test.expectDeleted, deleteds, "deleted collections") - assert.Equal(t, test.expectNewColls, news, "new collections") - assert.Equal(t, 1, metadatas, "metadata collections") - - // items in collections assertions - for k, expect := range test.getter { - coll := collections[k] - - if coll == nil { - continue - } - - exColl, ok := coll.(*Collection) - require.True(t, ok, "collection is an *exchange.Collection") - - ids := [][]string{ - make([]string, 0, len(exColl.added)), - make([]string, 0, len(exColl.removed)), - } - - for i, cIDs := range []map[string]struct{}{exColl.added, exColl.removed} { - for id := range cIDs { - ids[i] = append(ids[i], id) - } - } - - assert.ElementsMatch(t, expect.added, ids[0], "added items") - assert.ElementsMatch(t, expect.removed, ids[1], "removed items") - } - }) - } -} - func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repeatedItems() { newDelta := api.DeltaUpdate{URL: "delta_url"} diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 4d49e3df9..9e293ce5d 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -218,8 +218,7 @@ func RestoreMailMessage( return nil, err } - info := api.MailInfo(clone) - info.Size = int64(len(bits)) + info := api.MailInfo(clone, int64(len(bits))) return info, nil } @@ -436,16 +435,13 @@ func restoreCollection( metrics.Bytes += int64(len(byteArray)) metrics.Successes++ - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { errs.AddRecoverable(clues.Wrap(err, "building full path with item").WithClues(ctx)) continue } - locationRef := &path.Builder{} - if category == path.ContactsCategory { - locationRef = locationRef.Append(itemPath.Folders()...) - } + locationRef := path.Builder{}.Append(itemPath.Folders()...) err = deets.Add( itemPath, diff --git a/src/internal/connector/graph/collections.go b/src/internal/connector/graph/collections.go index ce93aa6c9..ee941f81c 100644 --- a/src/internal/connector/graph/collections.go +++ b/src/internal/connector/graph/collections.go @@ -11,14 +11,19 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -var _ data.BackupCollection = emptyCollection{} +var _ data.BackupCollection = prefixCollection{} -type emptyCollection struct { - p path.Path - su support.StatusUpdater +// TODO: move this out of graph. /data would be a much better owner +// for a generic struct like this. However, support.StatusUpdater makes +// it difficult to extract from this package in a generic way. +type prefixCollection struct { + full path.Path + prev path.Path + su support.StatusUpdater + state data.CollectionState } -func (c emptyCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { +func (c prefixCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { res := make(chan data.Stream) close(res) @@ -28,21 +33,19 @@ func (c emptyCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.St return res } -func (c emptyCollection) FullPath() path.Path { - return c.p +func (c prefixCollection) FullPath() path.Path { + return c.full } -func (c emptyCollection) PreviousPath() path.Path { - return c.p +func (c prefixCollection) PreviousPath() path.Path { + return c.prev } -func (c emptyCollection) State() data.CollectionState { - // This assumes we won't change the prefix path. Could probably use MovedState - // as well if we do need to change things around. - return data.NotMovedState +func (c prefixCollection) State() data.CollectionState { + return c.state } -func (c emptyCollection) DoNotMergeItems() bool { +func (c prefixCollection) DoNotMergeItems() bool { return false } @@ -76,7 +79,7 @@ func BaseCollections( for cat := range categories { ictx := clues.Add(ctx, "base_service", service, "base_category", cat) - p, err := path.ServicePrefix(tenant, rOwner, service, cat) + full, err := path.ServicePrefix(tenant, rOwner, service, cat) if err != nil { // Shouldn't happen. err = clues.Wrap(err, "making path").WithClues(ictx) @@ -87,8 +90,13 @@ func BaseCollections( } // only add this collection if it doesn't already exist in the set. - if _, ok := collKeys[p.String()]; !ok { - res = append(res, emptyCollection{p: p, su: su}) + if _, ok := collKeys[full.String()]; !ok { + res = append(res, &prefixCollection{ + prev: full, + full: full, + su: su, + state: data.StateOf(full, full), + }) } } @@ -99,45 +107,11 @@ func BaseCollections( // prefix migration // --------------------------------------------------------------------------- -var _ data.BackupCollection = prefixCollection{} - -// TODO: move this out of graph. /data would be a much better owner -// for a generic struct like this. However, support.StatusUpdater makes -// it difficult to extract from this package in a generic way. -type prefixCollection struct { - full, prev path.Path - su support.StatusUpdater - state data.CollectionState -} - -func (c prefixCollection) Items(ctx context.Context, _ *fault.Bus) <-chan data.Stream { - res := make(chan data.Stream) - close(res) - - s := support.CreateStatus(ctx, support.Backup, 0, support.CollectionMetrics{}, "") - c.su(s) - - return res -} - -func (c prefixCollection) FullPath() path.Path { - return c.full -} - -func (c prefixCollection) PreviousPath() path.Path { - return c.prev -} - -func (c prefixCollection) State() data.CollectionState { - return c.state -} - -func (c prefixCollection) DoNotMergeItems() bool { - return false -} - // Creates a new collection that only handles prefix pathing. -func NewPrefixCollection(prev, full path.Path, su support.StatusUpdater) (*prefixCollection, error) { +func NewPrefixCollection( + prev, full path.Path, + su support.StatusUpdater, +) (*prefixCollection, error) { if prev != nil { if len(prev.Item()) > 0 { return nil, clues.New("prefix collection previous path contains an item") diff --git a/src/internal/connector/graph/concurrency_limiter.go b/src/internal/connector/graph/concurrency_limiter.go deleted file mode 100644 index 6fe1ea0cd..000000000 --- a/src/internal/connector/graph/concurrency_limiter.go +++ /dev/null @@ -1,53 +0,0 @@ -package graph - -import ( - "net/http" - "sync" - - "github.com/alcionai/clues" - khttp "github.com/microsoft/kiota-http-go" -) - -// concurrencyLimiter middleware limits the number of concurrent requests to graph API -type concurrencyLimiter struct { - semaphore chan struct{} -} - -var ( - once sync.Once - concurrencyLim *concurrencyLimiter - maxConcurrentRequests = 4 -) - -func generateConcurrencyLimiter(capacity int) *concurrencyLimiter { - if capacity < 1 || capacity > maxConcurrentRequests { - capacity = maxConcurrentRequests - } - - return &concurrencyLimiter{ - semaphore: make(chan struct{}, capacity), - } -} - -func InitializeConcurrencyLimiter(capacity int) { - once.Do(func() { - concurrencyLim = generateConcurrencyLimiter(capacity) - }) -} - -func (cl *concurrencyLimiter) Intercept( - pipeline khttp.Pipeline, - middlewareIndex int, - req *http.Request, -) (*http.Response, error) { - if cl == nil || cl.semaphore == nil { - return nil, clues.New("nil concurrency limiter") - } - - cl.semaphore <- struct{}{} - defer func() { - <-cl.semaphore - }() - - return pipeline.Next(req, middlewareIndex) -} diff --git a/src/internal/connector/graph/concurrency_middleware.go b/src/internal/connector/graph/concurrency_middleware.go new file mode 100644 index 000000000..2756a60c6 --- /dev/null +++ b/src/internal/connector/graph/concurrency_middleware.go @@ -0,0 +1,202 @@ +package graph + +import ( + "context" + "net/http" + "sync" + + "github.com/alcionai/clues" + khttp "github.com/microsoft/kiota-http-go" + "golang.org/x/time/rate" + + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// Concurrency Limiter +// "how many calls at one time" +// --------------------------------------------------------------------------- + +// concurrencyLimiter middleware limits the number of concurrent requests to graph API +type concurrencyLimiter struct { + semaphore chan struct{} +} + +var ( + once sync.Once + concurrencyLim *concurrencyLimiter + maxConcurrentRequests = 4 +) + +func generateConcurrencyLimiter(capacity int) *concurrencyLimiter { + if capacity < 1 || capacity > maxConcurrentRequests { + capacity = maxConcurrentRequests + } + + return &concurrencyLimiter{ + semaphore: make(chan struct{}, capacity), + } +} + +func InitializeConcurrencyLimiter(capacity int) { + once.Do(func() { + concurrencyLim = generateConcurrencyLimiter(capacity) + }) +} + +func (cl *concurrencyLimiter) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + if cl == nil || cl.semaphore == nil { + return nil, clues.New("nil concurrency limiter") + } + + cl.semaphore <- struct{}{} + defer func() { + <-cl.semaphore + }() + + return pipeline.Next(req, middlewareIndex) +} + +//nolint:lll +// --------------------------------------------------------------------------- +// Rate Limiter +// "how many calls in a minute" +// https://learn.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online +// --------------------------------------------------------------------------- + +const ( + // Default goal is to keep calls below the 10k-per-10-minute threshold. + // 14 tokens every second nets 840 per minute. That's 8400 every 10 minutes, + // which is a bit below the mark. + // But suppose we have a minute-long dry spell followed by a 10 minute tsunami. + // We'll have built up 750 tokens in reserve, so the first 750 calls go through + // immediately. Over the next 10 minutes, we'll partition out the other calls + // at a rate of 840-per-minute, ending at a total of 9150. Theoretically, if + // the volume keeps up after that, we'll always stay between 8400 and 9150 out + // of 10k. Worst case scenario, we have an extra minute of padding to allow + // up to 9990. + defaultPerSecond = 14 // 14 * 60 = 840 + defaultMaxCap = 750 // real cap is 10k-per-10-minutes + // since drive runs on a per-minute, rather than per-10-minute bucket, we have + // to keep the max cap equal to the per-second cap. A large maxCap pool (say, + // 1200, similar to the per-minute cap) would allow us to make a flood of 2400 + // calls in the first minute, putting us over the per-minute limit. Keeping + // the cap at the per-second burst means we only dole out a max of 1240 in one + // minute (20 cap + 1200 per minute + one burst of padding). + drivePerSecond = 20 // 20 * 60 = 1200 + driveMaxCap = 20 // real cap is 1250-per-minute +) + +var ( + driveLimiter = rate.NewLimiter(drivePerSecond, driveMaxCap) + // also used as the exchange service limiter + defaultLimiter = rate.NewLimiter(defaultPerSecond, defaultMaxCap) +) + +type LimiterCfg struct { + Service path.ServiceType +} + +type limiterCfgKey string + +const limiterCfgCtxKey limiterCfgKey = "corsoGaphRateLimiterCfg" + +func BindRateLimiterConfig(ctx context.Context, lc LimiterCfg) context.Context { + return context.WithValue(ctx, limiterCfgCtxKey, lc) +} + +func ctxLimiter(ctx context.Context) *rate.Limiter { + lc, ok := extractRateLimiterConfig(ctx) + if !ok { + return defaultLimiter + } + + switch lc.Service { + case path.OneDriveService, path.SharePointService: + return driveLimiter + default: + return defaultLimiter + } +} + +func extractRateLimiterConfig(ctx context.Context) (LimiterCfg, bool) { + l := ctx.Value(limiterCfgCtxKey) + if l == nil { + return LimiterCfg{}, false + } + + lc, ok := l.(LimiterCfg) + + return lc, ok +} + +type limiterConsumptionKey string + +const limiterConsumptionCtxKey limiterConsumptionKey = "corsoGraphRateLimiterConsumption" + +const ( + defaultLC = 1 + driveDefaultLC = 2 + // limit consumption rate for single-item GETs requests, + // or delta-based multi-item GETs. + SingleGetOrDeltaLC = 1 + // limit consumption rate for anything permissions related + PermissionsLC = 5 +) + +// ConsumeNTokens ensures any calls using this context will consume +// n rate-limiter tokens. Default is 1, and this value does not need +// to be established in the context to consume the default tokens. +// This should only get used on a per-call basis, to avoid cross-pollination. +func ConsumeNTokens(ctx context.Context, n int) context.Context { + return context.WithValue(ctx, limiterConsumptionCtxKey, n) +} + +func ctxLimiterConsumption(ctx context.Context, defaultConsumption int) int { + l := ctx.Value(limiterConsumptionCtxKey) + if l == nil { + return defaultConsumption + } + + lc, ok := l.(int) + if !ok || lc < 1 { + return defaultConsumption + } + + return lc +} + +// QueueRequest will allow the request to occur immediately if we're under the +// calls-per-minute rate. Otherwise, the call will wait in a queue until +// the next token set is available. +func QueueRequest(ctx context.Context) { + limiter := ctxLimiter(ctx) + defaultConsumed := defaultLC + + if limiter == driveLimiter { + defaultConsumed = driveDefaultLC + } + + consume := ctxLimiterConsumption(ctx, defaultConsumed) + + if err := limiter.WaitN(ctx, consume); err != nil { + logger.CtxErr(ctx, err).Error("graph middleware waiting on the limiter") + } +} + +// RateLimiterMiddleware is used to ensure we don't overstep per-min request limits. +type RateLimiterMiddleware struct{} + +func (mw *RateLimiterMiddleware) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + QueueRequest(req.Context()) + return pipeline.Next(req, middlewareIndex) +} diff --git a/src/internal/connector/graph/concurrency_limiter_test.go b/src/internal/connector/graph/concurrency_middleware_test.go similarity index 100% rename from src/internal/connector/graph/concurrency_limiter_test.go rename to src/internal/connector/graph/concurrency_middleware_test.go diff --git a/src/internal/connector/graph/http_wrapper.go b/src/internal/connector/graph/http_wrapper.go index bc469c5f2..b0bca76e2 100644 --- a/src/internal/connector/graph/http_wrapper.go +++ b/src/internal/connector/graph/http_wrapper.go @@ -140,13 +140,20 @@ func defaultTransport() http.RoundTripper { } func internalMiddleware(cc *clientConfig) []khttp.Middleware { - return []khttp.Middleware{ + mw := []khttp.Middleware{ &RetryMiddleware{ MaxRetries: cc.maxRetries, Delay: cc.minDelay, }, + khttp.NewRedirectHandler(), &LoggingMiddleware{}, - &ThrottleControlMiddleware{}, + &RateLimiterMiddleware{}, &MetricsMiddleware{}, } + + if len(cc.appendMiddleware) > 0 { + mw = append(mw, cc.appendMiddleware...) + } + + return mw } diff --git a/src/internal/connector/graph/http_wrapper_test.go b/src/internal/connector/graph/http_wrapper_test.go index 483a5f0ba..40abea977 100644 --- a/src/internal/connector/graph/http_wrapper_test.go +++ b/src/internal/connector/graph/http_wrapper_test.go @@ -2,9 +2,11 @@ package graph import ( "net/http" + "strings" "testing" "github.com/alcionai/clues" + khttp "github.com/microsoft/kiota-http-go" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -43,3 +45,72 @@ func (suite *HTTPWrapperIntgSuite) TestNewHTTPWrapper() { require.NotNil(t, resp) require.Equal(t, http.StatusOK, resp.StatusCode) } + +type mwForceResp struct { + err error + resp *http.Response + alternate func(*http.Request) (bool, *http.Response, error) +} + +func (mw *mwForceResp) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + ok, r, e := mw.alternate(req) + if ok { + return r, e + } + + return mw.resp, mw.err +} + +type HTTPWrapperUnitSuite struct { + tester.Suite +} + +func TestHTTPWrapperUnitSuite(t *testing.T) { + suite.Run(t, &HTTPWrapperUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() { + ctx, flush := tester.NewContext() + defer flush() + + var ( + t = suite.T() + uri = "https://graph.microsoft.com" + path = "/fnords/beaux/regard" + url = uri + path + ) + + // can't use gock for this, or else it'll short-circuit the transport, + // and thus skip all the middleware + hdr := http.Header{} + hdr.Set("Location", "localhost:99999999/smarfs") + + toResp := &http.Response{ + StatusCode: 302, + Header: hdr, + } + + mwResp := mwForceResp{ + resp: toResp, + alternate: func(req *http.Request) (bool, *http.Response, error) { + if strings.HasSuffix(req.URL.String(), "smarfs") { + return true, &http.Response{StatusCode: http.StatusOK}, nil + } + + return false, nil, nil + }, + } + + hw := NewHTTPWrapper(appendMiddleware(&mwResp)) + + resp, err := hw.Request(ctx, http.MethodGet, url, nil, nil) + + require.NoError(t, err, clues.ToCore(err)) + require.NotNil(t, resp) + // require.Equal(t, 1, calledCorrectly, "test server was called with expected path") + require.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/src/internal/connector/graph/middleware.go b/src/internal/connector/graph/middleware.go index b1d4ad99f..108f03cac 100644 --- a/src/internal/connector/graph/middleware.go +++ b/src/internal/connector/graph/middleware.go @@ -2,7 +2,6 @@ package graph import ( "context" - "fmt" "io" "net/http" "net/http/httputil" @@ -15,7 +14,6 @@ import ( backoff "github.com/cenkalti/backoff/v4" khttp "github.com/microsoft/kiota-http-go" "golang.org/x/exp/slices" - "golang.org/x/time/rate" "github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/events" @@ -100,6 +98,9 @@ func LoggableURL(url string) pii.SafeURL { } } +// 1 MB +const logMBLimit = 1 * 1048576 + func (mw *LoggingMiddleware) Intercept( pipeline khttp.Pipeline, middlewareIndex int, @@ -122,42 +123,61 @@ func (mw *LoggingMiddleware) Intercept( return resp, err } - ctx = clues.Add(ctx, "status", resp.Status, "statusCode", resp.StatusCode) - log := logger.Ctx(ctx) + ctx = clues.Add( + ctx, + "status", resp.Status, + "statusCode", resp.StatusCode, + "content_len", resp.ContentLength) - // Return immediately if the response is good (2xx). - // If api logging is toggled, log a body-less dump of the request/resp. - if (resp.StatusCode / 100) == 2 { - if logger.DebugAPIFV || os.Getenv(log2xxGraphRequestsEnvKey) != "" { - log.Debugw("2xx graph api resp", "response", getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "")) - } + var ( + log = logger.Ctx(ctx) + respClass = resp.StatusCode / 100 + logExtra = logger.DebugAPIFV || os.Getenv(logGraphRequestsEnvKey) != "" + ) - return resp, err - } - - // Log errors according to api debugging configurations. - // When debugging is toggled, every non-2xx is recorded with a response dump. - // Otherwise, throttling cases and other non-2xx responses are logged - // with a slimmer reference for telemetry/supportability purposes. - if logger.DebugAPIFV || os.Getenv(logGraphRequestsEnvKey) != "" { - log.Errorw("non-2xx graph api response", "response", getRespDump(ctx, resp, true)) - return resp, err - } - - msg := fmt.Sprintf("graph api error: %s", resp.Status) - - // special case for supportability: log all throttling cases. + // special case: always info log 429 responses if resp.StatusCode == http.StatusTooManyRequests { - log = log.With( + log.Infow( + "graph api throttling", "limit", resp.Header.Get(rateLimitHeader), "remaining", resp.Header.Get(rateRemainingHeader), "reset", resp.Header.Get(rateResetHeader), "retry-after", resp.Header.Get(retryAfterHeader)) - } else if resp.StatusCode/100 == 4 || resp.StatusCode == http.StatusServiceUnavailable { - log = log.With("response", getRespDump(ctx, resp, true)) + + return resp, err } - log.Info(msg) + // special case: always dump status-400-bad-request + if resp.StatusCode == http.StatusBadRequest { + log.With("response", getRespDump(ctx, resp, true)). + Error("graph api error: " + resp.Status) + + return resp, err + } + + // Log api calls according to api debugging configurations. + switch respClass { + case 2: + if logExtra { + // only dump the body if it's under a size limit. We don't want to copy gigs into memory for a log. + dump := getRespDump(ctx, resp, os.Getenv(log2xxGraphResponseEnvKey) != "" && resp.ContentLength < logMBLimit) + log.Infow("2xx graph api resp", "response", dump) + } + case 3: + log.With("redirect_location", LoggableURL(resp.Header.Get(locationHeader))) + + if logExtra { + log.With("response", getRespDump(ctx, resp, false)) + } + + log.Info("graph api redirect: " + resp.Status) + default: + if logExtra { + log.With("response", getRespDump(ctx, resp, true)) + } + + log.Error("graph api error: " + resp.Status) + } return resp, err } @@ -359,50 +379,6 @@ func (mw RetryMiddleware) getRetryDelay( return exponentialBackoff.NextBackOff() } -// We're trying to keep calls below the 10k-per-10-minute threshold. -// 15 tokens every second nets 900 per minute. That's 9000 every 10 minutes, -// which is a bit below the mark. -// But suppose we have a minute-long dry spell followed by a 10 minute tsunami. -// We'll have built up 900 tokens in reserve, so the first 900 calls go through -// immediately. Over the next 10 minutes, we'll partition out the other calls -// at a rate of 900-per-minute, ending at a total of 9900. Theoretically, if -// the volume keeps up after that, we'll always stay between 9000 and 9900 out -// of 10k. -const ( - perSecond = 15 - maxCap = 900 -) - -// Single, global rate limiter at this time. Refinements for method (creates, -// versus reads) or service can come later. -var limiter = rate.NewLimiter(perSecond, maxCap) - -// QueueRequest will allow the request to occur immediately if we're under the -// 1k-calls-per-minute rate. Otherwise, the call will wait in a queue until -// the next token set is available. -func QueueRequest(ctx context.Context) { - if err := limiter.Wait(ctx); err != nil { - logger.CtxErr(ctx, err).Error("graph middleware waiting on the limiter") - } -} - -// --------------------------------------------------------------------------- -// Rate Limiting -// --------------------------------------------------------------------------- - -// ThrottleControlMiddleware is used to ensure we don't overstep 10k-per-10-min -// request limits. -type ThrottleControlMiddleware struct{} - -func (mw *ThrottleControlMiddleware) Intercept( - pipeline khttp.Pipeline, - middlewareIndex int, - req *http.Request, -) (*http.Response, error) { - QueueRequest(req.Context()) - return pipeline.Next(req, middlewareIndex) -} - // --------------------------------------------------------------------------- // Metrics // --------------------------------------------------------------------------- @@ -410,6 +386,8 @@ func (mw *ThrottleControlMiddleware) Intercept( // MetricsMiddleware aggregates per-request metrics on the events bus type MetricsMiddleware struct{} +const xmruHeader = "x-ms-resource-unit" + func (mw *MetricsMiddleware) Intercept( pipeline khttp.Pipeline, middlewareIndex int, @@ -430,5 +408,23 @@ func (mw *MetricsMiddleware) Intercept( events.Since(start, events.APICall) events.Since(start, events.APICall, status) + // track the graph "resource cost" for each call (if not provided, assume 1) + + // nil-pointer guard + if len(resp.Header) == 0 { + resp.Header = http.Header{} + } + + // from msoft throttling documentation: + // x-ms-resource-unit - Indicates the resource unit used for this request. Values are positive integer + xmru := resp.Header.Get(xmruHeader) + xmrui, e := strconv.Atoi(xmru) + + if len(xmru) == 0 || e != nil { + xmrui = 1 + } + + events.IncN(xmrui, events.APICall, xmruHeader) + return resp, err } diff --git a/src/internal/connector/graph/middleware_test.go b/src/internal/connector/graph/middleware_test.go index 0874a38f6..3aa77778c 100644 --- a/src/internal/connector/graph/middleware_test.go +++ b/src/internal/connector/graph/middleware_test.go @@ -17,10 +17,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "golang.org/x/time/rate" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/path" ) type mwReturns struct { @@ -132,9 +134,9 @@ func (suite *RetryMWIntgSuite) SetupSuite() { func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() { var ( - uri = "https://graph.microsoft.com" - path = "/v1.0/users/user/messages/foo" - url = uri + path + uri = "https://graph.microsoft.com" + urlPath = "/v1.0/users/user/messages/foo" + url = uri + urlPath ) tests := []struct { @@ -230,3 +232,107 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_RetryRequest_resetBodyAfter50 Post(ctx, body, nil) require.NoError(t, err, clues.ToCore(err)) } + +type MiddlewareUnitSuite struct { + tester.Suite +} + +func TestMiddlewareUnitSuite(t *testing.T) { + suite.Run(t, &MiddlewareUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *MiddlewareUnitSuite) TestBindExtractLimiterConfig() { + ctx, flush := tester.NewContext() + defer flush() + + // an unpopulated ctx should produce the default limiter + assert.Equal(suite.T(), defaultLimiter, ctxLimiter(ctx)) + + table := []struct { + name string + service path.ServiceType + expectOK require.BoolAssertionFunc + expectLimiter *rate.Limiter + }{ + { + name: "exchange", + service: path.ExchangeService, + expectLimiter: defaultLimiter, + }, + { + name: "oneDrive", + service: path.OneDriveService, + expectLimiter: driveLimiter, + }, + { + name: "sharePoint", + service: path.SharePointService, + expectLimiter: driveLimiter, + }, + { + name: "unknownService", + service: path.UnknownService, + expectLimiter: defaultLimiter, + }, + { + name: "badService", + service: path.ServiceType(-1), + expectLimiter: defaultLimiter, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + tctx := BindRateLimiterConfig(ctx, LimiterCfg{Service: test.service}) + lc, ok := extractRateLimiterConfig(tctx) + require.True(t, ok, "found rate limiter in ctx") + assert.Equal(t, test.service, lc.Service) + assert.Equal(t, test.expectLimiter, ctxLimiter(tctx)) + }) + } +} + +func (suite *MiddlewareUnitSuite) TestLimiterConsumption() { + ctx, flush := tester.NewContext() + defer flush() + + // an unpopulated ctx should produce the default consumption + assert.Equal(suite.T(), defaultLC, ctxLimiterConsumption(ctx, defaultLC)) + + table := []struct { + name string + n int + expect int + }{ + { + name: "matches default", + n: defaultLC, + expect: defaultLC, + }, + { + name: "default+1", + n: defaultLC + 1, + expect: defaultLC + 1, + }, + { + name: "zero", + n: 0, + expect: defaultLC, + }, + { + name: "negative", + n: -1, + expect: defaultLC, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + tctx := ConsumeNTokens(ctx, test.n) + lc := ctxLimiterConsumption(tctx, defaultLC) + assert.Equal(t, test.expect, lc) + }) + } +} diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 9db7fb825..dc5129ac4 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -21,13 +21,14 @@ const ( logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS" log2xxGraphRequestsEnvKey = "LOG_2XX_GRAPH_REQUESTS" log2xxGraphResponseEnvKey = "LOG_2XX_GRAPH_RESPONSES" - retryAttemptHeader = "Retry-Attempt" - retryAfterHeader = "Retry-After" defaultMaxRetries = 3 defaultDelay = 3 * time.Second + locationHeader = "Location" rateLimitHeader = "RateLimit-Limit" rateRemainingHeader = "RateLimit-Remaining" rateResetHeader = "RateLimit-Reset" + retryAfterHeader = "Retry-After" + retryAttemptHeader = "Retry-Attempt" defaultHTTPClientTimeout = 1 * time.Hour ) @@ -173,6 +174,8 @@ type clientConfig struct { // The minimum delay in seconds between retries minDelay time.Duration overrideRetryCount bool + + appendMiddleware []khttp.Middleware } type Option func(*clientConfig) @@ -225,6 +228,14 @@ func MinimumBackoff(dur time.Duration) Option { } } +func appendMiddleware(mw ...khttp.Middleware) Option { + return func(c *clientConfig) { + if len(mw) > 0 { + c.appendMiddleware = mw + } + } +} + // --------------------------------------------------------------------------- // Middleware Control // --------------------------------------------------------------------------- @@ -253,9 +264,13 @@ func kiotaMiddlewares( khttp.NewParametersNameDecodingHandler(), khttp.NewUserAgentHandler(), &LoggingMiddleware{}, - &ThrottleControlMiddleware{}, + &RateLimiterMiddleware{}, &MetricsMiddleware{}, }...) + if len(cc.appendMiddleware) > 0 { + mw = append(mw, cc.appendMiddleware...) + } + return mw } diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index b95f75335..23a6ab1dc 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" ) // --------------------------------------------------------------- @@ -82,19 +83,19 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices checkError: assert.NoError, excludes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Exclude(sel.Folders(selectors.Any())) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, filters: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Filter(sel.Folders(selectors.Any())) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, includes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"}) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) sel.DiscreteOwner = "elliotReid@someHospital.org" return sel.Selector }, @@ -104,17 +105,17 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices checkError: assert.NoError, excludes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Exclude(sel.Folders(selectors.Any())) + sel.Exclude(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, filters: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Filter(sel.Folders(selectors.Any())) + sel.Filter(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, includes: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"}) - sel.Include(sel.Folders(selectors.Any())) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) return sel.Selector }, }, diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 8f33b0247..58d838529 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -866,7 +866,36 @@ func checkHasCollections( } for _, g := range got { - gotNames = append(gotNames, g.FullPath().String()) + // TODO(ashmrtn): Remove when LocationPath is made part of BackupCollection + // interface. + if !assert.Implements(t, (*data.LocationPather)(nil), g) { + continue + } + + fp := g.FullPath() + loc := g.(data.LocationPather).LocationPath() + + if fp.Service() == path.OneDriveService || + (fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) { + dp, err := path.ToDrivePath(fp) + if !assert.NoError(t, err, clues.ToCore(err)) { + continue + } + + loc = path.BuildDriveLocation(dp.DriveID, loc.Elements()...) + } + + p, err := loc.ToDataLayerPath( + fp.Tenant(), + fp.ResourceOwner(), + fp.Service(), + fp.Category(), + false) + if !assert.NoError(t, err, clues.ToCore(err)) { + continue + } + + gotNames = append(gotNames, p.String()) } assert.ElementsMatch(t, expectedNames, gotNames, "returned collections") @@ -1047,7 +1076,7 @@ func makeSharePointBackupSel( } // backupSelectorForExpected creates a selector that can be used to backup the -// given Items in expected based on the item paths. Fails the test if items from +// given dests based on the item paths. Fails the test if items from // multiple services are in expected. func backupSelectorForExpected( t *testing.T, diff --git a/src/internal/connector/graph_connector_onedrive_test.go b/src/internal/connector/graph_connector_onedrive_test.go index 99e0fbbe1..683de3eaf 100644 --- a/src/internal/connector/graph_connector_onedrive_test.go +++ b/src/internal/connector/graph_connector_onedrive_test.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -351,35 +352,35 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( suite.BackupResourceOwner()) rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } subfolderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, } subfolderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, folderAName, } folderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderBName, } @@ -496,34 +497,34 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { folderCName := "folder-c" rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } folderBPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderBName, } // For skipped test // subfolderAPath := []string{ - // "drives", + // odConsts.DrivesPathDir, // driveID, - // rootFolder, + // odConsts.RootPathDir, // folderBName, // folderAName, // } folderCPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderCName, } @@ -707,9 +708,9 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { inputCols := []OnedriveColInfo{ { PathElements: []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, }, Files: []ItemData{ { @@ -728,9 +729,9 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { expectedCols := []OnedriveColInfo{ { PathElements: []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, }, Files: []ItemData{ { @@ -793,34 +794,34 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio folderCName := "empty" rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, } subfolderAAPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderAName, } subfolderABPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderBName, } subfolderACPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderAName, folderCName, } @@ -966,20 +967,20 @@ func testRestoreFolderNamedFolderRegression( suite.BackupResourceOwner()) rootPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, } folderFolderPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderNamedFolder, } subfolderPath := []string{ - "drives", + odConsts.DrivesPathDir, driveID, - rootFolder, + odConsts.RootPathDir, folderNamedFolder, folderBName, } diff --git a/src/internal/connector/graph_connector_onedrive_test_helper.go b/src/internal/connector/graph_connector_onedrive_test_helper.go index 26d71bfe8..f93cfd1a1 100644 --- a/src/internal/connector/graph_connector_onedrive_test_helper.go +++ b/src/internal/connector/graph_connector_onedrive_test_helper.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" + + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" ) // For any version post this(inclusive), we expect to be using IDs for @@ -230,7 +232,7 @@ func (c *onedriveCollection) withPermissions(perm PermData) *onedriveCollection metaName = "" } - if name == rootFolder { + if name == odConsts.RootPathDir { return c } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 7fcacf255..62840153d 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -294,7 +294,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() { var ( t = suite.T() acct = tester.NewM365Account(t) - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") sel = selectors.Selector{ Service: selectors.ServiceUnknown, } @@ -322,7 +322,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() { } func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") table := []struct { name string col []data.RestoreCollection @@ -542,7 +542,7 @@ func runRestoreBackupTest( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - Dest: tester.DefaultTestRestoreDestination(), + Dest: tester.DefaultTestRestoreDestination(""), } totalItems, totalKopiaItems, collections, expectedData := GetCollectionsAndExpected( @@ -588,7 +588,7 @@ func runRestoreTestWithVerion( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - Dest: tester.DefaultTestRestoreDestination(), + Dest: tester.DefaultTestRestoreDestination(""), } totalItems, _, collections, _ := GetCollectionsAndExpected( @@ -627,7 +627,7 @@ func runRestoreBackupTestVersions( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - Dest: tester.DefaultTestRestoreDestination(), + Dest: tester.DefaultTestRestoreDestination(""), } totalItems, _, collections, _ := GetCollectionsAndExpected( @@ -1005,7 +1005,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames for i, collection := range test.collections { // Get a dest per collection so they're independent. - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") expectedDests = append(expectedDests, destAndCats{ resourceOwner: suite.user, dest: dest.ContainerName, @@ -1177,9 +1177,7 @@ func (suite *GraphConnectorIntegrationSuite) TestBackup_CreatesPrefixCollections resource: Users, selectorFunc: func(t *testing.T) selectors.Selector { sel := selectors.NewOneDriveBackup([]string{suite.user}) - sel.Include( - sel.Folders([]string{selectors.NoneTgt}), - ) + sel.Include(sel.Folders([]string{selectors.NoneTgt})) return sel.Selector }, diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 3b2674553..d87546830 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -336,18 +336,33 @@ func GetItemPermission( return perm, nil } -func GetDriveByID( +func GetUsersDrive( ctx context.Context, srv graph.Servicer, - userID string, + user string, ) (models.Driveable, error) { - //revive:enable:context-as-argument d, err := srv.Client(). - UsersById(userID). + UsersById(user). Drive(). Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting drive") + return nil, graph.Wrap(ctx, err, "getting user's drive") + } + + return d, nil +} + +func GetSitesDefaultDrive( + ctx context.Context, + srv graph.Servicer, + site string, +) (models.Driveable, error) { + d, err := srv.Client(). + SitesById(site). + Drive(). + Get(ctx, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "getting site's drive") } return d, nil @@ -358,7 +373,10 @@ func GetDriveRoot( srv graph.Servicer, driveID string, ) (models.DriveItemable, error) { - root, err := srv.Client().DrivesById(driveID).Root().Get(ctx, nil) + root, err := srv.Client(). + DrivesById(driveID). + Root(). + Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting drive root") } diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index a4caafae2..26fd41283 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -131,7 +131,7 @@ func pathToLocation(p path.Path) (*path.Builder, error) { // NewCollection creates a Collection func NewCollection( itemClient graph.Requester, - folderPath path.Path, + currPath path.Path, prevPath path.Path, driveID string, service graph.Servicer, @@ -145,9 +145,9 @@ func NewCollection( // to be changed as we won't be able to extract path information from the // storage path. In that case, we'll need to start storing the location paths // like we do the previous path. - locPath, err := pathToLocation(folderPath) + locPath, err := pathToLocation(currPath) if err != nil { - return nil, clues.Wrap(err, "getting location").With("folder_path", folderPath.String()) + return nil, clues.Wrap(err, "getting location").With("curr_path", currPath.String()) } prevLocPath, err := pathToLocation(prevPath) @@ -157,7 +157,7 @@ func NewCollection( c := newColl( itemClient, - folderPath, + currPath, prevPath, driveID, service, @@ -175,7 +175,7 @@ func NewCollection( func newColl( gr graph.Requester, - folderPath path.Path, + currPath path.Path, prevPath path.Path, driveID string, service graph.Servicer, @@ -188,7 +188,7 @@ func newColl( c := &Collection{ itemClient: gr, itemGetter: api.GetDriveItem, - folderPath: folderPath, + folderPath: currPath, prevPath: prevPath, driveItems: map[string]models.DriveItemable{}, driveID: driveID, @@ -197,7 +197,7 @@ func newColl( data: make(chan data.Stream, graph.Parallelism(path.OneDriveMetadataService).CollectionBufferSize()), statusUpdater: statusUpdater, ctrl: ctrlOpts, - state: data.StateOf(prevPath, folderPath), + state: data.StateOf(prevPath, currPath), scope: colScope, doNotMergeItems: doNotMergeItems, } diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 8594e4a6f..52f29f879 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -101,6 +101,7 @@ type Collections struct { servicer graph.Servicer, driveID, link string, ) itemPager + servicePathPfxFunc pathPrefixerFunc // Track stats from drive enumeration. Represents the items backed up. NumItems int @@ -119,17 +120,18 @@ func NewCollections( ctrlOpts control.Options, ) *Collections { return &Collections{ - itemClient: itemClient, - tenant: tenant, - resourceOwner: resourceOwner, - source: source, - matcher: matcher, - CollectionMap: map[string]map[string]*Collection{}, - drivePagerFunc: PagerForSource, - itemPagerFunc: defaultItemPager, - service: service, - statusUpdater: statusUpdater, - ctrl: ctrlOpts, + itemClient: itemClient, + tenant: tenant, + resourceOwner: resourceOwner, + source: source, + matcher: matcher, + CollectionMap: map[string]map[string]*Collection{}, + drivePagerFunc: PagerForSource, + itemPagerFunc: defaultItemPager, + servicePathPfxFunc: pathPrefixerForSource(tenant, resourceOwner, source), + service: service, + statusUpdater: statusUpdater, + ctrl: ctrlOpts, } } @@ -280,6 +282,12 @@ func (c *Collections) Get( return nil, err } + driveTombstones := map[string]struct{}{} + + for driveID := range oldPathsByDriveID { + driveTombstones[driveID] = struct{}{} + } + driveComplete, closer := observe.MessageWithCompletion(ctx, observe.Bulletf("files")) defer closer() defer close(driveComplete) @@ -312,6 +320,8 @@ func (c *Collections) Get( ictx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) ) + delete(driveTombstones, driveID) + if _, ok := c.CollectionMap[driveID]; !ok { c.CollectionMap[driveID] = map[string]*Collection{} } @@ -408,7 +418,7 @@ func (c *Collections) Get( col, err := NewCollection( c.itemClient, - nil, + nil, // delete the folder prevPath, driveID, c.service, @@ -427,15 +437,41 @@ func (c *Collections) Get( observe.Message(ctx, fmt.Sprintf("Discovered %d items to backup", c.NumItems)) - // Add an extra for the metadata collection. collections := []data.BackupCollection{} + // add all the drives we found for _, driveColls := range c.CollectionMap { for _, coll := range driveColls { collections = append(collections, coll) } } + // generate tombstones for drives that were removed. + for driveID := range driveTombstones { + prevDrivePath, err := c.servicePathPfxFunc(driveID) + if err != nil { + return nil, clues.Wrap(err, "making drive tombstone previous path").WithClues(ctx) + } + + coll, err := NewCollection( + c.itemClient, + nil, // delete the drive + prevDrivePath, + driveID, + c.service, + c.statusUpdater, + c.source, + c.ctrl, + CollectionScopeUnknown, + true) + if err != nil { + return nil, clues.Wrap(err, "making drive tombstone").WithClues(ctx) + } + + collections = append(collections, coll) + } + + // add metadata collections service, category := c.source.toPathServiceCat() md, err := graph.MakeMetadataCollection( c.tenant, @@ -457,7 +493,6 @@ func (c *Collections) Get( collections = append(collections, md) } - // TODO(ashmrtn): Track and return the set of items to exclude. return collections, nil } @@ -642,7 +677,7 @@ func (c *Collections) getCollectionPath( return nil, clues.New("folder with empty name") } - collectionPath, err = collectionPath.Append(name, false) + collectionPath, err = collectionPath.Append(false, name) if err != nil { return nil, clues.Wrap(err, "making non-root folder path") } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 1baaed521..e55bf2db8 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -1246,16 +1246,15 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { user, path.OneDriveService, path.FilesCategory, - false, - ) + false) require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err)) - driveID1 := uuid.NewString() + driveID1 := "drive-1-" + uuid.NewString() drive1 := models.NewDrive() drive1.SetId(&driveID1) drive1.SetName(&driveID1) - driveID2 := uuid.NewString() + driveID2 := "drive-2-" + uuid.NewString() drive2 := models.NewDrive() drive2.SetId(&driveID2) drive2.SetName(&driveID2) @@ -1287,7 +1286,8 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedFolderPaths map[string]map[string]string expectedDelList *pmMock.PrefixMap expectedSkippedCount int - doNotMergeItems bool + // map full or previous path (prefers full) -> bool + doNotMergeItems map[string]bool }{ { name: "OneDrive_OneItemPage_DelFileOnly_NoFolders_NoErrors", @@ -1321,7 +1321,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }), }, { - name: "OneDrive_OneItemPage_NoFolders_NoErrors", + name: "OneDrive_OneItemPage_NoFolderDeltas_NoErrors", drives: []models.Driveable{drive1}, items: map[string][]deltaPagerResult{ driveID1: { @@ -1699,7 +1699,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + }, }, { name: "OneDrive_TwoItemPage_DeltaError", @@ -1741,7 +1743,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "OneDrive_TwoItemPage_NoDeltaError", @@ -1785,7 +1790,7 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{ rootFolderPath1: getDelList("file", "file2"), }), - doNotMergeItems: false, + doNotMergeItems: map[string]bool{}, }, { name: "OneDrive_OneItemPage_InvalidPrevDelta_DeleteNonExistentFolder", @@ -1827,7 +1832,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + expectedPath1("/folder2"): true, + }, }, { name: "OneDrive_OneItemPage_InvalidPrevDelta_AnotherFolderAtDeletedLocation", @@ -1873,7 +1882,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "OneDrive Two Item Pages with Malware", @@ -1973,7 +1985,11 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + expectedPath1("/folder2"): true, + }, }, { name: "One Drive Delta Error Random Folder Delete", @@ -2012,7 +2028,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + folderPath1: true, + }, }, { name: "One Drive Delta Error Random Item Delete", @@ -2049,7 +2068,9 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { }, }, expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), - doNotMergeItems: true, + doNotMergeItems: map[string]bool{ + rootFolderPath1: true, + }, }, { name: "One Drive Folder Made And Deleted", @@ -2200,6 +2221,37 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { rootFolderPath1: getDelList("file"), }), }, + { + name: "TwoPriorDrives_OneTombstoned", + drives: []models.Driveable{drive1}, + items: map[string][]deltaPagerResult{ + driveID1: { + { + items: []models.DriveItemable{ + driveRootItem("root"), // will be present + }, + deltaLink: &delta, + }, + }, + }, + errCheck: assert.NoError, + prevFolderPaths: map[string]map[string]string{ + driveID1: {"root": rootFolderPath1}, + driveID2: {"root": rootFolderPath2}, + }, + expectedCollections: map[string]map[data.CollectionState][]string{ + rootFolderPath1: {data.NotMovedState: {}}, + rootFolderPath2: {data.DeletedState: {}}, + }, + expectedDeltaURLs: map[string]string{driveID1: delta}, + expectedFolderPaths: map[string]map[string]string{ + driveID1: {"root": rootFolderPath1}, + }, + expectedDelList: pmMock.NewPrefixMap(map[string]map[string]struct{}{}), + doNotMergeItems: map[string]bool{ + rootFolderPath2: true, + }, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -2257,12 +2309,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { map[string]string{ driveID1: prevDelta, driveID2: prevDelta, - }, - ), + }), graph.NewMetadataEntry( graph.PreviousPathFileName, - test.prevFolderPaths, - ), + test.prevFolderPaths), }, func(*support.ConnectorOperationStatus) {}, ) @@ -2329,18 +2379,24 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() { "state: %d, path: %s", baseCol.State(), folderPath) - assert.Equal(t, test.doNotMergeItems, baseCol.DoNotMergeItems(), "DoNotMergeItems") + + p := baseCol.FullPath() + if p == nil { + p = baseCol.PreviousPath() + } + + assert.Equalf( + t, + test.doNotMergeItems[p.String()], + baseCol.DoNotMergeItems(), + "DoNotMergeItems in collection: %s", p) } expectedCollectionCount := 0 - for c := range test.expectedCollections { - for range test.expectedCollections[c] { - expectedCollectionCount++ - } + for _, ec := range test.expectedCollections { + expectedCollectionCount += len(ec) } - // This check is necessary to make sure we are all the - // collections we expect it to assert.Equal(t, expectedCollectionCount, collectionCount, "number of collections") test.expectedDelList.AssertEqual(t, delList) diff --git a/src/internal/connector/onedrive/consts/consts.go b/src/internal/connector/onedrive/consts/consts.go new file mode 100644 index 000000000..662354ad6 --- /dev/null +++ b/src/internal/connector/onedrive/consts/consts.go @@ -0,0 +1,10 @@ +package onedrive + +const ( + // const used as the root dir for the drive portion of a path prefix. + // eg: tid/onedrive/ro/files/drives/driveid/... + DrivesPathDir = "drives" + // const used as the root-of-drive dir for the drive portion of a path prefix. + // eg: tid/onedrive/ro/files/drives/driveid/root:/... + RootPathDir = "root:" +) diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 99487c66b..27bf2091c 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -14,8 +14,10 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" ) const ( @@ -55,6 +57,25 @@ func PagerForSource( } } +type pathPrefixerFunc func(driveID string) (path.Path, error) + +func pathPrefixerForSource( + tenantID, resourceOwner string, + source driveSource, +) pathPrefixerFunc { + cat := path.FilesCategory + serv := path.OneDriveService + + if source == SharePointSource { + cat = path.LibrariesCategory + serv = path.SharePointService + } + + return func(driveID string) (path.Path, error) { + return path.Build(tenantID, resourceOwner, serv, cat, false, odConsts.DrivesPathDir, driveID, odConsts.RootPathDir) + } +} + // itemCollector functions collect the items found in a drive type itemCollector func( ctx context.Context, @@ -137,7 +158,8 @@ func collectItems( } for { - page, err := pager.GetPage(ctx) + // assume delta urls here, which allows single-token consumption + page, err := pager.GetPage(graph.ConsumeNTokens(ctx, graph.SingleGetOrDeltaLC)) if graph.IsErrInvalidDelta(err) { logger.Ctx(ctx).Infow("Invalid previous delta link", "link", prevDelta) diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index d2f1a68b6..2a5d4b5a8 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -279,24 +279,24 @@ func (suite *OneDriveUnitSuite) TestDrives() { // Integration tests -type OneDriveSuite struct { +type OneDriveIntgSuite struct { tester.Suite userID string } func TestOneDriveSuite(t *testing.T) { - suite.Run(t, &OneDriveSuite{ + suite.Run(t, &OneDriveIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tester.M365AcctCredEnvs}), }) } -func (suite *OneDriveSuite) SetupSuite() { +func (suite *OneDriveIntgSuite) SetupSuite() { suite.userID = tester.SecondaryM365UserID(suite.T()) } -func (suite *OneDriveSuite) TestCreateGetDeleteFolder() { +func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { ctx, flush := tester.NewContext() defer flush() @@ -401,7 +401,7 @@ func (fm testFolderMatcher) Matches(p string) bool { return fm.scope.Matches(selectors.OneDriveFolder, p) } -func (suite *OneDriveSuite) TestOneDriveNewCollections() { +func (suite *OneDriveIntgSuite) TestOneDriveNewCollections() { creds, err := tester.NewM365Account(suite.T()).M365Config() require.NoError(suite.T(), err, clues.ToCore(err)) diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index c7cebc8c1..ac992e90a 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -333,7 +333,11 @@ func driveItemWriter( session := drives.NewItemItemsItemCreateUploadSessionPostRequestBody() ctx = clues.Add(ctx, "upload_item_id", itemID) - r, err := service.Client().DrivesById(driveID).ItemsById(itemID).CreateUploadSession().Post(ctx, session, nil) + r, err := service.Client(). + DrivesById(driveID). + ItemsById(itemID). + CreateUploadSession(). + Post(ctx, session, nil) if err != nil { return nil, graph.Wrap(ctx, err, "creating item upload session") } diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 65b69ede7..47feea0ff 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -128,8 +128,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { } // TestItemWriter is an integration test for uploading data to OneDrive -// It creates a new `testfolder_ item and writes data to it +// It creates a new folder with a new item and writes data to it func (suite *ItemIntegrationSuite) TestItemWriter() { table := []struct { name string @@ -155,24 +154,20 @@ func (suite *ItemIntegrationSuite) TestItemWriter() { root, err := srv.Client().DrivesById(test.driveID).Root().Get(ctx, nil) require.NoError(t, err, clues.ToCore(err)) - // Test Requirement 2: "Test Folder" should exist - folder, err := api.GetFolderByName(ctx, srv, test.driveID, ptr.Val(root.GetId()), "Test Folder") - require.NoError(t, err, clues.ToCore(err)) - - newFolderName := "testfolder_" + dttm.FormatNow(dttm.SafeForTesting) - t.Logf("Test will create folder %s", newFolderName) + newFolderName := tester.DefaultTestRestoreDestination("folder").ContainerName + t.Logf("creating folder %s", newFolderName) newFolder, err := CreateItem( ctx, srv, test.driveID, - ptr.Val(folder.GetId()), + ptr.Val(root.GetId()), newItem(newFolderName, true)) require.NoError(t, err, clues.ToCore(err)) require.NotNil(t, newFolder.GetId()) newItemName := "testItem_" + dttm.FormatNow(dttm.SafeForTesting) - t.Logf("Test will create item %s", newItemName) + t.Logf("creating item %s", newItemName) newItem, err := CreateItem( ctx, diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index 7cd4b530d..b67973be0 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -161,7 +161,7 @@ func UpdatePermissions( DrivesById(driveID). ItemsById(itemID). PermissionsById(pid). - Delete(ctx, nil) + Delete(graph.ConsumeNTokens(ctx, graph.PermissionsLC), nil) if err != nil { return graph.Wrap(ctx, err, "removing permissions") } @@ -207,7 +207,11 @@ func UpdatePermissions( pbody.SetRecipients([]models.DriveRecipientable{rec}) - np, err := service.Client().DrivesById(driveID).ItemsById(itemID).Invite().Post(ctx, pbody, nil) + np, err := service.Client(). + DrivesById(driveID). + ItemsById(itemID). + Invite(). + Post(graph.ConsumeNTokens(ctx, graph.PermissionsLC), pbody, nil) if err != nil { return graph.Wrap(ctx, err, "setting permissions") } diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 0cff8b465..41d037b13 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -231,7 +231,7 @@ func RestoreCollection( return metrics, nil } - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue @@ -781,17 +781,29 @@ func getMetadata(metar io.ReadCloser) (metadata.Metadata, error) { // Augment restore path to add extra files(meta) needed for restore as // well as do any other ordering operations on the paths -func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, error) { - colPaths := map[string]path.Path{} +// +// Only accepts StoragePath/RestorePath pairs where the RestorePath is +// at least as long as the StoragePath. If the RestorePath is longer than the +// StoragePath then the first few (closest to the root) directories will use +// default permissions during restore. +func AugmentRestorePaths( + backupVersion int, + paths []path.RestorePaths, +) ([]path.RestorePaths, error) { + // Keyed by each value's StoragePath.String() which corresponds to the RepoRef + // of the directory. + colPaths := map[string]path.RestorePaths{} for _, p := range paths { + first := true + for { - np, err := p.Dir() + sp, err := p.StoragePath.Dir() if err != nil { return nil, err } - drivePath, err := path.ToDrivePath(np) + drivePath, err := path.ToDrivePath(sp) if err != nil { return nil, err } @@ -800,8 +812,31 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err break } - colPaths[np.String()] = np - p = np + if len(p.RestorePath.Elements()) < len(sp.Elements()) { + return nil, clues.New("restorePath shorter than storagePath"). + With("restore_path", p.RestorePath, "storage_path", sp) + } + + rp := p.RestorePath + + // Make sure the RestorePath always points to the level of the current + // collection. We need to track if it's the first iteration because the + // RestorePath starts out at the collection level to begin with. + if !first { + rp, err = p.RestorePath.Dir() + if err != nil { + return nil, err + } + } + + paths := path.RestorePaths{ + StoragePath: sp, + RestorePath: rp, + } + + colPaths[sp.String()] = paths + p = paths + first = false } } @@ -814,32 +849,45 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err // As of now look up metadata for parent directories from a // collection. for _, p := range colPaths { - el := p.Elements() + el := p.StoragePath.Elements() if backupVersion >= version.OneDrive6NameInMeta { - mPath, err := p.Append(".dirmeta", true) + mPath, err := p.StoragePath.AppendItem(".dirmeta") if err != nil { return nil, err } - paths = append(paths, mPath) + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) } else if backupVersion >= version.OneDrive4DirIncludesPermissions { - mPath, err := p.Append(el[len(el)-1]+".dirmeta", true) + mPath, err := p.StoragePath.AppendItem(el.Last() + ".dirmeta") if err != nil { return nil, err } - paths = append(paths, mPath) + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: p.RestorePath}) } else if backupVersion >= version.OneDrive1DataAndMetaFiles { - pp, err := p.Dir() + pp, err := p.StoragePath.Dir() if err != nil { return nil, err } - mPath, err := pp.Append(el[len(el)-1]+".dirmeta", true) + + mPath, err := pp.AppendItem(el.Last() + ".dirmeta") if err != nil { return nil, err } - paths = append(paths, mPath) + + prp, err := p.RestorePath.Dir() + if err != nil { + return nil, err + } + + paths = append( + paths, + path.RestorePaths{StoragePath: mPath, RestorePath: prp}) } } @@ -847,8 +895,11 @@ func AugmentRestorePaths(backupVersion int, paths []path.Path) ([]path.Path, err // files. This is only a necessity for OneDrive as we are storing // metadata for files/folders in separate meta files and we the // data to be restored before we can restore the metadata. + // + // This sorting assumes stuff in the same StoragePath directory end up in the + // same RestorePath collection. sort.Slice(paths, func(i, j int) bool { - return paths[i].String() < paths[j].String() + return paths[i].StoragePath.String() < paths[j].StoragePath.String() }) return paths, nil diff --git a/src/internal/connector/onedrive/restore_test.go b/src/internal/connector/onedrive/restore_test.go index 56e5d467b..c085d689f 100644 --- a/src/internal/connector/onedrive/restore_test.go +++ b/src/internal/connector/onedrive/restore_test.go @@ -172,20 +172,30 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { base := "id/onedrive/user/files/drives/driveID/root:/" - inPaths := []path.Path{} + inPaths := []path.RestorePaths{} for _, ps := range test.input { p, err := path.FromDataLayerPath(base+ps, true) require.NoError(t, err, "creating path", clues.ToCore(err)) - inPaths = append(inPaths, p) + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) } - outPaths := []path.Path{} + outPaths := []path.RestorePaths{} for _, ps := range test.output { p, err := path.FromDataLayerPath(base+ps, true) require.NoError(t, err, "creating path", clues.ToCore(err)) - outPaths = append(outPaths, p) + pd, err := p.Dir() + require.NoError(t, err, "creating collection path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: pd}) } actual, err := AugmentRestorePaths(test.version, inPaths) @@ -197,3 +207,111 @@ func (suite *RestoreUnitSuite) TestAugmentRestorePaths() { }) } } + +// TestAugmentRestorePaths_DifferentRestorePath tests that RestorePath +// substitution works properly. Since it's only possible for future backup +// versions to need restore path substitution (i.e. due to storing folders by +// ID instead of name) this is only tested against the most recent backup +// version at the moment. +func (suite *RestoreUnitSuite) TestAugmentRestorePaths_DifferentRestorePath() { + // Adding a simple test here so that we can be sure that this + // function gets updated whenever we add a new version. + require.LessOrEqual(suite.T(), version.Backup, version.All8MigrateUserPNToID, "unsupported backup version") + + type pathPair struct { + storage string + restore string + } + + table := []struct { + name string + version int + input []pathPair + output []pathPair + errCheck assert.ErrorAssertionFunc + }{ + { + name: "nested folders", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "folder"}, + {storage: "folder-id/file.txt.data", restore: "folder"}, + {storage: "folder-id/folder2-id/.dirmeta", restore: "folder/folder2"}, + {storage: "folder-id/folder2-id/file.txt.data", restore: "folder/folder2"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path longer one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + output: []pathPair{ + {storage: "folder-id/.dirmeta", restore: "corso_restore/folder"}, + {storage: "folder-id/file.txt.data", restore: "corso_restore/folder"}, + }, + errCheck: assert.NoError, + }, + { + name: "restore path shorter one folder", + version: version.Backup, + input: []pathPair{ + {storage: "folder-id/file.txt.data", restore: ""}, + }, + errCheck: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + _, flush := tester.NewContext() + defer flush() + + base := "id/onedrive/user/files/drives/driveID/root:/" + + inPaths := []path.RestorePaths{} + for _, ps := range test.input { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + inPaths = append( + inPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + outPaths := []path.RestorePaths{} + for _, ps := range test.output { + p, err := path.FromDataLayerPath(base+ps.storage, true) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + r, err := path.FromDataLayerPath(base+ps.restore, false) + require.NoError(t, err, "creating path", clues.ToCore(err)) + + outPaths = append( + outPaths, + path.RestorePaths{StoragePath: p, RestorePath: r}) + } + + actual, err := AugmentRestorePaths(test.version, inPaths) + test.errCheck(t, err, "augmenting paths", clues.ToCore(err)) + + if err != nil { + return + } + + // Ordering of paths matter here as we need dirmeta files + // to show up before file in dir + assert.Equal(t, outPaths, actual, "augmented paths") + }) + } +} diff --git a/src/internal/connector/sharepoint/api/pages_test.go b/src/internal/connector/sharepoint/api/pages_test.go index 32d0aa07c..c56c3bc86 100644 --- a/src/internal/connector/sharepoint/api/pages_test.go +++ b/src/internal/connector/sharepoint/api/pages_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock" @@ -81,7 +80,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { t := suite.T() - destName := "Corso_Restore_" + dttm.FormatNow(dttm.SafeForTesting) + destName := tester.DefaultTestRestoreDestination("").ContainerName testName := "MockPage" // Create Test Page @@ -98,8 +97,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() { suite.service, pageData, suite.siteID, - destName, - ) + destName) require.NoError(t, err, clues.ToCore(err)) require.NotNil(t, info) diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index 6beb811f3..596b1bb34 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock" @@ -193,7 +192,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { info: sharePointListInfo(listing, int64(len(byteArray))), } - destName := "Corso_Restore_" + dttm.FormatNow(dttm.SafeForTesting) + destName := tester.DefaultTestRestoreDestination("").ContainerName deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName) assert.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 1c06e8ae3..2f64454da 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -125,7 +125,7 @@ func RestoreCollections( } // restoreListItem utility function restores a List to the siteID. -// The name is changed to to Corso_Restore_{timeStame}_name +// The name is changed to to {DestName}_{name} // API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http // Restored List can be verified within the Site contents. func restoreListItem( @@ -172,7 +172,11 @@ func restoreListItem( newList.SetItems(contents) // Restore to List base to M365 back store - restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil) + restoredList, err := service. + Client(). + SitesById(siteID). + Lists(). + Post(ctx, newList, nil) if err != nil { return dii, graph.Wrap(ctx, err, "restoring list") } @@ -247,7 +251,7 @@ func RestoreListCollection( metrics.Bytes += itemInfo.SharePoint.Size - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue @@ -335,7 +339,7 @@ func RestorePageCollection( metrics.Bytes += itemInfo.SharePoint.Size - itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + itemPath, err := dc.FullPath().AppendItem(itemData.UUID()) if err != nil { el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx)) continue diff --git a/src/internal/events/events.go b/src/internal/events/events.go index 47a15f5e9..f900c50c4 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -188,10 +188,12 @@ func tenantHash(tenID string) string { // metrics aggregation // --------------------------------------------------------------------------- -type m string +type metricsCategory string // metrics collection bucket -const APICall m = "api_call" +const ( + APICall metricsCategory = "api_call" +) // configurations const ( @@ -256,13 +258,19 @@ func dumpMetrics(ctx context.Context, stop <-chan struct{}, sig *metrics.InmemSi } // Inc increments the given category by 1. -func Inc(cat m, keys ...string) { +func Inc(cat metricsCategory, keys ...string) { cats := append([]string{string(cat)}, keys...) metrics.IncrCounter(cats, 1) } +// IncN increments the given category by N. +func IncN(n int, cat metricsCategory, keys ...string) { + cats := append([]string{string(cat)}, keys...) + metrics.IncrCounter(cats, float32(n)) +} + // Since records the duration between the provided time and now, in millis. -func Since(start time.Time, cat m, keys ...string) { +func Since(start time.Time, cat metricsCategory, keys ...string) { cats := append([]string{string(cat)}, keys...) metrics.MeasureSince(cats, start) } diff --git a/src/internal/kopia/merge_collection.go b/src/internal/kopia/merge_collection.go new file mode 100644 index 000000000..ab95dead8 --- /dev/null +++ b/src/internal/kopia/merge_collection.go @@ -0,0 +1,112 @@ +package kopia + +import ( + "context" + "errors" + + "github.com/alcionai/clues" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +var _ data.RestoreCollection = &mergeCollection{} + +type col struct { + storagePath string + data.RestoreCollection +} + +type mergeCollection struct { + cols []col + // Technically don't need to track this but it can help detect errors. + fullPath path.Path +} + +func (mc *mergeCollection) addCollection( + storagePath string, + c data.RestoreCollection, +) error { + if c == nil { + return clues.New("adding nil collection"). + With("current_path", mc.FullPath()) + } else if mc.FullPath().String() != c.FullPath().String() { + return clues.New("attempting to merge collection with different path"). + With("current_path", mc.FullPath(), "new_path", c.FullPath()) + } + + mc.cols = append(mc.cols, col{storagePath: storagePath, RestoreCollection: c}) + + // Keep a stable sorting of this merged collection set so we can say there's + // some deterministic behavior when Fetch is called. We don't expect to have + // to merge many collections. + slices.SortStableFunc(mc.cols, func(a, b col) bool { + return a.storagePath < b.storagePath + }) + + return nil +} + +func (mc mergeCollection) FullPath() path.Path { + return mc.fullPath +} + +func (mc *mergeCollection) Items( + ctx context.Context, + errs *fault.Bus, +) <-chan data.Stream { + res := make(chan data.Stream) + + go func() { + defer close(res) + + logger.Ctx(ctx).Infow( + "getting items for merged collection", + "merged_collection_count", len(mc.cols)) + + for _, c := range mc.cols { + // Unfortunately doesn't seem to be a way right now to see if the + // iteration failed and we should be exiting early. + ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + logger.Ctx(ictx).Debug("sending items from merged collection") + + for item := range c.Items(ictx, errs) { + res <- item + } + } + }() + + return res +} + +// Fetch goes through all the collections in this one and returns the first +// match found or the first error that is not data.ErrNotFound. If multiple +// collections have the requested item, the instance in the collection with the +// lexicographically smallest storage path is returned. +func (mc *mergeCollection) Fetch( + ctx context.Context, + name string, +) (data.Stream, error) { + logger.Ctx(ctx).Infow( + "fetching item in merged collection", + "merged_collection_count", len(mc.cols)) + + for _, c := range mc.cols { + ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + + logger.Ctx(ictx).Debug("looking for item in merged collection") + + s, err := c.Fetch(ictx, name) + if err == nil { + return s, nil + } else if err != nil && !errors.Is(err, data.ErrNotFound) { + return nil, clues.Wrap(err, "fetching from merged collection"). + WithClues(ictx) + } + } + + return nil, clues.Wrap(data.ErrNotFound, "merged collection fetch") +} diff --git a/src/internal/kopia/merge_collection_test.go b/src/internal/kopia/merge_collection_test.go new file mode 100644 index 000000000..e287452dc --- /dev/null +++ b/src/internal/kopia/merge_collection_test.go @@ -0,0 +1,297 @@ +package kopia + +import ( + "bytes" + "io" + "testing" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/connector/exchange/mock" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type MergeCollectionUnitSuite struct { + tester.Suite +} + +func TestMergeCollectionUnitSuite(t *testing.T) { + suite.Run(t, &MergeCollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *MergeCollectionUnitSuite) TestReturnsPath() { + t := suite.T() + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + c := mergeCollection{ + fullPath: pth, + } + + assert.Equal(t, pth, c.FullPath()) +} + +func (suite *MergeCollectionUnitSuite) TestItems() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + storagePaths := []string{ + "tenant-id/exchange/user-id/mail/some/folder/path1", + "tenant-id/exchange/user-id/mail/some/folder/path2", + } + + expectedItemNames := []string{"1", "2"} + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + c1 := mock.NewCollection(pth, nil, 1) + c1.Names[0] = expectedItemNames[0] + + c2 := mock.NewCollection(pth, nil, 1) + c2.Names[0] = expectedItemNames[1] + + // Not testing fetch here so safe to use this wrapper. + cols := []data.RestoreCollection{ + data.NotFoundRestoreCollection{Collection: c1}, + data.NotFoundRestoreCollection{Collection: c2}, + } + + dc := &mergeCollection{fullPath: pth} + + for i, c := range cols { + err := dc.addCollection(storagePaths[i], c) + require.NoError(t, err, "adding collection", clues.ToCore(err)) + } + + gotItemNames := []string{} + + for item := range dc.Items(ctx, fault.New(true)) { + gotItemNames = append(gotItemNames, item.UUID()) + } + + assert.ElementsMatch(t, expectedItemNames, gotItemNames) +} + +func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() { + t := suite.T() + + pth1, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(t, err, clues.ToCore(err)) + + pth2, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data2") + require.NoError(t, err, clues.ToCore(err)) + + dc := mergeCollection{fullPath: pth1} + + err = dc.addCollection("some/path", &kopiaDataCollection{path: pth2}) + assert.Error(t, err, clues.ToCore(err)) +} + +func (suite *MergeCollectionUnitSuite) TestFetch() { + var ( + fileData1 = []byte("abcdefghijklmnopqrstuvwxyz") + fileData2 = []byte("zyxwvutsrqponmlkjihgfedcba") + fileData3 = []byte("foo bar baz") + + fileName1 = "file1" + fileName2 = "file2" + fileLookupErrName = "errLookup" + fileOpenErrName = "errOpen" + + colPaths = []string{ + "tenant-id/exchange/user-id/mail/some/data/directory1", + "tenant-id/exchange/user-id/mail/some/data/directory2", + } + ) + + pth, err := path.Build( + "a-tenant", + "a-user", + path.ExchangeService, + path.EmailCategory, + false, + "some", "path", "for", "data") + require.NoError(suite.T(), err, clues.ToCore(err)) + + // Needs to be a function so the readers get refreshed each time. + layouts := []func() fs.Directory{ + // Has the following; + // - file1: data[0] + // - errOpen: (error opening file) + func() fs.Directory { + return virtualfs.NewStaticDirectory(encodeAsPath(colPaths[0]), []fs.Entry{ + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName1), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData1)), + ), + size: int64(len(fileData1) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileOpenErrName), + nil, + ), + openErr: assert.AnError, + }, + }) + }, + + // Has the following; + // - file1: data[1] + // - file2: data[0] + // - errOpen: data[2] + func() fs.Directory { + return virtualfs.NewStaticDirectory(encodeAsPath(colPaths[1]), []fs.Entry{ + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName1), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData2)), + ), + size: int64(len(fileData2) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileName2), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData1)), + ), + size: int64(len(fileData1) + versionSize), + }, + &mockFile{ + StreamingFile: virtualfs.StreamingFileFromReader( + encodeAsPath(fileOpenErrName), + nil, + ), + r: newBackupStreamReader( + serializationVersion, + io.NopCloser(bytes.NewReader(fileData3)), + ), + size: int64(len(fileData3) + versionSize), + }, + }) + }, + } + + table := []struct { + name string + fileName string + expectError assert.ErrorAssertionFunc + expectData []byte + notFoundErr bool + }{ + { + name: "Duplicate File, first collection", + fileName: fileName1, + expectError: assert.NoError, + expectData: fileData1, + }, + { + name: "Distinct File, second collection", + fileName: fileName2, + expectError: assert.NoError, + expectData: fileData1, + }, + { + name: "Error opening file", + fileName: fileOpenErrName, + expectError: assert.Error, + }, + { + name: "File not found", + fileName: fileLookupErrName, + expectError: assert.Error, + notFoundErr: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + c := &i64counter{} + + dc := mergeCollection{fullPath: pth} + + for i, layout := range layouts { + col := &kopiaDataCollection{ + path: pth, + dir: layout(), + counter: c, + expectedVersion: serializationVersion, + } + + err := dc.addCollection(colPaths[i], col) + require.NoError(t, err, "adding collection", clues.ToCore(err)) + } + + s, err := dc.Fetch(ctx, test.fileName) + test.expectError(t, err, clues.ToCore(err)) + + if err != nil { + if test.notFoundErr { + assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err)) + } + + return + } + + fileData, err := io.ReadAll(s.ToReader()) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + if err != nil { + return + } + + assert.Equal(t, test.expectData, fileData) + }) + } +} diff --git a/src/internal/kopia/s3.go b/src/internal/kopia/s3.go index 5810487dc..6b5c081d7 100644 --- a/src/internal/kopia/s3.go +++ b/src/internal/kopia/s3.go @@ -31,6 +31,10 @@ func s3BlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) Prefix: cfg.Prefix, DoNotUseTLS: cfg.DoNotUseTLS, DoNotVerifyTLS: cfg.DoNotVerifyTLS, + Tags: s.SessionTags, + SessionName: s.SessionName, + RoleARN: s.Role, + RoleDuration: s.SessionDuration, } store, err := s3.New(ctx, &opts, false) diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 6f7f5388c..a1cc0bed2 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -347,7 +347,7 @@ func collectionEntries( seen[encodedName] = struct{}{} // For now assuming that item IDs don't need escaping. - itemPath, err := streamedEnts.FullPath().Append(e.UUID(), true) + itemPath, err := streamedEnts.FullPath().AppendItem(e.UUID()) if err != nil { err = clues.Wrap(err, "getting full item path") progress.errs.AddRecoverable(err) @@ -464,7 +464,7 @@ func streamBaseEntries( } // For now assuming that item IDs don't need escaping. - itemPath, err := curPath.Append(entName, true) + itemPath, err := curPath.AppendItem(entName) if err != nil { return clues.Wrap(err, "getting full item path for base entry") } @@ -473,7 +473,7 @@ func streamBaseEntries( // backup details. If the item moved and we had only the new path, we'd be // unable to find it in the old backup details because we wouldn't know what // to look for. - prevItemPath, err := prevPath.Append(entName, true) + prevItemPath, err := prevPath.AppendItem(entName) if err != nil { return clues.Wrap(err, "getting previous full item path for base entry") } diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index e35d61cb6..e4d73bb4c 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -365,6 +365,11 @@ type ByteCounter interface { Count(numBytes int64) } +type restoreCollection struct { + restorePath path.Path + storageDirs map[string]*dirAndItems +} + type dirAndItems struct { dir path.Path items []string @@ -380,7 +385,7 @@ func loadDirsAndItems( ctx context.Context, snapshotRoot fs.Entry, bcounter ByteCounter, - toLoad map[string]*dirAndItems, + toLoad map[string]*restoreCollection, bus *fault.Bus, ) ([]data.RestoreCollection, error) { var ( @@ -389,50 +394,67 @@ func loadDirsAndItems( loadCount = 0 ) - for _, dirItems := range toLoad { + for _, col := range toLoad { if el.Failure() != nil { return nil, el.Failure() } - ictx := clues.Add(ctx, "directory_path", dirItems.dir) + ictx := clues.Add(ctx, "restore_path", col.restorePath) - dir, err := getDir(ictx, dirItems.dir, snapshotRoot) - if err != nil { - el.AddRecoverable(clues.Wrap(err, "loading directory"). - WithClues(ictx). - Label(fault.LabelForceNoBackupCreation)) + mergeCol := &mergeCollection{fullPath: col.restorePath} + res = append(res, mergeCol) - continue - } - - dc := &kopiaDataCollection{ - path: dirItems.dir, - dir: dir, - counter: bcounter, - expectedVersion: serializationVersion, - } - - res = append(res, dc) - - for _, item := range dirItems.items { + for _, dirItems := range col.storageDirs { if el.Failure() != nil { return nil, el.Failure() } - err := dc.addStream(ictx, item) + ictx = clues.Add(ictx, "storage_directory_path", dirItems.dir) + + dir, err := getDir(ictx, dirItems.dir, snapshotRoot) if err != nil { - el.AddRecoverable(clues.Wrap(err, "loading item"). + el.AddRecoverable(clues.Wrap(err, "loading storage directory"). WithClues(ictx). Label(fault.LabelForceNoBackupCreation)) continue } - loadCount++ - if loadCount%1000 == 0 { - logger.Ctx(ctx).Infow( - "loading items from kopia", - "loaded_items", loadCount) + dc := &kopiaDataCollection{ + path: col.restorePath, + dir: dir, + counter: bcounter, + expectedVersion: serializationVersion, + } + + if err := mergeCol.addCollection(dirItems.dir.String(), dc); err != nil { + el.AddRecoverable(clues.Wrap(err, "adding collection to merge collection"). + WithClues(ctx). + Label(fault.LabelForceNoBackupCreation)) + + continue + } + + for _, item := range dirItems.items { + if el.Failure() != nil { + return nil, el.Failure() + } + + err := dc.addStream(ictx, item) + if err != nil { + el.AddRecoverable(clues.Wrap(err, "loading item"). + WithClues(ictx). + Label(fault.LabelForceNoBackupCreation)) + + continue + } + + loadCount++ + if loadCount%1000 == 0 { + logger.Ctx(ctx).Infow( + "loading items from kopia", + "loaded_items", loadCount) + } } } } @@ -454,7 +476,7 @@ func loadDirsAndItems( func (w Wrapper) ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bcounter ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) { @@ -474,36 +496,53 @@ func (w Wrapper) ProduceRestoreCollections( var ( loadCount int - // Directory path -> set of items to load from the directory. - dirsToItems = map[string]*dirAndItems{} + // RestorePath -> []StoragePath directory -> set of items to load from the + // directory. + dirsToItems = map[string]*restoreCollection{} el = errs.Local() ) - for _, itemPath := range paths { + for _, itemPaths := range paths { if el.Failure() != nil { return nil, el.Failure() } - // Group things by directory so we can load all items from a single - // directory instance lower down. - ictx := clues.Add(ctx, "item_path", itemPath.String()) + // Group things by RestorePath and then StoragePath so we can load multiple + // items from a single directory instance lower down. + ictx := clues.Add( + ctx, + "item_path", itemPaths.StoragePath.String(), + "restore_path", itemPaths.RestorePath.String()) - parentPath, err := itemPath.Dir() + parentStoragePath, err := itemPaths.StoragePath.Dir() if err != nil { - el.AddRecoverable(clues.Wrap(err, "making directory collection"). + el.AddRecoverable(clues.Wrap(err, "getting storage directory path"). WithClues(ictx). Label(fault.LabelForceNoBackupCreation)) continue } - di := dirsToItems[parentPath.ShortRef()] - if di == nil { - dirsToItems[parentPath.ShortRef()] = &dirAndItems{dir: parentPath} - di = dirsToItems[parentPath.ShortRef()] + // Find the location this item is restored to. + rc := dirsToItems[itemPaths.RestorePath.ShortRef()] + if rc == nil { + dirsToItems[itemPaths.RestorePath.ShortRef()] = &restoreCollection{ + restorePath: itemPaths.RestorePath, + storageDirs: map[string]*dirAndItems{}, + } + rc = dirsToItems[itemPaths.RestorePath.ShortRef()] } - di.items = append(di.items, itemPath.Item()) + // Find the collection this item is sourced from. + di := rc.storageDirs[parentStoragePath.ShortRef()] + if di == nil { + rc.storageDirs[parentStoragePath.ShortRef()] = &dirAndItems{ + dir: parentStoragePath, + } + di = rc.storageDirs[parentStoragePath.ShortRef()] + } + + di.items = append(di.items, itemPaths.StoragePath.Item()) loadCount++ if loadCount%1000 == 0 { diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 67540aec7..48041cd91 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -59,14 +59,12 @@ var ( testFileData6 = testFileData ) -//revive:disable:context-as-argument func testForFiles( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument expected map[string][]byte, collections []data.RestoreCollection, ) { - //revive:enable:context-as-argument t.Helper() count := 0 @@ -75,7 +73,7 @@ func testForFiles( for s := range c.Items(ctx, fault.New(true)) { count++ - fullPath, err := c.FullPath().Append(s.UUID(), true) + fullPath, err := c.FullPath().AppendItem(s.UUID()) require.NoError(t, err, clues.ToCore(err)) expected, ok := expected[fullPath.String()] @@ -107,6 +105,19 @@ func checkSnapshotTags( assert.Equal(t, expectedTags, man.Tags) } +func toRestorePaths(t *testing.T, paths ...path.Path) []path.RestorePaths { + res := make([]path.RestorePaths, 0, len(paths)) + + for _, p := range paths { + dir, err := p.Dir() + require.NoError(t, err, clues.ToCore(err)) + + res = append(res, path.RestorePaths{StoragePath: p, RestorePath: dir}) + } + + return res +} + // --------------- // unit tests // --------------- @@ -678,10 +689,10 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1) dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1) - fp1, err := suite.storePath1.Append(dc1.Names[0], true) + fp1, err := suite.storePath1.AppendItem(dc1.Names[0]) require.NoError(t, err, clues.ToCore(err)) - fp2, err := suite.storePath2.Append(dc2.Names[0], true) + fp2, err := suite.storePath2.AppendItem(dc2.Names[0]) require.NoError(t, err, clues.ToCore(err)) stats, _, _, err := w.ConsumeBackupCollections( @@ -705,10 +716,7 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { result, err := w.ProduceRestoreCollections( ctx, string(stats.SnapshotID), - []path.Path{ - fp1, - fp2, - }, + toRestorePaths(t, fp1, fp2), nil, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -830,7 +838,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { // 5 file and 2 folder entries. assert.Len(t, deets.Details().Entries, 5+2) - failedPath, err := suite.storePath2.Append(testFileName4, true) + failedPath, err := suite.storePath2.AppendItem(testFileName4) require.NoError(t, err, clues.ToCore(err)) ic := i64counter{} @@ -838,7 +846,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { _, err = suite.w.ProduceRestoreCollections( suite.ctx, string(stats.SnapshotID), - []path.Path{failedPath}, + toRestorePaths(t, failedPath), &ic, fault.New(true)) // Files that had an error shouldn't make a dir entry in kopia. If they do we @@ -979,7 +987,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { } for _, item := range filesInfo { - pth, err := item.parentPath.Append(item.name, true) + pth, err := item.parentPath.AppendItem(item.name) require.NoError(suite.T(), err, clues.ToCore(err)) mapKey := item.parentPath.String() @@ -1219,9 +1227,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { _, err = suite.w.ProduceRestoreCollections( suite.ctx, string(stats.SnapshotID), - []path.Path{ - suite.files[suite.testPath1.String()][0].itemPath, - }, + toRestorePaths(t, suite.files[suite.testPath1.String()][0].itemPath), &ic, fault.New(true)) test.restoreCheck(t, err, clues.ToCore(err)) @@ -1322,7 +1328,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { result, err := suite.w.ProduceRestoreCollections( suite.ctx, string(suite.snapshotID), - test.inputPaths, + toRestorePaths(t, test.inputPaths...), &ic, fault.New(true)) test.expectedErr(t, err, clues.ToCore(err)) @@ -1338,14 +1344,201 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { } } +// TestProduceRestoreCollections_PathChanges tests that having different +// Restore and Storage paths works properly. Having the same Restore and Storage +// paths is tested by TestProduceRestoreCollections. +func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_PathChanges() { + rp1, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Inbox") + require.NoError(suite.T(), err) + + rp2, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Archive") + require.NoError(suite.T(), err) + + // Expected items is generated during the test by looking up paths in the + // suite's map of files. + table := []struct { + name string + inputPaths []path.RestorePaths + expectedCollections int + }{ + { + name: "SingleItem", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + { + name: "MultipleItemsSameCollection", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath1.String()][1].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + { + name: "MultipleItemsDifferentCollections", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp2, + }, + }, + expectedCollections: 2, + }, + { + name: "Multiple Items From Different Collections To Same Collection", + inputPaths: []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp1, + }, + }, + expectedCollections: 1, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + expected := make(map[string][]byte, len(test.inputPaths)) + + for _, pth := range test.inputPaths { + item, ok := suite.filesByPath[pth.StoragePath.String()] + require.True(t, ok, "getting expected file data") + + itemPath, err := pth.RestorePath.AppendItem(pth.StoragePath.Item()) + require.NoError(t, err, "getting expected item path") + + expected[itemPath.String()] = item.data + } + + ic := i64counter{} + + result, err := suite.w.ProduceRestoreCollections( + suite.ctx, + string(suite.snapshotID), + test.inputPaths, + &ic, + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + + assert.Len(t, result, test.expectedCollections) + assert.Less(t, int64(0), ic.i) + testForFiles(t, ctx, expected, result) + }) + } +} + +// TestProduceRestoreCollections_Fetch tests that the Fetch function still works +// properly even with different Restore and Storage paths and items from +// different kopia directories. +func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Fetch() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + + rp1, err := path.Build( + testTenant, + testUser, + path.ExchangeService, + path.EmailCategory, + false, + "corso_restore", "Inbox") + require.NoError(suite.T(), err) + + inputPaths := []path.RestorePaths{ + { + StoragePath: suite.files[suite.testPath1.String()][0].itemPath, + RestorePath: rp1, + }, + { + StoragePath: suite.files[suite.testPath2.String()][0].itemPath, + RestorePath: rp1, + }, + } + + // Really only interested in getting the collection so we can call fetch on + // it. + ic := i64counter{} + + result, err := suite.w.ProduceRestoreCollections( + suite.ctx, + string(suite.snapshotID), + inputPaths, + &ic, + fault.New(true)) + require.NoError(t, err, "getting collection", clues.ToCore(err)) + require.Len(t, result, 1) + + // Item from first kopia directory. + f := suite.files[suite.testPath1.String()][0] + + item, err := result[0].Fetch(ctx, f.itemPath.Item()) + require.NoError(t, err, "fetching file", clues.ToCore(err)) + + r := item.ToReader() + + buf, err := io.ReadAll(r) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + assert.Equal(t, f.data, buf) + + // Item from second kopia directory. + f = suite.files[suite.testPath2.String()][0] + + item, err = result[0].Fetch(ctx, f.itemPath.Item()) + require.NoError(t, err, "fetching file", clues.ToCore(err)) + + r = item.ToReader() + + buf, err = io.ReadAll(r) + require.NoError(t, err, "reading file data", clues.ToCore(err)) + + assert.Equal(t, f.data, buf) +} + func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Errors() { - itemPath, err := suite.testPath1.Append(testFileName, true) + itemPath, err := suite.testPath1.AppendItem(testFileName) require.NoError(suite.T(), err, clues.ToCore(err)) table := []struct { name string snapshotID string - paths []path.Path + paths []path.RestorePaths }{ { "NilPaths", @@ -1355,12 +1548,12 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Erro { "EmptyPaths", string(suite.snapshotID), - []path.Path{}, + []path.RestorePaths{}, }, { "NoSnapshot", "foo", - []path.Path{itemPath}, + toRestorePaths(suite.T(), itemPath), }, } @@ -1393,7 +1586,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestDeleteSnapshot() { c, err := suite.w.ProduceRestoreCollections( suite.ctx, string(suite.snapshotID), - []path.Path{itemPath}, + toRestorePaths(t, itemPath), &ic, fault.New(true)) assert.Error(t, err, "snapshot should be deleted", clues.ToCore(err)) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 2d926b692..6c6049156 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -512,11 +512,16 @@ func consumeBackupCollections( "kopia_ignored_errors", kopiaStats.IgnoredErrorCount) } - if kopiaStats.ErrorCount > 0 || - (kopiaStats.IgnoredErrorCount > kopiaStats.ExpectedIgnoredErrorCount) { - err = clues.New("building kopia snapshot").With( - "kopia_errors", kopiaStats.ErrorCount, - "kopia_ignored_errors", kopiaStats.IgnoredErrorCount) + ctx = clues.Add( + ctx, + "kopia_errors", kopiaStats.ErrorCount, + "kopia_ignored_errors", kopiaStats.IgnoredErrorCount, + "kopia_expected_ignored_errors", kopiaStats.ExpectedIgnoredErrorCount) + + if kopiaStats.ErrorCount > 0 { + err = clues.New("building kopia snapshot").WithClues(ctx) + } else if kopiaStats.IgnoredErrorCount > kopiaStats.ExpectedIgnoredErrorCount { + err = clues.New("downloading items for persistence").WithClues(ctx) } return kopiaStats, deets, itemsSourcedFromBase, err diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 38a28ac86..ddc59e6ce 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -3,6 +3,7 @@ package operations import ( "context" "fmt" + "strings" "testing" "time" @@ -22,11 +23,13 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/exchange" - "github.com/alcionai/corso/src/internal/connector/exchange/api" + exapi "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" + odapi "github.com/alcionai/corso/src/internal/connector/onedrive/api" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -41,15 +44,19 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" + deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/selectors/testdata" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/store" ) +// Does not use the tester.DefaultTestRestoreDestination syntax as some of these +// items are created directly, not as a result of restoration, and we want to ensure +// they get clearly selected without accidental overlap. const incrementalsDestContainerPrefix = "incrementals_ci_" // --------------------------------------------------------------------------- @@ -59,11 +66,9 @@ const incrementalsDestContainerPrefix = "incrementals_ci_" // prepNewTestBackupOp generates all clients required to run a backup operation, // returning both a backup operation created with those clients, as well as // the clients themselves. -// -//revive:disable:context-as-argument func prepNewTestBackupOp( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument bus events.Eventer, sel selectors.Selector, featureToggles control.Toggles, @@ -73,11 +78,11 @@ func prepNewTestBackupOp( account.Account, *kopia.Wrapper, *kopia.ModelStore, + streamstore.Streamer, *connector.GraphConnector, selectors.Selector, func(), ) { - //revive:enable:context-as-argument var ( acct = tester.NewM365Account(t) // need to initialize the repository before we can test connecting to it. @@ -123,18 +128,18 @@ func prepNewTestBackupOp( gc, sel := GCWithSelector(t, ctx, acct, connectorResource, sel, nil, closer) bo := newTestBackupOp(t, ctx, kw, ms, gc, acct, sel, bus, featureToggles, closer) - return bo, acct, kw, ms, gc, sel, closer + ss := streamstore.NewStreamer(kw, acct.ID(), sel.PathService()) + + return bo, acct, kw, ms, ss, gc, sel, closer } // newTestBackupOp accepts the clients required to compose a backup operation, plus // any other metadata, and uses them to generate a new backup operation. This // allows backup chains to utilize the same temp directory and configuration // details. -// -//revive:disable:context-as-argument func newTestBackupOp( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument kw *kopia.Wrapper, ms *kopia.ModelStore, gc *connector.GraphConnector, @@ -144,7 +149,6 @@ func newTestBackupOp( featureToggles control.Toggles, closer func(), ) BackupOperation { - //revive:enable:context-as-argument var ( sw = store.NewKopiaStore(ms) opts = control.Defaults() @@ -162,15 +166,13 @@ func newTestBackupOp( return bo } -//revive:disable:context-as-argument func runAndCheckBackup( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument bo *BackupOperation, mb *evmock.Bus, acceptNoData bool, ) { - //revive:enable:context-as-argument err := bo.Run(ctx) require.NoError(t, err, clues.ToCore(err)) require.NotEmpty(t, bo.Results, "the backup had non-zero results") @@ -203,17 +205,15 @@ func runAndCheckBackup( bo.Results.BackupID, "backupID pre-declaration") } -//revive:disable:context-as-argument func checkBackupIsInManifests( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument kw *kopia.Wrapper, bo *BackupOperation, sel selectors.Selector, resourceOwner string, categories ...path.CategoryType, ) { - //revive:enable:context-as-argument for _, category := range categories { t.Run(category.String(), func(t *testing.T) { var ( @@ -248,10 +248,9 @@ func checkBackupIsInManifests( } } -//revive:disable:context-as-argument func checkMetadataFilesExist( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument backupID model.StableID, kw *kopia.Wrapper, ms *kopia.ModelStore, @@ -259,7 +258,6 @@ func checkMetadataFilesExist( service path.ServiceType, filesByCat map[path.CategoryType][]string, ) { - //revive:enable:context-as-argument for category, files := range filesByCat { t.Run(category.String(), func(t *testing.T) { bup := &backup.Backup{} @@ -269,7 +267,7 @@ func checkMetadataFilesExist( return } - paths := []path.Path{} + paths := []path.RestorePaths{} pathsByRef := map[string][]string{} for _, fName := range files { @@ -285,11 +283,18 @@ func checkMetadataFilesExist( continue } - paths = append(paths, p) + paths = append( + paths, + path.RestorePaths{StoragePath: p, RestorePath: dir}) pathsByRef[dir.ShortRef()] = append(pathsByRef[dir.ShortRef()], fName) } - cols, err := kw.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, nil, fault.New(true)) + cols, err := kw.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + nil, + fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) for _, col := range cols { @@ -335,10 +340,9 @@ func checkMetadataFilesExist( // the callback provider can use them, or not, as wanted. type dataBuilderFunc func(id, timeStamp, subject, body string) []byte -//revive:disable:context-as-argument func generateContainerOfItems( t *testing.T, - ctx context.Context, + ctx context.Context, //revive:disable-line:context-as-argument gc *connector.GraphConnector, service path.ServiceType, acct account.Account, @@ -349,7 +353,6 @@ func generateContainerOfItems( backupVersion int, dbf dataBuilderFunc, ) *details.Details { - //revive:enable:context-as-argument t.Helper() items := make([]incrementalItem, 0, howManyItems) @@ -367,7 +370,7 @@ func generateContainerOfItems( switch service { case path.OneDriveService, path.SharePointService: - pathFolders = []string{"drives", driveID, "root:", destFldr} + pathFolders = []string{odConsts.DrivesPathDir, driveID, odConsts.RootPathDir, destFldr} } collections := []incrementalCollection{{ @@ -576,11 +579,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { defer flush() tests := []struct { - name string - selector func() *selectors.ExchangeBackup - category path.CategoryType - metadataFiles []string - runIncremental bool + name string + selector func() *selectors.ExchangeBackup + category path.CategoryType + metadataFiles []string }{ { name: "Mail", @@ -591,9 +593,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { return sel }, - category: path.EmailCategory, - metadataFiles: exchange.MetadataFileNames(path.EmailCategory), - runIncremental: true, + category: path.EmailCategory, + metadataFiles: exchange.MetadataFileNames(path.EmailCategory), }, { name: "Contacts", @@ -602,9 +603,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { sel.Include(sel.ContactFolders([]string{exchange.DefaultContactFolder}, selectors.PrefixMatch())) return sel }, - category: path.ContactsCategory, - metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), - runIncremental: true, + category: path.ContactsCategory, + metadataFiles: exchange.MetadataFileNames(path.ContactsCategory), }, { name: "Calendar Events", @@ -620,13 +620,14 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { for _, test := range tests { suite.Run(test.name, func() { var ( - t = suite.T() - mb = evmock.NewBus() - sel = test.selector().Selector - ffs = control.Toggles{} + t = suite.T() + mb = evmock.NewBus() + sel = test.selector().Selector + ffs = control.Toggles{} + whatSet = deeTD.CategoryFromRepoRef ) - bo, acct, kw, ms, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) + bo, acct, kw, ms, ss, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) defer closer() userID := sel.ID() @@ -648,9 +649,17 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { path.ExchangeService, map[path.CategoryType][]string{test.category: test.metadataFiles}) - if !test.runIncremental { - return - } + _, expectDeets := deeTD.GetDeetsInBackup( + t, + ctx, + bo.Results.BackupID, + acct.ID(), + userID, + path.ExchangeService, + whatSet, + ms, + ss) + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, whatSet, ms, ss, expectDeets, false) // Basic, happy path incremental test. No changes are dictated or expected. // This only tests that an incremental backup is runnable at all, and that it @@ -672,6 +681,15 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { userID, path.ExchangeService, map[path.CategoryType][]string{test.category: test.metadataFiles}) + deeTD.CheckBackupDetails( + t, + ctx, + incBO.Results.BackupID, + whatSet, + ms, + ss, + expectDeets, + false) // do some additional checks to ensure the incremental dealt with fewer items. assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written") @@ -692,7 +710,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { // TestBackup_Run ensures that Integration Testing works // for the following scopes: Contacts, Events, and Mail -func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalExchange() { ctx, flush := tester.NewContext() defer flush() @@ -704,6 +722,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ffs = control.Toggles{} mb = evmock.NewBus() now = dttm.Now() + service = path.ExchangeService categories = map[path.CategoryType][]string{ path.EmailCategory: exchange.MetadataFileNames(path.EmailCategory), path.ContactsCategory: exchange.MetadataFileNames(path.ContactsCategory), @@ -720,11 +739,12 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // at this point is harmless. containers = []string{container1, container2, container3, containerRename} sel = selectors.NewExchangeBackup([]string{suite.user}) + whatSet = deeTD.CategoryFromRepoRef ) gc, sels := GCWithSelector(t, ctx, acct, connector.Users, sel.Selector, nil, nil) - sel, err := sels.ToExchangeBackup() - require.NoError(t, err, clues.ToCore(err)) + sel.DiscreteOwner = sels.ID() + sel.DiscreteOwnerName = sels.Name() uidn := inMock.NewProvider(sels.ID(), sels.Name()) @@ -735,7 +755,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { m365, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(m365) + ac, err := exapi.NewClient(m365) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -746,7 +766,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // container into another generates a delta for both addition and deletion. type contDeets struct { containerID string - deets *details.Details + locRef string + itemRefs []string // cached for populating expected deets, otherwise not used } mailDBF := func(id, timeStamp, subject, body string) []byte { @@ -804,11 +825,14 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { // populate initial test data for category, gen := range dataset { for destName := range gen.dests { + // TODO: the details.Builder returned by restore can contain entries with + // incorrect information. non-representative repo-refs and the like. Until + // that gets fixed, we can't consume that info for testing. deets := generateContainerOfItems( t, ctx, gc, - path.ExchangeService, + service, acct, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, @@ -817,41 +841,103 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { version.Backup, gen.dbf) - dataset[category].dests[destName] = contDeets{"", deets} + itemRefs := []string{} + + for _, ent := range deets.Entries { + if ent.Exchange == nil || ent.Folder != nil { + continue + } + + if len(ent.ItemRef) > 0 { + itemRefs = append(itemRefs, ent.ItemRef) + } + } + + // save the item ids for building expectedDeets later on + cd := dataset[category].dests[destName] + cd.itemRefs = itemRefs + dataset[category].dests[destName] = cd + } + } + + bo, acct, kw, ms, ss, gc, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs, version.Backup) + defer closer() + + // run the initial backup + runAndCheckBackup(t, ctx, &bo, mb, false) + + rrPfx, err := path.ServicePrefix(acct.ID(), uidn.ID(), service, path.EmailCategory) + require.NoError(t, err, clues.ToCore(err)) + + // strip the category from the prefix; we primarily want the tenant and resource owner. + expectDeets := deeTD.NewInDeets(rrPfx.ToBuilder().Dir().String()) + bupDeets, _ := deeTD.GetDeetsInBackup(t, ctx, bo.Results.BackupID, acct.ID(), uidn.ID(), service, whatSet, ms, ss) + + // update the datasets with their location refs + for category, gen := range dataset { + for destName, cd := range gen.dests { + var longestLR string + + for _, ent := range bupDeets.Entries { + // generated destinations should always contain items + if ent.Folder != nil { + continue + } + + p, err := path.FromDataLayerPath(ent.RepoRef, false) + require.NoError(t, err, clues.ToCore(err)) + + // category must match, and the owning folder must be this destination + if p.Category() != category || strings.HasSuffix(ent.LocationRef, destName) { + continue + } + + // emails, due to folder nesting and our design for populating data via restore, + // will duplicate the dest folder as both the restore destination, and the "old parent + // folder". we'll get both a prefix/destName and a prefix/destName/destName folder. + // since we want future comparison to only use the leaf dir, we select for the longest match. + if len(ent.LocationRef) > len(longestLR) { + longestLR = ent.LocationRef + } + } + + require.NotEmptyf(t, longestLR, "must find an expected details entry matching the generated folder: %s", destName) + + cd.locRef = longestLR + + dataset[category].dests[destName] = cd + expectDeets.AddLocation(category.String(), cd.locRef) + + for _, i := range dataset[category].dests[destName].itemRefs { + expectDeets.AddItem(category.String(), cd.locRef, i) + } } } // verify test data was populated, and track it for comparisons + // TODO: this can be swapped out for InDeets checks if we add itemRefs to folder ents. for category, gen := range dataset { qp := graph.QueryParams{ Category: category, ResourceOwner: uidn, Credentials: m365, } + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating container resolver", category, clues.ToCore(err)) for destName, dest := range gen.dests { - p, err := path.FromDataLayerPath(dest.deets.Entries[0].RepoRef, true) - require.NoError(t, err, clues.ToCore(err)) + id, ok := cr.LocationInCache(dest.locRef) + require.True(t, ok, "dir %s found in %s cache", dest.locRef, category) - id, ok := cr.LocationInCache(p.Folder(false)) - require.True(t, ok, "dir %s found in %s cache", p.Folder(false), category) - - d := dataset[category].dests[destName] - d.containerID = id - dataset[category].dests[destName] = d + dest.containerID = id + dataset[category].dests[destName] = dest } } - bo, _, kw, ms, gc, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, ffs, version.Backup) - defer closer() - - sel, err = sels.ToExchangeBackup() - require.NoError(t, err, clues.ToCore(err)) - - // run the initial backup - runAndCheckBackup(t, ctx, &bo, mb, false) + // precheck to ensure the expectedDeets are correct. + // if we fail here, the expectedDeets were populated incorrectly. + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, whatSet, ms, ss, expectDeets, true) // Although established as a table, these tests are no isolated from each other. // Assume that every test's side effects cascade to all following test cases. @@ -873,20 +959,25 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { { name: "move an email folder to a subfolder", updateUserData: func(t *testing.T) { + cat := path.EmailCategory + // contacts and events cannot be sufoldered; this is an email-only change - toContainer := dataset[path.EmailCategory].dests[container1].containerID - fromContainer := dataset[path.EmailCategory].dests[container2].containerID + from := dataset[cat].dests[container2] + to := dataset[cat].dests[container1] body := users.NewItemMailFoldersItemMovePostRequestBody() - body.SetDestinationId(&toContainer) + body.SetDestinationId(ptr.To(to.containerID)) _, err := gc.Service. Client(). UsersById(uidn.ID()). - MailFoldersById(fromContainer). + MailFoldersById(from.containerID). Move(). Post(ctx, body, nil) require.NoError(t, err, clues.ToCore(err)) + + newLoc := expectDeets.MoveLocation(cat.String(), from.locRef, to.locRef) + from.locRef = newLoc }, itemsRead: 0, // zero because we don't count container reads itemsWritten: 2, @@ -908,6 +999,8 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err := ac.Events().DeleteContainer(ctx, uidn.ID(), containerID) require.NoError(t, err, "deleting a calendar", clues.ToCore(err)) } + + expectDeets.RemoveLocation(category.String(), d.dests[container2].locRef) } }, itemsRead: 0, @@ -921,7 +1014,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { t, ctx, gc, - path.ExchangeService, + service, acct, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, @@ -936,16 +1029,28 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { Credentials: m365, } + expectedLocRef := container3 + if category == path.EmailCategory { + expectedLocRef = path.Builder{}.Append(container3, container3).String() + } + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating container resolver", category, clues.ToCore(err)) - p, err := path.FromDataLayerPath(deets.Entries[0].RepoRef, true) - require.NoError(t, err, clues.ToCore(err)) + id, ok := cr.LocationInCache(expectedLocRef) + require.Truef(t, ok, "dir %s found in %s cache", expectedLocRef, category) - id, ok := cr.LocationInCache(p.Folder(false)) - require.Truef(t, ok, "dir %s found in %s cache", p.Folder(false), category) + dataset[category].dests[container3] = contDeets{ + containerID: id, + locRef: expectedLocRef, + itemRefs: nil, // not needed at this point + } - dataset[category].dests[container3] = contDeets{id, deets} + for _, ent := range deets.Entries { + if ent.Folder == nil { + expectDeets.AddItem(category.String(), expectedLocRef, ent.ItemRef) + } + } } }, itemsRead: 4, @@ -955,17 +1060,24 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { name: "rename a folder", updateUserData: func(t *testing.T) { for category, d := range dataset { - containerID := d.dests[container3].containerID cli := gc.Service.Client().UsersById(uidn.ID()) + containerID := d.dests[container3].containerID + newLoc := containerRename - // copy the container info, since both names should - // reference the same container by id. Though the - // details refs won't line up, so those get deleted. - d.dests[containerRename] = contDeets{ - containerID: d.dests[container3].containerID, - deets: nil, + if category == path.EmailCategory { + newLoc = path.Builder{}.Append(container3, containerRename).String() } + d.dests[containerRename] = contDeets{ + containerID: containerID, + locRef: newLoc, + } + + expectDeets.RenameLocation( + category.String(), + d.dests[container3].containerID, + newLoc) + switch category { case path.EmailCategory: cmf := cli.MailFoldersById(containerID) @@ -999,8 +1111,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { } } }, - itemsRead: 0, // containers are not counted as reads - itemsWritten: 4, // two items per category + itemsRead: 0, // containers are not counted as reads + // Renaming a folder doesn't cause kopia changes as the folder ID doesn't + // change. + itemsWritten: 0, }, { name: "add a new item", @@ -1015,24 +1129,39 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { body, err := support.CreateMessageFromBytes(itemData) require.NoError(t, err, "transforming mail bytes to messageable", clues.ToCore(err)) - _, err = cli.MailFoldersById(containerID).Messages().Post(ctx, body, nil) + itm, err := cli.MailFoldersById(containerID).Messages().Post(ctx, body, nil) require.NoError(t, err, "posting email item", clues.ToCore(err)) + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) + case path.ContactsCategory: _, itemData := generateItemData(t, category, uidn.ID(), contactDBF) body, err := support.CreateContactFromBytes(itemData) require.NoError(t, err, "transforming contact bytes to contactable", clues.ToCore(err)) - _, err = cli.ContactFoldersById(containerID).Contacts().Post(ctx, body, nil) + itm, err := cli.ContactFoldersById(containerID).Contacts().Post(ctx, body, nil) require.NoError(t, err, "posting contact item", clues.ToCore(err)) + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) + case path.EventsCategory: _, itemData := generateItemData(t, category, uidn.ID(), eventDBF) body, err := support.CreateEventFromBytes(itemData) require.NoError(t, err, "transforming event bytes to eventable", clues.ToCore(err)) - _, err = cli.CalendarsById(containerID).Events().Post(ctx, body, nil) + itm, err := cli.CalendarsById(containerID).Events().Post(ctx, body, nil) require.NoError(t, err, "posting events item", clues.ToCore(err)) + + expectDeets.AddItem( + category.String(), + d.dests[category.String()].locRef, + ptr.Val(itm.GetId())) } } }, @@ -1055,6 +1184,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.MessagesById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting email item", clues.ToCore(err)) + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) + case path.ContactsCategory: ids, _, _, err := ac.Contacts().GetAddedAndRemovedItemIDs(ctx, uidn.ID(), containerID, "", false) require.NoError(t, err, "getting contact ids", clues.ToCore(err)) @@ -1063,6 +1197,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.ContactsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting contact item", clues.ToCore(err)) + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) + case path.EventsCategory: ids, _, _, err := ac.Events().GetAddedAndRemovedItemIDs(ctx, uidn.ID(), containerID, "", false) require.NoError(t, err, "getting event ids", clues.ToCore(err)) @@ -1070,6 +1209,11 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { err = cli.CalendarsById(ids[0]).Delete(ctx, nil) require.NoError(t, err, "deleting calendar", clues.ToCore(err)) + + expectDeets.RemoveItem( + category.String(), + d.dests[category.String()].locRef, + ids[0]) } } }, @@ -1082,24 +1226,20 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { var ( t = suite.T() incMB = evmock.NewBus() - incBO = newTestBackupOp(t, ctx, kw, ms, gc, acct, sel.Selector, incMB, ffs, closer) + incBO = newTestBackupOp(t, ctx, kw, ms, gc, acct, sels, incMB, ffs, closer) + atid = m365.AzureTenantID ) test.updateUserData(t) err := incBO.Run(ctx) require.NoError(t, err, clues.ToCore(err)) - checkBackupIsInManifests(t, ctx, kw, &incBO, sel.Selector, uidn.ID(), maps.Keys(categories)...) - checkMetadataFilesExist( - t, - ctx, - incBO.Results.BackupID, - kw, - ms, - m365.AzureTenantID, - uidn.ID(), - path.ExchangeService, - categories) + + bupID := incBO.Results.BackupID + + checkBackupIsInManifests(t, ctx, kw, &incBO, sels, uidn.ID(), maps.Keys(categories)...) + checkMetadataFilesExist(t, ctx, bupID, kw, ms, atid, uidn.ID(), service, categories) + deeTD.CheckBackupDetails(t, ctx, bupID, whatSet, ms, ss, expectDeets, true) // do some additional checks to ensure the incremental dealt with fewer items. // +4 on read/writes to account for metadata: 1 delta and 1 path for each type. @@ -1111,7 +1251,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") assert.Equal(t, incMB.CalledWith[events.BackupStart][0][events.BackupID], - incBO.Results.BackupID, "incremental backupID pre-declaration") + bupID, "incremental backupID pre-declaration") }) } } @@ -1125,21 +1265,29 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() { defer flush() var ( - t = suite.T() - mb = evmock.NewBus() - m365UserID = tester.SecondaryM365UserID(t) - osel = selectors.NewOneDriveBackup([]string{m365UserID}) + t = suite.T() + tenID = tester.M365TenantID(t) + mb = evmock.NewBus() + userID = tester.SecondaryM365UserID(t) + osel = selectors.NewOneDriveBackup([]string{userID}) + ws = deeTD.DriveIDFromRepoRef + svc = path.OneDriveService ) - osel.Include(osel.AllData()) + osel.Include(selTD.OneDriveBackupFolderScope(osel)) - bo, _, _, _, _, _, closer := prepNewTestBackupOp(t, ctx, mb, osel.Selector, control.Toggles{}, version.Backup) + bo, _, _, ms, ss, _, sel, closer := prepNewTestBackupOp(t, ctx, mb, osel.Selector, control.Toggles{}, version.Backup) defer closer() runAndCheckBackup(t, ctx, &bo, mb, false) + + bID := bo.Results.BackupID + + _, expectDeets := deeTD.GetDeetsInBackup(t, ctx, bID, tenID, sel.ID(), svc, ws, ms, ss) + deeTD.CheckBackupDetails(t, ctx, bID, ws, ms, ss, expectDeets, false) } -func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalOneDrive() { sel := selectors.NewOneDriveRestore([]string{suite.user}) ic := func(cs []string) selectors.Selector { @@ -1150,9 +1298,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { gtdi := func( t *testing.T, ctx context.Context, - svc graph.Servicer, + gs graph.Servicer, ) string { - d, err := svc.Client().UsersById(suite.user).Drive().Get(ctx, nil) + d, err := odapi.GetUsersDrive(ctx, gs, suite.user) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). With("user", suite.user) @@ -1178,7 +1326,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveIncrementals() { false) } -func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePointIncrementals() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalSharePoint() { sel := selectors.NewSharePointRestore([]string{suite.site}) ic := func(cs []string) selectors.Selector { @@ -1189,9 +1337,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePointIncrementals() { gtdi := func( t *testing.T, ctx context.Context, - svc graph.Servicer, + gs graph.Servicer, ) string { - d, err := svc.Client().SitesById(suite.site).Drive().Get(ctx, nil) + d, err := odapi.GetSitesDefaultDrive(ctx, gs, suite.site) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). With("site", suite.site) @@ -1235,6 +1383,7 @@ func runDriveIncrementalTest( acct = tester.NewM365Account(t) ffs = control.Toggles{} mb = evmock.NewBus() + ws = deeTD.DriveIDFromRepoRef // `now` has to be formatted with SimpleDateTimeTesting as // some drives cannot have `:` in file/folder names @@ -1243,9 +1392,10 @@ func runDriveIncrementalTest( categories = map[path.CategoryType][]string{ category: {graph.DeltaURLsFileName, graph.PreviousPathFileName}, } - container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) - container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) - container3 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 3, now) + container1 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 1, now) + container2 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 2, now) + container3 = fmt.Sprintf("%s%d_%s", incrementalsDestContainerPrefix, 3, now) + containerRename = "renamed_folder" genDests = []string{container1, container2} @@ -1261,13 +1411,26 @@ func runDriveIncrementalTest( gc, sel := GCWithSelector(t, ctx, acct, resource, sel, nil, nil) + roidn := inMock.NewProvider(sel.ID(), sel.Name()) + var ( + atid = creds.AzureTenantID driveID = getTestDriveID(t, ctx, gc.Service) fileDBF = func(id, timeStamp, subject, body string) []byte { return []byte(id + subject) } + makeLocRef = func(flds ...string) string { + elems := append([]string{driveID, "root:"}, flds...) + return path.Builder{}.Append(elems...).String() + } ) + rrPfx, err := path.ServicePrefix(atid, roidn.ID(), service, category) + require.NoError(t, err, clues.ToCore(err)) + + // strip the category from the prefix; we primarily want the tenant and resource owner. + expectDeets := deeTD.NewInDeets(rrPfx.ToBuilder().Dir().String()) + // Populate initial test data. // Generate 2 new folders with two items each. Only the first two // folders will be part of the initial backup and @@ -1275,7 +1438,7 @@ func runDriveIncrementalTest( // through the changes. This should be enough to cover most delta // actions. for _, destName := range genDests { - generateContainerOfItems( + deets := generateContainerOfItems( t, ctx, gc, @@ -1283,11 +1446,19 @@ func runDriveIncrementalTest( acct, category, sel, - creds.AzureTenantID, owner, driveID, destName, + atid, roidn.ID(), driveID, destName, 2, // Use an old backup version so we don't need metadata files. 0, fileDBF) + + for _, ent := range deets.Entries { + if ent.Folder != nil { + continue + } + + expectDeets.AddItem(driveID, makeLocRef(destName), ent.ItemRef) + } } containerIDs := map[string]string{} @@ -1305,15 +1476,20 @@ func runDriveIncrementalTest( containerIDs[destName] = ptr.Val(resp.GetId()) } - bo, _, kw, ms, gc, _, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) + bo, _, kw, ms, ss, gc, _, closer := prepNewTestBackupOp(t, ctx, mb, sel, ffs, version.Backup) defer closer() // run the initial backup runAndCheckBackup(t, ctx, &bo, mb, false) + // precheck to ensure the expectedDeets are correct. + // if we fail here, the expectedDeets were populated incorrectly. + deeTD.CheckBackupDetails(t, ctx, bo.Results.BackupID, ws, ms, ss, expectDeets, true) + var ( newFile models.DriveItemable newFileName = "new_file.txt" + newFileID string permissionIDMappings = map[string]string{} writePerm = metadata.Permission{ @@ -1355,6 +1531,10 @@ func runDriveIncrementalTest( targetContainer, driveItem) require.NoErrorf(t, err, "creating new file %v", clues.ToCore(err)) + + newFileID = ptr.Val(newFile.GetId()) + + expectDeets.AddItem(driveID, makeLocRef(container1), newFileID) }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1374,8 +1554,10 @@ func runDriveIncrementalTest( *newFile.GetId(), []metadata.Permission{writePerm}, []metadata.Permission{}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 1, // .data file for newitem itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) @@ -1395,8 +1577,10 @@ func runDriveIncrementalTest( *newFile.GetId(), []metadata.Permission{}, []metadata.Permission{writePerm}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 1, // .data file for newitem itemsWritten: 2, // .meta for newitem, .dirmeta for parent (.data is not written as it is not updated) @@ -1417,8 +1601,10 @@ func runDriveIncrementalTest( targetContainer, []metadata.Permission{writePerm}, []metadata.Permission{}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked5tgb }, itemsRead: 0, itemsWritten: 1, // .dirmeta for collection @@ -1439,8 +1625,10 @@ func runDriveIncrementalTest( targetContainer, []metadata.Permission{}, []metadata.Permission{writePerm}, - permissionIDMappings) + permissionIDMappings, + ) require.NoErrorf(t, err, "adding permission to file %v", clues.ToCore(err)) + // no expectedDeets: metadata isn't tracked }, itemsRead: 0, itemsWritten: 1, // .dirmeta for collection @@ -1455,6 +1643,7 @@ func runDriveIncrementalTest( Content(). Put(ctx, []byte("new content"), nil) require.NoErrorf(t, err, "updating file contents: %v", clues.ToCore(err)) + // no expectedDeets: neither file id nor location changed }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1480,11 +1669,12 @@ func runDriveIncrementalTest( }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent + // no expectedDeets: neither file id nor location changed }, { name: "move a file between folders", updateFiles: func(t *testing.T) { - dest := containerIDs[container1] + dest := containerIDs[container2] driveItem := models.NewDriveItem() driveItem.SetName(&newFileName) @@ -1498,6 +1688,12 @@ func runDriveIncrementalTest( ItemsById(ptr.Val(newFile.GetId())). Patch(ctx, driveItem, nil) require.NoErrorf(t, err, "moving file between folders %v", clues.ToCore(err)) + + expectDeets.MoveItem( + driveID, + makeLocRef(container1), + makeLocRef(container2), + ptr.Val(newFile.GetId())) }, itemsRead: 1, // .data file for newitem itemsWritten: 3, // .data and .meta for newitem, .dirmeta for parent @@ -1513,6 +1709,8 @@ func runDriveIncrementalTest( ItemsById(ptr.Val(newFile.GetId())). Delete(ctx, nil) require.NoErrorf(t, err, "deleting file %v", clues.ToCore(err)) + + expectDeets.RemoveItem(driveID, makeLocRef(container2), ptr.Val(newFile.GetId())) }, itemsRead: 0, itemsWritten: 0, @@ -1520,21 +1718,26 @@ func runDriveIncrementalTest( { name: "move a folder to a subfolder", updateFiles: func(t *testing.T) { - dest := containerIDs[container1] - source := containerIDs[container2] + parent := containerIDs[container1] + child := containerIDs[container2] driveItem := models.NewDriveItem() driveItem.SetName(&container2) parentRef := models.NewItemReference() - parentRef.SetId(&dest) + parentRef.SetId(&parent) driveItem.SetParentReference(parentRef) _, err := gc.Service. Client(). DrivesById(driveID). - ItemsById(source). + ItemsById(child). Patch(ctx, driveItem, nil) require.NoError(t, err, "moving folder", clues.ToCore(err)) + + expectDeets.MoveLocation( + driveID, + makeLocRef(container2), + makeLocRef(container1)) }, itemsRead: 0, itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) @@ -1546,8 +1749,7 @@ func runDriveIncrementalTest( child := containerIDs[container2] driveItem := models.NewDriveItem() - name := "renamed_folder" - driveItem.SetName(&name) + driveItem.SetName(&containerRename) parentRef := models.NewItemReference() parentRef.SetId(&parent) driveItem.SetParentReference(parentRef) @@ -1558,6 +1760,13 @@ func runDriveIncrementalTest( ItemsById(child). Patch(ctx, driveItem, nil) require.NoError(t, err, "renaming folder", clues.ToCore(err)) + + containerIDs[containerRename] = containerIDs[container2] + + expectDeets.RenameLocation( + driveID, + makeLocRef(container1, container2), + makeLocRef(container1, containerRename)) }, itemsRead: 0, itemsWritten: 7, // 2*2(data and meta of 2 files) + 3 (dirmeta of two moved folders and target) @@ -1565,7 +1774,7 @@ func runDriveIncrementalTest( { name: "delete a folder", updateFiles: func(t *testing.T) { - container := containerIDs[container2] + container := containerIDs[containerRename] // deletes require unique http clients // https://github.com/alcionai/corso/issues/2707 err = newDeleteServicer(t). @@ -1574,6 +1783,8 @@ func runDriveIncrementalTest( ItemsById(container). Delete(ctx, nil) require.NoError(t, err, "deleting folder", clues.ToCore(err)) + + expectDeets.RemoveLocation(driveID, makeLocRef(container1, containerRename)) }, itemsRead: 0, itemsWritten: 0, @@ -1589,7 +1800,7 @@ func runDriveIncrementalTest( acct, category, sel, - creds.AzureTenantID, owner, driveID, container3, + atid, roidn.ID(), driveID, container3, 2, 0, fileDBF) @@ -1604,6 +1815,8 @@ func runDriveIncrementalTest( require.NoError(t, err, "getting drive folder ID", "folder name", container3, clues.ToCore(err)) containerIDs[container3] = ptr.Val(resp.GetId()) + + expectDeets.AddLocation(driveID, container3) }, itemsRead: 2, // 2 .data for 2 files itemsWritten: 6, // read items + 2 directory meta @@ -1631,17 +1844,11 @@ func runDriveIncrementalTest( err = incBO.Run(ctx) require.NoError(t, err, clues.ToCore(err)) - checkBackupIsInManifests(t, ctx, kw, &incBO, sel, sel.ID(), maps.Keys(categories)...) - checkMetadataFilesExist( - t, - ctx, - incBO.Results.BackupID, - kw, - ms, - creds.AzureTenantID, - sel.ID(), - service, - categories) + bupID := incBO.Results.BackupID + + checkBackupIsInManifests(t, ctx, kw, &incBO, sel, roidn.ID(), maps.Keys(categories)...) + checkMetadataFilesExist(t, ctx, bupID, kw, ms, atid, roidn.ID(), service, categories) + deeTD.CheckBackupDetails(t, ctx, bupID, ws, ms, ss, expectDeets, true) // do some additional checks to ensure the incremental dealt with fewer items. // +2 on read/writes to account for metadata: 1 delta and 1 path. @@ -1653,7 +1860,7 @@ func runDriveIncrementalTest( assert.Equal(t, 1, incMB.TimesCalled[events.BackupEnd], "incremental backup-end events") assert.Equal(t, incMB.CalledWith[events.BackupStart][0][events.BackupID], - incBO.Results.BackupID, "incremental backupID pre-declaration") + bupID, "incremental backupID pre-declaration") }) } } @@ -1689,9 +1896,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { uname := ptr.Val(userable.GetUserPrincipalName()) oldsel := selectors.NewOneDriveBackup([]string{uname}) - oldsel.Include(oldsel.Folders([]string{"test"}, selectors.ExactMatch())) + oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel)) - bo, _, kw, ms, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, ffs, 0) + bo, _, kw, ms, _, gc, sel, closer := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, ffs, 0) defer closer() // ensure the initial owner uses name in both cases @@ -1711,7 +1918,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDriveOwnerMigration() { runAndCheckBackup(t, ctx, &bo, mb, false) newsel := selectors.NewOneDriveBackup([]string{uid}) - newsel.Include(newsel.Folders([]string{"test"}, selectors.ExactMatch())) + newsel.Include(selTD.OneDriveBackupFolderScope(newsel)) sel = newsel.SetDiscreteOwnerIDName(uid, uname) var ( @@ -1790,9 +1997,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { sel = selectors.NewSharePointBackup([]string{suite.site}) ) - sel.Include(testdata.SharePointBackupFolderScope(sel)) + sel.Include(selTD.SharePointBackupFolderScope(sel)) - bo, _, kw, _, _, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, control.Toggles{}, version.Backup) + bo, _, kw, _, _, _, sels, closer := prepNewTestBackupOp(t, ctx, mb, sel.Selector, control.Toggles{}, version.Backup) defer closer() runAndCheckBackup(t, ctx, &bo, mb, false) diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index ea710fcf3..608f6a20a 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/connector/mock" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/data" evmock "github.com/alcionai/corso/src/internal/events/mock" "github.com/alcionai/corso/src/internal/kopia" @@ -46,16 +47,28 @@ type mockRestoreProducer struct { onRestore restoreFunc } -type restoreFunc func(id string, ps []path.Path) ([]data.RestoreCollection, error) +type restoreFunc func( + id string, + ps []path.RestorePaths, +) ([]data.RestoreCollection, error) func (mr *mockRestoreProducer) buildRestoreFunc( t *testing.T, oid string, ops []path.Path, ) { - mr.onRestore = func(id string, ps []path.Path) ([]data.RestoreCollection, error) { + mr.onRestore = func( + id string, + ps []path.RestorePaths, + ) ([]data.RestoreCollection, error) { + gotPaths := make([]path.Path, 0, len(ps)) + + for _, rp := range ps { + gotPaths = append(gotPaths, rp.StoragePath) + } + assert.Equal(t, oid, id, "manifest id") - checkPaths(t, ops, ps) + checkPaths(t, ops, gotPaths) return mr.colls, mr.err } @@ -64,11 +77,13 @@ func (mr *mockRestoreProducer) buildRestoreFunc( func (mr *mockRestoreProducer) ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bc kopia.ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) { - mr.gotPaths = append(mr.gotPaths, paths...) + for _, ps := range paths { + mr.gotPaths = append(mr.gotPaths, ps.StoragePath) + } if mr.onRestore != nil { return mr.onRestore(snapshotID, paths) @@ -643,15 +658,15 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems path.OneDriveService.String(), ro, path.FilesCategory.String(), - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "work", "item1", }, true, ) - locationPath1 = path.Builder{}.Append("root:", "work-display-name") + locationPath1 = path.Builder{}.Append(odConsts.RootPathDir, "work-display-name") itemPath2 = makePath( suite.T(), []string{ @@ -659,15 +674,15 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems path.OneDriveService.String(), ro, path.FilesCategory.String(), - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "personal", "item2", }, true, ) - locationPath2 = path.Builder{}.Append("root:", "personal-display-name") + locationPath2 = path.Builder{}.Append(odConsts.RootPathDir, "personal-display-name") itemPath3 = makePath( suite.T(), []string{ diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index 41f934692..55c472f7c 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -47,7 +47,7 @@ type ( ProduceRestoreCollections( ctx context.Context, snapshotID string, - paths []path.Path, + paths []path.RestorePaths, bc kopia.ByteCounter, errs *fault.Bus, ) ([]data.RestoreCollection, error) diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index a402808f2..16e2029f9 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -308,7 +308,7 @@ func collectMetadata( tenantID string, errs *fault.Bus, ) ([]data.RestoreCollection, error) { - paths := []path.Path{} + paths := []path.RestorePaths{} for _, fn := range fileNames { for _, reason := range man.Reasons { @@ -326,7 +326,14 @@ func collectMetadata( With("metadata_file", fn, "category", reason.Category) } - paths = append(paths, p) + dir, err := p.Dir() + if err != nil { + return nil, clues. + Wrap(err, "building metadata collection path"). + With("metadata_file", fn, "category", reason.Category) + } + + paths = append(paths, path.RestorePaths{StoragePath: p, RestorePath: dir}) } } diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index aa481ade7..ccef6e248 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -140,7 +140,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -163,7 +163,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -191,10 +191,10 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) - p, err = contactPath.Append(f, true) + p, err = contactPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } @@ -222,10 +222,10 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { ps := make([]path.Path, 0, len(files)) for _, f := range files { - p, err := emailPath.Append(f, true) + p, err := emailPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) - p, err = contactPath.Append(f, true) + p, err = contactPath.AppendItem(f) assert.NoError(t, err, clues.ToCore(err)) ps = append(ps, p) } diff --git a/src/internal/operations/pathtransformer/restore_path_transformer.go b/src/internal/operations/pathtransformer/restore_path_transformer.go new file mode 100644 index 000000000..8993328f3 --- /dev/null +++ b/src/internal/operations/pathtransformer/restore_path_transformer.go @@ -0,0 +1,181 @@ +package pathtransformer + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" +) + +func locationRef( + ent *details.Entry, + repoRef path.Path, + backupVersion int, +) (*path.Builder, error) { + loc := ent.LocationRef + + // At this backup version all data types should populate LocationRef. + if len(loc) > 0 || backupVersion >= version.OneDrive7LocationRef { + return path.Builder{}.SplitUnescapeAppend(loc) + } + + // We could get an empty LocationRef either because it wasn't populated or it + // was in the root of the data type. + elems := repoRef.Folders() + + if ent.OneDrive != nil || ent.SharePoint != nil { + dp, err := path.ToDrivePath(repoRef) + if err != nil { + return nil, clues.Wrap(err, "fallback for LocationRef") + } + + elems = append([]string{dp.Root}, dp.Folders...) + } + + return path.Builder{}.Append(elems...), nil +} + +func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, error) { + if len(locRef.Elements()) == 0 { + res, err := path.ServicePrefix( + repoRef.Tenant(), + repoRef.ResourceOwner(), + repoRef.Service(), + repoRef.Category()) + if err != nil { + return nil, clues.Wrap(err, "getting prefix for empty location") + } + + return res, nil + } + + return locRef.ToDataLayerPath( + repoRef.Tenant(), + repoRef.ResourceOwner(), + repoRef.Service(), + repoRef.Category(), + false) +} + +func drivePathMerge( + ent *details.Entry, + repoRef path.Path, + locRef *path.Builder, +) (path.Path, error) { + // Try getting the drive ID from the item. Not all details versions had it + // though. + var driveID string + + if ent.SharePoint != nil { + driveID = ent.SharePoint.DriveID + } else if ent.OneDrive != nil { + driveID = ent.OneDrive.DriveID + } + + // Fallback to trying to get from RepoRef. + if len(driveID) == 0 { + odp, err := path.ToDrivePath(repoRef) + if err != nil { + return nil, clues.Wrap(err, "fallback getting DriveID") + } + + driveID = odp.DriveID + } + + return basicLocationPath( + repoRef, + path.BuildDriveLocation(driveID, locRef.Elements()...)) +} + +func makeRestorePathsForEntry( + ctx context.Context, + backupVersion int, + ent *details.Entry, +) (path.RestorePaths, error) { + res := path.RestorePaths{} + + repoRef, err := path.FromDataLayerPath(ent.RepoRef, true) + if err != nil { + err = clues.Wrap(err, "parsing RepoRef"). + WithClues(ctx). + With("repo_ref", clues.Hide(ent.RepoRef), "location_ref", clues.Hide(ent.LocationRef)) + + return res, err + } + + res.StoragePath = repoRef + ctx = clues.Add(ctx, "repo_ref", repoRef) + + // Get the LocationRef so we can munge it onto our path. + locRef, err := locationRef(ent, repoRef, backupVersion) + if err != nil { + err = clues.Wrap(err, "parsing LocationRef after reduction"). + WithClues(ctx). + With("location_ref", clues.Hide(ent.LocationRef)) + + return res, err + } + + ctx = clues.Add(ctx, "location_ref", locRef) + + // Now figure out what type of ent it is and munge the path accordingly. + // Eventually we're going to need munging for: + // * Exchange Calendars (different folder handling) + // * Exchange Email/Contacts + // * OneDrive/SharePoint (needs drive information) + switch true { + case ent.Exchange != nil: + // TODO(ashmrtn): Eventually make Events have it's own function to handle + // setting the restore destination properly. + res.RestorePath, err = basicLocationPath(repoRef, locRef) + case ent.OneDrive != nil || + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.SharePointLibrary) || + (ent.SharePoint != nil && ent.SharePoint.ItemType == details.OneDriveItem): + res.RestorePath, err = drivePathMerge(ent, repoRef, locRef) + default: + return res, clues.New("unknown entry type").WithClues(ctx) + } + + if err != nil { + return res, clues.Wrap(err, "generating RestorePath").WithClues(ctx) + } + + return res, nil +} + +// GetPaths takes a set of filtered details entries and returns a set of +// RestorePaths for the entries. +func GetPaths( + ctx context.Context, + backupVersion int, + items []*details.Entry, + errs *fault.Bus, +) ([]path.RestorePaths, error) { + var ( + paths = make([]path.RestorePaths, len(items)) + el = errs.Local() + ) + + for i, ent := range items { + if el.Failure() != nil { + break + } + + restorePaths, err := makeRestorePathsForEntry(ctx, backupVersion, ent) + if err != nil { + el.AddRecoverable(clues.Wrap(err, "getting restore paths")) + continue + } + + paths[i] = restorePaths + } + + logger.Ctx(ctx).Infof("found %d details entries to restore", len(paths)) + + return paths, el.Failure() +} diff --git a/src/internal/operations/pathtransformer/restore_path_transformer_test.go b/src/internal/operations/pathtransformer/restore_path_transformer_test.go new file mode 100644 index 000000000..57381c3cf --- /dev/null +++ b/src/internal/operations/pathtransformer/restore_path_transformer_test.go @@ -0,0 +1,340 @@ +package pathtransformer_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/operations/pathtransformer" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/details/testdata" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type RestorePathTransformerUnitSuite struct { + tester.Suite +} + +func TestRestorePathTransformerUnitSuite(t *testing.T) { + suite.Run(t, &RestorePathTransformerUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *RestorePathTransformerUnitSuite) TestGetPaths() { + type expectPaths struct { + storage string + restore string + isRestorePrefix bool + } + + toRestore := func( + repoRef path.Path, + unescapedFolders ...string, + ) string { + return path.Builder{}. + Append( + repoRef.Tenant(), + repoRef.Service().String(), + repoRef.ResourceOwner(), + repoRef.Category().String()). + Append(unescapedFolders...). + String() + } + + var ( + driveID = "some-drive-id" + extraItemName = "some-item" + SharePointRootItemPath = testdata.SharePointRootPath.MustAppend(extraItemName, true) + ) + + table := []struct { + name string + backupVersion int + input []*details.Entry + expectErr assert.ErrorAssertionFunc + expected []expectPaths + }{ + { + name: "SharePoint List Errors", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointList, + }, + }, + }, + }, + expectErr: assert.Error, + }, + { + name: "SharePoint Page Errors", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointPage, + }, + }, + }, + }, + expectErr: assert.Error, + }, + { + name: "SharePoint old format, item in root", + // No version bump for the change so we always have to check for this. + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + LocationRef: SharePointRootItemPath.Loc.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.OneDriveItem, + DriveID: driveID, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: SharePointRootItemPath.RR.String(), + restore: toRestore( + SharePointRootItemPath.RR, + append( + []string{"drives", driveID}, + SharePointRootItemPath.Loc.Elements()...)...), + }, + }, + }, + { + name: "SharePoint, no LocationRef, no DriveID, item in root", + backupVersion: version.OneDrive6NameInMeta, + input: []*details.Entry{ + { + RepoRef: SharePointRootItemPath.RR.String(), + ItemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointLibrary, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: SharePointRootItemPath.RR.String(), + restore: toRestore( + SharePointRootItemPath.RR, + append( + []string{"drives"}, + // testdata path has '.d' on the drives folder we need to remove. + SharePointRootItemPath.RR.Folders()[1:]...)...), + }, + }, + }, + { + name: "OneDrive, nested item", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.OneDriveItemPath2.RR.String(), + LocationRef: testdata.OneDriveItemPath2.Loc.String(), + ItemInfo: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + ItemType: details.OneDriveItem, + DriveID: driveID, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.OneDriveItemPath2.RR.String(), + restore: toRestore( + testdata.OneDriveItemPath2.RR, + append( + []string{"drives", driveID}, + testdata.OneDriveItemPath2.Loc.Elements()...)...), + }, + }, + }, + { + name: "Exchange Email, extra / in path", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Email, no LocationRef, extra / in path", + backupVersion: version.OneDrive7LocationRef, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Contact", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeContactsItemPath1.RR.String(), + LocationRef: testdata.ExchangeContactsItemPath1.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeContact, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeContactsItemPath1.RR.String(), + restore: toRestore( + testdata.ExchangeContactsItemPath1.RR, + testdata.ExchangeContactsItemPath1.Loc.Elements()...), + }, + }, + }, + { + name: "Exchange Contact, root dir", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeContactsItemPath1.RR.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeContact, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeContactsItemPath1.RR.String(), + restore: toRestore(testdata.ExchangeContactsItemPath1.RR, "tmp"), + isRestorePrefix: true, + }, + }, + }, + { + name: "Exchange Event", + backupVersion: version.All8MigrateUserPNToID, + input: []*details.Entry{ + { + RepoRef: testdata.ExchangeEmailItemPath3.RR.String(), + LocationRef: testdata.ExchangeEmailItemPath3.Loc.String(), + ItemInfo: details.ItemInfo{ + Exchange: &details.ExchangeInfo{ + ItemType: details.ExchangeMail, + }, + }, + }, + }, + expectErr: assert.NoError, + expected: []expectPaths{ + { + storage: testdata.ExchangeEmailItemPath3.RR.String(), + restore: toRestore( + testdata.ExchangeEmailItemPath3.RR, + testdata.ExchangeEmailItemPath3.Loc.Elements()...), + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + + paths, err := pathtransformer.GetPaths( + ctx, + test.backupVersion, + test.input, + fault.New(true)) + test.expectErr(t, err, clues.ToCore(err)) + + if err != nil { + return + } + + expected := make([]path.RestorePaths, 0, len(test.expected)) + + for _, e := range test.expected { + tmp := path.RestorePaths{} + p, err := path.FromDataLayerPath(e.storage, true) + require.NoError(t, err, "parsing expected storage path", clues.ToCore(err)) + + tmp.StoragePath = p + + p, err = path.FromDataLayerPath(e.restore, false) + require.NoError(t, err, "parsing expected restore path", clues.ToCore(err)) + + if e.isRestorePrefix { + p, err = p.Dir() + require.NoError(t, err, "getting service prefix", clues.ToCore(err)) + } + + tmp.RestorePath = p + + expected = append(expected, tmp) + } + + assert.ElementsMatch(t, expected, paths) + }) + } +} diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 370869801..28dbb5e1a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/operations/pathtransformer" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/pkg/account" @@ -349,36 +350,15 @@ func formatDetailsForRestoration( sel selectors.Selector, deets *details.Details, errs *fault.Bus, -) ([]path.Path, error) { +) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) if err != nil { return nil, err } - var ( - fdsPaths = fds.Paths() - paths = make([]path.Path, len(fdsPaths)) - shortRefs = make([]string, len(fdsPaths)) - el = errs.Local() - ) - - for i := range fdsPaths { - if el.Failure() != nil { - break - } - - p, err := path.FromDataLayerPath(fdsPaths[i], true) - if err != nil { - el.AddRecoverable(clues. - Wrap(err, "parsing details path after reduction"). - WithMap(clues.In(ctx)). - With("path", fdsPaths[i])) - - continue - } - - paths[i] = p - shortRefs[i] = p.ShortRef() + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) + if err != nil { + return nil, clues.Wrap(err, "getting restore paths") } if sel.Service == selectors.ServiceOneDrive { @@ -388,7 +368,5 @@ func formatDetailsForRestoration( } } - logger.Ctx(ctx).With("short_refs", shortRefs).Infof("found %d details entries to restore", len(shortRefs)) - - return paths, el.Failure() + return paths, nil } diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 320f2933d..ea42a5c4f 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -55,7 +55,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { gc = &mock.GraphConnector{} acct = account.Account{} now = time.Now() - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") ) table := []struct { @@ -220,7 +220,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { sw = &store.Wrapper{} gc = &mock.GraphConnector{} acct = tester.NewM365Account(suite.T()) - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") opts = control.Defaults() ) @@ -392,7 +392,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { { name: "Exchange_Restore", owner: tester.M365UserID(suite.T()), - dest: tester.DefaultTestRestoreDestination(), + dest: tester.DefaultTestRestoreDestination(""), getSelector: func(t *testing.T, owners []string) selectors.Selector { rsel := selectors.NewExchangeRestore(owners) rsel.Include(rsel.AllData()) @@ -464,7 +464,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoResults() { var ( t = suite.T() - dest = tester.DefaultTestRestoreDestination() + dest = tester.DefaultTestRestoreDestination("") mb = evmock.NewBus() ) diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index bc86687ef..146f0d1c7 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -262,12 +262,22 @@ func read( return clues.Stack(err).WithClues(ctx) } + pd, err := p.Dir() + if err != nil { + return clues.Stack(err).WithClues(ctx) + } + ctx = clues.Add(ctx, "snapshot_id", snapshotID) cs, err := rer.ProduceRestoreCollections( ctx, snapshotID, - []path.Path{p}, + []path.RestorePaths{ + { + StoragePath: p, + RestorePath: pd, + }, + }, &stats.ByteCounter{}, errs) if err != nil { diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 8a002fd2c..14a4f54d9 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -106,7 +106,7 @@ func readTestConfig() (map[string]string, error) { testEnv := map[string]string{} fallbackTo(testEnv, TestCfgStorageProvider, vpr.GetString(TestCfgStorageProvider)) fallbackTo(testEnv, TestCfgAccountProvider, vpr.GetString(TestCfgAccountProvider)) - fallbackTo(testEnv, TestCfgBucket, vpr.GetString(TestCfgBucket), "test-corso-repo-init") + fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket), "test-corso-repo-init") fallbackTo(testEnv, TestCfgEndpoint, vpr.GetString(TestCfgEndpoint), "s3.amazonaws.com") fallbackTo(testEnv, TestCfgPrefix, vpr.GetString(TestCfgPrefix)) fallbackTo(testEnv, TestCfgAzureTenantID, os.Getenv(account.AzureTenantID), vpr.GetString(TestCfgAzureTenantID)) diff --git a/src/internal/tester/restore_destination.go b/src/internal/tester/restore_destination.go index b22e8593b..af247258d 100644 --- a/src/internal/tester/restore_destination.go +++ b/src/internal/tester/restore_destination.go @@ -1,11 +1,26 @@ package tester import ( + "strings" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/control" ) -func DefaultTestRestoreDestination() control.RestoreDestination { - // Use microsecond granularity to help reduce collisions. - return control.DefaultRestoreDestination(dttm.SafeForTesting) +const RestoreFolderPrefix = "Corso_Test" + +func DefaultTestRestoreDestination(namespace string) control.RestoreDestination { + var ( + dest = control.DefaultRestoreDestination(dttm.SafeForTesting) + sft = dttm.FormatNow(dttm.SafeForTesting) + ) + + parts := []string{RestoreFolderPrefix, namespace, sft} + if len(namespace) == 0 { + parts = []string{RestoreFolderPrefix, sft} + } + + dest.ContainerName = strings.Join(parts, "_") + + return dest } diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 7c6466d3c..d6aae6bbc 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" @@ -242,9 +243,9 @@ func oneDriveishEntry(t *testing.T, id string, size int, it ItemType) Entry { "tenant-id", "user-id", []string{ - "drives", + odConsts.DrivesPathDir, "drive-id", - "root:", + odConsts.RootPathDir, "Inbox", "folder1", id, @@ -408,7 +409,7 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_LocationFolders() { { ItemInfo: ItemInfo{ Folder: &FolderInfo{ - DisplayName: "root:", + DisplayName: odConsts.RootPathDir, ItemType: FolderItem, DriveName: "drive-name", DriveID: "drive-id", @@ -416,7 +417,7 @@ func (suite *DetailsUnitSuite) TestDetailsAdd_LocationFolders() { }, }, { - LocationRef: "root:", + LocationRef: odConsts.RootPathDir, ItemInfo: ItemInfo{ Folder: &FolderInfo{ DisplayName: "Inbox", @@ -958,7 +959,7 @@ func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() { "a-user", []string{ "drive-id", - "root:", + odConsts.RootPathDir, "folder", name + "-id", }) @@ -971,7 +972,7 @@ func (suite *DetailsUnitSuite) TestBuilder_Add_shortRefsUniqueFromFolder() { "a-user", []string{ "drive-id", - "root:", + odConsts.RootPathDir, "folder", name + "-id", name, @@ -1060,7 +1061,7 @@ func (suite *DetailsUnitSuite) TestUpdateItem() { ) newExchangePB := path.Builder{}.Append(folder2) - newOneDrivePB := path.Builder{}.Append("root:", folder2) + newOneDrivePB := path.Builder{}.Append(odConsts.RootPathDir, folder2) table := []struct { name string diff --git a/src/pkg/backup/details/testdata/in_deets.go b/src/pkg/backup/details/testdata/in_deets.go new file mode 100644 index 000000000..b15c50f17 --- /dev/null +++ b/src/pkg/backup/details/testdata/in_deets.go @@ -0,0 +1,368 @@ +package testdata + +import ( + "context" + "strings" + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// location set handling +// --------------------------------------------------------------------------- + +var exists = struct{}{} + +type locSet struct { + // map [locationRef] map [itemRef] {} + // refs may be either the canonical ent refs, or something else, + // so long as they are consistent for the test in question + Locations map[string]map[string]struct{} + Deleted map[string]map[string]struct{} +} + +func newLocSet() *locSet { + return &locSet{ + Locations: map[string]map[string]struct{}{}, + Deleted: map[string]map[string]struct{}{}, + } +} + +func (ls *locSet) AddItem(locationRef, itemRef string) { + ls.AddLocation(locationRef) + + ls.Locations[locationRef][itemRef] = exists + delete(ls.Deleted[locationRef], itemRef) +} + +func (ls *locSet) RemoveItem(locationRef, itemRef string) { + delete(ls.Locations[locationRef], itemRef) + + if _, ok := ls.Deleted[locationRef]; !ok { + ls.Deleted[locationRef] = map[string]struct{}{} + } + + ls.Deleted[locationRef][itemRef] = exists +} + +func (ls *locSet) MoveItem(fromLocation, toLocation, ir string) { + ls.RemoveItem(fromLocation, ir) + ls.AddItem(toLocation, ir) +} + +func (ls *locSet) AddLocation(locationRef string) { + if _, ok := ls.Locations[locationRef]; !ok { + ls.Locations[locationRef] = map[string]struct{}{} + } + // don't purge previously deleted items, or child locations. + // Assumption is that their itemRef is unique, and still deleted. + delete(ls.Deleted, locationRef) +} + +func (ls *locSet) RemoveLocation(locationRef string) { + ss := ls.Subset(locationRef) + + for lr := range ss.Locations { + items := ls.Locations[lr] + + delete(ls.Locations, lr) + + if _, ok := ls.Deleted[lr]; !ok { + ls.Deleted[lr] = map[string]struct{}{} + } + + for ir := range items { + ls.Deleted[lr][ir] = exists + } + } +} + +// MoveLocation takes the LAST elemet in the fromLocation (and all) +// children matching the prefix, and relocates it as a child of toLocation. +// ex: MoveLocation("/a/b/c", "/d") will move all entries with the prefix +// "/a/b/c" into "/d/c". This also deletes all "/a/b/c" entries and children. +// assumes item IDs don't change across the migration. If item IDs do change, +// that difference will need to be handled manually by the caller. +// returns the base folder's new location (ex: /d/c) +func (ls *locSet) MoveLocation(fromLocation, toLocation string) string { + fromBuilder := path.Builder{}.Append(path.Split(fromLocation)...) + toBuilder := path.Builder{}.Append(path.Split(toLocation)...).Append(fromBuilder.LastElem()) + + ls.RenameLocation(fromBuilder.String(), toBuilder.String()) + + return toBuilder.String() +} + +func (ls *locSet) RenameLocation(fromLocation, toLocation string) { + ss := ls.Subset(fromLocation) + fromBuilder := path.Builder{}.Append(path.Split(fromLocation)...) + toBuilder := path.Builder{}.Append(path.Split(toLocation)...) + + for lr, items := range ss.Locations { + lrBuilder := path.Builder{}.Append(path.Split(lr)...) + lrBuilder.UpdateParent(fromBuilder, toBuilder) + + newLoc := lrBuilder.String() + + for ir := range items { + ls.RemoveItem(lr, ir) + ls.AddItem(newLoc, ir) + } + + ls.RemoveLocation(lr) + ls.AddLocation(newLoc) + } +} + +// Subset produces a new locSet containing only Items and Locations +// whose location matches the locationPfx +func (ls *locSet) Subset(locationPfx string) *locSet { + ss := newLocSet() + + for lr, items := range ls.Locations { + if strings.HasPrefix(lr, locationPfx) { + ss.AddLocation(lr) + + for ir := range items { + ss.AddItem(lr, ir) + } + } + } + + return ss +} + +// --------------------------------------------------------------------------- +// The goal of InDeets is to provide a struct and interface which allows +// tests to predict not just the elements within a set of details entries, +// but also their changes (relocation, renaming, etc) in a way that consolidates +// building an "expected set" of details entries that can be compared against +// the details results after a backup. +// --------------------------------------------------------------------------- + +// InDeets is a helper for comparing details state in tests +// across backup instances. +type InDeets struct { + // only: tenantID/service/resourceOwnerID + RRPrefix string + // map of container setting the uniqueness boundary for location + // ref entries (eg, data type like email, contacts, etc, or + // drive id) to the unique entries in that set. + Sets map[string]*locSet +} + +func NewInDeets(repoRefPrefix string) *InDeets { + return &InDeets{ + RRPrefix: repoRefPrefix, + Sets: map[string]*locSet{}, + } +} + +func (id *InDeets) getSet(set string) *locSet { + s, ok := id.Sets[set] + if ok { + return s + } + + return newLocSet() +} + +func (id *InDeets) AddAll(deets details.Details, ws whatSet) { + if id.Sets == nil { + id.Sets = map[string]*locSet{} + } + + for _, ent := range deets.Entries { + set, err := ws(ent) + if err != nil { + set = err.Error() + } + + dir := ent.LocationRef + + if ent.Folder != nil { + dir = dir + ent.Folder.DisplayName + id.AddLocation(set, dir) + } else { + id.AddItem(set, ent.LocationRef, ent.ItemRef) + } + } +} + +func (id *InDeets) AddItem(set, locationRef, itemRef string) { + id.getSet(set).AddItem(locationRef, itemRef) +} + +func (id *InDeets) RemoveItem(set, locationRef, itemRef string) { + id.getSet(set).RemoveItem(locationRef, itemRef) +} + +func (id *InDeets) MoveItem(set, fromLocation, toLocation, ir string) { + id.getSet(set).MoveItem(fromLocation, toLocation, ir) +} + +func (id *InDeets) AddLocation(set, locationRef string) { + id.getSet(set).AddLocation(locationRef) +} + +// RemoveLocation removes the provided location, and all children +// of that location. +func (id *InDeets) RemoveLocation(set, locationRef string) { + id.getSet(set).RemoveLocation(locationRef) +} + +// MoveLocation takes the LAST elemet in the fromLocation (and all) +// children matching the prefix, and relocates it as a child of toLocation. +// ex: MoveLocation("/a/b/c", "/d") will move all entries with the prefix +// "/a/b/c" into "/d/c". This also deletes all "/a/b/c" entries and children. +// assumes item IDs don't change across the migration. If item IDs do change, +// that difference will need to be handled manually by the caller. +// returns the base folder's new location (ex: /d/c) +func (id *InDeets) MoveLocation(set, fromLocation, toLocation string) string { + return id.getSet(set).MoveLocation(fromLocation, toLocation) +} + +func (id *InDeets) RenameLocation(set, fromLocation, toLocation string) { + id.getSet(set).RenameLocation(fromLocation, toLocation) +} + +// Subset produces a new locSet containing only Items and Locations +// whose location matches the locationPfx +func (id *InDeets) Subset(set, locationPfx string) *locSet { + return id.getSet(set).Subset(locationPfx) +} + +// --------------------------------------------------------------------------- +// whatSet helpers for extracting a set identifier from an arbitrary repoRef +// --------------------------------------------------------------------------- + +type whatSet func(details.Entry) (string, error) + +// common whatSet parser that extracts the service category from +// a repoRef. +func CategoryFromRepoRef(ent details.Entry) (string, error) { + p, err := path.FromDataLayerPath(ent.RepoRef, false) + if err != nil { + return "", err + } + + return p.Category().String(), nil +} + +// common whatSet parser that extracts the driveID from a repoRef. +func DriveIDFromRepoRef(ent details.Entry) (string, error) { + p, err := path.FromDataLayerPath(ent.RepoRef, false) + if err != nil { + return "", err + } + + odp, err := path.ToDrivePath(p) + if err != nil { + return "", err + } + + return odp.DriveID, nil +} + +// --------------------------------------------------------------------------- +// helpers and comparators +// --------------------------------------------------------------------------- + +func CheckBackupDetails( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupID model.StableID, + ws whatSet, + ms *kopia.ModelStore, + ssr streamstore.Reader, + expect *InDeets, + // standard check is assert.Subset due to issues of external data cross- + // pollination. This should be true if the backup contains a unique directory + // of data. + mustEqualFolders bool, +) { + deets, result := GetDeetsInBackup(t, ctx, backupID, "", "", path.UnknownService, ws, ms, ssr) + + t.Log("details entries in result") + + for _, ent := range deets.Entries { + if ent.Folder == nil { + t.Log(ent.LocationRef) + t.Log(ent.ItemRef) + } + + assert.Truef( + t, + strings.HasPrefix(ent.RepoRef, expect.RRPrefix), + "all details should begin with the expected prefix\nwant: %s\ngot: %s", + expect.RRPrefix, ent.RepoRef) + } + + for set := range expect.Sets { + check := assert.Subsetf + + if mustEqualFolders { + check = assert.ElementsMatchf + } + + check( + t, + maps.Keys(result.Sets[set].Locations), + maps.Keys(expect.Sets[set].Locations), + "results in %s missing expected location", set) + + for lr, items := range expect.Sets[set].Deleted { + _, ok := result.Sets[set].Locations[lr] + assert.Falsef(t, ok, "deleted location in %s found in result: %s", set, lr) + + for ir := range items { + _, ok := result.Sets[set].Locations[lr][ir] + assert.Falsef(t, ok, "deleted item in %s found in result: %s", set, lr) + } + } + } +} + +func GetDeetsInBackup( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + backupID model.StableID, + tid, resourceOwner string, + service path.ServiceType, + ws whatSet, + ms *kopia.ModelStore, + ssr streamstore.Reader, +) (details.Details, *InDeets) { + bup := backup.Backup{} + + err := ms.Get(ctx, model.BackupSchema, backupID, &bup) + require.NoError(t, err, clues.ToCore(err)) + + ssid := bup.StreamStoreID + require.NotEmpty(t, ssid, "stream store ID") + + var deets details.Details + err = ssr.Read( + ctx, + ssid, + streamstore.DetailsReader(details.UnmarshalTo(&deets)), + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + + id := NewInDeets(path.Builder{}.Append(tid, service.String(), resourceOwner).String()) + id.AddAll(deets, ws) + + return deets, id +} diff --git a/src/pkg/backup/details/testdata/in_deets_test.go b/src/pkg/backup/details/testdata/in_deets_test.go new file mode 100644 index 000000000..81beb0b0f --- /dev/null +++ b/src/pkg/backup/details/testdata/in_deets_test.go @@ -0,0 +1,445 @@ +package testdata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/tester" +) + +type LocSetUnitSuite struct { + tester.Suite +} + +func TestLocSetUnitSuite(t *testing.T) { + suite.Run(t, &LocSetUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +const ( + l1 = "lr_1" + l2 = "lr_2" + l13 = "lr_1/lr_3" + l14 = "lr_1/lr_4" + i1 = "ir_1" + i2 = "ir_2" + i3 = "ir_3" + i4 = "ir_4" +) + +func (suite *LocSetUnitSuite) TestAdd() { + t := suite.T() + + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddLocation(l2) + + assert.ElementsMatch(t, []string{l1, l2}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) +} + +func (suite *LocSetUnitSuite) TestRemove() { + t := suite.T() + + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[l14])) + + // nop removal + ls.RemoveItem(l2, i1) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + + // item removal + ls.RemoveItem(l1, i2) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + + // nop location removal + ls.RemoveLocation(l2) + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + + // non-cascading location removal + ls.RemoveLocation(l13) + assert.ElementsMatch(t, []string{l1, l14}, maps.Keys(ls.Locations)) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[l14])) + + // cascading location removal + ls.RemoveLocation(l1) + assert.Empty(t, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) +} + +func (suite *LocSetUnitSuite) TestSubset() { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + table := []struct { + name string + locPfx string + expect func(*testing.T, *locSet) + }{ + { + name: "nop", + locPfx: l2, + expect: func(t *testing.T, ss *locSet) { + assert.Empty(t, maps.Keys(ss.Locations)) + }, + }, + { + name: "no items", + locPfx: l13, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l13}, maps.Keys(ss.Locations)) + assert.Empty(t, maps.Keys(ss.Locations[l13])) + }, + }, + { + name: "non-cascading", + locPfx: l14, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l14}, maps.Keys(ss.Locations)) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ss.Locations[l14])) + }, + }, + { + name: "cascading", + locPfx: l1, + expect: func(t *testing.T, ss *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ss.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ss.Locations[l1])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ss.Locations[l14])) + assert.Empty(t, maps.Keys(ss.Locations[l13])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + test.expect(t, ls.Subset(test.locPfx)) + }) + } +} + +func (suite *LocSetUnitSuite) TestRename() { + t := suite.T() + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + + table := []struct { + name string + from string + to string + expect func(*testing.T, *locSet) + }{ + { + name: "nop", + from: l2, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "no items", + from: l13, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, "foo", l14}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "with items", + from: l14, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, "foo"}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations["foo"])) + }, + }, + { + name: "cascading locations", + from: l1, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{"foo", "foo/lr_3", "foo/lr_4"}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations["foo"])) + assert.Empty(t, maps.Keys(ls.Locations["foo/lr_3"])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations["foo/lr_4"])) + }, + }, + { + name: "to existing location", + from: l14, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i1, i2, i3, i4}, maps.Keys(ls.Locations[l1])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + ls.RenameLocation(test.from, test.to) + test.expect(t, ls) + }) + } +} + +func (suite *LocSetUnitSuite) TestItem() { + t := suite.T() + b4 := "bar/lr_4" + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + ls.AddItem(b4, "fnord") + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + assert.ElementsMatch(t, []string{"fnord"}, maps.Keys(ts.Locations[b4])) + + table := []struct { + name string + item string + from string + to string + expect func(*testing.T, *locSet) + }{ + { + name: "nop item", + item: "floob", + from: l2, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2, "floob"}, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + }, + }, + { + name: "nop origin", + item: i1, + from: "smarf", + to: l2, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l2])) + assert.Empty(t, maps.Keys(ls.Locations["smarf"])) + }, + }, + { + name: "new location", + item: i1, + from: l1, + to: "fnords", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations["fnords"])) + }, + }, + { + name: "existing location", + item: i1, + from: l1, + to: l2, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i2}, maps.Keys(ls.Locations[l1])) + assert.ElementsMatch(t, []string{i1}, maps.Keys(ls.Locations[l2])) + }, + }, + { + name: "same location", + item: i1, + from: l1, + to: l1, + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[l1])) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + ls.MoveItem(test.from, test.to, test.item) + test.expect(t, ls) + }) + } +} + +func (suite *LocSetUnitSuite) TestMoveLocation() { + t := suite.T() + b4 := "bar/lr_4" + + makeSet := func() *locSet { + ls := newLocSet() + + ls.AddItem(l1, i1) + ls.AddItem(l1, i2) + ls.AddLocation(l13) + ls.AddItem(l14, i3) + ls.AddItem(l14, i4) + ls.AddItem(b4, "fnord") + + return ls + } + + ts := makeSet() + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ts.Locations)) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ts.Locations[l1])) + assert.Empty(t, maps.Keys(ts.Locations[l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ts.Locations[l14])) + assert.ElementsMatch(t, []string{"fnord"}, maps.Keys(ts.Locations[b4])) + + table := []struct { + name string + from string + to string + expect func(*testing.T, *locSet) + expectNewLoc string + }{ + { + name: "nop root", + from: l2, + to: "", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l2])) + }, + expectNewLoc: l2, + }, + { + name: "nop child", + from: l2, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations["foo"])) + assert.Empty(t, maps.Keys(ls.Locations["foo/"+l2])) + }, + expectNewLoc: "foo/" + l2, + }, + { + name: "no items", + from: l13, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + newLoc := "foo/lr_3" + assert.ElementsMatch(t, []string{l1, newLoc, l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.Empty(t, maps.Keys(ls.Locations[newLoc])) + }, + expectNewLoc: "foo/lr_3", + }, + { + name: "with items", + from: l14, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + newLoc := "foo/lr_4" + assert.ElementsMatch(t, []string{l1, l13, newLoc, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[newLoc])) + }, + expectNewLoc: "foo/lr_4", + }, + { + name: "cascading locations", + from: l1, + to: "foo", + expect: func(t *testing.T, ls *locSet) { + pfx := "foo/" + assert.ElementsMatch(t, []string{pfx + l1, pfx + l13, pfx + l14, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l1])) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations[l13])) + assert.ElementsMatch(t, []string{i1, i2}, maps.Keys(ls.Locations[pfx+l1])) + assert.Empty(t, maps.Keys(ls.Locations[pfx+l13])) + assert.ElementsMatch(t, []string{i3, i4}, maps.Keys(ls.Locations[pfx+l14])) + }, + expectNewLoc: "foo/" + l1, + }, + { + name: "to existing location", + from: l14, + to: "bar", + expect: func(t *testing.T, ls *locSet) { + assert.ElementsMatch(t, []string{l1, l13, b4}, maps.Keys(ls.Locations)) + assert.Empty(t, maps.Keys(ls.Locations[l14])) + assert.Empty(t, maps.Keys(ls.Locations["bar"])) + assert.ElementsMatch(t, []string{"fnord", i3, i4}, maps.Keys(ls.Locations[b4])) + }, + expectNewLoc: b4, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + ls := makeSet() + + newLoc := ls.MoveLocation(test.from, test.to) + test.expect(t, ls) + assert.Equal(t, test.expectNewLoc, newLoc) + }) + } +} diff --git a/src/pkg/backup/details/testdata/testdata.go b/src/pkg/backup/details/testdata/testdata.go index 0b770c050..0d98ec7df 100644 --- a/src/pkg/backup/details/testdata/testdata.go +++ b/src/pkg/backup/details/testdata/testdata.go @@ -25,7 +25,7 @@ func mustParsePath(ref string, isItem bool) path.Path { // path with the element appended to it. Panics if the path cannot be parsed. // Useful for simple variable assignments. func mustAppendPath(p path.Path, newElement string, isItem bool) path.Path { - newP, err := p.Append(newElement, isItem) + newP, err := p.Append(isItem, newElement) if err != nil { panic(err) } @@ -54,10 +54,10 @@ func locFromRepo(rr path.Path, isItem bool) *path.Builder { type repoRefAndLocRef struct { RR path.Path - loc *path.Builder + Loc *path.Builder } -func (p repoRefAndLocRef) mustAppend(newElement string, isItem bool) repoRefAndLocRef { +func (p repoRefAndLocRef) MustAppend(newElement string, isItem bool) repoRefAndLocRef { e := newElement + folderSuffix if isItem { @@ -68,7 +68,7 @@ func (p repoRefAndLocRef) mustAppend(newElement string, isItem bool) repoRefAndL RR: mustAppendPath(p.RR, e, isItem), } - res.loc = locFromRepo(res.RR, isItem) + res.Loc = locFromRepo(res.RR, isItem) return res } @@ -85,7 +85,7 @@ func (p repoRefAndLocRef) FolderLocation() string { lastElem = f[len(f)-2] } - return p.loc.Append(strings.TrimSuffix(lastElem, folderSuffix)).String() + return p.Loc.Append(strings.TrimSuffix(lastElem, folderSuffix)).String() } func mustPathRep(ref string, isItem bool) repoRefAndLocRef { @@ -115,7 +115,7 @@ func mustPathRep(ref string, isItem bool) repoRefAndLocRef { } res.RR = rr - res.loc = locFromRepo(rr, isItem) + res.Loc = locFromRepo(rr, isItem) return res } @@ -138,12 +138,12 @@ var ( Time4 = time.Date(2023, 10, 21, 10, 0, 0, 0, time.UTC) ExchangeEmailInboxPath = mustPathRep("tenant-id/exchange/user-id/email/Inbox", false) - ExchangeEmailBasePath = ExchangeEmailInboxPath.mustAppend("subfolder", false) - ExchangeEmailBasePath2 = ExchangeEmailInboxPath.mustAppend("othersubfolder/", false) - ExchangeEmailBasePath3 = ExchangeEmailBasePath2.mustAppend("subsubfolder", false) - ExchangeEmailItemPath1 = ExchangeEmailBasePath.mustAppend(ItemName1, true) - ExchangeEmailItemPath2 = ExchangeEmailBasePath2.mustAppend(ItemName2, true) - ExchangeEmailItemPath3 = ExchangeEmailBasePath3.mustAppend(ItemName3, true) + ExchangeEmailBasePath = ExchangeEmailInboxPath.MustAppend("subfolder", false) + ExchangeEmailBasePath2 = ExchangeEmailInboxPath.MustAppend("othersubfolder/", false) + ExchangeEmailBasePath3 = ExchangeEmailBasePath2.MustAppend("subsubfolder", false) + ExchangeEmailItemPath1 = ExchangeEmailBasePath.MustAppend(ItemName1, true) + ExchangeEmailItemPath2 = ExchangeEmailBasePath2.MustAppend(ItemName2, true) + ExchangeEmailItemPath3 = ExchangeEmailBasePath3.MustAppend(ItemName3, true) ExchangeEmailItems = []details.Entry{ { @@ -151,7 +151,7 @@ var ( ShortRef: ExchangeEmailItemPath1.RR.ShortRef(), ParentRef: ExchangeEmailItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath1.ItemLocation(), - LocationRef: ExchangeEmailItemPath1.loc.String(), + LocationRef: ExchangeEmailItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -166,7 +166,7 @@ var ( ShortRef: ExchangeEmailItemPath2.RR.ShortRef(), ParentRef: ExchangeEmailItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath2.ItemLocation(), - LocationRef: ExchangeEmailItemPath2.loc.String(), + LocationRef: ExchangeEmailItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -181,7 +181,7 @@ var ( ShortRef: ExchangeEmailItemPath3.RR.ShortRef(), ParentRef: ExchangeEmailItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEmailItemPath3.ItemLocation(), - LocationRef: ExchangeEmailItemPath3.loc.String(), + LocationRef: ExchangeEmailItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeMail, @@ -194,10 +194,10 @@ var ( } ExchangeContactsRootPath = mustPathRep("tenant-id/exchange/user-id/contacts/contacts", false) - ExchangeContactsBasePath = ExchangeContactsRootPath.mustAppend("contacts", false) - ExchangeContactsBasePath2 = ExchangeContactsRootPath.mustAppend("morecontacts", false) - ExchangeContactsItemPath1 = ExchangeContactsBasePath.mustAppend(ItemName1, true) - ExchangeContactsItemPath2 = ExchangeContactsBasePath2.mustAppend(ItemName2, true) + ExchangeContactsBasePath = ExchangeContactsRootPath.MustAppend("contacts", false) + ExchangeContactsBasePath2 = ExchangeContactsRootPath.MustAppend("morecontacts", false) + ExchangeContactsItemPath1 = ExchangeContactsBasePath.MustAppend(ItemName1, true) + ExchangeContactsItemPath2 = ExchangeContactsBasePath2.MustAppend(ItemName2, true) ExchangeContactsItems = []details.Entry{ { @@ -205,7 +205,7 @@ var ( ShortRef: ExchangeContactsItemPath1.RR.ShortRef(), ParentRef: ExchangeContactsItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeContactsItemPath1.ItemLocation(), - LocationRef: ExchangeContactsItemPath1.loc.String(), + LocationRef: ExchangeContactsItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -218,7 +218,7 @@ var ( ShortRef: ExchangeContactsItemPath2.RR.ShortRef(), ParentRef: ExchangeContactsItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeContactsItemPath2.ItemLocation(), - LocationRef: ExchangeContactsItemPath2.loc.String(), + LocationRef: ExchangeContactsItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeContact, @@ -228,11 +228,10 @@ var ( }, } - ExchangeEventsRootPath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false) - ExchangeEventsBasePath = ExchangeEventsRootPath.mustAppend("holidays", false) - ExchangeEventsBasePath2 = ExchangeEventsRootPath.mustAppend("moreholidays", false) - ExchangeEventsItemPath1 = ExchangeEventsBasePath.mustAppend(ItemName1, true) - ExchangeEventsItemPath2 = ExchangeEventsBasePath2.mustAppend(ItemName2, true) + ExchangeEventsBasePath = mustPathRep("tenant-id/exchange/user-id/events/holidays", false) + ExchangeEventsBasePath2 = mustPathRep("tenant-id/exchange/user-id/events/moreholidays", false) + ExchangeEventsItemPath1 = ExchangeEventsBasePath.MustAppend(ItemName1, true) + ExchangeEventsItemPath2 = ExchangeEventsBasePath2.MustAppend(ItemName2, true) ExchangeEventsItems = []details.Entry{ { @@ -240,7 +239,7 @@ var ( ShortRef: ExchangeEventsItemPath1.RR.ShortRef(), ParentRef: ExchangeEventsItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEventsItemPath1.ItemLocation(), - LocationRef: ExchangeEventsItemPath1.loc.String(), + LocationRef: ExchangeEventsItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -256,7 +255,7 @@ var ( ShortRef: ExchangeEventsItemPath2.RR.ShortRef(), ParentRef: ExchangeEventsItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: ExchangeEventsItemPath2.ItemLocation(), - LocationRef: ExchangeEventsItemPath2.loc.String(), + LocationRef: ExchangeEventsItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ Exchange: &details.ExchangeInfo{ ItemType: details.ExchangeEvent, @@ -270,17 +269,17 @@ var ( } OneDriveRootPath = mustPathRep("tenant-id/onedrive/user-id/files/drives/foo/root:", false) - OneDriveFolderPath = OneDriveRootPath.mustAppend("folder", false) - OneDriveBasePath1 = OneDriveFolderPath.mustAppend("a", false) - OneDriveBasePath2 = OneDriveFolderPath.mustAppend("b", false) + OneDriveFolderPath = OneDriveRootPath.MustAppend("folder", false) + OneDriveBasePath1 = OneDriveFolderPath.MustAppend("a", false) + OneDriveBasePath2 = OneDriveFolderPath.MustAppend("b", false) - OneDriveItemPath1 = OneDriveFolderPath.mustAppend(ItemName1, true) - OneDriveItemPath2 = OneDriveBasePath1.mustAppend(ItemName2, true) - OneDriveItemPath3 = OneDriveBasePath2.mustAppend(ItemName3, true) + OneDriveItemPath1 = OneDriveFolderPath.MustAppend(ItemName1, true) + OneDriveItemPath2 = OneDriveBasePath1.MustAppend(ItemName2, true) + OneDriveItemPath3 = OneDriveBasePath2.MustAppend(ItemName3, true) - OneDriveFolderFolder = OneDriveFolderPath.loc.PopFront().String() - OneDriveParentFolder1 = OneDriveBasePath1.loc.PopFront().String() - OneDriveParentFolder2 = OneDriveBasePath2.loc.PopFront().String() + OneDriveFolderFolder = OneDriveFolderPath.Loc.PopFront().String() + OneDriveParentFolder1 = OneDriveBasePath1.Loc.PopFront().String() + OneDriveParentFolder2 = OneDriveBasePath2.Loc.PopFront().String() OneDriveItems = []details.Entry{ { @@ -288,7 +287,7 @@ var ( ShortRef: OneDriveItemPath1.RR.ShortRef(), ParentRef: OneDriveItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath1.ItemLocation(), - LocationRef: OneDriveItemPath1.loc.String(), + LocationRef: OneDriveItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -306,7 +305,7 @@ var ( ShortRef: OneDriveItemPath2.RR.ShortRef(), ParentRef: OneDriveItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath2.ItemLocation(), - LocationRef: OneDriveItemPath2.loc.String(), + LocationRef: OneDriveItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -324,7 +323,7 @@ var ( ShortRef: OneDriveItemPath3.RR.ShortRef(), ParentRef: OneDriveItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: OneDriveItemPath3.ItemLocation(), - LocationRef: OneDriveItemPath3.loc.String(), + LocationRef: OneDriveItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, @@ -340,17 +339,17 @@ var ( } SharePointRootPath = mustPathRep("tenant-id/sharepoint/site-id/libraries/drives/foo/root:", false) - SharePointLibraryPath = SharePointRootPath.mustAppend("library", false) - SharePointBasePath1 = SharePointLibraryPath.mustAppend("a", false) - SharePointBasePath2 = SharePointLibraryPath.mustAppend("b", false) + SharePointLibraryPath = SharePointRootPath.MustAppend("library", false) + SharePointBasePath1 = SharePointLibraryPath.MustAppend("a", false) + SharePointBasePath2 = SharePointLibraryPath.MustAppend("b", false) - SharePointLibraryItemPath1 = SharePointLibraryPath.mustAppend(ItemName1, true) - SharePointLibraryItemPath2 = SharePointBasePath1.mustAppend(ItemName2, true) - SharePointLibraryItemPath3 = SharePointBasePath2.mustAppend(ItemName3, true) + SharePointLibraryItemPath1 = SharePointLibraryPath.MustAppend(ItemName1, true) + SharePointLibraryItemPath2 = SharePointBasePath1.MustAppend(ItemName2, true) + SharePointLibraryItemPath3 = SharePointBasePath2.MustAppend(ItemName3, true) - SharePointLibraryFolder = SharePointLibraryPath.loc.PopFront().String() - SharePointParentLibrary1 = SharePointBasePath1.loc.PopFront().String() - SharePointParentLibrary2 = SharePointBasePath2.loc.PopFront().String() + SharePointLibraryFolder = SharePointLibraryPath.Loc.PopFront().String() + SharePointParentLibrary1 = SharePointBasePath1.Loc.PopFront().String() + SharePointParentLibrary2 = SharePointBasePath2.Loc.PopFront().String() SharePointLibraryItems = []details.Entry{ { @@ -358,7 +357,7 @@ var ( ShortRef: SharePointLibraryItemPath1.RR.ShortRef(), ParentRef: SharePointLibraryItemPath1.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath1.ItemLocation(), - LocationRef: SharePointLibraryItemPath1.loc.String(), + LocationRef: SharePointLibraryItemPath1.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -376,7 +375,7 @@ var ( ShortRef: SharePointLibraryItemPath2.RR.ShortRef(), ParentRef: SharePointLibraryItemPath2.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath2.ItemLocation(), - LocationRef: SharePointLibraryItemPath2.loc.String(), + LocationRef: SharePointLibraryItemPath2.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -394,7 +393,7 @@ var ( ShortRef: SharePointLibraryItemPath3.RR.ShortRef(), ParentRef: SharePointLibraryItemPath3.RR.ToBuilder().Dir().ShortRef(), ItemRef: SharePointLibraryItemPath3.ItemLocation(), - LocationRef: SharePointLibraryItemPath3.loc.String(), + LocationRef: SharePointLibraryItemPath3.Loc.String(), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, diff --git a/src/pkg/logger/logger.go b/src/pkg/logger/logger.go index fde379430..cc632b422 100644 --- a/src/pkg/logger/logger.go +++ b/src/pkg/logger/logger.go @@ -34,20 +34,20 @@ const ( // flag names const ( - DebugAPIFN = "debug-api-calls" - LogFileFN = "log-file" - LogLevelFN = "log-level" - ReadableLogsFN = "readable-logs" - SensitiveInfoFN = "sensitive-info" + DebugAPIFN = "debug-api-calls" + LogFileFN = "log-file" + LogLevelFN = "log-level" + ReadableLogsFN = "readable-logs" + MaskSensitiveDataFN = "mask-sensitive-data" ) // flag values var ( - DebugAPIFV bool - logFileFV = "" - LogLevelFV = "info" - ReadableLogsFV bool - SensitiveInfoFV = PIIPlainText + DebugAPIFV bool + logFileFV = "" + LogLevelFV = "info" + ReadableLogsFV bool + MaskSensitiveDataFV bool LogFile string // logFileFV after processing ) @@ -83,9 +83,6 @@ func AddLoggingFlags(cmd *cobra.Command) { //nolint:errcheck fs.MarkHidden(ReadableLogsFN) - // TODO(keepers): unhide when we have sufficient/complete coverage of PII handling - //nolint:errcheck - fs.MarkHidden(SensitiveInfoFN) } // internal deduplication for adding flags @@ -106,11 +103,11 @@ func addFlags(fs *pflag.FlagSet, defaultFile string) { false, "minimizes log output for console readability: removes the file and date, colors the level") - fs.StringVar( - &SensitiveInfoFV, - SensitiveInfoFN, - PIIPlainText, - fmt.Sprintf("set the format for sensitive info in logs to %s|%s|%s", PIIHash, PIIMask, PIIPlainText)) + fs.BoolVar( + &MaskSensitiveDataFV, + MaskSensitiveDataFN, + false, + "anonymize personal data in log output") } // Settings records the user's preferred logging settings. @@ -136,7 +133,7 @@ func PreloadLoggingFlags(args []string) Settings { ls := Settings{ File: "", Level: LogLevelFV, - PIIHandling: SensitiveInfoFV, + PIIHandling: PIIPlainText, } // parse the os args list to find the log level flag @@ -144,6 +141,10 @@ func PreloadLoggingFlags(args []string) Settings { return ls } + if MaskSensitiveDataFV { + ls.PIIHandling = PIIHash + } + // retrieve the user's preferred log level // automatically defaults to "info" levelString, err := fs.GetString(LogLevelFN) @@ -165,7 +166,7 @@ func PreloadLoggingFlags(args []string) Settings { // retrieve the user's preferred PII handling algorithm // automatically defaults to default log location - pii, err := fs.GetString(SensitiveInfoFN) + pii, err := fs.GetString(MaskSensitiveDataFN) if err != nil { return ls } diff --git a/src/pkg/logger/logger_test.go b/src/pkg/logger/logger_test.go index 7cb7926fa..644c23aa0 100644 --- a/src/pkg/logger/logger_test.go +++ b/src/pkg/logger/logger_test.go @@ -33,7 +33,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { assert.True(t, logger.DebugAPIFV, logger.DebugAPIFN) assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, logger.LLError, logger.LogLevelFV, logger.LogLevelFN) - assert.Equal(t, logger.PIIMask, logger.SensitiveInfoFV, logger.SensitiveInfoFN) + assert.True(t, logger.MaskSensitiveDataFV, logger.MaskSensitiveDataFN) // empty assertion here, instead of matching "log-file", because the LogFile // var isn't updated by running the command (this is expected and correct), // while the logFileFV remains unexported. @@ -50,7 +50,7 @@ func (suite *LoggerUnitSuite) TestAddLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, }) err := cmd.Execute() @@ -68,7 +68,7 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { "--" + logger.LogFileFN, "log-file", "--" + logger.LogLevelFN, logger.LLError, "--" + logger.ReadableLogsFN, - "--" + logger.SensitiveInfoFN, logger.PIIMask, + "--" + logger.MaskSensitiveDataFN, } settings := logger.PreloadLoggingFlags(args) @@ -77,5 +77,5 @@ func (suite *LoggerUnitSuite) TestPreloadLoggingFlags() { assert.True(t, logger.ReadableLogsFV, logger.ReadableLogsFN) assert.Equal(t, "log-file", settings.File, "settings.File") assert.Equal(t, logger.LLError, settings.Level, "settings.Level") - assert.Equal(t, logger.PIIMask, settings.PIIHandling, "settings.PIIHandling") + assert.Equal(t, logger.PIIHash, settings.PIIHandling, "settings.PIIHandling") } diff --git a/src/pkg/path/drive.go b/src/pkg/path/drive.go index b073ff125..033f9934b 100644 --- a/src/pkg/path/drive.go +++ b/src/pkg/path/drive.go @@ -38,3 +38,13 @@ func GetDriveFolderPath(p Path) (string, error) { return Builder{}.Append(drivePath.Folders...).String(), nil } + +// BuildDriveLocation takes a driveID and a set of unescaped element names, +// including the root folder, and returns a *path.Builder containing the +// canonical path representation for the drive path. +func BuildDriveLocation( + driveID string, + unescapedElements ...string, +) *Builder { + return Builder{}.Append("drives", driveID).Append(unescapedElements...) +} diff --git a/src/pkg/path/drive_test.go b/src/pkg/path/drive_test.go index cddd050bf..5a6853caf 100644 --- a/src/pkg/path/drive_test.go +++ b/src/pkg/path/drive_test.go @@ -1,6 +1,7 @@ package path_test import ( + "strings" "testing" "github.com/alcionai/clues" @@ -8,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/path" ) @@ -21,8 +23,6 @@ func TestOneDrivePathSuite(t *testing.T) { } func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { - const root = "root:" - tests := []struct { name string pathElements []string @@ -31,20 +31,28 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }{ { name: "Not enough path elements", - pathElements: []string{"drive", "driveID"}, + pathElements: []string{odConsts.DrivesPathDir, "driveID"}, errCheck: assert.Error, }, { name: "Root path", - pathElements: []string{"drive", "driveID", root}, - expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{}}, - errCheck: assert.NoError, + pathElements: []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir}, + expected: &path.DrivePath{ + DriveID: "driveID", + Root: odConsts.RootPathDir, + Folders: []string{}, + }, + errCheck: assert.NoError, }, { name: "Deeper path", - pathElements: []string{"drive", "driveID", root, "folder1", "folder2"}, - expected: &path.DrivePath{DriveID: "driveID", Root: root, Folders: []string{"folder1", "folder2"}}, - errCheck: assert.NoError, + pathElements: []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "folder1", "folder2"}, + expected: &path.DrivePath{ + DriveID: "driveID", + Root: odConsts.RootPathDir, + Folders: []string{"folder1", "folder2"}, + }, + errCheck: assert.NoError, }, } for _, tt := range tests { @@ -63,3 +71,49 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() { }) } } + +func (suite *OneDrivePathSuite) TestFormatDriveFolders() { + const ( + driveID = "some-drive-id" + drivePrefix = "drives/" + driveID + ) + + table := []struct { + name string + input []string + expected string + }{ + { + name: "normal", + input: []string{ + "root:", + "foo", + "bar", + }, + expected: strings.Join( + append([]string{drivePrefix}, "root:", "foo", "bar"), + "/"), + }, + { + name: "has character that would be escaped", + input: []string{ + "root:", + "foo/", + "bar", + }, + // Element "foo/" should end up escaped in the string output. + expected: strings.Join( + append([]string{drivePrefix}, "root:", `foo\/`, "bar"), + "/"), + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + assert.Equal( + suite.T(), + test.expected, + path.BuildDriveLocation(driveID, test.input...).String()) + }) + } +} diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index 0a55bd8e4..a77ea3345 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -86,3 +86,12 @@ func (el Elements) String() string { func (el Elements) PlainString() string { return join(el) } + +// Last returns the last element. Returns "" if empty. +func (el Elements) Last() string { + if len(el) == 0 { + return "" + } + + return el[len(el)-1] +} diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index 52daa1e87..189e24449 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -85,7 +85,7 @@ type Path interface { Category() CategoryType Tenant() string ResourceOwner() string - Folder(bool) string + Folder(escaped bool) string Folders() Elements Item() string // UpdateParent updates parent from old to new if the item/folder was @@ -106,7 +106,9 @@ type Path interface { // Append returns a new Path object with the given element added to the end of // the old Path if possible. If the old Path is an item Path then Append // returns an error. - Append(element string, isItem bool) (Path, error) + Append(isItem bool, elems ...string) (Path, error) + // AppendItem is a shorthand for Append(true, someItem) + AppendItem(item string) (Path, error) // ShortRef returns a short reference representing this path. The short // reference is guaranteed to be unique. No guarantees are made about whether // a short reference can be converted back into the Path that generated it. @@ -130,6 +132,13 @@ var ( _ fmt.Stringer = &Builder{} ) +// RestorePaths denotes the location to find an item in kopia and the path of +// the collection to place the item in for restore. +type RestorePaths struct { + StoragePath Path + RestorePath Path +} + // Builder is a simple path representation that only tracks path elements. It // can join, escape, and unescape elements. Higher-level packages are expected // to wrap this struct to build resource-specific contexts (e.x. an diff --git a/src/pkg/path/path_test.go b/src/pkg/path/path_test.go index 21631f7bf..be43d3732 100644 --- a/src/pkg/path/path_test.go +++ b/src/pkg/path/path_test.go @@ -245,6 +245,26 @@ func (suite *PathUnitSuite) TestAppend() { } } +func (suite *PathUnitSuite) TestAppendItem() { + t := suite.T() + + p, err := Build("t", "ro", ExchangeService, EmailCategory, false, "foo", "bar") + require.NoError(t, err, clues.ToCore(err)) + + pb := p.ToBuilder() + assert.Equal(t, pb.String(), p.String()) + + pb = pb.Append("qux") + + p, err = p.AppendItem("qux") + + require.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, pb.String(), p.String()) + + _, err = p.AppendItem("fnords") + require.Error(t, err, clues.ToCore(err)) +} + func (suite *PathUnitSuite) TestUnescapeAndAppend() { table := append(append([]testData{}, genericCases...), basicEscapedInputs...) for _, test := range table { diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 47d481a46..923d66453 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -253,21 +253,25 @@ func (rp dataLayerResourcePath) Dir() (Path, error) { } func (rp dataLayerResourcePath) Append( - element string, isItem bool, + elems ...string, ) (Path, error) { if rp.hasItem { return nil, clues.New("appending to an item path") } return &dataLayerResourcePath{ - Builder: *rp.Builder.Append(element), + Builder: *rp.Builder.Append(elems...), service: rp.service, category: rp.category, hasItem: isItem, }, nil } +func (rp dataLayerResourcePath) AppendItem(item string) (Path, error) { + return rp.Append(true, item) +} + func (rp dataLayerResourcePath) ToBuilder() *Builder { // Safe to directly return the Builder because Builders are immutable. return &rp.Builder diff --git a/src/pkg/path/resource_path_test.go b/src/pkg/path/resource_path_test.go index 3453737e6..e49f797e2 100644 --- a/src/pkg/path/resource_path_test.go +++ b/src/pkg/path/resource_path_test.go @@ -547,7 +547,7 @@ func (suite *PopulatedDataLayerResourcePath) TestAppend() { suite.Run(test.name, func() { t := suite.T() - newPath, err := suite.paths[m.isItem].Append(newElement, test.hasItem) + newPath, err := suite.paths[m.isItem].Append(test.hasItem, newElement) // Items don't allow appending. if m.isItem { diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index 4d9b718c1..7ef56fdb0 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -24,6 +24,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" "github.com/alcionai/corso/src/pkg/storage" ) @@ -150,7 +151,7 @@ func runRestoreLoadTest( t.Skip("restore load test is toggled off") } - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") rst, err := r.NewRestore(ctx, backupID, restSel, dest) require.NoError(t, err, clues.ToCore(err)) @@ -541,7 +542,7 @@ func (suite *LoadOneDriveSuite) TestOneDrive() { defer flush() bsel := selectors.NewOneDriveBackup(suite.usersUnderTest) - bsel.Include(bsel.AllData()) + bsel.Include(selTD.OneDriveBackupFolderScope(bsel)) sel := bsel.Selector runLoadTest( @@ -588,7 +589,7 @@ func (suite *IndividualLoadOneDriveSuite) TestOneDrive() { defer flush() bsel := selectors.NewOneDriveBackup(suite.usersUnderTest) - bsel.Include(bsel.AllData()) + bsel.Include(selTD.OneDriveBackupFolderScope(bsel)) sel := bsel.Selector runLoadTest( diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 649601142..8efe44f31 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -1,7 +1,9 @@ package repository_test import ( + "os" "testing" + "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" @@ -145,6 +147,33 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { } } +const ( + roleARNEnvKey = "CORSO_TEST_S3_ROLE" + roleDuration = time.Minute * 20 +) + +func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { + if _, ok := os.LookupEnv(roleARNEnvKey); !ok { + suite.T().Skip(roleARNEnvKey + " not set") + } + + ctx, flush := tester.NewContext() + defer flush() + + st := tester.NewPrefixedS3Storage(suite.T()) + + st.Role = os.Getenv(roleARNEnvKey) + st.SessionName = "corso-repository-test" + st.SessionDuration = roleDuration.String() + + r, err := repository.Initialize(ctx, account.Account{}, st, control.Options{}) + require.NoError(suite.T(), err) + + defer func() { + r.Close(ctx) + }() +} + func (suite *RepositoryIntegrationSuite) TestConnect() { ctx, flush := tester.NewContext() defer flush() @@ -213,7 +242,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { t := suite.T() acct := tester.NewM365Account(t) - dest := tester.DefaultTestRestoreDestination() + dest := tester.DefaultTestRestoreDestination("") // need to initialize the repository before we can test connecting to it. st := tester.NewPrefixedS3Storage(t) diff --git a/src/pkg/selectors/onedrive_test.go b/src/pkg/selectors/onedrive_test.go index c91f27b04..41835875b 100644 --- a/src/pkg/selectors/onedrive_test.go +++ b/src/pkg/selectors/onedrive_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -315,7 +316,7 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() { fileName := "file" fileID := fileName + "-id" shortRef := "short" - elems := []string{"drive", "driveID", "root:", "dir1.d", "dir2.d", fileID} + elems := []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "dir1.d", "dir2.d", fileID} filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/pkg/selectors/selectors_reduce_test.go b/src/pkg/selectors/selectors_reduce_test.go index b72a4c65e..c57cde409 100644 --- a/src/pkg/selectors/selectors_reduce_test.go +++ b/src/pkg/selectors/selectors_reduce_test.go @@ -249,18 +249,6 @@ func (suite *SelectorReduceSuite) TestReduce() { }, expected: []details.Entry{testdata.ExchangeEventsItems[0]}, }, - { - name: "ExchangeEventsByFolderRoot", - selFunc: func() selectors.Reducer { - sel := selectors.NewExchangeRestore(selectors.Any()) - sel.Include(sel.EventCalendars( - []string{testdata.ExchangeEventsRootPath.FolderLocation()}, - )) - - return sel - }, - expected: testdata.ExchangeEventsItems, - }, } for _, test := range table { diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index b2c9e2344..2b8f3edf4 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -12,6 +12,7 @@ import ( "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/dttm" + odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -223,9 +224,9 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( prefixElems = []string{ - "drive", + odConsts.DrivesPathDir, "drive!id", - "root:", + odConsts.RootPathDir, } itemElems1 = []string{"folderA", "folderB"} itemElems2 = []string{"folderA", "folderC"} @@ -257,7 +258,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { { RepoRef: item, ItemRef: "item", - LocationRef: strings.Join(append([]string{"root:"}, itemElems1...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems1...), "/"), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -268,7 +269,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { }, { RepoRef: item2, - LocationRef: strings.Join(append([]string{"root:"}, itemElems2...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems2...), "/"), // ItemRef intentionally blank to test fallback case ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ @@ -281,7 +282,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { { RepoRef: item3, ItemRef: "item3", - LocationRef: strings.Join(append([]string{"root:"}, itemElems3...), "/"), + LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems3...), "/"), ItemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, @@ -415,8 +416,15 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { itemName = "item" itemID = "item-id" shortRef = "short" - driveElems = []string{"drive", "drive!id", "root:.d", "dir1.d", "dir2.d", itemID} - elems = []string{"dir1", "dir2", itemID} + driveElems = []string{ + odConsts.DrivesPathDir, + "drive!id", + odConsts.RootPathDir + ".d", + "dir1.d", + "dir2.d", + itemID, + } + elems = []string{"dir1", "dir2", itemID} ) table := []struct { diff --git a/src/pkg/selectors/testdata/onedrive.go b/src/pkg/selectors/testdata/onedrive.go new file mode 100644 index 000000000..8592d3d80 --- /dev/null +++ b/src/pkg/selectors/testdata/onedrive.go @@ -0,0 +1,9 @@ +package testdata + +import "github.com/alcionai/corso/src/pkg/selectors" + +// OneDriveBackupFolderScope is the standard folder scope that should be used +// in integration backups with onedrive. +func OneDriveBackupFolderScope(sel *selectors.OneDriveBackup) []selectors.OneDriveScope { + return sel.Folders([]string{"test"}, selectors.PrefixMatch()) +} diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index e635f9981..19cc9ddc7 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -36,6 +36,11 @@ const ( type Storage struct { Provider storageProvider Config map[string]string + // TODO: These are AWS S3 specific -> move these out + SessionTags map[string]string + Role string + SessionName string + SessionDuration string } // NewStorage aggregates all the supplied configurations into a single configuration. @@ -48,6 +53,28 @@ func NewStorage(p storageProvider, cfgs ...common.StringConfigurer) (Storage, er }, err } +// NewStorageUsingRole supports specifying an AWS IAM role the storage provider +// should assume. +func NewStorageUsingRole( + p storageProvider, + roleARN string, + sessionName string, + sessionTags map[string]string, + duration string, + cfgs ...common.StringConfigurer, +) (Storage, error) { + cs, err := common.UnionStringConfigs(cfgs...) + + return Storage{ + Provider: p, + Config: cs, + Role: roleARN, + SessionTags: sessionTags, + SessionName: sessionName, + SessionDuration: duration, + }, err +} + // Helper for parsing the values in a config object. // If the value is nil or not a string, returns an empty string. func orEmptyString(v any) string { diff --git a/website/docs/setup/configuration.md b/website/docs/setup/configuration.md index d9255f6b7..65c04e99b 100644 --- a/website/docs/setup/configuration.md +++ b/website/docs/setup/configuration.md @@ -132,7 +132,13 @@ directory within the container. Corso generates a unique log file named with its timestamp for every invocation. The default location of Corso's log file is shown below but the location can be overridden by using the `--log-file` flag. The log file will be appended to if multiple Corso invocations are pointed to the same file. + You can also use `stdout` or `stderr` as the `--log-file` location to redirect the logs to "stdout" and "stderr" respectively. +This setting can cause logs to compete with progress bar displays in the terminal. +We suggest using the `--hide-progress` option if you plan to log to stdout or stderr. + +Log entries, by default, include user names and file names. The `--mask-sensitive-data` option can be +used to replace this information with anonymized hashes. diff --git a/website/styles/Vocab/Base/accept.txt b/website/styles/Vocab/Base/accept.txt index 7f8d159c7..b915b5010 100644 --- a/website/styles/Vocab/Base/accept.txt +++ b/website/styles/Vocab/Base/accept.txt @@ -54,4 +54,5 @@ Demetrius Malbrough lockdowns exfiltrate -deduplicating \ No newline at end of file +deduplicating +anonymized