diff --git a/.github/actions/backup-restore-test/action.yml b/.github/actions/backup-restore-test/action.yml index 4e31ad836..95e41d1bb 100644 --- a/.github/actions/backup-restore-test/action.yml +++ b/.github/actions/backup-restore-test/action.yml @@ -24,6 +24,10 @@ inputs: log-dir: description: Folder to store test log files required: true + on-collision: + description: Value for the --collisions flag + requried: false + default: "replace" outputs: backup-id: @@ -57,6 +61,7 @@ runs: ./corso restore '${{ inputs.service }}' \ --no-stats \ --hide-progress \ + --collisions ${{ inputs.on-collision }} \ ${{ inputs.restore-args }} \ --backup '${{ steps.backup.outputs.result }}' \ 2>&1 | @@ -67,12 +72,78 @@ runs: cat /tmp/corsologs - - name: Check ${{ inputs.service }} ${{ inputs.kind }} + - name: Check restore ${{ inputs.service }} ${{ inputs.kind }} shell: bash working-directory: src env: - SANITY_RESTORE_FOLDER: ${{ steps.restore.outputs.result }} - SANITY_RESTORE_SERVICE: ${{ inputs.service }} + SANITY_TEST_KIND: restore + SANITY_TEST_FOLDER: ${{ steps.restore.outputs.result }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export ${{ inputs.service }} ${{ inputs.kind }} + id: export + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }} \ + --no-stats \ + --hide-progress \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + cat /tmp/corsologs + + - name: Check export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' || inputs.service == 'sharepoint' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }} + SANITY_TEST_SERVICE: ${{ inputs.service }} + TEST_DATA: ${{ inputs.test-folder }} + BASE_BACKUP: ${{ inputs.base-backup }} + run: | + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log + ./sanity-test + + - name: Export archive ${{ inputs.service }} ${{ inputs.kind }} + id: export-archive + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} # Export only available for OneDrive + run: | + set -euo pipefail + CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log + ./corso export '${{ inputs.service }}' \ + /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \ + --no-stats \ + --hide-progress \ + --archive \ + ${{ inputs.export-args }} \ + --backup '${{ steps.backup.outputs.result }}' + + unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \ + -d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + cat /tmp/corsologs + + - name: Check archive export ${{ inputs.service }} ${{ inputs.kind }} + shell: bash + working-directory: src + if: ${{ inputs.service == 'onedrive' }} + env: + SANITY_TEST_KIND: export + SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped + SANITY_TEST_SERVICE: ${{ inputs.service }} TEST_DATA: ${{ inputs.test-folder }} BASE_BACKUP: ${{ inputs.base-backup }} run: | diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index d7681e55b..cf013a054 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -19,7 +19,9 @@ inputs: site: description: Sharepoint site where data is to be purged. libraries: - description: List of library names within site where data is to be purged. + description: List of library names within the site where data is to be purged. + library-prefix: + description: List of library names within the site where the library will get deleted entirely. folder-prefix: description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: @@ -51,8 +53,7 @@ runs: AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} - run: | - ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Reset retention for all mailboxes to 0 if: ${{ inputs.user == '' }} @@ -61,8 +62,7 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./exchangeRetention.ps1 + run: ./exchangeRetention.ps1 ################################################################################################################ # OneDrive @@ -89,5 +89,4 @@ runs: env: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/actions/slack-message/action.yml b/.github/actions/slack-message/action.yml new file mode 100644 index 000000000..57091d430 --- /dev/null +++ b/.github/actions/slack-message/action.yml @@ -0,0 +1,58 @@ +name: Send a message to slack + +inputs: + msg: + description: The slack message text + slack_url: + description: passthrough for secrets.SLACK_WEBHOOK_URL + +runs: + using: composite + steps: + - uses: actions/checkout@v3 + + - name: set github ref + shell: bash + run: | + echo "github_reference=${{ github.ref }}" >> $GITHUB_ENV + + - name: trim github ref + shell: bash + run: | + echo "trimmed_ref=${github_reference#refs/}" >> $GITHUB_ENV + + - name: build urls + shell: bash + run: | + echo "logurl=$(printf '' ${{ github.run_id }})" >> $GITHUB_ENV + echo "commiturl=$(printf '' ${{ github.sha }})" >> $GITHUB_ENV + echo "refurl=$(printf '' ${{ env.trimmed_ref }})" >> $GITHUB_ENV + + - name: use url or blank val + shell: bash + run: | + echo "STEP=${{ github.action || '' }}" >> $GITHUB_ENV + echo "JOB=${{ github.job || '' }}" >> $GITHUB_ENV + echo "LOGS=${{ github.run_id && env.logurl || '-' }}" >> $GITHUB_ENV + echo "COMMIT=${{ github.sha && env.commiturl || '-' }}" >> $GITHUB_ENV + echo "REF=${{ env.trimmed_ref && env.refurl || '-' }}" >> $GITHUB_ENV + + - id: slack-message + uses: slackapi/slack-github-action@v1.24.0 + env: + SLACK_WEBHOOK_URL: ${{ inputs.slack_url }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "text": "${{ inputs.msg }} :: ${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "${{ inputs.msg }} :: ${{ env.JOB }} - ${{ env.STEP }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}" + } + } + ] + } diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 65e678e4b..3687c0e0c 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -12,18 +12,15 @@ jobs: continue-on-error: true strategy: matrix: - user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ] + user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, '' ] steps: - uses: actions/checkout@v3 # sets the maximum time to now-30m. - # CI test have a 10 minute timeout. - # At 20 minutes ago, we should be safe from conflicts. - # The additional 10 minutes is just to be good citizens. + # CI test have a 20 minute timeout. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Users uses: ./.github/actions/purge-m365-data @@ -37,34 +34,46 @@ jobs: m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} + - name: Notify failure in slack + if: failure() + uses: ./.github/actions/slack-message + with: + msg: "[FAILED] ${{ vars[matrix.user] }} CI Cleanup" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} + Test-Site-Data-Cleanup: environment: Testing runs-on: ubuntu-latest continue-on-error: true strategy: matrix: - site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] + site: [ CORSO_M365_TEST_SITE_URL ] steps: - uses: actions/checkout@v3 # sets the maximum time to now-30m. - # CI test have a 10 minute timeout. - # At 20 minutes ago, we should be safe from conflicts. - # The additional 10 minutes is just to be good citizens. + # CI test have a 20 minute timeout. - name: Set purge boundary - run: | - echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + run: echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge CI-Produced Folders for Sites uses: ./.github/actions/purge-m365-data with: site: ${{ vars[matrix.site] }} - folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} - libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} azure-tenant-id: ${{ secrets.TENANT_ID }} m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} + + - name: Notify failure in slack + if: failure() + uses: ./.github/actions/slack-message + with: + msg: "[FAILED] ${{ vars[matrix.site] }} CI Cleanup" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/longevity_test.yml b/.github/workflows/longevity_test.yml index 02b956be1..8d294107a 100644 --- a/.github/workflows/longevity_test.yml +++ b/.github/workflows/longevity_test.yml @@ -277,33 +277,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Longevity test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Longevity Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ steps.sha-info.outputs.COMMIT_URL }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + msg: "[FAILED] Longevity Test" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index 2ea556099..29b69ad20 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -94,6 +94,7 @@ jobs: CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} CORSO_LOG_FILE: ${{ github.workspace }}/testlog/run-nightly.log LOG_GRAPH_REQUESTS: true + S3_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} run: | set -euo pipefail go test \ @@ -119,33 +120,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Nightly test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Nightly Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ steps.sha-info.outputs.COMMIT_URL }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + msg: "[FAILED] Nightly Checks" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 9e210778d..292c358dd 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -333,33 +333,9 @@ jobs: if-no-files-found: error retention-days: 14 - - name: SHA info - id: sha-info + - name: Notify failure in slack if: failure() - run: | - echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA} - echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT - echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_OUTPUT - echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT - - - name: Send Github Action failure to Slack - id: slack-notification - if: failure() - uses: slackapi/slack-github-action@v1.24.0 + uses: ./.github/actions/slack-message with: - payload: | - { - "text": "Sanity test failure - build: ${{ job.status }} - SHA: ${{ steps.sha-info.outputs.SHA }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "[FAILED] Sanity Checks :: <${{ steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ github.event.pull_request.html_url || github.event.head_commit.url }}|[Base]>\nCommit: <${{ steps.sha-info.outputs.COMMIT_URL }}|${{ steps.sha-info.outputs.SHA }}>" - } - } - ] - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK \ No newline at end of file + msg: "[FAILED] Sanity Tests" + slack_url: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7fdca0d..906267535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] (beta) +### Added +- Restore commands now accept an optional resource override with the `--to-resource` flag. This allows restores to recreate backup data within different mailboxes, sites, and users. + +### Fixed +- SharePoint document libraries deleted after the last backup can now be restored. +- Restore requires the protected resource to have access to the service being restored. + +### Added +- Added option to export data from OneDrive and SharePoint backups as individual files or as a single zip file. + +## [v0.11.1] (beta) - 2023-07-20 + ### Fixed - Allow repo connect to succeed when a `corso.toml` file was not provided but configuration is specified using environment variables and flags. @@ -21,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Return a ServiceNotEnabled error when a tenant has no active SharePoint license. - Added retries for http/2 stream connection failures when downloading large item content. +- SharePoint document libraries that were deleted after the last backup can now be restored. ### Known issues - If a link share is created for an item with inheritance disabled @@ -328,7 +341,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Miscellaneous - Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35)) -[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.0...HEAD +[Unreleased]: https://github.com/alcionai/corso/compare/v0.11.1...HEAD +[v0.11.1]: https://github.com/alcionai/corso/compare/v0.11.0...v0.11.1 [v0.11.0]: https://github.com/alcionai/corso/compare/v0.10.0...v0.11.0 [v0.10.0]: https://github.com/alcionai/corso/compare/v0.9.0...v0.10.0 [v0.9.0]: https://github.com/alcionai/corso/compare/v0.8.1...v0.9.0 diff --git a/src/.golangci.yml b/src/.golangci.yml index 06ccaa3dd..da54f6217 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -3,14 +3,20 @@ run: linters: enable: + - errcheck + - forbidigo - gci - gofmt - gofumpt - - errcheck - - forbidigo + - gosimple + - govet + - ineffassign - lll + - loggercheck - misspell - revive + - unused + - usestdlibvars - wsl disable: diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 99bb0ff78..0f11bd6bd 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -94,6 +94,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { flags.AddDisableDeltaFlag(c) flags.AddEnableImmutableIDFlag(c) flags.AddDisableConcurrencyLimiterFlag(c) + flags.AddDeltaPageSizeFlag(c) case listCommand: c, fs = utils.AddCommand(cmd, exchangeListCmd()) @@ -175,7 +176,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { sel := exchangeBackupCreateSelectors(flags.UserFV, flags.CategoryDataFV) - ins, err := utils.UsersMap(ctx, *acct, fault.New(true)) + ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true)) if err != nil { return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users")) } diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index 6bd078797..d260ca290 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -37,11 +37,11 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { expectRunE func(*cobra.Command, []string) error }{ { - "create exchange", - createCommand, - expectUse + " " + exchangeServiceCommandCreateUseSuffix, - exchangeCreateCmd().Short, - []string{ + name: "create exchange", + use: createCommand, + expectUse: expectUse + " " + exchangeServiceCommandCreateUseSuffix, + expectShort: exchangeCreateCmd().Short, + flags: []string{ flags.UserFN, flags.CategoryDataFN, flags.DisableIncrementalsFN, @@ -50,28 +50,29 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { flags.FetchParallelismFN, flags.SkipReduceFN, flags.NoStatsFN, + flags.DeltaPageSizeFN, }, - createExchangeCmd, + expectRunE: createExchangeCmd, }, { - "list exchange", - listCommand, - expectUse, - exchangeListCmd().Short, - []string{ + name: "list exchange", + use: listCommand, + expectUse: expectUse, + expectShort: exchangeListCmd().Short, + flags: []string{ flags.BackupFN, flags.FailedItemsFN, flags.SkippedItemsFN, flags.RecoveredErrorsFN, }, - listExchangeCmd, + expectRunE: listExchangeCmd, }, { - "details exchange", - detailsCommand, - expectUse + " " + exchangeServiceCommandDetailsUseSuffix, - exchangeDetailsCmd().Short, - []string{ + name: "details exchange", + use: detailsCommand, + expectUse: expectUse + " " + exchangeServiceCommandDetailsUseSuffix, + expectShort: exchangeDetailsCmd().Short, + flags: []string{ flags.BackupFN, flags.ContactFN, flags.ContactFolderFN, @@ -90,7 +91,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { flags.EventStartsBeforeFN, flags.EventSubjectFN, }, - detailsExchangeCmd, + expectRunE: detailsExchangeCmd, }, { "delete exchange", diff --git a/src/cli/backup/help_e2e_test.go b/src/cli/backup/help_e2e_test.go index d99d3769a..f6b4edd83 100644 --- a/src/cli/backup/help_e2e_test.go +++ b/src/cli/backup/help_e2e_test.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -47,7 +48,12 @@ func prepM365Test( vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) ctx = config.SetViper(ctx, vpr) - repo, err := repository.Initialize(ctx, acct, st, control.Defaults()) + repo, err := repository.Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return acct, st, repo, vpr, recorder, cfgFP diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 62ce242d4..b9d94fc41 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -157,7 +157,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error { sel := oneDriveBackupCreateSelectors(flags.UserFV) - ins, err := utils.UsersMap(ctx, *acct, fault.New(true)) + ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true)) if err != nil { return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users")) } diff --git a/src/cli/cli.go b/src/cli/cli.go index 5f03364a2..68f454407 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/cli/backup" "github.com/alcionai/corso/src/cli/config" + "github.com/alcionai/corso/src/cli/export" "github.com/alcionai/corso/src/cli/flags" "github.com/alcionai/corso/src/cli/help" "github.com/alcionai/corso/src/cli/print" @@ -53,7 +54,7 @@ func preRun(cc *cobra.Command, args []string) error { } avoidTheseCommands := []string{ - "corso", "env", "help", "backup", "details", "list", "restore", "delete", "repo", "init", "connect", + "corso", "env", "help", "backup", "details", "list", "restore", "export", "delete", "repo", "init", "connect", } if len(logger.ResolvedLogFile) > 0 && !slices.Contains(avoidTheseCommands, cc.Use) { @@ -150,6 +151,7 @@ func BuildCommandTree(cmd *cobra.Command) { repo.AddCommands(cmd) backup.AddCommands(cmd) restore.AddCommands(cmd) + export.AddCommands(cmd) help.AddCommands(cmd) } diff --git a/src/cli/export/export.go b/src/cli/export/export.go new file mode 100644 index 000000000..e0deed014 --- /dev/null +++ b/src/cli/export/export.go @@ -0,0 +1,108 @@ +package export + +import ( + "context" + "errors" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" +) + +var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ + addOneDriveCommands, + addSharePointCommands, +} + +// AddCommands attaches all `corso export * *` commands to the parent. +func AddCommands(cmd *cobra.Command) { + exportC := exportCmd() + cmd.AddCommand(exportC) + + for _, addExportTo := range exportCommands { + addExportTo(exportC) + } +} + +const exportCommand = "export" + +// The export category of commands. +// `corso export [] [...]` +func exportCmd() *cobra.Command { + return &cobra.Command{ + Use: exportCommand, + Short: "Export your service data", + Long: `Export the data stored in one of your M365 services.`, + RunE: handleExportCmd, + Args: cobra.NoArgs, + } +} + +// Handler for flat calls to `corso export`. +// Produces the same output as `corso export --help`. +func handleExportCmd(cmd *cobra.Command, args []string) error { + return cmd.Help() +} + +func runExport( + ctx context.Context, + cmd *cobra.Command, + args []string, + ueco utils.ExportCfgOpts, + sel selectors.Selector, + backupID, serviceName string, +) error { + r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if len(exportLocation) == 0 { + // This should not be possible, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + eo, err := r.NewExport( + ctx, + backupID, + sel, + utils.MakeExportConfig(ctx, ueco)) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+backupID)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have any way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer close(diskWriteComplete) + + err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors) + if err != nil { + return Only(ctx, err) + } + + return nil +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go new file mode 100644 index 000000000..593149bd9 --- /dev/null +++ b/src/cli/export/onedrive.go @@ -0,0 +1,96 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, oneDriveExportCmd()) + + c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddOneDriveDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + oneDriveServiceCommand = "onedrive" + oneDriveServiceCommandUseSuffix = "--backup " + + //nolint:lll + oneDriveServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef + +# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory +corso export onedrive . --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + +# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports +corso export onedrive my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd + --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` +) + +// `corso export onedrive [...] ` +func oneDriveExportCmd() *cobra.Command { + return &cobra.Command{ + Use: oneDriveServiceCommand, + Short: "Export M365 OneDrive service data", + RunE: exportOneDriveCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: oneDriveServiceCommandExportExamples, + } +} + +// processes an onedrive service export. +func exportOneDriveCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeOneDriveOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateOneDriveRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + sel := utils.IncludeOneDriveRestoreDataSelectors(opts) + utils.FilterOneDriveRestoreInfoSelectors(sel, opts) + + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "OneDrive") +} diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go new file mode 100644 index 000000000..59ab966e8 --- /dev/null +++ b/src/cli/export/onedrive_test.go @@ -0,0 +1,106 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type OneDriveUnitSuite struct { + tester.Suite +} + +func TestOneDriveUnitSuite(t *testing.T) { + suite.Run(t, &OneDriveUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { + expectUse := oneDriveServiceCommand + " " + oneDriveServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export onedrive", exportCommand, expectUse, oneDriveExportCmd().Short, exportOneDriveCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addOneDriveCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "onedrive", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeOneDriveOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/export/sharepoint.go b/src/cli/export/sharepoint.go new file mode 100644 index 000000000..ec71a5f2b --- /dev/null +++ b/src/cli/export/sharepoint.go @@ -0,0 +1,100 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addSharePointCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, sharePointExportCmd()) + + c.Use = c.Use + " " + sharePointServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddSharePointDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + sharePointServiceCommand = "sharepoint" + sharePointServiceCommandUseSuffix = "--backup " + + //nolint:lll + sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to my-exports directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef my-exports + +# Export files named "ServerRenderTemplate.xsl" in the folder "Display Templates/Style Sheets". as archive to current directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" --archive . + +# Export all files in the folder "Display Templates/Style Sheets" that were created before 2020 to my-exports directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" my-exports + +# Export all files in the "Documents" library to current directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --library Documents --folder "Display Templates/Style Sheets" .` +) + +// `corso export sharepoint [...] ` +func sharePointExportCmd() *cobra.Command { + return &cobra.Command{ + Use: sharePointServiceCommand, + Short: "Export M365 SharePoint service data", + RunE: exportSharePointCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: sharePointServiceCommandExportExamples, + } +} + +// processes an sharepoint service export. +func exportSharePointCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeSharePointOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateSharePointRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + utils.FilterSharePointRestoreInfoSelectors(sel, opts) + + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "SharePoint") +} diff --git a/src/cli/export/sharepoint_test.go b/src/cli/export/sharepoint_test.go new file mode 100644 index 000000000..48ce28f5c --- /dev/null +++ b/src/cli/export/sharepoint_test.go @@ -0,0 +1,118 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type SharePointUnitSuite struct { + tester.Suite +} + +func TestSharePointUnitSuite(t *testing.T) { + suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharePointUnitSuite) TestAddSharePointCommands() { + expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export sharepoint", exportCommand, expectUse, sharePointExportCmd().Short, exportSharePointCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addSharePointCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "sharepoint", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.LibraryFN, testdata.LibraryInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + "--" + flags.ListItemFN, testdata.FlgInputs(testdata.ListItemInput), + "--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput), + "--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput), + "--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput), + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeSharePointOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.LibraryInput, opts.Library) + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.ElementsMatch(t, testdata.ListItemInput, opts.ListItem) + assert.ElementsMatch(t, testdata.ListFolderInput, opts.ListFolder) + + assert.ElementsMatch(t, testdata.PageInput, opts.Page) + assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/flags/export.go b/src/cli/flags/export.go new file mode 100644 index 000000000..b9af7e141 --- /dev/null +++ b/src/cli/flags/export.go @@ -0,0 +1,15 @@ +package flags + +import ( + "github.com/spf13/cobra" +) + +const ArchiveFN = "archive" + +var ArchiveFV bool + +// AddExportConfigFlags adds the restore config flag set. +func AddExportConfigFlags(cmd *cobra.Command) { + fs := cmd.Flags() + fs.BoolVar(&ArchiveFV, ArchiveFN, false, "Export data as an archive instead of individual files") +} diff --git a/src/cli/flags/options.go b/src/cli/flags/options.go index 046d3c8d7..81a893f93 100644 --- a/src/cli/flags/options.go +++ b/src/cli/flags/options.go @@ -5,6 +5,7 @@ import ( ) const ( + DeltaPageSizeFN = "delta-page-size" DisableConcurrencyLimiterFN = "disable-concurrency-limiter" DisableDeltaFN = "disable-delta" DisableIncrementalsFN = "disable-incrementals" @@ -21,6 +22,7 @@ const ( ) var ( + DeltaPageSizeFV int DisableConcurrencyLimiterFV bool DisableDeltaFV bool DisableIncrementalsFV bool @@ -72,6 +74,18 @@ func AddSkipReduceFlag(cmd *cobra.Command) { cobra.CheckErr(fs.MarkHidden(SkipReduceFN)) } +// AddDeltaPageSizeFlag adds a hidden flag that allows callers to reduce delta +// query page sizes below 500. +func AddDeltaPageSizeFlag(cmd *cobra.Command) { + fs := cmd.Flags() + fs.IntVar( + &DeltaPageSizeFV, + DeltaPageSizeFN, + 500, + "Control quantity of items returned in paged queries. Valid range is [1-500]. Default: 500") + cobra.CheckErr(fs.MarkHidden(DeltaPageSizeFN)) +} + // AddFetchParallelismFlag adds a hidden flag that allows callers to reduce call // paralellism (ie, the corso worker pool size) from 4 to as low as 1. func AddFetchParallelismFlag(cmd *cobra.Command) { diff --git a/src/cli/flags/restore_config.go b/src/cli/flags/restore_config.go index a2b8c3a86..4a1868d01 100644 --- a/src/cli/flags/restore_config.go +++ b/src/cli/flags/restore_config.go @@ -9,11 +9,13 @@ import ( const ( CollisionsFN = "collisions" DestinationFN = "destination" + ToResourceFN = "to-resource" ) var ( CollisionsFV string DestinationFV string + ToResourceFV string ) // AddRestoreConfigFlags adds the restore config flag set. @@ -25,5 +27,8 @@ func AddRestoreConfigFlags(cmd *cobra.Command) { "Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace)) fs.StringVar( &DestinationFV, DestinationFN, "", - "Overrides the destination where items get restored; '/' places items into their original location") + "Overrides the folder where items get restored; '/' places items into their original location") + fs.StringVar( + &ToResourceFV, ToResourceFN, "", + "Overrides the protected resource (mailbox, site, user, etc) where data gets restored") } diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 58f04764c..b611e93e5 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/account" + rep "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -158,7 +159,13 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) } - r, err := repository.Initialize(ctx, cfg.Account, cfg.Storage, opt) + // TODO(ashmrtn): Wire to flags for retention during repo init. + r, err := repository.Initialize( + ctx, + cfg.Account, + cfg.Storage, + opt, + rep.Retention{}) if err != nil { if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index 540d08836..4b4975a6a 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) @@ -200,7 +201,12 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { ctx = config.SetViper(ctx, vpr) // init the repo first - _, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults()) + _, err = repository.Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 56b0e7255..904025116 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" @@ -83,7 +84,12 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() { ) // init the repo first - suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{}) + suite.repo, err = repository.Initialize( + ctx, + suite.acct, + suite.st, + control.Options{}, + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) diff --git a/src/cli/restore/exchange_test.go b/src/cli/restore/exchange_test.go index 0dd022a81..a257fff7a 100644 --- a/src/cli/restore/exchange_test.go +++ b/src/cli/restore/exchange_test.go @@ -84,6 +84,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -125,6 +126,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) diff --git a/src/cli/restore/onedrive_test.go b/src/cli/restore/onedrive_test.go index a21f1191d..8a9fc7a94 100644 --- a/src/cli/restore/onedrive_test.go +++ b/src/cli/restore/onedrive_test.go @@ -70,6 +70,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -80,6 +81,9 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -99,6 +103,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -109,6 +114,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index b4547077f..c9bc8277f 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -34,7 +34,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { expectShort string expectRunE func(*cobra.Command, []string) error }{ - {"restore onedrive", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, + {"restore sharepoint", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, } for _, test := range table { suite.Run(test.name, func() { @@ -75,6 +75,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.DestinationFN, testdata.Destination, + "--" + flags.ToResourceFN, testdata.ToResource, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, @@ -85,6 +86,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.RestorePermissionsFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output @@ -111,6 +115,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) + assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) @@ -121,6 +126,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + + // bool flags + assert.True(t, flags.RestorePermissionsFV) }) } } diff --git a/src/cli/utils/export_config.go b/src/cli/utils/export_config.go new file mode 100644 index 000000000..2fd3827bd --- /dev/null +++ b/src/cli/utils/export_config.go @@ -0,0 +1,38 @@ +package utils + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgOpts struct { + Archive bool + + Populated flags.PopulatedFlags +} + +func makeExportCfgOpts(cmd *cobra.Command) ExportCfgOpts { + return ExportCfgOpts{ + Archive: flags.ArchiveFV, + + // populated contains the list of flags that appear in the + // command, according to pflags. Use this to differentiate + // between an "empty" and a "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +func MakeExportConfig( + ctx context.Context, + opts ExportCfgOpts, +) control.ExportConfig { + exportCfg := control.DefaultExportConfig() + + exportCfg.Archive = opts.Archive + + return exportCfg +} diff --git a/src/cli/utils/export_config_test.go b/src/cli/utils/export_config_test.go new file mode 100644 index 000000000..d25d6629b --- /dev/null +++ b/src/cli/utils/export_config_test.go @@ -0,0 +1,54 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" +) + +type ExportCfgUnitSuite struct { + tester.Suite +} + +func TestExportCfgUnitSuite(t *testing.T) { + suite.Run(t, &ExportCfgUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportCfgUnitSuite) TestMakeExportConfig() { + rco := &ExportCfgOpts{Archive: true} + + table := []struct { + name string + populated flags.PopulatedFlags + expect control.ExportConfig + }{ + { + name: "archive populated", + populated: flags.PopulatedFlags{ + flags.ArchiveFN: {}, + }, + expect: control.ExportConfig{ + Archive: true, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := *rco + opts.Populated = test.populated + + result := MakeExportConfig(ctx, opts) + assert.Equal(t, test.expect.Archive, result.Archive) + }) + } +} diff --git a/src/cli/utils/onedrive.go b/src/cli/utils/onedrive.go index 06699adca..e12f35230 100644 --- a/src/cli/utils/onedrive.go +++ b/src/cli/utils/onedrive.go @@ -19,6 +19,7 @@ type OneDriveOpts struct { FileModifiedBefore string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -35,6 +36,7 @@ func MakeOneDriveOpts(cmd *cobra.Command) OneDriveOpts { FileModifiedBefore: flags.FileModifiedBeforeFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 0cc44c839..932c56b6b 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -8,14 +8,19 @@ import ( // Control produces the control options based on the user's flags. func Control() control.Options { - opt := control.Defaults() + opt := control.DefaultOptions() if flags.FailFastFV { opt.FailureHandling = control.FailFast } + dps := int32(flags.DeltaPageSizeFV) + if dps > 500 || dps < 1 { + dps = 500 + } + + opt.DeltaPageSize = dps opt.DisableMetrics = flags.NoStatsFV - opt.RestorePermissions = flags.RestorePermissionsFV opt.SkipReduce = flags.SkipReduceFV opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV diff --git a/src/cli/utils/options_test.go b/src/cli/utils/options_test.go index 746558aa1..1a8f7ddcd 100644 --- a/src/cli/utils/options_test.go +++ b/src/cli/utils/options_test.go @@ -35,6 +35,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { assert.True(t, flags.SkipReduceFV, flags.SkipReduceFN) assert.Equal(t, 2, flags.FetchParallelismFV, flags.FetchParallelismFN) assert.True(t, flags.DisableConcurrencyLimiterFV, flags.DisableConcurrencyLimiterFN) + assert.Equal(t, 499, flags.DeltaPageSizeFV, flags.DeltaPageSizeFN) }, } @@ -48,6 +49,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { flags.AddSkipReduceFlag(cmd) flags.AddFetchParallelismFlag(cmd) flags.AddDisableConcurrencyLimiterFlag(cmd) + flags.AddDeltaPageSizeFlag(cmd) // Test arg parsing for few args cmd.SetArgs([]string{ @@ -60,6 +62,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { "--" + flags.SkipReduceFN, "--" + flags.FetchParallelismFN, "2", "--" + flags.DisableConcurrencyLimiterFN, + "--" + flags.DeltaPageSizeFN, "499", }) err := cmd.Execute() diff --git a/src/cli/utils/restore_config.go b/src/cli/utils/restore_config.go index fa036e3f9..6be54f1ab 100644 --- a/src/cli/utils/restore_config.go +++ b/src/cli/utils/restore_config.go @@ -18,16 +18,20 @@ type RestoreCfgOpts struct { // DTTMFormat is the timestamp format appended // to the default folder name. Defaults to // dttm.HumanReadable. - DTTMFormat dttm.TimeFormat + DTTMFormat dttm.TimeFormat + ProtectedResource string + RestorePermissions bool Populated flags.PopulatedFlags } func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts { return RestoreCfgOpts{ - Collisions: flags.CollisionsFV, - Destination: flags.DestinationFV, - DTTMFormat: dttm.HumanReadable, + Collisions: flags.CollisionsFV, + Destination: flags.DestinationFV, + DTTMFormat: dttm.HumanReadable, + ProtectedResource: flags.ToResourceFV, + RestorePermissions: flags.RestorePermissionsFV, // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate @@ -67,6 +71,9 @@ func MakeRestoreConfig( restoreCfg.Location = opts.Destination } + restoreCfg.ProtectedResource = opts.ProtectedResource + restoreCfg.IncludePermissions = opts.RestorePermissions + Infof(ctx, "Restoring to folder %s", restoreCfg.Location) return restoreCfg diff --git a/src/cli/utils/restore_config_test.go b/src/cli/utils/restore_config_test.go index 1324c9571..c3509e360 100644 --- a/src/cli/utils/restore_config_test.go +++ b/src/cli/utils/restore_config_test.go @@ -68,18 +68,18 @@ func (suite *RestoreCfgUnitSuite) TestValidateRestoreConfigFlags() { } func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { - rco := &RestoreCfgOpts{ - Collisions: "collisions", - Destination: "destination", - } - table := []struct { name string + rco *RestoreCfgOpts populated flags.PopulatedFlags expect control.RestoreConfig }{ { - name: "not populated", + name: "not populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{}, expect: control.RestoreConfig{ OnCollision: control.Skip, @@ -88,6 +88,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "collision populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, }, @@ -98,6 +102,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "destination populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.DestinationFN: {}, }, @@ -108,6 +116,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { }, { name: "both populated", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + }, populated: flags.PopulatedFlags{ flags.CollisionsFN: {}, flags.DestinationFN: {}, @@ -117,6 +129,23 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { Location: "destination", }, }, + { + name: "with restore permissions", + rco: &RestoreCfgOpts{ + Collisions: "collisions", + Destination: "destination", + RestorePermissions: true, + }, + populated: flags.PopulatedFlags{ + flags.CollisionsFN: {}, + flags.DestinationFN: {}, + }, + expect: control.RestoreConfig{ + OnCollision: control.CollisionPolicy("collisions"), + Location: "destination", + IncludePermissions: true, + }, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -125,12 +154,13 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { ctx, flush := tester.NewContext(t) defer flush() - opts := *rco + opts := *test.rco opts.Populated = test.populated result := MakeRestoreConfig(ctx, opts) assert.Equal(t, test.expect.OnCollision, result.OnCollision) assert.Contains(t, result.Location, test.expect.Location) + assert.Equal(t, test.expect.IncludePermissions, result.IncludePermissions) }) } } diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index 6672beda1..2ab43d90c 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -32,6 +32,7 @@ type SharePointOpts struct { Page []string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -56,6 +57,7 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts { PageFolder: flags.PageFolderFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/cli/utils/testdata/flags.go b/src/cli/utils/testdata/flags.go index f97529b57..b04a9ab63 100644 --- a/src/cli/utils/testdata/flags.go +++ b/src/cli/utils/testdata/flags.go @@ -46,8 +46,13 @@ var ( Collisions = "collisions" Destination = "destination" + ToResource = "toResource" RestorePermissions = true + DeltaPageSize = "deltaPageSize" + + Archive = true + AzureClientID = "testAzureClientId" AzureTenantID = "testAzureTenantId" AzureClientSecret = "testAzureClientSecret" @@ -57,4 +62,6 @@ var ( AWSSessionToken = "testAWSSessionToken" CorsoPassphrase = "testCorsoPassphrase" + + RestoreDestination = "test-restore-destination" ) diff --git a/src/cli/utils/users.go b/src/cli/utils/users.go index 610f0e2c6..affa520fd 100644 --- a/src/cli/utils/users.go +++ b/src/cli/utils/users.go @@ -7,6 +7,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -15,9 +16,10 @@ import ( func UsersMap( ctx context.Context, acct account.Account, + co control.Options, errs *fault.Bus, ) (idname.Cacher, error) { - au, err := makeUserAPI(acct) + au, err := makeUserAPI(acct, co) if err != nil { return nil, clues.Wrap(err, "constructing a graph client") } @@ -25,13 +27,13 @@ func UsersMap( return au.GetAllIDsAndNames(ctx, errs) } -func makeUserAPI(acct account.Account) (api.Users, error) { +func makeUserAPI(acct account.Account, co control.Options) (api.Users, error) { creds, err := acct.M365Config() if err != nil { return api.Users{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds) + cli, err := api.NewClient(creds, co) if err != nil { return api.Users{}, clues.Wrap(err, "constructing api client") } diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 04967dc2a..5904e09d4 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -21,12 +21,12 @@ import ( odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" "github.com/alcionai/corso/src/internal/m365/resource" m365Stub "github.com/alcionai/corso/src/internal/m365/stub" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" @@ -104,7 +104,15 @@ func generateAndRestoreItems( print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, dataColls, errs, ctr) } // ------------------------------------------------------------------------------------------ @@ -144,7 +152,7 @@ func getControllerAndVerifyResourceOwner( return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") } - id, _, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, resourceOwner, nil) + id, _, err := ctrl.PopulateProtectedResourceIDAndName(ctx, resourceOwner, nil) if err != nil { return nil, account.Account{}, nil, clues.Wrap(err, "verifying user") } @@ -216,7 +224,8 @@ var ( func generateAndRestoreDriveItems( ctrl *m365.Controller, - resourceOwner, secondaryUserID, secondaryUserName string, + protectedResource idname.Provider, + secondaryUserID, secondaryUserName string, acct account.Account, service path.ServiceType, cat path.CategoryType, @@ -240,14 +249,23 @@ func generateAndRestoreDriveItems( switch service { case path.SharePointService: - d, err := ctrl.AC.Stable.Client().Sites().BySiteId(resourceOwner).Drive().Get(ctx, nil) + d, err := ctrl.AC.Stable. + Client(). + Sites(). + BySiteId(protectedResource.ID()). + Drive(). + Get(ctx, nil) if err != nil { return nil, clues.Wrap(err, "getting site's default drive") } driveID = ptr.Val(d.GetId()) default: - d, err := ctrl.AC.Stable.Client().Users().ByUserId(resourceOwner).Drive().Get(ctx, nil) + d, err := ctrl.AC.Stable.Client(). + Users(). + ByUserId(protectedResource.ID()). + Drive(). + Get(ctx, nil) if err != nil { return nil, clues.Wrap(err, "getting user's default drive") } @@ -407,18 +425,16 @@ func generateAndRestoreDriveItems( // input, // version.Backup) - opts := control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - } + opts := control.DefaultOptions() + restoreCfg.IncludePermissions = true config := m365Stub.ConfigInfo{ Opts: opts, Resource: resource.Users, Service: service, Tenant: tenantID, - ResourceOwners: []string{resourceOwner}, - RestoreCfg: testdata.DefaultRestoreConfig(""), + ResourceOwners: []string{protectedResource.ID()}, + RestoreCfg: restoreCfg, } _, _, collections, _, err := m365Stub.GetCollectionsAndExpected( @@ -429,5 +445,13 @@ func generateAndRestoreDriveItems( return nil, err } - return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs, ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: opts, + ProtectedResource: protectedResource, + RestoreConfig: restoreCfg, + Selector: sel, + } + + return ctrl.ConsumeRestoreCollections(ctx, rcc, collections, errs, ctr) } diff --git a/src/cmd/factory/impl/exchange.go b/src/cmd/factory/impl/exchange.go index eb923969b..b7ad4840d 100644 --- a/src/cmd/factory/impl/exchange.go +++ b/src/cmd/factory/impl/exchange.go @@ -72,7 +72,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error { subject, body, body, now, now, now, now) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -121,7 +121,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error exchMock.NoAttachments, exchMock.NoCancelledOccurrences, exchMock.NoExceptionOccurrences) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { @@ -172,7 +172,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error { "123-456-7890", ) }, - control.Defaults(), + control.DefaultOptions(), errs, count.New()) if err != nil { diff --git a/src/cmd/factory/impl/onedrive.go b/src/cmd/factory/impl/onedrive.go index e647863dd..6ccc98977 100644 --- a/src/cmd/factory/impl/onedrive.go +++ b/src/cmd/factory/impl/onedrive.go @@ -47,7 +47,7 @@ func handleOneDriveFileFactory(cmd *cobra.Command, args []string) error { deets, err := generateAndRestoreDriveItems( ctrl, - inp.ID(), + inp, SecondaryUser, strings.ToLower(SecondaryUser), acct, diff --git a/src/cmd/factory/impl/sharepoint.go b/src/cmd/factory/impl/sharepoint.go index eeec57c95..ab9d3fc92 100644 --- a/src/cmd/factory/impl/sharepoint.go +++ b/src/cmd/factory/impl/sharepoint.go @@ -47,7 +47,7 @@ func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error deets, err := generateAndRestoreDriveItems( ctrl, - inp.ID(), + inp, SecondaryUser, strings.ToLower(SecondaryUser), acct, diff --git a/src/cmd/getM365/exchange/get_item.go b/src/cmd/getM365/exchange/get_item.go deleted file mode 100644 index cc6e8cd6a..000000000 --- a/src/cmd/getM365/exchange/get_item.go +++ /dev/null @@ -1,157 +0,0 @@ -// get_item.go is a source file designed to retrieve an m365 object from an -// existing M365 account. Data displayed is representative of the current -// serialization abstraction versioning used by Microsoft Graph and stored by Corso. - -package exchange - -import ( - "context" - "fmt" - "os" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - kw "github.com/microsoft/kiota-serialization-json-go" - "github.com/spf13/cobra" - - "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -// Required inputs from user for command execution -var ( - user, tenant, m365ID, category string -) - -func AddCommands(parent *cobra.Command) { - exCmd := &cobra.Command{ - Use: "exchange", - Short: "Get an M365ID item JSON", - RunE: handleExchangeCmd, - } - - fs := exCmd.PersistentFlags() - fs.StringVar(&m365ID, "id", "", "m365 identifier for object") - fs.StringVar(&category, "category", "", "type of M365 data (contacts, email, events)") - fs.StringVar(&user, "user", "", "m365 user id of M365 user") - fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") - - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("category")) - - parent.AddCommand(exCmd) -} - -func handleExchangeCmd(cmd *cobra.Command, args []string) error { - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - tid := str.First(tenant, os.Getenv(account.AzureTenantID)) - - ctx := clues.Add( - cmd.Context(), - "item_id", m365ID, - "resource_owner", user, - "tenant", tid) - - creds := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: tid, - } - - err := runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true)) - if err != nil { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - - return clues.Wrap(err, "getting item") - } - - return nil -} - -func runDisplayM365JSON( - ctx context.Context, - creds account.M365Config, - user, itemID string, - errs *fault.Bus, -) error { - var ( - bs []byte - err error - cat = path.ToCategoryType(category) - sw = kw.NewJsonSerializationWriter() - ) - - ac, err := api.NewClient(creds) - if err != nil { - return err - } - - switch cat { - case path.EmailCategory: - bs, err = getItem(ctx, ac.Mail(), user, itemID, true, errs) - case path.EventsCategory: - bs, err = getItem(ctx, ac.Events(), user, itemID, true, errs) - case path.ContactsCategory: - bs, err = getItem(ctx, ac.Contacts(), user, itemID, true, errs) - default: - return fmt.Errorf("unable to process category: %s", cat) - } - - if err != nil { - return err - } - - err = sw.WriteStringValue("", ptr.To(string(bs))) - if err != nil { - return clues.Wrap(err, "Error writing string value: "+itemID) - } - - array, err := sw.GetSerializedContent() - if err != nil { - return clues.Wrap(err, "Error serializing item: "+itemID) - } - - fmt.Println(string(array)) - - return nil -} - -type itemer interface { - GetItem( - ctx context.Context, - user, itemID string, - immutableID bool, - errs *fault.Bus, - ) (serialization.Parsable, *details.ExchangeInfo, error) - Serialize( - ctx context.Context, - item serialization.Parsable, - user, itemID string, - ) ([]byte, error) -} - -func getItem( - ctx context.Context, - itm itemer, - user, itemID string, - immutableIDs bool, - errs *fault.Bus, -) ([]byte, error) { - sp, _, err := itm.GetItem(ctx, user, itemID, immutableIDs, errs) - if err != nil { - return nil, clues.Wrap(err, "getting item") - } - - return itm.Serialize(ctx, sp, user, itemID) -} diff --git a/src/cmd/getM365/main.go b/src/cmd/getM365/main.go deleted file mode 100644 index c7acd3175..000000000 --- a/src/cmd/getM365/main.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "context" - "os" - - "github.com/spf13/cobra" - - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cmd/getM365/exchange" - "github.com/alcionai/corso/src/cmd/getM365/onedrive" - "github.com/alcionai/corso/src/pkg/logger" -) - -var rootCmd = &cobra.Command{ - Use: "getM365", -} - -func main() { - ls := logger.Settings{ - Level: logger.LLDebug, - Format: logger.LFText, - } - ctx, _ := logger.CtxOrSeed(context.Background(), ls) - - ctx = SetRootCmd(ctx, rootCmd) - defer logger.Flush(ctx) - - exchange.AddCommands(rootCmd) - onedrive.AddCommands(rootCmd) - - if err := rootCmd.Execute(); err != nil { - Err(ctx, err) - os.Exit(1) - } -} diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go deleted file mode 100644 index 05b5395ce..000000000 --- a/src/cmd/getM365/onedrive/get_item.go +++ /dev/null @@ -1,207 +0,0 @@ -// get_item.go is a source file designed to retrieve an m365 object from an -// existing M365 account. Data displayed is representative of the current -// serialization abstraction versioning used by Microsoft Graph and stored by Corso. - -package onedrive - -import ( - "context" - "encoding/json" - "io" - "net/http" - "os" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - kjson "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/spf13/cobra" - - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/common/str" - "github.com/alcionai/corso/src/internal/m365/graph" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/services/m365/api" -) - -const downloadURLKey = "@microsoft.graph.downloadUrl" - -// Required inputs from user for command execution -var ( - user, tenant, m365ID string -) - -func AddCommands(parent *cobra.Command) { - exCmd := &cobra.Command{ - Use: "onedrive", - Short: "Get an M365ID item", - RunE: handleOneDriveCmd, - } - - fs := exCmd.PersistentFlags() - fs.StringVar(&m365ID, "id", "", "m365 identifier for object") - fs.StringVar(&user, "user", "", "m365 user id of M365 user") - fs.StringVar(&tenant, "tenant", "", "m365 identifier for the tenant") - - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("user")) - cobra.CheckErr(exCmd.MarkPersistentFlagRequired("id")) - - parent.AddCommand(exCmd) -} - -func handleOneDriveCmd(cmd *cobra.Command, args []string) error { - if utils.HasNoFlagsAndShownHelp(cmd) { - return nil - } - - tid := str.First(tenant, os.Getenv(account.AzureTenantID)) - - ctx := clues.Add( - cmd.Context(), - "item_id", m365ID, - "resource_owner", user, - "tenant", tid) - - // get account info - creds := account.M365Config{ - M365: credentials.GetM365(), - AzureTenantID: tid, - } - - gr := graph.NewNoTimeoutHTTPWrapper() - - ac, err := api.NewClient(creds) - if err != nil { - return Only(ctx, clues.Wrap(err, "getting api client")) - } - - err = runDisplayM365JSON(ctx, ac, gr, creds, user, m365ID) - if err != nil { - cmd.SilenceUsage = true - cmd.SilenceErrors = true - - return Only(ctx, clues.Wrap(err, "getting item")) - } - - return nil -} - -type itemData struct { - Size int `json:"size"` -} - -type itemPrintable struct { - Info json.RawMessage `json:"info"` - Permissions json.RawMessage `json:"permissions"` - Data itemData `json:"data"` -} - -func (i itemPrintable) MinimumPrintable() any { - return i -} - -func runDisplayM365JSON( - ctx context.Context, - ac api.Client, - gr graph.Requester, - creds account.M365Config, - userID, itemID string, -) error { - drive, err := ac.Users().GetDefaultDrive(ctx, userID) - if err != nil { - return err - } - - driveID := ptr.Val(drive.GetId()) - - it := itemPrintable{} - - item, err := ac.Drives().GetItem(ctx, driveID, itemID) - if err != nil { - return err - } - - if item != nil { - content, err := getDriveItemContent(ctx, gr, item) - if err != nil { - return err - } - - // We could get size from item.GetSize(), but the - // getDriveItemContent call is to ensure that we are able to - // download the file. - it.Data.Size = len(content) - } - - sInfo, err := serializeObject(item) - if err != nil { - return err - } - - err = json.Unmarshal([]byte(sInfo), &it.Info) - if err != nil { - return err - } - - perms, err := ac.Drives().GetItemPermission(ctx, driveID, itemID) - if err != nil { - return err - } - - sPerms, err := serializeObject(perms) - if err != nil { - return err - } - - err = json.Unmarshal([]byte(sPerms), &it.Permissions) - if err != nil { - return err - } - - PrettyJSON(ctx, it) - - return nil -} - -func serializeObject(data serialization.Parsable) (string, error) { - sw := kjson.NewJsonSerializationWriter() - - err := sw.WriteObjectValue("", data) - if err != nil { - return "", clues.Wrap(err, "writing serializing info") - } - - content, err := sw.GetSerializedContent() - if err != nil { - return "", clues.Wrap(err, "getting serializing info") - } - - return string(content), err -} - -func getDriveItemContent( - ctx context.Context, - gr graph.Requester, - item models.DriveItemable, -) ([]byte, error) { - url, ok := item.GetAdditionalData()[downloadURLKey].(*string) - if !ok { - return nil, clues.New("retrieving download url") - } - - resp, err := gr.Request(ctx, http.MethodGet, *url, nil, nil) - if err != nil { - return nil, clues.New("requesting item content").With("error", err) - } - defer resp.Body.Close() - - content, err := io.ReadAll(resp.Body) - if err != nil { - return nil, clues.New("reading item content").With("error", err) - } - - return content, nil -} diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index 33de060f8..8a5c01537 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -7,15 +7,122 @@ import ( "strconv" "time" + "github.com/alcionai/clues" "github.com/spf13/cobra" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/store" ) +// deleteBackups connects to the repository and deletes all backups for +// service that are at least deletionDays old. Returns the IDs of all backups +// that were deleted. +func deleteBackups( + ctx context.Context, + service path.ServiceType, + deletionDays int, +) ([]string, error) { + ctx = clues.Add(ctx, "cutoff_days", deletionDays) + + r, _, _, _, err := utils.GetAccountAndConnect(ctx, service, nil) + if err != nil { + return nil, clues.Wrap(err, "connecting to account").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return nil, clues.Wrap(err, "listing backups").WithClues(ctx) + } + + var ( + deleted []string + cutoff = time.Now().Add(-time.Hour * 24 * time.Duration(deletionDays)) + ) + + for _, backup := range backups { + if backup.StartAndEndTime.CompletedAt.Before(cutoff) { + if err := r.DeleteBackup(ctx, backup.ID.String()); err != nil { + return nil, clues.Wrap( + err, + "deleting backup"). + With("backup_id", backup.ID). + WithClues(ctx) + } + + deleted = append(deleted, backup.ID.String()) + logAndPrint(ctx, "Deleted backup %s", backup.ID.String()) + } + } + + return deleted, nil +} + +// pitrListBackups connects to the repository at the given point in time and +// lists the backups for service. It then checks the list of backups contains +// the backups in backupIDs. +// +//nolint:unused +//lint:ignore U1000 Waiting for full support. +func pitrListBackups( + ctx context.Context, + service path.ServiceType, + pitr time.Time, + backupIDs []string, +) error { + if len(backupIDs) == 0 { + return nil + } + + ctx = clues.Add(ctx, "pitr_time", pitr, "search_backups", backupIDs) + + // TODO(ashmrtn): This may be moved into CLI layer at some point when we add + // flags for opening a repo at a point in time. + cfg, err := config.GetConfigRepoDetails(ctx, true, true, nil) + if err != nil { + return clues.Wrap(err, "getting config info") + } + + opts := utils.ControlWithConfig(cfg) + opts.Repo.ViewTimestamp = &pitr + + r, err := repository.Connect(ctx, cfg.Account, cfg.Storage, cfg.RepoID, opts) + if err != nil { + return clues.Wrap(err, "connecting to repo").WithClues(ctx) + } + + defer r.Close(ctx) + + backups, err := r.BackupsByTag(ctx, store.Service(service)) + if err != nil { + return clues.Wrap(err, "listing backups").WithClues(ctx) + } + + bups := map[string]struct{}{} + + for _, backup := range backups { + bups[backup.ID.String()] = struct{}{} + } + + ctx = clues.Add(ctx, "found_backups", maps.Keys(bups)) + + for _, backupID := range backupIDs { + if _, ok := bups[backupID]; !ok { + return clues.New("looking for backup"). + With("search_backup_id", backupID). + WithClues(ctx) + } + } + + return nil +} + func main() { var ( service path.ServiceType @@ -39,31 +146,16 @@ func main() { fatal(cc.Context(), "unknown service", nil) } - r, _, _, _, err := utils.GetAccountAndConnect(cc.Context(), service, nil) - if err != nil { - fatal(cc.Context(), "unable to connect account", err) - } - - defer r.Close(cc.Context()) - - backups, err := r.BackupsByTag(cc.Context(), store.Service(service)) - if err != nil { - fatal(cc.Context(), "unable to find backups", err) - } + ctx := clues.Add(cc.Context(), "service", service) days, err := strconv.Atoi(os.Getenv("DELETION_DAYS")) if err != nil { - fatal(cc.Context(), "invalid no of days provided", nil) + fatal(ctx, "invalid number of days provided", nil) } - for _, backup := range backups { - if backup.StartAndEndTime.CompletedAt.Before(time.Now().AddDate(0, 0, -days)) { - if err := r.DeleteBackup(cc.Context(), backup.ID.String()); err != nil { - fatal(cc.Context(), "deleting backup", err) - } - - logAndPrint(cc.Context(), "Deleted backup %s", backup.ID.String()) - } + _, err = deleteBackups(ctx, service, days) + if err != nil { + fatal(cc.Context(), "deleting backups", clues.Stack(err)) } } diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 7372245aa..e8f258b95 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -19,14 +19,17 @@ Param ( [datetime]$PurgeBeforeTimestamp, [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] - [String[]]$FolderPrefixPurgeList + [String[]]$FolderPrefixPurgeList, + + [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] + [String[]]$LibraryPrefixDeleteList = @() ) Set-StrictMode -Version 2.0 # Attempt to set network timeout to 10min [System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000 -function Get-TimestampFromName { +function Get-TimestampFromFolderName { param ( [Parameter(Mandatory = $True, HelpMessage = "Folder ")] [Microsoft.SharePoint.Client.Folder]$folder @@ -54,6 +57,36 @@ function Get-TimestampFromName { return $timestamp } + +function Get-TimestampFromListName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "List ")] + [Microsoft.SharePoint.Client.List]$list + ) + + $name = $list.Title + + #fallback on list create time + [datetime]$timestamp = $list.LastItemUserModifiedDate + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "List: $name, create timestamp: $timestamp" + + return $timestamp +} + function Purge-Library { [CmdletBinding(SupportsShouldProcess)] Param ( @@ -77,7 +110,7 @@ function Purge-Library { foreach ($f in $folders) { $folderName = $f.Name - $createTime = Get-TimestampFromName -Folder $f + $createTime = Get-TimestampFromFolderName -Folder $f if ($PurgeBeforeTimestamp -gt $createTime) { foreach ($p in $FolderPrefixPurgeList) { @@ -97,7 +130,7 @@ function Purge-Library { if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") { $siteRelativeParentPath = $Matches.1 } - + if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) { Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath" try { @@ -110,6 +143,54 @@ function Purge-Library { } } +function Delete-LibraryByPrefix { + [CmdletBinding(SupportsShouldProcess)] + Param ( + [Parameter(Mandatory = $True, HelpMessage = "Document library root")] + [String]$LibraryNamePrefix, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Site suffix")] + [String[]]$SiteSuffix + ) + + Write-Host "`nDeleting library: $LibraryNamePrefix" + + $listsToDelete = @() + $lists = Get-PnPList + + foreach ($l in $lists) { + $listName = $l.Title + $createTime = Get-TimestampFromListName -List $l + + if ($PurgeBeforeTimestamp -gt $createTime) { + foreach ($p in $FolderPrefixPurgeList) { + if ($listName -like "$p*") { + $listsToDelete += $l + } + } + } + } + + Write-Host "Found"$listsToDelete.count"lists to delete" + + foreach ($l in $listsToDelete) { + $listName = $l.Title + + if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) { + Write-Host "Deleting list: "$l.Title + try { + Remove-PnPList -Identity $l.Id -Force + } + catch [ System.Management.Automation.ItemNotFoundException ] { + Write-Host "List: "$f.Name" is already deleted. Skipping..." + } + } + } +} + ######## MAIN ######### # Setup SharePointPnP @@ -176,4 +257,8 @@ $FolderPrefixPurgeList = $FolderPrefixPurgeList | ForEach-Object { @($_.Split(', foreach ($library in $LibraryNameList) { Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSuffix -} \ No newline at end of file +} + +foreach ($libraryPfx in $LibraryPrefixDeleteList) { + Delete-LibraryByPrefix -LibraryNamePrefix $libraryPfx -PurgeBeforeTimestamp $PurgeBeforeTimestamp -SiteSuffix $siteSuffix +} diff --git a/src/cmd/sanity_test/common/common.go b/src/cmd/sanity_test/common/common.go new file mode 100644 index 000000000..344d6dc19 --- /dev/null +++ b/src/cmd/sanity_test/common/common.go @@ -0,0 +1,6 @@ +package common + +type PermissionInfo struct { + EntityID string + Roles []string +} diff --git a/src/cmd/sanity_test/common/utils.go b/src/cmd/sanity_test/common/utils.go new file mode 100644 index 000000000..e14fa86c6 --- /dev/null +++ b/src/cmd/sanity_test/common/utils.go @@ -0,0 +1,82 @@ +package common + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} diff --git a/src/cmd/sanity_test/export/onedrive.go b/src/cmd/sanity_test/export/onedrive.go new file mode 100644 index 000000000..3d5564bcc --- /dev/null +++ b/src/cmd/sanity_test/export/onedrive.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckOneDriveExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/export/sharepoint.go b/src/cmd/sanity_test/export/sharepoint.go new file mode 100644 index 000000000..55ab8ed5c --- /dev/null +++ b/src/cmd/sanity_test/export/sharepoint.go @@ -0,0 +1,88 @@ +package export + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/internal/common/ptr" +) + +func CheckSharePointExport( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, folderName, dataFolder string, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + // map itemID -> item size + var ( + fileSizes = make(map[string]int64) + exportFileSizes = make(map[string]int64) + startTime = time.Now() + ) + + err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error { + if err != nil { + return clues.Stack(err) + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(folderName, path) + if err != nil { + return clues.Stack(err) + } + + exportFileSizes[relPath] = info.Size() + if startTime.After(info.ModTime()) { + startTime = info.ModTime() + } + + return nil + }) + if err != nil { + fmt.Println("Error walking the path:", err) + } + + _ = restore.PopulateDriveDetails( + ctx, + client, + ptr.Val(drive.GetId()), + folderName, + dataFolder, + fileSizes, + map[string][]common.PermissionInfo{}, + startTime) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := exportFileSizes[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go new file mode 100644 index 000000000..2dc65e6e1 --- /dev/null +++ b/src/cmd/sanity_test/restore/exchange.go @@ -0,0 +1,219 @@ +package restore + +import ( + "context" + "fmt" + stdpath "path" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-sdk-go/users" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/filters" +) + +// CheckEmailRestoration verifies that the emails count in restored folder is equivalent to +// emails in actual m365 account +func CheckEmailRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser, folderName, dataFolder, baseBackupFolder string, + startTime time.Time, +) { + var ( + restoreFolder models.MailFolderable + itemCount = make(map[string]int32) + restoreItemCount = make(map[string]int32) + builder = client.Users().ByUserId(testUser).MailFolders() + ) + + for { + result, err := builder.Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting mail folders", err) + } + + values := result.GetValue() + + for _, v := range values { + itemName := ptr.Val(v.GetDisplayName()) + + if itemName == folderName { + restoreFolder = v + continue + } + + if itemName == dataFolder || itemName == baseBackupFolder { + // otherwise, recursively aggregate all child folders. + getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) + + itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) + } + } + + link, ok := ptr.ValOK(result.GetOdataNextLink()) + if !ok { + break + } + + builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) + } + + folderID := ptr.Val(restoreFolder.GetId()) + folderName = ptr.Val(restoreFolder.GetDisplayName()) + ctx = clues.Add( + ctx, + "restore_folder_id", folderID, + "restore_folder_name", folderName) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting restore folder child folders", err) + } + + for _, fld := range childFolder.GetValue() { + restoreDisplayName := ptr.Val(fld.GetDisplayName()) + + // check if folder is the data folder we loaded or the base backup to verify + // the incremental backup worked fine + if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { + count, _ := ptr.ValOK(fld.GetTotalItemCount()) + + restoreItemCount[restoreDisplayName] = count + checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) + } + } + + verifyEmailData(ctx, restoreItemCount, itemCount) +} + +func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { + for fldName, expected := range messageCount { + got := restoreMessageCount[fldName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("Restore item counts do not match: %s", fldName), + expected, + got) + } +} + +// getAllSubFolder will recursively check for all subfolders and get the corresponding +// email count. +func getAllMailSubFolders( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + testUser string, + r models.MailFolderable, + parentFolder, + dataFolder string, + messageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + ctx = clues.Add(ctx, "parent_folder_id", folderID) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + common.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + childFolderCount = ptr.Val(child.GetChildFolderCount()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) + // recursively check for subfolders + if childFolderCount > 0 { + parentFolder := fullFolderName + + getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) + } + } + } +} + +// checkAllSubFolder will recursively traverse inside the restore folder and +// verify that data matched in all subfolders +func checkAllSubFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + r models.MailFolderable, + testUser, + parentFolder, + dataFolder string, + restoreMessageCount map[string]int32, +) { + var ( + folderID = ptr.Val(r.GetId()) + count int32 = 99 + options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ + Top: &count, + }, + } + ) + + childFolder, err := client. + Users(). + ByUserId(testUser). + MailFolders(). + ByMailFolderId(folderID). + ChildFolders(). + Get(ctx, options) + if err != nil { + common.Fatal(ctx, "getting mail subfolders", err) + } + + for _, child := range childFolder.GetValue() { + var ( + childDisplayName = ptr.Val(child.GetDisplayName()) + //nolint:forbidigo + fullFolderName = stdpath.Join(parentFolder, childDisplayName) + ) + + if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { + childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) + restoreMessageCount[fullFolderName] = childTotalCount + } + + childFolderCount := ptr.Val(child.GetChildFolderCount()) + + if childFolderCount > 0 { + parentFolder := fullFolderName + checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) + } + } +} diff --git a/src/cmd/sanity_test/restore/onedrive.go b/src/cmd/sanity_test/restore/onedrive.go new file mode 100644 index 000000000..87ea424c3 --- /dev/null +++ b/src/cmd/sanity_test/restore/onedrive.go @@ -0,0 +1,369 @@ +package restore + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/alcionai/clues" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "golang.org/x/exp/slices" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +const ( + owner = "owner" +) + +func CheckOneDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Users(). + ByUserId(userID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.OneDriveService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + false) +} + +func checkDriveRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + service path.ServiceType, + folderName, + driveID, + driveName, + dataFolder string, + startTime time.Time, + skipPermissionTest bool, +) { + var ( + // map itemID -> item size + fileSizes = make(map[string]int64) + // map itemID -> permission id -> []permission roles + folderPermissions = make(map[string][]common.PermissionInfo) + restoreFile = make(map[string]int64) + restoredFolderPermissions = make(map[string][]common.PermissionInfo) + ) + + ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) + + restoreFolderID := PopulateDriveDetails( + ctx, + client, + driveID, + folderName, + dataFolder, + fileSizes, + folderPermissions, + startTime) + + getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) + + checkRestoredDriveItemPermissions( + ctx, + service, + skipPermissionTest, + folderPermissions, + restoredFolderPermissions) + + for fileName, expected := range fileSizes { + common.LogAndPrint(ctx, "checking for file: %s", fileName) + + got := restoreFile[fileName] + + common.Assert( + ctx, + func() bool { return expected == got }, + fmt.Sprintf("different file size: %s", fileName), + expected, + got) + } + + fmt.Println("Success") +} + +func PopulateDriveDetails( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, folderName, dataFolder string, + fileSizes map[string]int64, + folderPermissions map[string][]common.PermissionInfo, + startTime time.Time, +) string { + var restoreFolderID string + + response, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId("root"). + Children(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting drive by id", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + ) + + if itemName == folderName { + restoreFolderID = itemID + continue + } + + if itemName != dataFolder { + common.LogAndPrint(ctx, "test data for folder: %s", dataFolder) + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[itemName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + common.LogAndPrint(ctx, "skipped empty folder: %s", itemName) + continue + } + + folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) + } + + return restoreFolderID +} + +func checkRestoredDriveItemPermissions( + ctx context.Context, + service path.ServiceType, + skip bool, + folderPermissions map[string][]common.PermissionInfo, + restoredFolderPermissions map[string][]common.PermissionInfo, +) { + if skip { + return + } + + /** + TODO: replace this check with testElementsMatch + from internal/connecter/graph_connector_helper_test.go + **/ + + for folderName, permissions := range folderPermissions { + common.LogAndPrint(ctx, "checking for folder: %s", folderName) + + restoreFolderPerm := restoredFolderPermissions[folderName] + + if len(permissions) < 1 { + common.LogAndPrint(ctx, "no permissions found in: %s", folderName) + continue + } + + permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } + + if service == path.SharePointService { + permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } + } + + common.Assert( + ctx, + permCheck, + fmt.Sprintf("wrong number of restored permissions: %s", folderName), + permissions, + restoreFolderPerm) + + for _, perm := range permissions { + eqID := func(pi common.PermissionInfo) bool { return strings.EqualFold(pi.EntityID, perm.EntityID) } + i := slices.IndexFunc(restoreFolderPerm, eqID) + + common.Assert( + ctx, + func() bool { return i >= 0 }, + fmt.Sprintf("permission was restored in: %s", folderName), + perm.EntityID, + restoreFolderPerm) + + // permissions should be sorted, so a by-index comparison works + restored := restoreFolderPerm[i] + + common.Assert( + ctx, + func() bool { return slices.Equal(perm.Roles, restored.Roles) }, + fmt.Sprintf("different roles restored: %s", folderName), + perm.Roles, + restored.Roles) + } + } +} + +func getOneDriveChildFolder( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID, parentName string, + fileSizes map[string]int64, + folderPermission map[string][]common.PermissionInfo, + startTime time.Time, +) { + response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting child folder", err) + } + + for _, driveItem := range response.GetValue() { + var ( + itemID = ptr.Val(driveItem.GetId()) + itemName = ptr.Val(driveItem.GetName()) + fullName = parentName + "/" + itemName + ) + + folderTime, hasTime := common.MustGetTimeFromName(ctx, itemName) + if !common.IsWithinTimeBound(ctx, startTime, folderTime, hasTime) { + continue + } + + // if it's a file check the size + if driveItem.GetFile() != nil { + fileSizes[fullName] = ptr.Val(driveItem.GetSize()) + } + + if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { + continue + } + + // currently we don't restore blank folders. + // skip permission check for empty folders + if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { + common.LogAndPrint(ctx, "skipped empty folder: %s", fullName) + + continue + } + + folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) + } +} + +func getRestoredDrive( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, restoreFolderID string, + restoreFile map[string]int64, + restoreFolder map[string][]common.PermissionInfo, + startTime time.Time, +) { + restored, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(restoreFolderID). + Children(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting child folder", err) + } + + for _, item := range restored.GetValue() { + var ( + itemID = ptr.Val(item.GetId()) + itemName = ptr.Val(item.GetName()) + itemSize = ptr.Val(item.GetSize()) + ) + + if item.GetFile() != nil { + restoreFile[itemName] = itemSize + continue + } + + if item.GetFolder() == nil && item.GetPackage() == nil { + continue + } + + restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) + getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) + } +} + +// --------------------------------------------------------------------------- +// permission helpers +// --------------------------------------------------------------------------- + +func permissionIn( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + driveID, itemID string, +) []common.PermissionInfo { + pi := []common.PermissionInfo{} + + pcr, err := client. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(itemID). + Permissions(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting permission", err) + } + + for _, perm := range pcr.GetValue() { + if perm.GetGrantedToV2() == nil { + continue + } + + var ( + gv2 = perm.GetGrantedToV2() + permInfo = common.PermissionInfo{} + entityID string + ) + + // TODO: replace with filterUserPermissions in onedrive item.go + if gv2.GetUser() != nil { + entityID = ptr.Val(gv2.GetUser().GetId()) + } else if gv2.GetGroup() != nil { + entityID = ptr.Val(gv2.GetGroup().GetId()) + } + + roles := common.FilterSlice(perm.GetRoles(), owner) + for _, role := range roles { + permInfo.EntityID = entityID + permInfo.Roles = append(permInfo.Roles, role) + } + + if len(roles) > 0 { + slices.Sort(permInfo.Roles) + pi = append(pi, permInfo) + } + } + + return pi +} diff --git a/src/cmd/sanity_test/restore/sharepoint.go b/src/cmd/sanity_test/restore/sharepoint.go new file mode 100644 index 000000000..a5146d7a4 --- /dev/null +++ b/src/cmd/sanity_test/restore/sharepoint.go @@ -0,0 +1,39 @@ +package restore + +import ( + "context" + "time" + + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/path" +) + +func CheckSharePointRestoration( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + siteID, userID, folderName, dataFolder string, + startTime time.Time, +) { + drive, err := client. + Sites(). + BySiteId(siteID). + Drive(). + Get(ctx, nil) + if err != nil { + common.Fatal(ctx, "getting the drive:", err) + } + + checkDriveRestoration( + ctx, + client, + path.SharePointService, + folderName, + ptr.Val(drive.GetId()), + ptr.Val(drive.GetName()), + dataFolder, + startTime, + true) +} diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index ab89586b4..84bce47a0 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -2,45 +2,21 @@ package main import ( "context" - "errors" - "fmt" "os" - stdpath "path" "strings" "time" "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" - "golang.org/x/exp/slices" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/export" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester/tconfig" - "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/path" ) -// --------------------------------------------------------------------------- -// types, consts, etc -// --------------------------------------------------------------------------- - -type permissionInfo struct { - entityID string - roles []string -} - -const ( - owner = "owner" -) - -// --------------------------------------------------------------------------- -// main -// --------------------------------------------------------------------------- - func main() { ls := logger.Settings{ File: logger.GetLogFile(""), @@ -60,16 +36,16 @@ func main() { os.Getenv("AZURE_CLIENT_ID"), os.Getenv("AZURE_CLIENT_SECRET")) if err != nil { - fatal(ctx, "creating adapter", err) + common.Fatal(ctx, "creating adapter", err) } var ( client = msgraphsdk.NewGraphServiceClient(adapter) testUser = tconfig.GetM365UserID(ctx) testSite = tconfig.GetM365SiteID(ctx) - testService = os.Getenv("SANITY_RESTORE_SERVICE") - folder = strings.TrimSpace(os.Getenv("SANITY_RESTORE_FOLDER")) - startTime, _ = mustGetTimeFromName(ctx, folder) + testKind = os.Getenv("SANITY_TEST_KIND") // restore or export (cli arg?) + testService = os.Getenv("SANITY_TEST_SERVICE") + folder = strings.TrimSpace(os.Getenv("SANITY_TEST_FOLDER")) dataFolder = os.Getenv("TEST_DATA") baseBackupFolder = os.Getenv("BASE_BACKUP") ) @@ -78,664 +54,35 @@ func main() { ctx, "resource_owner", testUser, "service", testService, - "sanity_restore_folder", folder, - "start_time", startTime.Format(time.RFC3339Nano)) + "sanity_restore_folder", folder) logger.Ctx(ctx).Info("starting sanity test check") - switch testService { - case "exchange": - checkEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) - case "onedrive": - checkOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) - case "sharepoint": - checkSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + switch testKind { + case "restore": + startTime, _ := common.MustGetTimeFromName(ctx, folder) + clues.Add(ctx, "sanity_restore_start_time", startTime.Format(time.RFC3339)) + + switch testService { + case "exchange": + restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime) + case "onedrive": + restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime) + case "sharepoint": + restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime) + default: + common.Fatal(ctx, "unknown service for restore sanity tests", nil) + } + case "export": + switch testService { + case "onedrive": + export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder) + case "sharepoint": + export.CheckSharePointExport(ctx, client, testSite, folder, dataFolder) + default: + common.Fatal(ctx, "unknown service for export sanity tests", nil) + } default: - fatal(ctx, "no service specified", nil) + common.Fatal(ctx, "unknown test kind (expected restore or export)", nil) } } - -// --------------------------------------------------------------------------- -// exchange -// --------------------------------------------------------------------------- - -// checkEmailRestoration verifies that the emails count in restored folder is equivalent to -// emails in actual m365 account -func checkEmailRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser, folderName, dataFolder, baseBackupFolder string, - startTime time.Time, -) { - var ( - restoreFolder models.MailFolderable - itemCount = make(map[string]int32) - restoreItemCount = make(map[string]int32) - builder = client.Users().ByUserId(testUser).MailFolders() - ) - - for { - result, err := builder.Get(ctx, nil) - if err != nil { - fatal(ctx, "getting mail folders", err) - } - - values := result.GetValue() - - for _, v := range values { - itemName := ptr.Val(v.GetDisplayName()) - - if itemName == folderName { - restoreFolder = v - continue - } - - if itemName == dataFolder || itemName == baseBackupFolder { - // otherwise, recursively aggregate all child folders. - getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount) - - itemCount[itemName] = ptr.Val(v.GetTotalItemCount()) - } - } - - link, ok := ptr.ValOK(result.GetOdataNextLink()) - if !ok { - break - } - - builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter()) - } - - folderID := ptr.Val(restoreFolder.GetId()) - folderName = ptr.Val(restoreFolder.GetDisplayName()) - ctx = clues.Add( - ctx, - "restore_folder_id", folderID, - "restore_folder_name", folderName) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting restore folder child folders", err) - } - - for _, fld := range childFolder.GetValue() { - restoreDisplayName := ptr.Val(fld.GetDisplayName()) - - // check if folder is the data folder we loaded or the base backup to verify - // the incremental backup worked fine - if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) { - count, _ := ptr.ValOK(fld.GetTotalItemCount()) - - restoreItemCount[restoreDisplayName] = count - checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount) - } - } - - verifyEmailData(ctx, restoreItemCount, itemCount) -} - -func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) { - for fldName, expected := range messageCount { - got := restoreMessageCount[fldName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("Restore item counts do not match: %s", fldName), - expected, - got) - } -} - -// getAllSubFolder will recursively check for all subfolders and get the corresponding -// email count. -func getAllMailSubFolders( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - testUser string, - r models.MailFolderable, - parentFolder, - dataFolder string, - messageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - ctx = clues.Add(ctx, "parent_folder_id", folderID) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - childFolderCount = ptr.Val(child.GetChildFolderCount()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount()) - // recursively check for subfolders - if childFolderCount > 0 { - parentFolder := fullFolderName - - getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount) - } - } - } -} - -// checkAllSubFolder will recursively traverse inside the restore folder and -// verify that data matched in all subfolders -func checkAllSubFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - r models.MailFolderable, - testUser, - parentFolder, - dataFolder string, - restoreMessageCount map[string]int32, -) { - var ( - folderID = ptr.Val(r.GetId()) - count int32 = 99 - options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{ - Top: &count, - }, - } - ) - - childFolder, err := client. - Users(). - ByUserId(testUser). - MailFolders(). - ByMailFolderId(folderID). - ChildFolders(). - Get(ctx, options) - if err != nil { - fatal(ctx, "getting mail subfolders", err) - } - - for _, child := range childFolder.GetValue() { - var ( - childDisplayName = ptr.Val(child.GetDisplayName()) - //nolint:forbidigo - fullFolderName = stdpath.Join(parentFolder, childDisplayName) - ) - - if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) { - childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount()) - restoreMessageCount[fullFolderName] = childTotalCount - } - - childFolderCount := ptr.Val(child.GetChildFolderCount()) - - if childFolderCount > 0 { - parentFolder := fullFolderName - checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount) - } - } -} - -// --------------------------------------------------------------------------- -// oneDrive -// --------------------------------------------------------------------------- - -func checkOneDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Users(). - ByUserId(userID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.OneDriveService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - false) -} - -// --------------------------------------------------------------------------- -// sharePoint -// --------------------------------------------------------------------------- - -func checkSharePointRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - siteID, userID, folderName, dataFolder string, - startTime time.Time, -) { - drive, err := client. - Sites(). - BySiteId(siteID). - Drive(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting the drive:", err) - } - - checkDriveRestoration( - ctx, - client, - path.SharePointService, - folderName, - ptr.Val(drive.GetId()), - ptr.Val(drive.GetName()), - dataFolder, - startTime, - true) -} - -// --------------------------------------------------------------------------- -// shared drive tests -// --------------------------------------------------------------------------- - -func checkDriveRestoration( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - service path.ServiceType, - folderName, - driveID, - driveName, - dataFolder string, - startTime time.Time, - skipPermissionTest bool, -) { - var ( - // map itemID -> item size - fileSizes = make(map[string]int64) - // map itemID -> permission id -> []permission roles - folderPermissions = make(map[string][]permissionInfo) - restoreFile = make(map[string]int64) - restoredFolderPermissions = make(map[string][]permissionInfo) - ) - - var restoreFolderID string - - ctx = clues.Add(ctx, "drive_id", driveID, "drive_name", driveName) - - response, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId("root"). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting drive by id", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - ) - - if itemName == folderName { - restoreFolderID = itemID - continue - } - - if itemName != dataFolder { - logAndPrint(ctx, "test data for folder: %s", dataFolder) - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[itemName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", itemName) - continue - } - - folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime) - } - - getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime) - - checkRestoredDriveItemPermissions( - ctx, - service, - skipPermissionTest, - folderPermissions, - restoredFolderPermissions) - - for fileName, expected := range fileSizes { - logAndPrint(ctx, "checking for file: %s", fileName) - - got := restoreFile[fileName] - - assert( - ctx, - func() bool { return expected == got }, - fmt.Sprintf("different file size: %s", fileName), - expected, - got) - } - - fmt.Println("Success") -} - -func checkRestoredDriveItemPermissions( - ctx context.Context, - service path.ServiceType, - skip bool, - folderPermissions map[string][]permissionInfo, - restoredFolderPermissions map[string][]permissionInfo, -) { - if skip { - return - } - - /** - TODO: replace this check with testElementsMatch - from internal/connecter/graph_connector_helper_test.go - **/ - - for folderName, permissions := range folderPermissions { - logAndPrint(ctx, "checking for folder: %s", folderName) - - restoreFolderPerm := restoredFolderPermissions[folderName] - - if len(permissions) < 1 { - logAndPrint(ctx, "no permissions found in: %s", folderName) - continue - } - - permCheck := func() bool { return len(permissions) == len(restoreFolderPerm) } - - if service == path.SharePointService { - permCheck = func() bool { return len(permissions) <= len(restoreFolderPerm) } - } - - assert( - ctx, - permCheck, - fmt.Sprintf("wrong number of restored permissions: %s", folderName), - permissions, - restoreFolderPerm) - - for _, perm := range permissions { - eqID := func(pi permissionInfo) bool { return strings.EqualFold(pi.entityID, perm.entityID) } - i := slices.IndexFunc(restoreFolderPerm, eqID) - - assert( - ctx, - func() bool { return i >= 0 }, - fmt.Sprintf("permission was restored in: %s", folderName), - perm.entityID, - restoreFolderPerm) - - // permissions should be sorted, so a by-index comparison works - restored := restoreFolderPerm[i] - - assert( - ctx, - func() bool { return slices.Equal(perm.roles, restored.roles) }, - fmt.Sprintf("different roles restored: %s", folderName), - perm.roles, - restored.roles) - } - } -} - -func getOneDriveChildFolder( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID, parentName string, - fileSizes map[string]int64, - folderPermission map[string][]permissionInfo, - startTime time.Time, -) { - response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, driveItem := range response.GetValue() { - var ( - itemID = ptr.Val(driveItem.GetId()) - itemName = ptr.Val(driveItem.GetName()) - fullName = parentName + "/" + itemName - ) - - folderTime, hasTime := mustGetTimeFromName(ctx, itemName) - if !isWithinTimeBound(ctx, startTime, folderTime, hasTime) { - continue - } - - // if it's a file check the size - if driveItem.GetFile() != nil { - fileSizes[fullName] = ptr.Val(driveItem.GetSize()) - } - - if driveItem.GetFolder() == nil && driveItem.GetPackage() == nil { - continue - } - - // currently we don't restore blank folders. - // skip permission check for empty folders - if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 { - logAndPrint(ctx, "skipped empty folder: %s", fullName) - - continue - } - - folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime) - } -} - -func getRestoredDrive( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, restoreFolderID string, - restoreFile map[string]int64, - restoreFolder map[string][]permissionInfo, - startTime time.Time, -) { - restored, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(restoreFolderID). - Children(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting child folder", err) - } - - for _, item := range restored.GetValue() { - var ( - itemID = ptr.Val(item.GetId()) - itemName = ptr.Val(item.GetName()) - itemSize = ptr.Val(item.GetSize()) - ) - - if item.GetFile() != nil { - restoreFile[itemName] = itemSize - continue - } - - if item.GetFolder() == nil && item.GetPackage() == nil { - continue - } - - restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID) - getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime) - } -} - -// --------------------------------------------------------------------------- -// permission helpers -// --------------------------------------------------------------------------- - -func permissionIn( - ctx context.Context, - client *msgraphsdk.GraphServiceClient, - driveID, itemID string, -) []permissionInfo { - pi := []permissionInfo{} - - pcr, err := client. - Drives(). - ByDriveId(driveID). - Items(). - ByDriveItemId(itemID). - Permissions(). - Get(ctx, nil) - if err != nil { - fatal(ctx, "getting permission", err) - } - - for _, perm := range pcr.GetValue() { - if perm.GetGrantedToV2() == nil { - continue - } - - var ( - gv2 = perm.GetGrantedToV2() - permInfo = permissionInfo{} - entityID string - ) - - // TODO: replace with filterUserPermissions in onedrive item.go - if gv2.GetUser() != nil { - entityID = ptr.Val(gv2.GetUser().GetId()) - } else if gv2.GetGroup() != nil { - entityID = ptr.Val(gv2.GetGroup().GetId()) - } - - roles := filterSlice(perm.GetRoles(), owner) - for _, role := range roles { - permInfo.entityID = entityID - permInfo.roles = append(permInfo.roles, role) - } - - if len(roles) > 0 { - slices.Sort(permInfo.roles) - pi = append(pi, permInfo) - } - } - - return pi -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -func fatal(ctx context.Context, msg string, err error) { - logger.CtxErr(ctx, err).Error("test failure: " + msg) - fmt.Println(msg+": ", err) - os.Exit(1) -} - -func mustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { - t, err := dttm.ExtractTime(name) - if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { - fatal(ctx, "extracting time from name: "+name, err) - } - - return t, !errors.Is(err, dttm.ErrNoTimeString) -} - -func isWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { - if hasTime { - if bound.Before(check) { - logger.Ctx(ctx). - With("boundary_time", bound, "check_time", check). - Info("skipping restore folder: not older than time bound") - - return false - } - } - - return true -} - -func filterSlice(sl []string, remove string) []string { - r := []string{} - - for _, s := range sl { - if !strings.EqualFold(s, remove) { - r = append(r, s) - } - } - - return r -} - -func assert( - ctx context.Context, - passes func() bool, - header string, - expect, current any, -) { - if passes() { - return - } - - header = "Error: " + header - expected := fmt.Sprintf("* Expected: %+v", expect) - got := fmt.Sprintf("* Current: %+v", current) - - logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) - - fmt.Println(header) - fmt.Println(expected) - fmt.Println(got) - - os.Exit(1) -} - -func logAndPrint(ctx context.Context, tmpl string, vs ...any) { - logger.Ctx(ctx).Infof(tmpl, vs...) - fmt.Printf(tmpl+"\n", vs...) -} diff --git a/src/cmd/sanity_test/utils/utils.go b/src/cmd/sanity_test/utils/utils.go new file mode 100644 index 000000000..d0a877d61 --- /dev/null +++ b/src/cmd/sanity_test/utils/utils.go @@ -0,0 +1,82 @@ +package utils + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/logger" +) + +func Assert( + ctx context.Context, + passes func() bool, + header string, + expect, current any, +) { + if passes() { + return + } + + header = "Error: " + header + expected := fmt.Sprintf("* Expected: %+v", expect) + got := fmt.Sprintf("* Current: %+v", current) + + logger.Ctx(ctx).Info(strings.Join([]string{header, expected, got}, " ")) + + fmt.Println(header) + fmt.Println(expected) + fmt.Println(got) + + os.Exit(1) +} + +func Fatal(ctx context.Context, msg string, err error) { + logger.CtxErr(ctx, err).Error("test failure: " + msg) + fmt.Println(msg+": ", err) + os.Exit(1) +} + +func MustGetTimeFromName(ctx context.Context, name string) (time.Time, bool) { + t, err := dttm.ExtractTime(name) + if err != nil && !errors.Is(err, dttm.ErrNoTimeString) { + Fatal(ctx, "extracting time from name: "+name, err) + } + + return t, !errors.Is(err, dttm.ErrNoTimeString) +} + +func IsWithinTimeBound(ctx context.Context, bound, check time.Time, hasTime bool) bool { + if hasTime { + if bound.Before(check) { + logger.Ctx(ctx). + With("boundary_time", bound, "check_time", check). + Info("skipping restore folder: not older than time bound") + + return false + } + } + + return true +} + +func FilterSlice(sl []string, remove string) []string { + r := []string{} + + for _, s := range sl { + if !strings.EqualFold(s, remove) { + r = append(r, s) + } + } + + return r +} + +func LogAndPrint(ctx context.Context, tmpl string, vs ...any) { + logger.Ctx(ctx).Infof(tmpl, vs...) + fmt.Printf(tmpl+"\n", vs...) +} diff --git a/src/go.mod b/src/go.mod index 99581228f..9b03cf95b 100644 --- a/src/go.mod +++ b/src/go.mod @@ -5,21 +5,21 @@ go 1.20 replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 require ( - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go v1.44.302 + github.com/aws/aws-sdk-go v1.44.311 github.com/aws/aws-xray-sdk-go v1.8.1 github.com/cenkalti/backoff/v4 v4.2.1 github.com/google/uuid v1.3.0 github.com/h2non/gock v1.2.0 github.com/kopia/kopia v0.12.2-0.20230327171220-747baeebdab1 - github.com/microsoft/kiota-abstractions-go v1.0.0 + github.com/microsoft/kiota-abstractions-go v1.1.0 github.com/microsoft/kiota-authentication-azure-go v1.0.0 - github.com/microsoft/kiota-http-go v1.0.0 + github.com/microsoft/kiota-http-go v1.0.1 github.com/microsoft/kiota-serialization-form-go v1.0.0 - github.com/microsoft/kiota-serialization-json-go v1.0.2 - github.com/microsoftgraph/msgraph-sdk-go v1.4.0 + github.com/microsoft/kiota-serialization-json-go v1.0.4 + github.com/microsoftgraph/msgraph-sdk-go v1.12.0 github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 github.com/pkg/errors v0.9.1 github.com/puzpuzpuz/xsync/v2 v2.4.1 @@ -42,7 +42,6 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect @@ -62,9 +61,9 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/src/go.sum b/src/go.sum index 34860c7c1..74f71dadf 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,14 +36,14 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= @@ -53,8 +53,8 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a h1:mtJyeK/FhArTn06M5Lfgxk/GWnu8yqCGNN1BY16vjaA= -github.com/alcionai/clues v0.0.0-20230630194723-e24d7940e07a/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4 h1:husF7eAYw2HEzgjfAmNy+ZLzyztJV2SyoUngSUo829Y= +github.com/alcionai/clues v0.0.0-20230728164842-7dc4795a43e4/go.mod h1:MLEWSZ0cjEMg6hiGCRvE7AtrOhs7deBcm7ZrJBpfGRM= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377 h1:w50/aVU+zRP5lvE86TSSCCYrrEyuXOlJA06R5RdTS8Y= github.com/alcionai/kopia v0.12.2-0.20230713235606-4c85869e9377/go.mod h1:WH725ws0BYpZpTkVh4uqFHHPiiJuirl1Cm73jv5RYyA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -66,12 +66,11 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.44.302 h1:ST3ko6GrJKn3Xi+nAvxjG3uk/V1pW8KC52WLeIxqqNk= -github.com/aws/aws-sdk-go v1.44.302/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.311 h1:60i8hyVMOXqabKJQPCq4qKRBQ6hRafI/WOcDxGM+J7Q= +github.com/aws/aws-sdk-go v1.44.311/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.1 h1:O4pXV+hnCskaamGsZnFpzHyAmgPGusBMN6i7nnsy0Fo= github.com/aws/aws-xray-sdk-go v1.8.1/go.mod h1:wMmVYzej3sykAttNBkXQHK/+clAPWTOrPiajEk7Cp3A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -103,7 +102,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= @@ -124,7 +122,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -227,7 +224,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -235,7 +231,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -276,20 +271,20 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/microsoft/kiota-abstractions-go v1.0.0 h1:teQS3yOmcTyps+O48AD17LI8TR1B3wCEwGFcwC6K75c= -github.com/microsoft/kiota-abstractions-go v1.0.0/go.mod h1:2yaRQnx2KU7UaenYSApiTT4pf7fFkPV0B71Rm2uYynQ= +github.com/microsoft/kiota-abstractions-go v1.1.0 h1:X1aKlsYCRs/0RSChr/fbq4j/+kxRzbSY5GeWhtHQNYI= +github.com/microsoft/kiota-abstractions-go v1.1.0/go.mod h1:RkxyZ5x87Njik7iVeQY9M2wtrrL1MJZcXiI/BxD/82g= github.com/microsoft/kiota-authentication-azure-go v1.0.0 h1:29FNZZ/4nnCOwFcGWlB/sxPvWz487HA2bXH8jR5k2Rk= github.com/microsoft/kiota-authentication-azure-go v1.0.0/go.mod h1:rnx3PRlkGdXDcA/0lZQTbBwyYGmc+3POt7HpE/e4jGw= -github.com/microsoft/kiota-http-go v1.0.0 h1:F1hd6gMlLeEgH2CkRB7z13ow7LxMKMWEmms/t0VfS+k= -github.com/microsoft/kiota-http-go v1.0.0/go.mod h1:eujxJliqodotsYepIc6ihhK+vXMMt5Q8YiSNL7+7M7U= +github.com/microsoft/kiota-http-go v1.0.1 h1:818u3aiLpxj35hZgfUSqphQ18IUTK3gVdTE4cQ5vjLw= +github.com/microsoft/kiota-http-go v1.0.1/go.mod h1:H0cg+ly+5ZSR8z4swj5ea9O/GB5ll2YuYeQ0/pJs7AY= github.com/microsoft/kiota-serialization-form-go v1.0.0 h1:UNdrkMnLFqUCccQZerKjblsyVgifS11b3WCx+eFEsAI= github.com/microsoft/kiota-serialization-form-go v1.0.0/go.mod h1:h4mQOO6KVTNciMF6azi1J9QB19ujSw3ULKcSNyXXOMA= -github.com/microsoft/kiota-serialization-json-go v1.0.2 h1:RXan8v7yWBD88XxVZ2W38BBcqu2UqWtgS54nCbOS5ow= -github.com/microsoft/kiota-serialization-json-go v1.0.2/go.mod h1:AUItT9exyxmjZQE8IeFD9ygP77q9GKVb+AQE2V5Ikho= +github.com/microsoft/kiota-serialization-json-go v1.0.4 h1:5TaISWwd2Me8clrK7SqNATo0tv9seOq59y4I5953egQ= +github.com/microsoft/kiota-serialization-json-go v1.0.4/go.mod h1:rM4+FsAY+9AEpBsBzkFFis+b/LZLlNKKewuLwK9Q6Mg= github.com/microsoft/kiota-serialization-text-go v1.0.0 h1:XOaRhAXy+g8ZVpcq7x7a0jlETWnWrEum0RhmbYrTFnA= github.com/microsoft/kiota-serialization-text-go v1.0.0/go.mod h1:sM1/C6ecnQ7IquQOGUrUldaO5wj+9+v7G2W3sQ3fy6M= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0 h1:ibNwMDEZ6HikA9BVXu+TljCzCiE+yFsD6wLpJbTc1tc= -github.com/microsoftgraph/msgraph-sdk-go v1.4.0/go.mod h1:JIDL1xENx92B60NjO2ACyqGeKvtYkdl9rirgajIgryw= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0 h1:/jZJ1KCtVlvxStKq31VsEPOQQ5Iy26R1pgvc+RYt7XI= +github.com/microsoftgraph/msgraph-sdk-go v1.12.0/go.mod h1:ccLv84FJFtwdSzYWM/HlTes5FLzkzzBsYh9kg93/WS8= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0 h1:7NWTfyXvOjoizW7PmxNp3+8wCKPgpODs/D1cUZ3fkAY= github.com/microsoftgraph/msgraph-sdk-go-core v1.0.0/go.mod h1:tQb4q3YMIj2dWhhXhQSJ4ELpol931ANKzHSYK5kX1qE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -307,9 +302,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= @@ -446,7 +439,6 @@ go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLk go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -798,7 +790,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/internal/archive/zip.go b/src/internal/archive/zip.go new file mode 100644 index 000000000..f3e02ad66 --- /dev/null +++ b/src/internal/archive/zip.go @@ -0,0 +1,99 @@ +package archive + +import ( + "archive/zip" + "context" + "io" + "path" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/pkg/export" +) + +const ( + // ZipCopyBufferSize is the size of the copy buffer for zip + // write operations + // TODO(meain): tweak this value + ZipCopyBufferSize = 5 * 1024 * 1024 +) + +type zipCollection struct { + reader io.ReadCloser +} + +func (z zipCollection) BasePath() string { + return "" +} + +func (z zipCollection) Items(ctx context.Context) <-chan export.Item { + rc := make(chan export.Item, 1) + defer close(rc) + + rc <- export.Item{ + Data: export.ItemData{ + Name: "Corso_Export_" + dttm.FormatNow(dttm.HumanReadable) + ".zip", + Body: z.reader, + }, + } + + return rc +} + +// ZipExportCollection takes a list of export collections and zips +// them into a single collection. +func ZipExportCollection( + ctx context.Context, + expCollections []export.Collection, +) (export.Collection, error) { + if len(expCollections) == 0 { + return nil, clues.New("no export collections provided") + } + + reader, writer := io.Pipe() + wr := zip.NewWriter(writer) + + go func() { + defer writer.Close() + defer wr.Close() + + buf := make([]byte, ZipCopyBufferSize) + + for _, ec := range expCollections { + folder := ec.BasePath() + items := ec.Items(ctx) + + for item := range items { + err := item.Error + if err != nil { + writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID)) + return + } + + name := item.Data.Name + + // We assume folder and name to not contain any path separators. + // Also, this should always use `/` as this is + // created within a zip file and not written to disk. + // TODO(meain): Exchange paths might contain a path + // separator and will have to have special handling. + + //nolint:forbidigo + f, err := wr.Create(path.Join(folder, name)) + if err != nil { + writer.CloseWithError(clues.Wrap(err, "creating zip entry").With("name", name).With("id", item.ID)) + return + } + + _, err = io.CopyBuffer(f, item.Data.Body, buf) + if err != nil { + writer.CloseWithError(clues.Wrap(err, "writing zip entry").With("name", name).With("id", item.ID)) + return + } + } + } + }() + + return zipCollection{reader}, nil +} diff --git a/src/internal/common/idname/idname.go b/src/internal/common/idname/idname.go index d56fab025..e2a48fca3 100644 --- a/src/internal/common/idname/idname.go +++ b/src/internal/common/idname/idname.go @@ -28,6 +28,10 @@ type is struct { name string } +func NewProvider(id, name string) *is { + return &is{id, name} +} + func (is is) ID() string { return is.id } func (is is) Name() string { return is.name } @@ -40,6 +44,11 @@ type Cacher interface { ProviderForName(id string) Provider } +type CacheBuilder interface { + Add(id, name string) + Cacher +} + var _ Cacher = &cache{} type cache struct { @@ -47,17 +56,29 @@ type cache struct { nameToID map[string]string } -func NewCache(idToName map[string]string) cache { - nti := make(map[string]string, len(idToName)) - - for id, name := range idToName { - nti[name] = id +func NewCache(idToName map[string]string) *cache { + c := cache{ + idToName: map[string]string{}, + nameToID: map[string]string{}, } - return cache{ - idToName: idToName, - nameToID: nti, + if len(idToName) > 0 { + nti := make(map[string]string, len(idToName)) + + for id, name := range idToName { + nti[name] = id + } + + c.idToName = idToName + c.nameToID = nti } + + return &c +} + +func (c *cache) Add(id, name string) { + c.idToName[strings.ToLower(id)] = name + c.nameToID[strings.ToLower(name)] = id } // IDOf returns the id associated with the given name. diff --git a/src/internal/common/idname/idname_test.go b/src/internal/common/idname/idname_test.go new file mode 100644 index 000000000..229177d61 --- /dev/null +++ b/src/internal/common/idname/idname_test.go @@ -0,0 +1,60 @@ +package idname + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" +) + +type IDNameUnitSuite struct { + tester.Suite +} + +func TestIDNameUnitSuite(t *testing.T) { + suite.Run(t, &IDNameUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *IDNameUnitSuite) TestAdd() { + table := []struct { + name string + inID string + inName string + searchID string + searchName string + }{ + { + name: "basic", + inID: "foo", + inName: "bar", + searchID: "foo", + searchName: "bar", + }, + { + name: "change casing", + inID: "FNORDS", + inName: "SMARF", + searchID: "fnords", + searchName: "smarf", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cache := NewCache(nil) + + cache.Add(test.inID, test.inName) + + id, found := cache.IDOf(test.searchName) + assert.True(t, found) + assert.Equal(t, test.inID, id) + + name, found := cache.NameOf(test.searchID) + assert.True(t, found) + assert.Equal(t, test.inName, name) + }) + } +} diff --git a/src/internal/common/pii/url.go b/src/internal/common/pii/url.go index 339bee19f..de6e5ed8a 100644 --- a/src/internal/common/pii/url.go +++ b/src/internal/common/pii/url.go @@ -78,6 +78,10 @@ func (u SafeURL) Format(fs fmt.State, _ rune) { fmt.Fprint(fs, u.Conceal()) } +func (u SafeURL) PlainString() string { + return u.URL +} + // String complies with Stringer to ensure the Conceal() version // of the url is printed anytime it gets transformed to a string. func (u SafeURL) String() string { diff --git a/src/internal/data/data_collection.go b/src/internal/data/data_collection.go index b85e1e977..cec096783 100644 --- a/src/internal/data/data_collection.go +++ b/src/internal/data/data_collection.go @@ -91,6 +91,11 @@ func (c NoFetchRestoreCollection) FetchItemByName(context.Context, string) (Stre return nil, ErrNotFound } +type FetchRestoreCollection struct { + Collection + FetchItemByNamer +} + // Stream represents a single item within a Collection // that can be consumed as a stream (it embeds io.Reader) type Stream interface { diff --git a/src/internal/data/mock/collection.go b/src/internal/data/mock/collection.go index 63f2b2dd8..55f291a7f 100644 --- a/src/internal/data/mock/collection.go +++ b/src/internal/data/mock/collection.go @@ -1,12 +1,24 @@ package mock import ( + "context" "io" "time" + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" ) +// --------------------------------------------------------------------------- +// stream +// --------------------------------------------------------------------------- + +var _ data.Stream = &Stream{} + type Stream struct { ID string Reader io.ReadCloser @@ -52,3 +64,39 @@ type errReader struct { func (er errReader) Read([]byte) (int, error) { return 0, er.readErr } + +// --------------------------------------------------------------------------- +// collection +// --------------------------------------------------------------------------- + +var ( + _ data.Collection = &Collection{} + _ data.BackupCollection = &Collection{} + _ data.RestoreCollection = &Collection{} +) + +type Collection struct{} + +func (c Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream { + return nil +} + +func (c Collection) FullPath() path.Path { + return nil +} + +func (c Collection) PreviousPath() path.Path { + return nil +} + +func (c Collection) State() data.CollectionState { + return data.NewState +} + +func (c Collection) DoNotMergeItems() bool { + return true +} + +func (c Collection) FetchItemByName(ctx context.Context, name string) (data.Stream, error) { + return &Stream{}, clues.New("not implemented") +} diff --git a/src/internal/events/events.go b/src/internal/events/events.go index baa2c2117..99c1651ac 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -35,6 +35,8 @@ const ( BackupEnd = "Backup End" RestoreStart = "Restore Start" RestoreEnd = "Restore End" + ExportStart = "Export Start" + ExportEnd = "Export End" MaintenanceStart = "Maintenance Start" MaintenanceEnd = "Maintenance End" @@ -49,6 +51,7 @@ const ( ItemsWritten = "items_written" Resources = "resources" RestoreID = "restore_id" + ExportID = "export_id" Service = "service" StartTime = "start_time" Status = "status" @@ -82,8 +85,8 @@ var ( RudderStackDataPlaneURL string ) -func NewBus(ctx context.Context, s storage.Storage, tenID string, opts control.Options) (Bus, error) { - if opts.DisableMetrics { +func NewBus(ctx context.Context, s storage.Storage, tenID string, co control.Options) (Bus, error) { + if co.DisableMetrics { return Bus{}, nil } diff --git a/src/internal/events/events_test.go b/src/internal/events/events_test.go index 7cc47f607..f8b1d6beb 100644 --- a/src/internal/events/events_test.go +++ b/src/internal/events/events_test.go @@ -52,7 +52,7 @@ func (suite *EventsIntegrationSuite) TestNewBus() { ) require.NoError(t, err, clues.ToCore(err)) - b, err := events.NewBus(ctx, s, a.ID(), control.Defaults()) + b, err := events.NewBus(ctx, s, a.ID(), control.DefaultOptions()) require.NotEmpty(t, b) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/kopia/backup_bases.go b/src/internal/kopia/backup_bases.go index 0505fc829..c0b8ecfaa 100644 --- a/src/internal/kopia/backup_bases.go +++ b/src/internal/kopia/backup_bases.go @@ -24,7 +24,7 @@ type BackupBases interface { MergeBackupBases( ctx context.Context, other BackupBases, - reasonToKey func(Reason) string, + reasonToKey func(Reasoner) string, ) BackupBases } @@ -109,10 +109,10 @@ func (bb *backupBases) ClearAssistBases() { // some migration that disrupts lookup), and that the BackupBases used to call // this function contains the current version. // -// reasonToKey should be a function that, given a Reason, will produce some -// string that represents Reason in the context of the merge operation. For -// example, to merge BackupBases across a ResourceOwner migration, the Reason's -// service and category can be used as the key. +// reasonToKey should be a function that, given a Reasoner, will produce some +// string that represents Reasoner in the context of the merge operation. For +// example, to merge BackupBases across a ProtectedResource migration, the +// Reasoner's service and category can be used as the key. // // Selection priority, for each reason key generated by reasonsToKey, follows // these rules: @@ -125,7 +125,7 @@ func (bb *backupBases) ClearAssistBases() { func (bb *backupBases) MergeBackupBases( ctx context.Context, other BackupBases, - reasonToKey func(reason Reason) string, + reasonToKey func(reason Reasoner) string, ) BackupBases { if other == nil || (len(other.MergeBases()) == 0 && len(other.AssistBases()) == 0) { return bb @@ -159,7 +159,7 @@ func (bb *backupBases) MergeBackupBases( // Calculate the set of mergeBases to pull from other into this one. for _, m := range other.MergeBases() { - useReasons := []Reason{} + useReasons := []Reasoner{} for _, r := range m.Reasons { k := reasonToKey(r) @@ -210,7 +210,7 @@ func (bb *backupBases) MergeBackupBases( // Add assistBases from other to this one as needed. for _, m := range other.AssistBases() { - useReasons := []Reason{} + useReasons := []Reasoner{} // Assume that all complete manifests in assist overlap with MergeBases. if len(m.IncompleteReason) == 0 { @@ -267,8 +267,8 @@ func findNonUniqueManifests( } for _, reason := range man.Reasons { - reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String() - reasons[reasonKey] = append(reasons[reasonKey], man) + mapKey := reasonKey(reason) + reasons[mapKey] = append(reasons[mapKey], man) } } diff --git a/src/internal/kopia/backup_bases_test.go b/src/internal/kopia/backup_bases_test.go index f902d4e37..04afb5408 100644 --- a/src/internal/kopia/backup_bases_test.go +++ b/src/internal/kopia/backup_bases_test.go @@ -16,7 +16,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -func makeManifest(id, incmpl, bID string, reasons ...Reason) ManifestEntry { +func makeManifest(id, incmpl, bID string, reasons ...Reasoner) ManifestEntry { bIDKey, _ := makeTagKV(TagBackupID) return ManifestEntry{ @@ -223,14 +223,10 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { ir = "checkpoint" } - reasons := make([]Reason, 0, len(i.cat)) + reasons := make([]Reasoner, 0, len(i.cat)) for _, c := range i.cat { - reasons = append(reasons, Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: c, - }) + reasons = append(reasons, NewReason("", ro, path.ExchangeService, c)) } m := makeManifest(baseID, ir, "b"+baseID, reasons...) @@ -457,8 +453,8 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() { got := bb.MergeBackupBases( ctx, other, - func(reason Reason) string { - return reason.Service.String() + reason.Category.String() + func(r Reasoner) string { + return r.Service().String() + r.Category().String() }) AssertBackupBasesEqual(t, expect, got) }) @@ -469,13 +465,8 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { ro := "resource_owner" makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry { - reason := Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: pct, - } - - return makeManifest(id, incmpl, bID, reason) + r := NewReason("", ro, path.ExchangeService, pct) + return makeManifest(id, incmpl, bID, r) } // Make a function so tests can modify things without messing with each other. @@ -606,11 +597,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }) + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) res.assistBases = res.mergeBases return res @@ -619,11 +606,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() { res := validMail1() res.mergeBases[0].Reasons = append( res.mergeBases[0].Reasons, - Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }) + NewReason("", ro, path.ExchangeService, path.ContactsCategory)) res.assistBases = res.mergeBases return res diff --git a/src/internal/kopia/base_finder.go b/src/internal/kopia/base_finder.go index 9ac651512..00561c833 100644 --- a/src/internal/kopia/base_finder.go +++ b/src/internal/kopia/base_finder.go @@ -29,39 +29,94 @@ const ( userTagPrefix = "tag:" ) -type Reason struct { - ResourceOwner string - Service path.ServiceType - Category path.CategoryType +// TODO(ashmrtn): Move this into some inject package. Here to avoid import +// cycles. +type Reasoner interface { + Tenant() string + ProtectedResource() string + Service() path.ServiceType + Category() path.CategoryType + // SubtreePath returns the path prefix for data in existing backups that have + // parameters (tenant, protected resourced, etc) that match this Reasoner. + SubtreePath() (path.Path, error) } -func (r Reason) TagKeys() []string { - return []string{ - r.ResourceOwner, - serviceCatString(r.Service, r.Category), +func NewReason( + tenant, resource string, + service path.ServiceType, + category path.CategoryType, +) Reasoner { + return reason{ + tenant: tenant, + resource: resource, + service: service, + category: category, } } -// Key is the concatenation of the ResourceOwner, Service, and Category. -func (r Reason) Key() string { - return r.ResourceOwner + r.Service.String() + r.Category.String() +type reason struct { + // tenant appears here so that when this is moved to an inject package nothing + // needs changed. However, kopia itself is blind to the fields in the reason + // struct and relies on helper functions to get the information it needs. + tenant string + resource string + service path.ServiceType + category path.CategoryType +} + +func (r reason) Tenant() string { + return r.tenant +} + +func (r reason) ProtectedResource() string { + return r.resource +} + +func (r reason) Service() path.ServiceType { + return r.service +} + +func (r reason) Category() path.CategoryType { + return r.category +} + +func (r reason) SubtreePath() (path.Path, error) { + p, err := path.ServicePrefix( + r.Tenant(), + r.ProtectedResource(), + r.Service(), + r.Category()) + + return p, clues.Wrap(err, "building path").OrNil() +} + +func tagKeys(r Reasoner) []string { + return []string{ + r.ProtectedResource(), + serviceCatString(r.Service(), r.Category()), + } +} + +// reasonKey returns the concatenation of the ProtectedResource, Service, and Category. +func reasonKey(r Reasoner) string { + return r.ProtectedResource() + r.Service().String() + r.Category().String() } type BackupEntry struct { *backup.Backup - Reasons []Reason + Reasons []Reasoner } type ManifestEntry struct { *snapshot.Manifest - // Reason contains the ResourceOwners and Service/Categories that caused this + // Reasons contains the ResourceOwners and Service/Categories that caused this // snapshot to be selected as a base. We can't reuse OwnersCats here because // it's possible some ResourceOwners will have a subset of the Categories as // the reason for selecting a snapshot. For example: // 1. backup user1 email,contacts -> B1 // 2. backup user1 contacts -> B2 (uses B1 as base) // 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts) - Reasons []Reason + Reasons []Reasoner } func (me ManifestEntry) GetTag(key string) (string, bool) { @@ -157,7 +212,7 @@ func (b *baseFinder) getBackupModel( // most recent complete backup as the base. func (b *baseFinder) findBasesInSet( ctx context.Context, - reason Reason, + reason Reasoner, metas []*manifest.EntryMetadata, ) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { // Sort manifests by time so we can go through them sequentially. The code in @@ -190,7 +245,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Info("found incomplete backup") @@ -211,7 +266,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Info("found incomplete backup") @@ -235,7 +290,7 @@ func (b *baseFinder) findBasesInSet( kopiaAssistSnaps = append(kopiaAssistSnaps, ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }) logger.Ctx(ictx).Infow( @@ -253,13 +308,13 @@ func (b *baseFinder) findBasesInSet( me := ManifestEntry{ Manifest: man, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, } kopiaAssistSnaps = append(kopiaAssistSnaps, me) return &BackupEntry{ Backup: bup, - Reasons: []Reason{reason}, + Reasons: []Reasoner{reason}, }, &me, kopiaAssistSnaps, nil } @@ -270,12 +325,12 @@ func (b *baseFinder) findBasesInSet( func (b *baseFinder) getBase( ctx context.Context, - reason Reason, + r Reasoner, tags map[string]string, ) (*BackupEntry, *ManifestEntry, []ManifestEntry, error) { allTags := map[string]string{} - for _, k := range reason.TagKeys() { + for _, k := range tagKeys(r) { allTags[k] = "" } @@ -292,12 +347,12 @@ func (b *baseFinder) getBase( return nil, nil, nil, nil } - return b.findBasesInSet(ctx, reason, metas) + return b.findBasesInSet(ctx, r, metas) } func (b *baseFinder) FindBases( ctx context.Context, - reasons []Reason, + reasons []Reasoner, tags map[string]string, ) BackupBases { var ( @@ -310,14 +365,14 @@ func (b *baseFinder) FindBases( kopiaAssistSnaps = map[manifest.ID]ManifestEntry{} ) - for _, reason := range reasons { + for _, searchReason := range reasons { ictx := clues.Add( ctx, - "search_service", reason.Service.String(), - "search_category", reason.Category.String()) + "search_service", searchReason.Service().String(), + "search_category", searchReason.Category().String()) logger.Ctx(ictx).Info("searching for previous manifests") - baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, reason, tags) + baseBackup, baseSnap, assistSnaps, err := b.getBase(ictx, searchReason, tags) if err != nil { logger.Ctx(ctx).Info( "getting base, falling back to full backup for reason", diff --git a/src/internal/kopia/base_finder_test.go b/src/internal/kopia/base_finder_test.go index f76b3c81a..cb3239ca1 100644 --- a/src/internal/kopia/base_finder_test.go +++ b/src/internal/kopia/base_finder_test.go @@ -39,61 +39,24 @@ var ( testUser2 = "user2" testUser3 = "user3" - testAllUsersAllCats = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + testAllUsersAllCats = []Reasoner{ + // User1 email and events. + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + // User2 email and events. + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + // User3 email and events. + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), } - testAllUsersMail = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + testAllUsersMail = []Reasoner{ + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), } - testUser1Mail = []Reason{ - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + testUser1Mail = []Reasoner{ + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), } ) @@ -322,12 +285,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() { sm: mockEmptySnapshotManager{}, bg: mockEmptyModelGetter{}, } - reasons := []Reason{ - { - ResourceOwner: "a-user", - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons := []Reasoner{ + NewReason("", "a-user", path.ExchangeService, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -345,12 +304,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { sm: &mockSnapshotManager{findErr: assert.AnError}, bg: mockEmptyModelGetter{}, } - reasons := []Reason{ - { - ResourceOwner: "a-user", - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons := []Reasoner{ + NewReason("", "a-user", path.ExchangeService, path.EmailCategory), } bb := bf.FindBases(ctx, reasons, nil) @@ -361,14 +316,14 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() { func (suite *BaseFinderUnitSuite) TestGetBases() { table := []struct { name string - input []Reason + input []Reasoner manifestData []manifestInfo // Use this to denote the Reasons a base backup or base manifest is // selected. The int maps to the index of the backup or manifest in data. - expectedBaseReasons map[int][]Reason + expectedBaseReasons map[int][]Reasoner // Use this to denote the Reasons a kopia assised incrementals manifest is // selected. The int maps to the index of the manifest in data. - expectedAssistManifestReasons map[int][]Reason + expectedAssistManifestReasons map[int][]Reasoner backupData []backupInfo }{ { @@ -394,10 +349,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -428,10 +383,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -463,10 +418,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -492,10 +447,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -519,10 +474,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testAllUsersAllCats, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testAllUsersAllCats, }, backupData: []backupInfo{ @@ -557,76 +512,28 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser3, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: { - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), }, 1: { - Reason{ - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), }, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: { - { - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EmailCategory), + NewReason("", testUser2, path.ExchangeService, path.EmailCategory), + NewReason("", testUser3, path.ExchangeService, path.EmailCategory), }, 1: { - Reason{ - ResourceOwner: testUser1, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser2, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, - Reason{ - ResourceOwner: testUser3, - Service: path.ExchangeService, - Category: path.EventsCategory, - }, + NewReason("", testUser1, path.ExchangeService, path.EventsCategory), + NewReason("", testUser2, path.ExchangeService, path.EventsCategory), + NewReason("", testUser3, path.ExchangeService, path.EventsCategory), }, }, backupData: []backupInfo{ @@ -657,10 +564,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, 1: testUser1Mail, }, @@ -693,10 +600,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 1: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -728,8 +635,8 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{}, - expectedAssistManifestReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{}, + expectedAssistManifestReasons: map[int][]Reasoner{ 1: testUser1Mail, }, backupData: []backupInfo{ @@ -752,10 +659,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -787,10 +694,10 @@ func (suite *BaseFinderUnitSuite) TestGetBases() { testUser1, ), }, - expectedBaseReasons: map[int][]Reason{ + expectedBaseReasons: map[int][]Reasoner{ 0: testUser1Mail, }, - expectedAssistManifestReasons: map[int][]Reason{ + expectedAssistManifestReasons: map[int][]Reasoner{ 0: testUser1Mail, }, backupData: []backupInfo{ @@ -857,17 +764,17 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { table := []struct { name string - input []Reason + input []Reasoner tags map[string]string // Use this to denote which manifests in data should be expected. Allows // defining data in a table while not repeating things between data and // expected. - expectedIdxs map[int][]Reason + expectedIdxs map[int][]Reasoner }{ { name: "no tags specified", tags: nil, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, @@ -877,14 +784,14 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() { "fnords": "", "smarf": "", }, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, { name: "subset of custom tags", tags: map[string]string{"fnords": ""}, - expectedIdxs: map[int][]Reason{ + expectedIdxs: map[int][]Reasoner{ 0: testUser1Mail, }, }, @@ -925,7 +832,7 @@ func checkManifestEntriesMatch( t *testing.T, retSnaps []ManifestEntry, allExpected []manifestInfo, - expectedIdxsAndReasons map[int][]Reason, + expectedIdxsAndReasons map[int][]Reasoner, ) { // Check the proper snapshot manifests were returned. expected := make([]*snapshot.Manifest, 0, len(expectedIdxsAndReasons)) @@ -941,7 +848,7 @@ func checkManifestEntriesMatch( assert.ElementsMatch(t, expected, got) // Check the reasons for selecting each manifest are correct. - expectedReasons := make(map[manifest.ID][]Reason, len(expectedIdxsAndReasons)) + expectedReasons := make(map[manifest.ID][]Reasoner, len(expectedIdxsAndReasons)) for idx, reasons := range expectedIdxsAndReasons { expectedReasons[allExpected[idx].man.ID] = reasons } @@ -967,7 +874,7 @@ func checkBackupEntriesMatch( t *testing.T, retBups []BackupEntry, allExpected []backupInfo, - expectedIdxsAndReasons map[int][]Reason, + expectedIdxsAndReasons map[int][]Reasoner, ) { // Check the proper snapshot manifests were returned. expected := make([]*backup.Backup, 0, len(expectedIdxsAndReasons)) @@ -983,7 +890,7 @@ func checkBackupEntriesMatch( assert.ElementsMatch(t, expected, got) // Check the reasons for selecting each manifest are correct. - expectedReasons := make(map[model.StableID][]Reason, len(expectedIdxsAndReasons)) + expectedReasons := make(map[model.StableID][]Reasoner, len(expectedIdxsAndReasons)) for idx, reasons := range expectedIdxsAndReasons { expectedReasons[allExpected[idx].b.ID] = reasons } diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index d28001f3f..7eac9df5c 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -12,12 +12,16 @@ import ( "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/compression" "github.com/kopia/kopia/repo/content" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/kopia/retention" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" ) @@ -70,7 +74,11 @@ func NewConn(s storage.Storage) *conn { } } -func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { +func (w *conn) Initialize( + ctx context.Context, + opts repository.Options, + retentionOpts repository.Retention, +) error { bst, err := blobStoreByProvider(ctx, opts, w.storage) if err != nil { return clues.Wrap(err, "initializing storage") @@ -82,8 +90,23 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - // todo - issue #75: nil here should be a storage.NewRepoOptions() - if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassphrase); err != nil { + rOpts := retention.NewOpts() + if err := rOpts.Set(retentionOpts); err != nil { + return clues.Wrap(err, "setting retention configuration").WithClues(ctx) + } + + blobCfg, _, err := rOpts.AsConfigs(ctx) + if err != nil { + return clues.Stack(err) + } + + // Minimal config for retention if caller requested it. + kopiaOpts := repo.NewRepositoryOptions{ + RetentionMode: blobCfg.RetentionMode, + RetentionPeriod: blobCfg.RetentionPeriod, + } + + if err = repo.Initialize(ctx, bst, &kopiaOpts, cfg.CorsoPassphrase); err != nil { if errors.Is(err, repo.ErrAlreadyInitialized) { return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) } @@ -107,7 +130,10 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error { return clues.Stack(err).WithClues(ctx) } - return nil + // Calling with all parameters here will set extend object locks for + // maintenance. Parameters for actual retention should have been set during + // initialization and won't be updated again. + return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil() } func (w *conn) Connect(ctx context.Context, opts repository.Options) error { @@ -326,12 +352,12 @@ func updateCompressionOnPolicy(compressor string, p *policy.Policy) (bool, error return true, nil } -func updateRetentionOnPolicy(retention policy.RetentionPolicy, p *policy.Policy) bool { - if retention == p.RetentionPolicy { +func updateRetentionOnPolicy(retPolicy policy.RetentionPolicy, p *policy.Policy) bool { + if retPolicy == p.RetentionPolicy { return false } - p.RetentionPolicy = retention + p.RetentionPolicy = retPolicy return true } @@ -410,6 +436,118 @@ func checkCompressor(compressor compression.Name) error { return clues.Stack(clues.New("unknown compressor type"), clues.New(string(compressor))) } +func (w *conn) setRetentionParameters( + ctx context.Context, + rrOpts repository.Retention, +) error { + if rrOpts.Mode == nil && rrOpts.Duration == nil && rrOpts.Extend == nil { + return nil + } + + // Somewhat confusing case, when we have no retention but a non-zero duration + // it acts like we passed in only the duration and returns an error about + // having to set both. Return a clearer error here instead. + if ptr.Val(rrOpts.Mode) == repository.NoRetention && ptr.Val(rrOpts.Duration) != 0 { + return clues.New("duration must be 0 if rrOpts is disabled").WithClues(ctx) + } + + dr, ok := w.Repository.(repo.DirectRepository) + if !ok { + return clues.New("getting handle to repo").WithClues(ctx) + } + + blobCfg, params, err := getRetentionConfigs(ctx, dr) + if err != nil { + return clues.Stack(err) + } + + opts := retention.OptsFromConfigs(*blobCfg, *params) + if err := opts.Set(rrOpts); err != nil { + return clues.Stack(err).WithClues(ctx) + } + + return clues.Stack(persistRetentionConfigs(ctx, dr, opts)).OrNil() +} + +func getRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, +) (*format.BlobStorageConfiguration, *maintenance.Params, error) { + blobCfg, err := dr.FormatManager().BlobCfgBlob() + if err != nil { + return nil, nil, clues.Wrap(err, "getting storage config").WithClues(ctx) + } + + params, err := maintenance.GetParams(ctx, dr) + if err != nil { + return nil, nil, clues.Wrap(err, "getting maintenance config").WithClues(ctx) + } + + return &blobCfg, params, nil +} + +func persistRetentionConfigs( + ctx context.Context, + dr repo.DirectRepository, + opts *retention.Opts, +) error { + // Persist changes. + if !opts.BlobChanged() && !opts.ParamsChanged() { + return nil + } + + blobCfg, params, err := opts.AsConfigs(ctx) + if err != nil { + return clues.Stack(err) + } + + mp, err := dr.FormatManager().GetMutableParameters() + if err != nil { + return clues.Wrap(err, "getting mutable parameters").WithClues(ctx) + } + + requiredFeatures, err := dr.FormatManager().RequiredFeatures() + if err != nil { + return clues.Wrap(err, "getting required features").WithClues(ctx) + } + + // Must be the case that only blob changed. + if !opts.ParamsChanged() { + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, blobCfg, requiredFeatures), + "persisting storage config", + ).WithClues(ctx).OrNil() + } + + // Both blob and maintenance changed. A DirectWriteSession is required to + // update the maintenance config but not the blob config. + err = repo.DirectWriteSession( + ctx, + dr, + repo.WriteSessionOptions{ + Purpose: "Corso immutable backups config", + }, + func(ctx context.Context, dw repo.DirectRepositoryWriter) error { + // Set the maintenance config first as we can bail out of the write + // session later. + if err := maintenance.SetParams(ctx, dw, ¶ms); err != nil { + return clues.Wrap(err, "maintenance config"). + WithClues(ctx) + } + + if !opts.BlobChanged() { + return nil + } + + return clues.Wrap( + dr.FormatManager().SetParameters(ctx, mp, blobCfg, requiredFeatures), + "storage config", + ).WithClues(ctx).OrNil() + }) + + return clues.Wrap(err, "persisting config changes").WithClues(ctx).OrNil() +} + func (w *conn) LoadSnapshot( ctx context.Context, id manifest.ID, diff --git a/src/internal/kopia/conn_test.go b/src/internal/kopia/conn_test.go index 899447e00..36a2bcdfd 100644 --- a/src/internal/kopia/conn_test.go +++ b/src/internal/kopia/conn_test.go @@ -7,12 +7,15 @@ import ( "time" "github.com/alcionai/clues" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -26,7 +29,7 @@ func openKopiaRepo( st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - if err := k.Initialize(ctx, repository.Options{}); err != nil { + if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil { return nil, err } @@ -82,13 +85,13 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Close(ctx) require.NoError(t, err, clues.ToCore(err)) - err = k.Initialize(ctx, repository.Options{}) + err = k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) assert.ErrorIs(t, err, ErrorRepoAlreadyExists) } @@ -103,7 +106,7 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() { st.Provider = storage.ProviderUnknown k := NewConn(st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) assert.Error(t, err, clues.ToCore(err)) } @@ -413,7 +416,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { st := storeTD.NewPrefixedS3Storage(t) k := NewConn(st) - err := k.Initialize(ctx, opts) + err := k.Initialize(ctx, opts, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kopiaOpts := k.ClientOptions() @@ -453,3 +456,72 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() { err = k.Close(ctx) assert.NoError(t, err, clues.ToCore(err)) } + +// --------------- +// integration tests that require object locking to be enabled on the bucket. +// --------------- +type ConnRetentionIntegrationSuite struct { + tester.Suite +} + +func TestConnRetentionIntegrationSuite(t *testing.T) { + suite.Run(t, &ConnRetentionIntegrationSuite{ + Suite: tester.NewRetentionSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}, + ), + }) +} + +// Test that providing retention doesn't change anything but retention values +// from the default values that kopia uses. +func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + st1 := storeTD.NewPrefixedS3Storage(t) + + k1 := NewConn(st1) + err := k1.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err)) + + st2 := storeTD.NewPrefixedS3Storage(t) + + k2 := NewConn(st2) + err = k2.Initialize( + ctx, + repository.Options{}, + repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }) + require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err)) + + dr1, ok := k1.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 1") + + dr2, ok := k2.Repository.(repo.DirectRepository) + require.True(t, ok, "getting direct repo 2") + + format1 := dr1.FormatManager().ScrubbedContentFormat() + format2 := dr2.FormatManager().ScrubbedContentFormat() + + assert.Equal(t, format1, format2) + + blobCfg1, err := dr1.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting blob config 1: %v", clues.ToCore(err)) + + blobCfg2, err := dr2.FormatManager().BlobCfgBlob() + require.NoError(t, err, "getting retention config 2: %v", clues.ToCore(err)) + + assert.NotEqual(t, blobCfg1, blobCfg2) + + // Check to make sure retention not enabled unexpectedly. + checkRetentionParams(t, ctx, k1, blob.RetentionMode(""), 0, assert.False) + + // Some checks to make sure retention was fully initialized as expected. + checkRetentionParams(t, ctx, k2, blob.Governance, time.Hour*48, assert.True) +} diff --git a/src/internal/kopia/inject/inject.go b/src/internal/kopia/inject/inject.go index 6921c353d..5d8dd3bc7 100644 --- a/src/internal/kopia/inject/inject.go +++ b/src/internal/kopia/inject/inject.go @@ -15,7 +15,8 @@ type ( BackupConsumer interface { ConsumeBackupCollections( ctx context.Context, - bases []kopia.IncrementalBase, + backupReasons []kopia.Reasoner, + bases kopia.BackupBases, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, tags map[string]string, @@ -37,7 +38,7 @@ type ( BaseFinder interface { FindBases( ctx context.Context, - reasons []kopia.Reason, + reasons []kopia.Reasoner, tags map[string]string, ) kopia.BackupBases } diff --git a/src/internal/kopia/merge_collection.go b/src/internal/kopia/merge_collection.go index ff32c4e73..25897fd58 100644 --- a/src/internal/kopia/merge_collection.go +++ b/src/internal/kopia/merge_collection.go @@ -70,7 +70,9 @@ func (mc *mergeCollection) Items( for _, c := range mc.cols { // Unfortunately doesn't seem to be a way right now to see if the // iteration failed and we should be exiting early. - ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + ictx := clues.Add( + ctx, + "merged_collection_storage_path", path.LoggableDir(c.storagePath)) logger.Ctx(ictx).Debug("sending items from merged collection") for item := range c.Items(ictx, errs) { @@ -95,7 +97,9 @@ func (mc *mergeCollection) FetchItemByName( "merged_collection_count", len(mc.cols)) for _, c := range mc.cols { - ictx := clues.Add(ctx, "merged_collection_storage_path", c.storagePath) + ictx := clues.Add( + ctx, + "merged_collection_storage_path", path.LoggableDir(c.storagePath)) logger.Ctx(ictx).Debug("looking for item in merged collection") diff --git a/src/internal/kopia/model_store_test.go b/src/internal/kopia/model_store_test.go index bd7d651ca..aa929ea9d 100644 --- a/src/internal/kopia/model_store_test.go +++ b/src/internal/kopia/model_store_test.go @@ -808,7 +808,7 @@ func openConnAndModelStore( st := storeTD.NewPrefixedS3Storage(t) c := NewConn(st) - err := c.Initialize(ctx, repository.Options{}) + err := c.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { diff --git a/src/internal/kopia/retention/opts.go b/src/internal/kopia/retention/opts.go new file mode 100644 index 000000000..b63a6a6a3 --- /dev/null +++ b/src/internal/kopia/retention/opts.go @@ -0,0 +1,139 @@ +package retention + +import ( + "context" + "time" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" + + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type Opts struct { + blobCfg format.BlobStorageConfiguration + params maintenance.Params + + blobChanged bool + paramsChanged bool +} + +func NewOpts() *Opts { + return &Opts{} +} + +func OptsFromConfigs( + blobCfg format.BlobStorageConfiguration, + params maintenance.Params, +) *Opts { + return &Opts{ + blobCfg: blobCfg, + params: params, + } +} + +func (r *Opts) AsConfigs( + ctx context.Context, +) (format.BlobStorageConfiguration, maintenance.Params, error) { + // Check the new config is valid. + if r.blobCfg.IsRetentionEnabled() { + if err := maintenance.CheckExtendRetention(ctx, r.blobCfg, &r.params); err != nil { + return format.BlobStorageConfiguration{}, maintenance.Params{}, clues.Wrap( + err, + "invalid retention config", + ).WithClues(ctx) + } + } + + return r.blobCfg, r.params, nil +} + +func (r *Opts) BlobChanged() bool { + return r.blobChanged +} + +func (r *Opts) ParamsChanged() bool { + return r.paramsChanged +} + +func (r *Opts) Set(opts repository.Retention) error { + r.setMaintenanceParams(opts.Extend) + + return clues.Wrap( + r.setBlobConfigParams(opts.Mode, opts.Duration), + "setting mode or duration", + ).OrNil() +} + +func (r *Opts) setMaintenanceParams(extend *bool) { + if extend != nil && r.params.ExtendObjectLocks != *extend { + r.params.ExtendObjectLocks = *extend + r.paramsChanged = true + } +} + +func (r *Opts) setBlobConfigParams( + mode *repository.RetentionMode, + duration *time.Duration, +) error { + err := r.setBlobConfigMode(mode) + if err != nil { + return clues.Stack(err) + } + + r.setBlobConfigDuration(duration) + + return nil +} + +func (r *Opts) setBlobConfigDuration(duration *time.Duration) { + if duration != nil && r.blobCfg.RetentionPeriod != *duration { + r.blobCfg.RetentionPeriod = *duration + r.blobChanged = true + } +} + +func (r *Opts) setBlobConfigMode( + mode *repository.RetentionMode, +) error { + if mode == nil { + return nil + } + + startMode := r.blobCfg.RetentionMode + + switch *mode { + case repository.NoRetention: + if !r.blobCfg.IsRetentionEnabled() { + return nil + } + + r.blobCfg.RetentionMode = "" + r.blobCfg.RetentionPeriod = 0 + + case repository.GovernanceRetention: + r.blobCfg.RetentionMode = blob.Governance + + case repository.ComplianceRetention: + r.blobCfg.RetentionMode = blob.Compliance + + default: + return clues.New("unknown retention mode"). + With("provided_retention_mode", mode.String()) + } + + // Only check if the retention mode is not empty. IsValid errors out if it's + // empty. + if len(r.blobCfg.RetentionMode) > 0 && !r.blobCfg.RetentionMode.IsValid() { + return clues.New("invalid retention mode"). + With("retention_mode", r.blobCfg.RetentionMode) + } + + // Take into account previous operations on r that could have already updated + // blobChanged. + r.blobChanged = r.blobChanged || startMode != r.blobCfg.RetentionMode + + return nil +} diff --git a/src/internal/kopia/retention/opts_test.go b/src/internal/kopia/retention/opts_test.go new file mode 100644 index 000000000..8b250c79a --- /dev/null +++ b/src/internal/kopia/retention/opts_test.go @@ -0,0 +1,204 @@ +package retention_test + +import ( + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/format" + "github.com/kopia/kopia/repo/maintenance" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/kopia/retention" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type OptsUnitSuite struct { + tester.Suite +} + +func TestOptsUnitSuite(t *testing.T) { + suite.Run(t, &OptsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *OptsUnitSuite) TestOptsFromConfigs() { + var ( + t = suite.T() + + mode = blob.Governance + duration = time.Hour * 48 + extend = true + + blobCfgInput = format.BlobStorageConfiguration{ + RetentionMode: mode, + RetentionPeriod: duration, + } + paramsInput = maintenance.Params{ExtendObjectLocks: extend} + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := retention.OptsFromConfigs(blobCfgInput, paramsInput) + + assert.False(t, opts.BlobChanged(), "BlobChanged") + assert.False(t, opts.ParamsChanged(), "ParamsChanged") + + blobCfg, params, err := opts.AsConfigs(ctx) + require.NoError(t, err, "AsConfigs: %v", clues.ToCore(err)) + assert.Equal(t, blobCfgInput, blobCfg) + assert.Equal(t, paramsInput, params) +} + +func (suite *OptsUnitSuite) TestSet() { + var ( + kopiaMode = blob.Governance + mode = repository.GovernanceRetention + duration = time.Hour * 48 + ) + + table := []struct { + name string + inputBlob format.BlobStorageConfiguration + inputParams maintenance.Params + ctrlOpts repository.Retention + setErr require.ErrorAssertionFunc + expectMode blob.RetentionMode + expectDuration time.Duration + expectExtend bool + expectBlobChanged bool + expectParamsChanged bool + }{ + { + name: "All Nils", + setErr: require.NoError, + }, + { + name: "All Off", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.NoRetention), + Duration: ptr.To(time.Duration(0)), + Extend: ptr.To(false), + }, + setErr: require.NoError, + }, + { + name: "UnknownRetention", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.UnknownRetention), + Duration: ptr.To(duration), + }, + setErr: require.Error, + }, + { + name: "Invalid Retention Mode", + ctrlOpts: repository.Retention{ + Mode: ptr.To(repository.RetentionMode(-1)), + Duration: ptr.To(duration), + }, + setErr: require.Error, + }, + { + name: "Valid Set All", + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + expectExtend: true, + expectBlobChanged: true, + expectParamsChanged: true, + }, + { + name: "Valid Set BlobConfig", + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + expectBlobChanged: true, + }, + { + name: "Valid Set Params", + ctrlOpts: repository.Retention{ + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectExtend: true, + expectParamsChanged: true, + }, + { + name: "Partial BlobConfig Change", + inputBlob: format.BlobStorageConfiguration{ + RetentionMode: kopiaMode, + RetentionPeriod: duration, + }, + ctrlOpts: repository.Retention{ + Duration: ptr.To(duration + time.Hour), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration + time.Hour, + expectBlobChanged: true, + }, + { + name: "No BlobConfig Change", + inputBlob: format.BlobStorageConfiguration{ + RetentionMode: kopiaMode, + RetentionPeriod: duration, + }, + ctrlOpts: repository.Retention{ + Mode: ptr.To(mode), + Duration: ptr.To(duration), + }, + setErr: require.NoError, + expectMode: kopiaMode, + expectDuration: duration, + }, + { + name: "No Params Change", + inputParams: maintenance.Params{ExtendObjectLocks: true}, + ctrlOpts: repository.Retention{ + Extend: ptr.To(true), + }, + setErr: require.NoError, + expectExtend: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + opts := retention.OptsFromConfigs(test.inputBlob, test.inputParams) + err := opts.Set(test.ctrlOpts) + test.setErr(t, err, "setting params: %v", clues.ToCore(err)) + + if err != nil { + return + } + + blobCfg, params, err := opts.AsConfigs(ctx) + require.NoError(t, err, "getting configs: %v", clues.ToCore(err)) + + assert.Equal(t, test.expectMode, blobCfg.RetentionMode, "mode") + assert.Equal(t, test.expectDuration, blobCfg.RetentionPeriod, "duration") + assert.Equal(t, test.expectExtend, params.ExtendObjectLocks, "extend locks") + assert.Equal(t, test.expectBlobChanged, opts.BlobChanged(), "blob changed") + assert.Equal(t, test.expectParamsChanged, opts.ParamsChanged(), "params changed") + }) + } +} diff --git a/src/internal/kopia/upload.go b/src/internal/kopia/upload.go index 8a91367c6..9df1327f9 100644 --- a/src/internal/kopia/upload.go +++ b/src/internal/kopia/upload.go @@ -20,6 +20,7 @@ import ( "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/snapshotfs" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/data" @@ -27,6 +28,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph/metadata" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -250,7 +252,9 @@ func (cp *corsoProgress) FinishedHashingFile(fname string, bs int64) { sl[i] = string(rdt) } - logger.Ctx(context.Background()).Debugw("finished hashing file", "path", sl[2:]) + logger.Ctx(cp.ctx).Debugw( + "finished hashing file", + "path", clues.Hide(path.Elements(sl[2:]))) atomic.AddInt64(&cp.totalBytes, bs) } @@ -440,12 +444,12 @@ func streamBaseEntries( ctx = clues.Add( ctx, - "current_item_path", curPath, - "longest_prefix", longest) + "current_directory_path", curPath, + "longest_prefix", path.LoggableDir(longest)) err := dir.IterateEntries(ctx, func(innerCtx context.Context, entry fs.Entry) error { if err := innerCtx.Err(); err != nil { - return err + return clues.Stack(err).WithClues(ctx) } // Don't walk subdirectories in this function. @@ -462,7 +466,9 @@ func streamBaseEntries( entName, err := decodeElement(entry.Name()) if err != nil { - return clues.Wrap(err, "decoding entry name: "+entry.Name()) + return clues.Wrap(err, "decoding entry name"). + WithClues(ctx). + With("entry_name", entry.Name()) } // This entry was marked as deleted by a service that can't tell us the @@ -474,7 +480,7 @@ func streamBaseEntries( // For now assuming that item IDs don't need escaping. itemPath, err := curPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting full item path for base entry") + return clues.Wrap(err, "getting full item path for base entry").WithClues(ctx) } // We need the previous path so we can find this item in the base snapshot's @@ -483,7 +489,7 @@ func streamBaseEntries( // to look for. prevItemPath, err := prevPath.AppendItem(entName) if err != nil { - return clues.Wrap(err, "getting previous full item path for base entry") + return clues.Wrap(err, "getting previous full item path for base entry").WithClues(ctx) } // Meta files aren't in backup details since it's the set of items the user @@ -507,13 +513,15 @@ func streamBaseEntries( } if err := ctr(ctx, entry); err != nil { - return clues.Wrap(err, "executing callback on item").With("item_path", itemPath) + return clues.Wrap(err, "executing callback on item"). + WithClues(ctx). + With("item_path", itemPath) } return nil }) if err != nil { - return clues.Wrap(err, "traversing items in base snapshot directory") + return clues.Wrap(err, "traversing items in base snapshot directory").WithClues(ctx) } return nil @@ -824,7 +832,9 @@ func inflateCollectionTree( } if node.collection != nil && node.collection.State() == data.NotMovedState { - return nil, nil, clues.New("conflicting states for collection").With("changed_path", p) + return nil, nil, clues.New("conflicting states for collection"). + WithClues(ctx). + With("changed_path", p) } } @@ -851,13 +861,14 @@ func traverseBaseDir( expectedDirPath *path.Builder, dir fs.Directory, roots map[string]*treeMap, + stats *count.Bus, ) error { ctx = clues.Add(ctx, "old_dir_path", oldDirPath, "expected_dir_path", expectedDirPath) if depth >= maxInflateTraversalDepth { - return clues.New("base snapshot tree too tall") + return clues.New("base snapshot tree too tall").WithClues(ctx) } // Wrapper base64 encodes all file and folder names to avoid issues with @@ -865,7 +876,9 @@ func traverseBaseDir( // from kopia we need to do the decoding here. dirName, err := decodeElement(dir.Name()) if err != nil { - return clues.Wrap(err, "decoding base directory name").With("dir_name", dir.Name()) + return clues.Wrap(err, "decoding base directory name"). + WithClues(ctx). + With("dir_name", clues.Hide(dir.Name())) } // Form the path this directory would be at if the hierarchy remained the same @@ -884,14 +897,29 @@ func traverseBaseDir( currentPath = currentPath.Append(dirName) } + var explicitMention bool + if upb, ok := updatedPaths[oldDirPath.String()]; ok { // This directory was deleted. if upb == nil { currentPath = nil + + stats.Inc(statDel) } else { - // This directory was moved/renamed and the new location is in upb. + // This directory was explicitly mentioned and the new (possibly + // unchanged) location is in upb. currentPath = upb.ToBuilder() + + // Below we check if the collection was marked as new or DoNotMerge which + // disables merging behavior. That means we can't directly update stats + // here else we'll miss delta token refreshes and whatnot. Instead note + // that we did see the path explicitly so it's not counted as a recursive + // operation. + explicitMention = true } + } else if currentPath == nil { + // Just stats tracking stuff. + stats.Inc(statRecursiveDel) } ctx = clues.Add(ctx, "new_path", currentPath) @@ -919,10 +947,11 @@ func traverseBaseDir( oldDirPath, currentPath, dEntry, - roots) + roots, + stats) }) if err != nil { - return clues.Wrap(err, "traversing base directory") + return clues.Wrap(err, "traversing base directory").WithClues(ctx) } // We only need to add this base directory to the tree we're building if it @@ -939,7 +968,7 @@ func traverseBaseDir( // in the if-block though as that is an optimization. node := getTreeNode(roots, currentPath.Elements()) if node == nil { - return clues.New("getting tree node") + return clues.New("getting tree node").WithClues(ctx) } // Now that we have the node we need to check if there is a collection @@ -949,17 +978,28 @@ func traverseBaseDir( // directories. The expected usecase for this is delta token expiry in M365. if node.collection != nil && (node.collection.DoNotMergeItems() || node.collection.State() == data.NewState) { + stats.Inc(statSkipMerge) + return nil } + // Just stats tracking stuff. + if oldDirPath.String() == currentPath.String() { + stats.Inc(statNoMove) + } else if explicitMention { + stats.Inc(statMove) + } else { + stats.Inc(statRecursiveMove) + } + curP, err := path.FromDataLayerPath(currentPath.String(), false) if err != nil { - return clues.New("converting current path to path.Path") + return clues.New("converting current path to path.Path").WithClues(ctx) } oldP, err := path.FromDataLayerPath(oldDirPath.String(), false) if err != nil { - return clues.New("converting old path to path.Path") + return clues.New("converting old path to path.Path").WithClues(ctx) } node.baseDir = dir @@ -970,10 +1010,50 @@ func traverseBaseDir( return nil } +func logBaseInfo(ctx context.Context, m ManifestEntry) { + svcs := map[string]struct{}{} + cats := map[string]struct{}{} + + for _, r := range m.Reasons { + svcs[r.Service().String()] = struct{}{} + cats[r.Category().String()] = struct{}{} + } + + mbID, _ := m.GetTag(TagBackupID) + if len(mbID) == 0 { + mbID = "no_backup_id_tag" + } + + logger.Ctx(ctx).Infow( + "using base for backup", + "base_snapshot_id", m.ID, + "services", maps.Keys(svcs), + "categories", maps.Keys(cats), + "base_backup_id", mbID) +} + +const ( + // statNoMove denotes an directory that wasn't moved at all. + statNoMove = "directories_not_moved" + // statMove denotes an directory that was explicitly moved. + statMove = "directories_explicitly_moved" + // statRecursiveMove denotes an directory that moved because one or more or + // its ancestors moved and it wasn't explicitly mentioned. + statRecursiveMove = "directories_recursively_moved" + // statDel denotes a directory that was explicitly deleted. + statDel = "directories_explicitly_deleted" + // statRecursiveDel denotes a directory that was deleted because one or more + // of its ancestors was deleted and it wasn't explicitly mentioned. + statRecursiveDel = "directories_recursively_deleted" + // statSkipMerge denotes the number of directories that weren't merged because + // they were marked either DoNotMerge or New. + statSkipMerge = "directories_skipped_merging" +) + func inflateBaseTree( ctx context.Context, loader snapshotLoader, - snap IncrementalBase, + snap ManifestEntry, updatedPaths map[string]path.Path, roots map[string]*treeMap, ) error { @@ -996,13 +1076,25 @@ func inflateBaseTree( return clues.New("snapshot root is not a directory").WithClues(ctx) } + // Some logging to help track things. + logBaseInfo(ctx, snap) + // For each subtree corresponding to the tuple // (resource owner, service, category) merge the directories in the base with // what has been reported in the collections we got. - for _, subtreePath := range snap.SubtreePaths { + for _, r := range snap.Reasons { + ictx := clues.Add( + ctx, + "subtree_service", r.Service().String(), + "subtree_category", r.Category().String()) + + subtreePath, err := r.SubtreePath() + if err != nil { + return clues.Wrap(err, "building subtree path").WithClues(ictx) + } + // We're starting from the root directory so don't need it in the path. pathElems := encodeElements(subtreePath.PopFront().Elements()...) - ictx := clues.Add(ctx, "subtree_path", subtreePath) ent, err := snapshotfs.GetNestedEntry(ictx, dir, pathElems) if err != nil { @@ -1022,22 +1114,35 @@ func inflateBaseTree( // This ensures that a migration on the directory prefix can complete. // The prefix is the tenant/service/owner/category set, which remains // otherwise unchecked in tree inflation below this point. - newSubtreePath := subtreePath + newSubtreePath := subtreePath.ToBuilder() + if p, ok := updatedPaths[subtreePath.String()]; ok { newSubtreePath = p.ToBuilder() } + stats := count.New() + if err = traverseBaseDir( ictx, 0, updatedPaths, - subtreePath.Dir(), + subtreePath.ToBuilder().Dir(), newSubtreePath.Dir(), subtreeDir, roots, + stats, ); err != nil { return clues.Wrap(err, "traversing base snapshot").WithClues(ictx) } + + logger.Ctx(ctx).Infow( + "merge subtree stats", + statNoMove, stats.Get(statNoMove), + statMove, stats.Get(statMove), + statRecursiveMove, stats.Get(statRecursiveMove), + statDel, stats.Get(statDel), + statRecursiveDel, stats.Get(statRecursiveDel), + statSkipMerge, stats.Get(statSkipMerge)) } return nil @@ -1059,7 +1164,7 @@ func inflateBaseTree( func inflateDirTree( ctx context.Context, loader snapshotLoader, - baseSnaps []IncrementalBase, + baseSnaps []ManifestEntry, collections []data.BackupCollection, globalExcludeSet prefixmatcher.StringSetReader, progress *corsoProgress, @@ -1089,7 +1194,7 @@ func inflateDirTree( } if len(roots) > 1 { - return nil, clues.New("multiple root directories") + return nil, clues.New("multiple root directories").WithClues(ctx) } var res fs.Directory diff --git a/src/internal/kopia/upload_test.go b/src/internal/kopia/upload_test.go index 0ac10ec6b..bbdbe9e6f 100644 --- a/src/internal/kopia/upload_test.go +++ b/src/internal/kopia/upload_test.go @@ -946,21 +946,22 @@ func (msw *mockSnapshotWalker) SnapshotRoot(*snapshot.Manifest) (fs.Entry, error return msw.snapshotRoot, nil } -func mockIncrementalBase( +func makeManifestEntry( id, tenant, resourceOwner string, service path.ServiceType, categories ...path.CategoryType, -) IncrementalBase { - stps := []*path.Builder{} +) ManifestEntry { + var reasons []Reasoner + for _, c := range categories { - stps = append(stps, path.Builder{}.Append(tenant, service.String(), resourceOwner, c.String())) + reasons = append(reasons, NewReason(tenant, resourceOwner, service, c)) } - return IncrementalBase{ + return ManifestEntry{ Manifest: &snapshot.Manifest{ ID: manifest.ID(id), }, - SubtreePaths: stps, + Reasons: reasons, } } @@ -1331,8 +1332,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() { dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, test.inputCollections(), pmMock.NewPrefixMap(nil), @@ -2260,8 +2261,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, test.inputCollections(t), ie, @@ -2425,8 +2426,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2531,8 +2532,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_HandleEmptyBase() dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2782,9 +2783,9 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("id1", testTenant, testUser, path.ExchangeService, path.ContactsCategory), - mockIncrementalBase("id2", testTenant, testUser, path.ExchangeService, path.EmailCategory), + []ManifestEntry{ + makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.ContactsCategory), + makeManifestEntry("id2", testTenant, testUser, path.ExchangeService, path.EmailCategory), }, collections, pmMock.NewPrefixMap(nil), @@ -2948,8 +2949,8 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt dirTree, err := inflateDirTree( ctx, msw, - []IncrementalBase{ - mockIncrementalBase("id1", testTenant, testUser, path.ExchangeService, path.EmailCategory, path.ContactsCategory), + []ManifestEntry{ + makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.EmailCategory, path.ContactsCategory), }, []data.BackupCollection{mce, mcc}, pmMock.NewPrefixMap(nil), diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index f65827f76..7b1feca44 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -4,22 +4,19 @@ import ( "context" "errors" "strings" - "time" "github.com/alcionai/clues" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/repo/blob" - "github.com/kopia/kopia/repo/format" "github.com/kopia/kopia/repo/maintenance" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" "github.com/kopia/kopia/snapshot/snapshotfs" "github.com/kopia/kopia/snapshot/snapshotmaintenance" + "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/common/prefixmatcher" - "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/observe" @@ -132,11 +129,6 @@ func (w *Wrapper) Close(ctx context.Context) error { return nil } -type IncrementalBase struct { - *snapshot.Manifest - SubtreePaths []*path.Builder -} - // ConsumeBackupCollections takes a set of collections and creates a kopia snapshot // with the data that they contain. previousSnapshots is used for incremental // backups and should represent the base snapshot from which metadata is sourced @@ -145,10 +137,11 @@ type IncrementalBase struct { // complete backup of all data. func (w Wrapper) ConsumeBackupCollections( ctx context.Context, - previousSnapshots []IncrementalBase, + backupReasons []Reasoner, + bases BackupBases, collections []data.BackupCollection, globalExcludeSet prefixmatcher.StringSetReader, - tags map[string]string, + additionalTags map[string]string, buildTreeWithBase bool, errs *fault.Bus, ) (*BackupStats, *details.Builder, DetailsMergeInfoer, error) { @@ -174,15 +167,23 @@ func (w Wrapper) ConsumeBackupCollections( // When running an incremental backup, we need to pass the prior // snapshot bases into inflateDirTree so that the new snapshot // includes historical data. - var base []IncrementalBase - if buildTreeWithBase { - base = previousSnapshots + var ( + mergeBase []ManifestEntry + assistBase []ManifestEntry + ) + + if bases != nil { + if buildTreeWithBase { + mergeBase = bases.MergeBases() + } + + assistBase = bases.AssistBases() } dirTree, err := inflateDirTree( ctx, w.c, - base, + mergeBase, collections, globalExcludeSet, progress) @@ -190,9 +191,22 @@ func (w Wrapper) ConsumeBackupCollections( return nil, nil, nil, clues.Wrap(err, "building kopia directories") } + // Add some extra tags so we can look things up by reason. + tags := maps.Clone(additionalTags) + if tags == nil { + // Some platforms seem to return nil if the input is nil. + tags = map[string]string{} + } + + for _, r := range backupReasons { + for _, k := range tagKeys(r) { + tags[k] = "" + } + } + s, err := w.makeSnapshotWithRoot( ctx, - previousSnapshots, + assistBase, dirTree, tags, progress) @@ -205,7 +219,7 @@ func (w Wrapper) ConsumeBackupCollections( func (w Wrapper) makeSnapshotWithRoot( ctx context.Context, - prevSnapEntries []IncrementalBase, + prevSnapEntries []ManifestEntry, root fs.Directory, addlTags map[string]string, progress *corsoProgress, @@ -225,8 +239,8 @@ func (w Wrapper) makeSnapshotWithRoot( ctx = clues.Add( ctx, - "len_prev_base_snapshots", len(prevSnapEntries), - "assist_snap_ids", snapIDs, + "num_assist_snapshots", len(prevSnapEntries), + "assist_snapshot_ids", snapIDs, "additional_tags", addlTags) if len(snapIDs) > 0 { @@ -310,7 +324,7 @@ func (w Wrapper) makeSnapshotWithRoot( // Telling kopia to always flush may hide other errors if it fails while // flushing the write session (hence logging above). if err != nil { - return nil, clues.Wrap(err, "kopia backup") + return nil, clues.Wrap(err, "kopia backup").WithClues(ctx) } res := manifestToStats(man, progress, bc) @@ -355,7 +369,7 @@ func getDir( encodeElements(dirPath.PopFront().Elements()...)) if err != nil { if isErrEntryNotFound(err) { - err = clues.Stack(data.ErrNotFound, err) + err = clues.Stack(data.ErrNotFound, err).WithClues(ctx) } return nil, clues.Wrap(err, "getting nested object handle").WithClues(ctx) @@ -473,7 +487,7 @@ func (w Wrapper) ProduceRestoreCollections( // load it here. snapshotRoot, err := w.getSnapshotRoot(ctx, snapshotID) if err != nil { - return nil, clues.Wrap(err, "loading snapshot root") + return nil, clues.Wrap(err, "loading snapshot root").WithClues(ctx) } var ( @@ -493,8 +507,8 @@ func (w Wrapper) ProduceRestoreCollections( // items from a single directory instance lower down. ictx := clues.Add( ctx, - "item_path", itemPaths.StoragePath.String(), - "restore_path", itemPaths.RestorePath.String()) + "item_path", itemPaths.StoragePath, + "restore_path", itemPaths.RestorePath) parentStoragePath, err := itemPaths.StoragePath.Dir() if err != nil { @@ -538,7 +552,7 @@ func (w Wrapper) ProduceRestoreCollections( // then load the items from the directory. res, err := loadDirsAndItems(ctx, snapshotRoot, bcounter, dirsToItems, errs) if err != nil { - return nil, clues.Wrap(err, "loading items") + return nil, clues.Wrap(err, "loading items").WithClues(ctx) } return res, el.Failure() @@ -596,12 +610,12 @@ func (w Wrapper) RepoMaintenance( ) error { kopiaSafety, err := translateSafety(opts.Safety) if err != nil { - return clues.Wrap(err, "identifying safety level") + return clues.Wrap(err, "identifying safety level").WithClues(ctx) } mode, err := translateMode(opts.Type) if err != nil { - return clues.Wrap(err, "identifying maintenance mode") + return clues.Wrap(err, "identifying maintenance mode").WithClues(ctx) } currentOwner := w.c.ClientOptions().UsernameAtHost() @@ -722,202 +736,5 @@ func (w *Wrapper) SetRetentionParameters( ctx context.Context, retention repository.Retention, ) error { - if retention.Mode == nil && retention.Duration == nil && retention.Extend == nil { - return nil - } - - // Somewhat confusing case, when we have no retention but a non-zero duration - // it acts like we passed in only the duration and returns an error about - // having to set both. Return a clearer error here instead. Check if mode is - // set so we still allow changing duration if mode is already set. - if m, ok := ptr.ValOK(retention.Mode); ok && m == repository.NoRetention && ptr.Val(retention.Duration) != 0 { - return clues.New("duration must be 0 if retention is disabled").WithClues(ctx) - } - - dr, ok := w.c.Repository.(repo.DirectRepository) - if !ok { - return clues.New("getting handle to repo").WithClues(ctx) - } - - blobCfg, params, err := getRetentionConfigs(ctx, dr) - if err != nil { - return clues.Stack(err) - } - - // Update blob config information. - blobChanged, err := w.setBlobConfigParams(retention.Mode, retention.Duration, blobCfg) - if err != nil { - return clues.Wrap(err, "setting retention mode or duration").WithClues(ctx) - } - - // Update maintenance config information. - var maintenanceChanged bool - - if retention.Extend != nil && params.ExtendObjectLocks != *retention.Extend { - params.ExtendObjectLocks = *retention.Extend - maintenanceChanged = true - } - - // Check the new config is valid. - if blobCfg.IsRetentionEnabled() { - if err := maintenance.CheckExtendRetention(ctx, *blobCfg, params); err != nil { - return clues.Wrap(err, "invalid retention config").WithClues(ctx) - } - } - - return clues.Stack(persistRetentionConfigs( - ctx, - dr, - blobCfg, - blobChanged, - params, - maintenanceChanged, - )).OrNil() -} - -func getRetentionConfigs( - ctx context.Context, - dr repo.DirectRepository, -) (*format.BlobStorageConfiguration, *maintenance.Params, error) { - blobCfg, err := dr.FormatManager().BlobCfgBlob() - if err != nil { - return nil, nil, clues.Wrap(err, "getting storage config").WithClues(ctx) - } - - params, err := maintenance.GetParams(ctx, dr) - if err != nil { - return nil, nil, clues.Wrap(err, "getting maintenance config").WithClues(ctx) - } - - return &blobCfg, params, nil -} - -func persistRetentionConfigs( - ctx context.Context, - dr repo.DirectRepository, - blobCfg *format.BlobStorageConfiguration, - blobChanged bool, - params *maintenance.Params, - maintenanceChanged bool, -) error { - // Persist changes. - if !blobChanged && !maintenanceChanged { - return nil - } - - mp, err := dr.FormatManager().GetMutableParameters() - if err != nil { - return clues.Wrap(err, "getting mutable parameters") - } - - requiredFeatures, err := dr.FormatManager().RequiredFeatures() - if err != nil { - return clues.Wrap(err, "getting required features").WithClues(ctx) - } - - // Must be the case that only blob changed. - if !maintenanceChanged { - return clues.Wrap( - dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), - "persisting storage config", - ).WithClues(ctx).OrNil() - } - - // Both blob and maintenance changed. A DirectWriteSession is required to - // update the maintenance config but not the blob config. - err = repo.DirectWriteSession( - ctx, - dr, - repo.WriteSessionOptions{ - Purpose: "Corso immutable backups config", - }, - func(ctx context.Context, dw repo.DirectRepositoryWriter) error { - // Set the maintenance config first as we can bail out of the write - // session later. - if err := maintenance.SetParams(ctx, dw, params); err != nil { - return clues.Wrap(err, "maintenance config"). - WithClues(ctx) - } - - if !blobChanged { - return nil - } - - return clues.Wrap( - dr.FormatManager().SetParameters(ctx, mp, *blobCfg, requiredFeatures), - "storage config", - ).WithClues(ctx).OrNil() - }) - - return clues.Wrap(err, "persisting config changes").WithClues(ctx).OrNil() -} - -func (w Wrapper) setBlobConfigParams( - mode *repository.RetentionMode, - duration *time.Duration, - blobCfg *format.BlobStorageConfiguration, -) (bool, error) { - changed, err := setBlobConfigMode(mode, blobCfg) - if err != nil { - return false, clues.Stack(err) - } - - tmp := setBlobConfigDuration(duration, blobCfg) - changed = changed || tmp - - return changed, nil -} - -func setBlobConfigDuration( - duration *time.Duration, - blobCfg *format.BlobStorageConfiguration, -) bool { - var changed bool - - if duration != nil && blobCfg.RetentionPeriod != *duration { - blobCfg.RetentionPeriod = *duration - changed = true - } - - return changed -} - -func setBlobConfigMode( - mode *repository.RetentionMode, - blobCfg *format.BlobStorageConfiguration, -) (bool, error) { - if mode == nil { - return false, nil - } - - startMode := blobCfg.RetentionMode - - switch *mode { - case repository.NoRetention: - if !blobCfg.IsRetentionEnabled() { - return false, nil - } - - blobCfg.RetentionMode = "" - blobCfg.RetentionPeriod = 0 - - case repository.GovernanceRetention: - blobCfg.RetentionMode = blob.Governance - - case repository.ComplianceRetention: - blobCfg.RetentionMode = blob.Compliance - - default: - return false, clues.New("unknown retention mode"). - With("provided_retention_mode", mode.String()) - } - - // Only check if the retention mode is not empty. IsValid errors out if it's - // empty. - if len(blobCfg.RetentionMode) > 0 && !blobCfg.RetentionMode.IsValid() { - return false, clues.New("invalid retention mode"). - With("retention_mode", blobCfg.RetentionMode) - } - - return startMode != blobCfg.RetentionMode, nil + return clues.Stack(w.c.setRetentionParameters(ctx, retention)).OrNil() } diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index 12857904f..5014e07c1 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -696,6 +696,24 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { 42), } + c1 := exchMock.NewCollection( + suite.storePath1, + suite.locPath1, + 0) + c1.ColState = data.NotMovedState + c1.PrevPath = suite.storePath1 + + c2 := exchMock.NewCollection( + suite.storePath2, + suite.locPath2, + 0) + c2.ColState = data.NotMovedState + c2.PrevPath = suite.storePath2 + + // Make empty collections at the same locations to force a backup with no + // changes. Needed to ensure we force a backup even if nothing has changed. + emptyCollections := []data.BackupCollection{c1, c2} + // tags that are supplied by the caller. This includes basic tags to support // lookups and extra tags the caller may want to apply. tags := map[string]string{ @@ -703,108 +721,246 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { "brunhilda": "", } - reasons := []Reason{ - { - ResourceOwner: suite.storePath1.ResourceOwner(), - Service: suite.storePath1.Service(), - Category: suite.storePath1.Category(), - }, - { - ResourceOwner: suite.storePath2.ResourceOwner(), - Service: suite.storePath2.Service(), - Category: suite.storePath2.Category(), - }, - } - - for _, r := range reasons { - for _, k := range r.TagKeys() { - tags[k] = "" - } + reasons := []Reasoner{ + NewReason( + testTenant, + suite.storePath1.ResourceOwner(), + suite.storePath1.Service(), + suite.storePath1.Category(), + ), + NewReason( + testTenant, + suite.storePath2.ResourceOwner(), + suite.storePath2.Service(), + suite.storePath2.Category(), + ), } expectedTags := map[string]string{} - maps.Copy(expectedTags, normalizeTagKVs(tags)) + maps.Copy(expectedTags, tags) - table := []struct { - name string - expectedUploadedFiles int - expectedCachedFiles int - // Whether entries in the resulting details should be marked as updated. - deetsUpdated bool - }{ - { - name: "Uncached", - expectedUploadedFiles: 47, - expectedCachedFiles: 0, - deetsUpdated: true, - }, - { - name: "Cached", - expectedUploadedFiles: 0, - expectedCachedFiles: 47, - deetsUpdated: false, - }, + for _, r := range reasons { + for _, k := range tagKeys(r) { + expectedTags[k] = "" + } } - prevSnaps := []IncrementalBase{} + expectedTags = normalizeTagKVs(expectedTags) + + type testCase struct { + name string + baseBackups func(base ManifestEntry) BackupBases + collections []data.BackupCollection + expectedUploadedFiles int + expectedCachedFiles int + // We're either going to get details entries or entries in the details + // merger. Details is populated when there's entries in the collection. The + // details merger is populated for cached entries. The details merger + // doesn't count folders, only items. + // + // Setting this to true looks for details merger entries. Setting it to + // false looks for details entries. + expectMerge bool + // Whether entries in the resulting details should be marked as updated. + deetsUpdated assert.BoolAssertionFunc + hashedBytesCheck assert.ValueAssertionFunc + // Range of bytes (inclusive) to expect as uploaded. A little fragile, but + // allows us to differentiate between content that wasn't uploaded due to + // being cached/deduped/skipped due to existing dir entries and stuff that + // was actually pushed to S3. + uploadedBytes []int64 + } + + // Initial backup. All files should be considered new by kopia. + baseBackupCase := testCase{ + name: "Uncached", + baseBackups: func(ManifestEntry) BackupBases { + return NewMockBackupBases() + }, + collections: collections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + deetsUpdated: assert.True, + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{8000, 10000}, + } + + runAndTestBackup := func(test testCase, base ManifestEntry) ManifestEntry { + var res ManifestEntry - for _, test := range table { suite.Run(test.name, func() { t := suite.T() - stats, deets, _, err := suite.w.ConsumeBackupCollections( - suite.ctx, - prevSnaps, - collections, + ctx, flush := tester.NewContext(t) + defer flush() + + bbs := test.baseBackups(base) + + stats, deets, deetsMerger, err := suite.w.ConsumeBackupCollections( + ctx, + reasons, + bbs, + test.collections, nil, tags, true, fault.New(true)) - assert.NoError(t, err, clues.ToCore(err)) + require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files") assert.Equal(t, test.expectedUploadedFiles, stats.UncachedFileCount, "uncached files") assert.Equal(t, test.expectedCachedFiles, stats.CachedFileCount, "cached files") - assert.Equal(t, 6, stats.TotalDirectoryCount) + assert.Equal(t, 4+len(test.collections), stats.TotalDirectoryCount, "directory count") assert.Equal(t, 0, stats.IgnoredErrorCount) assert.Equal(t, 0, stats.ErrorCount) assert.False(t, stats.Incomplete) - - // 47 file and 2 folder entries. - details := deets.Details().Entries - assert.Len( + test.hashedBytesCheck(t, stats.TotalHashedBytes, "hashed bytes") + assert.LessOrEqual( t, - details, - test.expectedUploadedFiles+test.expectedCachedFiles+2, - ) + test.uploadedBytes[0], + stats.TotalUploadedBytes, + "low end of uploaded bytes") + assert.GreaterOrEqual( + t, + test.uploadedBytes[1], + stats.TotalUploadedBytes, + "high end of uploaded bytes") - for _, entry := range details { - assert.Equal(t, test.deetsUpdated, entry.Updated) + if test.expectMerge { + assert.Empty(t, deets.Details().Entries, "details entries") + assert.Equal( + t, + test.expectedUploadedFiles+test.expectedCachedFiles, + deetsMerger.ItemsToMerge(), + "details merger entries") + } else { + assert.Zero(t, deetsMerger.ItemsToMerge(), "details merger entries") + + details := deets.Details().Entries + assert.Len( + t, + details, + // 47 file and 2 folder entries. + test.expectedUploadedFiles+test.expectedCachedFiles+2, + ) + + for _, entry := range details { + test.deetsUpdated(t, entry.Updated) + } } checkSnapshotTags( t, - suite.ctx, + ctx, suite.w.c, expectedTags, stats.SnapshotID, ) snap, err := snapshot.LoadSnapshot( - suite.ctx, + ctx, suite.w.c, manifest.ID(stats.SnapshotID), ) require.NoError(t, err, clues.ToCore(err)) - prevSnaps = append(prevSnaps, IncrementalBase{ + res = ManifestEntry{ Manifest: snap, - SubtreePaths: []*path.Builder{ - suite.storePath1.ToBuilder().Dir(), - }, - }) + Reasons: reasons, + } }) + + return res + } + + base := runAndTestBackup(baseBackupCase, ManifestEntry{}) + + table := []testCase{ + { + name: "Kopia Assist And Merge All Files Changed", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base) + }, + collections: collections, + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Kopia Assist And Merge No Files Changed", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base) + }, + // Pass in empty collections to force a backup. Otherwise we'll skip + // actually trying to do anything because we'll see there's nothing that + // changed. The real goal is to get it to deal with the merged collections + // again though. + collections: emptyCollections, + // Should hit cached check prior to dir entry check so we see them as + // cached. + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + // Entries go into the details merger because we never materialize details + // info for the items since they're from the base. + expectMerge: true, + // Not used since there's no details entries. + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Kopia Assist Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithAssistBases(base) + }, + collections: collections, + expectedUploadedFiles: 0, + expectedCachedFiles: 47, + deetsUpdated: assert.False, + hashedBytesCheck: assert.Zero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Merge Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases().WithMergeBases(base).ClearMockAssistBases() + }, + // Pass in empty collections to force a backup. Otherwise we'll skip + // actually trying to do anything because we'll see there's nothing that + // changed. The real goal is to get it to deal with the merged collections + // again though. + collections: emptyCollections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + expectMerge: true, + // Not used since there's no details entries. + deetsUpdated: assert.False, + // Kopia still counts these bytes as "hashed" even though it shouldn't + // read the file data since they already have dir entries it can reuse. + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{4000, 6000}, + }, + { + name: "Content Hash Only", + baseBackups: func(base ManifestEntry) BackupBases { + return NewMockBackupBases() + }, + collections: collections, + expectedUploadedFiles: 47, + expectedCachedFiles: 0, + // Marked as updated because we still fall into the uploadFile handler in + // kopia instead of the cachedFile handler. + deetsUpdated: assert.True, + hashedBytesCheck: assert.NotZero, + uploadedBytes: []int64{4000, 6000}, + }, + } + + for _, test := range table { + runAndTestBackup(test, base) } } @@ -837,23 +993,25 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { "brunhilda": "", } - reasons := []Reason{ - { - ResourceOwner: storePath.ResourceOwner(), - Service: storePath.Service(), - Category: storePath.Category(), - }, - } - - for _, r := range reasons { - for _, k := range r.TagKeys() { - tags[k] = "" - } + reasons := []Reasoner{ + NewReason( + testTenant, + storePath.ResourceOwner(), + storePath.Service(), + storePath.Category()), } expectedTags := map[string]string{} - maps.Copy(expectedTags, normalizeTagKVs(tags)) + maps.Copy(expectedTags, tags) + + for _, r := range reasons { + for _, k := range tagKeys(r) { + expectedTags[k] = "" + } + } + + expectedTags = normalizeTagKVs(expectedTags) table := []struct { name string @@ -931,7 +1089,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { }, } - prevSnaps := []IncrementalBase{} + prevSnaps := NewMockBackupBases() for _, test := range table { suite.Run(test.name, func() { @@ -940,6 +1098,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { stats, deets, prevShortRefs, err := suite.w.ConsumeBackupCollections( suite.ctx, + reasons, prevSnaps, collections, nil, @@ -992,12 +1151,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { manifest.ID(stats.SnapshotID)) require.NoError(t, err, clues.ToCore(err)) - prevSnaps = append(prevSnaps, IncrementalBase{ - Manifest: snap, - SubtreePaths: []*path.Builder{ - storePath.ToBuilder().Dir(), + prevSnaps.WithMergeBases( + ManifestEntry{ + Manifest: snap, + Reasons: reasons, }, - }) + ) }) } } @@ -1016,16 +1175,7 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { w := &Wrapper{k} - tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - - for _, k := range reason.TagKeys() { - tags[k] = "" - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1) dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1) @@ -1038,10 +1188,11 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { stats, _, _, err := w.ConsumeBackupCollections( ctx, + []Reasoner{r}, nil, []data.BackupCollection{dc1, dc2}, nil, - tags, + nil, true, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1112,16 +1263,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { loc1 := path.Builder{}.Append(suite.storePath1.Folders()...) loc2 := path.Builder{}.Append(suite.storePath2.Folders()...) - tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - - for _, k := range reason.TagKeys() { - tags[k] = "" - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) collections := []data.BackupCollection{ &mockBackupCollection{ @@ -1164,10 +1306,11 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + []Reasoner{r}, nil, collections, nil, - tags, + nil, true, fault.New(true)) require.Error(t, err, clues.ToCore(err)) @@ -1239,6 +1382,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections() s, d, _, err := suite.w.ConsumeBackupCollections( ctx, nil, + nil, test.collections, nil, nil, @@ -1391,23 +1535,15 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { collections = append(collections, collection) } - tags := map[string]string{} - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - - for _, k := range reason.TagKeys() { - tags[k] = "" - } + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) stats, deets, _, err := suite.w.ConsumeBackupCollections( suite.ctx, + []Reasoner{r}, nil, collections, nil, - tags, + nil, false, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1437,32 +1573,11 @@ func (c *i64counter) Count(i int64) { } func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { - reason := Reason{ - ResourceOwner: testUser, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - - subtreePathTmp, err := path.Build( - testTenant, - testUser, - path.ExchangeService, - path.EmailCategory, - false, - "tmp") - require.NoError(suite.T(), err, clues.ToCore(err)) - - subtreePath := subtreePathTmp.ToBuilder().Dir() + r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory) man, err := suite.w.c.LoadSnapshot(suite.ctx, suite.snapshotID) require.NoError(suite.T(), err, "getting base snapshot: %v", clues.ToCore(err)) - tags := map[string]string{} - - for _, k := range reason.TagKeys() { - tags[k] = "" - } - table := []struct { name string excludeItem bool @@ -1551,17 +1666,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() { stats, _, _, err := suite.w.ConsumeBackupCollections( suite.ctx, - []IncrementalBase{ - { + []Reasoner{r}, + NewMockBackupBases().WithMergeBases( + ManifestEntry{ Manifest: man, - SubtreePaths: []*path.Builder{ - subtreePath, - }, + Reasons: []Reasoner{r}, }, - }, + ), test.cols(), excluded, - tags, + nil, true, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 50514e4ad..a5a89f134 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -2,7 +2,6 @@ package m365 import ( "context" - "strings" "github.com/alcionai/clues" @@ -44,7 +43,7 @@ func (ctrl *Controller) ProduceBackupCollections( ctx, end := diagnostics.Span( ctx, "m365:produceBackupCollections", - diagnostics.Index("service", sels.Service.String())) + diagnostics.Index("service", sels.PathService().String())) defer end() ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) @@ -61,8 +60,8 @@ func (ctrl *Controller) ProduceBackupCollections( serviceEnabled, canMakeDeltaQueries, err := checkServiceEnabled( ctx, ctrl.AC.Users(), - path.ServiceType(sels.Service), - sels.DiscreteOwner) + sels.PathService(), + owner.ID()) if err != nil { return nil, nil, false, err } @@ -194,10 +193,8 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error { ids = siteIDs } - resourceOwner := strings.ToLower(sels.DiscreteOwner) - - if !filters.Equal(ids).Compare(resourceOwner) { - return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_resource_owner", sels.DiscreteOwner) + if !filters.Contains(ids).Compare(sels.ID()) { + return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner) } return nil diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index bb59741f8..87c9c766d 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -57,7 +57,7 @@ func (suite *DataCollectionIntgSuite) SetupSuite() { suite.tenantID = creds.AzureTenantID - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -120,7 +120,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() { sel := test.getSelector(t) uidn := inMock.NewProvider(sel.ID(), sel.Name()) - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( @@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner() test.getSelector(t), nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.Error(t, err, clues.ToCore(err)) assert.False(t, canUsePreviousBackup, "can use previous backup") @@ -296,7 +296,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { nil, ctrl.credentials, ctrl, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -367,7 +367,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -381,7 +381,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -414,7 +414,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { siteIDs = []string{siteID} ) - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil) require.NoError(t, err, clues.ToCore(err)) sel := selectors.NewSharePointBackup(siteIDs) @@ -428,7 +428,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { sel.Selector, nil, version.NoBackup, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index b0c8792e5..174148a76 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -23,6 +24,7 @@ import ( var ( _ inject.BackupProducer = &Controller{} _ inject.RestoreConsumer = &Controller{} + _ inject.ExportConsumer = &Controller{} ) // Controller is a struct used to wrap the GraphServiceClient and @@ -47,6 +49,11 @@ type Controller struct { // mutex used to synchronize updates to `status` mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status + + // backupDriveIDNames is populated on restore. It maps the backup's + // drive names to their id. Primarily for use when creating or looking + // up a new drive. + backupDriveIDNames idname.CacheBuilder } func NewController( @@ -63,7 +70,7 @@ func NewController( return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx) } - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, co) if err != nil { return nil, clues.Wrap(err, "creating api client").WithClues(ctx) } @@ -77,10 +84,11 @@ func NewController( AC: ac, IDNameLookup: idname.NewCache(nil), - credentials: creds, - ownerLookup: rCli, - tenant: acct.ID(), - wg: &sync.WaitGroup{}, + credentials: creds, + ownerLookup: rCli, + tenant: acct.ID(), + wg: &sync.WaitGroup{}, + backupDriveIDNames: idname.NewCache(nil), } return &ctrl, nil @@ -142,6 +150,16 @@ func (ctrl *Controller) incrementAwaitingMessages() { ctrl.wg.Add(1) } +func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { + if dii.SharePoint != nil { + ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + } + + if dii.OneDrive != nil { + ctrl.backupDriveIDNames.Add(dii.OneDrive.DriveID, dii.OneDrive.DriveName) + } +} + // --------------------------------------------------------------------------- // Resource Lookup Handling // --------------------------------------------------------------------------- @@ -228,15 +246,15 @@ func (r resourceClient) getOwnerIDAndNameFrom( return id, name, nil } -// PopulateOwnerIDAndNamesFrom takes the provided owner identifier and produces +// PopulateProtectedResourceIDAndName takes the provided owner identifier and produces // the owner's name and ID from that value. Returns an error if the owner is // not recognized by the current tenant. // -// The id-name swapper is optional. Some processes will look up all owners in +// The id-name cacher is optional. Some processes will look up all owners in // the tenant before reaching this step. In that case, the data gets handed // down for this func to consume instead of performing further queries. The // data gets stored inside the controller instance for later re-use. -func (ctrl *Controller) PopulateOwnerIDAndNamesFrom( +func (ctrl *Controller) PopulateProtectedResourceIDAndName( ctx context.Context, owner string, // input value, can be either id or name ins idname.Cacher, diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 6d04b7e9e..f4ff3c032 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -12,16 +12,22 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/data" + dataMock "github.com/alcionai/corso/src/internal/data/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" @@ -220,7 +226,7 @@ func (suite *ControllerUnitSuite) TestPopulateOwnerIDAndNamesFrom() { ctrl := &Controller{ownerLookup: test.rc} - rID, rName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, test.owner, test.ins) + rID, rName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, test.owner, test.ins) test.expectErr(t, err, clues.ToCore(err)) assert.Equal(t, test.expectID, rID, "id") assert.Equal(t, test.expectName, rName, "name") @@ -260,6 +266,82 @@ func (suite *ControllerUnitSuite) TestController_Wait() { assert.Equal(t, int64(4), result.Bytes) } +func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { + var ( + odid = "od-id" + odname = "od-name" + spid = "sp-id" + spname = "sp-name" + // intentionally declared outside the test loop + ctrl = &Controller{ + wg: &sync.WaitGroup{}, + region: &trace.Region{}, + backupDriveIDNames: idname.NewCache(nil), + } + ) + + table := []struct { + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectID string + expectName string + }{ + { + name: "exchange", + dii: details.ItemInfo{ + Exchange: &details.ExchangeInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "folder", + dii: details.ItemInfo{ + Folder: &details.FolderInfo{}, + }, + expectID: "", + expectName: "", + }, + { + name: "onedrive", + dii: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + DriveID: odid, + DriveName: odname, + }, + }, + expectID: odid, + expectName: odname, + }, + { + name: "sharepoint", + dii: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + DriveID: spid, + DriveName: spname, + }, + }, + expectID: spid, + expectName: spname, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctrl.CacheItemInfo(test.dii) + + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) + assert.Equal(t, test.expectName, name) + + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) + assert.Equal(t, test.expectID, id) + }) + } +} + // --------------------------------------------------------------------------- // Integration tests // --------------------------------------------------------------------------- @@ -306,20 +388,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { } ) + restoreCfg.IncludePermissions = true + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - nil, + rcc, + []data.RestoreCollection{&dataMock.Collection{}}, fault.New(true), count.New()) - assert.Error(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) + assert.Error(t, err, graph.ErrServiceNotEnabled, clues.ToCore(err)) + assert.Nil(t, deets) status := suite.ctrl.Wait() assert.Equal(t, 0, status.Objects) @@ -329,6 +415,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() { func (suite *ControllerIntegrationSuite) TestEmptyCollections() { restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + table := []struct { name string col []data.RestoreCollection @@ -385,25 +473,22 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() { ctx, flush := tester.NewContext(t) defer flush() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: test.sel, + RestoreConfig: restoreCfg, + Selector: test.sel, + } + deets, err := suite.ctrl.ConsumeRestoreCollections( ctx, - version.Backup, - test.sel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, test.col, fault.New(true), count.New()) - require.NoError(t, err, clues.ToCore(err)) - assert.NotNil(t, deets) - - stats := suite.ctrl.Wait() - assert.Zero(t, stats.Objects) - assert.Zero(t, stats.Folders) - assert.Zero(t, stats.Successes) + require.Error(t, err, clues.ToCore(err)) + assert.Nil(t, deets) }) } } @@ -429,12 +514,18 @@ func runRestore( restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService) restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: sci.RestoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - backupVersion, - restoreSel, - sci.RestoreCfg, - sci.Opts, + rcc, collections, fault.New(true), count.New()) @@ -536,6 +627,7 @@ func runRestoreBackupTest( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -546,7 +638,7 @@ func runRestoreBackupTest( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( @@ -581,6 +673,7 @@ func runRestoreTestWithVersion( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -591,7 +684,7 @@ func runRestoreTestWithVersion( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -618,6 +711,7 @@ func runRestoreBackupTestVersions( tenant string, resourceOwners []string, opts control.Options, + restoreCfg control.RestoreConfig, ) { ctx, flush := tester.NewContext(t) defer flush() @@ -628,7 +722,7 @@ func runRestoreBackupTestVersions( Service: test.service, Tenant: tenant, ResourceOwners: resourceOwners, - RestoreCfg: testdata.DefaultRestoreConfig(""), + RestoreCfg: restoreCfg, } totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( @@ -662,7 +756,7 @@ func runRestoreBackupTestVersions( test.collectionsLatest) } -func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { +func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_core() { bodyText := "This email has some text. However, all the text is on the same line." subjectText := "Test message for restore" @@ -921,10 +1015,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)) }) } } @@ -1005,6 +1097,8 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { for i, collection := range test.collections { // Get a restoreCfg per collection so they're independent. restoreCfg := testdata.DefaultRestoreConfig("") + restoreCfg.IncludePermissions = true + expectedDests = append(expectedDests, destAndCats{ resourceOwner: suite.user, dest: restoreCfg.Location, @@ -1037,15 +1131,18 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { ) restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService) + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + ProtectedResource: restoreSel, + RestoreConfig: restoreCfg, + Selector: restoreSel, + } + deets, err := restoreCtrl.ConsumeRestoreCollections( ctx, - version.Backup, - restoreSel, - restoreCfg, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + rcc, collections, fault.New(true), count.New()) @@ -1077,10 +1174,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") @@ -1089,10 +1183,13 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() { t.Log("Backup enumeration complete") + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + ci := stub.ConfigInfo{ - Opts: control.Options{RestorePermissions: true}, + Opts: control.DefaultOptions(), // Alright to be empty, needed for OneDrive. - RestoreCfg: control.RestoreConfig{}, + RestoreCfg: restoreCfg, } // Pull the data prior to waiting for the status as otherwise it will @@ -1130,16 +1227,16 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen }, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreBackupTest( suite.T(), test, suite.ctrl.tenant, []string{suite.user}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) } func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { @@ -1158,8 +1255,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { sel.Include( sel.ContactFolders([]string{selectors.NoneTgt}), sel.EventCalendars([]string{selectors.NoneTgt}), - sel.MailFolders([]string{selectors.NoneTgt}), - ) + sel.MailFolders([]string{selectors.NoneTgt})) return sel.Selector }, @@ -1222,23 +1318,20 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { start = time.Now() ) - id, name, err := backupCtrl.PopulateOwnerIDAndNamesFrom(ctx, backupSel.DiscreteOwner, nil) + id, name, err := backupCtrl.PopulateProtectedResourceIDAndName(ctx, backupSel.DiscreteOwner, nil) require.NoError(t, err, clues.ToCore(err)) backupSel.SetDiscreteOwnerIDName(id, name) dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( ctx, - inMock.NewProvider(id, name), + idname.NewProvider(id, name), backupSel, nil, version.NoBackup, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }, + control.DefaultOptions(), fault.New(true)) - require.NoError(t, err) + require.NoError(t, err, clues.ToCore(err)) assert.True(t, canUsePreviousBackup, "can use previous backup") // No excludes yet because this isn't an incremental backup. assert.True(t, excludes.Empty()) diff --git a/src/internal/m365/exchange/backup_test.go b/src/internal/m365/exchange/backup_test.go index 8ac8c14dd..fa4b87d9d 100644 --- a/src/internal/m365/exchange/backup_test.go +++ b/src/internal/m365/exchange/backup_test.go @@ -414,7 +414,7 @@ func (suite *BackupIntgSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.tenantID = creds.AzureTenantID @@ -466,7 +466,7 @@ func (suite *BackupIntgSuite) TestMailFetch() { ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries collections, err := createCollections( @@ -554,7 +554,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -587,7 +587,7 @@ func (suite *BackupIntgSuite) TestDelta() { inMock.NewProvider(userID, userID), test.scope, dps, - control.Defaults(), + control.DefaultOptions(), func(status *support.ControllerOperationStatus) {}, fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -633,7 +633,7 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() { inMock.NewProvider(suite.user, suite.user), sel.Scopes()[0], DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -709,7 +709,7 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -834,7 +834,7 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() { inMock.NewProvider(suite.user, suite.user), test.scope, DeltaPaths{}, - control.Defaults(), + control.DefaultOptions(), newStatusUpdater(t, &wg), fault.New(true)) require.NoError(t, err, clues.ToCore(err)) @@ -1995,7 +1995,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i ctx, flush := tester.NewContext(t) defer flush() - ctrlOpts := control.Defaults() + ctrlOpts := control.DefaultOptions() ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter getter := test.getter diff --git a/src/internal/m365/exchange/collection_test.go b/src/internal/m365/exchange/collection_test.go index 2c023d703..7c5a4adab 100644 --- a/src/internal/m365/exchange/collection_test.go +++ b/src/internal/m365/exchange/collection_test.go @@ -178,7 +178,7 @@ func (suite *CollectionSuite) TestNewCollection_state() { test.curr, test.prev, test.loc, 0, &mockItemer{}, nil, - control.Defaults(), + control.DefaultOptions(), false) assert.Equal(t, test.expect, c.State(), "collection state") assert.Equal(t, test.curr, c.fullPath, "full path") diff --git a/src/internal/m365/exchange/container_resolver_test.go b/src/internal/m365/exchange/container_resolver_test.go index 8b5fa7c95..b2ff30830 100644 --- a/src/internal/m365/exchange/container_resolver_test.go +++ b/src/internal/m365/exchange/container_resolver_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -698,7 +699,7 @@ func (suite *ContainerResolverSuite) SetupSuite() { } func (suite *ContainerResolverSuite) TestPopulate() { - ac, err := api.NewClient(suite.credentials) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(suite.T(), err, clues.ToCore(err)) eventFunc := func(t *testing.T) graph.ContainerResolver { diff --git a/src/internal/m365/exchange/helper_test.go b/src/internal/m365/exchange/helper_test.go index 7e604c466..9b1583b9c 100644 --- a/src/internal/m365/exchange/helper_test.go +++ b/src/internal/m365/exchange/helper_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -30,7 +31,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.creds = creds - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.userID = tconfig.GetM365UserID(ctx) diff --git a/src/internal/m365/exchange/mail_container_cache_test.go b/src/internal/m365/exchange/mail_container_cache_test.go index b95a9a170..de0694749 100644 --- a/src/internal/m365/exchange/mail_container_cache_test.go +++ b/src/internal/m365/exchange/mail_container_cache_test.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -83,7 +84,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { ctx, flush := tester.NewContext(t) defer flush() - ac, err := api.NewClient(suite.credentials) + ac, err := api.NewClient(suite.credentials, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) acm := ac.Mail() diff --git a/src/internal/m365/exchange/restore.go b/src/internal/m365/exchange/restore.go index 5a5dbfcbc..7871f68d4 100644 --- a/src/internal/m365/exchange/restore.go +++ b/src/internal/m365/exchange/restore.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -28,7 +29,7 @@ import ( func ConsumeRestoreCollections( ctx context.Context, ac api.Client, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, @@ -39,16 +40,13 @@ func ConsumeRestoreCollections( } var ( - userID = dcs[0].FullPath().ResourceOwner() + resourceID = rcc.ProtectedResource.ID() directoryCache = make(map[path.CategoryType]graph.ContainerResolver) handlers = restoreHandlers(ac) metrics support.CollectionMetrics el = errs.Local() ) - // FIXME: should be user name - ctx = clues.Add(ctx, "resource_owner", clues.Hide(userID)) - for _, dc := range dcs { if el.Failure() != nil { break @@ -69,7 +67,7 @@ func ConsumeRestoreCollections( } if directoryCache[category] == nil { - gcr := handler.newContainerCache(userID) + gcr := handler.newContainerCache(resourceID) if err := gcr.Populate(ctx, errs, handler.defaultRootContainer()); err != nil { return nil, clues.Wrap(err, "populating container cache") } @@ -80,8 +78,8 @@ func ConsumeRestoreCollections( containerID, gcc, err := createDestination( ictx, handler, - handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()), - userID, + handler.formatRestoreDestination(rcc.RestoreConfig.Location, dc.FullPath()), + resourceID, directoryCache[category], errs) if err != nil { @@ -92,7 +90,7 @@ func ConsumeRestoreCollections( directoryCache[category] = gcc ictx = clues.Add(ictx, "restore_destination_id", containerID) - collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, userID, containerID) + collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, resourceID, containerID) if err != nil { el.AddRecoverable(ctx, clues.Wrap(err, "building item collision cache")) continue @@ -102,10 +100,10 @@ func ConsumeRestoreCollections( ictx, handler, dc, - userID, + resourceID, containerID, collisionKeyToItemID, - restoreCfg.OnCollision, + rcc.RestoreConfig.OnCollision, deets, errs, ctr) @@ -126,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), metrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -136,7 +134,7 @@ func restoreCollection( ctx context.Context, ir itemRestorer, dc data.RestoreCollection, - userID, destinationID string, + resourceID, destinationID string, collisionKeyToItemID map[string]string, collisionPolicy control.CollisionPolicy, deets *details.Builder, @@ -187,7 +185,7 @@ func restoreCollection( info, err := ir.restore( ictx, body, - userID, + resourceID, destinationID, collisionKeyToItemID, collisionPolicy, @@ -240,7 +238,7 @@ func createDestination( ctx context.Context, ca containerAPI, destination *path.Builder, - userID string, + resourceID string, gcr graph.ContainerResolver, errs *fault.Bus, ) (string, graph.ContainerResolver, error) { @@ -264,7 +262,7 @@ func createDestination( ca, cache, restoreLoc, - userID, + resourceID, containerParentID, container, errs) @@ -285,7 +283,7 @@ func getOrPopulateContainer( ca containerAPI, gcr graph.ContainerResolver, restoreLoc *path.Builder, - userID, containerParentID, containerName string, + resourceID, containerParentID, containerName string, errs *fault.Bus, ) (string, error) { cached, ok := gcr.LocationInCache(restoreLoc.String()) @@ -293,7 +291,7 @@ func getOrPopulateContainer( return cached, nil } - c, err := ca.CreateContainer(ctx, userID, containerParentID, containerName) + c, err := ca.CreateContainer(ctx, resourceID, containerParentID, containerName) // 409 handling case: // attempt to fetch the container by name and add that result to the cache. @@ -301,7 +299,7 @@ func getOrPopulateContainer( // sometimes the backend will create the folder despite the 5xx response, // leaving our local containerResolver with inconsistent state. if graph.IsErrFolderExists(err) { - cc, e := ca.GetContainerByName(ctx, userID, containerParentID, containerName) + cc, e := ca.GetContainerByName(ctx, resourceID, containerParentID, containerName) if e != nil { err = clues.Stack(err, e) } else { @@ -327,7 +325,7 @@ func uploadAttachments( ctx context.Context, ap attachmentPoster, as []models.Attachmentable, - userID, destinationID, itemID string, + resourceID, destinationID, itemID string, errs *fault.Bus, ) error { el := errs.Local() @@ -340,7 +338,7 @@ func uploadAttachments( err := uploadAttachment( ctx, ap, - userID, + resourceID, destinationID, itemID, a) diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/exchange/restore_test.go index 4d91329e9..a30d56dd0 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/exchange/restore_test.go @@ -44,7 +44,7 @@ func (suite *RestoreIntgSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) suite.credentials = m365 - suite.ac, err = api.NewClient(m365) + suite.ac, err = api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go new file mode 100644 index 000000000..4da037e26 --- /dev/null +++ b/src/internal/m365/export.go @@ -0,0 +1,62 @@ +package m365 + +import ( + "context" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/onedrive" + "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// ProduceExportCollections exports data from the specified collections +func (ctrl *Controller) ProduceExportCollections( + ctx context.Context, + backupVersion int, + sels selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, +) ([]export.Collection, error) { + ctx, end := diagnostics.Span(ctx, "m365:export") + defer end() + + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) + ctx = clues.Add(ctx, "export_config", exportCfg) // TODO(meain): needs PII control + + var ( + expCollections []export.Collection + status *support.ControllerOperationStatus + deets = &details.Builder{} + err error + ) + + switch sels.Service { + case selectors.ServiceOneDrive, selectors.ServiceSharePoint: + // OneDrive and SharePoint can share the code to create collections + expCollections, err = onedrive.ProduceExportCollections( + ctx, + backupVersion, + exportCfg, + opts, + dcs, + deets, + errs) + default: + err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + } + + ctrl.incrementAwaitingMessages() + ctrl.UpdateStatus(status) + + return expCollections, err +} diff --git a/src/internal/m365/graph/errors.go b/src/internal/m365/graph/errors.go index f0df6b4ec..f35b91385 100644 --- a/src/internal/m365/graph/errors.go +++ b/src/internal/m365/graph/errors.go @@ -271,7 +271,9 @@ func Wrap(ctx context.Context, e error, msg string) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Wrap(e, msg).WithClues(ctx).With(data...), innerMsg) + ce := clues.Wrap(e, msg).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // Stack is a helper function that extracts ODataError metadata from @@ -292,7 +294,9 @@ func Stack(ctx context.Context, e error) *clues.Err { e = clues.Stack(e, clues.New(mainMsg)) } - return setLabels(clues.Stack(e).WithClues(ctx).With(data...), innerMsg) + ce := clues.Stack(e).WithClues(ctx).With(data...).WithTrace(1) + + return setLabels(ce, innerMsg) } // stackReq is a helper function that extracts ODataError metadata from @@ -361,7 +365,7 @@ func errData(err odataerrors.ODataErrorable) (string, []any, string) { msgConcat += ptr.Val(d.GetMessage()) } - inner := mainErr.GetInnererror() + inner := mainErr.GetInnerError() if inner != nil { data = appendIf(data, "odataerror_inner_cli_req_id", inner.GetClientRequestId()) data = appendIf(data, "odataerror_inner_req_id", inner.GetRequestId()) diff --git a/src/internal/m365/graph/http_wrapper_test.go b/src/internal/m365/graph/http_wrapper_test.go index 594eb75cd..19711edc4 100644 --- a/src/internal/m365/graph/http_wrapper_test.go +++ b/src/internal/m365/graph/http_wrapper_test.go @@ -93,7 +93,7 @@ func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() { hdr.Set("Location", "localhost:99999999/smarfs") toResp := &http.Response{ - StatusCode: 302, + StatusCode: http.StatusFound, Header: hdr, } diff --git a/src/internal/m365/helper_test.go b/src/internal/m365/helper_test.go index 25f4bb18c..a3f1d7d5e 100644 --- a/src/internal/m365/helper_test.go +++ b/src/internal/m365/helper_test.go @@ -796,8 +796,8 @@ func compareDriveItem( assert.Equal(t, expectedMeta.FileName, itemMeta.FileName) } - if !mci.Opts.RestorePermissions { - assert.Equal(t, 0, len(itemMeta.Permissions)) + if !mci.RestoreCfg.IncludePermissions { + assert.Empty(t, itemMeta.Permissions, "no permissions should be included in restore") return true } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index 05cb8e159..5510d2f9a 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -26,6 +27,10 @@ type Controller struct { Err error Stats data.CollectionStats + + ProtectedResourceID string + ProtectedResourceName string + ProtectedResourceErr error } func (ctrl Controller) ProduceBackupCollections( @@ -59,13 +64,34 @@ func (ctrl Controller) Wait() *data.CollectionStats { func (ctrl Controller) ConsumeRestoreCollections( _ context.Context, - _ int, - _ selectors.Selector, - _ control.RestoreConfig, - _ control.Options, + _ inject.RestoreConsumerConfig, _ []data.RestoreCollection, _ *fault.Bus, _ *count.Bus, ) (*details.Details, error) { return ctrl.Deets, ctrl.Err } + +func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} + +func (ctrl Controller) ProduceExportCollections( + _ context.Context, + _ int, + _ selectors.Selector, + _ control.ExportConfig, + _ control.Options, + _ []data.RestoreCollection, + _ *fault.Bus, +) ([]export.Collection, error) { + return nil, ctrl.Err +} + +func (ctrl Controller) PopulateProtectedResourceIDAndName( + ctx context.Context, + protectedResource string, // input value, can be either id or name + ins idname.Cacher, +) (string, string, error) { + return ctrl.ProtectedResourceID, + ctrl.ProtectedResourceName, + ctrl.ProtectedResourceErr +} diff --git a/src/internal/m365/onedrive/collection_test.go b/src/internal/m365/onedrive/collection_test.go index bcd4da4b6..3c30cac22 100644 --- a/src/internal/m365/onedrive/collection_test.go +++ b/src/internal/m365/onedrive/collection_test.go @@ -945,7 +945,7 @@ func (suite *CollectionUnitTestSuite) TestItemExtensions() { nil, } - opts := control.Defaults() + opts := control.DefaultOptions() opts.ItemExtensionFactory = append( opts.ItemExtensionFactory, test.factories...) diff --git a/src/internal/m365/onedrive/export.go b/src/internal/m365/onedrive/export.go new file mode 100644 index 000000000..9868a9b71 --- /dev/null +++ b/src/internal/m365/onedrive/export.go @@ -0,0 +1,166 @@ +package onedrive + +import ( + "context" + "strings" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +var _ export.Collection = &exportCollection{} + +// exportCollection is the implementation of export.ExportCollection for OneDrive +type exportCollection struct { + // baseDir contains the path of the collection + baseDir string + + // backingCollection is the restore collection from which we will + // create the export collection. + backingCollection data.RestoreCollection + + // backupVersion is the backupVersion of the backup this collection was part + // of. This is required to figure out how to get the name of the + // item. + backupVersion int +} + +func (ec exportCollection) BasePath() string { + return ec.baseDir +} + +func (ec exportCollection) Items(ctx context.Context) <-chan export.Item { + ch := make(chan export.Item) + go items(ctx, ec, ch) + + return ch +} + +// items converts items in backing collection to export items +func items(ctx context.Context, ec exportCollection, ch chan<- export.Item) { + defer close(ch) + + errs := fault.New(false) + + // There will only be a single item in the backingCollections + // for OneDrive + for item := range ec.backingCollection.Items(ctx, errs) { + itemUUID := item.UUID() + if isMetadataFile(itemUUID, ec.backupVersion) { + continue + } + + name, err := getItemName(ctx, itemUUID, ec.backupVersion, ec.backingCollection) + + ch <- export.Item{ + ID: itemUUID, + Data: export.ItemData{ + Name: name, + Body: item.ToReader(), + }, + Error: err, + } + } + + eitems, erecovereable := errs.ItemsAndRecovered() + + // Return all the items that we failed to get from kopia at the end + for _, err := range eitems { + ch <- export.Item{ + ID: err.ID, + Error: &err, + } + } + + for _, ec := range erecovereable { + ch <- export.Item{ + Error: ec, + } + } +} + +// isMetadataFile is used to determine if a path corresponds to a +// metadata file. This is OneDrive specific logic and depends on the +// version of the backup unlike metadata.IsMetadataFile which only has +// to be concerned about the current version. +func isMetadataFile(id string, backupVersion int) bool { + if backupVersion < version.OneDrive1DataAndMetaFiles { + return false + } + + return strings.HasSuffix(id, metadata.MetaFileSuffix) || + strings.HasSuffix(id, metadata.DirMetaFileSuffix) +} + +// getItemName is used to get the name of the item. +// How we get the name depends on the version of the backup. +func getItemName( + ctx context.Context, + id string, + backupVersion int, + fin data.FetchItemByNamer, +) (string, error) { + if backupVersion < version.OneDrive1DataAndMetaFiles { + return id, nil + } + + if backupVersion < version.OneDrive5DirMetaNoName { + return strings.TrimSuffix(id, metadata.DataFileSuffix), nil + } + + if strings.HasSuffix(id, metadata.DataFileSuffix) { + trimmedName := strings.TrimSuffix(id, metadata.DataFileSuffix) + metaName := trimmedName + metadata.MetaFileSuffix + + meta, err := fetchAndReadMetadata(ctx, fin, metaName) + if err != nil { + return "", clues.Wrap(err, "getting metadata").WithClues(ctx) + } + + return meta.FileName, nil + } + + return "", clues.New("invalid item id").WithClues(ctx) +} + +// ProduceExportCollections will create the export collections for the +// given restore collections. +func ProduceExportCollections( + ctx context.Context, + backupVersion int, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + deets *details.Builder, + errs *fault.Bus, +) ([]export.Collection, error) { + var ( + el = errs.Local() + ec = make([]export.Collection, 0, len(dcs)) + ) + + for _, dc := range dcs { + drivePath, err := path.ToDrivePath(dc.FullPath()) + if err != nil { + return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) + } + + baseDir := path.Builder{}.Append(drivePath.Folders...) + + ec = append(ec, exportCollection{ + baseDir: baseDir.String(), + backingCollection: dc, + backupVersion: backupVersion, + }) + } + + return ec, el.Failure() +} diff --git a/src/internal/m365/onedrive/export_test.go b/src/internal/m365/onedrive/export_test.go new file mode 100644 index 000000000..ce707885f --- /dev/null +++ b/src/internal/m365/onedrive/export_test.go @@ -0,0 +1,463 @@ +package onedrive + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/data" + odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/path" +) + +type ExportUnitSuite struct { + tester.Suite +} + +func TestExportUnitSuite(t *testing.T) { + suite.Run(t, &ExportUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportUnitSuite) TestIsMetadataFile() { + table := []struct { + name string + id string + backupVersion int + isMeta bool + }{ + { + name: "legacy", + backupVersion: version.OneDrive1DataAndMetaFiles, + isMeta: false, + }, + { + name: "metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.MetaFileSuffix, + isMeta: true, + }, + { + name: "dir metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.DirMetaFileSuffix, + isMeta: true, + }, + { + name: "non metadata file", + backupVersion: version.OneDrive3IsMetaMarker, + id: "name" + metadata.DataFileSuffix, + isMeta: false, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + assert.Equal(suite.T(), test.isMeta, isMetadataFile(test.id, test.backupVersion), "is metadata") + }) + } +} + +type metadataStream struct { + id string + name string +} + +func (ms metadataStream) ToReader() io.ReadCloser { + return io.NopCloser(bytes.NewBufferString(`{"filename": "` + ms.name + `"}`)) +} +func (ms metadataStream) UUID() string { return ms.id } +func (ms metadataStream) Deleted() bool { return false } + +type finD struct { + id string + name string + err error +} + +func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Stream, error) { + if fd.err != nil { + return nil, fd.err + } + + if name == fd.id { + return metadataStream{id: fd.id, name: fd.name}, nil + } + + return nil, assert.AnError +} + +func (suite *ExportUnitSuite) TestGetItemName() { + table := []struct { + tname string + id string + backupVersion int + name string + fin data.FetchItemByNamer + errFunc assert.ErrorAssertionFunc + }{ + { + tname: "legacy", + id: "name", + backupVersion: version.OneDrive1DataAndMetaFiles, + name: "name", + errFunc: assert.NoError, + }, + { + tname: "name in filename", + id: "name.data", + backupVersion: version.OneDrive4DirIncludesPermissions, + name: "name", + errFunc: assert.NoError, + }, + { + tname: "name in metadata", + id: "id.data", + backupVersion: version.Backup, + name: "name", + fin: finD{id: "id.meta", name: "name"}, + errFunc: assert.NoError, + }, + { + tname: "name in metadata but error", + id: "id.data", + backupVersion: version.Backup, + name: "", + fin: finD{err: assert.AnError}, + errFunc: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.tname, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + name, err := getItemName( + ctx, + test.id, + test.backupVersion, + test.fin, + ) + test.errFunc(t, err) + + assert.Equal(t, test.name, name, "name") + }) + } +} + +type mockRestoreCollection struct { + path path.Path + items []mockDataStream +} + +func (rc mockRestoreCollection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream { + ch := make(chan data.Stream) + + go func() { + defer close(ch) + + el := errs.Local() + + for _, item := range rc.items { + if item.err != nil { + el.AddRecoverable(ctx, item.err) + continue + } + + ch <- item + } + }() + + return ch +} + +func (rc mockRestoreCollection) FullPath() path.Path { + return rc.path +} + +type mockDataStream struct { + id string + data string + err error +} + +func (ms mockDataStream) ToReader() io.ReadCloser { + if ms.data != "" { + return io.NopCloser(bytes.NewBufferString(ms.data)) + } + + return nil +} +func (ms mockDataStream) UUID() string { return ms.id } +func (ms mockDataStream) Deleted() bool { return false } + +func (suite *ExportUnitSuite) TestGetItems() { + table := []struct { + name string + version int + backingCollection data.RestoreCollection + expectedItems []export.Item + }{ + { + name: "single item", + version: 1, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1", data: "body1"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "multiple items", + version: 1, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1", data: "body1"}, + {id: "name2", data: "body2"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + { + ID: "name2", + Data: export.ItemData{ + Name: "name2", + Body: io.NopCloser((bytes.NewBufferString("body2"))), + }, + }, + }, + }, + { + name: "single item with data suffix", + version: 2, + backingCollection: data.NoFetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name1.data", data: "body1"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "single item name from metadata", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + expectedItems: []export.Item{ + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + }, + { + name: "single item name from metadata with error", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "id1.data"}, + }, + }, + FetchItemByNamer: finD{err: assert.AnError}, + }, + expectedItems: []export.Item{ + { + ID: "id1.data", + Error: assert.AnError, + }, + }, + }, + { + name: "items with success and metadata read error", + version: version.Backup, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "missing.data"}, + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + expectedItems: []export.Item{ + { + ID: "missing.data", + Error: assert.AnError, + }, + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser(bytes.NewBufferString("body1")), + }, + }, + }, + }, + { + name: "items with success and fetch error", + version: version.OneDrive1DataAndMetaFiles, + backingCollection: data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + items: []mockDataStream{ + {id: "name0", data: "body0"}, + {id: "name1", err: assert.AnError}, + {id: "name2", data: "body2"}, + }, + }, + }, + expectedItems: []export.Item{ + { + ID: "name0", + Data: export.ItemData{ + Name: "name0", + Body: io.NopCloser(bytes.NewBufferString("body0")), + }, + }, + { + ID: "name2", + Data: export.ItemData{ + Name: "name2", + Body: io.NopCloser(bytes.NewBufferString("body2")), + }, + }, + { + ID: "", + Error: assert.AnError, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ec := exportCollection{ + baseDir: "", + backingCollection: test.backingCollection, + backupVersion: test.version, + } + + items := ec.Items(ctx) + + fitems := []export.Item{} + for item := range items { + fitems = append(fitems, item) + } + + assert.Len(t, fitems, len(test.expectedItems), "num of items") + + // We do not have any grantees about the ordering of the + // items in the SDK, but leaving the test this way for now + // to simplify testing. + for i, item := range fitems { + assert.Equal(t, test.expectedItems[i].ID, item.ID, "id") + assert.Equal(t, test.expectedItems[i].Data.Name, item.Data.Name, "name") + assert.Equal(t, test.expectedItems[i].Data.Body, item.Data.Body, "body") + assert.ErrorIs(t, item.Error, test.expectedItems[i].Error) + } + }) + } +} + +func (suite *ExportUnitSuite) TestExportRestoreCollections() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + dpb := odConsts.DriveFolderPrefixBuilder("driveID1") + + p, err := dpb.ToDataLayerOneDrivePath("t", "u", false) + assert.NoError(t, err, "build path") + + dcs := []data.RestoreCollection{ + data.FetchRestoreCollection{ + Collection: mockRestoreCollection{ + path: p, + items: []mockDataStream{ + {id: "id1.data", data: "body1"}, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, + }, + } + + expectedItems := []export.Item{ + { + ID: "id1.data", + Data: export.ItemData{ + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + } + + exportCfg := control.ExportConfig{} + ecs, err := ProduceExportCollections(ctx, int(version.Backup), exportCfg, control.Options{}, dcs, nil, fault.New(true)) + assert.NoError(t, err, "export collections error") + + assert.Len(t, ecs, 1, "num of collections") + + items := ecs[0].Items(ctx) + + fitems := []export.Item{} + for item := range items { + fitems = append(fitems, item) + } + + assert.Equal(t, expectedItems, fitems, "items") +} diff --git a/src/internal/m365/onedrive/handlers.go b/src/internal/m365/onedrive/handlers.go index dfea5ee17..cb33b373d 100644 --- a/src/internal/m365/onedrive/handlers.go +++ b/src/internal/m365/onedrive/handlers.go @@ -35,6 +35,7 @@ type BackupHandler interface { api.Getter GetItemPermissioner GetItemer + NewDrivePagerer // PathPrefix constructs the service and category specific path prefix for // the given values. @@ -49,7 +50,6 @@ type BackupHandler interface { // ServiceCat returns the service and category used by this implementation. ServiceCat() (path.ServiceType, path.CategoryType) - NewDrivePager(resourceOwner string, fields []string) api.DrivePager NewItemPager(driveID, link string, fields []string) api.DriveItemDeltaEnumerator // FormatDisplayPath creates a human-readable string to represent the // provided path. @@ -61,6 +61,10 @@ type BackupHandler interface { IncludesDir(dir string) bool } +type NewDrivePagerer interface { + NewDrivePager(resourceOwner string, fields []string) api.DrivePager +} + type GetItemPermissioner interface { GetItemPermission( ctx context.Context, @@ -86,7 +90,9 @@ type RestoreHandler interface { GetItemsByCollisionKeyser GetRootFolderer ItemInfoAugmenter + NewDrivePagerer NewItemContentUploader + PostDriver PostItemInContainerer DeleteItemPermissioner UpdateItemPermissioner @@ -145,6 +151,13 @@ type UpdateItemLinkSharer interface { ) (models.Permissionable, error) } +type PostDriver interface { + PostDrive( + ctx context.Context, + protectedResourceID, driveName string, + ) (models.Driveable, error) +} + type PostItemInContainerer interface { PostItemInContainer( ctx context.Context, diff --git a/src/internal/m365/onedrive/item_collector_test.go b/src/internal/m365/onedrive/item_collector_test.go index 6e4a79be6..6078517c7 100644 --- a/src/internal/m365/onedrive/item_collector_test.go +++ b/src/internal/m365/onedrive/item_collector_test.go @@ -313,7 +313,7 @@ func (suite *OneDriveIntgSuite) SetupSuite() { suite.creds = creds - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -361,8 +361,8 @@ func (suite *OneDriveIntgSuite) TestCreateGetDeleteFolder() { Folders: folderElements, } - caches := NewRestoreCaches() - caches.DriveIDToRootFolderID[driveID] = ptr.Val(rootFolder.GetId()) + caches := NewRestoreCaches(nil) + caches.DriveIDToDriveInfo[driveID] = driveInfo{rootFolderID: ptr.Val(rootFolder.GetId())} rh := NewRestoreHandler(suite.ac) diff --git a/src/internal/m365/onedrive/item_handler.go b/src/internal/m365/onedrive/item_handler.go index 0b1420cf0..64701da8f 100644 --- a/src/internal/m365/onedrive/item_handler.go +++ b/src/internal/m365/onedrive/item_handler.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -133,6 +134,19 @@ func NewRestoreHandler(ac api.Client) *itemRestoreHandler { return &itemRestoreHandler{ac.Drives()} } +func (h itemRestoreHandler) PostDrive( + context.Context, + string, string, +) (models.Driveable, error) { + return nil, clues.New("creating drives in oneDrive is not supported") +} + +func (h itemRestoreHandler) NewDrivePager( + resourceOwner string, fields []string, +) api.DrivePager { + return h.ac.NewUserDrivePager(resourceOwner, fields) +} + // AugmentItemInfo will populate a details.OneDriveInfo struct // with properties from the drive item. ItemSize is specified // separately for restore processes because the local itemable diff --git a/src/internal/m365/onedrive/mock/handlers.go b/src/internal/m365/onedrive/mock/handlers.go index 92b4573e6..75dd3c3f1 100644 --- a/src/internal/m365/onedrive/mock/handlers.go +++ b/src/internal/m365/onedrive/mock/handlers.go @@ -249,9 +249,25 @@ type RestoreHandler struct { PostItemResp models.DriveItemable PostItemErr error + DrivePagerV api.DrivePager + + PostDriveResp models.Driveable + PostDriveErr error + UploadSessionErr error } +func (h RestoreHandler) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + return h.PostDriveResp, h.PostDriveErr +} + +func (h RestoreHandler) NewDrivePager(string, []string) api.DrivePager { + return h.DrivePagerV +} + func (h *RestoreHandler) AugmentItemInfo( details.ItemInfo, models.DriveItemable, diff --git a/src/internal/m365/onedrive/restore.go b/src/internal/m365/onedrive/restore.go index 84b8f1cd0..7c920e6ff 100644 --- a/src/internal/m365/onedrive/restore.go +++ b/src/internal/m365/onedrive/restore.go @@ -15,6 +15,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -22,6 +23,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" @@ -37,54 +39,30 @@ const ( maxUploadRetries = 3 ) -type restoreCaches struct { - collisionKeyToItemID map[string]api.DriveItemIDType - DriveIDToRootFolderID map[string]string - Folders *folderCache - OldLinkShareIDToNewID map[string]string - OldPermIDToNewID map[string]string - ParentDirToMeta map[string]metadata.Metadata - - pool sync.Pool -} - -func NewRestoreCaches() *restoreCaches { - return &restoreCaches{ - collisionKeyToItemID: map[string]api.DriveItemIDType{}, - DriveIDToRootFolderID: map[string]string{}, - Folders: NewFolderCache(), - OldLinkShareIDToNewID: map[string]string{}, - OldPermIDToNewID: map[string]string{}, - ParentDirToMeta: map[string]metadata.Metadata{}, - // Buffer pool for uploads - pool: sync.Pool{ - New: func() any { - b := make([]byte, graph.CopyBufferSize) - return &b - }, - }, - } -} - // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, rh RestoreHandler, - backupVersion int, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( - restoreMetrics support.CollectionMetrics - caches = NewRestoreCaches() - el = errs.Local() + restoreMetrics support.CollectionMetrics + el = errs.Local() + caches = NewRestoreCaches(backupDriveIDNames) + fallbackDriveName = rcc.RestoreConfig.Location ) - ctx = clues.Add(ctx, "backup_version", backupVersion) + ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion) + + err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID()) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. @@ -102,19 +80,17 @@ func ConsumeRestoreCollections( ictx = clues.Add( ctx, "category", dc.FullPath().Category(), - "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "full_path", dc.FullPath()) ) metrics, err = RestoreCollection( ictx, rh, - restoreCfg, - backupVersion, + rcc, dc, caches, deets, - opts.RestorePermissions, + fallbackDriveName, errs, ctr.Local()) if err != nil { @@ -133,7 +109,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } @@ -146,12 +122,11 @@ func ConsumeRestoreCollections( func RestoreCollection( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, - backupVersion int, + rcc inject.RestoreConsumerConfig, dc data.RestoreCollection, caches *restoreCaches, deets *details.Builder, - restorePerms bool, // TODD: move into restoreConfig + fallbackDriveName string, errs *fault.Bus, ctr *count.Bus, ) (support.CollectionMetrics, error) { @@ -174,23 +149,31 @@ func RestoreCollection( return metrics, clues.Wrap(err, "creating drive path").WithClues(ctx) } - if _, ok := caches.DriveIDToRootFolderID[drivePath.DriveID]; !ok { - root, err := rh.GetRootFolder(ctx, drivePath.DriveID) - if err != nil { - return metrics, clues.Wrap(err, "getting drive root id") - } - - caches.DriveIDToRootFolderID[drivePath.DriveID] = ptr.Val(root.GetId()) + di, err := ensureDriveExists( + ctx, + rh, + caches, + drivePath, + rcc.ProtectedResource.ID(), + fallbackDriveName) + if err != nil { + return metrics, clues.Wrap(err, "ensuring drive exists") } + // clobber the drivePath details with the details retrieved + // in the ensure func, as they might have changed to reflect + // a different drive as a restore location. + drivePath.DriveID = di.id + drivePath.Root = di.rootFolderID + // Assemble folder hierarchy we're going to restore into (we recreate the folder hierarchy // from the backup under this the restore folder instead of root) // i.e. Restore into `/` // the drive into which this folder gets restored is tracked separately in drivePath. restoreDir := &path.Builder{} - if len(restoreCfg.Location) > 0 { - restoreDir = restoreDir.Append(restoreCfg.Location) + if len(rcc.RestoreConfig.Location) > 0 { + restoreDir = restoreDir.Append(rcc.RestoreConfig.Location) } restoreDir = restoreDir.Append(drivePath.Folders...) @@ -209,8 +192,8 @@ func RestoreCollection( drivePath, dc, caches, - backupVersion, - restorePerms) + rcc.BackupVersion, + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx) } @@ -224,7 +207,7 @@ func RestoreCollection( dc.FullPath(), colMeta, caches, - restorePerms) + rcc.RestoreConfig.IncludePermissions) if err != nil { return metrics, clues.Wrap(err, "creating folders for restore") } @@ -298,14 +281,12 @@ func RestoreCollection( itemInfo, skipped, err := restoreItem( ictx, rh, - restoreCfg, + rcc, dc, - backupVersion, drivePath, restoreFolderID, copyBuffer, caches, - restorePerms, itemData, itemPath, ctr) @@ -348,14 +329,12 @@ func RestoreCollection( func restoreItem( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, fibn data.FetchItemByNamer, - backupVersion int, drivePath *path.DrivePath, restoreFolderID string, copyBuffer []byte, caches *restoreCaches, - restorePerms bool, itemData data.Stream, itemPath path.Path, ctr *count.Bus, @@ -363,11 +342,11 @@ func restoreItem( itemUUID := itemData.UUID() ctx = clues.Add(ctx, "item_id", itemUUID) - if backupVersion < version.OneDrive1DataAndMetaFiles { + if rcc.BackupVersion < version.OneDrive1DataAndMetaFiles { itemInfo, err := restoreV0File( ctx, rh, - restoreCfg, + rcc.RestoreConfig, drivePath, fibn, restoreFolderID, @@ -376,7 +355,7 @@ func restoreItem( itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -399,7 +378,7 @@ func restoreItem( // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // permission for child folders here. Later versions can request // permissions inline when processing the collection. - if !restorePerms || backupVersion >= version.OneDrive4DirIncludesPermissions { + if !rcc.RestoreConfig.IncludePermissions || rcc.BackupVersion >= version.OneDrive4DirIncludesPermissions { return details.ItemInfo{}, true, nil } @@ -419,22 +398,21 @@ func restoreItem( // only items with DataFileSuffix from this point on - if backupVersion < version.OneDrive6NameInMeta { + if rcc.BackupVersion < version.OneDrive6NameInMeta { itemInfo, err := restoreV1File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -449,18 +427,17 @@ func restoreItem( itemInfo, err := restoreV6File( ctx, rh, - restoreCfg, + rcc, drivePath, fibn, restoreFolderID, copyBuffer, - restorePerms, caches, itemPath, itemData, ctr) if err != nil { - if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { + if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip { return details.ItemInfo{}, true, nil } @@ -504,12 +481,11 @@ func restoreV0File( func restoreV1File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -519,7 +495,7 @@ func restoreV1File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, trimmedName, @@ -535,7 +511,7 @@ func restoreV1File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } @@ -565,12 +541,11 @@ func restoreV1File( func restoreV6File( ctx context.Context, rh RestoreHandler, - restoreCfg control.RestoreConfig, + rcc inject.RestoreConsumerConfig, drivePath *path.DrivePath, fibn data.FetchItemByNamer, restoreFolderID string, copyBuffer []byte, - restorePerms bool, caches *restoreCaches, itemPath path.Path, itemData data.Stream, @@ -604,7 +579,7 @@ func restoreV6File( itemID, itemInfo, err := restoreFile( ctx, - restoreCfg, + rcc.RestoreConfig, rh, fibn, meta.FileName, @@ -620,10 +595,12 @@ func restoreV6File( // Mark it as success without processing .meta // file if we are not restoring permissions - if !restorePerms { + if !rcc.RestoreConfig.IncludePermissions { return itemInfo, nil } + fmt.Printf("\n-----\nrestorev6 %+v\n-----\n", rcc.RestoreConfig.IncludePermissions) + err = RestorePermissions( ctx, rh, @@ -673,6 +650,8 @@ func CreateRestoreFolders( return id, nil } + fmt.Printf("\n-----\ncreatefolders %+v\n-----\n", restorePerms) + err = RestorePermissions( ctx, rh, @@ -704,7 +683,7 @@ func createRestoreFolders( driveID = drivePath.DriveID folders = restoreDir.Elements() location = path.Builder{}.Append(driveID) - parentFolderID = caches.DriveIDToRootFolderID[drivePath.DriveID] + parentFolderID = caches.DriveIDToDriveInfo[drivePath.DriveID].rootFolderID ) ctx = clues.Add( @@ -1113,3 +1092,79 @@ func AugmentRestorePaths( return paths, nil } + +type PostDriveAndGetRootFolderer interface { + PostDriver + GetRootFolderer +} + +// ensureDriveExists looks up the drive by its id. If no drive is found with +// that ID, a new drive is generated with the same name. If the name collides +// with an existing drive, a number is appended to the drive name. Eg: foo -> +// foo 1. This will repeat as many times as is needed. +// Returns the root folder of the drive +func ensureDriveExists( + ctx context.Context, + pdagrf PostDriveAndGetRootFolderer, + caches *restoreCaches, + drivePath *path.DrivePath, + protectedResourceID, fallbackDriveName string, +) (driveInfo, error) { + driveID := drivePath.DriveID + + // the drive might already be cached by ID. it's okay + // if the name has changed. the ID is a better reference + // anyway. + if di, ok := caches.DriveIDToDriveInfo[driveID]; ok { + return di, nil + } + + var ( + newDriveName = fallbackDriveName + newDrive models.Driveable + err error + ) + + // if the drive wasn't found by ID, maybe we can find a + // drive with the same name but different ID. + // start by looking up the old drive's name + oldName, ok := caches.BackupDriveIDName.NameOf(driveID) + if ok { + // check for drives that currently have the same name + if di, ok := caches.DriveNameToDriveInfo[oldName]; ok { + return di, nil + } + + // if no current drives have the same name, we'll make + // a new drive with that name. + newDriveName = oldName + } + + nextDriveName := newDriveName + + // For sharepoint, document libraries can collide by name with + // item types beyond just drive. Lists, for example, cannot share + // names with document libraries (they're the same type, actually). + // In those cases we need to rename the drive until we can create + // one without a collision. + for i := 1; ; i++ { + ictx := clues.Add(ctx, "new_drive_name", clues.Hide(nextDriveName)) + + newDrive, err = pdagrf.PostDrive(ictx, protectedResourceID, nextDriveName) + if err != nil && !errors.Is(err, graph.ErrItemAlreadyExistsConflict) { + return driveInfo{}, clues.Wrap(err, "creating new drive") + } + + if err == nil { + break + } + + nextDriveName = fmt.Sprintf("%s %d", newDriveName, i) + } + + if err := caches.AddDrive(ctx, newDrive, pdagrf); err != nil { + return driveInfo{}, clues.Wrap(err, "adding drive to cache").OrNil() + } + + return caches.DriveIDToDriveInfo[ptr.Val(newDrive.GetId())], nil +} diff --git a/src/internal/m365/onedrive/restore_caches.go b/src/internal/m365/onedrive/restore_caches.go new file mode 100644 index 000000000..6951a8bfe --- /dev/null +++ b/src/internal/m365/onedrive/restore_caches.go @@ -0,0 +1,116 @@ +package onedrive + +import ( + "context" + "sync" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/onedrive/metadata" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type driveInfo struct { + id string + name string + rootFolderID string +} + +type restoreCaches struct { + BackupDriveIDName idname.Cacher + collisionKeyToItemID map[string]api.DriveItemIDType + DriveIDToDriveInfo map[string]driveInfo + DriveNameToDriveInfo map[string]driveInfo + Folders *folderCache + OldLinkShareIDToNewID map[string]string + OldPermIDToNewID map[string]string + ParentDirToMeta map[string]metadata.Metadata + + pool sync.Pool +} + +func (rc *restoreCaches) AddDrive( + ctx context.Context, + md models.Driveable, + grf GetRootFolderer, +) error { + di := driveInfo{ + id: ptr.Val(md.GetId()), + name: ptr.Val(md.GetName()), + } + + ctx = clues.Add(ctx, "drive_info", di) + + root, err := grf.GetRootFolder(ctx, di.id) + if err != nil { + return clues.Wrap(err, "getting drive root id") + } + + di.rootFolderID = ptr.Val(root.GetId()) + + rc.DriveIDToDriveInfo[di.id] = di + rc.DriveNameToDriveInfo[di.name] = di + + return nil +} + +// Populate looks up drive items available to the protectedResource +// and adds their info to the caches. +func (rc *restoreCaches) Populate( + ctx context.Context, + gdparf GetDrivePagerAndRootFolderer, + protectedResourceID string, +) error { + drives, err := api.GetAllDrives( + ctx, + gdparf.NewDrivePager(protectedResourceID, nil), + true, + maxDrivesRetries) + if err != nil { + return clues.Wrap(err, "getting drives") + } + + for _, md := range drives { + if err := rc.AddDrive(ctx, md, gdparf); err != nil { + return clues.Wrap(err, "caching drive") + } + } + + return nil +} + +type GetDrivePagerAndRootFolderer interface { + GetRootFolderer + NewDrivePagerer +} + +func NewRestoreCaches( + backupDriveIDNames idname.Cacher, +) *restoreCaches { + // avoid nil panics + if backupDriveIDNames == nil { + backupDriveIDNames = idname.NewCache(nil) + } + + return &restoreCaches{ + BackupDriveIDName: backupDriveIDNames, + collisionKeyToItemID: map[string]api.DriveItemIDType{}, + DriveIDToDriveInfo: map[string]driveInfo{}, + DriveNameToDriveInfo: map[string]driveInfo{}, + Folders: NewFolderCache(), + OldLinkShareIDToNewID: map[string]string{}, + OldPermIDToNewID: map[string]string{}, + ParentDirToMeta: map[string]metadata.Metadata{}, + // Buffer pool for uploads + pool: sync.Pool{ + New: func() any { + b := make([]byte, graph.CopyBufferSize) + return &b + }, + }, + } +} diff --git a/src/internal/m365/onedrive/restore_test.go b/src/internal/m365/onedrive/restore_test.go index 4128661f5..301a1b01e 100644 --- a/src/internal/m365/onedrive/restore_test.go +++ b/src/internal/m365/onedrive/restore_test.go @@ -11,16 +11,19 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" "github.com/alcionai/corso/src/internal/m365/onedrive/mock" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" + apiMock "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type RestoreUnitSuite struct { @@ -491,7 +494,7 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { mndi.SetId(ptr.To(mndiID)) var ( - caches = NewRestoreCaches() + caches = NewRestoreCaches(nil) rh = &mock.RestoreHandler{ PostItemResp: models.NewDriveItem(), DeleteItemErr: test.deleteErr, @@ -510,21 +513,25 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() { ctr := count.New() + rcc := inject.RestoreConsumerConfig{ + BackupVersion: version.Backup, + Options: control.DefaultOptions(), + RestoreConfig: restoreCfg, + } + _, skip, err := restoreItem( ctx, rh, - restoreCfg, + rcc, mock.FetchItemByName{ Item: &mock.Data{ Reader: mock.FileRespReadCloser(mock.DriveFileMetaData), }, }, - version.Backup, dp, "", make([]byte, graph.CopyBufferSize), caches, - false, &mock.Data{ ID: uuid.NewString(), Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), @@ -617,3 +624,435 @@ func (suite *RestoreUnitSuite) TestCreateFolder() { }) } } + +type mockGRF struct { + err error + rootFolder models.DriveItemable +} + +func (m *mockGRF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_AddDrive() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *mockGRF + expectErr require.ErrorAssertionFunc + expectID string + checkValues bool + }{ + { + name: "good", + mock: &mockGRF{rootFolder: rf}, + expectErr: require.NoError, + expectID: rfID, + checkValues: true, + }, + { + name: "err", + mock: &mockGRF{err: assert.AnError}, + expectErr: require.Error, + expectID: "", + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := NewRestoreCaches(nil) + err := rc.AddDrive(ctx, md, test.mock) + test.expectErr(t, err, clues.ToCore(err)) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, test.expectID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, test.expectID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockGDPARF struct { + err error + rootFolder models.DriveItemable + pager *apiMock.DrivePager +} + +func (m *mockGDPARF) GetRootFolder( + context.Context, + string, +) (models.DriveItemable, error) { + return m.rootFolder, m.err +} + +func (m *mockGDPARF) NewDrivePager( + string, + []string, +) api.DrivePager { + return m.pager +} + +func (suite *RestoreUnitSuite) TestRestoreCaches_Populate() { + rfID := "this-is-id" + driveID := "another-id" + name := "name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + table := []struct { + name string + mock *apiMock.DrivePager + expectErr require.ErrorAssertionFunc + expectLen int + checkValues bool + }{ + { + name: "no results", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{}}, + }, + }, + expectErr: require.NoError, + expectLen: 0, + }, + { + name: "one result", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Drives: []models.Driveable{md}}, + }, + }, + expectErr: require.NoError, + expectLen: 1, + checkValues: true, + }, + { + name: "error", + mock: &apiMock.DrivePager{ + ToReturn: []apiMock.PagerResult{ + {Err: assert.AnError}, + }, + }, + expectErr: require.Error, + expectLen: 0, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + gdparf := &mockGDPARF{ + rootFolder: rf, + pager: test.mock, + } + + rc := NewRestoreCaches(nil) + err := rc.Populate(ctx, gdparf, "shmoo") + test.expectErr(t, err, clues.ToCore(err)) + + assert.Len(t, rc.DriveIDToDriveInfo, test.expectLen) + assert.Len(t, rc.DriveNameToDriveInfo, test.expectLen) + + if test.checkValues { + idResult := rc.DriveIDToDriveInfo[driveID] + assert.Equal(t, driveID, idResult.id, "drive id") + assert.Equal(t, name, idResult.name, "drive name") + assert.Equal(t, rfID, idResult.rootFolderID, "root folder id") + + nameResult := rc.DriveNameToDriveInfo[name] + assert.Equal(t, driveID, nameResult.id, "drive id") + assert.Equal(t, name, nameResult.name, "drive name") + assert.Equal(t, rfID, nameResult.rootFolderID, "root folder id") + } + }) + } +} + +type mockPDAGRF struct { + i int + postResp []models.Driveable + postErr []error + + grf mockGRF +} + +func (m *mockPDAGRF) PostDrive( + ctx context.Context, + protectedResourceID, driveName string, +) (models.Driveable, error) { + defer func() { m.i++ }() + + md := m.postResp[m.i] + if md != nil { + md.SetName(&driveName) + } + + return md, m.postErr[m.i] +} + +func (m *mockPDAGRF) GetRootFolder( + ctx context.Context, + driveID string, +) (models.DriveItemable, error) { + return m.grf.rootFolder, m.grf.err +} + +func (suite *RestoreUnitSuite) TestEnsureDriveExists() { + rfID := "this-is-id" + driveID := "another-id" + oldID := "old-id" + name := "name" + otherName := "other name" + + rf := models.NewDriveItem() + rf.SetId(&rfID) + + grf := mockGRF{rootFolder: rf} + + makeMD := func() models.Driveable { + md := models.NewDrive() + md.SetId(&driveID) + md.SetName(&name) + + return md + } + + dp := &path.DrivePath{ + DriveID: driveID, + Root: "root:", + Folders: path.Elements{}, + } + + oldDP := &path.DrivePath{ + DriveID: oldID, + Root: "root:", + Folders: path.Elements{}, + } + + populatedCache := func(id string) *restoreCaches { + rc := NewRestoreCaches(nil) + di := driveInfo{ + id: id, + name: name, + } + rc.DriveIDToDriveInfo[id] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + oldDriveIDNames := idname.NewCache(nil) + oldDriveIDNames.Add(oldID, name) + + idSwitchedCache := func() *restoreCaches { + rc := NewRestoreCaches(oldDriveIDNames) + di := driveInfo{ + id: "diff", + name: name, + } + rc.DriveIDToDriveInfo["diff"] = di + rc.DriveNameToDriveInfo[name] = di + + return rc + } + + table := []struct { + name string + dp *path.DrivePath + mock *mockPDAGRF + rc *restoreCaches + expectErr require.ErrorAssertionFunc + fallbackName string + expectName string + expectID string + skipValueChecks bool + }{ + { + name: "drive already in cache", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "drive with same name but different id exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: idSwitchedCache(), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: "diff", + }, + { + name: "drive created with old name", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: otherName, + expectName: name, + expectID: driveID, + }, + { + name: "drive created with fallback name", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: otherName, + expectName: otherName, + expectID: driveID, + }, + { + name: "error creating drive", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil}, + postErr: []error{assert.AnError}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.Error, + fallbackName: name, + expectName: "", + skipValueChecks: true, + expectID: driveID, + }, + { + name: "drive name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{makeMD()}, + postErr: []error{nil}, + grf: grf, + }, + rc: populatedCache("beaux"), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + { + name: "list with name already exists", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(nil), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "list with old name already exists", + dp: oldDP, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: NewRestoreCaches(oldDriveIDNames), + expectErr: require.NoError, + fallbackName: name, + expectName: name + " 1", + expectID: driveID, + }, + { + name: "drive and list with name already exist", + dp: dp, + mock: &mockPDAGRF{ + postResp: []models.Driveable{nil, makeMD()}, + postErr: []error{graph.ErrItemAlreadyExistsConflict, nil}, + grf: grf, + }, + rc: populatedCache(driveID), + expectErr: require.NoError, + fallbackName: name, + expectName: name, + expectID: driveID, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := test.rc + + di, err := ensureDriveExists( + ctx, + test.mock, + rc, + test.dp, + "prID", + test.fallbackName) + test.expectErr(t, err, clues.ToCore(err)) + + if !test.skipValueChecks { + assert.Equal(t, test.expectName, di.name, "ensured drive has expected name") + assert.Equal(t, test.expectID, di.id, "ensured drive has expected id") + + nameResult := rc.DriveNameToDriveInfo[test.expectName] + assert.Equal(t, test.expectName, nameResult.name, "found drive entry with expected name") + } + }) + } +} diff --git a/src/internal/m365/onedrive/service_test.go b/src/internal/m365/onedrive/service_test.go index 4569acffc..a2766b8ee 100644 --- a/src/internal/m365/onedrive/service_test.go +++ b/src/internal/m365/onedrive/service_test.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -20,7 +21,7 @@ type oneDriveService struct { } func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) { - ac, err := api.NewClient(credentials) + ac, err := api.NewClient(credentials, control.DefaultOptions()) if err != nil { return nil, err } diff --git a/src/internal/m365/onedrive/url_cache_test.go b/src/internal/m365/onedrive/url_cache_test.go index 8adcf36cc..bf4f25350 100644 --- a/src/internal/m365/onedrive/url_cache_test.go +++ b/src/internal/m365/onedrive/url_cache_test.go @@ -53,7 +53,7 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() { creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user) diff --git a/src/internal/m365/onedrive_test.go b/src/internal/m365/onedrive_test.go index 3fbd5f531..ba81a477a 100644 --- a/src/internal/m365/onedrive_test.go +++ b/src/internal/m365/onedrive_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" @@ -22,6 +23,7 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" ) @@ -222,9 +224,9 @@ func (suite *SharePointIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *SharePointIntegrationSuite) TestPermissionsBackupAndNoRestore() { +func (suite *SharePointIntegrationSuite) TestRestoreNoPermissionsAndBackup() { suite.T().Skip("Temporarily disabled due to CI issues") - testPermissionsBackupAndNoRestore(suite, version.Backup) + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *SharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -289,8 +291,8 @@ func (suite *OneDriveIntegrationSuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.Backup) } -func (suite *OneDriveIntegrationSuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.Backup) +func (suite *OneDriveIntegrationSuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.Backup) } func (suite *OneDriveIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -353,8 +355,8 @@ func (suite *OneDriveNightlySuite) TestPermissionsRestoreAndBackup() { testPermissionsRestoreAndBackup(suite, version.OneDrive1DataAndMetaFiles) } -func (suite *OneDriveNightlySuite) TestPermissionsBackupAndNoRestore() { - testPermissionsBackupAndNoRestore(suite, version.OneDrive1DataAndMetaFiles) +func (suite *OneDriveNightlySuite) TestRestoreNoPermissionsAndBackup() { + testRestoreNoPermissionsAndBackup(suite, version.OneDrive1DataAndMetaFiles) } func (suite *OneDriveNightlySuite) TestPermissionsInheritanceRestoreAndBackup() { @@ -516,15 +518,17 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions( collectionsLatest: expected, } + restoreCfg := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true + runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + restoreCfg) }) } } @@ -763,20 +767,22 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } + restoreCfg := testdata.DefaultRestoreConfig("perms_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true + runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + restoreCfg) }) } } -func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { +func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) { t := suite.T() ctx, flush := tester.NewContext(t) @@ -851,15 +857,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { collectionsLatest: expected, } + restoreCfg := testdata.DefaultRestoreConfig("perms_backup_no_restore") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = false + + fmt.Printf("\n-----\nrcfg %+v\n-----\n", restoreCfg.IncludePermissions) + runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: false, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1054,15 +1064,17 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio collectionsLatest: expected, } + restoreCfg := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true + runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1247,15 +1259,17 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion collectionsLatest: expected, } + restoreCfg := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") + restoreCfg.OnCollision = control.Replace + restoreCfg.IncludePermissions = true + runRestoreBackupTestVersions( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }) + control.DefaultOptions(), + restoreCfg) }) } } @@ -1362,16 +1376,16 @@ func testRestoreFolderNamedFolderRegression( collectionsLatest: expected, } + restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem) + restoreCfg.IncludePermissions = true + runRestoreTestWithVersion( t, testData, suite.Tenant(), []string{suite.ResourceOwner()}, - control.Options{ - RestorePermissions: true, - ToggleFeatures: control.Toggles{}, - }, - ) + control.DefaultOptions(), + restoreCfg) }) } } diff --git a/src/internal/m365/restore.go b/src/internal/m365/restore.go index 3c5e3e646..de9e0bb13 100644 --- a/src/internal/m365/restore.go +++ b/src/internal/m365/restore.go @@ -12,11 +12,11 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/sharepoint" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/path" ) // ConsumeRestoreCollections restores data from the specified collections @@ -24,10 +24,7 @@ import ( // SideEffect: status is updated at the completion of operation func (ctrl *Controller) ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - sels selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc inject.RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, @@ -35,42 +32,64 @@ func (ctrl *Controller) ConsumeRestoreCollections( ctx, end := diagnostics.Span(ctx, "m365:restore") defer end() - ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) - ctx = clues.Add(ctx, "restore_config", restoreCfg) // TODO(rkeepers): needs PII control + ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: rcc.Selector.PathService()}) + ctx = clues.Add(ctx, "restore_config", rcc.RestoreConfig) + + if len(dcs) == 0 { + return nil, clues.New("no data collections to restore") + } + + serviceEnabled, _, err := checkServiceEnabled( + ctx, + ctrl.AC.Users(), + rcc.Selector.PathService(), + rcc.ProtectedResource.ID()) + if err != nil { + return nil, err + } + + if !serviceEnabled { + return nil, clues.Stack(graph.ErrServiceNotEnabled).WithClues(ctx) + } var ( - status *support.ControllerOperationStatus - deets = &details.Builder{} - err error + service = rcc.Selector.PathService() + status *support.ControllerOperationStatus + deets = &details.Builder{} ) - switch sels.Service { - case selectors.ServiceExchange: - status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, restoreCfg, dcs, deets, errs, ctr) - case selectors.ServiceOneDrive: - status, err = onedrive.ConsumeRestoreCollections( + switch service { + case path.ExchangeService: + status, err = exchange.ConsumeRestoreCollections( ctx, - onedrive.NewRestoreHandler(ctrl.AC), - backupVersion, - restoreCfg, - opts, + ctrl.AC, + rcc, dcs, deets, errs, ctr) - case selectors.ServiceSharePoint: + case path.OneDriveService: + status, err = onedrive.ConsumeRestoreCollections( + ctx, + onedrive.NewRestoreHandler(ctrl.AC), + rcc, + ctrl.backupDriveIDNames, + dcs, + deets, + errs, + ctr) + case path.SharePointService: status, err = sharepoint.ConsumeRestoreCollections( ctx, - backupVersion, + rcc, ctrl.AC, - restoreCfg, - opts, + ctrl.backupDriveIDNames, dcs, deets, errs, ctr) default: - err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + err = clues.Wrap(clues.New(service.String()), "service not supported") } ctrl.incrementAwaitingMessages() diff --git a/src/internal/m365/sharepoint/backup_test.go b/src/internal/m365/sharepoint/backup_test.go index 6e878f0b9..348b15dfd 100644 --- a/src/internal/m365/sharepoint/backup_test.go +++ b/src/internal/m365/sharepoint/backup_test.go @@ -107,7 +107,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() { tenantID, site, nil, - control.Defaults()) + control.DefaultOptions()) c.CollectionMap = collMap @@ -201,7 +201,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) col, err := collectPages( @@ -210,7 +210,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() { ac, mock.NewProvider(siteID, siteID), &MockGraphService{}, - control.Defaults(), + control.DefaultOptions(), fault.New(true)) assert.NoError(t, err, clues.ToCore(err)) assert.NotEmpty(t, col) diff --git a/src/internal/m365/sharepoint/collection_test.go b/src/internal/m365/sharepoint/collection_test.go index babe6114e..0462a5c8e 100644 --- a/src/internal/m365/sharepoint/collection_test.go +++ b/src/internal/m365/sharepoint/collection_test.go @@ -43,7 +43,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() { suite.creds = m365 - ac, err := api.NewClient(m365) + ac, err := api.NewClient(m365, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) suite.ac = ac @@ -168,7 +168,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { suite.ac, test.category, nil, - control.Defaults()) + control.DefaultOptions()) col.data <- test.getItem(t, test.itemName) readItems := []data.Stream{} diff --git a/src/internal/m365/sharepoint/library_handler.go b/src/internal/m365/sharepoint/library_handler.go index 07c997fcb..3f16c6eae 100644 --- a/src/internal/m365/sharepoint/library_handler.go +++ b/src/internal/m365/sharepoint/library_handler.go @@ -157,11 +157,25 @@ func (h libraryBackupHandler) IncludesDir(dir string) bool { var _ onedrive.RestoreHandler = &libraryRestoreHandler{} type libraryRestoreHandler struct { - ac api.Drives + ac api.Client +} + +func (h libraryRestoreHandler) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + return h.ac.Lists().PostDrive(ctx, siteID, driveName) } func NewRestoreHandler(ac api.Client) *libraryRestoreHandler { - return &libraryRestoreHandler{ac.Drives()} + return &libraryRestoreHandler{ac} +} + +func (h libraryRestoreHandler) NewDrivePager( + resourceOwner string, + fields []string, +) api.DrivePager { + return h.ac.Drives().NewSiteDrivePager(resourceOwner, fields) } func (h libraryRestoreHandler) AugmentItemInfo( @@ -177,21 +191,21 @@ func (h libraryRestoreHandler) DeleteItem( ctx context.Context, driveID, itemID string, ) error { - return h.ac.DeleteItem(ctx, driveID, itemID) + return h.ac.Drives().DeleteItem(ctx, driveID, itemID) } func (h libraryRestoreHandler) DeleteItemPermission( ctx context.Context, driveID, itemID, permissionID string, ) error { - return h.ac.DeleteItemPermission(ctx, driveID, itemID, permissionID) + return h.ac.Drives().DeleteItemPermission(ctx, driveID, itemID, permissionID) } func (h libraryRestoreHandler) GetItemsInContainerByCollisionKey( ctx context.Context, driveID, containerID string, ) (map[string]api.DriveItemIDType, error) { - m, err := h.ac.GetItemsInContainerByCollisionKey(ctx, driveID, containerID) + m, err := h.ac.Drives().GetItemsInContainerByCollisionKey(ctx, driveID, containerID) if err != nil { return nil, err } @@ -203,7 +217,7 @@ func (h libraryRestoreHandler) NewItemContentUpload( ctx context.Context, driveID, itemID string, ) (models.UploadSessionable, error) { - return h.ac.NewItemContentUpload(ctx, driveID, itemID) + return h.ac.Drives().NewItemContentUpload(ctx, driveID, itemID) } func (h libraryRestoreHandler) PostItemPermissionUpdate( @@ -211,7 +225,7 @@ func (h libraryRestoreHandler) PostItemPermissionUpdate( driveID, itemID string, body *drives.ItemItemsItemInvitePostRequestBody, ) (drives.ItemItemsItemInviteResponseable, error) { - return h.ac.PostItemPermissionUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemPermissionUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemLinkShareUpdate( @@ -219,7 +233,7 @@ func (h libraryRestoreHandler) PostItemLinkShareUpdate( driveID, itemID string, body *drives.ItemItemsItemCreateLinkPostRequestBody, ) (models.Permissionable, error) { - return h.ac.PostItemLinkShareUpdate(ctx, driveID, itemID, body) + return h.ac.Drives().PostItemLinkShareUpdate(ctx, driveID, itemID, body) } func (h libraryRestoreHandler) PostItemInContainer( @@ -228,21 +242,21 @@ func (h libraryRestoreHandler) PostItemInContainer( newItem models.DriveItemable, onCollision control.CollisionPolicy, ) (models.DriveItemable, error) { - return h.ac.PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) + return h.ac.Drives().PostItemInContainer(ctx, driveID, parentFolderID, newItem, onCollision) } func (h libraryRestoreHandler) GetFolderByName( ctx context.Context, driveID, parentFolderID, folderName string, ) (models.DriveItemable, error) { - return h.ac.GetFolderByName(ctx, driveID, parentFolderID, folderName) + return h.ac.Drives().GetFolderByName(ctx, driveID, parentFolderID, folderName) } func (h libraryRestoreHandler) GetRootFolder( ctx context.Context, driveID string, ) (models.DriveItemable, error) { - return h.ac.GetRootFolder(ctx, driveID) + return h.ac.Drives().GetRootFolder(ctx, driveID) } // --------------------------------------------------------------------------- diff --git a/src/internal/m365/sharepoint/restore.go b/src/internal/m365/sharepoint/restore.go index 417d6d87c..bb894f5ea 100644 --- a/src/internal/m365/sharepoint/restore.go +++ b/src/internal/m365/sharepoint/restore.go @@ -10,6 +10,8 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" @@ -17,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/onedrive" betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" "github.com/alcionai/corso/src/internal/m365/support" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/count" @@ -29,21 +32,26 @@ import ( // ConsumeRestoreCollections will restore the specified data collections into OneDrive func ConsumeRestoreCollections( ctx context.Context, - backupVersion int, + rcc inject.RestoreConsumerConfig, ac api.Client, - restoreCfg control.RestoreConfig, - opts control.Options, + backupDriveIDNames idname.Cacher, dcs []data.RestoreCollection, deets *details.Builder, errs *fault.Bus, ctr *count.Bus, ) (*support.ControllerOperationStatus, error) { var ( + lrh = libraryRestoreHandler{ac} restoreMetrics support.CollectionMetrics - caches = onedrive.NewRestoreCaches() + caches = onedrive.NewRestoreCaches(backupDriveIDNames) el = errs.Local() ) + err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID()) + if err != nil { + return nil, clues.Wrap(err, "initializing restore caches") + } + // Reorder collections so that the parents directories are created // before the child directories; a requirement for permissions. data.SortRestoreCollections(dcs) @@ -60,7 +68,7 @@ func ConsumeRestoreCollections( metrics support.CollectionMetrics ictx = clues.Add(ctx, "category", category, - "restore_location", restoreCfg.Location, + "restore_location", clues.Hide(rcc.RestoreConfig.Location), "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "full_path", dc.FullPath()) ) @@ -69,13 +77,12 @@ func ConsumeRestoreCollections( case path.LibrariesCategory: metrics, err = onedrive.RestoreCollection( ictx, - libraryRestoreHandler{ac.Drives()}, - restoreCfg, - backupVersion, + lrh, + rcc, dc, caches, deets, - opts.RestorePermissions, + control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), errs, ctr) @@ -84,7 +91,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -93,7 +100,7 @@ func ConsumeRestoreCollections( ictx, ac.Stable, dc, - restoreCfg.Location, + rcc.RestoreConfig.Location, deets, errs) @@ -117,7 +124,7 @@ func ConsumeRestoreCollections( support.Restore, len(dcs), restoreMetrics, - restoreCfg.Location) + rcc.RestoreConfig.Location) return status, el.Failure() } diff --git a/src/internal/m365/stub/stub.go b/src/internal/m365/stub/stub.go index 601e57722..da3340f60 100644 --- a/src/internal/m365/stub/stub.go +++ b/src/internal/m365/stub/stub.go @@ -68,8 +68,7 @@ func GetCollectionsAndExpected( owner, config.RestoreCfg, testCollections, - backupVersion, - ) + backupVersion) if err != nil { return totalItems, totalKopiaItems, collections, expectedData, err } diff --git a/src/internal/m365/support/status.go b/src/internal/m365/support/status.go index f241909fe..b1a7d2449 100644 --- a/src/internal/m365/support/status.go +++ b/src/internal/m365/support/status.go @@ -40,6 +40,7 @@ const ( OpUnknown Operation = iota Backup Restore + Export ) // Constructor for ConnectorOperationStatus. If the counts do not agree, an error is returned. diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index bd94b6531..50994eacb 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -133,6 +133,7 @@ func Complete() { const ( ItemBackupMsg = "Backing up item" ItemRestoreMsg = "Restoring item" + ItemExportMsg = "Exporting item" ItemQueueMsg = "Queuing items" ) @@ -281,6 +282,51 @@ func ItemProgress( return bar.ProxyReader(rc), abort } +// ItemSpinner is similar to ItemProgress, but for use in cases where +// we don't know the file size but want to show progress. +func ItemSpinner( + ctx context.Context, + rc io.ReadCloser, + header string, + iname any, +) (io.ReadCloser, func()) { + plain := plainString(iname) + log := logger.Ctx(ctx).With("item", iname) + log.Debug(header) + + if cfg.hidden() || rc == nil { + defer log.Debug("done - " + header) + return rc, func() {} + } + + wg.Add(1) + + barOpts := []mpb.BarOption{ + mpb.PrependDecorators( + decor.Name(header, decor.WCSyncSpaceR), + decor.Name(plain, decor.WCSyncSpaceR), + decor.CurrentKibiByte(" %.1f", decor.WC{W: 8})), + } + + if !cfg.keepBarsAfterComplete { + barOpts = append(barOpts, mpb.BarRemoveOnComplete()) + } + + bar := progress.New(-1, mpb.NopStyle(), barOpts...) + + go waitAndCloseBar(bar, func() { + // might be overly chatty, we can remove if needed. + log.Debug("done - " + header) + })() + + abort := func() { + bar.SetTotal(-1, true) + bar.Abort(true) + } + + return bar.ProxyReader(rc), abort +} + // ProgressWithCount tracks the display of a bar that tracks the completion // of the specified count. // Each write to the provided channel counts as a single increment. @@ -517,8 +563,8 @@ func (b bulletf) String() string { // observe progress bar. Logged values should only use // the fmt %v to ensure Concealers hide PII. func plainString(v any) string { - if ps, ok := v.(clues.PlainStringer); ok { - return ps.PlainString() + if c, ok := v.(clues.Concealer); ok { + return c.PlainString() } return fmt.Sprintf("%v", v) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 00eb82884..82ae79fb6 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -6,7 +6,6 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - "github.com/kopia/kopia/repo/manifest" "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/dttm" @@ -280,8 +279,8 @@ func (op *BackupOperation) do( backupID model.StableID, ) (*details.Builder, error) { var ( - reasons = selectorToReasons(op.Selectors, false) - fallbackReasons = makeFallbackReasons(op.Selectors) + reasons = selectorToReasons(op.account.ID(), op.Selectors, false) + fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors) lastBackupVersion = version.NoBackup ) @@ -370,10 +369,10 @@ func (op *BackupOperation) do( return deets, nil } -func makeFallbackReasons(sel selectors.Selector) []kopia.Reason { +func makeFallbackReasons(tenant string, sel selectors.Selector) []kopia.Reasoner { if sel.PathService() != path.SharePointService && sel.DiscreteOwner != sel.DiscreteOwnerName { - return selectorToReasons(sel, true) + return selectorToReasons(tenant, sel, true) } return nil @@ -420,9 +419,13 @@ func produceBackupDataCollections( // Consumer funcs // --------------------------------------------------------------------------- -func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.Reason { +func selectorToReasons( + tenant string, + sel selectors.Selector, + useOwnerNameForID bool, +) []kopia.Reasoner { service := sel.PathService() - reasons := []kopia.Reason{} + reasons := []kopia.Reasoner{} pcs, err := sel.PathCategories() if err != nil { @@ -438,43 +441,19 @@ func selectorToReasons(sel selectors.Selector, useOwnerNameForID bool) []kopia.R for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} { for _, cat := range sl { - reasons = append(reasons, kopia.Reason{ - ResourceOwner: owner, - Service: service, - Category: cat, - }) + reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat)) } } return reasons } -func builderFromReason(ctx context.Context, tenant string, r kopia.Reason) (*path.Builder, error) { - ctx = clues.Add(ctx, "category", r.Category.String()) - - // This is hacky, but we want the path package to format the path the right - // way (e.x. proper order for service, category, etc), but we don't care about - // the folders after the prefix. - p, err := path.Build( - tenant, - r.ResourceOwner, - r.Service, - r.Category, - false, - "tmp") - if err != nil { - return nil, clues.Wrap(err, "building path").WithClues(ctx) - } - - return p.ToBuilder().Dir(), nil -} - // calls kopia to backup the collections of data func consumeBackupCollections( ctx context.Context, bc kinject.BackupConsumer, tenantID string, - reasons []kopia.Reason, + reasons []kopia.Reasoner, bbs kopia.BackupBases, cs []data.BackupCollection, pmr prefixmatcher.StringSetReader, @@ -495,90 +474,10 @@ func consumeBackupCollections( kopia.TagBackupCategory: "", } - for _, reason := range reasons { - for _, k := range reason.TagKeys() { - tags[k] = "" - } - } - - // AssistBases should be the upper bound for how many snapshots we pass in. - bases := make([]kopia.IncrementalBase, 0, len(bbs.AssistBases())) - // Track IDs we've seen already so we don't accidentally duplicate some - // manifests. This can be removed when we move the code below into the kopia - // package. - ids := map[manifest.ID]struct{}{} - - var mb []kopia.ManifestEntry - - if bbs != nil { - mb = bbs.MergeBases() - } - - // TODO(ashmrtn): Make a wrapper for Reson that allows adding a tenant and - // make a function that will spit out a prefix that includes the tenant. With - // that done this code can be moved to kopia wrapper since it's really more - // specific to that. - for _, m := range mb { - paths := make([]*path.Builder, 0, len(m.Reasons)) - services := map[string]struct{}{} - categories := map[string]struct{}{} - - for _, reason := range m.Reasons { - pb, err := builderFromReason(ctx, tenantID, reason) - if err != nil { - return nil, nil, nil, clues.Wrap(err, "getting subtree paths for bases") - } - - paths = append(paths, pb) - services[reason.Service.String()] = struct{}{} - categories[reason.Category.String()] = struct{}{} - } - - ids[m.ID] = struct{}{} - - bases = append(bases, kopia.IncrementalBase{ - Manifest: m.Manifest, - SubtreePaths: paths, - }) - - svcs := make([]string, 0, len(services)) - for k := range services { - svcs = append(svcs, k) - } - - cats := make([]string, 0, len(categories)) - for k := range categories { - cats = append(cats, k) - } - - mbID, ok := m.GetTag(kopia.TagBackupID) - if !ok { - mbID = "no_backup_id_tag" - } - - logger.Ctx(ctx).Infow( - "using base for backup", - "base_snapshot_id", m.ID, - "services", svcs, - "categories", cats, - "base_backup_id", mbID) - } - - // At the moment kopia assisted snapshots are in the same set as merge bases. - // When we fixup generating subtree paths we can remove this. - if bbs != nil { - for _, ab := range bbs.AssistBases() { - if _, ok := ids[ab.ID]; ok { - continue - } - - bases = append(bases, kopia.IncrementalBase{Manifest: ab.Manifest}) - } - } - kopiaStats, deets, itemsSourcedFromBase, err := bc.ConsumeBackupCollections( ctx, - bases, + reasons, + bbs, cs, pmr, tags, @@ -586,7 +485,7 @@ func consumeBackupCollections( errs) if err != nil { if kopiaStats == nil { - return nil, nil, nil, err + return nil, nil, nil, clues.Stack(err) } return nil, nil, nil, clues.Stack(err).With( @@ -609,11 +508,11 @@ func consumeBackupCollections( return kopiaStats, deets, itemsSourcedFromBase, err } -func matchesReason(reasons []kopia.Reason, p path.Path) bool { +func matchesReason(reasons []kopia.Reasoner, p path.Path) bool { for _, reason := range reasons { - if p.ResourceOwner() == reason.ResourceOwner && - p.Service() == reason.Service && - p.Category() == reason.Category { + if p.ResourceOwner() == reason.ProtectedResource() && + p.Service() == reason.Service() && + p.Category() == reason.Category() { return true } } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index ffa164c81..a2783e92e 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -107,7 +107,8 @@ func checkPaths(t *testing.T, expected, got []path.Path) { type mockBackupConsumer struct { checkFunc func( - bases []kopia.IncrementalBase, + backupReasons []kopia.Reasoner, + bases kopia.BackupBases, cs []data.BackupCollection, tags map[string]string, buildTreeWithBase bool) @@ -115,7 +116,8 @@ type mockBackupConsumer struct { func (mbu mockBackupConsumer) ConsumeBackupCollections( ctx context.Context, - bases []kopia.IncrementalBase, + backupReasons []kopia.Reasoner, + bases kopia.BackupBases, cs []data.BackupCollection, excluded prefixmatcher.StringSetReader, tags map[string]string, @@ -123,7 +125,7 @@ func (mbu mockBackupConsumer) ConsumeBackupCollections( errs *fault.Bus, ) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) { if mbu.checkFunc != nil { - mbu.checkFunc(bases, cs, tags, buildTreeWithBase) + mbu.checkFunc(backupReasons, bases, cs, tags, buildTreeWithBase) } return &kopia.BackupStats{}, &details.Builder{}, nil, nil @@ -360,7 +362,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() { op, err := NewBackupOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -388,31 +390,25 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() { func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections_Paths() { var ( + t = suite.T() + tenant = "a-tenant" resourceOwner = "a-user" - emailBuilder = path.Builder{}.Append( + emailReason = kopia.NewReason( tenant, - path.ExchangeService.String(), resourceOwner, - path.EmailCategory.String(), - ) - contactsBuilder = path.Builder{}.Append( + path.ExchangeService, + path.EmailCategory) + contactsReason = kopia.NewReason( tenant, - path.ExchangeService.String(), resourceOwner, - path.ContactsCategory.String(), - ) + path.ExchangeService, + path.ContactsCategory) - emailReason = kopia.Reason{ - ResourceOwner: resourceOwner, - Service: path.ExchangeService, - Category: path.EmailCategory, - } - contactsReason = kopia.Reason{ - ResourceOwner: resourceOwner, - Service: path.ExchangeService, - Category: path.ContactsCategory, + reasons = []kopia.Reasoner{ + emailReason, + contactsReason, } manifest1 = &snapshot.Manifest{ @@ -421,147 +417,57 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections manifest2 = &snapshot.Manifest{ ID: "id2", } + + bases = kopia.NewMockBackupBases().WithMergeBases( + kopia.ManifestEntry{ + Manifest: manifest1, + Reasons: []kopia.Reasoner{ + emailReason, + }, + }).WithAssistBases( + kopia.ManifestEntry{ + Manifest: manifest2, + Reasons: []kopia.Reasoner{ + contactsReason, + }, + }) + + backupID = model.StableID("foo") + expectedTags = map[string]string{ + kopia.TagBackupID: string(backupID), + kopia.TagBackupCategory: "", + } ) - table := []struct { - name string - // Backup model is untouched in this test so there's no need to populate it. - input kopia.BackupBases - expected []kopia.IncrementalBase - }{ - { - name: "SingleManifestSingleReason", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reason{ - emailReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - }, - }, - }, - }, - { - name: "SingleManifestMultipleReasons", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reason{ - emailReason, - contactsReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - }, - }, - { - name: "MultipleManifestsMultipleReasons", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reason{ - emailReason, - contactsReason, - }, - }, - kopia.ManifestEntry{ - Manifest: manifest2, - Reasons: []kopia.Reason{ - emailReason, - contactsReason, - }, - }).ClearMockAssistBases(), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - { - Manifest: manifest2, - SubtreePaths: []*path.Builder{ - emailBuilder, - contactsBuilder, - }, - }, - }, - }, - { - name: "Single Manifest Single Reason With Assist Base", - input: kopia.NewMockBackupBases().WithMergeBases( - kopia.ManifestEntry{ - Manifest: manifest1, - Reasons: []kopia.Reason{ - emailReason, - }, - }).WithAssistBases( - kopia.ManifestEntry{ - Manifest: manifest2, - Reasons: []kopia.Reason{ - contactsReason, - }, - }), - expected: []kopia.IncrementalBase{ - { - Manifest: manifest1, - SubtreePaths: []*path.Builder{ - emailBuilder, - }, - }, - { - Manifest: manifest2, - }, - }, + mbu := &mockBackupConsumer{ + checkFunc: func( + backupReasons []kopia.Reasoner, + gotBases kopia.BackupBases, + cs []data.BackupCollection, + gotTags map[string]string, + buildTreeWithBase bool, + ) { + kopia.AssertBackupBasesEqual(t, bases, gotBases) + assert.Equal(t, expectedTags, gotTags) + assert.ElementsMatch(t, reasons, backupReasons) }, } - for _, test := range table { - suite.Run(test.name, func() { - t := suite.T() + ctx, flush := tester.NewContext(t) + defer flush() - ctx, flush := tester.NewContext(t) - defer flush() - - mbu := &mockBackupConsumer{ - checkFunc: func( - bases []kopia.IncrementalBase, - cs []data.BackupCollection, - tags map[string]string, - buildTreeWithBase bool, - ) { - assert.ElementsMatch(t, test.expected, bases) - }, - } - - //nolint:errcheck - consumeBackupCollections( - ctx, - mbu, - tenant, - nil, - test.input, - nil, - nil, - model.StableID(""), - true, - fault.New(true)) - }) - } + //nolint:errcheck + consumeBackupCollections( + ctx, + mbu, + tenant, + reasons, + bases, + nil, + nil, + backupID, + true, + fault.New(true)) } func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() { @@ -629,16 +535,16 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems DetailsID: "did2", } - pathReason1 = kopia.Reason{ - ResourceOwner: itemPath1.ResourceOwner(), - Service: itemPath1.Service(), - Category: itemPath1.Category(), - } - pathReason3 = kopia.Reason{ - ResourceOwner: itemPath3.ResourceOwner(), - Service: itemPath3.Service(), - Category: itemPath3.Category(), - } + pathReason1 = kopia.NewReason( + "", + itemPath1.ResourceOwner(), + itemPath1.Service(), + itemPath1.Category()) + pathReason3 = kopia.NewReason( + "", + itemPath3.ResourceOwner(), + itemPath3.Service(), + itemPath3.Category()) ) itemParents1, err := path.GetDriveFolderPath(itemPath1) @@ -684,7 +590,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems }, DetailsID: "foo", }, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -703,7 +609,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -730,13 +636,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -763,7 +669,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -822,7 +728,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -849,7 +755,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -879,7 +785,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -909,7 +815,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -940,7 +846,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, @@ -971,13 +877,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems inputBackups: []kopia.BackupEntry{ { Backup: &backup1, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, }, { Backup: &backup2, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason3, }, }, @@ -1064,11 +970,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde locPath1 = path.Builder{}.Append(itemPath1.Folders()...) - pathReason1 = kopia.Reason{ - ResourceOwner: itemPath1.ResourceOwner(), - Service: itemPath1.Service(), - Category: itemPath1.Category(), - } + pathReason1 = kopia.NewReason( + "", + itemPath1.ResourceOwner(), + itemPath1.Service(), + itemPath1.Category()) backup1 = kopia.BackupEntry{ Backup: &backup.Backup{ @@ -1077,7 +983,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde }, DetailsID: "did1", }, - Reasons: []kopia.Reason{ + Reasons: []kopia.Reasoner{ pathReason1, }, } @@ -1231,7 +1137,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - suite.ac, err = api.NewClient(creds) + suite.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) } @@ -1241,7 +1147,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} acct = tconfig.NewM365Account(suite.T()) - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { diff --git a/src/internal/operations/export.go b/src/internal/operations/export.go new file mode 100644 index 000000000..772ba44f2 --- /dev/null +++ b/src/internal/operations/export.go @@ -0,0 +1,359 @@ +package operations + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/alcionai/clues" + "github.com/google/uuid" + + "github.com/alcionai/corso/src/internal/archive" + "github.com/alcionai/corso/src/internal/common/crash" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/diagnostics" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +const ( + // CopyBufferSize is the size of the copy buffer for disk + // write operations + // TODO(meain): tweak this value + CopyBufferSize = 5 * 1024 * 1024 +) + +// ExportOperation wraps an operation with export-specific props. +type ExportOperation struct { + operation + + BackupID model.StableID + Results RestoreResults + Selectors selectors.Selector + ExportCfg control.ExportConfig + Version string + + acct account.Account + ec inject.ExportConsumer +} + +// NewExportOperation constructs and validates a export operation. +func NewExportOperation( + ctx context.Context, + opts control.Options, + kw *kopia.Wrapper, + sw *store.Wrapper, + ec inject.ExportConsumer, + acct account.Account, + backupID model.StableID, + sel selectors.Selector, + exportCfg control.ExportConfig, + bus events.Eventer, +) (ExportOperation, error) { + op := ExportOperation{ + operation: newOperation(opts, bus, count.New(), kw, sw), + acct: acct, + BackupID: backupID, + ExportCfg: exportCfg, + Selectors: sel, + Version: "v0", + ec: ec, + } + if err := op.validate(); err != nil { + return ExportOperation{}, err + } + + return op, nil +} + +func (op ExportOperation) validate() error { + if op.ec == nil { + return clues.New("missing export consumer") + } + + return op.operation.validate() +} + +// aggregates stats from the export.Run(). +// primarily used so that the defer can take in a +// pointer wrapping the values, while those values +// get populated asynchronously. +type exportStats struct { + cs []data.RestoreCollection + ctrl *data.CollectionStats + bytesRead *stats.ByteCounter + resourceCount int + + // a transient value only used to pair up start-end events. + exportID string +} + +// Run begins a synchronous export operation. +func (op *ExportOperation) Run(ctx context.Context) ( + expColl []export.Collection, + err error, +) { + defer func() { + if crErr := crash.Recovery(ctx, recover(), "export"); crErr != nil { + err = crErr + } + }() + + var ( + opStats = exportStats{ + bytesRead: &stats.ByteCounter{}, + exportID: uuid.NewString(), + } + start = time.Now() + sstore = streamstore.NewStreamer(op.kopia, op.acct.ID(), op.Selectors.PathService()) + ) + + // ----- + // Setup + // ----- + + ctx, end := diagnostics.Span(ctx, "operations:export:run") + defer func() { + end() + // wait for the progress display to clean up + observe.Complete() + }() + + ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx}) + defer flushMetrics() + + ctx = clues.Add( + ctx, + "tenant_id", clues.Hide(op.acct.ID()), + "backup_id", op.BackupID, + "service", op.Selectors.Service) + + defer func() { + op.bus.Event( + ctx, + events.ExportEnd, + map[string]any{ + events.BackupID: op.BackupID, + events.DataRetrieved: op.Results.BytesRead, + events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), + events.EndTime: dttm.Format(op.Results.CompletedAt), + events.ItemsRead: op.Results.ItemsRead, + events.ItemsWritten: op.Results.ItemsWritten, + events.Resources: op.Results.ResourceOwners, + events.ExportID: opStats.exportID, + events.Service: op.Selectors.Service.String(), + events.StartTime: dttm.Format(op.Results.StartedAt), + events.Status: op.Status.String(), + }) + }() + + // ----- + // Execution + // ----- + + expCollections, err := op.do(ctx, &opStats, sstore, start) + if err != nil { + // No return here! We continue down to persistResults, even in case of failure. + logger.CtxErr(ctx, err).Error("running export") + + if errors.Is(err, kopia.ErrNoRestorePath) { + op.Errors.Fail(clues.New("empty backup or unknown path provided")) + } + + op.Errors.Fail(clues.Wrap(err, "running export")) + } + + finalizeErrorHandling(ctx, op.Options, op.Errors, "running export") + LogFaultErrors(ctx, op.Errors.Errors(), "running export") + + // ----- + // Persistence + // ----- + + err = op.finalizeMetrics(ctx, start, &opStats) + if err != nil { + op.Errors.Fail(clues.Wrap(err, "finalizing export metrics")) + return nil, op.Errors.Failure() + } + + logger.Ctx(ctx).Infow("completed export", "results", op.Results) + + return expCollections, nil +} + +func (op *ExportOperation) do( + ctx context.Context, + opStats *exportStats, + detailsStore streamstore.Reader, + start time.Time, +) ([]export.Collection, error) { + logger.Ctx(ctx). + With("control_options", op.Options, "selectors", op.Selectors). + Info("exporting selection") + + bup, deets, err := getBackupAndDetailsFromID( + ctx, + op.BackupID, + op.store, + detailsStore, + op.Errors) + if err != nil { + return nil, clues.Wrap(err, "getting backup and details") + } + + observe.Message(ctx, "Exporting", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) + + paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.ec, op.Errors) + if err != nil { + return nil, clues.Wrap(err, "formatting paths from details") + } + + ctx = clues.Add( + ctx, + "resource_owner_id", bup.Selector.ID(), + "resource_owner_name", clues.Hide(bup.Selector.Name()), + "details_entries", len(deets.Entries), + "details_paths", len(paths), + "backup_snapshot_id", bup.SnapshotID, + "backup_version", bup.Version) + + op.bus.Event( + ctx, + events.ExportStart, + map[string]any{ + events.StartTime: start, + events.BackupID: op.BackupID, + events.BackupCreateTime: bup.CreationTime, + events.ExportID: opStats.exportID, + }) + + observe.Message(ctx, fmt.Sprintf("Discovered %d items in backup %s to export", len(paths), op.BackupID)) + + kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") + defer close(kopiaComplete) + + dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) + if err != nil { + return nil, clues.Wrap(err, "producing collections to export") + } + + kopiaComplete <- struct{}{} + + ctx = clues.Add(ctx, "coll_count", len(dcs)) + + // should always be 1, since backups are 1:1 with resourceOwners. + opStats.resourceCount = 1 + opStats.cs = dcs + + expCollections, err := exportRestoreCollections( + ctx, + op.ec, + bup.Version, + op.Selectors, + op.ExportCfg, + op.Options, + dcs, + op.Errors) + if err != nil { + return nil, clues.Wrap(err, "exporting collections") + } + + opStats.ctrl = op.ec.Wait() + + logger.Ctx(ctx).Debug(opStats.ctrl) + + if op.ExportCfg.Archive { + zc, err := archive.ZipExportCollection(ctx, expCollections) + if err != nil { + return nil, clues.Wrap(err, "zipping export collections") + } + + return []export.Collection{zc}, nil + } + + return expCollections, nil +} + +// persists details and statistics about the export operation. +func (op *ExportOperation) finalizeMetrics( + ctx context.Context, + started time.Time, + opStats *exportStats, +) error { + op.Results.StartedAt = started + op.Results.CompletedAt = time.Now() + + op.Status = Completed + + if op.Errors.Failure() != nil { + op.Status = Failed + } + + op.Results.BytesRead = opStats.bytesRead.NumBytes + op.Results.ItemsRead = len(opStats.cs) // TODO: file count, not collection count + op.Results.ResourceOwners = opStats.resourceCount + + if opStats.ctrl == nil { + op.Status = Failed + return clues.New("restoration never completed") + } + + if op.Status != Failed && opStats.ctrl.IsZero() { + op.Status = NoData + } + + // We don't have data on what all items were written + // op.Results.ItemsWritten = opStats.ctrl.Successes + + return op.Errors.Failure() +} + +// --------------------------------------------------------------------------- +// Exporter funcs +// --------------------------------------------------------------------------- + +func exportRestoreCollections( + ctx context.Context, + ec inject.ExportConsumer, + backupVersion int, + sel selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, +) ([]export.Collection, error) { + complete := observe.MessageWithCompletion(ctx, "Preparing export") + defer func() { + complete <- struct{}{} + close(complete) + }() + + expCollections, err := ec.ProduceExportCollections( + ctx, + backupVersion, + sel, + exportCfg, + opts, + dcs, + errs) + if err != nil { + return nil, clues.Wrap(err, "exporting collections") + } + + return expCollections, nil +} diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go new file mode 100644 index 000000000..10dec2ab1 --- /dev/null +++ b/src/internal/operations/export_test.go @@ -0,0 +1,321 @@ +package operations + +import ( + "archive/zip" + "bytes" + "context" + "io" + "strings" + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/archive" + "github.com/alcionai/corso/src/internal/data" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" + "github.com/alcionai/corso/src/internal/m365/mock" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +type ExportOpSuite struct { + tester.Suite +} + +func TestExportOpSuite(t *testing.T) { + suite.Run(t, &ExportOpSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportOpSuite) TestExportOperation_PersistResults() { + var ( + kw = &kopia.Wrapper{} + sw = &store.Wrapper{} + ctrl = &mock.Controller{} + now = time.Now() + exportCfg = control.DefaultExportConfig() + ) + + table := []struct { + expectStatus OpStatus + expectErr assert.ErrorAssertionFunc + stats exportStats + fail error + }{ + { + expectStatus: Completed, + expectErr: assert.NoError, + stats: exportStats{ + resourceCount: 1, + bytesRead: &stats.ByteCounter{ + NumBytes: 42, + }, + cs: []data.RestoreCollection{ + data.NoFetchRestoreCollection{ + Collection: &exchMock.DataCollection{}, + }, + }, + ctrl: &data.CollectionStats{ + Objects: 1, + Successes: 1, + }, + }, + }, + { + expectStatus: Failed, + expectErr: assert.Error, + fail: assert.AnError, + stats: exportStats{ + bytesRead: &stats.ByteCounter{}, + ctrl: &data.CollectionStats{}, + }, + }, + { + expectStatus: NoData, + expectErr: assert.NoError, + stats: exportStats{ + bytesRead: &stats.ByteCounter{}, + cs: []data.RestoreCollection{}, + ctrl: &data.CollectionStats{}, + }, + }, + } + for _, test := range table { + suite.Run(test.expectStatus.String(), func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + op, err := NewExportOperation( + ctx, + control.DefaultOptions(), + kw, + sw, + ctrl, + account.Account{}, + "foo", + selectors.Selector{DiscreteOwner: "test"}, + exportCfg, + evmock.NewBus()) + require.NoError(t, err, clues.ToCore(err)) + + op.Errors.Fail(test.fail) + + err = op.finalizeMetrics(ctx, now, &test.stats) + test.expectErr(t, err, clues.ToCore(err)) + + assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status") + assert.Equal(t, len(test.stats.cs), op.Results.ItemsRead, "items read") + assert.Equal(t, test.stats.bytesRead.NumBytes, op.Results.BytesRead, "resource owners") + assert.Equal(t, test.stats.resourceCount, op.Results.ResourceOwners, "resource owners") + assert.Equal(t, now, op.Results.StartedAt, "started at") + assert.Less(t, now, op.Results.CompletedAt, "completed at") + }) + } +} + +type expCol struct { + base string + items []export.Item +} + +func (ec expCol) BasePath() string { return ec.base } +func (ec expCol) Items(ctx context.Context) <-chan export.Item { + ch := make(chan export.Item) + + go func() { + defer close(ch) + + for _, item := range ec.items { + ch <- item + } + }() + + return ch +} + +// ReadSeekCloser implements io.ReadSeekCloser. +type ReadSeekCloser struct { + *bytes.Reader +} + +// NewReadSeekCloser creates a new ReadSeekCloser from a byte slice. +func NewReadSeekCloser(byts []byte) *ReadSeekCloser { + return &ReadSeekCloser{ + Reader: bytes.NewReader(byts), + } +} + +// Close implements the io.Closer interface. +func (r *ReadSeekCloser) Close() error { + // Nothing to close for a byte slice. + return nil +} + +func (suite *ExportOpSuite) TestZipExports() { + table := []struct { + name string + collection []export.Collection + shouldErr bool + readErr bool + }{ + { + name: "nothing", + collection: []export.Collection{}, + shouldErr: true, + }, + { + name: "empty", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{}, + }, + }, + }, + { + name: "one item", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id1", + Data: export.ItemData{ + Name: "test", + Body: NewReadSeekCloser([]byte("test")), + }, + }, + }, + }, + }, + }, + { + name: "multiple items", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id1", + Data: export.ItemData{ + Name: "test", + Body: NewReadSeekCloser([]byte("test")), + }, + }, + }, + }, + expCol{ + base: "/fold", + items: []export.Item{ + { + ID: "id2", + Data: export.ItemData{ + Name: "test2", + Body: NewReadSeekCloser([]byte("test2")), + }, + }, + }, + }, + }, + }, + { + name: "one item with err", + collection: []export.Collection{ + expCol{ + base: "", + items: []export.Item{ + { + ID: "id3", + Error: assert.AnError, + }, + }, + }, + }, + readErr: true, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + zc, err := archive.ZipExportCollection(ctx, test.collection) + + if test.shouldErr { + assert.Error(t, err, "error") + return + } + + require.NoError(t, err, "error") + assert.Empty(t, zc.BasePath(), "base path") + + zippedItems := []export.ItemData{} + + count := 0 + for item := range zc.Items(ctx) { + assert.True(t, strings.HasPrefix(item.Data.Name, "Corso_Export_"), "name prefix") + assert.True(t, strings.HasSuffix(item.Data.Name, ".zip"), "name suffix") + + data, err := io.ReadAll(item.Data.Body) + if test.readErr { + assert.Error(t, err, "read error") + return + } + + size := int64(len(data)) + + item.Data.Body.Close() + + reader, err := zip.NewReader(bytes.NewReader(data), size) + require.NoError(t, err, "zip reader") + + for _, f := range reader.File { + rc, err := f.Open() + assert.NoError(t, err, "open file in zip") + + data, err := io.ReadAll(rc) + require.NoError(t, err, "read zip file content") + + rc.Close() + + zippedItems = append(zippedItems, export.ItemData{ + Name: f.Name, + Body: NewReadSeekCloser([]byte(data)), + }) + } + + count++ + } + + assert.Equal(t, 1, count, "single item") + + expectedZippedItems := []export.ItemData{} + for _, col := range test.collection { + for item := range col.Items(ctx) { + if col.BasePath() != "" { + item.Data.Name = strings.Join([]string{col.BasePath(), item.Data.Name}, "/") + } + _, err := item.Data.Body.(io.ReadSeeker).Seek(0, io.SeekStart) + require.NoError(t, err, "seek") + expectedZippedItems = append(expectedZippedItems, item.Data) + } + } + assert.Equal(t, expectedZippedItems, zippedItems, "items") + }) + } +} diff --git a/src/internal/operations/help_test.go b/src/internal/operations/help_test.go index 0951572ba..bd8509a1a 100644 --- a/src/internal/operations/help_test.go +++ b/src/internal/operations/help_test.go @@ -27,7 +27,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() @@ -36,7 +36,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail() diff --git a/src/internal/operations/inject/containers.go b/src/internal/operations/inject/containers.go new file mode 100644 index 000000000..f44bb7e66 --- /dev/null +++ b/src/internal/operations/inject/containers.go @@ -0,0 +1,18 @@ +package inject + +import ( + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// RestoreConsumerConfig is a container-of-things for holding options and +// configurations from various packages, which are widely used by all +// restore consumers independent of service or data category. +type RestoreConsumerConfig struct { + BackupVersion int + Options control.Options + ProtectedResource idname.Provider + RestoreConfig control.RestoreConfig + Selector selectors.Selector +} diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index ae2c8d534..5a30fe8a3 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -36,16 +37,60 @@ type ( RestoreConsumer interface { ConsumeRestoreCollections( ctx context.Context, - backupVersion int, - selector selectors.Selector, - restoreCfg control.RestoreConfig, - opts control.Options, + rcc RestoreConsumerConfig, dcs []data.RestoreCollection, errs *fault.Bus, ctr *count.Bus, ) (*details.Details, error) Wait() *data.CollectionStats + + CacheItemInfoer + PopulateProtectedResourceIDAndNamer + } + + CacheItemInfoer interface { + // CacheItemInfo is used by the consumer to cache metadata that is + // sourced from per-item info, but may be valuable to the restore at + // large. + // Ex: pairing drive ids with drive names as they appeared at the time + // of backup. + CacheItemInfo(v details.ItemInfo) + } + + ExportConsumer interface { + ProduceExportCollections( + ctx context.Context, + backupVersion int, + selector selectors.Selector, + exportCfg control.ExportConfig, + opts control.Options, + dcs []data.RestoreCollection, + errs *fault.Bus, + ) ([]export.Collection, error) + + Wait() *data.CollectionStats + + CacheItemInfoer + } + + PopulateProtectedResourceIDAndNamer interface { + // PopulateProtectedResourceIDAndName takes the provided owner identifier and produces + // the owner's name and ID from that value. Returns an error if the owner is + // not recognized by the current tenant. + // + // The id-name cacher should be optional. Some processes will look up all owners in + // the tenant before reaching this step. In that case, the data gets handed + // down for this func to consume instead of performing further queries. The + // data gets stored inside the controller instance for later re-use. + PopulateProtectedResourceIDAndName( + ctx context.Context, + owner string, // input value, can be either id or name + ins idname.Cacher, + ) ( + id, name string, + err error, + ) } RepoMaintenancer interface { diff --git a/src/internal/operations/maintenance_test.go b/src/internal/operations/maintenance_test.go index a625af482..6c822e1cd 100644 --- a/src/internal/operations/maintenance_test.go +++ b/src/internal/operations/maintenance_test.go @@ -40,7 +40,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { ctx, flush := tester.NewContext(t) defer flush() - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) kw, err := kopia.NewWrapper(k) @@ -54,7 +54,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() { mo, err := NewMaintenanceOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, repository.Maintenance{ Type: repository.MetadataMaintenance, diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 5e1c79e4f..1c5d1716c 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -23,7 +23,7 @@ func produceManifestsAndMetadata( ctx context.Context, bf inject.BaseFinder, rp inject.RestoreProducer, - reasons, fallbackReasons []kopia.Reason, + reasons, fallbackReasons []kopia.Reasoner, tenantID string, getMetadata bool, ) (kopia.BackupBases, []data.RestoreCollection, bool, error) { @@ -47,8 +47,8 @@ func produceManifestsAndMetadata( bb = bb.MergeBackupBases( ctx, fbb, - func(r kopia.Reason) string { - return r.Service.String() + r.Category.String() + func(r kopia.Reasoner) string { + return r.Service().String() + r.Category().String() }) if !getMetadata { @@ -115,9 +115,9 @@ func collectMetadata( Append(fn). ToServiceCategoryMetadataPath( tenantID, - reason.ResourceOwner, - reason.Service, - reason.Category, + reason.ProtectedResource(), + reason.Service(), + reason.Category(), true) if err != nil { return nil, clues. diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index e4ae9b6d3..5fdf22424 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -47,7 +47,7 @@ type mockBackupFinder struct { func (bf *mockBackupFinder) FindBases( _ context.Context, - reasons []kopia.Reason, + reasons []kopia.Reasoner, _ map[string]string, ) kopia.BackupBases { if len(reasons) == 0 { @@ -58,7 +58,7 @@ func (bf *mockBackupFinder) FindBases( return kopia.NewMockBackupBases() } - b := bf.data[reasons[0].ResourceOwner] + b := bf.data[reasons[0].ProtectedResource()] if b == nil { return kopia.NewMockBackupBases() } @@ -102,7 +102,7 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { table := []struct { name string manID string - reasons []kopia.Reason + reasons []kopia.Reasoner fileNames []string expectPaths func(*testing.T, []string) []path.Path expectErr error @@ -110,12 +110,8 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "single reason, single file", manID: "single single", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -133,12 +129,8 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "single reason, multiple files", manID: "single multi", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -156,17 +148,9 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "multiple reasons, single file", manID: "multi single", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -187,17 +171,9 @@ func (suite *OperationsManifestsUnitSuite) TestCollectMetadata() { { name: "multiple reasons, multiple file", manID: "multi multi", - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory), }, expectPaths: func(t *testing.T, files []string) []path.Path { ps := make([]path.Path, 0, len(files)) @@ -243,17 +219,13 @@ func buildReasons( ro string, service path.ServiceType, cats ...path.CategoryType, -) []kopia.Reason { - var reasons []kopia.Reason +) []kopia.Reasoner { + var reasons []kopia.Reasoner for _, cat := range cats { reasons = append( reasons, - kopia.Reason{ - ResourceOwner: ro, - Service: service, - Category: cat, - }) + kopia.NewReason("", ro, service, cat)) } return reasons @@ -280,7 +252,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { name string bf *mockBackupFinder rp mockRestoreProducer - reasons []kopia.Reason + reasons []kopia.Reasoner getMeta bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc @@ -291,7 +263,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { { name: "don't get metadata, no mans", rp: mockRestoreProducer{}, - reasons: []kopia.Reason{}, + reasons: []kopia.Reasoner{}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -308,12 +280,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: false, assertErr: assert.NoError, @@ -333,12 +301,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -365,17 +329,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id1"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -421,12 +377,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -454,12 +406,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, }, }, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.NoError, @@ -480,12 +428,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { }, }, rp: mockRestoreProducer{err: assert.AnError}, - reasons: []kopia.Reason{ - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), }, getMeta: true, assertErr: assert.Error, @@ -588,24 +532,24 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb } } - emailReason := kopia.Reason{ - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + emailReason := kopia.NewReason( + "", + ro, + path.ExchangeService, + path.EmailCategory) - fbEmailReason := kopia.Reason{ - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.EmailCategory, - } + fbEmailReason := kopia.NewReason( + "", + fbro, + path.ExchangeService, + path.EmailCategory) table := []struct { name string bf *mockBackupFinder rp mockRestoreProducer - reasons []kopia.Reason - fallbackReasons []kopia.Reason + reasons []kopia.Reasoner + fallbackReasons []kopia.Reasoner getMeta bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc @@ -624,7 +568,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb }, }, rp: mockRestoreProducer{}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: false, assertErr: assert.NoError, assertB: assert.False, @@ -649,7 +593,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - fallbackReasons: []kopia.Reason{fbEmailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -680,8 +624,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -708,8 +652,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -744,8 +688,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -776,8 +720,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -808,8 +752,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id2"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{fbEmailReason}, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, getMeta: true, assertErr: assert.NoError, assertB: assert.True, @@ -838,21 +782,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{ + reasons: []kopia.Reasoner{ emailReason, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, - fallbackReasons: []kopia.Reason{ + fallbackReasons: []kopia.Reasoner{ fbEmailReason, - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -882,13 +818,9 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{emailReason}, - fallbackReasons: []kopia.Reason{ - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{ + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, @@ -921,21 +853,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, }, }, - reasons: []kopia.Reason{ + reasons: []kopia.Reasoner{ emailReason, - { - ResourceOwner: ro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory), }, - fallbackReasons: []kopia.Reason{ + fallbackReasons: []kopia.Reasoner{ fbEmailReason, - { - ResourceOwner: fbro, - Service: path.ExchangeService, - Category: path.ContactsCategory, - }, + kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory), }, getMeta: true, assertErr: assert.NoError, diff --git a/src/internal/operations/operation_test.go b/src/internal/operations/operation_test.go index 4cbbe2a0c..b615a492e 100644 --- a/src/internal/operations/operation_test.go +++ b/src/internal/operations/operation_test.go @@ -26,7 +26,7 @@ func TestOperationSuite(t *testing.T) { func (suite *OperationSuite) TestNewOperation() { t := suite.T() - op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) + op := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, nil, nil) assert.Greater(t, op.CreatedAt, time.Time{}) } @@ -46,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() { } for _, test := range table { suite.Run(test.name, func() { - err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() + err := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() test.errCheck(suite.T(), err, clues.ToCore(err)) }) } diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index e77b1104b..141300f6a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/events" @@ -172,7 +173,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De logger.CtxErr(ctx, err).Error("running restore") if errors.Is(err, kopia.ErrNoRestorePath) { - op.Errors.Fail(clues.New("empty backup or unknown path provided")) + op.Errors.Fail(clues.Wrap(err, "empty backup or unknown path provided")) } op.Errors.Fail(clues.Wrap(err, "running restore")) @@ -217,17 +218,33 @@ func (op *RestoreOperation) do( return nil, clues.Wrap(err, "getting backup and details") } - observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) + restoreToProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector) + if err != nil { + return nil, clues.Wrap(err, "getting destination protected resource") + } - paths, err := formatDetailsForRestoration(ctx, bup.Version, op.Selectors, deets, op.Errors) + ctx = clues.Add( + ctx, + "backup_protected_resource_id", bup.Selector.ID(), + "backup_protected_resource_name", clues.Hide(bup.Selector.Name()), + "restore_protected_resource_id", restoreToProtectedResource.ID(), + "restore_protected_resource_name", clues.Hide(restoreToProtectedResource.Name())) + + observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreToProtectedResource.Name())) + + paths, err := formatDetailsForRestoration( + ctx, + bup.Version, + op.Selectors, + deets, + op.rc, + op.Errors) if err != nil { return nil, clues.Wrap(err, "formatting paths from details") } ctx = clues.Add( ctx, - "resource_owner_id", bup.Selector.ID(), - "resource_owner_name", clues.Hide(bup.Selector.Name()), "details_entries", len(deets.Entries), "details_paths", len(paths), "backup_snapshot_id", bup.SnapshotID, @@ -248,7 +265,12 @@ func (op *RestoreOperation) do( kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") defer close(kopiaComplete) - dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) + dcs, err := op.kopia.ProduceRestoreCollections( + ctx, + bup.SnapshotID, + paths, + opStats.bytesRead, + op.Errors) if err != nil { return nil, clues.Wrap(err, "producing collections to restore") } @@ -265,6 +287,7 @@ func (op *RestoreOperation) do( ctx, op.rc, bup.Version, + restoreToProtectedResource, op.Selectors, op.RestoreCfg, op.Options, @@ -315,6 +338,24 @@ func (op *RestoreOperation) persistResults( return op.Errors.Failure() } +func chooseRestoreResource( + ctx context.Context, + pprian inject.PopulateProtectedResourceIDAndNamer, + restoreCfg control.RestoreConfig, + orig idname.Provider, +) (idname.Provider, error) { + if len(restoreCfg.ProtectedResource) == 0 { + return orig, nil + } + + id, name, err := pprian.PopulateProtectedResourceIDAndName( + ctx, + restoreCfg.ProtectedResource, + nil) + + return idname.NewProvider(id, name), clues.Stack(err).OrNil() +} + // --------------------------------------------------------------------------- // Restorer funcs // --------------------------------------------------------------------------- @@ -323,6 +364,7 @@ func consumeRestoreCollections( ctx context.Context, rc inject.RestoreConsumer, backupVersion int, + toProtectedResource idname.Provider, sel selectors.Selector, restoreCfg control.RestoreConfig, opts control.Options, @@ -336,15 +378,15 @@ func consumeRestoreCollections( close(complete) }() - deets, err := rc.ConsumeRestoreCollections( - ctx, - backupVersion, - sel, - restoreCfg, - opts, - dcs, - errs, - ctr) + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: toProtectedResource, + RestoreConfig: restoreCfg, + Selector: sel, + } + + deets, err := rc.ConsumeRestoreCollections(ctx, rcc, dcs, errs, ctr) if err != nil { return nil, clues.Wrap(err, "restoring collections") } @@ -359,6 +401,7 @@ func formatDetailsForRestoration( backupVersion int, sel selectors.Selector, deets *details.Details, + cii inject.CacheItemInfoer, errs *fault.Bus, ) ([]path.RestorePaths, error) { fds, err := sel.Reduce(ctx, deets, errs) @@ -366,6 +409,11 @@ func formatDetailsForRestoration( return nil, err } + // allow restore controllers to iterate over item metadata + for _, ent := range fds.Entries { + cii.CacheItemInfo(ent.ItemInfo) + } + paths, err := pathtransformer.GetPaths(ctx, backupVersion, fds.Items(), errs) if err != nil { return nil, clues.Wrap(err, "getting restore paths") diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index f02ee1731..c97812a63 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/dttm" - inMock "github.com/alcionai/corso/src/internal/common/idname/mock" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -21,7 +21,6 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/resource" - "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/internal/tester" @@ -32,7 +31,6 @@ import ( "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/selectors" - "github.com/alcionai/corso/src/pkg/services/m365/api" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" "github.com/alcionai/corso/src/pkg/store" ) @@ -41,15 +39,15 @@ import ( // unit // --------------------------------------------------------------------------- -type RestoreOpSuite struct { +type RestoreOpUnitSuite struct { tester.Suite } -func TestRestoreOpSuite(t *testing.T) { - suite.Run(t, &RestoreOpSuite{Suite: tester.NewUnitSuite(t)}) +func TestRestoreOpUnitSuite(t *testing.T) { + suite.Run(t, &RestoreOpUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { +func (suite *RestoreOpUnitSuite) TestRestoreOperation_PersistResults() { var ( kw = &kopia.Wrapper{} sw = &store.Wrapper{} @@ -111,7 +109,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { op, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), kw, sw, ctrl, @@ -139,17 +137,79 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { } } +func (suite *RestoreOpUnitSuite) TestChooseRestoreResource() { + var ( + id = "id" + name = "name" + cfgWithPR = control.DefaultRestoreConfig(dttm.HumanReadable) + ) + + cfgWithPR.ProtectedResource = "cfgid" + + table := []struct { + name string + cfg control.RestoreConfig + ctrl *mock.Controller + orig idname.Provider + expectErr assert.ErrorAssertionFunc + expectProvider assert.ValueAssertionFunc + expectID string + expectName string + }{ + { + name: "use original", + cfg: control.DefaultRestoreConfig(dttm.HumanReadable), + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: "oid", + expectName: "oname", + }, + { + name: "look up resource with iface", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceID: id, + ProtectedResourceName: name, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.NoError, + expectID: id, + expectName: name, + }, + { + name: "error looking up protected resource", + cfg: cfgWithPR, + ctrl: &mock.Controller{ + ProtectedResourceErr: assert.AnError, + }, + orig: idname.NewProvider("oid", "oname"), + expectErr: assert.Error, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + result, err := chooseRestoreResource(ctx, test.ctrl, test.cfg, test.orig) + test.expectErr(t, err, clues.ToCore(err)) + require.NotNil(t, result) + assert.Equal(t, test.expectID, result.ID()) + assert.Equal(t, test.expectName, result.Name()) + }) + } +} + // --------------------------------------------------------------------------- // integration // --------------------------------------------------------------------------- -type bupResults struct { - selectorResourceOwners []string - backupID model.StableID - items int - ctrl *m365.Controller -} - type RestoreOpIntegrationSuite struct { tester.Suite @@ -183,7 +243,7 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { suite.acct = tconfig.NewM365Account(t) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) suite.kopiaCloser = func(ctx context.Context) { @@ -227,7 +287,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { sw = &store.Wrapper{} ctrl = &mock.Controller{} restoreCfg = testdata.DefaultRestoreConfig("") - opts = control.Defaults() + opts = control.DefaultOptions() ) table := []struct { @@ -267,192 +327,6 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() { } } -func setupExchangeBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - users = []string{owner} - esel = selectors.NewExchangeBackup(users) - ) - - esel.DiscreteOwner = owner - esel.Include( - esel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()), - esel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), - esel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Users, esel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: users, - backupID: bo.Results.BackupID, - // Discount metadata collection files (1 delta and one prev path for each category). - // These meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func setupSharePointBackup( - t *testing.T, - kw *kopia.Wrapper, - sw *store.Wrapper, - acct account.Account, - owner string, -) bupResults { - ctx, flush := tester.NewContext(t) - defer flush() - - var ( - sites = []string{owner} - ssel = selectors.NewSharePointBackup(sites) - ) - - // assume a folder name "test" exists in the drive. - // this is brittle, and requires us to backfill anytime - // the site under test changes, but also prevents explosive - // growth from re-backup/restore of restored files. - ssel.Include(ssel.LibraryFolders([]string{"test"}, selectors.PrefixMatch())) - ssel.DiscreteOwner = owner - - ctrl, sel := ControllerWithSelector(t, ctx, acct, resource.Sites, ssel.Selector, nil, nil) - - bo, err := NewBackupOperation( - ctx, - control.Defaults(), - kw, - sw, - ctrl, - acct, - sel, - inMock.NewProvider(owner, owner), - evmock.NewBus()) - require.NoError(t, err, clues.ToCore(err)) - - err = bo.Run(ctx) - require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, bo.Results.BackupID) - - return bupResults{ - selectorResourceOwners: sites, - backupID: bo.Results.BackupID, - // Discount metadata files (2: 1 delta, 1 prev path) - // assume only one folder, and therefore 1 dirmeta per drive - // (2 drives: documents and more documents) - // assume only one file in each folder, and therefore 1 meta per drive - // (2 drives: documents and more documents) - // Meta files are used to aid restore, but are not themselves - // restored (ie: counted as writes). - items: bo.Results.ItemsWritten - 6, - ctrl: ctrl, - } -} - -func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { - tables := []struct { - name string - owner string - restoreCfg control.RestoreConfig - getSelector func(t *testing.T, owners []string) selectors.Selector - setup func(t *testing.T, kw *kopia.Wrapper, sw *store.Wrapper, acct account.Account, owner string) bupResults - }{ - { - name: "Exchange_Restore", - owner: tconfig.M365UserID(suite.T()), - restoreCfg: testdata.DefaultRestoreConfig(""), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewExchangeRestore(owners) - rsel.Include(rsel.AllData()) - - return rsel.Selector - }, - setup: setupExchangeBackup, - }, - { - name: "SharePoint_Restore", - owner: tconfig.M365SiteID(suite.T()), - restoreCfg: control.DefaultRestoreConfig(dttm.SafeForTesting), - getSelector: func(t *testing.T, owners []string) selectors.Selector { - rsel := selectors.NewSharePointRestore(owners) - rsel.Include(rsel.Library(tconfig.LibraryDocuments), rsel.Library(tconfig.LibraryMoreDocuments)) - - return rsel.Selector - }, - setup: setupSharePointBackup, - }, - } - - for _, test := range tables { - suite.Run(test.name, func() { - var ( - t = suite.T() - mb = evmock.NewBus() - bup = test.setup(t, suite.kw, suite.sw, suite.acct, test.owner) - ) - - ctx, flush := tester.NewContext(t) - defer flush() - - require.NotZero(t, bup.items) - require.NotEmpty(t, bup.backupID) - - ro, err := NewRestoreOperation( - ctx, - control.Options{FailureHandling: control.FailFast}, - suite.kw, - suite.sw, - bup.ctrl, - tconfig.NewM365Account(t), - bup.backupID, - test.getSelector(t, bup.selectorResourceOwners), - test.restoreCfg, - mb, - count.New()) - require.NoError(t, err, clues.ToCore(err)) - - ds, err := ro.Run(ctx) - - require.NoError(t, err, "restoreOp.Run() %+v", clues.ToCore(err)) - require.NotEmpty(t, ro.Results, "restoreOp results") - require.NotNil(t, ds, "restored details") - assert.Equal(t, ro.Status, Completed, "restoreOp status") - assert.Equal(t, ro.Results.ItemsWritten, len(ds.Items()), "item write count matches len details") - assert.Less(t, 0, ro.Results.ItemsRead, "restore items read") - assert.Less(t, int64(0), ro.Results.BytesRead, "bytes read") - assert.Equal(t, 1, ro.Results.ResourceOwners, "resource Owners") - assert.NoError(t, ro.Errors.Failure(), "non-recoverable error", clues.ToCore(ro.Errors.Failure())) - assert.Empty(t, ro.Errors.Recovered(), "recoverable errors") - assert.Equal(t, bup.items, ro.Results.ItemsWritten, "backup and restore wrote the same num of items") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreStart], "restore-start events") - assert.Equal(t, 1, mb.TimesCalled[events.RestoreEnd], "restore-end events") - }) - } -} - func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { t := suite.T() @@ -472,12 +346,12 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() { suite.acct, resource.Users, rsel.PathService(), - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) ro, err := NewRestoreOperation( ctx, - control.Defaults(), + control.DefaultOptions(), suite.kw, suite.sw, ctrl, diff --git a/src/internal/operations/retention_config.go b/src/internal/operations/retention_config.go new file mode 100644 index 000000000..ca8504382 --- /dev/null +++ b/src/internal/operations/retention_config.go @@ -0,0 +1,77 @@ +package operations + +import ( + "context" + "time" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/crash" + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/stats" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + "github.com/alcionai/corso/src/pkg/count" +) + +// RetentionConfigOperation wraps an operation with restore-specific props. +type RetentionConfigOperation struct { + operation + Results RetentionConfigResults + rcOpts repository.Retention +} + +// RetentionConfigResults aggregate the details of the results of the operation. +type RetentionConfigResults struct { + stats.StartAndEndTime +} + +// NewRetentionConfigOperation constructs and validates an operation to change +// retention parameters. +func NewRetentionConfigOperation( + ctx context.Context, + opts control.Options, + kw *kopia.Wrapper, + rcOpts repository.Retention, + bus events.Eventer, +) (RetentionConfigOperation, error) { + op := RetentionConfigOperation{ + operation: newOperation(opts, bus, count.New(), kw, nil), + rcOpts: rcOpts, + } + + // Don't run validation because we don't populate the model store. + + return op, nil +} + +func (op *RetentionConfigOperation) Run(ctx context.Context) (err error) { + defer func() { + if crErr := crash.Recovery(ctx, recover(), "retention_config"); crErr != nil { + err = crErr + } + }() + + op.Results.StartedAt = time.Now() + + // TODO(ashmrtn): Send telemetry? + + return op.do(ctx) +} + +func (op *RetentionConfigOperation) do(ctx context.Context) error { + defer func() { + op.Results.CompletedAt = time.Now() + }() + + err := op.operation.kopia.SetRetentionParameters(ctx, op.rcOpts) + if err != nil { + op.Status = Failed + return clues.Wrap(err, "running retention config operation") + } + + op.Status = Completed + + return nil +} diff --git a/src/internal/operations/retention_config_test.go b/src/internal/operations/retention_config_test.go new file mode 100644 index 000000000..ce57cd879 --- /dev/null +++ b/src/internal/operations/retention_config_test.go @@ -0,0 +1,74 @@ +package operations + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + evmock "github.com/alcionai/corso/src/internal/events/mock" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/control/repository" + storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" +) + +type RetentionConfigOpIntegrationSuite struct { + tester.Suite +} + +func TestRetentionConfigOpIntegrationSuite(t *testing.T) { + suite.Run(t, &RetentionConfigOpIntegrationSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{storeTD.AWSStorageCredEnvs}), + }) +} + +func (suite *RetentionConfigOpIntegrationSuite) TestRepoRetentionConfig() { + var ( + t = suite.T() + // need to initialize the repository before we can test connecting to it. + st = storeTD.NewPrefixedS3Storage(t) + k = kopia.NewConn(st) + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) + + kw, err := kopia.NewWrapper(k) + // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe + // to close here. + k.Close(ctx) + + require.NoError(t, err, clues.ToCore(err)) + + defer kw.Close(ctx) + + // Only set extend locks parameter as other retention options require a bucket + // with object locking enabled. There's more complete tests in the kopia + // package. + rco, err := NewRetentionConfigOperation( + ctx, + control.DefaultOptions(), + kw, + repository.Retention{ + Extend: ptr.To(true), + }, + evmock.NewBus()) + require.NoError(t, err, clues.ToCore(err)) + + err = rco.Run(ctx) + assert.NoError(t, err, clues.ToCore(err)) + assert.Equal(t, Completed, rco.Status) + assert.NotZero(t, rco.Results.StartedAt) + assert.NotZero(t, rco.Results.CompletedAt) + assert.NotEqual(t, rco.Results.StartedAt, rco.Results.CompletedAt) +} diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index 647c7a397..33bd18a6a 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -67,9 +67,9 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Mail", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID return sel }, @@ -79,7 +79,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Contacts", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())) return sel }, @@ -89,7 +89,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { { name: "Calendar Events", selector: func() *selectors.ExchangeBackup { - sel := selectors.NewExchangeBackup([]string{suite.its.userID}) + sel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) return sel }, @@ -107,7 +107,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() { var ( mb = evmock.NewBus() sel = test.selector().Selector - opts = control.Defaults() + opts = control.DefaultOptions() whatSet = deeTD.CategoryFromRepoRef ) @@ -258,9 +258,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr // later on during the tests. Putting their identifiers into the selector // at this point is harmless. containers = []string{container1, container2, container3, containerRename} - sel = selectors.NewExchangeBackup([]string{suite.its.userID}) + sel = selectors.NewExchangeBackup([]string{suite.its.user.ID}) whatSet = deeTD.CategoryFromRepoRef - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ToggleFeatures = toggles @@ -278,7 +278,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr creds, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := api.NewClient(creds) + ac, err := api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -295,7 +295,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr mailDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.MessageWith( - suite.its.userID, suite.its.userID, suite.its.userID, + suite.its.user.ID, suite.its.user.ID, suite.its.user.ID, subject, body, body, now, now, now, now) } @@ -312,7 +312,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr eventDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.EventWith( - suite.its.userID, subject, body, body, + suite.its.user.ID, subject, body, body, exchMock.NoOriginalStartDate, now, now, exchMock.NoRecurrence, exchMock.NoAttendees, exchMock.NoAttachments, exchMock.NoCancelledOccurrences, @@ -578,7 +578,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr service, category, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, - creds.AzureTenantID, suite.its.userID, "", container3, + creds.AzureTenantID, suite.its.user.ID, "", container3, 2, version.Backup, gen.dbf) @@ -889,6 +889,26 @@ func (suite *ExchangeRestoreIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } +type clientItemPager interface { + GetItemsInContainerByCollisionKeyer[string] + GetItemIDsInContainer( + ctx context.Context, + userID, containerID string, + ) (map[string]struct{}, error) + GetContainerByName( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + userID, containerID string, + ) (map[string]string, error) + CreateContainer( + ctx context.Context, + userID, parentContainerID, containerName string, + ) (graph.Container, error) +} + func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptions() { t := suite.T() @@ -897,7 +917,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // a backup is required to run restores - baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) baseSel.Include( // events cannot be run, for the same reason as incremental backups: the user needs // to have their account recycled. @@ -905,11 +925,11 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) - baseSel.DiscreteOwner = suite.its.userID + baseSel.DiscreteOwner = suite.its.user.ID var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) @@ -921,25 +941,28 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio require.NoError(t, err, clues.ToCore(err)) var ( - restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") - sel = rsel.Selector - userID = sel.ID() - cIDs = map[path.CategoryType]string{ - path.ContactsCategory: "", - path.EmailCategory: "", - path.EventsCategory: "", + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_adv_restore") + sel = rsel.Selector + userID = sel.ID() + countItemsInRestore int + + itemIDs = map[path.CategoryType]map[string]struct{}{} + collisionKeys = map[path.CategoryType]map[string]string{} + containerIDs = map[path.CategoryType]string{} + parentContainerIDs = map[path.CategoryType]string{ + path.EmailCategory: api.MsgFolderRoot, + } + parentContainerNames = map[path.CategoryType][]string{ + path.EmailCategory: {api.MailInbox}, + path.ContactsCategory: {}, + path.EventsCategory: {}, + } + + testCategories = map[path.CategoryType]clientItemPager{ + path.ContactsCategory: suite.its.ac.Contacts(), + path.EmailCategory: suite.its.ac.Mail(), + // path.EventsCategory: suite.its.ac.Events(), } - collKeys = map[path.CategoryType]map[string]string{} - countContactsInRestore int - acCont = suite.its.ac.Contacts() - contactIDs map[string]struct{} - countEmailsInRestore int - acMail = suite.its.ac.Mail() - mailIDs map[string]struct{} - countItemsInRestore int - // countEventsInRestore int - // acEvts = suite.its.ac.Events() - // eventIDs = []string{} ) // initial restore @@ -971,61 +994,27 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio // get all files in folder, use these as the base // set of files to compare against. - // --- contacts + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - contGC, err := acCont.GetContainerByName(ctx, userID, "", restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - cIDs[path.ContactsCategory] = ptr.Val(contGC.GetId()) + containers := append([]string{restoreCfg.Location}, parentContainerNames[cat]...) - collKeys[path.ContactsCategory], err = acCont.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - countContactsInRestore = len(collKeys[path.ContactsCategory]) - t.Log(countContactsInRestore, "contacts restored") + itemIDs[cat], collisionKeys[cat], containerIDs[cat] = getCollKeysAndItemIDs( + t, + ctx, + ac, + userID, + parentContainerIDs[cat], + containers...) - contactIDs, err = acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + countItemsInRestore += len(collisionKeys[cat]) + }) + } - // --- events - - // gc, err = acEvts.GetContainerByName(ctx, userID, "", restoreCfg.Location) - // require.NoError(t, err, clues.ToCore(err)) - - // restoredContainerID[path.EventsCategory] = ptr.Val(gc.GetId()) - - // collKeys[path.EventsCategory], err = acEvts.GetItemsInContainerByCollisionKey( - // ctx, - // userID, - // cIDs[path.EventsCategory]) - // require.NoError(t, err, clues.ToCore(err)) - // countEventsInRestore = len(collKeys[path.EventsCategory]) - // t.Log(countContactsInRestore, "events restored") - - mailGC, err := acMail.GetContainerByName(ctx, userID, api.MsgFolderRoot, restoreCfg.Location) - require.NoError(t, err, clues.ToCore(err)) - - mailGC, err = acMail.GetContainerByName(ctx, userID, ptr.Val(mailGC.GetId()), api.MailInbox) - require.NoError(t, err, clues.ToCore(err)) - - cIDs[path.EmailCategory] = ptr.Val(mailGC.GetId()) - - // --- mail - - collKeys[path.EmailCategory], err = acMail.GetItemsInContainerByCollisionKey( - ctx, - userID, - cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - countEmailsInRestore = len(collKeys[path.EmailCategory]) - t.Log(countContactsInRestore, "emails restored") - - mailIDs, err = acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - countItemsInRestore = countContactsInRestore + countEmailsInRestore // + countEventsInRestore checkRestoreCounts(t, ctr1, 0, 0, countItemsInRestore) }) @@ -1062,43 +1051,30 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio checkRestoreCounts(t, ctr2, countItemsInRestore, 0, 0) - // --- contacts + result := map[string]string{} - // get all files in folder, use these as the base - // set of files to compare against. - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + ctx, flush := tester.NewContext(t) + defer flush() - assert.Equal(t, contactIDs, currentContactIDs, "ids are equal") + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - // --- events + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) - - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, mailIDs, currentMailIDs, "ids are equal") + assert.Equal(t, itemIDs[cat], currentIDs, "ids are equal") + }) + } assert.Len(t, result, 0, "no new items should get added") }) @@ -1136,60 +1112,40 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been replaced") - - // --- contacts - - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) - - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, len(contactIDs), len(currentContactIDs), "count of ids are equal") - for orig := range contactIDs { - assert.NotContains(t, currentContactIDs, orig, "original item should not exist after replacement") - } - - contactIDs = currentContactIDs - - // --- events - - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) - - // --- mail - - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been replaced") checkRestoreCounts(t, ctr3, 0, countItemsInRestore, 0) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) + result := map[string]string{} - assert.Equal(t, len(mailIDs), len(currentMailIDs), "count of ids are equal") - for orig := range mailIDs { - assert.NotContains(t, currentMailIDs, orig, "original item should not exist after replacement") + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) + + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) + + assert.Equal(t, len(itemIDs[cat]), len(currentIDs), "count of ids are equal") + for orig := range itemIDs[cat] { + assert.NotContains(t, currentIDs, orig, "original item should not exist after replacement") + } + + itemIDs[cat] = currentIDs + }) } - mailIDs = currentMailIDs - assert.Len(t, result, 0, "all items should have been replaced") }) @@ -1226,45 +1182,35 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio } } - assert.Len( - t, - filtEnts, - countItemsInRestore, - "every item should have been copied") + assert.Len(t, filtEnts, countItemsInRestore, "every item should have been copied") checkRestoreCounts(t, ctr4, 0, 0, countItemsInRestore) - result := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.ContactsCategory], - GetItemsInContainerByCollisionKeyer[string](acCont), - collKeys[path.ContactsCategory]) + result := map[string]string{} - currentContactIDs, err := acCont.GetItemIDsInContainer(ctx, userID, cIDs[path.ContactsCategory]) - require.NoError(t, err, clues.ToCore(err)) + for cat, ac := range testCategories { + suite.Run(cat.String(), func() { + t := suite.T() - assert.Equal(t, 2*len(contactIDs), len(currentContactIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentContactIDs), maps.Keys(contactIDs), "original item should exist after copy") + ctx, flush := tester.NewContext(t) + defer flush() - // m = checkCollisionKeyResults(t, ctx, userID, cIDs[path.EventsCategory], acEvts, collKeys[path.EventsCategory]) - // maps.Copy(result, m) + m := filterCollisionKeyResults( + t, + ctx, + userID, + containerIDs[cat], + GetItemsInContainerByCollisionKeyer[string](ac), + collisionKeys[cat]) + maps.Copy(result, m) - m := filterCollisionKeyResults( - t, - ctx, - userID, - cIDs[path.EmailCategory], - GetItemsInContainerByCollisionKeyer[string](acMail), - collKeys[path.EmailCategory]) - maps.Copy(result, m) + currentIDs, err := ac.GetItemIDsInContainer(ctx, userID, containerIDs[cat]) + require.NoError(t, err, clues.ToCore(err)) - currentMailIDs, err := acMail.GetItemIDsInContainer(ctx, userID, cIDs[path.EmailCategory]) - require.NoError(t, err, clues.ToCore(err)) - - assert.Equal(t, 2*len(mailIDs), len(currentMailIDs), "count of ids should be double from before") - assert.Subset(t, maps.Keys(currentMailIDs), maps.Keys(mailIDs), "original item should exist after copy") + assert.Equal(t, 2*len(itemIDs[cat]), len(currentIDs), "count of ids should be double from before") + assert.Subset(t, maps.Keys(currentIDs), maps.Keys(itemIDs[cat]), "original item should exist after copy") + }) + } // TODO: we have the option of modifying copy creations in exchange // so that the results don't collide. But we haven't made that @@ -1272,3 +1218,195 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio assert.Len(t, result, 0, "no items should have been added as copies") }) } + +func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtectedResource() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID}) + baseSel.Include( + // events cannot be run, for the same reason as incremental backups: the user needs + // to have their account recycled. + // base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()), + baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), + baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) + + baseSel.DiscreteOwner = suite.its.user.ID + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + rsel, err := baseSel.ToExchangeRestore() + require.NoError(t, err, clues.ToCore(err)) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_restore_to_user") + sel = rsel.Selector + userID = suite.its.user.ID + secondaryUserID = suite.its.secondaryUser.ID + uid = userID + acCont = suite.its.ac.Contacts() + acMail = suite.its.ac.Mail() + // acEvts = suite.its.ac.Events() + firstCtr = count.New() + ) + + restoreCfg.OnCollision = control.Copy + mb = evmock.NewBus() + + // first restore to the current user + + ro1, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + firstCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro1, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + + var ( + userItemIDs = map[path.CategoryType]map[string]struct{}{} + userCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat := path.ContactsCategory + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + userItemIDs[cat], userCollisionKeys[cat], _ = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // then restore to the secondary user + + uid = secondaryUserID + mb = evmock.NewBus() + secondCtr := count.New() + restoreCfg.ProtectedResource = uid + + ro2, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + secondCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro2, mb, false) + + var ( + secondaryItemIDs = map[path.CategoryType]map[string]struct{}{} + secondaryCollisionKeys = map[path.CategoryType]map[string]string{} + ) + + // --- contacts + cat = path.ContactsCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( + t, + ctx, + acCont, + uid, + "", + restoreCfg.Location) + + // --- events + // cat = path.EventsCategory + // secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( + // t, + // ctx, + // acEvts, + // uid, + // "", + // restoreCfg.Location) + + // --- mail + cat = path.EmailCategory + secondaryItemIDs[cat], secondaryCollisionKeys[cat], _ = getCollKeysAndItemIDs( + t, + ctx, + acMail, + uid, + "", + restoreCfg.Location, + api.MailInbox) + + // compare restore results + for _, cat := range []path.CategoryType{path.ContactsCategory, path.EmailCategory, path.EventsCategory} { + assert.Equal(t, len(userItemIDs[cat]), len(secondaryItemIDs[cat])) + assert.ElementsMatch(t, maps.Keys(userCollisionKeys[cat]), maps.Keys(secondaryCollisionKeys[cat])) + } +} + +func getCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + cip clientItemPager, + userID, parentContainerID string, + containerNames ...string, +) (map[string]struct{}, map[string]string, string) { + var ( + c graph.Container + err error + cID = parentContainerID + ) + + for _, cn := range containerNames { + c, err = cip.GetContainerByName(ctx, userID, cID, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := cip.GetItemIDsInContainer(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := cip.GetItemsInContainerByCollisionKey(ctx, userID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys, cID +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 93a609365..c826b3e44 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -25,6 +25,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" @@ -101,7 +102,7 @@ func prepNewTestBackupOp( k := kopia.NewConn(bod.st) - err := k.Initialize(ctx, repository.Options{}) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) require.NoError(t, err, clues.ToCore(err)) defer func() { @@ -242,13 +243,7 @@ func checkBackupIsInManifests( for _, category := range categories { t.Run(category.String(), func(t *testing.T) { var ( - reasons = []kopia.Reason{ - { - ResourceOwner: resourceOwner, - Service: sel.PathService(), - Category: category, - }, - } + r = kopia.NewReason("", resourceOwner, sel.PathService(), category) tags = map[string]string{kopia.TagBackupCategory: ""} found bool ) @@ -256,7 +251,7 @@ func checkBackupIsInManifests( bf, err := kw.NewBaseFinder(sw) require.NoError(t, err, clues.ToCore(err)) - mans := bf.FindBases(ctx, reasons, tags) + mans := bf.FindBases(ctx, []kopia.Reasoner{r}, tags) for _, man := range mans.MergeBases() { bID, ok := man.GetTag(kopia.TagBackupID) if !assert.Truef(t, ok, "snapshot manifest %s missing backup ID tag", man.ID) { @@ -406,6 +401,7 @@ func generateContainerOfItems( restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg.Location = destFldr + restoreCfg.IncludePermissions = true dataColls := buildCollections( t, @@ -414,15 +410,19 @@ func generateContainerOfItems( restoreCfg, collections) - opts := control.Defaults() - opts.RestorePermissions = true + opts := control.DefaultOptions() + + rcc := inject.RestoreConsumerConfig{ + BackupVersion: backupVersion, + Options: opts, + ProtectedResource: sel, + RestoreConfig: restoreCfg, + Selector: sel, + } deets, err := ctrl.ConsumeRestoreCollections( ctx, - backupVersion, - sel, - restoreCfg, - opts, + rcc, dataColls, fault.New(true), count.New()) @@ -541,7 +541,7 @@ func ControllerWithSelector( ins idname.Cacher, onFail func(*testing.T, context.Context), ) (*m365.Controller, selectors.Selector) { - ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) + ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions()) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -550,7 +550,7 @@ func ControllerWithSelector( t.FailNow() } - id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if !assert.NoError(t, err, clues.ToCore(err)) { if onFail != nil { onFail(t, ctx) @@ -568,15 +568,19 @@ func ControllerWithSelector( // Suite Setup // --------------------------------------------------------------------------- +type ids struct { + ID string + DriveID string + DriveRootFolderID string +} + type intgTesterSetup struct { - ac api.Client - gockAC api.Client - userID string - userDriveID string - userDriveRootFolderID string - siteID string - siteDriveID string - siteDriveRootFolderID string + ac api.Client + gockAC api.Client + user ids + secondaryUser ids + site ids + secondarySite ids } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -591,43 +595,58 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) require.NoError(t, err, clues.ToCore(err)) - // user drive - - its.userID = tconfig.M365UserID(t) - - userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveID = ptr.Val(userDrive.GetId()) - - userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId()) - - its.siteID = tconfig.M365SiteID(t) - - // site - - siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveID = ptr.Val(siteDrive.GetId()) - - siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID) - require.NoError(t, err, clues.ToCore(err)) - - its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) + its.user = userIDs(t, tconfig.M365UserID(t), its.ac) + its.secondaryUser = userIDs(t, tconfig.SecondaryM365UserID(t), its.ac) + its.site = siteIDs(t, tconfig.M365SiteID(t), its.ac) + its.secondarySite = siteIDs(t, tconfig.SecondaryM365SiteID(t), its.ac) return its } +func userIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Users().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + +func siteIDs(t *testing.T, id string, ac api.Client) ids { + ctx, flush := tester.NewContext(t) + defer flush() + + r := ids{ID: id} + + drive, err := ac.Sites().GetDefaultDrive(ctx, id) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveID = ptr.Val(drive.GetId()) + + driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID) + require.NoError(t, err, clues.ToCore(err)) + + r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId()) + + return r +} + func getTestExtensionFactories() []extensions.CreateItemExtensioner { return []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, diff --git a/src/internal/operations/test/onedrive_test.go b/src/internal/operations/test/onedrive_test.go index 41aab489d..b5057be31 100644 --- a/src/internal/operations/test/onedrive_test.go +++ b/src/internal/operations/test/onedrive_test.go @@ -72,7 +72,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) osel.Include(selTD.OneDriveBackupFolderScope(osel)) @@ -106,7 +106,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() { } func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { - sel := selectors.NewOneDriveRestore([]string{suite.its.userID}) + sel := selectors.NewOneDriveRestore([]string{suite.its.user.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.Folders(cs, selectors.PrefixMatch())) @@ -117,10 +117,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.userID) + d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.user.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). - With("user", suite.its.userID) + With("user", suite.its.user.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -137,8 +137,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { runDriveIncrementalTest( suite, - suite.its.userID, - suite.its.userID, + suite.its.user.ID, + suite.its.user.ID, resource.Users, path.OneDriveService, path.FilesCategory, @@ -166,7 +166,7 @@ func runDriveIncrementalTest( var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() ws = deeTD.DriveIDFromRepoRef @@ -683,7 +683,7 @@ func runDriveIncrementalTest( } for _, test := range table { suite.Run(test.name, func() { - cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) + cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) bod.ctrl = cleanCtrl @@ -785,7 +785,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { var ( acct = tconfig.NewM365Account(t) - opts = control.Defaults() + opts = control.DefaultOptions() mb = evmock.NewBus() categories = map[path.CategoryType][]string{ @@ -801,10 +801,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() { acct, resource.Users, path.OneDriveService, - control.Defaults()) + control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) - userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.userID) + userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID) require.NoError(t, err, clues.ToCore(err)) uid := ptr.Val(userable.GetId()) @@ -922,7 +922,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() { osel = selectors.NewOneDriveBackup([]string{userID}) ws = deeTD.DriveIDFromRepoRef svc = path.OneDriveService - opts = control.Defaults() + opts = control.DefaultOptions() ) opts.ItemExtensionFactory = getTestExtensionFactories() @@ -982,17 +982,17 @@ func (suite *OneDriveRestoreIntgSuite) SetupSuite() { } func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { - sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) sel.Include(selTD.OneDriveBackupFolderScope(sel)) - sel.DiscreteOwner = suite.its.userID + sel.DiscreteOwner = suite.its.user.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.userDriveID, - suite.its.userDriveRootFolderID) + suite.its.user.DriveID, + suite.its.user.DriveRootFolderID) } func runDriveRestoreWithAdvancedOptions( @@ -1009,7 +1009,7 @@ func runDriveRestoreWithAdvancedOptions( var ( mb = evmock.NewBus() - opts = control.Defaults() + opts = control.DefaultOptions() ) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) @@ -1250,3 +1250,173 @@ func runDriveRestoreWithAdvancedOptions( assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") }) } + +func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveAlternateProtectedResource() { + sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID}) + sel.Include(selTD.OneDriveBackupFolderScope(sel)) + sel.DiscreteOwner = suite.its.user.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.user, + suite.its.secondaryUser) +} + +func runDriveRestoreToAlternateProtectedResource( + t *testing.T, + suite tester.Suite, + ac api.Client, + sel selectors.Selector, // owner should match 'from', both Restore and Backup types work. + from, to ids, +) { + ctx, flush := tester.NewContext(t) + defer flush() + + // a backup is required to run restores + + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + ) + + bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + var ( + restoreCfg = ctrlTD.DefaultRestoreConfig("drive_restore_to_resource") + fromCollisionKeys map[string]api.DriveItemIDType + fromItemIDs map[string]api.DriveItemIDType + acd = ac.Drives() + ) + + // first restore to the 'from' resource + + suite.Run("restore original resource", func() { + mb = evmock.NewBus() + fromCtr := count.New() + driveID := from.DriveID + rootFolderID := from.DriveRootFolderID + restoreCfg.OnCollision = control.Copy + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + fromCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + fromItemIDs, fromCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // then restore to the 'to' resource + var ( + toCollisionKeys map[string]api.DriveItemIDType + toItemIDs map[string]api.DriveItemIDType + ) + + suite.Run("restore to alternate resource", func() { + mb = evmock.NewBus() + toCtr := count.New() + driveID := to.DriveID + rootFolderID := to.DriveRootFolderID + restoreCfg.ProtectedResource = to.ID + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + toCtr, + sel, + opts, + restoreCfg) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + // get all files in folder, use these as the base + // set of files to compare against. + toItemIDs, toCollisionKeys = getDriveCollKeysAndItemIDs( + t, + ctx, + acd, + driveID, + rootFolderID, + restoreCfg.Location, + selTD.TestFolderName) + }) + + // compare restore results + assert.Equal(t, len(fromItemIDs), len(toItemIDs)) + assert.ElementsMatch(t, maps.Keys(fromCollisionKeys), maps.Keys(toCollisionKeys)) +} + +type GetItemsKeysAndFolderByNameer interface { + GetItemIDsInContainer( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) + GetFolderByName( + ctx context.Context, + driveID, parentFolderID, folderName string, + ) (models.DriveItemable, error) + GetItemsInContainerByCollisionKey( + ctx context.Context, + driveID, containerID string, + ) (map[string]api.DriveItemIDType, error) +} + +func getDriveCollKeysAndItemIDs( + t *testing.T, + ctx context.Context, //revive:disable-line:context-as-argument + gikafbn GetItemsKeysAndFolderByNameer, + driveID, parentContainerID string, + containerNames ...string, +) (map[string]api.DriveItemIDType, map[string]api.DriveItemIDType) { + var ( + c models.DriveItemable + err error + cID string + ) + + for _, cn := range containerNames { + pcid := parentContainerID + + if len(cID) != 0 { + pcid = cID + } + + c, err = gikafbn.GetFolderByName(ctx, driveID, pcid, cn) + require.NoError(t, err, clues.ToCore(err)) + + cID = ptr.Val(c.GetId()) + } + + itemIDs, err := gikafbn.GetItemIDsInContainer(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + collisionKeys, err := gikafbn.GetItemsInContainerByCollisionKey(ctx, driveID, cID) + require.NoError(t, err, clues.ToCore(err)) + + return itemIDs, collisionKeys +} diff --git a/src/internal/operations/test/sharepoint_test.go b/src/internal/operations/test/sharepoint_test.go index 1b5a52dc2..08cc4cb1c 100644 --- a/src/internal/operations/test/sharepoint_test.go +++ b/src/internal/operations/test/sharepoint_test.go @@ -5,6 +5,9 @@ import ( "testing" "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -19,6 +22,8 @@ import ( "github.com/alcionai/corso/src/internal/version" deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" + ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" + "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" @@ -44,7 +49,7 @@ func (suite *SharePointBackupIntgSuite) SetupSuite() { } func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { - sel := selectors.NewSharePointRestore([]string{suite.its.siteID}) + sel := selectors.NewSharePointRestore([]string{suite.its.site.ID}) ic := func(cs []string) selectors.Selector { sel.Include(sel.LibraryFolders(cs, selectors.PrefixMatch())) @@ -55,10 +60,10 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { t *testing.T, ctx context.Context, ) string { - d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.siteID) + d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.site.ID) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). - With("site", suite.its.siteID) + With("site", suite.its.site.ID) } require.NoError(t, err, clues.ToCore(err)) @@ -75,8 +80,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { runDriveIncrementalTest( suite, - suite.its.siteID, - suite.its.userID, + suite.its.site.ID, + suite.its.user.ID, resource.Sites, path.SharePointService, path.LibrariesCategory, @@ -86,7 +91,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { true) } -func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { +func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -94,8 +99,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() ) sel.Include(selTD.SharePointBackupFolderScope(sel)) @@ -111,7 +116,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) } @@ -123,8 +128,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { var ( mb = evmock.NewBus() - sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) - opts = control.Defaults() + sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) + opts = control.DefaultOptions() tenID = tconfig.M365TenantID(t) svc = path.SharePointService ws = deeTD.DriveIDFromRepoRef @@ -145,7 +150,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() { bod.sw, &bo, bod.sel, - suite.its.siteID, + bod.sel.ID(), path.LibrariesCategory) bID := bo.Results.BackupID @@ -196,16 +201,268 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() { } func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { - sel := selectors.NewSharePointBackup([]string{suite.its.userID}) + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Filter(sel.Library("documents")) - sel.DiscreteOwner = suite.its.siteID + sel.DiscreteOwner = suite.its.site.ID runDriveRestoreWithAdvancedOptions( suite.T(), suite, suite.its.ac, sel.Selector, - suite.its.siteDriveID, - suite.its.siteDriveRootFolderID) + suite.its.site.DriveID, + suite.its.site.DriveRootFolderID) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointAlternateProtectedResource() { + sel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) + sel.Include(selTD.SharePointBackupFolderScope(sel)) + sel.Filter(sel.Library("documents")) + sel.DiscreteOwner = suite.its.site.ID + + runDriveRestoreToAlternateProtectedResource( + suite.T(), + suite, + suite.its.ac, + sel.Selector, + suite.its.site, + suite.its.secondarySite) +} + +func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { + t := suite.T() + + // despite the client having a method for drive.Patch and drive.Delete, both only return + // the error code and message `invalidRequest`. + t.Skip("graph api doesn't allow patch or delete on drives, so we cannot run any conditions") + + ctx, flush := tester.NewContext(t) + defer flush() + + rc := ctrlTD.DefaultRestoreConfig("restore_deleted_drives") + rc.OnCollision = control.Copy + + // create a new drive + md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.site.ID, rc.Location) + require.NoError(t, err, clues.ToCore(err)) + + driveID := ptr.Val(md.GetId()) + + // get the root folder + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID := ptr.Val(mdi.GetId()) + + // add an item to it + itemName := uuid.NewString() + + item := models.NewDriveItem() + item.SetName(ptr.To(itemName + ".txt")) + + file := models.NewFile() + item.SetFile(file) + + _, err = suite.its.ac.Drives().PostItemInContainer( + ctx, + driveID, + rootFolderID, + item, + control.Copy) + require.NoError(t, err, clues.ToCore(err)) + + // run a backup + var ( + mb = evmock.NewBus() + opts = control.DefaultOptions() + graphClient = suite.its.ac.Stable.Client() + ) + + bsel := selectors.NewSharePointBackup([]string{suite.its.site.ID}) + bsel.Include(selTD.SharePointBackupFolderScope(bsel)) + bsel.Filter(bsel.Library(rc.Location)) + bsel.DiscreteOwner = suite.its.site.ID + + bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) + defer bod.close(t, ctx) + + runAndCheckBackup(t, ctx, &bo, mb, false) + + // test cases: + + // first test, we take the current drive and rename it. + // the restore should find the drive by id and restore items + // into it like normal. Due to collision handling, this should + // create a copy of the current item. + suite.Run("renamed drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + patchBody := models.NewDrive() + patchBody.SetName(ptr.To("some other name")) + + md, err = graphClient. + Drives(). + ByDriveId(driveID). + Patch(ctx, patchBody, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) + + // second test, we delete the drive altogether. the restore should find + // no existing drives, but it should have the old drive's name and attempt + // to recreate that drive by name. + suite.Run("deleted drive", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + err = graphClient. + Drives(). + ByDriveId(driveID). + Delete(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + pgr := suite.its.ac. + Drives(). + NewSiteDrivePager(suite.its.site.ID, []string{"id", "name"}) + + drives, err := api.GetAllDrives(ctx, pgr, false, -1) + require.NoError(t, err, clues.ToCore(err)) + + var created models.Driveable + + for _, drive := range drives { + if ptr.Val(drive.GetName()) == ptr.Val(created.GetName()) && + ptr.Val(drive.GetId()) != driveID { + created = drive + break + } + } + + require.NotNil(t, created, "found the restored drive by name") + md = created + driveID = ptr.Val(md.GetId()) + + mdi, err := suite.its.ac.Drives().GetRootFolder(ctx, driveID) + require.NoError(t, err, clues.ToCore(err)) + + rootFolderID = ptr.Val(mdi.GetId()) + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 1) + + assert.Equal(t, ptr.Val(items[0].GetName()), itemName+".txt") + }) + + // final test, run a follow-up restore. This should match the + // drive we created in the prior test by name, but not by ID. + suite.Run("different drive - same name", func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + mb = evmock.NewBus() + ctr = count.New() + ) + + ro, _ := prepNewTestRestoreOp( + t, + ctx, + bod.st, + bo.Results.BackupID, + mb, + ctr, + bod.sel, + opts, + rc) + + runAndCheckRestore(t, ctx, &ro, mb, false) + + assert.Equal(t, 1, ctr.Get(count.NewItemCreated), "restored an item") + + resp, err := graphClient. + Drives(). + ByDriveId(driveID). + Items(). + ByDriveItemId(rootFolderID). + Children(). + Get(ctx, nil) + require.NoError(t, err, clues.ToCore(graph.Stack(ctx, err))) + + items := resp.GetValue() + assert.Len(t, items, 2) + + for _, item := range items { + assert.Contains(t, ptr.Val(item.GetName()), itemName) + } + }) } diff --git a/src/internal/streamstore/collectables_test.go b/src/internal/streamstore/collectables_test.go index d8897c146..ccbeab7c2 100644 --- a/src/internal/streamstore/collectables_test.go +++ b/src/internal/streamstore/collectables_test.go @@ -44,7 +44,8 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() { st := storeTD.NewPrefixedS3Storage(t) k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx, repository.Options{})) + err := k.Initialize(ctx, repository.Options{}, repository.Retention{}) + require.NoError(t, err, clues.ToCore(err)) suite.kcloser = func() { k.Close(ctx) } diff --git a/src/internal/streamstore/streamstore.go b/src/internal/streamstore/streamstore.go index 9deb0176d..6f5918c81 100644 --- a/src/internal/streamstore/streamstore.go +++ b/src/internal/streamstore/streamstore.go @@ -234,6 +234,7 @@ func write( backupStats, _, _, err := bup.ConsumeBackupCollections( ctx, nil, + nil, dbcs, prefixmatcher.NopReader[map[string]struct{}](), nil, diff --git a/src/internal/tester/tconfig/config.go b/src/internal/tester/tconfig/config.go index c6bcd6e4b..a900f26f2 100644 --- a/src/internal/tester/tconfig/config.go +++ b/src/internal/tester/tconfig/config.go @@ -23,8 +23,11 @@ const ( // M365 config TestCfgAzureTenantID = "azure_tenantid" + TestCfgSecondarySiteID = "secondarym365siteid" TestCfgSiteID = "m365siteid" TestCfgSiteURL = "m365siteurl" + TestCfgTeamID = "m365teamid" + TestCfgGroupID = "m365groupid" TestCfgUserID = "m365userid" TestCfgSecondaryUserID = "secondarym365userid" TestCfgTertiaryUserID = "tertiarym365userid" @@ -36,13 +39,16 @@ const ( // test specific env vars const ( - EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" - EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" - EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" - EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" - EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" + EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" + EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" + EnvCorsoM365TestTeamID = "CORSO_M365_TEST_TEAM_ID" + EnvCorsoM365TestGroupID = "CORSO_M365_TEST_GROUP_ID" + EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" + EnvCorsoSecondaryM365TestSiteID = "CORSO_SECONDARY_M365_TEST_SITE_ID" + EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" + EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE" EnvCorsoUnlicensedM365TestUserID = "CORSO_M365_TEST_UNLICENSED_USER" ) @@ -108,7 +114,7 @@ func ReadTestConfig() (map[string]string, error) { testEnv := map[string]string{} fallbackTo(testEnv, TestCfgStorageProvider, vpr.GetString(TestCfgStorageProvider)) fallbackTo(testEnv, TestCfgAccountProvider, vpr.GetString(TestCfgAccountProvider)) - fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket), "test-corso-repo-init") + fallbackTo(testEnv, TestCfgBucket, os.Getenv("S3_BUCKET"), vpr.GetString(TestCfgBucket)) fallbackTo(testEnv, TestCfgEndpoint, vpr.GetString(TestCfgEndpoint), "s3.amazonaws.com") fallbackTo(testEnv, TestCfgPrefix, vpr.GetString(TestCfgPrefix)) fallbackTo(testEnv, TestCfgAzureTenantID, os.Getenv(account.AzureTenantID), vpr.GetString(TestCfgAzureTenantID)) @@ -147,13 +153,31 @@ func ReadTestConfig() (map[string]string, error) { TestCfgSiteID, os.Getenv(EnvCorsoM365TestSiteID), vpr.GetString(TestCfgSiteID), - "10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") + "4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") + fallbackTo( + testEnv, + TestCfgTeamID, + os.Getenv(EnvCorsoM365TestTeamID), + vpr.GetString(TestCfgTeamID), + "6f24b40d-b13d-4752-980f-f5fb9fba7aa0") + fallbackTo( + testEnv, + TestCfgGroupID, + os.Getenv(EnvCorsoM365TestGroupID), + vpr.GetString(TestCfgGroupID), + "6f24b40d-b13d-4752-980f-f5fb9fba7aa0") fallbackTo( testEnv, TestCfgSiteURL, os.Getenv(EnvCorsoM365TestSiteURL), vpr.GetString(TestCfgSiteURL), "https://10rqc2.sharepoint.com/sites/CorsoCI") + fallbackTo( + testEnv, + TestCfgSecondarySiteID, + os.Getenv(EnvCorsoSecondaryM365TestSiteID), + vpr.GetString(TestCfgSecondarySiteID), + "053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c") fallbackTo( testEnv, TestCfgUnlicensedUserID, diff --git a/src/internal/tester/tconfig/protected_resources.go b/src/internal/tester/tconfig/protected_resources.go index b9e31ce06..caac0c586 100644 --- a/src/internal/tester/tconfig/protected_resources.go +++ b/src/internal/tester/tconfig/protected_resources.go @@ -198,6 +198,17 @@ func GetM365SiteID(ctx context.Context) string { return strings.ToLower(cfg[TestCfgSiteID]) } +// SecondaryM365SiteID returns a siteID string representing the secondarym365SiteID described +// by either the env var CORSO_SECONDARY_M365_TEST_SITE_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func SecondaryM365SiteID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving secondary m365 site id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgSecondarySiteID]) +} + // UnlicensedM365UserID returns an userID string representing the m365UserID // described by either the env var CORSO_M365_TEST_UNLICENSED_USER, the // corso_test.toml config file or the default value (in that order of priority). @@ -209,3 +220,29 @@ func UnlicensedM365UserID(t *testing.T) string { return strings.ToLower(cfg[TestCfgSecondaryUserID]) } + +// Teams + +// M365TeamsID returns a teamID string representing the m365TeamsID described +// by either the env var CORSO_M365_TEST_TEAM_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func M365TeamsID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving m365 team id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgTeamID]) +} + +// Groups + +// M365GroupID returns a groupID string representing the m365GroupID described +// by either the env var CORSO_M365_TEST_GROUP_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func M365GroupID(t *testing.T) string { + cfg, err := ReadTestConfig() + require.NoError(t, err, "retrieving m365 group id from test configuration: %+v", clues.ToCore(err)) + + return strings.ToLower(cfg[TestCfgTeamID]) +} diff --git a/src/pkg/control/export.go b/src/pkg/control/export.go new file mode 100644 index 000000000..e58633a5f --- /dev/null +++ b/src/pkg/control/export.go @@ -0,0 +1,22 @@ +package control + +// ExportConfig contains config for exports +type ExportConfig struct { + // Archive decides if we should create an archive from the data + // instead of just returning all the files. If Archive is set to + // true, we return a single collection with a single file which is + // the archive. + Archive bool + + // DataFormat decides the format in which we return the data. This is + // only useful for outlook exports, for example they can be in eml + // or pst for emails. + // TODO: Enable once we support outlook exports + // DataFormat string +} + +func DefaultExportConfig() ExportConfig { + return ExportConfig{ + Archive: false, + } +} diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index 23375f229..01c88b5eb 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -7,14 +7,16 @@ import ( // Options holds the optional configurations for a process type Options struct { + // DeltaPageSize controls the quantity of items fetched in each page + // during multi-page queries, such as graph api delta endpoints. + DeltaPageSize int32 `json:"deltaPageSize"` DisableMetrics bool `json:"disableMetrics"` FailureHandling FailurePolicy `json:"failureHandling"` - RestorePermissions bool `json:"restorePermissions"` - SkipReduce bool `json:"skipReduce"` - ToggleFeatures Toggles `json:"toggleFeatures"` + ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` Parallelism Parallelism `json:"parallelism"` Repo repository.Options `json:"repo"` - ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` + SkipReduce bool `json:"skipReduce"` + ToggleFeatures Toggles `json:"toggleFeatures"` } type Parallelism struct { @@ -35,10 +37,11 @@ const ( BestEffort FailurePolicy = "best-effort" ) -// Defaults provides an Options with the default values set. -func Defaults() Options { +// DefaultOptions provides an Options with the default values set. +func DefaultOptions() Options { return Options{ FailureHandling: FailAfterRecovery, + DeltaPageSize: 500, ToggleFeatures: Toggles{}, Parallelism: Parallelism{ CollectionBuffer: 4, diff --git a/src/pkg/control/restore.go b/src/pkg/control/restore.go index 5fc5f7be8..cce3bea9b 100644 --- a/src/pkg/control/restore.go +++ b/src/pkg/control/restore.go @@ -2,17 +2,21 @@ package control import ( "context" + "encoding/json" + "fmt" "strings" + "github.com/alcionai/clues" "golang.org/x/exp/maps" "golang.org/x/exp/slices" "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" ) const ( - defaultRestoreLocation = "Corso_Restore_" + DefaultRestoreLocation = "Corso_Restore_" ) // CollisionPolicy describes how the datalayer behaves in case of a collision. @@ -39,32 +43,41 @@ const RootLocation = "/" type RestoreConfig struct { // Defines the per-item collision handling policy. // Defaults to Skip. - OnCollision CollisionPolicy + OnCollision CollisionPolicy `json:"onCollision"` // ProtectedResource specifies which resource the data will be restored to. // If empty, restores to the same resource that was backed up. // Defaults to empty. - ProtectedResource string + ProtectedResource string `json:"protectedResource"` // Location specifies the container into which the data will be restored. // Only accepts container names, does not accept IDs. // If empty or "/", data will get restored in place, beginning at the root. // Defaults to "Corso_Restore_" - Location string + Location string `json:"location"` - // Drive specifies the drive into which the data will be restored. - // If empty, data is restored to the same drive that was backed up. + // Drive specifies the name of the drive into which the data will be + // restored. If empty, data is restored to the same drive that was backed + // up. // Defaults to empty. - Drive string + Drive string `json:"drive"` + + // IncludePermissions toggles whether the restore will include the original + // folder- and item-level permissions. + IncludePermissions bool `json:"includePermissions"` } func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { return RestoreConfig{ OnCollision: Skip, - Location: defaultRestoreLocation + dttm.FormatNow(timeFormat), + Location: DefaultRestoreLocation + dttm.FormatNow(timeFormat), } } +func DefaultRestoreContainerName(timeFormat dttm.TimeFormat) string { + return DefaultRestoreLocation + dttm.FormatNow(timeFormat) +} + // EnsureRestoreConfigDefaults sets all non-supported values in the config // struct to the default value. func EnsureRestoreConfigDefaults( @@ -85,3 +98,55 @@ func EnsureRestoreConfigDefaults( return rc } + +// --------------------------------------------------------------------------- +// pii control +// --------------------------------------------------------------------------- + +var ( + // interface compliance required for handling PII + _ clues.Concealer = &RestoreConfig{} + _ fmt.Stringer = &RestoreConfig{} +) + +func (rc RestoreConfig) marshal() string { + bs, err := json.Marshal(rc) + if err != nil { + return "err marshalling" + } + + return string(bs) +} + +func (rc RestoreConfig) concealed() RestoreConfig { + return RestoreConfig{ + OnCollision: rc.OnCollision, + ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), + Location: path.LoggableDir(rc.Location), + Drive: clues.Hide(rc.Drive).Conceal(), + IncludePermissions: rc.IncludePermissions, + } +} + +// Conceal produces a concealed representation of the config, suitable for +// logging, storing in errors, and other output. +func (rc RestoreConfig) Conceal() string { + return rc.concealed().marshal() +} + +// Format produces a concealed representation of the config, even when +// used within a PrintF, suitable for logging, storing in errors, +// and other output. +func (rc RestoreConfig) Format(fs fmt.State, _ rune) { + fmt.Fprint(fs, rc.concealed()) +} + +// String returns a plain text version of the restoreConfig. +func (rc RestoreConfig) String() string { + return rc.PlainString() +} + +// PlainString returns an unescaped, unmodified string of the restore configuration. +func (rc RestoreConfig) PlainString() string { + return rc.marshal() +} diff --git a/src/pkg/export/consume.go b/src/pkg/export/consume.go new file mode 100644 index 000000000..899f9c3ba --- /dev/null +++ b/src/pkg/export/consume.go @@ -0,0 +1,79 @@ +package export + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/fault" +) + +func ConsumeExportCollections( + ctx context.Context, + exportLocation string, + expColl []Collection, + errs *fault.Bus, +) error { + el := errs.Local() + + for _, col := range expColl { + if el.Failure() != nil { + break + } + + folder := filepath.Join(exportLocation, col.BasePath()) + ictx := clues.Add(ctx, "dir_name", folder) + + for item := range col.Items(ctx) { + if item.Error != nil { + el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item").WithClues(ctx)) + } + + if err := writeItem(ictx, item, folder); err != nil { + el.AddRecoverable( + ictx, + clues.Wrap(err, "writing item").With("file_name", item.Data.Name).WithClues(ctx)) + } + } + } + + return el.Failure() +} + +// writeItem writes an ExportItem to disk in the specified folder. +func writeItem(ctx context.Context, item Item, folder string) error { + name := item.Data.Name + fpath := filepath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return clues.Wrap(err, "creating directory") + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return clues.Wrap(err, "creating file") + } + + _, err = io.Copy(f, progReader) + if err != nil { + return clues.Wrap(err, "writing data") + } + + return nil +} diff --git a/src/pkg/export/consume_test.go b/src/pkg/export/consume_test.go new file mode 100644 index 000000000..7d22dc237 --- /dev/null +++ b/src/pkg/export/consume_test.go @@ -0,0 +1,175 @@ +package export + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/fault" +) + +type ExportE2ESuite struct { + tester.Suite + called bool +} + +func TestExportE2ESuite(t *testing.T) { + suite.Run(t, &ExportE2ESuite{Suite: tester.NewE2ESuite(t, nil)}) +} + +func (suite *ExportE2ESuite) SetupSuite() { + suite.called = true +} + +type mockExportCollection struct { + path string + items []Item +} + +func (mec mockExportCollection) BasePath() string { return mec.path } +func (mec mockExportCollection) Items(context.Context) <-chan Item { + ch := make(chan Item) + + go func() { + defer close(ch) + + for _, item := range mec.items { + ch <- item + } + }() + + return ch +} + +func (suite *ExportE2ESuite) TestConsumeExportCollection() { + type ei struct { + name string + body string + } + + type i struct { + path string + items []ei + } + + table := []struct { + name string + cols []i + }{ + { + name: "single root collection single item", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + }, + }, + }, + }, + { + name: "single root collection multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + }, + }, + { + name: "multiple collections multiple items", + cols: []i{ + { + path: "", + items: []ei{ + { + name: "name1", + body: "body1", + }, + { + name: "name2", + body: "body2", + }, + }, + }, + { + path: "folder", + items: []ei{ + { + name: "name3", + body: "body3", + }, + }, + }, + }, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ecs := []Collection{} + for _, col := range test.cols { + items := []Item{} + for _, item := range col.items { + items = append(items, Item{ + Data: ItemData{ + Name: item.name, + Body: io.NopCloser((bytes.NewBufferString(item.body))), + }, + }) + } + + ecs = append(ecs, mockExportCollection{ + path: col.path, + items: items, + }) + } + + dir, err := os.MkdirTemp("", "export-test") + require.NoError(t, err) + defer os.RemoveAll(dir) + + err = ConsumeExportCollections(ctx, dir, ecs, fault.New(true)) + require.NoError(t, err, "writing data") + + for _, col := range test.cols { + for _, item := range col.items { + f, err := os.Open(filepath.Join(dir, col.path, item.name)) + require.NoError(t, err, "opening file") + + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(f) + require.NoError(t, err, "reading file") + + assert.Equal(t, item.body, buf.String(), "file contents") + } + } + }) + } +} diff --git a/src/pkg/export/export.go b/src/pkg/export/export.go new file mode 100644 index 000000000..73c173e04 --- /dev/null +++ b/src/pkg/export/export.go @@ -0,0 +1,47 @@ +package export + +import ( + "context" + "io" +) + +// Collection is the interface that is returned to the SDK consumer +type Collection interface { + // BasePath gets the base path of the collection. This is derived + // from FullPath, but trim out thing like drive id or any other part + // that is not needed to show the path to the collection. + BasePath() string + + // Items gets the items within the collection(folder) + Items(context.Context) <-chan Item +} + +// ItemData is the data for an individual item. +type ItemData struct { + // Name is the name of the item. This is the name that the item + // would have had in the service. + Name string + + // Body is the body of the item. This is an io.ReadCloser and the + // SDK consumer is responsible for closing it. + Body io.ReadCloser +} + +// Item is the item that is returned to the SDK consumer +type Item struct { + // ID will be a unique id for the item. This is same as the id + // that is used to store the data. This is not the name and is + // mostly used just for tracking. + ID string + + // Data contains the actual data of the item. It will have both + // the name of the item and an io.ReadCloser which contains the + // body of the item. + Data ItemData + + // Error will contain any error that happened while trying to get + // the item/items like when trying to resolve the name of the item. + // In case we have the error bound to a particular item, we will + // also return the id of the item. + Error error +} diff --git a/src/pkg/fault/fault.go b/src/pkg/fault/fault.go index 97cc0bae3..488656fa4 100644 --- a/src/pkg/fault/fault.go +++ b/src/pkg/fault/fault.go @@ -208,6 +208,32 @@ func (e *Bus) Errors() *Errors { } } +// ItemsAndRecovered returns the items that failed along with other +// recoverable errors +func (e *Bus) ItemsAndRecovered() ([]Item, []error) { + var ( + is = map[string]Item{} + non = []error{} + ) + + for _, err := range e.recoverable { + var ie *Item + if !errors.As(err, &ie) { + non = append(non, err) + continue + } + + is[ie.dedupeID()] = *ie + } + + var ie *Item + if errors.As(e.failure, &ie) { + is[ie.dedupeID()] = *ie + } + + return maps.Values(is), non +} + // --------------------------------------------------------------------------- // Errors Data // --------------------------------------------------------------------------- diff --git a/src/pkg/filters/filters.go b/src/pkg/filters/filters.go index 40c12a630..c697203b7 100644 --- a/src/pkg/filters/filters.go +++ b/src/pkg/filters/filters.go @@ -501,7 +501,7 @@ func suffixed(target, input string) bool { // Printers and PII control // ---------------------------------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Filter{} +var _ clues.Concealer = &Filter{} var safeFilterValues = map[string]struct{}{"*": {}} diff --git a/src/pkg/path/elements.go b/src/pkg/path/elements.go index d1ca932dc..838cea114 100644 --- a/src/pkg/path/elements.go +++ b/src/pkg/path/elements.go @@ -2,7 +2,6 @@ package path import ( "fmt" - "strings" "github.com/alcionai/clues" @@ -54,10 +53,6 @@ var ( // interface compliance required for handling PII _ clues.Concealer = &Elements{} _ fmt.Stringer = &Elements{} - - // interface compliance for the observe package to display - // values without concealing PII. - _ clues.PlainStringer = &Elements{} ) // Elements are a PII Concealer-compliant slice of elements within a path. @@ -123,16 +118,8 @@ func (el Elements) Last() string { // LoggableDir takes in a path reference (of any structure) and conceals any // non-standard elements (ids, filenames, foldernames, etc). func LoggableDir(ref string) string { - r := ref - n := strings.TrimSuffix(r, string(PathSeparator)) - - for n != r { - r = n - n = strings.TrimSuffix(r, string(PathSeparator)) - } - - elems := Split(r) - elems = pii.ConcealElements(elems, piiSafePathElems) - - return join(elems) + // Can't directly use Builder since that could return an error. Instead split + // into elements and use that. + split := Split(TrimTrailingSlash(ref)) + return Elements(split).Conceal() } diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index ca63bf8fe..b7cd38da0 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -120,10 +120,6 @@ type Path interface { // is appropriately hidden from logging, errors, and other outputs. clues.Concealer fmt.Stringer - - // In the rare case that the path needs to get printed as a plain string, - // without obscuring values for PII. - clues.PlainStringer } // interface compliance required for handling PII diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 88f025371..c2702faed 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -30,8 +30,8 @@ const ( ExchangeMetadataService // exchangeMetadata OneDriveMetadataService // onedriveMetadata SharePointMetadataService // sharepointMetadata - TeamsService // teams - TeamsMetadataService // teamsMetadata + GroupsService // groups + GroupsMetadataService // groupsMetadata ) func toServiceType(service string) ServiceType { diff --git a/src/pkg/path/servicetype_string.go b/src/pkg/path/servicetype_string.go index 6d6b960d8..6fa499364 100644 --- a/src/pkg/path/servicetype_string.go +++ b/src/pkg/path/servicetype_string.go @@ -15,11 +15,13 @@ func _() { _ = x[ExchangeMetadataService-4] _ = x[OneDriveMetadataService-5] _ = x[SharePointMetadataService-6] + _ = x[GroupsService-7] + _ = x[GroupsMetadataService-8] } -const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadata" +const _ServiceType_name = "UnknownServiceexchangeonedrivesharepointexchangeMetadataonedriveMetadatasharepointMetadatagroupsgroupsMetadata" -var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90} +var _ServiceType_index = [...]uint8{0, 14, 22, 30, 40, 56, 72, 90, 96, 110} func (i ServiceType) String() string { if i < 0 || i >= ServiceType(len(_ServiceType_index)-1) { diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index c723955d1..718ac678d 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -21,6 +21,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -102,7 +103,7 @@ func initM365Repo(t *testing.T) ( FailureHandling: control.FailFast, } - repo, err := repository.Initialize(ctx, ac, st, opts) + repo, err := repository.Initialize(ctx, ac, st, opts, ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) return ctx, repo, ac, st diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 8a8abb33e..d4b0f6eaa 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -25,7 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - rep "github.com/alcionai/corso/src/pkg/control/repository" + ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" @@ -74,10 +74,20 @@ type Repository interface { sel selectors.Selector, restoreCfg control.RestoreConfig, ) (operations.RestoreOperation, error) + NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, + ) (operations.ExportOperation, error) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) + NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, + ) (operations.RetentionConfigOperation, error) DeleteBackup(ctx context.Context, id string) error BackupGetter // ConnectToM365 establishes graph api connections @@ -111,7 +121,8 @@ func (r repository) GetID() string { // - validate the m365 account & secrets // - connect to the m365 account to ensure communication capability // - validate the provider config & secrets -// - initialize the kopia repo with the provider +// - initialize the kopia repo with the provider and retention parameters +// - update maintenance retention parameters as needed // - store the configuration details // - connect to the provider // - return the connected repository @@ -120,6 +131,7 @@ func Initialize( acct account.Account, s storage.Storage, opts control.Options, + retentionOpts ctrlRepo.Retention, ) (repo Repository, err error) { ctx = clues.Add( ctx, @@ -134,7 +146,7 @@ func Initialize( }() kopiaRef := kopia.NewConn(s) - if err := kopiaRef.Initialize(ctx, opts.Repo); err != nil { + if err := kopiaRef.Initialize(ctx, opts.Repo, retentionOpts); err != nil { // replace common internal errors so that sdk users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return nil, clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -329,7 +341,7 @@ func (r repository) NewBackupWithLookup( return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") } - ownerID, ownerName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) + ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) if err != nil { return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") } @@ -349,6 +361,31 @@ func (r repository) NewBackupWithLookup( r.Bus) } +// NewExport generates a exportOperation runner. +func (r repository) NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, +) (operations.ExportOperation, error) { + ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts) + if err != nil { + return operations.ExportOperation{}, clues.Wrap(err, "connecting to m365") + } + + return operations.NewExportOperation( + ctx, + r.Opts, + r.dataLayer, + store.NewKopiaStore(r.modelStore), + ctrl, + r.Account, + model.StableID(backupID), + sel, + exportCfg, + r.Bus) +} + // NewRestore generates a restoreOperation runner. func (r repository) NewRestore( ctx context.Context, @@ -377,7 +414,7 @@ func (r repository) NewRestore( func (r repository) NewMaintenance( ctx context.Context, - mOpts rep.Maintenance, + mOpts ctrlRepo.Maintenance, ) (operations.MaintenanceOperation, error) { return operations.NewMaintenanceOperation( ctx, @@ -387,6 +424,18 @@ func (r repository) NewMaintenance( r.Bus) } +func (r repository) NewRetentionConfig( + ctx context.Context, + rcOpts ctrlRepo.Retention, +) (operations.RetentionConfigOperation, error) { + return operations.NewRetentionConfigOperation( + ctx, + r.Opts, + r.dataLayer, + rcOpts, + r.Bus) +} + // Backup retrieves a backup by id. func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) { return getBackup(ctx, id, store.NewKopiaStore(r.modelStore)) @@ -585,8 +634,13 @@ func deleteBackup( } } - if len(b.DetailsID) > 0 { - if err := kw.DeleteSnapshot(ctx, b.DetailsID); err != nil { + ssid := b.StreamStoreID + if len(ssid) == 0 { + ssid = b.DetailsID + } + + if len(ssid) > 0 { + if err := kw.DeleteSnapshot(ctx, ssid); err != nil { return err } } diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index 80c2b71f6..2ee90b5da 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -60,7 +60,12 @@ func (suite *RepositoryUnitSuite) TestInitialize() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Initialize(ctx, test.account, st, control.Defaults()) + _, err = Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -94,7 +99,7 @@ func (suite *RepositoryUnitSuite) TestConnect() { st, err := test.storage() assert.NoError(t, err, clues.ToCore(err)) - _, err = Connect(ctx, test.account, st, "not_found", control.Defaults()) + _, err = Connect(ctx, test.account, st, "not_found", control.DefaultOptions()) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -137,7 +142,12 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { defer flush() st := test.storage(t) - r, err := Initialize(ctx, test.account, st, control.Defaults()) + r, err := Initialize( + ctx, + test.account, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) if err == nil { defer func() { err := r.Close(ctx) @@ -169,7 +179,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { st.SessionName = "corso-repository-test" st.SessionDuration = roleDuration.String() - r, err := Initialize(ctx, account.Account{}, st, control.Options{}) + r, err := Initialize(ctx, account.Account{}, st, control.Options{}, ctrlRepo.Retention{}) require.NoError(suite.T(), err) defer func() { @@ -186,11 +196,16 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) // now re-connect - _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.Defaults()) + _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.DefaultOptions()) assert.NoError(t, err, clues.ToCore(err)) } @@ -203,7 +218,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + r, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -212,7 +232,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { require.NoError(t, err, clues.ToCore(err)) // now re-connect - r, err = Connect(ctx, account.Account{}, st, oldID, control.Defaults()) + r, err = Connect(ctx, account.Account{}, st, oldID, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, oldID, r.GetID()) } @@ -228,7 +248,12 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -250,7 +275,12 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) @@ -269,7 +299,12 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - r, err := Initialize(ctx, acct, st, control.Defaults()) + r, err := Initialize( + ctx, + acct, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -286,7 +321,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() { // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) - repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) + repo, err := Initialize( + ctx, + account.Account{}, + st, + control.DefaultOptions(), + ctrlRepo.Retention{}) require.NoError(t, err) // now re-connect @@ -308,14 +348,14 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "default options", opts: func() control.Options { - return control.Defaults() + return control.DefaultOptions() }, expectedLen: 0, }, { name: "options with an extension factory", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() o.ItemExtensionFactory = append( o.ItemExtensionFactory, &extensions.MockItemExtensionFactory{}) @@ -327,7 +367,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { { name: "options with multiple extension factories", opts: func() control.Options { - o := control.Defaults() + o := control.DefaultOptions() f := []extensions.CreateItemExtensioner{ &extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{}, @@ -350,7 +390,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { ctx, flush := tester.NewContext(t) defer flush() - repo, err := Initialize(ctx, acct, st, test.opts()) + repo, err := Initialize(ctx, acct, st, test.opts(), ctrlRepo.Retention{}) require.NoError(t, err) r := repo.(*repository) diff --git a/src/pkg/repository/repository_unexported_test.go b/src/pkg/repository/repository_unexported_test.go index e284fad30..0e600157d 100644 --- a/src/pkg/repository/repository_unexported_test.go +++ b/src/pkg/repository/repository_unexported_test.go @@ -240,7 +240,7 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() { require.NotNil(t, k) - err = k.Initialize(ctx, rep.Options{}) + err = k.Initialize(ctx, rep.Options{}, rep.Retention{}) require.NoError(t, err, clues.ToCore(err)) err = k.Connect(ctx, rep.Options{}) @@ -291,8 +291,11 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() { k = kopia.NewConn(s) ) - require.NoError(t, k.Initialize(ctx, rep.Options{})) - require.NoError(t, k.Connect(ctx, rep.Options{})) + err := k.Initialize(ctx, rep.Options{}, rep.Retention{}) + require.NoError(t, err, "initializing repo: %v", clues.ToCore(err)) + + err = k.Connect(ctx, rep.Options{}) + require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err)) defer k.Close(ctx) diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index 2a860892b..aebd0f156 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -153,9 +153,6 @@ type ( // Primarily to ensure that root- or mid-tier scopes (such as folders) // cascade 'Any' matching to more granular categories. setDefaults() - - // Scopes need to comply with PII printing controls. - clues.PlainConcealer } // scopeT is the generic type interface of a scoper. scopeT interface { diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 1e148e04b..936bc3a32 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -341,7 +341,7 @@ func selectorAsIface[T any](s Selector) (T, error) { // Stringers and Concealers // --------------------------------------------------------------------------- -var _ clues.PlainConcealer = &Selector{} +var _ clues.Concealer = &Selector{} type loggableSelector struct { Service service `json:"service,omitempty"` diff --git a/src/pkg/services/m365/api/client.go b/src/pkg/services/m365/api/client.go index 957da03db..c74bf215b 100644 --- a/src/pkg/services/m365/api/client.go +++ b/src/pkg/services/m365/api/client.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" ) @@ -36,11 +37,13 @@ type Client struct { // arbitrary urls instead of constructing queries using the // graph api client. Requester graph.Requester + + options control.Options } // NewClient produces a new exchange api client. Must be used in // place of creating an ad-hoc client struct. -func NewClient(creds account.M365Config) (Client, error) { +func NewClient(creds account.M365Config, co control.Options) (Client, error) { s, err := NewService(creds) if err != nil { return Client{}, err @@ -53,7 +56,11 @@ func NewClient(creds account.M365Config) (Client, error) { rqr := graph.NewNoTimeoutHTTPWrapper() - return Client{creds, s, li, rqr}, nil + if co.DeltaPageSize < 1 || co.DeltaPageSize > maxDeltaPageSize { + co.DeltaPageSize = maxDeltaPageSize + } + + return Client{creds, s, li, rqr, co}, nil } // initConcurrencyLimit ensures that the graph concurrency limiter is diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index f997bd2e7..9a86f1e00 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -277,7 +277,7 @@ func (c Contacts) NewContactDeltaIDsPager( Select: idAnd(parentFolderID), // do NOT set Top. It limits the total items received. }, - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), } var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder diff --git a/src/pkg/services/m365/api/events_pager.go b/src/pkg/services/m365/api/events_pager.go index d70e1d281..2874d37e5 100644 --- a/src/pkg/services/m365/api/events_pager.go +++ b/src/pkg/services/m365/api/events_pager.go @@ -244,7 +244,7 @@ func (c Events) NewEventDeltaIDsPager( immutableIDs bool, ) (itemIDPager, error) { options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), QueryParameters: &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{ // do NOT set Top. It limits the total items received. }, diff --git a/src/pkg/services/m365/api/groups.go b/src/pkg/services/m365/api/groups.go index 19c381323..589da1a45 100644 --- a/src/pkg/services/m365/api/groups.go +++ b/src/pkg/services/m365/api/groups.go @@ -4,15 +4,20 @@ import ( "context" "github.com/alcionai/clues" + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/common/tform" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/fault" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/pkg/logger" ) -const teamService = "Team" +const ( + teamsAdditionalDataLabel = "Team" + ResourceProvisioningOptions = "resourceProvisioningOptions" +) // --------------------------------------------------------------------------- // controller @@ -22,20 +27,18 @@ func (c Client) Groups() Groups { return Groups{c} } -// On creation of each Microsoft Teams a corrsponding group gets created from them. -// Most of the information like events, drive and mail info will be fetched directly -// from groups. So we pull in group and process only the once which are associated with -// a team for further proccessing of teams. +// On creation of each Teams team a corrsponding group gets created. +// The group acts as the protected resource, and all teams data like events, +// drive and mail messages are owned by that group. -// Teams is an interface-compliant provider of the client. +// Groups is an interface-compliant provider of the client. type Groups struct { Client } -// GetAll retrieves all groups. +// GetAllGroups retrieves all groups. func (c Groups) GetAll( ctx context.Context, - filterTeams bool, errs *fault.Bus, ) ([]models.Groupable, error) { service, err := c.Service() @@ -43,17 +46,29 @@ func (c Groups) GetAll( return nil, err } - return getGroups(ctx, filterTeams, errs, service) + return getGroups(ctx, func(ctx context.Context, g models.Groupable) bool { return true }, errs, service) +} + +// GetTeams retrieves all Teams. +func (c Groups) GetTeams( + ctx context.Context, + errs *fault.Bus, +) ([]models.Groupable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + return getGroups(ctx, FetchOnlyTeams, errs, service) } // GetAll retrieves all groups. func getGroups( ctx context.Context, - filterTeams bool, + filterGroupsData func(ctx context.Context, g models.Groupable) bool, errs *fault.Bus, service graph.Servicer, ) ([]models.Groupable, error) { - resp, err := service.Client().Groups().Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting all groups") @@ -62,7 +77,7 @@ func getGroups( iter, err := msgraphgocore.NewPageIterator[models.Groupable]( resp, service.Adapter(), - models.CreateTeamCollectionResponseFromDiscriminatorValue) + models.CreateGroupCollectionResponseFromDiscriminatorValue) if err != nil { return nil, graph.Wrap(ctx, err, "creating groups iterator") } @@ -81,8 +96,7 @@ func getGroups( if err != nil { el.AddRecoverable(ctx, graph.Wrap(ctx, err, "validating groups")) } else { - isTeam := IsTeam(item) - if !filterTeams || isTeam { + if filterGroupsData(ctx, item) { groups = append(groups, item) } } @@ -97,23 +111,28 @@ func getGroups( return groups, el.Failure() } -func IsTeam(g models.Groupable) bool { - if g.GetAdditionalData()["resourceProvisioningOptions"] != nil { - val, _ := tform.AnyValueToT[[]any]("resourceProvisioningOptions", g.GetAdditionalData()) +func FetchOnlyTeams(ctx context.Context, g models.Groupable) bool { + log := logger.Ctx(ctx) + + if g.GetAdditionalData()[ResourceProvisioningOptions] != nil { + val, _ := tform.AnyValueToT[[]any](ResourceProvisioningOptions, g.GetAdditionalData()) for _, v := range val { s, err := str.AnyToString(v) if err != nil { + log.Debug("could not be converted to string value: ", ResourceProvisioningOptions) return false } - if s == teamService { + + if s == teamsAdditionalDataLabel { return true } } } + return false } -// GetID retrieves team by groupID/teamID. +// GetID retrieves group by groupID. func (c Groups) GetByID( ctx context.Context, identifier string, @@ -125,21 +144,38 @@ func (c Groups) GetByID( resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting group by ID") - } - - if err != nil { - err := graph.Wrap(ctx, err, "getting teams by id") - - // TODO: check if its applicable here - if graph.IsErrItemNotFound(err) { - err = clues.Stack(graph.ErrResourceOwnerNotFound, err) - } + err := graph.Wrap(ctx, err, "getting group by id") return nil, err } - return resp, err + return resp, graph.Stack(ctx, err).OrNil() +} + +// GetTeamByID retrieves group by groupID. +func (c Groups) GetTeamByID( + ctx context.Context, + identifier string, +) (models.Groupable, error) { + service, err := c.Service() + if err != nil { + return nil, err + } + + resp, err := service.Client().Groups().ByGroupId(identifier).Get(ctx, nil) + if err != nil { + err := graph.Wrap(ctx, err, "getting group by id") + + return nil, err + } + + if !FetchOnlyTeams(ctx, resp) { + err := clues.New("given teamID is not related to any team") + + return nil, err + } + + return resp, graph.Stack(ctx, err).OrNil() } // --------------------------------------------------------------------------- @@ -154,7 +190,7 @@ func ValidateGroup(item models.Groupable) error { } if item.GetDisplayName() == nil { - return clues.New("missing principalName") + return clues.New("missing display name") } return nil diff --git a/src/pkg/services/m365/api/groups_test.go b/src/pkg/services/m365/api/groups_test.go index 96e131273..5d43ea08d 100644 --- a/src/pkg/services/m365/api/groups_test.go +++ b/src/pkg/services/m365/api/groups_test.go @@ -3,53 +3,238 @@ package api_test import ( "testing" + "github.com/alcionai/clues" + "github.com/google/uuid" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/fault" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) -type TeamsUnitSuite struct { +type GroupUnitSuite struct { tester.Suite } -func TestTeamsUnitSuite(t *testing.T) { - suite.Run(t, &TeamsUnitSuite{Suite: tester.NewUnitSuite(t)}) +func TestGroupsUnitSuite(t *testing.T) { + suite.Run(t, &GroupUnitSuite{Suite: tester.NewUnitSuite(t)}) } -type TeamsIntgSuite struct { +func (suite *GroupUnitSuite) TestValidateGroup() { + group := models.NewGroup() + group.SetDisplayName(ptr.To("testgroup")) + group.SetId(ptr.To("testID")) + + tests := []struct { + name string + args models.Groupable + errCheck assert.ErrorAssertionFunc + errIsSkippable bool + }{ + { + name: "Valid group ", + args: func() *models.Group { + s := models.NewGroup() + s.SetId(ptr.To("id")) + s.SetDisplayName(ptr.To("testgroup")) + return s + }(), + errCheck: assert.NoError, + }, + { + name: "No name", + args: func() *models.Group { + s := models.NewGroup() + s.SetId(ptr.To("id")) + return s + }(), + errCheck: assert.Error, + }, + { + name: "No ID", + args: func() *models.Group { + s := models.NewGroup() + s.SetDisplayName(ptr.To("testgroup")) + return s + }(), + errCheck: assert.Error, + }, + } + + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + err := api.ValidateGroup(test.args) + test.errCheck(t, err, clues.ToCore(err)) + + if test.errIsSkippable { + assert.ErrorIs(t, err, api.ErrKnownSkippableCase) + } + }) + } +} + +type GroupsIntgSuite struct { tester.Suite its intgTesterSetup } -func TestTeamsIntgSuite(t *testing.T) { - suite.Run(t, &TeamsIntgSuite{ +func TestGroupsIntgSuite(t *testing.T) { + suite.Run(t, &GroupsIntgSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tconfig.M365AcctCredEnvs}), }) } -func (suite *TeamsIntgSuite) SetupSuite() { +func (suite *GroupsIntgSuite) SetupSuite() { suite.its = newIntegrationTesterSetup(suite.T()) } -func (suite *TeamsIntgSuite) TestGetAll() { +func (suite *GroupsIntgSuite) TestGetAllGroups() { t := suite.T() ctx, flush := tester.NewContext(t) defer flush() - teams, err := suite.its.ac. + groups, err := suite.its.ac. Groups(). - GetAll(ctx, true, fault.New(true)) + GetAll(ctx, fault.New(true)) require.NoError(t, err) - require.NotZero(t, len(teams), "must have at least one team") + require.NotZero(t, len(groups), "must have at least one group") +} - for _, team := range teams { - assert.NotEmpty(t, ptr.Val(team.GetDisplayName()), "must not return onedrive teams") +func (suite *GroupsIntgSuite) TestGetAllTeams() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + groups, err := suite.its.ac. + Groups(). + GetTeams(ctx, fault.New(true)) + require.NoError(t, err) + require.NotZero(t, len(groups), "must have at least one group") + + for _, team := range groups { + assert.True(t, api.FetchOnlyTeams(ctx, team), "must not return non groups groups") + } +} + +func (suite *GroupsIntgSuite) TestTeams_GetByID() { + var ( + t = suite.T() + teamID = tconfig.M365TeamsID(t) + ) + + teamsAPI := suite.its.ac.Groups() + + table := []struct { + name string + id string + expectErr func(*testing.T, error) + }{ + { + name: "3 part id", + id: teamID, + expectErr: func(t *testing.T, err error) { + assert.NoError(t, err, clues.ToCore(err)) + }, + }, + { + name: "malformed id", + id: uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + { + name: "random id", + id: uuid.NewString() + "," + uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + + { + name: "malformed url", + id: "barunihlda", + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + _, err := teamsAPI.GetTeamByID(ctx, test.id) + test.expectErr(t, err) + }) + } +} + +func (suite *GroupsIntgSuite) TestGroups_GetByID() { + var ( + t = suite.T() + groupID = tconfig.M365GroupID(t) + ) + + groupsAPI := suite.its.ac.Groups() + + table := []struct { + name string + id string + expectErr func(*testing.T, error) + }{ + { + name: "3 part id", + id: groupID, + expectErr: func(t *testing.T, err error) { + assert.NoError(t, err, clues.ToCore(err)) + }, + }, + { + name: "malformed id", + id: uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + { + name: "random id", + id: uuid.NewString() + "," + uuid.NewString(), + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + + { + name: "malformed url", + id: "barunihlda", + expectErr: func(t *testing.T, err error) { + assert.Error(t, err, clues.ToCore(err)) + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + _, err := groupsAPI.GetByID(ctx, test.id) + test.expectErr(t, err) + }) } } diff --git a/src/pkg/services/m365/api/helper_test.go b/src/pkg/services/m365/api/helper_test.go index 05e16b00e..cf61c0277 100644 --- a/src/pkg/services/m365/api/helper_test.go +++ b/src/pkg/services/m365/api/helper_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) @@ -82,6 +83,7 @@ type intgTesterSetup struct { siteID string siteDriveID string siteDriveRootFolderID string + teamID string } func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { @@ -96,7 +98,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { creds, err := a.M365Config() require.NoError(t, err, clues.ToCore(err)) - its.ac, err = api.NewClient(creds) + its.ac, err = api.NewClient(creds, control.DefaultOptions()) require.NoError(t, err, clues.ToCore(err)) its.gockAC, err = mock.NewClient(creds) @@ -130,5 +132,13 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId()) + // teams + its.teamID = tconfig.M365TeamsID(t) + + team, err := its.ac.Groups().GetTeamByID(ctx, its.teamID) + require.NoError(t, err, clues.ToCore(err)) + + its.teamID = ptr.Val(team.GetId()) + return its } diff --git a/src/pkg/services/m365/api/lists.go b/src/pkg/services/m365/api/lists.go new file mode 100644 index 000000000..fb6abaa48 --- /dev/null +++ b/src/pkg/services/m365/api/lists.go @@ -0,0 +1,64 @@ +package api + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Lists() Lists { + return Lists{c} +} + +// Lists is an interface-compliant provider of the client. +type Lists struct { + Client +} + +// PostDrive creates a new list of type drive. Specifically used to create +// documentLibraries for SharePoint Sites. +func (c Lists) PostDrive( + ctx context.Context, + siteID, driveName string, +) (models.Driveable, error) { + list := models.NewList() + list.SetDisplayName(&driveName) + list.SetDescription(ptr.To("corso auto-generated restore destination")) + + li := models.NewListInfo() + li.SetTemplate(ptr.To("documentLibrary")) + list.SetList(li) + + // creating a list of type documentLibrary will result in the creation + // of a new drive owned by the given site. + builder := c.Stable. + Client(). + Sites(). + BySiteId(siteID). + Lists() + + newList, err := builder.Post(ctx, list, nil) + if graph.IsErrItemAlreadyExistsConflict(err) { + return nil, clues.Stack(graph.ErrItemAlreadyExistsConflict, err).WithClues(ctx) + } + + if err != nil { + return nil, graph.Wrap(ctx, err, "creating documentLibrary list") + } + + // drive information is not returned by the list creation. + drive, err := builder. + ByListId(ptr.Val(newList.GetId())). + Drive(). + Get(ctx, nil) + + return drive, graph.Wrap(ctx, err, "fetching created documentLibrary").OrNil() +} diff --git a/src/pkg/services/m365/api/lists_test.go b/src/pkg/services/m365/api/lists_test.go new file mode 100644 index 000000000..5864427f2 --- /dev/null +++ b/src/pkg/services/m365/api/lists_test.go @@ -0,0 +1,57 @@ +package api_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/control/testdata" +) + +type ListsAPIIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func (suite *ListsAPIIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func TestListsAPIIntgSuite(t *testing.T) { + suite.Run(t, &ListsAPIIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *ListsAPIIntgSuite) TestLists_PostDrive() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + acl = suite.its.ac.Lists() + driveName = testdata.DefaultRestoreConfig("list_api_post_drive").Location + siteID = suite.its.siteID + ) + + // first post, should have no errors + list, err := acl.PostDrive(ctx, siteID, driveName) + require.NoError(t, err, clues.ToCore(err)) + // the site name cannot be set when posting, only its DisplayName. + // so we double check here that we're still getting the name we expect. + assert.Equal(t, driveName, ptr.Val(list.GetName())) + + // second post, same name, should error on name conflict] + _, err = acl.PostDrive(ctx, siteID, driveName) + require.ErrorIs(t, err, graph.ErrItemAlreadyExistsConflict, clues.ToCore(err)) +} diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index 5472239f8..0648a906c 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -310,7 +310,7 @@ func (c Mail) NewMailDeltaIDsPager( Select: idAnd("isRead"), // do NOT set Top. It limits the total items received. }, - Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)), + Headers: newPreferHeaders(preferPageSize(c.options.DeltaPageSize), preferImmutableIDs(immutableIDs)), } var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder diff --git a/src/pkg/services/m365/api/sites.go b/src/pkg/services/m365/api/sites.go index e573cfc07..4e13ebcfb 100644 --- a/src/pkg/services/m365/api/sites.go +++ b/src/pkg/services/m365/api/sites.go @@ -225,13 +225,13 @@ func ValidateSite(item models.Siteable) error { wURL := ptr.Val(item.GetWebUrl()) if len(wURL) == 0 { - return clues.New("missing webURL").With("site_id", id) // TODO: pii + return clues.New("missing webURL").With("site_id", clues.Hide(id)) } // personal (ie: oneDrive) sites have to be filtered out server-side. if strings.Contains(wURL, PersonalSitePath) { return clues.Stack(ErrKnownSkippableCase). - With("site_id", id, "site_web_url", wURL) // TODO: pii + With("site_id", clues.Hide(id), "site_web_url", clues.Hide(wURL)) } name := ptr.Val(item.GetDisplayName()) @@ -239,10 +239,10 @@ func ValidateSite(item models.Siteable) error { // the built-in site at "https://{tenant-domain}/search" never has a name. if strings.HasSuffix(wURL, "/search") { return clues.Stack(ErrKnownSkippableCase). - With("site_id", id, "site_web_url", wURL) // TODO: pii + With("site_id", clues.Hide(id), "site_web_url", clues.Hide(wURL)) } - return clues.New("missing site display name").With("site_id", id) + return clues.New("missing site display name").With("site_id", clues.Hide(id)) } return nil diff --git a/src/pkg/services/m365/api/sites_test.go b/src/pkg/services/m365/api/sites_test.go index d8f49614d..8c4ccf17f 100644 --- a/src/pkg/services/m365/api/sites_test.go +++ b/src/pkg/services/m365/api/sites_test.go @@ -143,12 +143,16 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { var ( t = suite.T() siteID = tconfig.M365SiteID(t) - host = strings.Split(siteID, ",")[0] - shortID = strings.TrimPrefix(siteID, host+",") + parts = strings.Split(siteID, ",") + uuids = siteID siteURL = tconfig.M365SiteURL(t) modifiedSiteURL = siteURL + "foo" ) + if len(parts) == 3 { + uuids = strings.Join(parts[1:], ",") + } + sitesAPI := suite.its.ac.Sites() table := []struct { @@ -165,7 +169,7 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { }, { name: "2 part id", - id: shortID, + id: uuids, expectErr: func(t *testing.T, err error) { assert.NoError(t, err, clues.ToCore(err)) }, @@ -191,13 +195,6 @@ func (suite *SitesIntgSuite) TestSites_GetByID() { assert.NoError(t, err, clues.ToCore(err)) }, }, - { - name: "host only", - id: host, - expectErr: func(t *testing.T, err error) { - assert.NoError(t, err, clues.ToCore(err)) - }, - }, { name: "malformed url", id: "barunihlda", diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index b08cf7926..752454363 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/services/m365/api" @@ -408,7 +409,7 @@ func makeAC( return api.Client{}, clues.Wrap(err, "getting m365 account creds") } - cli, err := api.NewClient(creds) + cli, err := api.NewClient(creds, control.DefaultOptions()) if err != nil { return api.Client{}, clues.Wrap(err, "constructing api client") } diff --git a/website/blog/2023-07-24-multi-tenant-backup-with-corso.md b/website/blog/2023-07-24-multi-tenant-backup-with-corso.md new file mode 100644 index 000000000..e828bad2c --- /dev/null +++ b/website/blog/2023-07-24-multi-tenant-backup-with-corso.md @@ -0,0 +1,175 @@ +--- +slug: multi-tenant-backup-with-corso +title: "Using Corso to Build a Self-Hosted Multi-Tenant Office 365 Backup Solution" +description: "" +authors: + - name: meuchels + title: Corso Community Member, IT Lead + url: https://github.com/meuchels + image_url: https://avatars.githubusercontent.com/u/77171293?v=4 +tags: [corso, microsoft 365, backups, msp, multi-tenant] +date: 2023-07-24 +image: ./images/data-center.jpg +--- + +![A woman engineer holding a laptop in front of a data center](./images/data-center.jpg) + +This community-contributed blog post shows how MSPs in the community are using Corso to build out a multi-tenant backup +solution for their Microsoft 365 customers. If you have questions, come find the author (or us) on +[Discord](https://www.alcion.ai/discord). + + + +First of all, I offer a fully managed backup solution. My clients have no access to the backup software or the data. I +require them to request recovery in a ticket. For my use case I have a self-hosted instance of MinIO that I won't be +going over but there is [another blog post on it](./2023-2-4-where-to-store-corso.md#local-s3-testing). I will show the +layout and an example of how to backup emails using the exchange option in Corso. + +## Organizing the file structure on your storage + +I wanted my S3 bucket to be laid out in the following fashion utilizing 1 bucket with prefixes for the tenants. For now, +all I did is create a bucket with access to a user for corso. While it's possible to use a single bucket and use prefix +paths per tenant within it, I didn't do that in my setup. The will be generated later with the backup initialization. + +```bash +BUCKET + tenant1-exchange + tenant1-onedrive + tenant1-sharepoint + tenant2-exchange + tenant2-onedrive + tenant2-sharepoint +``` + +If I don’t backup a particular service for a client, it will be clear by looking at whether the bucket exists or not. + +I have a short name for each tenant to differentiate them. + +## The backup compute server layout + +I utilize Ubuntu Server for this task. In my setup, everything is done as the root user. I have put the corso +executable in `/opt/corso/` and will be building everything under there. Here is the folder layout before I go into +usage. + +```bash +# For logs +/opt/corso/logs +# For config files +/opt/corso/toml +# Root of the scripts folder +/opt/corso/scripts +# For building out the environment loaders +/opt/corso/scripts/environments +# For building out the backup scripts +/opt/corso/scripts/back-available +# For adding a link to the backups that will be run +/opt/corso/scripts/back-active +``` + +## The environment files + +For [configuration](../../docs/setup/configuration/), create an environment file +`/opt/corso/scripts/environments/blank-exchange` with the following content for a template. You can copy this template +to `-exchange` in the same folder to setup your client exchange backup environment. + +```bash +##################################### +#EDIT THIS SECTION TO MEET YOUR NEEDS +##################################### + +# this is a shortname for your tenant to setup storage +export tenantshortname="" + +# this is your tenant info from the app setup on O365 +export AZURE_TENANT_ID="" +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" + +# this is your credentials for your s3 storage +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="{ - `corso restore onedrive --backup abcd` + `corso restore onedrive --backup a422895c-c20c-4b06-883d-b866db9f86ef` } If the flag isn't provided, Corso will create a new folder with a standard name: @@ -29,7 +29,7 @@ data integrity then this is always the safest option. ### An alternate destination { - `corso restore onedrive --backup abcd --destination /my-latest-restore` + `corso restore onedrive --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a destination is manually specified, all restored will appear in that top-level @@ -41,14 +41,14 @@ folder multiple times. ### The original location { - `corso restore onedrive --backup abcd --destination /` + `corso restore onedrive --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } You can restore items back to their original location by setting the destination to `/`. This skips the creation of a top-level folder, and all restored items will appear back in their location at the time of backup. -### Limitations +### Destination Limitations * Destination won't create N-depth folder structures. `--destination a/b/c` doesn't create three folders; it creates a single, top-level folder named `a/b/c`. @@ -79,19 +79,19 @@ it still collides. Collisions can be handled with three different configurations: `Skip`, `Copy`, and `Replace`. -## Skip (default) +### Skip (default) { - `corso restore onedrive --backup abcd --collisions skip --destination /` + `corso restore onedrive --collisions skip --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } When a collision is identified, the item is skipped and no restore is attempted. -## Copy +### Copy { - `corso restore onedrive --backup abcd --collisions copy --destination /my-latest-restore` + `corso restore onedrive --collisions copy --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Item collisions create a copy of the item in the backup. The copy holds the backup @@ -99,12 +99,31 @@ version of the item, leaving the current version unchanged. If necessary, change item properties (such as filenames) to avoid additional collisions. Eg: the copy of`reports.txt` is named `reports 1.txt`. -## Replace +### Replace { - `corso restore onedrive --backup abcd --collisions replace --destination /` + `corso restore onedrive --collisions replace --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef` } Collisions will entirely replace the current version of the item with the backup version. If multiple existing items collide with the backup item, only one of the existing items is replaced. + +## Restore to target resource + +The `--to-resource` flag lets you select which resource will receive the restored data. +A resource can be a mailbox, user, sharepoint site, or other owner of data. + +When restoring to a target resource, all other restore configuration behaves normally. +Data is restored into the default folder: `Corso_Restore_` (unless a +`--destination` flag is added). When restoring in-place, collision policies are followed. + +{ + `corso restore onedrive --to-resource adelev@alcion.ai --backup a422895c-c20c-4b06-883d-b866db9f86ef` +} + +### Resource Limitations + +* The resource must exist. Corso won't create new mailboxes, users, or sites. +* The resource must have access to the service being restored. No restore will be +performed for an unlicensed resource. diff --git a/website/docs/support/known-issues.md b/website/docs/support/known-issues.md index 754bddfb6..e6bc12809 100644 --- a/website/docs/support/known-issues.md +++ b/website/docs/support/known-issues.md @@ -16,8 +16,6 @@ Below is a list of known Corso issues and limitations: from M365 while a backup creation is running. The next backup creation will correct any missing data. -* SharePoint document library data can't be restored after the library has been deleted. - * Sharing information of items in OneDrive/SharePoint using sharing links aren't backed up and restored. * Permissions/Access given to a site group can't be restored. diff --git a/website/package-lock.json b/website/package-lock.json index 21df1ab55..23f62b662 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -16,15 +16,15 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", - "mermaid": "^10.2.4", + "mermaid": "^10.3.0", "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -33,7 +33,7 @@ "@docusaurus/module-type-aliases": "2.4.1", "@iconify/react": "^4.1.1", "autoprefixer": "^10.4.14", - "postcss": "^8.4.26", + "postcss": "^8.4.27", "tailwindcss": "^3.3.3" } }, @@ -3377,6 +3377,24 @@ "@types/node": "*" } }, + "node_modules/@types/d3-scale": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", + "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "node_modules/@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, "node_modules/@types/debug": { "version": "4.1.8", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz", @@ -6079,6 +6097,41 @@ "node": ">=12" } }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" + }, "node_modules/d3-scale": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", @@ -6541,14 +6594,14 @@ } }, "node_modules/docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "dependencies": { "sass-loader": "^10.1.1" }, "peerDependencies": { - "@docusaurus/core": "^2.0.0-beta", + "@docusaurus/core": "^2.0.0-beta || ^3.0.0-alpha", "sass": "^1.30.0" } }, @@ -6612,9 +6665,9 @@ } }, "node_modules/dompurify": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.3.tgz", - "integrity": "sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-F9e6wPGtY+8KNMRAVfxeCOHU0/NPWMSENNq4pQctuXRqqdEPW7q3CrLbR5Nse044WwacyjHGOMlvNsBe1y6z9A==" }, "node_modules/domutils": { "version": "2.8.0", @@ -9279,18 +9332,21 @@ } }, "node_modules/mermaid": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.2.4.tgz", - "integrity": "sha512-zHGjEI7lBvWZX+PQYmlhSA2p40OzW6QbGodTCSzDeVpqaTnyAC+2sRGqrpXO+uQk3CnoeClHQPraQUMStdqy2g==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.3.0.tgz", + "integrity": "sha512-H5quxuQjwXC8M1WuuzhAp2TdqGg74t5skfDBrNKJ7dt3z8Wprl5S6h9VJsRhoBUTSs1TMtHEdplLhCqXleZZLw==", "dependencies": { "@braintree/sanitize-url": "^6.0.2", + "@types/d3-scale": "^4.0.3", + "@types/d3-scale-chromatic": "^3.0.0", "cytoscape": "^3.23.0", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.1.0", "d3": "^7.4.0", + "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "3.0.3", + "dompurify": "3.0.5", "elkjs": "^0.8.2", "khroma": "^2.0.0", "lodash-es": "^4.17.21", @@ -10656,9 +10712,9 @@ } }, "node_modules/postcss": { - "version": "8.4.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.26.tgz", - "integrity": "sha512-jrXHFF8iTloAenySjM/ob3gSj7pCu0Ji49hnjqzsgSRa50hkWCKD0HQ+gMNJkW38jBI68MpAAg7ZWwHwX8NMMw==", + "version": "8.4.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.27.tgz", + "integrity": "sha512-gY/ACJtJPSmUFPDCHtX78+01fHa64FaU4zaaWfuh1MhGJISufJAH4cun6k/8fwsHYeK4UQmENQK+tRLCFJE8JQ==", "funding": [ { "type": "opencollective", @@ -12571,9 +12627,9 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.1.tgz", + "integrity": "sha512-16rRACSOFEE8VN7SCgBu1MpYCyN7urj9At898tyzdXFhC+a+yOX5dXwAR7L8/IdPJ1NB8OYoXmD55DM30B2kEQ==", "dependencies": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", @@ -17620,6 +17676,24 @@ "@types/node": "*" } }, + "@types/d3-scale": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.3.tgz", + "integrity": "sha512-PATBiMCpvHJSMtZAMEhc2WyL+hnzarKzI6wAHYjhsonjWJYGq5BXTzQjv4l8m2jO183/4wZ90rKvSeT7o72xNQ==", + "requires": { + "@types/d3-time": "*" + } + }, + "@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, "@types/debug": { "version": "4.1.8", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz", @@ -19542,6 +19616,43 @@ "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==" }, + "d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "requires": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + }, + "dependencies": { + "d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "requires": { + "internmap": "^1.0.0" + } + }, + "d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==" + }, + "d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "requires": { + "d3-path": "1" + } + }, + "internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==" + } + } + }, "d3-scale": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", @@ -19867,9 +19978,9 @@ } }, "docusaurus-plugin-sass": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.4.tgz", - "integrity": "sha512-r9bLXW6X2z64bzQUQZB1SxmNlGvSO9swTFALgiMjr/1O4FRDti6BseU4Sw2mlZkYvVQTq8cJMJIP6w7z/5We8Q==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/docusaurus-plugin-sass/-/docusaurus-plugin-sass-0.2.5.tgz", + "integrity": "sha512-Z+D0fLFUKcFpM+bqSUmqKIU+vO+YF1xoEQh5hoFreg2eMf722+siwXDD+sqtwU8E4MvVpuvsQfaHwODNlxJAEg==", "requires": { "sass-loader": "^10.1.1" } @@ -19913,9 +20024,9 @@ } }, "dompurify": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.3.tgz", - "integrity": "sha512-axQ9zieHLnAnHh0sfAamKYiqXMJAVwu+LM/alQ7WDagoWessyWvMSFyW65CqF3owufNu8HBcE4cM2Vflu7YWcQ==" + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-F9e6wPGtY+8KNMRAVfxeCOHU0/NPWMSENNq4pQctuXRqqdEPW7q3CrLbR5Nse044WwacyjHGOMlvNsBe1y6z9A==" }, "domutils": { "version": "2.8.0", @@ -21729,18 +21840,21 @@ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==" }, "mermaid": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.2.4.tgz", - "integrity": "sha512-zHGjEI7lBvWZX+PQYmlhSA2p40OzW6QbGodTCSzDeVpqaTnyAC+2sRGqrpXO+uQk3CnoeClHQPraQUMStdqy2g==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-10.3.0.tgz", + "integrity": "sha512-H5quxuQjwXC8M1WuuzhAp2TdqGg74t5skfDBrNKJ7dt3z8Wprl5S6h9VJsRhoBUTSs1TMtHEdplLhCqXleZZLw==", "requires": { "@braintree/sanitize-url": "^6.0.2", + "@types/d3-scale": "^4.0.3", + "@types/d3-scale-chromatic": "^3.0.0", "cytoscape": "^3.23.0", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.1.0", "d3": "^7.4.0", + "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.10", "dayjs": "^1.11.7", - "dompurify": "3.0.3", + "dompurify": "3.0.5", "elkjs": "^0.8.2", "khroma": "^2.0.0", "lodash-es": "^4.17.21", @@ -22569,9 +22683,9 @@ "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==" }, "postcss": { - "version": "8.4.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.26.tgz", - "integrity": "sha512-jrXHFF8iTloAenySjM/ob3gSj7pCu0Ji49hnjqzsgSRa50hkWCKD0HQ+gMNJkW38jBI68MpAAg7ZWwHwX8NMMw==", + "version": "8.4.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.27.tgz", + "integrity": "sha512-gY/ACJtJPSmUFPDCHtX78+01fHa64FaU4zaaWfuh1MhGJISufJAH4cun6k/8fwsHYeK4UQmENQK+tRLCFJE8JQ==", "requires": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", @@ -23802,9 +23916,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sass": { - "version": "1.63.6", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.63.6.tgz", - "integrity": "sha512-MJuxGMHzaOW7ipp+1KdELtqKbfAWbH7OLIdoSMnVe3EXPMTmxTmlaZDCTsgIpPCs3w99lLo9/zDKkOrJuT5byw==", + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.64.1.tgz", + "integrity": "sha512-16rRACSOFEE8VN7SCgBu1MpYCyN7urj9At898tyzdXFhC+a+yOX5dXwAR7L8/IdPJ1NB8OYoXmD55DM30B2kEQ==", "requires": { "chokidar": ">=3.0.0 <4.0.0", "immutable": "^4.0.0", diff --git a/website/package.json b/website/package.json index 2c7272ce9..470fbf579 100644 --- a/website/package.json +++ b/website/package.json @@ -22,15 +22,15 @@ "animate.css": "^4.1.1", "clsx": "^2.0.0", "docusaurus-plugin-image-zoom": "^1.0.1", - "docusaurus-plugin-sass": "^0.2.4", + "docusaurus-plugin-sass": "^0.2.5", "feather-icons": "^4.29.0", "jarallax": "^2.1.3", "mdx-mermaid": "^1.3.2", - "mermaid": "^10.2.4", + "mermaid": "^10.3.0", "prism-react-renderer": "^1.3.5", "react": "^17.0.2", "react-dom": "^17.0.2", - "sass": "^1.63.6", + "sass": "^1.64.1", "tiny-slider": "^2.9.4", "tw-elements": "^1.0.0-alpha13", "wow.js": "^1.2.2" @@ -39,7 +39,7 @@ "@docusaurus/module-type-aliases": "2.4.1", "@iconify/react": "^4.1.1", "autoprefixer": "^10.4.14", - "postcss": "^8.4.26", + "postcss": "^8.4.27", "tailwindcss": "^3.3.3" }, "browserslist": {