Compare commits
No commits in common. "main" and "v0.17.0" have entirely different histories.
69
.github/actions/backup-restore-test/action.yml
vendored
69
.github/actions/backup-restore-test/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Backup Restore Test
|
||||
description: Run various backup/restore/export tests for a service.
|
||||
|
||||
inputs:
|
||||
service:
|
||||
@ -19,10 +18,6 @@ inputs:
|
||||
description: Arguments to pass for restore; restore is skipped when missing.
|
||||
required: false
|
||||
default: ""
|
||||
export-args:
|
||||
description: Arguments to pass for export.
|
||||
required: false
|
||||
default: ""
|
||||
restore-container:
|
||||
description: Folder to use for testing
|
||||
required: true
|
||||
@ -37,9 +32,6 @@ inputs:
|
||||
description: Runs export tests when true
|
||||
required: false
|
||||
default: false
|
||||
category:
|
||||
description: category of data for given service
|
||||
required: false
|
||||
|
||||
outputs:
|
||||
backup-id:
|
||||
@ -57,9 +49,7 @@ runs:
|
||||
echo Backup ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-backup-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-backup-${{inputs.kind }}.log
|
||||
./corso backup create '${{ inputs.service }}' \
|
||||
--no-stats --hide-progress --json \
|
||||
${{ inputs.backup-args }} |
|
||||
@ -78,9 +68,7 @@ runs:
|
||||
echo Restore ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
|
||||
./corso restore '${{ inputs.service }}' \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
@ -103,17 +91,11 @@ runs:
|
||||
SANITY_TEST_RESTORE_CONTAINER: ${{ steps.restore.outputs.result }}
|
||||
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
|
||||
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
|
||||
# lists are not restored to a different folder. they get created adjacent to their originals
|
||||
# hence SANITY_TEST_RESTORE_CONTAINER_PREFIX is necessary to differentiate restored from original
|
||||
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
|
||||
SANITY_TEST_CATEGORY: ${{ inputs.category }}
|
||||
run: |
|
||||
echo "---------------------------"
|
||||
echo Sanity Test Restore ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
|
||||
./sanity-test restore ${{ inputs.service }}
|
||||
|
||||
- name: Export ${{ inputs.service }} ${{ inputs.kind }}
|
||||
@ -126,11 +108,9 @@ runs:
|
||||
echo Export ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
|
||||
./corso export '${{ inputs.service }}' \
|
||||
/tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }} \
|
||||
/tmp/export-${{ inputs.service }}-${{inputs.kind }} \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
${{ inputs.export-args }} \
|
||||
@ -143,19 +123,14 @@ runs:
|
||||
shell: bash
|
||||
working-directory: src
|
||||
env:
|
||||
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{ inputs.kind }}
|
||||
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}
|
||||
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
|
||||
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
|
||||
# applies only for sharepoint lists
|
||||
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
|
||||
SANITY_TEST_CATEGORY: ${{ inputs.category }}
|
||||
run: |
|
||||
echo "---------------------------"
|
||||
echo Sanity-Test Export ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
|
||||
./sanity-test export ${{ inputs.service }}
|
||||
|
||||
- name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
|
||||
@ -168,19 +143,17 @@ runs:
|
||||
echo Export Archive ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
|
||||
./corso export '${{ inputs.service }}' \
|
||||
/tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive \
|
||||
/tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
--archive \
|
||||
${{ inputs.export-args }} \
|
||||
--backup '${{ steps.backup.outputs.result }}'
|
||||
|
||||
unzip /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive/*.zip \
|
||||
-d /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-unzipped
|
||||
unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \
|
||||
-d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
|
||||
cat /tmp/corsologs
|
||||
|
||||
- name: Check archive export ${{ inputs.service }} ${{ inputs.kind }}
|
||||
@ -188,19 +161,14 @@ runs:
|
||||
shell: bash
|
||||
working-directory: src
|
||||
env:
|
||||
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{inputs.kind }}-unzipped
|
||||
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
|
||||
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
|
||||
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
|
||||
# applies only for sharepoint lists
|
||||
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
|
||||
SANITY_TEST_CATEGORY: ${{ inputs.category }}
|
||||
run: |
|
||||
echo "---------------------------"
|
||||
echo Sanity-Test Export Archive ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
|
||||
./sanity-test export ${{ inputs.service }}
|
||||
|
||||
- name: List ${{ inputs.service }} ${{ inputs.kind }}
|
||||
@ -211,9 +179,7 @@ runs:
|
||||
echo Backup list ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}${CATEGORY_SUFFIX}-list-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}-list-${{inputs.kind }}.log
|
||||
./corso backup list ${{ inputs.service }} \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
@ -234,10 +200,7 @@ runs:
|
||||
echo Backup List w/ Backup ${{ inputs.service }} ${{ inputs.kind }}
|
||||
echo "---------------------------"
|
||||
set -euo pipefail
|
||||
# Include category in the log file name if present
|
||||
CATEGORY_SUFFIX=""
|
||||
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}${CATEGORY_SUFFIX}-single-${{inputs.kind }}.log
|
||||
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}-single-${{inputs.kind }}.log
|
||||
./corso backup list ${{ inputs.service }} \
|
||||
--no-stats \
|
||||
--hide-progress \
|
||||
@ -265,4 +228,4 @@ runs:
|
||||
name: "${{ inputs.service }}-${{ inputs.kind }}-logs"
|
||||
path: ${{ inputs.log-dir }}/*
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
retention-days: 14
|
||||
1
.github/actions/go-setup-cache/action.yml
vendored
1
.github/actions/go-setup-cache/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Setup and Cache Golang
|
||||
description: Build golang binaries for later use in CI.
|
||||
|
||||
# clone of: https://github.com/magnetikonline/action-golang-cache/blob/main/action.yaml
|
||||
#
|
||||
|
||||
1
.github/actions/publish-binary/action.yml
vendored
1
.github/actions/publish-binary/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Publish Binary
|
||||
description: Publish binary artifacts.
|
||||
|
||||
inputs:
|
||||
version:
|
||||
|
||||
1
.github/actions/publish-website/action.yml
vendored
1
.github/actions/publish-website/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Publish Website
|
||||
description: Publish website artifacts.
|
||||
|
||||
inputs:
|
||||
aws-iam-role:
|
||||
|
||||
44
.github/actions/purge-m365-data/action.yml
vendored
44
.github/actions/purge-m365-data/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Purge M365 User Data
|
||||
description: Deletes M365 data generated during CI tests.
|
||||
|
||||
# Hard deletion of an m365 user's data. Our CI processes create a lot
|
||||
# of data churn (creation and immediate deletion) of files, the likes
|
||||
@ -31,19 +30,12 @@ inputs:
|
||||
description: Secret value of for AZURE_CLIENT_ID
|
||||
azure-client-secret:
|
||||
description: Secret value of for AZURE_CLIENT_SECRET
|
||||
azure-pnp-client-id:
|
||||
description: Secret value of AZURE_PNP_CLIENT_ID
|
||||
azure-pnp-client-cert:
|
||||
description: Base64 encoded private certificate for the azure-pnp-client-id (Secret value of AZURE_PNP_CLIENT_CERT)
|
||||
azure-tenant-id:
|
||||
description: Secret value of AZURE_TENANT_ID
|
||||
description: Secret value of for AZURE_TENANT_ID
|
||||
m365-admin-user:
|
||||
description: Secret value of for M365_TENANT_ADMIN_USER
|
||||
m365-admin-password:
|
||||
description: Secret value of for M365_TENANT_ADMIN_PASSWORD
|
||||
tenant-domain:
|
||||
description: The domain of the tenant (ex. 10rqc2.onmicrosft.com)
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
@ -61,13 +53,7 @@ runs:
|
||||
AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
|
||||
AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
|
||||
AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
|
||||
run: |
|
||||
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
|
||||
{
|
||||
if (./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
run: ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
|
||||
|
||||
# TODO(ashmrtn): Re-enable when we figure out errors we're seeing with Get-Mailbox call.
|
||||
#- name: Reset retention for all mailboxes to 0
|
||||
@ -88,16 +74,10 @@ runs:
|
||||
shell: pwsh
|
||||
working-directory: ./src/cmd/purge/scripts
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
|
||||
AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
|
||||
TENANT_DOMAIN: ${{ inputs.tenant-domain }}
|
||||
M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
|
||||
M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
|
||||
run: |
|
||||
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
|
||||
{
|
||||
if (./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
|
||||
|
||||
################################################################################################################
|
||||
# Sharepoint
|
||||
@ -108,14 +88,6 @@ runs:
|
||||
shell: pwsh
|
||||
working-directory: ./src/cmd/purge/scripts
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
|
||||
AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
|
||||
TENANT_DOMAIN: ${{ inputs.tenant-domain }}
|
||||
run: |
|
||||
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
|
||||
{
|
||||
if (./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
|
||||
M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
|
||||
run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
|
||||
|
||||
1
.github/actions/teams-message/action.yml
vendored
1
.github/actions/teams-message/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Send a message to Teams
|
||||
description: Send messages to communication apps.
|
||||
|
||||
inputs:
|
||||
msg:
|
||||
|
||||
1
.github/actions/website-linting/action.yml
vendored
1
.github/actions/website-linting/action.yml
vendored
@ -1,5 +1,4 @@
|
||||
name: Lint Website
|
||||
description: Lint website content.
|
||||
|
||||
inputs:
|
||||
version:
|
||||
|
||||
2
.github/workflows/_filechange_checker.yml
vendored
2
.github/workflows/_filechange_checker.yml
vendored
@ -28,7 +28,7 @@ jobs:
|
||||
|
||||
# only run CI tests if the src folder or workflow actions have changed
|
||||
- name: Check for file changes in src/ or .github/workflows/
|
||||
uses: dorny/paths-filter@v3
|
||||
uses: dorny/paths-filter@v2
|
||||
id: dornycheck
|
||||
with:
|
||||
list-files: json
|
||||
|
||||
2
.github/workflows/binary-publish.yml
vendored
2
.github/workflows/binary-publish.yml
vendored
@ -40,5 +40,5 @@ jobs:
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[CORSO FAILED] Publishing Binary"
|
||||
msg: "[FAILED] Publishing Binary"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
35
.github/workflows/ci.yml
vendored
35
.github/workflows/ci.yml
vendored
@ -189,7 +189,7 @@ jobs:
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-test-log
|
||||
path: src/testlog/*
|
||||
@ -260,7 +260,7 @@ jobs:
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ci-retention-test-log
|
||||
path: src/testlog/*
|
||||
@ -315,7 +315,7 @@ jobs:
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: unit-test-log
|
||||
path: src/testlog/*
|
||||
@ -404,7 +404,7 @@ jobs:
|
||||
# Upload the original go test log as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: fork-test-log
|
||||
path: src/testlog/*
|
||||
@ -463,7 +463,7 @@ jobs:
|
||||
go-version-file: src/go.mod
|
||||
|
||||
- name: Go Lint
|
||||
uses: golangci/golangci-lint-action@v4
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
# Keep pinned to a verson as sometimes updates will add new lint
|
||||
# failures in unchanged code.
|
||||
@ -492,8 +492,8 @@ jobs:
|
||||
|
||||
# I could not find a way to install tree-grepper without nix
|
||||
# https://github.com/BrianHicks/tree-grepper/issues/293
|
||||
- uses: cachix/install-nix-action@v25
|
||||
- uses: cachix/cachix-action@v14
|
||||
- uses: cachix/install-nix-action@v24
|
||||
- uses: cachix/cachix-action@v13
|
||||
with:
|
||||
name: tree-grepper
|
||||
- run: nix-env -if https://github.com/BrianHicks/tree-grepper/archive/refs/heads/main.tar.gz
|
||||
@ -511,27 +511,6 @@ jobs:
|
||||
echo "Use len check instead of empty string comparison"
|
||||
exit 1
|
||||
fi
|
||||
- name: Check for cases where errors are not propagated
|
||||
run: |
|
||||
# Using `grep .` as the exit codes are always true for correct grammar
|
||||
if tree-grepper -q go '((if_statement (binary_expression) @_if (block (return_statement (expression_list (call_expression (selector_expression) @_fun ) @ret .)))) (#match? @_if "err != nil") (#match? @_fun "clues.NewWC"))' | grep .; then
|
||||
echo "Make sure to propagate errors with clues"
|
||||
exit 1
|
||||
fi
|
||||
- name: Check if clues without context are used when context is passed in
|
||||
run: |
|
||||
# Using `grep .` as the exit codes are always true for correct grammar
|
||||
if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue))) . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#match? @clue "WC$"))' | grep .; then
|
||||
echo "Do not use clues.*WC when context is passed in"
|
||||
exit 1
|
||||
fi
|
||||
- name: Check clues with context is used when context is not passed in
|
||||
run: |
|
||||
# Using `grep .` as the exit codes are always true for correct grammar
|
||||
if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue))) . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#not-eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#not-match? @clue "WC$"))' | grep .; then
|
||||
echo "Use clues.*WC when context is not passed in"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------
|
||||
# --- GitHub Actions Linting -------------------------------------------------------------------------
|
||||
|
||||
14
.github/workflows/ci_test_cleanup.yml
vendored
14
.github/workflows/ci_test_cleanup.yml
vendored
@ -12,7 +12,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
matrix:
|
||||
user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, ""]
|
||||
user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, '' ]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@ -33,15 +33,12 @@ jobs:
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
|
||||
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
|
||||
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
|
||||
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
|
||||
tenant-domain: ${{ vars.TENANT_DOMAIN }}
|
||||
|
||||
- name: Notify failure in teams
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[CORSO FAILED] ${{ vars[matrix.user] }} CI Cleanup"
|
||||
msg: "[FAILED] ${{ vars[matrix.user] }} CI Cleanup"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
Test-Site-Data-Cleanup:
|
||||
@ -50,7 +47,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
matrix:
|
||||
site: [CORSO_M365_TEST_SITE_URL, CORSO_M365_TEST_GROUPS_SITE_URL]
|
||||
site: [ CORSO_M365_TEST_SITE_URL, CORSO_M365_TEST_GROUPS_SITE_URL ]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@ -73,13 +70,10 @@ jobs:
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
|
||||
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
|
||||
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
|
||||
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
|
||||
tenant-domain: ${{ vars.TENANT_DOMAIN }}
|
||||
|
||||
- name: Notify failure in teams
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[CORSO FAILED] ${{ vars[matrix.site] }} CI Cleanup"
|
||||
msg: "[FAILED] ${{ vars[matrix.site] }} CI Cleanup"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
5
.github/workflows/load_test.yml
vendored
5
.github/workflows/load_test.yml
vendored
@ -107,7 +107,7 @@ jobs:
|
||||
# package all artifacts for later review
|
||||
- name: Upload Log, Profilers, Traces
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: load-test-profiling
|
||||
path: ${{ github.workspace }}/testlog/*
|
||||
@ -155,6 +155,3 @@ jobs:
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
|
||||
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
|
||||
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
|
||||
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
|
||||
tenant-domain: ${{ vars.TENANT_DOMAIN }}
|
||||
|
||||
19
.github/workflows/longevity_test.yml
vendored
19
.github/workflows/longevity_test.yml
vendored
@ -6,7 +6,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
user:
|
||||
description: "User to run longevity test on"
|
||||
description: 'User to run longevity test on'
|
||||
|
||||
permissions:
|
||||
# required to retrieve AWS credentials
|
||||
@ -23,7 +23,7 @@ jobs:
|
||||
uses: alcionai/corso/.github/workflows/accSelector.yaml@main
|
||||
|
||||
Longevity-Tests:
|
||||
needs: [SetM365App]
|
||||
needs: [ SetM365App ]
|
||||
environment: Testing
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
@ -37,7 +37,7 @@ jobs:
|
||||
CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-longevity.log
|
||||
RESTORE_DEST_PFX: Corso_Test_Longevity_
|
||||
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
|
||||
PREFIX: "longevity"
|
||||
PREFIX: 'longevity'
|
||||
|
||||
# Options for retention.
|
||||
RETENTION_MODE: GOVERNANCE
|
||||
@ -46,7 +46,7 @@ jobs:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: src
|
||||
|
||||
|
||||
############################################################################
|
||||
# setup
|
||||
steps:
|
||||
@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
- run: go build -o corso
|
||||
timeout-minutes: 10
|
||||
|
||||
|
||||
- run: mkdir ${CORSO_LOG_DIR}
|
||||
|
||||
# Use shorter-lived credentials obtained from assume-role since these
|
||||
@ -113,6 +113,7 @@ jobs:
|
||||
--extend-retention \
|
||||
--prefix ${{ env.PREFIX }} \
|
||||
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
|
||||
--succeed-if-exists \
|
||||
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
|
||||
|
||||
if grep -q 'Failed to' ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
|
||||
@ -163,7 +164,7 @@ jobs:
|
||||
|
||||
data=$( echo $resultjson | jq -r '.[0] | .id' )
|
||||
echo result=$data >> $GITHUB_OUTPUT
|
||||
|
||||
|
||||
##########################################################################
|
||||
# Onedrive
|
||||
|
||||
@ -328,7 +329,7 @@ jobs:
|
||||
--hide-progress \
|
||||
--force \
|
||||
--json \
|
||||
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_metadata.txt
|
||||
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_metadata.txt
|
||||
|
||||
- name: Maintenance test Weekly
|
||||
id: maintenance-test-weekly
|
||||
@ -381,7 +382,7 @@ jobs:
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: longevity-test-log
|
||||
path: src/testlog/*
|
||||
@ -392,5 +393,5 @@ jobs:
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[CORSO FAILED] Longevity Test"
|
||||
msg: "[FAILED] Longevity Test"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
10
.github/workflows/nightly_test.yml
vendored
10
.github/workflows/nightly_test.yml
vendored
@ -48,7 +48,7 @@ jobs:
|
||||
# ----------------------------------------------------------------------------------------------------
|
||||
|
||||
Test-Suite-Trusted:
|
||||
needs: [Checkout, SetM365App]
|
||||
needs: [ Checkout, SetM365App]
|
||||
environment: Testing
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
@ -100,14 +100,14 @@ jobs:
|
||||
-timeout 2h \
|
||||
./... 2>&1 | tee ./testlog/gotest-nightly.log | gotestfmt -hide successful-tests
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Logging & Notifications
|
||||
# Logging & Notifications
|
||||
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: nightly-test-log
|
||||
path: src/testlog/*
|
||||
@ -118,5 +118,5 @@ jobs:
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[COROS FAILED] Nightly Checks"
|
||||
msg: "[FAILED] Nightly Checks"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
2
.github/workflows/ok-to-test.yml
vendored
2
.github/workflows/ok-to-test.yml
vendored
@ -19,7 +19,7 @@ jobs:
|
||||
private_key: ${{ secrets.PRIVATE_KEY }}
|
||||
|
||||
- name: Slash Command Dispatch
|
||||
uses: peter-evans/slash-command-dispatch@v4
|
||||
uses: peter-evans/slash-command-dispatch@v3
|
||||
env:
|
||||
TOKEN: ${{ steps.generate_token.outputs.token }}
|
||||
with:
|
||||
|
||||
189
.github/workflows/sanity-test.yaml
vendored
189
.github/workflows/sanity-test.yaml
vendored
@ -6,7 +6,7 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
user:
|
||||
description: "User to run sanity test on"
|
||||
description: 'User to run sanity test on'
|
||||
|
||||
permissions:
|
||||
# required to retrieve AWS credentials
|
||||
@ -23,7 +23,7 @@ jobs:
|
||||
uses: alcionai/corso/.github/workflows/accSelector.yaml@main
|
||||
|
||||
Sanity-Tests:
|
||||
needs: [SetM365App]
|
||||
needs: [ SetM365App ]
|
||||
environment: Testing
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
@ -43,11 +43,12 @@ jobs:
|
||||
defaults:
|
||||
run:
|
||||
working-directory: src
|
||||
|
||||
##########################################################################################################################################
|
||||
|
||||
##########################################################################################################################################
|
||||
|
||||
# setup
|
||||
# setup
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Golang with cache
|
||||
@ -63,9 +64,9 @@ jobs:
|
||||
|
||||
- run: mkdir ${CORSO_LOG_DIR}
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Pre-Run cleanup
|
||||
# Pre-Run cleanup
|
||||
|
||||
# unlike CI tests, sanity tests are not expected to run concurrently.
|
||||
# however, the sanity yaml concurrency is set to a maximum of 1 run, preferring
|
||||
@ -90,9 +91,6 @@ jobs:
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
|
||||
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
|
||||
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
|
||||
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
|
||||
tenant-domain: ${{ vars.TENANT_DOMAIN }}
|
||||
|
||||
- name: Purge CI-Produced Folders for Sites
|
||||
timeout-minutes: 30
|
||||
@ -101,20 +99,17 @@ jobs:
|
||||
with:
|
||||
site: ${{ vars.CORSO_M365_TEST_SITE_URL }}
|
||||
folder-prefix: ${{ env.RESTORE_DEST_PFX }}
|
||||
libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }}
|
||||
libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }}
|
||||
older-than: ${{ env.NOW }}
|
||||
azure-client-id: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
|
||||
azure-client-secret: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
|
||||
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
|
||||
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
|
||||
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
|
||||
tenant-domain: ${{ vars.TENANT_DOMAIN }}
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Repository commands
|
||||
# Repository commands
|
||||
|
||||
- name: Version Test
|
||||
timeout-minutes: 10
|
||||
@ -174,9 +169,9 @@ jobs:
|
||||
--mode complete \
|
||||
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-maintenance.log
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Exchange
|
||||
# Exchange
|
||||
|
||||
# generate new entries to roll into the next load test
|
||||
# only runs if the test was successful
|
||||
@ -198,8 +193,8 @@ jobs:
|
||||
service: exchange
|
||||
kind: first-backup
|
||||
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
|
||||
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
@ -211,8 +206,8 @@ jobs:
|
||||
service: exchange
|
||||
kind: incremental
|
||||
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
|
||||
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
@ -225,8 +220,8 @@ jobs:
|
||||
service: exchange
|
||||
kind: non-delta
|
||||
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email" --disable-delta'
|
||||
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
@ -239,15 +234,16 @@ jobs:
|
||||
service: exchange
|
||||
kind: non-delta-incremental
|
||||
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
|
||||
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
|
||||
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
|
||||
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
##########################################################################################################################################
|
||||
|
||||
# Onedrive
|
||||
##########################################################################################################################################
|
||||
|
||||
# Onedrive
|
||||
|
||||
# generate new entries for test
|
||||
- name: OneDrive - Create new data
|
||||
@ -274,8 +270,8 @@ jobs:
|
||||
service: onedrive
|
||||
kind: first-backup
|
||||
backup-args: '--user "${{ env.TEST_USER }}"'
|
||||
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
|
||||
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
@ -299,14 +295,14 @@ jobs:
|
||||
service: onedrive
|
||||
kind: incremental
|
||||
backup-args: '--user "${{ env.TEST_USER }}"'
|
||||
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
|
||||
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Sharepoint Library
|
||||
# Sharepoint
|
||||
|
||||
# generate new entries for test
|
||||
- name: SharePoint - Create new data
|
||||
@ -333,12 +329,11 @@ jobs:
|
||||
with:
|
||||
service: sharepoint
|
||||
kind: first-backup
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
|
||||
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"'
|
||||
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
category: libraries
|
||||
|
||||
# generate some more enteries for incremental check
|
||||
- name: SharePoint - Create new data (for incremental)
|
||||
@ -360,107 +355,15 @@ jobs:
|
||||
with:
|
||||
service: sharepoint
|
||||
kind: incremental
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
|
||||
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"'
|
||||
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
category: libraries
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Sharepoint Lists
|
||||
|
||||
# generate new entries for test
|
||||
# The `awk | tr | sed` command chain is used to get a comma separated list of SharePoint list names.
|
||||
- name: SharePoint Lists - Create new data
|
||||
id: new-data-creation-sharepoint-lists
|
||||
timeout-minutes: 30
|
||||
working-directory: ./src/cmd/factory
|
||||
run: |
|
||||
suffix=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
|
||||
go run . sharepoint lists \
|
||||
--site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
|
||||
--user ${{ env.TEST_USER }} \
|
||||
--secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
|
||||
--tenant ${{ secrets.TENANT_ID }} \
|
||||
--destination ${{ env.RESTORE_DEST_PFX }}$suffix \
|
||||
--count 4 |
|
||||
awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
|
||||
tee $GITHUB_OUTPUT
|
||||
# Extracts the common prefix for the Sharepoint list names.
|
||||
- name: SharePoint Lists - Store restore container
|
||||
id: sharepoint-lists-store-restore-container
|
||||
run: |
|
||||
echo ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} |
|
||||
cut -d',' -f1 |
|
||||
cut -d'_' -f1,2,3,4,5 |
|
||||
sed -e 's/^/result=/' |
|
||||
tee $GITHUB_OUTPUT
|
||||
|
||||
- name: SharePoint Lists - Backup
|
||||
id: sharepoint-lists-backup
|
||||
timeout-minutes: 30
|
||||
uses: ./.github/actions/backup-restore-test
|
||||
with:
|
||||
service: sharepoint
|
||||
kind: first-backup-lists
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
|
||||
restore-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
|
||||
export-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
|
||||
restore-container: "${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
category: lists
|
||||
on-collision: copy
|
||||
|
||||
# generate some more enteries for incremental check
|
||||
- name: SharePoint Lists - Create new data (for incremental)
|
||||
id: inc-data-creation-sharepoint-lists
|
||||
timeout-minutes: 30
|
||||
working-directory: ./src/cmd/factory
|
||||
run: |
|
||||
suffix=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
|
||||
go run . sharepoint lists \
|
||||
--site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
|
||||
--user ${{ env.TEST_USER }} \
|
||||
--secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
|
||||
--tenant ${{ secrets.TENANT_ID }} \
|
||||
--destination ${{ env.RESTORE_DEST_PFX }}$suffix \
|
||||
--count 4 |
|
||||
awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
|
||||
tee $GITHUB_OUTPUT
|
||||
|
||||
- name: SharePoint Lists - Store restore container (for incremental)
|
||||
id: sharepoint-lists-store-restore-container-inc
|
||||
run: |
|
||||
echo ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }} |
|
||||
cut -d',' -f1 |
|
||||
cut -d'_' -f1,2,3,4,5 |
|
||||
sed -e 's/^/result=/' |
|
||||
tee $GITHUB_OUTPUT
|
||||
|
||||
- name: SharePoint Lists - Incremental backup
|
||||
id: sharepoint-lists-incremental
|
||||
timeout-minutes: 30
|
||||
uses: ./.github/actions/backup-restore-test
|
||||
with:
|
||||
service: sharepoint
|
||||
kind: incremental-lists
|
||||
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
|
||||
restore-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
|
||||
export-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
|
||||
restore-container: "${{ steps.sharepoint-lists-store-restore-container-inc.outputs.result }},${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
category: lists
|
||||
on-collision: copy
|
||||
|
||||
##########################################################################################################################################
|
||||
|
||||
# Groups and Teams
|
||||
# Groups and Teams
|
||||
|
||||
# generate new entries for test
|
||||
- name: Groups - Create new data
|
||||
@ -487,8 +390,8 @@ jobs:
|
||||
with:
|
||||
service: groups
|
||||
kind: first-backup
|
||||
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
|
||||
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}"'
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
@ -512,20 +415,20 @@ jobs:
|
||||
with:
|
||||
service: groups
|
||||
kind: incremental
|
||||
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
|
||||
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}"'
|
||||
restore-args: '--site "${{ vars.CORSO_M365_TEST_GROUPS_SITE_URL }}" --folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
|
||||
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
|
||||
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
|
||||
log-dir: ${{ env.CORSO_LOG_DIR }}
|
||||
with-export: true
|
||||
|
||||
##########################################################################################################################################
|
||||
##########################################################################################################################################
|
||||
|
||||
# Logging & Notifications
|
||||
# Logging & Notifications
|
||||
|
||||
# Upload the original go test output as an artifact for later review.
|
||||
- name: Upload test log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: sanity-test-log
|
||||
path: ${{ env.CORSO_LOG_DIR }}/*
|
||||
@ -536,5 +439,5 @@ jobs:
|
||||
if: failure()
|
||||
uses: ./.github/actions/teams-message
|
||||
with:
|
||||
msg: "[CORSO FAILED] Sanity Tests"
|
||||
msg: "[FAILED] Sanity Tests"
|
||||
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|
||||
|
||||
63
CHANGELOG.md
63
CHANGELOG.md
@ -6,71 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased] (beta)
|
||||
### Fixed
|
||||
- Handle the case where an email or event cannot be retrieved from Exchange due to an `ErrorCorruptData` error. Corso will skip over the item but report it in the backup summary.
|
||||
- Emails attached within other emails are now correctly exported
|
||||
- Gracefully handle email and post attachments without name when exporting to eml
|
||||
- Use correct timezone for event start and end times in Exchange exports (helps fix issues in relative recurrence patterns)
|
||||
- Fixed an issue causing exports dealing with calendar data to have high memory usage
|
||||
|
||||
## [v0.19.0] (beta) - 2024-02-06
|
||||
|
||||
### Added
|
||||
- Events can now be exported from Exchange backups as .ics files.
|
||||
- Update repo init configuration to reduce the total number of GET requests sent
|
||||
to the object store when using corso. This affects repos that have many
|
||||
backups created in them per day the most.
|
||||
- Feature Preview: Corso now supports backup, export & restore of SharePoint lists. Lists backup can be initiated using `corso backup create sharepoint --site <site-url> --data lists`.
|
||||
- Group mailbox(aka conversations) backup and export support is now officially available. Group mailbox posts can be exported as `.eml` files.
|
||||
|
||||
### Fixed
|
||||
- Retry transient 400 "invalidRequest" errors during onedrive & sharepoint backup.
|
||||
- Backup attachments associated with group mailbox items.
|
||||
- Groups and Teams backups no longer fail when a resource has no display name.
|
||||
- Contacts in-place restore failed if the restore destination was empty.
|
||||
- Link shares with external users are now backed up and restored as expected
|
||||
- Ensure persistent repo config is populated on repo init if repo init failed partway through during the previous init attempt.
|
||||
|
||||
### Changed
|
||||
- When running `backup details` on an empty backup returns a more helpful error message.
|
||||
- Backup List additionally shows the data category for each backup.
|
||||
- Remove hidden `--succeed-if-exists` flag for repo init. Repo init will now succeed without error if run on an existing repo with the same passphrase.
|
||||
|
||||
### Known issues
|
||||
- Backing up a group mailbox item may fail if it has a very large number of attachments (500+).
|
||||
- Event description for exchange exports might look slightly different for certain events.
|
||||
- Exchange in-place restore may restore items in well-known folders to different folders if the user has well-known folder names change based on locale and has updated the locale since the backup was created.
|
||||
- In-place Exchange contacts restore will merge items in folders named "Contacts" or "contacts" into the default folder.
|
||||
- External users with access through shared links will not receive these links as they are not sent via email during restore.
|
||||
- Graph API has limited support for certain column types such as `location`, `hyperlink/picture`, and `metadata`. Restoring SharePoint list items containing these columns will result in differences compared to the original items.
|
||||
- SharePoint list item attachments are not available due to graph API limitations.
|
||||
- Group mailbox restore is not supported due to limited Graph API support for creating mailbox items.
|
||||
- Due to Graph API limitations, any group mailbox items present in subfolders other than Inbox aren't backed up.
|
||||
|
||||
## [v0.18.0] (beta) - 2024-01-02
|
||||
|
||||
### Fixed
|
||||
- Handle the case where an email cannot be retrieved from Exchange due to an `ErrorInvalidRecipients` error. In
|
||||
this case, Corso will skip over the item but report this in the backup summary.
|
||||
- Fix `ErrorItemNotFound` errors when restoring emails with multiple attachments.
|
||||
- Avoid Graph SDK `Requests must contain extension changes exclusively.` errors by removing server-populated field from restored event items.
|
||||
- Improve Group mailbox(conversations) backup performance by only downloading new items or items with modified content.
|
||||
- Handle cases where Exchange backup stored invalid JSON blobs if there were special characters in the user content. These would result in errors during restore.
|
||||
|
||||
### Known issues
|
||||
- Restoring OneDrive, SharePoint, or Teams & Groups items shared with external users while the tenant or site is configured to not allow sharing with external users will not restore permissions.
|
||||
|
||||
### Added
|
||||
- Contacts can now be exported from Exchange backups as .vcf files
|
||||
|
||||
## [v0.17.0] (beta) - 2023-12-11
|
||||
|
||||
### Changed
|
||||
- Memory optimizations for large scale OneDrive and Sharepoint backups.
|
||||
|
||||
### Fixed
|
||||
- Resolved a possible deadlock when backing up Teams Channel Messages.
|
||||
- Fixed an attachment download failure(ErrorTooManyObjectsOpened) during exchange backup.
|
||||
|
||||
## [v0.16.0] (beta) - 2023-11-28
|
||||
|
||||
@ -502,11 +445,7 @@ this case, Corso will skip over the item but report this in the backup summary.
|
||||
- Miscellaneous
|
||||
- Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35))
|
||||
|
||||
[Unreleased]: https://github.com/alcionai/corso/compare/v0.19.0...HEAD
|
||||
[v0.19.0]: https://github.com/alcionai/corso/compare/v0.18.0...v0.19.0
|
||||
[v0.18.0]: https://github.com/alcionai/corso/compare/v0.17.0...v0.18.0
|
||||
[v0.17.0]: https://github.com/alcionai/corso/compare/v0.16.0...v0.17.0
|
||||
[v0.16.0]: https://github.com/alcionai/corso/compare/v0.15.0...v0.16.0
|
||||
[Unreleased]: https://github.com/alcionai/corso/compare/v0.15.0...HEAD
|
||||
[v0.15.0]: https://github.com/alcionai/corso/compare/v0.14.0...v0.15.0
|
||||
[v0.14.0]: https://github.com/alcionai/corso/compare/v0.13.0...v0.14.0
|
||||
[v0.13.0]: https://github.com/alcionai/corso/compare/v0.12.0...v0.13.0
|
||||
|
||||
@ -1,6 +1,3 @@
|
||||
> [!NOTE]
|
||||
> **The Corso project is no longer actively maintained and has been archived**.
|
||||
|
||||
<p align="center">
|
||||
<img src="https://github.com/alcionai/corso/blob/main/website/static/img/corso_logo.svg?raw=true" alt="Corso Logo" width="100" />
|
||||
</p>
|
||||
|
||||
@ -4,7 +4,6 @@ run:
|
||||
linters:
|
||||
enable:
|
||||
- errcheck
|
||||
- exhaustive
|
||||
- forbidigo
|
||||
- gci
|
||||
- gofmt
|
||||
@ -26,11 +25,6 @@ linters:
|
||||
- staticcheck
|
||||
|
||||
linters-settings:
|
||||
exhaustive:
|
||||
check:
|
||||
- switch
|
||||
default-signifies-exhaustive: false
|
||||
explicit-exhaustive-switch: true
|
||||
gci:
|
||||
sections:
|
||||
- standard
|
||||
@ -55,7 +49,7 @@ linters-settings:
|
||||
# String formatting should be avoided in favor of structured errors (ie: err.With(k, v)).
|
||||
- '(errors|fmt)\.(New|Stack|Wrap|Error)f?\((# error handling should use clues pkg)?'
|
||||
# Avoid Warn-level logging in favor of Info or Error.
|
||||
- 'Warnw?f?\((# logging should use Info or Error)?'
|
||||
- 'Warn[wf]?\((# logging should use Info or Error)?'
|
||||
# Prefer suite.Run(name, func() {}) for subtests as testify has it instead
|
||||
# of suite.T().Run(name, func(t *testing.T) {}).
|
||||
- '(T\(\)|\st[a-zA-Z0-9]*)\.Run(# prefer testify suite.Run(name, func()) )?'
|
||||
|
||||
@ -2,6 +2,7 @@ package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
@ -19,16 +20,14 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/errs/core"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
"github.com/alcionai/corso/src/pkg/store"
|
||||
)
|
||||
|
||||
var ErrEmptyBackup = clues.New("no items in backup")
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// adding commands to cobra
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -45,7 +44,6 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{
|
||||
addOneDriveCommands,
|
||||
addSharePointCommands,
|
||||
addGroupsCommands,
|
||||
addTeamsChatsCommands,
|
||||
}
|
||||
|
||||
// AddCommands attaches all `corso backup * *` commands to the parent.
|
||||
@ -190,47 +188,45 @@ func genericCreateCommand(
|
||||
ictx = clues.Add(ctx, "resource_owner_selected", owner)
|
||||
)
|
||||
|
||||
logger.Ctx(ictx).Infof("setting up backup")
|
||||
|
||||
bo, err := r.NewBackupWithLookup(ictx, discSel, ins)
|
||||
if err != nil {
|
||||
cerr := clues.WrapWC(ictx, err, owner)
|
||||
errs = append(errs, cerr)
|
||||
|
||||
Errf(
|
||||
ictx,
|
||||
"%s\nCause: %s",
|
||||
"Unable to initiate backup",
|
||||
err.Error())
|
||||
meta, err := json.Marshal(cerr.Core().Values)
|
||||
if err != nil {
|
||||
meta = []byte("Unable to marshal error metadata")
|
||||
}
|
||||
|
||||
Errf(ictx, "%s\nMessage: %v\nMetadata:%s", "Unable to complete backup", err, meta)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
ictx = clues.Add(
|
||||
ictx,
|
||||
ctx,
|
||||
"resource_owner_id", bo.ResourceOwner.ID(),
|
||||
"resource_owner_name", clues.Hide(bo.ResourceOwner.Name()))
|
||||
|
||||
logger.Ctx(ictx).Infof("running backup")
|
||||
"resource_owner_name", bo.ResourceOwner.Name())
|
||||
|
||||
err = bo.Run(ictx)
|
||||
if err != nil {
|
||||
if errors.Is(err, core.ErrServiceNotEnabled) {
|
||||
logger.Ctx(ictx).Infow("service not enabled",
|
||||
if errors.Is(err, graph.ErrServiceNotEnabled) {
|
||||
logger.Ctx(ctx).Infow("service not enabled",
|
||||
"resource_owner_id", bo.ResourceOwner.ID(),
|
||||
"service", serviceName)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
cerr := clues.Wrap(err, owner)
|
||||
cerr := clues.WrapWC(ictx, err, owner)
|
||||
errs = append(errs, cerr)
|
||||
|
||||
Errf(
|
||||
ictx,
|
||||
"%s\nCause: %s",
|
||||
"Unable to complete backup",
|
||||
err.Error())
|
||||
meta, err := json.Marshal(cerr.Core().Values)
|
||||
if err != nil {
|
||||
meta = []byte("Unable to marshal error metadata")
|
||||
}
|
||||
|
||||
Errf(ictx, "%s\nMessage: %v\nMetadata:%s", "Unable to complete backup", err, meta)
|
||||
|
||||
continue
|
||||
}
|
||||
@ -238,10 +234,10 @@ func genericCreateCommand(
|
||||
bIDs = append(bIDs, string(bo.Results.BackupID))
|
||||
|
||||
if !DisplayJSONFormat() {
|
||||
Infof(ictx, fmt.Sprintf("Backup complete %s %s", observe.Bullet, color.BlueOutput(bo.Results.BackupID)))
|
||||
printBackupStats(ictx, r, string(bo.Results.BackupID))
|
||||
Infof(ctx, fmt.Sprintf("Backup complete %s %s", observe.Bullet, color.BlueOutput(bo.Results.BackupID)))
|
||||
printBackupStats(ctx, r, string(bo.Results.BackupID))
|
||||
} else {
|
||||
Infof(ictx, "Backup complete - ID: %v\n", bo.Results.BackupID)
|
||||
Infof(ctx, "Backup complete - ID: %v\n", bo.Results.BackupID)
|
||||
}
|
||||
}
|
||||
|
||||
@ -338,7 +334,7 @@ func genericListCommand(
|
||||
fe.PrintItems(
|
||||
ctx,
|
||||
!ifShow(flags.ListAlertsFV),
|
||||
!ifShow(flags.FailedItemsFV),
|
||||
!ifShow(flags.ListFailedItemsFV),
|
||||
!ifShow(flags.ListSkippedItemsFV),
|
||||
!ifShow(flags.ListRecoveredErrorsFV))
|
||||
|
||||
@ -398,10 +394,6 @@ func genericDetailsCore(
|
||||
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
|
||||
}
|
||||
|
||||
if len(d.Entries) == 0 {
|
||||
return nil, ErrEmptyBackup
|
||||
}
|
||||
|
||||
if opts.SkipReduce {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
@ -5,12 +5,10 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/utils/testdata"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
@ -68,30 +66,3 @@ func (suite *BackupUnitSuite) TestGenericDetailsCore() {
|
||||
assert.NoError(t, err, clues.ToCore(err))
|
||||
assert.ElementsMatch(t, expected, output.Entries)
|
||||
}
|
||||
|
||||
func (suite *BackupUnitSuite) TestGenericDetailsCore_empty() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
bg := testdata.VersionedBackupGetter{
|
||||
Details: &details.Details{
|
||||
DetailsModel: details.DetailsModel{
|
||||
Entries: []details.Entry{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
sel := selectors.NewExchangeBackup([]string{"user-id"})
|
||||
sel.Include(sel.AllData())
|
||||
|
||||
_, err := genericDetailsCore(
|
||||
ctx,
|
||||
bg,
|
||||
"backup-ID",
|
||||
sel.Selector,
|
||||
control.DefaultOptions())
|
||||
require.Error(t, err, "has error")
|
||||
assert.ErrorIs(t, err, ErrEmptyBackup, clues.ToCore(err))
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package backup
|
||||
import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
@ -61,11 +62,15 @@ corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
|
||||
// called by backup.go to map subcommands to provider-specific handling.
|
||||
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case createCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeCreateCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeCreateCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandCreateUseSuffix
|
||||
c.Example = exchangeServiceCommandCreateExamples
|
||||
@ -82,13 +87,15 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddDisableSlidingWindowLimiterFlag(c)
|
||||
|
||||
case listCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeListCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeListCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
flags.AddAllBackupListFlags(c)
|
||||
|
||||
case detailsCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeDetailsCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeDetailsCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandDetailsUseSuffix
|
||||
c.Example = exchangeServiceCommandDetailsExamples
|
||||
@ -101,7 +108,8 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddExchangeDetailsAndRestoreFlags(c, false)
|
||||
|
||||
case deleteCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeDeleteCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeDeleteCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandDeleteUseSuffix
|
||||
c.Example = exchangeServiceCommandDeleteExamples
|
||||
|
||||
@ -12,15 +12,14 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/its"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
@ -40,7 +39,7 @@ var (
|
||||
type NoBackupExchangeE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestNoBackupExchangeE2ESuite(t *testing.T) {
|
||||
@ -55,7 +54,7 @@ func (suite *NoBackupExchangeE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
|
||||
}
|
||||
|
||||
@ -70,7 +69,7 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_noBackups() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
@ -94,7 +93,7 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_noBackups() {
|
||||
type BackupExchangeE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestBackupExchangeE2ESuite(t *testing.T) {
|
||||
@ -109,7 +108,7 @@ func (suite *BackupExchangeE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
|
||||
}
|
||||
|
||||
@ -139,7 +138,7 @@ func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category path.
|
||||
cmd, ctx := buildExchangeBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
suite.m365.User.ID,
|
||||
suite.its.user.ID,
|
||||
category.String(),
|
||||
&recorder)
|
||||
|
||||
@ -150,11 +149,8 @@ func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category path.
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
|
||||
// As an offhand check: the result should contain the m365 user's email.
|
||||
assert.Contains(
|
||||
t,
|
||||
strings.ToLower(result),
|
||||
strings.ToLower(suite.m365.User.Provider.Name()))
|
||||
// as an offhand check: the result should contain the m365 user id
|
||||
assert.Contains(t, result, suite.its.user.ID)
|
||||
}
|
||||
|
||||
func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_ServiceNotEnabled_email() {
|
||||
@ -177,7 +173,7 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
|
||||
cmd, ctx := buildExchangeBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
fmt.Sprintf("%s,%s", tconfig.UnlicensedM365UserID(suite.T()), suite.m365.User.ID),
|
||||
fmt.Sprintf("%s,%s", tconfig.UnlicensedM365UserID(suite.T()), suite.its.user.ID),
|
||||
category.String(),
|
||||
&recorder)
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
@ -186,11 +182,8 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
|
||||
// As an offhand check: the result should contain the m365 user's email.
|
||||
assert.Contains(
|
||||
t,
|
||||
strings.ToLower(result),
|
||||
strings.ToLower(suite.m365.User.Provider.Name()))
|
||||
// as an offhand check: the result should contain the m365 user id
|
||||
assert.Contains(t, result, suite.its.user.ID)
|
||||
}
|
||||
|
||||
func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_userNotFound_email() {
|
||||
@ -229,8 +222,7 @@ func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category p
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"not found",
|
||||
"error missing user not found")
|
||||
"not found in tenant", "error missing user not found")
|
||||
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
|
||||
|
||||
t.Logf("backup error message: %s", err.Error())
|
||||
@ -249,7 +241,7 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAzureClientIDFl
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "exchange",
|
||||
"--user", suite.m365.User.ID,
|
||||
"--user", suite.its.user.ID,
|
||||
"--azure-client-id", "invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -273,8 +265,8 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_fromConfigFile() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "exchange",
|
||||
"--user", suite.m365.User.ID,
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--user", suite.its.user.ID,
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
@ -288,11 +280,8 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_fromConfigFile() {
|
||||
result := suite.dpnd.recorder.String()
|
||||
t.Log("backup results", result)
|
||||
|
||||
// As an offhand check: the result should contain the m365 user's email.
|
||||
assert.Contains(
|
||||
t,
|
||||
strings.ToLower(result),
|
||||
strings.ToLower(suite.m365.User.Provider.Name()))
|
||||
// as an offhand check: the result should contain the m365 user id
|
||||
assert.Contains(t, result, suite.its.user.ID)
|
||||
}
|
||||
|
||||
// AWS flags
|
||||
@ -306,7 +295,7 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAWSFlags() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "exchange",
|
||||
"--user", suite.m365.User.ID,
|
||||
"--user", suite.its.user.ID,
|
||||
"--aws-access-key", "invalid-value",
|
||||
"--aws-secret-access-key", "some-invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -329,7 +318,7 @@ type PreparedBackupExchangeE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
backupOps map[path.CategoryType]string
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestPreparedBackupExchangeE2ESuite(t *testing.T) {
|
||||
@ -346,13 +335,13 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
|
||||
suite.backupOps = make(map[path.CategoryType]string)
|
||||
|
||||
var (
|
||||
users = []string{suite.m365.User.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
|
||||
users = []string{suite.its.user.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.its.user.ID: suite.its.user.ID})
|
||||
)
|
||||
|
||||
for _, set := range []path.CategoryType{email, contacts, events} {
|
||||
@ -420,7 +409,7 @@ func runExchangeListCmdTest(suite *PreparedBackupExchangeE2ESuite, category path
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
@ -461,7 +450,7 @@ func runExchangeListSingleCmdTest(suite *PreparedBackupExchangeE2ESuite, categor
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -488,7 +477,7 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeListCmd_badID() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", "smarfs")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -530,7 +519,7 @@ func runExchangeDetailsCmdTest(suite *PreparedBackupExchangeE2ESuite, category p
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "details", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN, string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
@ -620,7 +609,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN,
|
||||
fmt.Sprintf("%s,%s",
|
||||
string(suite.backupOps[0].Results.BackupID),
|
||||
@ -634,7 +623,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[0].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -644,7 +633,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[1].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -662,7 +651,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_SingleID(
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN,
|
||||
string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -674,7 +663,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_SingleID(
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -692,7 +681,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_UnknownID
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN, uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -711,7 +700,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_NoBackupI
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// empty backupIDs should error since no data provided
|
||||
|
||||
@ -116,20 +116,8 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
|
||||
|
||||
opts := utils.MakeExchangeOpts(cmd)
|
||||
co := utils.Control()
|
||||
backupOpts := utils.ParseBackupOptions()
|
||||
|
||||
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
|
||||
// restore flags are switched over too and we no longer parse flags beyond
|
||||
// connection info into control.Options.
|
||||
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
|
||||
assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(backupOpts.M365.DeltaPageSize)))
|
||||
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
|
||||
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
|
||||
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
|
||||
assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
|
||||
assert.True(t, backupOpts.M365.ExchangeImmutableIDs)
|
||||
assert.True(t, backupOpts.ServiceRateLimiter.DisableSlidingWindowLimiter)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
|
||||
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
|
||||
assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(co.DeltaPageSize)))
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
@ -138,8 +126,6 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
|
||||
assert.True(t, co.ToggleFeatures.DisableDelta)
|
||||
assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs)
|
||||
assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
|
||||
flagsTD.AssertGenericBackupFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
@ -35,12 +36,9 @@ const (
|
||||
groupsServiceCommandCreateExamples = `# Backup all Groups and Teams data for the Marketing group
|
||||
corso backup create groups --group Marketing
|
||||
|
||||
# Backup only Teams channel messages
|
||||
# Backup only Teams conversations messages
|
||||
corso backup create groups --group Marketing --data messages
|
||||
|
||||
# Backup only group mailbox posts
|
||||
corso backup create groups --group Marketing --data conversations
|
||||
|
||||
# Backup all Groups and Teams data for all groups
|
||||
corso backup create groups --group '*'`
|
||||
|
||||
@ -53,19 +51,20 @@ corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd
|
||||
|
||||
# Explore Marketing messages posted after the start of 2022
|
||||
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--last-message-reply-after 2022-01-01T00:00:00
|
||||
|
||||
# Explore group mailbox posts with conversation subject "hello world"
|
||||
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"`
|
||||
--last-message-reply-after 2022-01-01T00:00:00`
|
||||
)
|
||||
|
||||
// called by backup.go to map subcommands to provider-specific handling.
|
||||
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case createCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreviewCommand())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandCreateUseSuffix
|
||||
c.Example = groupsServiceCommandCreateExamples
|
||||
@ -76,16 +75,17 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddFetchParallelismFlag(c)
|
||||
flags.AddDisableDeltaFlag(c)
|
||||
flags.AddGenericBackupFlags(c)
|
||||
flags.AddDisableLazyItemReader(c)
|
||||
|
||||
case listCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreviewCommand())
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
flags.AddAllBackupListFlags(c)
|
||||
|
||||
case detailsCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreviewCommand())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandDetailsUseSuffix
|
||||
c.Example = groupsServiceCommandDetailsExamples
|
||||
@ -99,7 +99,8 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddSharePointDetailsAndRestoreFlags(c)
|
||||
|
||||
case deleteCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreviewCommand())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandDeleteUseSuffix
|
||||
c.Example = groupsServiceCommandDeleteExamples
|
||||
@ -160,7 +161,7 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error {
|
||||
return Only(ctx, clues.Stack(err))
|
||||
}
|
||||
|
||||
ins, err := svcCli.AC.Groups().GetAllIDsAndNames(ctx, errs)
|
||||
ins, err := svcCli.GroupsMap(ctx, errs)
|
||||
if err != nil {
|
||||
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 groups"))
|
||||
}
|
||||
@ -316,7 +317,7 @@ func groupsBackupCreateSelectors(
|
||||
group, cats []string,
|
||||
) *selectors.GroupsBackup {
|
||||
if filters.PathContains(group).Compare(flags.Wildcard) {
|
||||
return includeAllGroupsWithCategories(ins, cats)
|
||||
return includeAllGroupWithCategories(ins, cats)
|
||||
}
|
||||
|
||||
sel := selectors.NewGroupsBackup(slices.Clone(group))
|
||||
@ -324,6 +325,6 @@ func groupsBackupCreateSelectors(
|
||||
return utils.AddGroupsCategories(sel, cats)
|
||||
}
|
||||
|
||||
func includeAllGroupsWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
|
||||
func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
|
||||
return utils.AddGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories)
|
||||
}
|
||||
|
||||
@ -14,15 +14,14 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/its"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||
@ -36,7 +35,7 @@ import (
|
||||
type NoBackupGroupsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestNoBackupGroupsE2ESuite(t *testing.T) {
|
||||
@ -51,7 +50,7 @@ func (suite *NoBackupGroupsE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
|
||||
}
|
||||
|
||||
@ -66,7 +65,7 @@ func (suite *NoBackupGroupsE2ESuite) TestGroupsBackupListCmd_noBackups() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
@ -90,7 +89,7 @@ func (suite *NoBackupGroupsE2ESuite) TestGroupsBackupListCmd_noBackups() {
|
||||
type BackupGroupsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestBackupGroupsE2ESuite(t *testing.T) {
|
||||
@ -105,7 +104,7 @@ func (suite *BackupGroupsE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
|
||||
}
|
||||
|
||||
@ -114,8 +113,6 @@ func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_channelMessages() {
|
||||
}
|
||||
|
||||
func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_conversations() {
|
||||
// skip
|
||||
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
|
||||
runGroupsBackupCategoryTest(suite, flags.DataConversations)
|
||||
}
|
||||
|
||||
@ -137,7 +134,7 @@ func runGroupsBackupCategoryTest(suite *BackupGroupsE2ESuite, category string) {
|
||||
cmd, ctx := buildGroupsBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
suite.m365.Group.ID,
|
||||
suite.its.group.ID,
|
||||
category,
|
||||
&recorder)
|
||||
|
||||
@ -185,8 +182,7 @@ func runGroupsBackupGroupNotFoundTest(suite *BackupGroupsE2ESuite, category stri
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"not found",
|
||||
"error missing user not found")
|
||||
"not found in tenant", "error missing group not found")
|
||||
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
|
||||
|
||||
t.Logf("backup error message: %s", err.Error())
|
||||
@ -205,7 +201,7 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAzureClientIDFlag()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "groups",
|
||||
"--group", suite.m365.Group.ID,
|
||||
"--group", suite.its.group.ID,
|
||||
"--azure-client-id", "invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -219,9 +215,6 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAzureClientIDFlag()
|
||||
}
|
||||
|
||||
func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_fromConfigFile() {
|
||||
// Skip
|
||||
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
|
||||
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
@ -232,8 +225,8 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_fromConfigFile() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "groups",
|
||||
"--group", suite.m365.Group.ID,
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--group", suite.its.group.ID,
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
@ -256,7 +249,7 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAWSFlags() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "groups",
|
||||
"--group", suite.m365.Group.ID,
|
||||
"--group", suite.its.group.ID,
|
||||
"--aws-access-key", "invalid-value",
|
||||
"--aws-secret-access-key", "some-invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -279,7 +272,7 @@ type PreparedBackupGroupsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
backupOps map[path.CategoryType]string
|
||||
m365 its.M365IntgTestSetup
|
||||
its intgTesterSetup
|
||||
}
|
||||
|
||||
func TestPreparedBackupGroupsE2ESuite(t *testing.T) {
|
||||
@ -296,19 +289,16 @@ func (suite *PreparedBackupGroupsE2ESuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.its = newIntegrationTesterSetup(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
|
||||
suite.backupOps = make(map[path.CategoryType]string)
|
||||
|
||||
var (
|
||||
groups = []string{suite.m365.Group.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.m365.Group.ID: suite.m365.Group.ID})
|
||||
groups = []string{suite.its.group.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.its.group.ID: suite.its.group.ID})
|
||||
cats = []path.CategoryType{
|
||||
path.ChannelMessagesCategory,
|
||||
// TODO(pandeyabs): CorsoCITeam group mailbox backup is currently broken because of invalid
|
||||
// odata.NextLink which causes an infinite loop during paging. Disabling conversations tests while
|
||||
// we go fix the group mailbox.
|
||||
// path.ConversationPostsCategory,
|
||||
path.ConversationPostsCategory,
|
||||
path.LibrariesCategory,
|
||||
}
|
||||
)
|
||||
@ -378,7 +368,7 @@ func runGroupsListCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.Cat
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
@ -419,7 +409,7 @@ func runGroupsListSingleCmdTest(suite *PreparedBackupGroupsE2ESuite, category pa
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -446,7 +436,7 @@ func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_badID() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", "smarfs")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -462,8 +452,6 @@ func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_channelMessages(
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_conversations() {
|
||||
// skip
|
||||
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
|
||||
runGroupsDetailsCmdTest(suite, path.ConversationPostsCategory)
|
||||
}
|
||||
|
||||
@ -476,6 +464,10 @@ func runGroupsDetailsCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.
|
||||
|
||||
t := suite.T()
|
||||
|
||||
if category == path.ConversationPostsCategory {
|
||||
t.Skip("skipping conversation details test, see issue #4780")
|
||||
}
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
@ -490,7 +482,7 @@ func runGroupsDetailsCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "details", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN, string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
@ -580,7 +572,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN,
|
||||
fmt.Sprintf("%s,%s",
|
||||
string(suite.backupOps[0].Results.BackupID),
|
||||
@ -594,7 +586,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd() {
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backups", string(suite.backupOps[0].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -612,7 +604,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_SingleID() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN,
|
||||
string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -624,7 +616,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_SingleID() {
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -642,7 +634,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_UnknownID() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN, uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -661,7 +653,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_NoBackupID()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// empty backupIDs should error since no data provided
|
||||
@ -680,7 +672,7 @@ func buildGroupsBackupCmd(
|
||||
) (*cobra.Command, context.Context) {
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "groups",
|
||||
"--"+flags.ConfigFileFN, configFile,
|
||||
"--config-file", configFile,
|
||||
"--"+flags.GroupFN, group,
|
||||
"--"+flags.CategoryDataFN, category)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -153,7 +153,6 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
|
||||
"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.GroupsCategoryDataInput),
|
||||
"--" + flags.FetchParallelismFN, flagsTD.FetchParallelism,
|
||||
"--" + flags.DisableDeltaFN,
|
||||
"--" + flags.DisableLazyItemReaderFN,
|
||||
},
|
||||
flagsTD.PreparedGenericBackupFlags(),
|
||||
flagsTD.PreparedProviderFlags(),
|
||||
@ -161,25 +160,13 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
|
||||
|
||||
opts := utils.MakeGroupsOpts(cmd)
|
||||
co := utils.Control()
|
||||
backupOpts := utils.ParseBackupOptions()
|
||||
|
||||
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
|
||||
// restore flags are switched over too and we no longer parse flags beyond
|
||||
// connection info into control.Options.
|
||||
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
|
||||
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
|
||||
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
|
||||
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
|
||||
assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
|
||||
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
assert.True(t, co.ToggleFeatures.DisableIncrementals)
|
||||
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
|
||||
assert.True(t, co.ToggleFeatures.DisableDelta)
|
||||
assert.True(t, co.ToggleFeatures.DisableLazyItemReader)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
|
||||
flagsTD.AssertGenericBackupFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
|
||||
@ -11,19 +11,145 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
gmock "github.com/alcionai/corso/src/pkg/services/m365/api/graph/mock"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
"github.com/alcionai/corso/src/pkg/storage/testdata"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Gockable client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// GockClient produces a new exchange api client that can be
|
||||
// mocked using gock.
|
||||
func gockClient(creds account.M365Config, counter *count.Bus) (api.Client, error) {
|
||||
s, err := gmock.NewService(creds, counter)
|
||||
if err != nil {
|
||||
return api.Client{}, err
|
||||
}
|
||||
|
||||
li, err := gmock.NewService(creds, counter, graph.NoTimeout())
|
||||
if err != nil {
|
||||
return api.Client{}, err
|
||||
}
|
||||
|
||||
return api.Client{
|
||||
Credentials: creds,
|
||||
Stable: s,
|
||||
LargeItem: li,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Suite Setup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type ids struct {
|
||||
ID string
|
||||
DriveID string
|
||||
DriveRootFolderID string
|
||||
}
|
||||
|
||||
type intgTesterSetup struct {
|
||||
acct account.Account
|
||||
ac api.Client
|
||||
gockAC api.Client
|
||||
user ids
|
||||
site ids
|
||||
group ids
|
||||
team ids
|
||||
}
|
||||
|
||||
func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
|
||||
its := intgTesterSetup{}
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
||||
|
||||
its.acct = tconfig.NewM365Account(t)
|
||||
creds, err := its.acct.M365Config()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
its.ac, err = api.NewClient(
|
||||
creds,
|
||||
control.DefaultOptions(),
|
||||
count.New())
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
its.gockAC, err = gockClient(creds, count.New())
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// user drive
|
||||
|
||||
uids := ids{}
|
||||
|
||||
uids.ID = tconfig.M365UserID(t)
|
||||
|
||||
userDrive, err := its.ac.Users().GetDefaultDrive(ctx, uids.ID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
uids.DriveID = ptr.Val(userDrive.GetId())
|
||||
|
||||
userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, uids.DriveID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
uids.DriveRootFolderID = ptr.Val(userDriveRootFolder.GetId())
|
||||
|
||||
its.user = uids
|
||||
|
||||
// site
|
||||
|
||||
sids := ids{}
|
||||
|
||||
sids.ID = tconfig.M365SiteID(t)
|
||||
|
||||
siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, sids.ID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
sids.DriveID = ptr.Val(siteDrive.GetId())
|
||||
|
||||
siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, sids.DriveID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
sids.DriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId())
|
||||
|
||||
its.site = sids
|
||||
|
||||
// group
|
||||
|
||||
gids := ids{}
|
||||
|
||||
// use of the TeamID is intentional here, so that we are assured
|
||||
// the group has full usage of the teams api.
|
||||
gids.ID = tconfig.M365TeamID(t)
|
||||
|
||||
its.group = gids
|
||||
|
||||
// team
|
||||
|
||||
tids := ids{}
|
||||
tids.ID = tconfig.M365TeamID(t)
|
||||
its.team = tids
|
||||
|
||||
return its
|
||||
}
|
||||
|
||||
type dependencies struct {
|
||||
st storage.Storage
|
||||
repo repository.Repositoryer
|
||||
@ -88,7 +214,7 @@ func buildExchangeBackupCmd(
|
||||
) (*cobra.Command, context.Context) {
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "exchange",
|
||||
"--"+flags.ConfigFileFN, configFile,
|
||||
"--config-file", configFile,
|
||||
"--"+flags.UserFN, user,
|
||||
"--"+flags.CategoryDataFN, category)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -60,6 +60,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
switch cmd.Use {
|
||||
case createCommand:
|
||||
c, fs = utils.AddCommand(cmd, oneDriveCreateCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandCreateUseSuffix
|
||||
c.Example = oneDriveServiceCommandCreateExamples
|
||||
@ -67,20 +68,22 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddUserFlag(c)
|
||||
flags.AddGenericBackupFlags(c)
|
||||
fs.BoolVar(
|
||||
&flags.UseOldDeltaProcessFV,
|
||||
flags.UseOldDeltaProcessFN,
|
||||
&flags.UseDeltaTreeFV,
|
||||
flags.UseDeltaTreeFN,
|
||||
false,
|
||||
"process backups using the old delta processor instead of tree-based enumeration")
|
||||
cobra.CheckErr(fs.MarkHidden(flags.UseOldDeltaProcessFN))
|
||||
"process backups using the delta tree instead of standard enumeration")
|
||||
cobra.CheckErr(fs.MarkHidden(flags.UseDeltaTreeFN))
|
||||
|
||||
case listCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveListCmd())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveListCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
flags.AddAllBackupListFlags(c)
|
||||
|
||||
case detailsCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveDetailsCmd())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveDetailsCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandDetailsUseSuffix
|
||||
c.Example = oneDriveServiceCommandDetailsExamples
|
||||
@ -90,7 +93,8 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddOneDriveDetailsAndRestoreFlags(c)
|
||||
|
||||
case deleteCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveDeleteCmd())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveDeleteCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandDeleteUseSuffix
|
||||
c.Example = oneDriveServiceCommandDeleteExamples
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
@ -19,7 +20,6 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||
@ -64,7 +64,7 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupListCmd_empty() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
@ -93,8 +93,8 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_userNotInTenant() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--"+flags.UserFN, "foo@not-there.com")
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.UserFN, "foo@nothere.com")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&recorder)
|
||||
@ -107,8 +107,7 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_userNotInTenant() {
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"not found",
|
||||
"error missing user not found")
|
||||
"not found in tenant", "error missing user not found")
|
||||
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
|
||||
|
||||
t.Logf("backup error message: %s", err.Error())
|
||||
@ -176,7 +175,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN,
|
||||
fmt.Sprintf("%s,%s",
|
||||
string(suite.backupOps[0].Results.BackupID),
|
||||
@ -201,7 +200,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backups", string(suite.backupOps[0].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -221,7 +220,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_SingleID(
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN,
|
||||
string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -243,7 +242,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_SingleID(
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[0].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -261,7 +260,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_unknownID
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN, uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -280,7 +279,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_NoBackupI
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "onedrive",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// empty backupIDs should error since no data provided
|
||||
|
||||
@ -108,20 +108,11 @@ func (suite *OneDriveUnitSuite) TestBackupCreateFlags() {
|
||||
|
||||
opts := utils.MakeOneDriveOpts(cmd)
|
||||
co := utils.Control()
|
||||
backupOpts := utils.ParseBackupOptions()
|
||||
|
||||
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
|
||||
// restore flags are switched over too and we no longer parse flags beyond
|
||||
// connection info into control.Options.
|
||||
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
|
||||
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
|
||||
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
assert.True(t, co.ToggleFeatures.DisableIncrementals)
|
||||
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
|
||||
flagsTD.AssertGenericBackupFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
@ -37,11 +38,7 @@ corso backup create sharepoint --site https://example.com/hr
|
||||
corso backup create sharepoint --site https://example.com/hr,https://example.com/team
|
||||
|
||||
# Backup all SharePoint data for all Sites
|
||||
corso backup create sharepoint --site '*'
|
||||
|
||||
# Backup all SharePoint list data for a Site
|
||||
corso backup create sharepoint --site https://example.com/hr --data lists
|
||||
`
|
||||
corso backup create sharepoint --site '*'`
|
||||
|
||||
sharePointServiceCommandDeleteExamples = `# Delete SharePoint backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
and 1234abcd-12ab-cd34-56de-1234abce
|
||||
@ -61,54 +58,39 @@ corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
# Explore all files within the document library "Work Documents"
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--library "Work Documents"
|
||||
|
||||
# Explore lists by their name(s)
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list "list-name-1,list-name-2"
|
||||
|
||||
# Explore lists created after a given time
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-after 2024-01-01T12:23:34
|
||||
|
||||
# Explore lists created before a given time
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-before 2024-01-01T12:23:34
|
||||
|
||||
# Explore lists modified before a given time
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-before 2024-01-01T12:23:34
|
||||
|
||||
# Explore lists modified after a given time
|
||||
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-after 2024-01-01T12:23:34`
|
||||
`
|
||||
)
|
||||
|
||||
// called by backup.go to map subcommands to provider-specific handling.
|
||||
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case createCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointCreateCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointCreateCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandCreateUseSuffix
|
||||
c.Example = sharePointServiceCommandCreateExamples
|
||||
|
||||
flags.AddSiteFlag(c, true)
|
||||
flags.AddSiteIDFlag(c, true)
|
||||
// [TODO](hitesh) to add lists flag to invoke backup for lists
|
||||
// when explicit invoke is not required anymore
|
||||
flags.AddDataFlag(c, []string{flags.DataLibraries}, true)
|
||||
flags.AddGenericBackupFlags(c)
|
||||
|
||||
case listCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointListCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointListCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
flags.AddAllBackupListFlags(c)
|
||||
|
||||
case detailsCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointDetailsCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointDetailsCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandDetailsUseSuffix
|
||||
c.Example = sharePointServiceCommandDetailsExamples
|
||||
@ -118,7 +100,8 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
|
||||
flags.AddSharePointDetailsAndRestoreFlags(c)
|
||||
|
||||
case deleteCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointDeleteCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointDeleteCmd())
|
||||
fs.SortFlags = false
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandDeleteUseSuffix
|
||||
c.Example = sharePointServiceCommandDeleteExamples
|
||||
@ -211,12 +194,10 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
|
||||
flags.SiteFN + " *")
|
||||
}
|
||||
|
||||
allowedCats := utils.SharePointAllowedCategories()
|
||||
|
||||
for _, d := range cats {
|
||||
if _, ok := allowedCats[d]; !ok {
|
||||
if d != flags.DataLibraries && d != flags.DataPages {
|
||||
return clues.New(
|
||||
d + " is an unrecognized data type; only " + flags.DataLibraries + " supported")
|
||||
d + " is an unrecognized data type; either " + flags.DataLibraries + "or " + flags.DataPages)
|
||||
}
|
||||
}
|
||||
|
||||
@ -243,11 +224,29 @@ func sharePointBackupCreateSelectors(
|
||||
|
||||
sel := selectors.NewSharePointBackup(append(slices.Clone(sites), weburls...))
|
||||
|
||||
return utils.AddCategories(sel, cats), nil
|
||||
return addCategories(sel, cats), nil
|
||||
}
|
||||
|
||||
func includeAllSitesWithCategories(ins idname.Cacher, categories []string) *selectors.SharePointBackup {
|
||||
return utils.AddCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
|
||||
return addCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
|
||||
}
|
||||
|
||||
func addCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
|
||||
// Issue #2631: Libraries are the only supported feature for SharePoint at this time.
|
||||
if len(cats) == 0 {
|
||||
sel.Include(sel.LibraryFolders(selectors.Any()))
|
||||
}
|
||||
|
||||
for _, d := range cats {
|
||||
switch d {
|
||||
case flags.DataLibraries:
|
||||
sel.Include(sel.LibraryFolders(selectors.Any()))
|
||||
case flags.DataPages:
|
||||
sel.Include(sel.Pages(selectors.Any()))
|
||||
}
|
||||
}
|
||||
|
||||
return sel
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
@ -1,29 +1,25 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/its"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||
@ -31,7 +27,7 @@ import (
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests that require no existing backups
|
||||
// tests with no prior backup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type NoBackupSharePointE2ESuite struct {
|
||||
@ -66,7 +62,7 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "sharepoint",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
@ -83,297 +79,6 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
|
||||
assert.True(t, strings.HasSuffix(result, "No backups available\n"))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests with no prior backup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type BackupSharepointE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestBackupSharepointE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &BackupSharepointE2ESuite{Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
|
||||
}
|
||||
|
||||
func (suite *BackupSharepointE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
|
||||
}
|
||||
|
||||
func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_lists() {
|
||||
// Issue: https://github.com/alcionai/corso/issues/4754
|
||||
suite.T().Skip("unskip when sharepoint lists support is enabled")
|
||||
runSharepointBackupCategoryTest(suite, flags.DataLists)
|
||||
}
|
||||
|
||||
func runSharepointBackupCategoryTest(suite *BackupSharepointE2ESuite, category string) {
|
||||
recorder := strings.Builder{}
|
||||
recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd, ctx := buildSharepointBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
suite.m365.Site.ID,
|
||||
category,
|
||||
&recorder)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
}
|
||||
|
||||
func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_siteNotFound_lists() {
|
||||
// Issue: https://github.com/alcionai/corso/issues/4754
|
||||
suite.T().Skip("un-skip test when lists support is enabled")
|
||||
runSharepointBackupSiteNotFoundTest(suite, flags.DataLists)
|
||||
}
|
||||
|
||||
func runSharepointBackupSiteNotFoundTest(suite *BackupSharepointE2ESuite, category string) {
|
||||
recorder := strings.Builder{}
|
||||
recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd, ctx := buildSharepointBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
uuid.NewString(),
|
||||
category,
|
||||
&recorder)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"Invalid hostname for this tenancy", "error missing site not found")
|
||||
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
|
||||
|
||||
t.Logf("backup error message: %s", err.Error())
|
||||
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests prepared with a previous backup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type PreparedBackupSharepointE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
backupOps map[path.CategoryType]string
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestPreparedBackupSharepointE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &PreparedBackupSharepointE2ESuite{
|
||||
Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupSharepointE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
|
||||
suite.backupOps = make(map[path.CategoryType]string)
|
||||
|
||||
var (
|
||||
sites = []string{suite.m365.Site.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.m365.Site.ID: suite.m365.Site.ID})
|
||||
cats = []path.CategoryType{
|
||||
path.ListsCategory,
|
||||
}
|
||||
)
|
||||
|
||||
for _, set := range cats {
|
||||
var (
|
||||
sel = selectors.NewSharePointBackup(sites)
|
||||
scopes []selectors.SharePointScope
|
||||
)
|
||||
|
||||
switch set {
|
||||
case path.ListsCategory:
|
||||
scopes = testdata.SharePointBackupListsScope(sel)
|
||||
}
|
||||
|
||||
sel.Include(scopes)
|
||||
|
||||
bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
err = bop.Run(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
bIDs := string(bop.Results.BackupID)
|
||||
|
||||
// sanity check, ensure we can find the backup and its details immediately
|
||||
b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
|
||||
require.NoError(t, err, "retrieving recent backup by ID")
|
||||
require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
|
||||
|
||||
_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
|
||||
require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
|
||||
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
|
||||
require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
|
||||
|
||||
suite.backupOps[set] = string(b.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_lists() {
|
||||
runSharepointListCmdTest(suite, path.ListsCategory)
|
||||
}
|
||||
|
||||
func runSharepointListCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "sharepoint",
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// compare the output
|
||||
result := suite.dpnd.recorder.String()
|
||||
assert.Contains(t, result, suite.backupOps[category])
|
||||
|
||||
t.Log("backup results", result)
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_badID() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "sharepoint",
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--backup", uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointDetailsCmd_lists() {
|
||||
runSharepointDetailsCmdTest(suite, path.ListsCategory)
|
||||
}
|
||||
|
||||
func runSharepointDetailsCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
bID := suite.backupOps[category]
|
||||
|
||||
// fetch the details from the repo first
|
||||
deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
|
||||
require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
|
||||
require.Empty(t, errs.Recovered())
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "details", "sharepoint",
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN, string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// compare the output
|
||||
result := suite.dpnd.recorder.String()
|
||||
|
||||
i := 0
|
||||
findings := make(map[path.CategoryType]int)
|
||||
|
||||
incrementor := func(cond bool, cat path.CategoryType) {
|
||||
if cond {
|
||||
findings[cat]++
|
||||
}
|
||||
}
|
||||
|
||||
for _, ent := range deets.Entries {
|
||||
if ent.SharePoint == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
isSharePointList := ent.SharePoint.ItemType == details.SharePointList
|
||||
hasListName := isSharePointList && len(ent.SharePoint.List.Name) > 0
|
||||
hasItemName := !isSharePointList && len(ent.SharePoint.ItemName) > 0
|
||||
|
||||
incrementor(hasListName, category)
|
||||
incrementor(hasItemName, category)
|
||||
|
||||
suite.Run(fmt.Sprintf("detail %d", i), func() {
|
||||
assert.Contains(suite.T(), result, ent.ShortRef)
|
||||
})
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
assert.GreaterOrEqual(t, findings[category], 1)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests for deleting backups
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -441,7 +146,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "sharepoint",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN,
|
||||
fmt.Sprintf("%s,%s",
|
||||
string(suite.backupOp.Results.BackupID),
|
||||
@ -468,7 +173,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
|
||||
// // a follow-up details call should fail, due to the backup ID being deleted
|
||||
// cmd = cliTD.StubRootCmd(
|
||||
// "backup", "details", "sharepoint",
|
||||
// "--"+flags.ConfigFileFN, suite.cfgFP,
|
||||
// "--config-file", suite.cfgFP,
|
||||
// "--backup", string(suite.backupOp.Results.BackupID))
|
||||
// cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -485,7 +190,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_unkno
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "sharepoint",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--config-file", suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN, uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -504,30 +209,10 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_NoBac
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "groups",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
"--config-file", suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// empty backupIDs should error since no data provided
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func buildSharepointBackupCmd(
|
||||
ctx context.Context,
|
||||
configFile, site, category string,
|
||||
recorder *strings.Builder,
|
||||
) (*cobra.Command, context.Context) {
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "sharepoint",
|
||||
"--config-file", configFile,
|
||||
"--"+flags.SiteIDFN, site,
|
||||
"--"+flags.CategoryDataFN, category)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(recorder)
|
||||
|
||||
return cmd, print.SetRootCmd(ctx, cmd)
|
||||
}
|
||||
|
||||
@ -112,21 +112,12 @@ func (suite *SharePointUnitSuite) TestBackupCreateFlags() {
|
||||
|
||||
opts := utils.MakeSharePointOpts(cmd)
|
||||
co := utils.Control()
|
||||
backupOpts := utils.ParseBackupOptions()
|
||||
|
||||
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
|
||||
// restore flags are switched over too and we no longer parse flags beyond
|
||||
// connection info into control.Options.
|
||||
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
|
||||
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
|
||||
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
|
||||
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
assert.True(t, co.ToggleFeatures.DisableIncrementals)
|
||||
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
|
||||
|
||||
assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID)
|
||||
assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL)
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
assert.True(t, co.ToggleFeatures.DisableIncrementals)
|
||||
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
|
||||
flagsTD.AssertGenericBackupFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
@ -218,7 +209,6 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
|
||||
name string
|
||||
site []string
|
||||
weburl []string
|
||||
cats []string
|
||||
expect assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
@ -226,61 +216,25 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "sites but no category",
|
||||
name: "sites",
|
||||
site: []string{"smarf"},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "web urls but no category",
|
||||
name: "urls",
|
||||
weburl: []string{"fnord"},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "both web urls and sites but no category",
|
||||
name: "both",
|
||||
site: []string{"smarf"},
|
||||
weburl: []string{"fnord"},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "site with libraries category",
|
||||
site: []string{"smarf"},
|
||||
cats: []string{flags.DataLibraries},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "site with invalid category",
|
||||
site: []string{"smarf"},
|
||||
cats: []string{"invalid category"},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "site with lists category",
|
||||
site: []string{"smarf"},
|
||||
cats: []string{flags.DataLists},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
|
||||
// [TODO]: Uncomment when pages are enabled
|
||||
|
||||
// {
|
||||
// name: "site with pages category",
|
||||
// site: []string{"smarf"},
|
||||
// cats: []string{flags.DataPages},
|
||||
// expect: assert.NoError,
|
||||
// },
|
||||
|
||||
// [TODO]: Uncomment when pages & lists are enabled
|
||||
|
||||
// {
|
||||
// name: "site with all categories",
|
||||
// site: []string{"smarf"},
|
||||
// cats: []string{flags.DataLists, flags.DataPages, flags.DataLibraries},
|
||||
// expect: assert.NoError,
|
||||
// },
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
err := validateSharePointBackupCreateFlags(test.site, test.weburl, test.cats)
|
||||
err := validateSharePointBackupCreateFlags(test.site, test.weburl, nil)
|
||||
test.expect(suite.T(), err, clues.ToCore(err))
|
||||
})
|
||||
}
|
||||
@ -366,12 +320,6 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
|
||||
data: []string{flags.DataPages},
|
||||
expect: bothIDs,
|
||||
},
|
||||
{
|
||||
name: "Lists",
|
||||
site: bothIDs,
|
||||
data: []string{flags.DataLists},
|
||||
expect: bothIDs,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
|
||||
@ -1,305 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/filters"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365"
|
||||
)
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// setup and globals
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
teamschatsServiceCommand = "chats"
|
||||
teamschatsServiceCommandCreateUseSuffix = "--user <userEmail> | '" + flags.Wildcard + "'"
|
||||
teamschatsServiceCommandDeleteUseSuffix = "--backups <backupId>"
|
||||
teamschatsServiceCommandDetailsUseSuffix = "--backup <backupId>"
|
||||
)
|
||||
|
||||
const (
|
||||
teamschatsServiceCommandCreateExamples = `# Backup all chats with bob@company.hr
|
||||
corso backup create chats --user bob@company.hr
|
||||
|
||||
# Backup all chats for all users
|
||||
corso backup create chats --user '*'`
|
||||
|
||||
teamschatsServiceCommandDeleteExamples = `# Delete chats backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
and 1234abcd-12ab-cd34-56de-1234abce
|
||||
corso backup delete chats --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
|
||||
|
||||
teamschatsServiceCommandDetailsExamples = `# Explore chats in Bob's latest backup (1234abcd...)
|
||||
corso backup details chats --backup 1234abcd-12ab-cd34-56de-1234abcd`
|
||||
)
|
||||
|
||||
// called by backup.go to map subcommands to provider-specific handling.
|
||||
func addTeamsChatsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
|
||||
switch cmd.Use {
|
||||
case createCommand:
|
||||
c, _ = utils.AddCommand(cmd, teamschatsCreateCmd(), utils.MarkPreReleaseCommand())
|
||||
|
||||
c.Use = c.Use + " " + teamschatsServiceCommandCreateUseSuffix
|
||||
c.Example = teamschatsServiceCommandCreateExamples
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
flags.AddUserFlag(c)
|
||||
flags.AddDataFlag(c, []string{flags.DataChats}, false)
|
||||
flags.AddGenericBackupFlags(c)
|
||||
|
||||
case listCommand:
|
||||
c, _ = utils.AddCommand(cmd, teamschatsListCmd(), utils.MarkPreReleaseCommand())
|
||||
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
flags.AddAllBackupListFlags(c)
|
||||
|
||||
case detailsCommand:
|
||||
c, _ = utils.AddCommand(cmd, teamschatsDetailsCmd(), utils.MarkPreReleaseCommand())
|
||||
|
||||
c.Use = c.Use + " " + teamschatsServiceCommandDetailsUseSuffix
|
||||
c.Example = teamschatsServiceCommandDetailsExamples
|
||||
|
||||
flags.AddSkipReduceFlag(c)
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddTeamsChatsDetailsAndRestoreFlags(c)
|
||||
|
||||
case deleteCommand:
|
||||
c, _ = utils.AddCommand(cmd, teamschatsDeleteCmd(), utils.MarkPreReleaseCommand())
|
||||
|
||||
c.Use = c.Use + " " + teamschatsServiceCommandDeleteUseSuffix
|
||||
c.Example = teamschatsServiceCommandDeleteExamples
|
||||
|
||||
flags.AddMultipleBackupIDsFlag(c, false)
|
||||
flags.AddBackupIDFlag(c, false)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// backup create
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
// `corso backup create chats [<flag>...]`
|
||||
func teamschatsCreateCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: teamschatsServiceCommand,
|
||||
Aliases: []string{teamsServiceCommand},
|
||||
Short: "Backup M365 Chats data",
|
||||
RunE: createTeamsChatsCmd,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
}
|
||||
|
||||
// processes a teamschats backup.
|
||||
func createTeamsChatsCmd(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
if utils.HasNoFlagsAndShownHelp(cmd) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if flags.RunModeFV == flags.RunModeFlagTest {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := validateTeamsChatsBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r, acct, err := utils.AccountConnectAndWriteRepoConfig(
|
||||
ctx,
|
||||
cmd,
|
||||
path.TeamsChatsService)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
defer utils.CloseRepo(ctx, r)
|
||||
|
||||
// TODO: log/print recoverable errors
|
||||
errs := fault.New(false)
|
||||
|
||||
svcCli, err := m365.NewM365Client(ctx, *acct)
|
||||
if err != nil {
|
||||
return Only(ctx, clues.Stack(err))
|
||||
}
|
||||
|
||||
ins, err := svcCli.AC.Users().GetAllIDsAndNames(ctx, errs)
|
||||
if err != nil {
|
||||
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 teamschats"))
|
||||
}
|
||||
|
||||
sel := teamschatsBackupCreateSelectors(ctx, ins, flags.UserFV, flags.CategoryDataFV)
|
||||
selectorSet := []selectors.Selector{}
|
||||
|
||||
for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) {
|
||||
selectorSet = append(selectorSet, discSel.Selector)
|
||||
}
|
||||
|
||||
return genericCreateCommand(
|
||||
ctx,
|
||||
r,
|
||||
"Chats",
|
||||
selectorSet,
|
||||
ins)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// backup list
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
// `corso backup list teamschats [<flag>...]`
|
||||
func teamschatsListCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: teamschatsServiceCommand,
|
||||
Short: "List the history of M365 Chats backups",
|
||||
RunE: listTeamsChatsCmd,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
}
|
||||
|
||||
// lists the history of backup operations
|
||||
func listTeamsChatsCmd(cmd *cobra.Command, args []string) error {
|
||||
return genericListCommand(cmd, flags.BackupIDFV, path.TeamsChatsService, args)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// backup details
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
// `corso backup details teamschats [<flag>...]`
|
||||
func teamschatsDetailsCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: teamschatsServiceCommand,
|
||||
Short: "Shows the details of a M365 Chats backup",
|
||||
RunE: detailsTeamsChatsCmd,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
}
|
||||
|
||||
// processes a teamschats backup.
|
||||
func detailsTeamsChatsCmd(cmd *cobra.Command, args []string) error {
|
||||
if utils.HasNoFlagsAndShownHelp(cmd) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if flags.RunModeFV == flags.RunModeFlagTest {
|
||||
return nil
|
||||
}
|
||||
|
||||
return runDetailsTeamsChatsCmd(cmd)
|
||||
}
|
||||
|
||||
func runDetailsTeamsChatsCmd(cmd *cobra.Command) error {
|
||||
ctx := cmd.Context()
|
||||
opts := utils.MakeTeamsChatsOpts(cmd)
|
||||
|
||||
sel := utils.IncludeTeamsChatsRestoreDataSelectors(ctx, opts)
|
||||
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
|
||||
utils.FilterTeamsChatsRestoreInfoSelectors(sel, opts)
|
||||
|
||||
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
if len(ds.Entries) > 0 {
|
||||
ds.PrintEntries(ctx)
|
||||
} else {
|
||||
Info(ctx, selectors.ErrorNoMatchingItems)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// backup delete
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
|
||||
// `corso backup delete teamschats [<flag>...]`
|
||||
func teamschatsDeleteCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: teamschatsServiceCommand,
|
||||
Short: "Delete backed-up M365 Chats data",
|
||||
RunE: deleteTeamsChatsCmd,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
}
|
||||
|
||||
// deletes an teamschats backup.
|
||||
func deleteTeamsChatsCmd(cmd *cobra.Command, args []string) error {
|
||||
backupIDValue := []string{}
|
||||
|
||||
if len(flags.BackupIDsFV) > 0 {
|
||||
backupIDValue = flags.BackupIDsFV
|
||||
} else if len(flags.BackupIDFV) > 0 {
|
||||
backupIDValue = append(backupIDValue, flags.BackupIDFV)
|
||||
} else {
|
||||
return clues.New("either --backup or --backups flag is required")
|
||||
}
|
||||
|
||||
return genericDeleteCommand(cmd, path.TeamsChatsService, "TeamsChats", backupIDValue, args)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func validateTeamsChatsBackupCreateFlags(teamschats, cats []string) error {
|
||||
if len(teamschats) == 0 {
|
||||
return clues.New(
|
||||
"requires one or more --" +
|
||||
flags.UserFN + " ids, or the wildcard --" +
|
||||
flags.UserFN + " *")
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf(
|
||||
" is an unrecognized data type; only %s is supported",
|
||||
flags.DataChats)
|
||||
|
||||
allowedCats := utils.TeamsChatsAllowedCategories()
|
||||
|
||||
for _, d := range cats {
|
||||
if _, ok := allowedCats[d]; !ok {
|
||||
return clues.New(d + msg)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func teamschatsBackupCreateSelectors(
|
||||
ctx context.Context,
|
||||
ins idname.Cacher,
|
||||
users, cats []string,
|
||||
) *selectors.TeamsChatsBackup {
|
||||
if filters.PathContains(users).Compare(flags.Wildcard) {
|
||||
return includeAllTeamsChatsWithCategories(ins, cats)
|
||||
}
|
||||
|
||||
sel := selectors.NewTeamsChatsBackup(slices.Clone(users))
|
||||
|
||||
return utils.AddTeamsChatsCategories(sel, cats)
|
||||
}
|
||||
|
||||
func includeAllTeamsChatsWithCategories(ins idname.Cacher, categories []string) *selectors.TeamsChatsBackup {
|
||||
return utils.AddTeamsChatsCategories(selectors.NewTeamsChatsBackup(ins.IDs()), categories)
|
||||
}
|
||||
@ -1,636 +0,0 @@
|
||||
package backup_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/its"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
|
||||
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests that require no existing backups
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type NoBackupTeamsChatsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestNoBackupTeamsChatsE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
|
||||
}
|
||||
|
||||
func (suite *NoBackupTeamsChatsE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
t.Skip("not fully implemented")
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
|
||||
}
|
||||
|
||||
func (suite *NoBackupTeamsChatsE2ESuite) TestTeamsChatsBackupListCmd_noBackups() {
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
result := suite.dpnd.recorder.String()
|
||||
|
||||
// as an offhand check: the result should contain the m365 teamschat id
|
||||
assert.True(t, strings.HasSuffix(result, "No backups available\n"))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests with no prior backup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type BackupTeamsChatsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestBackupTeamsChatsE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
|
||||
}
|
||||
|
||||
func (suite *BackupTeamsChatsE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
t.Skip("not fully implemented")
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
|
||||
}
|
||||
|
||||
func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_chats() {
|
||||
runTeamsChatsBackupCategoryTest(suite, flags.DataChats)
|
||||
}
|
||||
|
||||
func runTeamsChatsBackupCategoryTest(suite *BackupTeamsChatsE2ESuite, category string) {
|
||||
recorder := strings.Builder{}
|
||||
recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd, ctx := buildTeamsChatsBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
suite.m365.User.ID,
|
||||
category,
|
||||
&recorder)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
}
|
||||
|
||||
func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_teamschatNotFound_chats() {
|
||||
runTeamsChatsBackupTeamsChatNotFoundTest(suite, flags.DataChats)
|
||||
}
|
||||
|
||||
func runTeamsChatsBackupTeamsChatNotFoundTest(suite *BackupTeamsChatsE2ESuite, category string) {
|
||||
recorder := strings.Builder{}
|
||||
recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd, ctx := buildTeamsChatsBackupCmd(
|
||||
ctx,
|
||||
suite.dpnd.configFilePath,
|
||||
"foo@not-there.com",
|
||||
category,
|
||||
&recorder)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
assert.Contains(
|
||||
t,
|
||||
err.Error(),
|
||||
"not found",
|
||||
"error missing user not found")
|
||||
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
|
||||
|
||||
t.Logf("backup error message: %s", err.Error())
|
||||
|
||||
result := recorder.String()
|
||||
t.Log("backup results", result)
|
||||
}
|
||||
|
||||
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAzureClientIDFlag() {
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext(t)
|
||||
|
||||
defer flush()
|
||||
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "chats",
|
||||
"--teamschat", suite.m365.User.ID,
|
||||
"--azure-client-id", "invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetErr(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_fromConfigFile() {
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "chats",
|
||||
"--teamschat", suite.m365.User.ID,
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
// AWS flags
|
||||
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAWSFlags() {
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext(t)
|
||||
|
||||
defer flush()
|
||||
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "chats",
|
||||
"--teamschat", suite.m365.User.ID,
|
||||
"--aws-access-key", "invalid-value",
|
||||
"--aws-secret-access-key", "some-invalid-value")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
// since invalid aws creds are explicitly set, should see a failure
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests prepared with a previous backup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type PreparedBackupTeamsChatsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
backupOps map[path.CategoryType]string
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestPreparedBackupTeamsChatsE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &PreparedBackupTeamsChatsE2ESuite{
|
||||
Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupTeamsChatsE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
t.Skip("not fully implemented")
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.m365 = its.GetM365(t)
|
||||
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
|
||||
suite.backupOps = make(map[path.CategoryType]string)
|
||||
|
||||
var (
|
||||
teamschats = []string{suite.m365.User.ID}
|
||||
ins = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
|
||||
cats = []path.CategoryType{
|
||||
path.ChatsCategory,
|
||||
}
|
||||
)
|
||||
|
||||
for _, set := range cats {
|
||||
var (
|
||||
sel = selectors.NewTeamsChatsBackup(teamschats)
|
||||
scopes []selectors.TeamsChatsScope
|
||||
)
|
||||
|
||||
switch set {
|
||||
case path.ChatsCategory:
|
||||
scopes = selTD.TeamsChatsBackupChatScope(sel)
|
||||
}
|
||||
|
||||
sel.Include(scopes)
|
||||
|
||||
bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
err = bop.Run(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
bIDs := string(bop.Results.BackupID)
|
||||
|
||||
// sanity check, ensure we can find the backup and its details immediately
|
||||
b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
|
||||
require.NoError(t, err, "retrieving recent backup by ID")
|
||||
require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
|
||||
|
||||
_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
|
||||
require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
|
||||
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
|
||||
require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
|
||||
|
||||
suite.backupOps[set] = string(b.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_chats() {
|
||||
runTeamsChatsListCmdTest(suite, path.ChatsCategory)
|
||||
}
|
||||
|
||||
func runTeamsChatsListCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// compare the output
|
||||
result := suite.dpnd.recorder.String()
|
||||
assert.Contains(t, result, suite.backupOps[category])
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_singleID_chats() {
|
||||
runTeamsChatsListSingleCmdTest(suite, path.ChatsCategory)
|
||||
}
|
||||
|
||||
func runTeamsChatsListSingleCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
bID := suite.backupOps[category]
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--backup", string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// compare the output
|
||||
result := suite.dpnd.recorder.String()
|
||||
assert.Contains(t, result, bID)
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_badID() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "list", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--backup", "smarfs")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsDetailsCmd_chats() {
|
||||
runTeamsChatsDetailsCmdTest(suite, path.ChatsCategory)
|
||||
}
|
||||
|
||||
func runTeamsChatsDetailsCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
|
||||
suite.dpnd.recorder.Reset()
|
||||
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
bID := suite.backupOps[category]
|
||||
|
||||
// fetch the details from the repo first
|
||||
deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
|
||||
require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
|
||||
require.Empty(t, errs.Recovered())
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "details", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN, string(bID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(&suite.dpnd.recorder)
|
||||
|
||||
ctx = print.SetRootCmd(ctx, cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// compare the output
|
||||
result := suite.dpnd.recorder.String()
|
||||
|
||||
i := 0
|
||||
foundFolders := 0
|
||||
|
||||
for _, ent := range deets.Entries {
|
||||
// Skip folders as they don't mean anything to the end teamschat.
|
||||
if ent.Folder != nil {
|
||||
foundFolders++
|
||||
continue
|
||||
}
|
||||
|
||||
suite.Run(fmt.Sprintf("detail %d", i), func() {
|
||||
assert.Contains(suite.T(), result, ent.ShortRef)
|
||||
})
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
// We only backup the default folder for each category so there should be at
|
||||
// least that folder (we don't make details entries for prefix folders).
|
||||
assert.GreaterOrEqual(t, foundFolders, 1)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests for deleting backups
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type BackupDeleteTeamsChatsE2ESuite struct {
|
||||
tester.Suite
|
||||
dpnd dependencies
|
||||
backupOps [3]operations.BackupOperation
|
||||
}
|
||||
|
||||
func TestBackupDeleteTeamsChatsE2ESuite(t *testing.T) {
|
||||
suite.Run(t, &BackupDeleteTeamsChatsE2ESuite{
|
||||
Suite: tester.NewE2ESuite(
|
||||
t,
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *BackupDeleteTeamsChatsE2ESuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
t.Skip("not fully implemented")
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
|
||||
|
||||
m365TeamsChatID := tconfig.M365TeamID(t)
|
||||
teamschats := []string{m365TeamsChatID}
|
||||
|
||||
// some tests require an existing backup
|
||||
sel := selectors.NewTeamsChatsBackup(teamschats)
|
||||
sel.Include(selTD.TeamsChatsBackupChatScope(sel))
|
||||
|
||||
for i := 0; i < cap(suite.backupOps); i++ {
|
||||
backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
suite.backupOps[i] = backupOp
|
||||
|
||||
err = suite.backupOps[i].Run(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN,
|
||||
fmt.Sprintf("%s,%s",
|
||||
string(suite.backupOps[0].Results.BackupID),
|
||||
string(suite.backupOps[1].Results.BackupID)))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--backups", string(suite.backupOps[0].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_SingleID() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupFN,
|
||||
string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// run the command
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// a follow-up details call should fail, due to the backup ID being deleted
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"backup", "details", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--backup", string(suite.backupOps[2].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_UnknownID() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
|
||||
"--"+flags.BackupIDsFN, uuid.NewString())
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// unknown backupIDs should error since the modelStore can't find the backup
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_NoBackupID() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
ctx = config.SetViper(ctx, suite.dpnd.vpr)
|
||||
|
||||
defer flush()
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "delete", "chats",
|
||||
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// empty backupIDs should error since no data provided
|
||||
err := cmd.ExecuteContext(ctx)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func buildTeamsChatsBackupCmd(
|
||||
ctx context.Context,
|
||||
configFile, resource, category string,
|
||||
recorder *strings.Builder,
|
||||
) (*cobra.Command, context.Context) {
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"backup", "create", "chats",
|
||||
"--"+flags.ConfigFileFN, configFile,
|
||||
"--"+flags.UserFN, resource,
|
||||
"--"+flags.CategoryDataFN, category)
|
||||
cli.BuildCommandTree(cmd)
|
||||
cmd.SetOut(recorder)
|
||||
|
||||
return cmd, print.SetRootCmd(ctx, cmd)
|
||||
}
|
||||
@ -1,248 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
)
|
||||
|
||||
type TeamsChatsUnitSuite struct {
|
||||
tester.Suite
|
||||
}
|
||||
|
||||
func TestTeamsChatsUnitSuite(t *testing.T) {
|
||||
suite.Run(t, &TeamsChatsUnitSuite{Suite: tester.NewUnitSuite(t)})
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestAddTeamsChatsCommands() {
|
||||
expectUse := teamschatsServiceCommand
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
use string
|
||||
expectUse string
|
||||
expectShort string
|
||||
expectRunE func(*cobra.Command, []string) error
|
||||
}{
|
||||
{
|
||||
name: "create teamschats",
|
||||
use: createCommand,
|
||||
expectUse: expectUse + " " + teamschatsServiceCommandCreateUseSuffix,
|
||||
expectShort: teamschatsCreateCmd().Short,
|
||||
expectRunE: createTeamsChatsCmd,
|
||||
},
|
||||
{
|
||||
name: "list teamschats",
|
||||
use: listCommand,
|
||||
expectUse: expectUse,
|
||||
expectShort: teamschatsListCmd().Short,
|
||||
expectRunE: listTeamsChatsCmd,
|
||||
},
|
||||
{
|
||||
name: "details teamschats",
|
||||
use: detailsCommand,
|
||||
expectUse: expectUse + " " + teamschatsServiceCommandDetailsUseSuffix,
|
||||
expectShort: teamschatsDetailsCmd().Short,
|
||||
expectRunE: detailsTeamsChatsCmd,
|
||||
},
|
||||
{
|
||||
name: "delete teamschats",
|
||||
use: deleteCommand,
|
||||
expectUse: expectUse + " " + teamschatsServiceCommandDeleteUseSuffix,
|
||||
expectShort: teamschatsDeleteCmd().Short,
|
||||
expectRunE: deleteTeamsChatsCmd,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
cmd := &cobra.Command{Use: test.use}
|
||||
|
||||
c := addTeamsChatsCommands(cmd)
|
||||
require.NotNil(t, c)
|
||||
|
||||
cmds := cmd.Commands()
|
||||
require.Len(t, cmds, 1)
|
||||
|
||||
child := cmds[0]
|
||||
assert.Equal(t, test.expectUse, child.Use)
|
||||
assert.Equal(t, test.expectShort, child.Short)
|
||||
tester.AreSameFunc(t, test.expectRunE, child.RunE)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestValidateTeamsChatsBackupCreateFlags() {
|
||||
table := []struct {
|
||||
name string
|
||||
cats []string
|
||||
expect assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "none",
|
||||
cats: []string{},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "chats",
|
||||
cats: []string{flags.DataChats},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "all allowed",
|
||||
cats: []string{
|
||||
flags.DataChats,
|
||||
},
|
||||
expect: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "bad inputs",
|
||||
cats: []string{"foo"},
|
||||
expect: assert.Error,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
err := validateTeamsChatsBackupCreateFlags([]string{"*"}, test.cats)
|
||||
test.expect(suite.T(), err, clues.ToCore(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestBackupCreateFlags() {
|
||||
t := suite.T()
|
||||
|
||||
cmd := cliTD.SetUpCmdHasFlags(
|
||||
t,
|
||||
&cobra.Command{Use: createCommand},
|
||||
addTeamsChatsCommands,
|
||||
[]cliTD.UseCobraCommandFn{
|
||||
flags.AddAllProviderFlags,
|
||||
flags.AddAllStorageFlags,
|
||||
},
|
||||
flagsTD.WithFlags(
|
||||
teamschatsServiceCommand,
|
||||
[]string{
|
||||
"--" + flags.RunModeFN, flags.RunModeFlagTest,
|
||||
"--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput),
|
||||
"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.TeamsChatsCategoryDataInput),
|
||||
},
|
||||
flagsTD.PreparedGenericBackupFlags(),
|
||||
flagsTD.PreparedProviderFlags(),
|
||||
flagsTD.PreparedStorageFlags()))
|
||||
|
||||
opts := utils.MakeTeamsChatsOpts(cmd)
|
||||
co := utils.Control()
|
||||
backupOpts := utils.ParseBackupOptions()
|
||||
|
||||
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
|
||||
// restore flags are switched over too and we no longer parse flags beyond
|
||||
// connection info into control.Options.
|
||||
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
|
||||
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
|
||||
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
|
||||
|
||||
assert.Equal(t, control.FailFast, co.FailureHandling)
|
||||
assert.True(t, co.ToggleFeatures.DisableIncrementals)
|
||||
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
|
||||
|
||||
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
|
||||
flagsTD.AssertGenericBackupFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestBackupListFlags() {
|
||||
t := suite.T()
|
||||
|
||||
cmd := cliTD.SetUpCmdHasFlags(
|
||||
t,
|
||||
&cobra.Command{Use: listCommand},
|
||||
addTeamsChatsCommands,
|
||||
[]cliTD.UseCobraCommandFn{
|
||||
flags.AddAllProviderFlags,
|
||||
flags.AddAllStorageFlags,
|
||||
},
|
||||
flagsTD.WithFlags(
|
||||
teamschatsServiceCommand,
|
||||
[]string{
|
||||
"--" + flags.RunModeFN, flags.RunModeFlagTest,
|
||||
"--" + flags.BackupFN, flagsTD.BackupInput,
|
||||
},
|
||||
flagsTD.PreparedBackupListFlags(),
|
||||
flagsTD.PreparedProviderFlags(),
|
||||
flagsTD.PreparedStorageFlags()))
|
||||
|
||||
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
|
||||
flagsTD.AssertBackupListFlags(t, cmd)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestBackupDetailsFlags() {
|
||||
t := suite.T()
|
||||
|
||||
cmd := cliTD.SetUpCmdHasFlags(
|
||||
t,
|
||||
&cobra.Command{Use: detailsCommand},
|
||||
addTeamsChatsCommands,
|
||||
[]cliTD.UseCobraCommandFn{
|
||||
flags.AddAllProviderFlags,
|
||||
flags.AddAllStorageFlags,
|
||||
},
|
||||
flagsTD.WithFlags(
|
||||
teamschatsServiceCommand,
|
||||
[]string{
|
||||
"--" + flags.RunModeFN, flags.RunModeFlagTest,
|
||||
"--" + flags.BackupFN, flagsTD.BackupInput,
|
||||
"--" + flags.SkipReduceFN,
|
||||
},
|
||||
flagsTD.PreparedTeamsChatsFlags(),
|
||||
flagsTD.PreparedProviderFlags(),
|
||||
flagsTD.PreparedStorageFlags()))
|
||||
|
||||
co := utils.Control()
|
||||
|
||||
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
|
||||
assert.True(t, co.SkipReduce)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
flagsTD.AssertTeamsChatsFlags(t, cmd)
|
||||
}
|
||||
|
||||
func (suite *TeamsChatsUnitSuite) TestBackupDeleteFlags() {
|
||||
t := suite.T()
|
||||
|
||||
cmd := cliTD.SetUpCmdHasFlags(
|
||||
t,
|
||||
&cobra.Command{Use: deleteCommand},
|
||||
addTeamsChatsCommands,
|
||||
[]cliTD.UseCobraCommandFn{
|
||||
flags.AddAllProviderFlags,
|
||||
flags.AddAllStorageFlags,
|
||||
},
|
||||
flagsTD.WithFlags(
|
||||
teamschatsServiceCommand,
|
||||
[]string{
|
||||
"--" + flags.RunModeFN, flags.RunModeFlagTest,
|
||||
"--" + flags.BackupFN, flagsTD.BackupInput,
|
||||
},
|
||||
flagsTD.PreparedProviderFlags(),
|
||||
flagsTD.PreparedStorageFlags()))
|
||||
|
||||
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
flagsTD.AssertStorageFlags(t, cmd)
|
||||
}
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/backup"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/debug"
|
||||
"github.com/alcionai/corso/src/cli/export"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
@ -19,7 +20,6 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/restore"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
@ -38,7 +38,7 @@ var corsoCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
func preRun(cc *cobra.Command, args []string) error {
|
||||
if err := config.InitCmd(cc, args); err != nil {
|
||||
if err := config.InitFunc(cc, args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -33,9 +33,10 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
defaultConfigFilePath string
|
||||
configDir string
|
||||
displayDefaultFP = filepath.Join("$HOME", ".corso.toml")
|
||||
configFilePath string
|
||||
configFilePathFlag string
|
||||
configDir string
|
||||
displayDefaultFP = filepath.Join("$HOME", ".corso.toml")
|
||||
)
|
||||
|
||||
// RepoDetails holds the repository configuration retrieved from
|
||||
@ -57,7 +58,7 @@ func init() {
|
||||
Infof(context.Background(), "cannot stat CORSO_CONFIG_DIR [%s]: %v", envDir, err)
|
||||
} else {
|
||||
configDir = envDir
|
||||
defaultConfigFilePath = filepath.Join(configDir, ".corso.toml")
|
||||
configFilePath = filepath.Join(configDir, ".corso.toml")
|
||||
}
|
||||
}
|
||||
|
||||
@ -68,7 +69,7 @@ func init() {
|
||||
|
||||
if len(configDir) == 0 {
|
||||
configDir = homeDir
|
||||
defaultConfigFilePath = filepath.Join(configDir, ".corso.toml")
|
||||
configFilePath = filepath.Join(configDir, ".corso.toml")
|
||||
}
|
||||
}
|
||||
|
||||
@ -76,63 +77,40 @@ func init() {
|
||||
func AddConfigFlags(cmd *cobra.Command) {
|
||||
pf := cmd.PersistentFlags()
|
||||
pf.StringVar(
|
||||
&flags.ConfigFileFV,
|
||||
flags.ConfigFileFN, displayDefaultFP, "config file location")
|
||||
&configFilePathFlag,
|
||||
"config-file", displayDefaultFP, "config file location")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Initialization & Storage
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// InitCmd provides a func that lazily initializes viper and
|
||||
// InitFunc provides a func that lazily initializes viper and
|
||||
// verifies that the configuration was able to read a file.
|
||||
func InitCmd(cmd *cobra.Command, args []string) error {
|
||||
_, err := commonInit(cmd.Context(), flags.ConfigFileFV)
|
||||
return clues.Stack(err).OrNil()
|
||||
}
|
||||
func InitFunc(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
// InitConfig allows sdk consumers to initialize viper.
|
||||
func InitConfig(
|
||||
ctx context.Context,
|
||||
userDefinedConfigFile string,
|
||||
) (context.Context, error) {
|
||||
return commonInit(ctx, userDefinedConfigFile)
|
||||
}
|
||||
|
||||
func commonInit(
|
||||
ctx context.Context,
|
||||
userDefinedConfigFile string,
|
||||
) (context.Context, error) {
|
||||
fp := userDefinedConfigFile
|
||||
fp := configFilePathFlag
|
||||
if len(fp) == 0 || fp == displayDefaultFP {
|
||||
fp = defaultConfigFilePath
|
||||
fp = configFilePath
|
||||
}
|
||||
|
||||
vpr := GetViper(ctx)
|
||||
if err := initWithViper(ctx, vpr, fp); err != nil {
|
||||
return ctx, err
|
||||
|
||||
if err := initWithViper(vpr, fp); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return SetViper(ctx, vpr), clues.Stack(Read(ctx)).OrNil()
|
||||
ctx = SetViper(ctx, vpr)
|
||||
|
||||
return Read(ctx)
|
||||
}
|
||||
|
||||
// initWithViper implements InitConfig, but takes in a viper
|
||||
// struct for testing.
|
||||
func initWithViper(
|
||||
ctx context.Context,
|
||||
vpr *viper.Viper,
|
||||
configFP string,
|
||||
) error {
|
||||
logger.Ctx(ctx).Debugw("initializing viper", "config_file_path", configFP)
|
||||
|
||||
defer func() {
|
||||
logger.Ctx(ctx).Debugw("initialized config", "config_file_path", configFP)
|
||||
}()
|
||||
|
||||
func initWithViper(vpr *viper.Viper, configFP string) error {
|
||||
// Configure default config file location
|
||||
if len(configFP) == 0 || configFP == displayDefaultFP {
|
||||
configFP = defaultConfigFilePath
|
||||
|
||||
// Find home directory.
|
||||
_, err := os.Stat(configDir)
|
||||
if err != nil {
|
||||
@ -264,17 +242,19 @@ func writeRepoConfigWithViper(
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadCorsoConfig creates a storage and account instance by mediating all the possible
|
||||
// GetStorageAndAccount creates a storage and account instance by mediating all the possible
|
||||
// data sources (config file, env vars, flag overrides) and the config file.
|
||||
func ReadCorsoConfig(
|
||||
func GetConfigRepoDetails(
|
||||
ctx context.Context,
|
||||
provider storage.ProviderType,
|
||||
readFromFile bool,
|
||||
mustMatchFromConfig bool,
|
||||
overrides map[string]string,
|
||||
) (RepoDetails, error) {
|
||||
) (
|
||||
RepoDetails,
|
||||
error,
|
||||
) {
|
||||
config, err := getStorageAndAccountWithViper(
|
||||
ctx,
|
||||
GetViper(ctx),
|
||||
provider,
|
||||
readFromFile,
|
||||
@ -287,13 +267,15 @@ func ReadCorsoConfig(
|
||||
// getSorageAndAccountWithViper implements GetSorageAndAccount, but takes in a viper
|
||||
// struct for testing.
|
||||
func getStorageAndAccountWithViper(
|
||||
ctx context.Context,
|
||||
vpr *viper.Viper,
|
||||
provider storage.ProviderType,
|
||||
readFromFile bool,
|
||||
mustMatchFromConfig bool,
|
||||
overrides map[string]string,
|
||||
) (RepoDetails, error) {
|
||||
) (
|
||||
RepoDetails,
|
||||
error,
|
||||
) {
|
||||
var (
|
||||
config RepoDetails
|
||||
err error
|
||||
@ -303,9 +285,6 @@ func getStorageAndAccountWithViper(
|
||||
|
||||
// possibly read the prior config from a .corso file
|
||||
if readFromFile {
|
||||
ctx = clues.Add(ctx, "viper_config_file", vpr.ConfigFileUsed())
|
||||
logger.Ctx(ctx).Debug("reading config from file")
|
||||
|
||||
if err := vpr.ReadInConfig(); err != nil {
|
||||
configNotSet := errors.As(err, &viper.ConfigFileNotFoundError{})
|
||||
configNotFound := errors.Is(err, fs.ErrNotExist)
|
||||
@ -314,8 +293,6 @@ func getStorageAndAccountWithViper(
|
||||
return config, clues.Wrap(err, "reading corso config file: "+vpr.ConfigFileUsed())
|
||||
}
|
||||
|
||||
logger.Ctx(ctx).Info("config file not found")
|
||||
|
||||
readConfigFromViper = false
|
||||
}
|
||||
|
||||
@ -16,7 +16,6 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/internal/common/str"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/its"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/control/repository"
|
||||
@ -150,9 +149,6 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
|
||||
testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
|
||||
)
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
const (
|
||||
bkt = "write-read-config-bucket"
|
||||
tid = "3c0748d2-470e-444c-9064-1268e52609d5"
|
||||
@ -161,7 +157,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
|
||||
host = "some-host"
|
||||
)
|
||||
|
||||
err := initWithViper(ctx, vpr, testConfigFilePath)
|
||||
err := initWithViper(vpr, testConfigFilePath)
|
||||
require.NoError(t, err, "initializing repo config", clues.ToCore(err))
|
||||
|
||||
s3Cfg := &storage.S3Config{
|
||||
@ -209,15 +205,12 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
|
||||
testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
|
||||
)
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
const (
|
||||
bkt = "must-match-config-bucket"
|
||||
tid = "dfb12063-7598-458b-85ab-42352c5c25e2"
|
||||
)
|
||||
|
||||
err := initWithViper(ctx, vpr, testConfigFilePath)
|
||||
err := initWithViper(vpr, testConfigFilePath)
|
||||
require.NoError(t, err, "initializing repo config")
|
||||
|
||||
s3Cfg := &storage.S3Config{Bucket: bkt}
|
||||
@ -292,9 +285,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
|
||||
vpr = viper.New()
|
||||
)
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
const (
|
||||
b = "read-repo-config-basic-bucket"
|
||||
tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba"
|
||||
@ -359,7 +349,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
|
||||
flags.PassphraseFV = "passphrase-flags"
|
||||
|
||||
repoDetails, err := getStorageAndAccountWithViper(
|
||||
ctx,
|
||||
vpr,
|
||||
storage.ProviderS3,
|
||||
true,
|
||||
@ -400,7 +389,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
|
||||
|
||||
type ConfigIntegrationSuite struct {
|
||||
tester.Suite
|
||||
m365 its.M365IntgTestSetup
|
||||
}
|
||||
|
||||
func TestConfigIntegrationSuite(t *testing.T) {
|
||||
@ -409,27 +397,21 @@ func TestConfigIntegrationSuite(t *testing.T) {
|
||||
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
|
||||
}
|
||||
|
||||
func (suite *ConfigIntegrationSuite) SetupSuite() {
|
||||
suite.m365 = its.GetM365(suite.T())
|
||||
}
|
||||
|
||||
func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
|
||||
t := suite.T()
|
||||
vpr := viper.New()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
const (
|
||||
bkt = "get-storage-and-account-bucket"
|
||||
end = "https://get-storage-and-account.com"
|
||||
pfx = "get-storage-and-account-prefix/"
|
||||
tid = "3a2faa4e-a882-445c-9d27-f552ef189381"
|
||||
)
|
||||
|
||||
// Configure viper to read test config file
|
||||
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
|
||||
|
||||
err := initWithViper(ctx, vpr, testConfigFilePath)
|
||||
err := initWithViper(vpr, testConfigFilePath)
|
||||
require.NoError(t, err, "initializing repo config", clues.ToCore(err))
|
||||
|
||||
s3Cfg := &storage.S3Config{
|
||||
@ -439,10 +421,9 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
|
||||
DoNotVerifyTLS: true,
|
||||
DoNotUseTLS: true,
|
||||
}
|
||||
m365 := account.M365Config{AzureTenantID: tid}
|
||||
|
||||
creds := suite.m365.Creds
|
||||
|
||||
err = writeRepoConfigWithViper(vpr, s3Cfg, creds, repository.Options{}, "repoid")
|
||||
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
|
||||
require.NoError(t, err, "writing repo config", clues.ToCore(err))
|
||||
|
||||
require.Equal(
|
||||
@ -454,7 +435,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
|
||||
err = vpr.ReadInConfig()
|
||||
require.NoError(t, err, "reading repo config", clues.ToCore(err))
|
||||
|
||||
cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, true, true, nil)
|
||||
cfg, err := getStorageAndAccountWithViper(vpr, storage.ProviderS3, true, true, nil)
|
||||
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
|
||||
|
||||
readS3Cfg, err := cfg.Storage.ToS3Config()
|
||||
@ -483,19 +464,17 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
|
||||
t := suite.T()
|
||||
vpr := viper.New()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
const (
|
||||
bkt = "get-storage-and-account-no-file-bucket"
|
||||
end = "https://get-storage-and-account.com/no-file"
|
||||
pfx = "get-storage-and-account-no-file-prefix/"
|
||||
tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1"
|
||||
)
|
||||
|
||||
creds := suite.m365.Creds
|
||||
m365 := account.M365Config{AzureTenantID: tid}
|
||||
|
||||
overrides := map[string]string{
|
||||
account.AzureTenantID: suite.m365.TenantID,
|
||||
account.AzureTenantID: tid,
|
||||
account.AccountProviderTypeKey: account.ProviderM365.String(),
|
||||
storage.Bucket: bkt,
|
||||
storage.Endpoint: end,
|
||||
@ -505,7 +484,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
|
||||
storage.StorageProviderTypeKey: storage.ProviderS3.String(),
|
||||
}
|
||||
|
||||
cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, false, true, overrides)
|
||||
cfg, err := getStorageAndAccountWithViper(vpr, storage.ProviderS3, false, true, overrides)
|
||||
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
|
||||
|
||||
readS3Cfg, err := cfg.Storage.ToS3Config()
|
||||
@ -524,7 +503,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
|
||||
|
||||
readM365, err := cfg.Account.M365Config()
|
||||
require.NoError(t, err, "reading m365 config from account", clues.ToCore(err))
|
||||
assert.Equal(t, readM365.AzureTenantID, creds.AzureTenantID)
|
||||
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
|
||||
assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
|
||||
assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
|
||||
}
|
||||
@ -2,6 +2,7 @@ package debug
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,13 +11,21 @@ import (
|
||||
|
||||
// called by debug.go to map subcommands to provider-specific handling.
|
||||
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case metadataFilesCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
c, fs = utils.AddCommand(cmd, exchangeMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package debug
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,14 +11,21 @@ import (
|
||||
|
||||
// called by debug.go to map subcommands to provider-specific handling.
|
||||
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case metadataFilesCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package debug
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,13 +11,21 @@ import (
|
||||
|
||||
// called by debug.go to map subcommands to provider-specific handling.
|
||||
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case metadataFilesCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package debug
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,13 +11,21 @@ import (
|
||||
|
||||
// called by debug.go to map subcommands to provider-specific handling.
|
||||
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case metadataFilesCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
c, fs = utils.AddCommand(cmd, sharePointMetadataFilesCmd(), utils.MarkDebugCommand())
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package export
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,14 +11,21 @@ import (
|
||||
|
||||
// called by export.go to map subcommands to provider-specific handling.
|
||||
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case exportCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeExportCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeExportCmd())
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddExchangeDetailsAndRestoreFlags(c, true)
|
||||
flags.AddExportConfigFlags(c)
|
||||
|
||||
@ -11,11 +11,10 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/export"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -97,7 +96,7 @@ func runExport(
|
||||
return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export"))
|
||||
}
|
||||
|
||||
collections, err := eo.Run(ctx)
|
||||
expColl, err := eo.Run(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
return Only(ctx, clues.New("Backup or backup details missing for id "+backupID))
|
||||
@ -106,18 +105,20 @@ func runExport(
|
||||
return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export"))
|
||||
}
|
||||
|
||||
if err = showExportProgress(ctx, eo, collections, exportLocation); err != nil {
|
||||
return err
|
||||
}
|
||||
// It would be better to give a progressbar than a spinner, but we
|
||||
// have any way of knowing how many files are available as of now.
|
||||
diskWriteComplete := observe.MessageWithCompletion(ctx, observe.ProgressCfg{}, "Writing data to disk")
|
||||
|
||||
if len(eo.Errors.Recovered()) > 0 {
|
||||
Infof(ctx, "\nExport failures")
|
||||
err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors)
|
||||
|
||||
for _, i := range eo.Errors.Recovered() {
|
||||
Err(ctx, i.Error())
|
||||
}
|
||||
// The progressbar has to be closed before we move on as the Infof
|
||||
// below flushes progressbar to prevent clobbering the output and
|
||||
// that causes the entire export operation to stall indefinitely.
|
||||
// https://github.com/alcionai/corso/blob/8102523dc62c001b301cd2ab4e799f86146ab1a0/src/cli/print/print.go#L151
|
||||
close(diskWriteComplete)
|
||||
|
||||
return Only(ctx, clues.New("Incomplete export of "+serviceName+" data"))
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
stats := eo.GetStats()
|
||||
@ -131,23 +132,3 @@ func runExport(
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// slim wrapper that allows us to defer the progress bar closure with the expected scope.
|
||||
func showExportProgress(
|
||||
ctx context.Context,
|
||||
op operations.ExportOperation,
|
||||
collections []export.Collectioner,
|
||||
exportLocation string,
|
||||
) error {
|
||||
// It would be better to give a progressbar than a spinner, but we
|
||||
// have any way of knowing how many files are available as of now.
|
||||
progressMessage := observe.MessageWithCompletion(ctx, observe.DefaultCfg(), "Writing data to disk")
|
||||
defer close(progressMessage)
|
||||
|
||||
err := export.ConsumeExportCollections(ctx, exportLocation, collections, op.Errors)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package export
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -11,14 +12,21 @@ import (
|
||||
|
||||
// called by export.go to map subcommands to provider-specific handling.
|
||||
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case exportCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreviewCommand())
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddSiteFlag(c, false)
|
||||
flags.AddSiteIDFlag(c, false)
|
||||
@ -50,13 +58,7 @@ corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd
|
||||
|
||||
# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to /my-exports
|
||||
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00
|
||||
|
||||
# Export all posts from a conversation with topic "hello world" from group mailbox's last backup to /my-exports
|
||||
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"
|
||||
|
||||
# Export post with ID 98765abcdef from a conversation from group mailbox's last backup to /my-exports
|
||||
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world" --post 98765abcdef`
|
||||
--folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00`
|
||||
)
|
||||
|
||||
// `corso export groups [<flag>...] <destination>`
|
||||
|
||||
@ -3,6 +3,7 @@ package export
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,14 +11,21 @@ import (
|
||||
|
||||
// called by export.go to map subcommands to provider-specific handling.
|
||||
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case exportCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveExportCmd())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveExportCmd())
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddOneDriveDetailsAndRestoreFlags(c)
|
||||
flags.AddExportConfigFlags(c)
|
||||
|
||||
@ -3,6 +3,7 @@ package export
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -10,14 +11,21 @@ import (
|
||||
|
||||
// called by export.go to map subcommands to provider-specific handling.
|
||||
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case exportCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointExportCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointExportCmd())
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddSharePointDetailsAndRestoreFlags(c)
|
||||
flags.AddExportConfigFlags(c)
|
||||
@ -45,27 +53,7 @@ corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
|
||||
# Export all files in the "Documents" library to the current directory.
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--library Documents --folder "Display Templates/Style Sheets" .
|
||||
|
||||
# Export lists by their name(s)
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list "list-name-1,list-name-2" .
|
||||
|
||||
# Export lists created after a given time
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-after 2024-01-01T12:23:34 .
|
||||
|
||||
# Export lists created before a given time
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-before 2024-01-01T12:23:34 .
|
||||
|
||||
# Export lists modified before a given time
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-before 2024-01-01T12:23:34 .
|
||||
|
||||
# Export lists modified after a given time
|
||||
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-after 2024-01-01T12:23:34 .`
|
||||
--library Documents --folder "Display Templates/Style Sheets" .`
|
||||
)
|
||||
|
||||
// `corso export sharepoint [<flag>...] <destination>`
|
||||
|
||||
@ -60,11 +60,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
|
||||
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
|
||||
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
|
||||
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
|
||||
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
|
||||
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
|
||||
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
|
||||
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
|
||||
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
|
||||
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
|
||||
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
|
||||
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
|
||||
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
|
||||
"--" + flags.FormatFN, flagsTD.FormatType,
|
||||
@ -91,11 +88,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
|
||||
assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
|
||||
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
|
||||
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
|
||||
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
|
||||
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
|
||||
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
|
||||
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
|
||||
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
|
||||
assert.ElementsMatch(t, flagsTD.ListItemInput, opts.ListItem)
|
||||
assert.ElementsMatch(t, flagsTD.ListFolderInput, opts.ListFolder)
|
||||
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
|
||||
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
|
||||
assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)
|
||||
|
||||
@ -12,11 +12,9 @@ func AddAllBackupListFlags(cmd *cobra.Command) {
|
||||
}
|
||||
|
||||
func AddFailedItemsFN(cmd *cobra.Command) {
|
||||
fs := cmd.Flags()
|
||||
fs.StringVar(
|
||||
&FailedItemsFV, FailedItemsFN, Show,
|
||||
cmd.Flags().StringVar(
|
||||
&ListFailedItemsFV, FailedItemsFN, Show,
|
||||
"Toggles showing or hiding the list of items that failed.")
|
||||
cobra.CheckErr(fs.MarkHidden(FailedItemsFN))
|
||||
}
|
||||
|
||||
func AddSkippedItemsFN(cmd *cobra.Command) {
|
||||
|
||||
@ -28,6 +28,13 @@ func AddFilesystemFlags(cmd *cobra.Command) {
|
||||
"",
|
||||
"path to local or network storage")
|
||||
cobra.CheckErr(cmd.MarkFlagRequired(FilesystemPathFN))
|
||||
|
||||
fs.BoolVar(
|
||||
&SucceedIfExistsFV,
|
||||
SucceedIfExistsFN,
|
||||
false,
|
||||
"Exit with success if the repo has already been initialized.")
|
||||
cobra.CheckErr(fs.MarkHidden("succeed-if-exists"))
|
||||
}
|
||||
|
||||
func FilesystemFlagOverrides(cmd *cobra.Command) map[string]string {
|
||||
|
||||
@ -13,7 +13,7 @@ const (
|
||||
FileModifiedAfterFN = "file-modified-after"
|
||||
FileModifiedBeforeFN = "file-modified-before"
|
||||
|
||||
UseOldDeltaProcessFN = "use-old-delta-process"
|
||||
UseDeltaTreeFN = "use-delta-tree"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -25,7 +25,7 @@ var (
|
||||
FileModifiedAfterFV string
|
||||
FileModifiedBeforeFV string
|
||||
|
||||
UseOldDeltaProcessFV bool
|
||||
UseDeltaTreeFV bool
|
||||
)
|
||||
|
||||
// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the
|
||||
|
||||
@ -6,46 +6,42 @@ import (
|
||||
|
||||
const (
|
||||
AlertsFN = "alerts"
|
||||
ConfigFileFN = "config-file"
|
||||
DeltaPageSizeFN = "delta-page-size"
|
||||
DisableDeltaFN = "disable-delta"
|
||||
DisableIncrementalsFN = "disable-incrementals"
|
||||
DisableLazyItemReaderFN = "disable-lazy-item-reader"
|
||||
DisableSlidingWindowLimiterFN = "disable-sliding-window-limiter"
|
||||
ForceItemDataDownloadFN = "force-item-data-download"
|
||||
EnableImmutableIDFN = "enable-immutable-id"
|
||||
FailFastFN = "fail-fast"
|
||||
FailedItemsFN = "failed-items"
|
||||
FetchParallelismFN = "fetch-parallelism"
|
||||
NoPermissionsFN = "no-permissions"
|
||||
NoStatsFN = "no-stats"
|
||||
RecoveredErrorsFN = "recovered-errors"
|
||||
NoPermissionsFN = "no-permissions"
|
||||
RunModeFN = "run-mode"
|
||||
SkippedItemsFN = "skipped-items"
|
||||
SkipReduceFN = "skip-reduce"
|
||||
)
|
||||
|
||||
var (
|
||||
ConfigFileFV string
|
||||
DeltaPageSizeFV int
|
||||
DisableDeltaFV bool
|
||||
DisableIncrementalsFV bool
|
||||
DisableLazyItemReaderFV bool
|
||||
DisableSlidingWindowLimiterFV bool
|
||||
ForceItemDataDownloadFV bool
|
||||
EnableImmutableIDFV bool
|
||||
FailFastFV bool
|
||||
FailedItemsFV string
|
||||
FetchParallelismFV int
|
||||
ListAlertsFV string
|
||||
ListFailedItemsFV string
|
||||
ListSkippedItemsFV string
|
||||
ListRecoveredErrorsFV string
|
||||
NoPermissionsFV bool
|
||||
NoStatsFV bool
|
||||
// RunMode describes the type of run, such as:
|
||||
// flagtest, dry, run. Should default to 'run'.
|
||||
RunModeFV string
|
||||
SkipReduceFV bool
|
||||
RunModeFV string
|
||||
NoPermissionsFV bool
|
||||
SkipReduceFV bool
|
||||
)
|
||||
|
||||
// well-known flag values
|
||||
@ -180,19 +176,3 @@ func AddDisableSlidingWindowLimiterFlag(cmd *cobra.Command) {
|
||||
"Disable sliding window rate limiter.")
|
||||
cobra.CheckErr(fs.MarkHidden(DisableSlidingWindowLimiterFN))
|
||||
}
|
||||
|
||||
// AddDisableLazyItemReader disables lazy item reader, such that we fall back to
|
||||
// prefetch reader. This flag is currently only meant for groups conversations
|
||||
// backup. Although it can be utilized for other services in future.
|
||||
//
|
||||
// This flag should only be used if lazy item reader is the default choice and
|
||||
// we want to fallback to prefetch reader.
|
||||
func AddDisableLazyItemReader(cmd *cobra.Command) {
|
||||
fs := cmd.Flags()
|
||||
fs.BoolVar(
|
||||
&DisableLazyItemReaderFV,
|
||||
DisableLazyItemReaderFN,
|
||||
false,
|
||||
"Disable lazy item reader.")
|
||||
cobra.CheckErr(fs.MarkHidden(DisableLazyItemReaderFN))
|
||||
}
|
||||
|
||||
@ -12,8 +12,9 @@ const (
|
||||
AWSSessionTokenFN = "aws-session-token"
|
||||
|
||||
// Corso Flags
|
||||
PassphraseFN = "passphrase"
|
||||
NewPassphraseFN = "new-passphrase"
|
||||
PassphraseFN = "passphrase"
|
||||
NewPassphraseFN = "new-passphrase"
|
||||
SucceedIfExistsFN = "succeed-if-exists"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -24,6 +25,7 @@ var (
|
||||
AWSSessionTokenFV string
|
||||
PassphraseFV string
|
||||
NewPhasephraseFV string
|
||||
SucceedIfExistsFV bool
|
||||
)
|
||||
|
||||
// AddMultipleBackupIDsFlag adds the --backups flag.
|
||||
|
||||
@ -38,6 +38,11 @@ func AddS3BucketFlags(cmd *cobra.Command) {
|
||||
fs.StringVar(&EndpointFV, EndpointFN, "", "S3 service endpoint.")
|
||||
fs.BoolVar(&DoNotUseTLSFV, DoNotUseTLSFN, false, "Disable TLS (HTTPS)")
|
||||
fs.BoolVar(&DoNotVerifyTLSFV, DoNotVerifyTLSFN, false, "Disable TLS (HTTPS) certificate verification.")
|
||||
|
||||
// In general, we don't want to expose this flag to users and have them mistake it
|
||||
// for a broad-scale idempotency solution. We can un-hide it later the need arises.
|
||||
fs.BoolVar(&SucceedIfExistsFV, SucceedIfExistsFN, false, "Exit with success if the repo has already been initialized.")
|
||||
cobra.CheckErr(fs.MarkHidden("succeed-if-exists"))
|
||||
}
|
||||
|
||||
func S3FlagOverrides(cmd *cobra.Command) map[string]string {
|
||||
|
||||
@ -7,39 +7,26 @@ import (
|
||||
const (
|
||||
DataLibraries = "libraries"
|
||||
DataPages = "pages"
|
||||
DataLists = "lists"
|
||||
)
|
||||
|
||||
const (
|
||||
LibraryFN = "library"
|
||||
|
||||
ListFN = "list"
|
||||
ListModifiedAfterFN = "list-modified-after"
|
||||
ListModifiedBeforeFN = "list-modified-before"
|
||||
ListCreatedAfterFN = "list-created-after"
|
||||
ListCreatedBeforeFN = "list-created-before"
|
||||
|
||||
LibraryFN = "library"
|
||||
ListFolderFN = "list"
|
||||
ListItemFN = "list-item"
|
||||
PageFolderFN = "page-folder"
|
||||
PageFN = "page"
|
||||
|
||||
SiteFN = "site" // site only accepts WebURL values
|
||||
SiteIDFN = "site-id" // site-id accepts actual site ids
|
||||
SiteFN = "site" // site only accepts WebURL values
|
||||
SiteIDFN = "site-id" // site-id accepts actual site ids
|
||||
)
|
||||
|
||||
var (
|
||||
LibraryFV string
|
||||
|
||||
ListFV []string
|
||||
ListModifiedAfterFV string
|
||||
ListModifiedBeforeFV string
|
||||
ListCreatedAfterFV string
|
||||
ListCreatedBeforeFV string
|
||||
|
||||
LibraryFV string
|
||||
ListFolderFV []string
|
||||
ListItemFV []string
|
||||
PageFolderFV []string
|
||||
PageFV []string
|
||||
|
||||
SiteIDFV []string
|
||||
WebURLFV []string
|
||||
SiteIDFV []string
|
||||
WebURLFV []string
|
||||
)
|
||||
|
||||
// AddSharePointDetailsAndRestoreFlags adds flags that are common to both the
|
||||
@ -79,26 +66,17 @@ func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) {
|
||||
"Select files modified before this datetime.")
|
||||
|
||||
// lists
|
||||
|
||||
fs.StringSliceVar(
|
||||
&ListFV,
|
||||
ListFN, nil,
|
||||
"Select lists by name.")
|
||||
fs.StringVar(
|
||||
&ListModifiedAfterFV,
|
||||
ListModifiedAfterFN, "",
|
||||
"Select lists modified after this datetime.")
|
||||
fs.StringVar(
|
||||
&ListModifiedBeforeFV,
|
||||
ListModifiedBeforeFN, "",
|
||||
"Select lists modified before this datetime.")
|
||||
fs.StringVar(
|
||||
&ListCreatedAfterFV,
|
||||
ListCreatedAfterFN, "",
|
||||
"Select lists created after this datetime.")
|
||||
fs.StringVar(
|
||||
&ListCreatedBeforeFV,
|
||||
ListCreatedBeforeFN, "",
|
||||
"Select lists created before this datetime.")
|
||||
&ListFolderFV,
|
||||
ListFolderFN, nil,
|
||||
"Select lists by name; accepts '"+Wildcard+"' to select all lists.")
|
||||
cobra.CheckErr(fs.MarkHidden(ListFolderFN))
|
||||
fs.StringSliceVar(
|
||||
&ListItemFV,
|
||||
ListItemFN, nil,
|
||||
"Select lists by item name; accepts '"+Wildcard+"' to select all lists.")
|
||||
cobra.CheckErr(fs.MarkHidden(ListItemFN))
|
||||
|
||||
// pages
|
||||
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
package flags
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const (
|
||||
DataChats = "chats"
|
||||
)
|
||||
|
||||
func AddTeamsChatsDetailsAndRestoreFlags(cmd *cobra.Command) {
|
||||
// TODO: add details flags
|
||||
}
|
||||
2
src/cli/flags/testdata/backup_list.go
vendored
2
src/cli/flags/testdata/backup_list.go
vendored
@ -20,7 +20,7 @@ func PreparedBackupListFlags() []string {
|
||||
|
||||
func AssertBackupListFlags(t *testing.T, cmd *cobra.Command) {
|
||||
assert.Equal(t, flags.Show, flags.ListAlertsFV)
|
||||
assert.Equal(t, flags.Show, flags.FailedItemsFV)
|
||||
assert.Equal(t, flags.Show, flags.ListFailedItemsFV)
|
||||
assert.Equal(t, flags.Show, flags.ListSkippedItemsFV)
|
||||
assert.Equal(t, flags.Show, flags.ListRecoveredErrorsFV)
|
||||
}
|
||||
|
||||
8
src/cli/flags/testdata/flags.go
vendored
8
src/cli/flags/testdata/flags.go
vendored
@ -21,7 +21,6 @@ var (
|
||||
ExchangeCategoryDataInput = []string{"email", "events", "contacts"}
|
||||
SharepointCategoryDataInput = []string{"files", "lists", "pages"}
|
||||
GroupsCategoryDataInput = []string{"files", "lists", "pages", "messages"}
|
||||
TeamsChatsCategoryDataInput = []string{"chats"}
|
||||
|
||||
ChannelInput = []string{"channel1", "channel2"}
|
||||
MessageInput = []string{"message1", "message2"}
|
||||
@ -60,11 +59,8 @@ var (
|
||||
FileModifiedAfterInput = "fileModifiedAfter"
|
||||
FileModifiedBeforeInput = "fileModifiedBefore"
|
||||
|
||||
ListsInput = []string{"listName1", "listName2"}
|
||||
ListCreatedAfterInput = "listCreatedAfter"
|
||||
ListCreatedBeforeInput = "listCreatedBefore"
|
||||
ListModifiedAfterInput = "listModifiedAfter"
|
||||
ListModifiedBeforeInput = "listModifiedBefore"
|
||||
ListFolderInput = []string{"listFolder1", "listFolder2"}
|
||||
ListItemInput = []string{"listItem1", "listItem2"}
|
||||
|
||||
PageFolderInput = []string{"pageFolder1", "pageFolder2"}
|
||||
PageInput = []string{"page1", "page2"}
|
||||
|
||||
25
src/cli/flags/testdata/teamschats.go
vendored
25
src/cli/flags/testdata/teamschats.go
vendored
@ -1,25 +0,0 @@
|
||||
package testdata
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func PreparedTeamsChatsFlags() []string {
|
||||
return []string{
|
||||
// FIXME: populate when adding filters
|
||||
// "--" + flags.ChatCreatedAfterFN, ChatCreatedAfterInput,
|
||||
// "--" + flags.ChatCreatedBeforeFN, ChatCreatedBeforeInput,
|
||||
// "--" + flags.ChatLastMessageAfterFN, ChatLastMessageAfterInput,
|
||||
// "--" + flags.ChatLastMessageBeforeFN, ChatLastMessageBeforeInput,
|
||||
}
|
||||
}
|
||||
|
||||
func AssertTeamsChatsFlags(t *testing.T, cmd *cobra.Command) {
|
||||
// FIXME: populate when adding filters
|
||||
// assert.Equal(t, ChatCreatedAfterInput, flags.ChatCreatedAfterFV)
|
||||
// assert.Equal(t, ChatCreatedBeforeInput, flags.ChatCreatedBeforeFV)
|
||||
// assert.Equal(t, ChatLastMessageAfterInput, flags.ChatLastMessageAfterFV)
|
||||
// assert.Equal(t, ChatLastMessageBeforeInput, flags.ChatLastMessageBeforeFV)
|
||||
}
|
||||
@ -133,7 +133,7 @@ func Pretty(ctx context.Context, a any) {
|
||||
return
|
||||
}
|
||||
|
||||
printPrettyJSON(ctx, getRootCmd(ctx).ErrOrStderr(), a)
|
||||
printPrettyJSON(getRootCmd(ctx).ErrOrStderr(), a)
|
||||
}
|
||||
|
||||
// PrettyJSON prettifies and prints the value.
|
||||
@ -143,7 +143,7 @@ func PrettyJSON(ctx context.Context, p minimumPrintabler) {
|
||||
return
|
||||
}
|
||||
|
||||
outputJSON(ctx, getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
|
||||
outputJSON(getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
|
||||
}
|
||||
|
||||
// out is the testable core of exported print funcs
|
||||
@ -193,56 +193,56 @@ type minimumPrintabler interface {
|
||||
|
||||
// Item prints the printable, according to the caller's requested format.
|
||||
func Item(ctx context.Context, p Printable) {
|
||||
printItem(ctx, getRootCmd(ctx).OutOrStdout(), p)
|
||||
printItem(getRootCmd(ctx).OutOrStdout(), p)
|
||||
}
|
||||
|
||||
// print prints the printable items,
|
||||
// according to the caller's requested format.
|
||||
func printItem(ctx context.Context, w io.Writer, p Printable) {
|
||||
func printItem(w io.Writer, p Printable) {
|
||||
if outputAsJSON || outputAsJSONDebug {
|
||||
outputJSON(ctx, w, p, outputAsJSONDebug)
|
||||
outputJSON(w, p, outputAsJSONDebug)
|
||||
return
|
||||
}
|
||||
|
||||
outputTable(ctx, w, []Printable{p})
|
||||
outputTable(w, []Printable{p})
|
||||
}
|
||||
|
||||
// ItemProperties prints the printable either as in a single line or a json
|
||||
// The difference between this and Item is that this one does not print the ID
|
||||
func ItemProperties(ctx context.Context, p Printable) {
|
||||
printItemProperties(ctx, getRootCmd(ctx).OutOrStdout(), p)
|
||||
printItemProperties(getRootCmd(ctx).OutOrStdout(), p)
|
||||
}
|
||||
|
||||
// print prints the printable items,
|
||||
// according to the caller's requested format.
|
||||
func printItemProperties(ctx context.Context, w io.Writer, p Printable) {
|
||||
func printItemProperties(w io.Writer, p Printable) {
|
||||
if outputAsJSON || outputAsJSONDebug {
|
||||
outputJSON(ctx, w, p, outputAsJSONDebug)
|
||||
outputJSON(w, p, outputAsJSONDebug)
|
||||
return
|
||||
}
|
||||
|
||||
outputOneLine(ctx, w, []Printable{p})
|
||||
outputOneLine(w, []Printable{p})
|
||||
}
|
||||
|
||||
// All prints the slice of printable items,
|
||||
// according to the caller's requested format.
|
||||
func All(ctx context.Context, ps ...Printable) {
|
||||
printAll(ctx, getRootCmd(ctx).OutOrStdout(), ps)
|
||||
printAll(getRootCmd(ctx).OutOrStdout(), ps)
|
||||
}
|
||||
|
||||
// printAll prints the slice of printable items,
|
||||
// according to the caller's requested format.
|
||||
func printAll(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
func printAll(w io.Writer, ps []Printable) {
|
||||
if len(ps) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if outputAsJSON || outputAsJSONDebug {
|
||||
outputJSONArr(ctx, w, ps, outputAsJSONDebug)
|
||||
outputJSONArr(w, ps, outputAsJSONDebug)
|
||||
return
|
||||
}
|
||||
|
||||
outputTable(ctx, w, ps)
|
||||
outputTable(w, ps)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------------------
|
||||
@ -252,11 +252,11 @@ func printAll(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
// Table writes the printables in a tabular format. Takes headers from
|
||||
// the 0th printable only.
|
||||
func Table(ctx context.Context, ps []Printable) {
|
||||
outputTable(ctx, getRootCmd(ctx).OutOrStdout(), ps)
|
||||
outputTable(getRootCmd(ctx).OutOrStdout(), ps)
|
||||
}
|
||||
|
||||
// output to stdout the list of printable structs in a table
|
||||
func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
func outputTable(w io.Writer, ps []Printable) {
|
||||
t := table.Table{
|
||||
Headers: ps[0].Headers(false),
|
||||
Rows: [][]string{},
|
||||
@ -266,9 +266,6 @@ func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
t.Rows = append(t.Rows, p.Values(false))
|
||||
}
|
||||
|
||||
// observe bars needs to be flushed before printing
|
||||
observe.Flush(ctx)
|
||||
|
||||
_ = t.WriteTable(
|
||||
w,
|
||||
&table.Config{
|
||||
@ -282,20 +279,20 @@ func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
// JSON
|
||||
// ------------------------------------------------------------------------------------------
|
||||
|
||||
func outputJSON(ctx context.Context, w io.Writer, p minimumPrintabler, debug bool) {
|
||||
func outputJSON(w io.Writer, p minimumPrintabler, debug bool) {
|
||||
if debug {
|
||||
printJSON(ctx, w, p)
|
||||
printJSON(w, p)
|
||||
return
|
||||
}
|
||||
|
||||
if debug {
|
||||
printJSON(ctx, w, p)
|
||||
printJSON(w, p)
|
||||
} else {
|
||||
printJSON(ctx, w, p.MinimumPrintable())
|
||||
printJSON(w, p.MinimumPrintable())
|
||||
}
|
||||
}
|
||||
|
||||
func outputJSONArr(ctx context.Context, w io.Writer, ps []Printable, debug bool) {
|
||||
func outputJSONArr(w io.Writer, ps []Printable, debug bool) {
|
||||
sl := make([]any, 0, len(ps))
|
||||
|
||||
for _, p := range ps {
|
||||
@ -306,14 +303,11 @@ func outputJSONArr(ctx context.Context, w io.Writer, ps []Printable, debug bool)
|
||||
}
|
||||
}
|
||||
|
||||
printJSON(ctx, w, sl)
|
||||
printJSON(w, sl)
|
||||
}
|
||||
|
||||
// output to stdout the list of printable structs as json.
|
||||
func printJSON(ctx context.Context, w io.Writer, a any) {
|
||||
// observe bars needs to be flushed before printing
|
||||
observe.Flush(ctx)
|
||||
|
||||
func printJSON(w io.Writer, a any) {
|
||||
bs, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "error formatting results to json: %v\n", err)
|
||||
@ -324,10 +318,7 @@ func printJSON(ctx context.Context, w io.Writer, a any) {
|
||||
}
|
||||
|
||||
// output to stdout the list of printable structs as prettified json.
|
||||
func printPrettyJSON(ctx context.Context, w io.Writer, a any) {
|
||||
// observe bars needs to be flushed before printing
|
||||
observe.Flush(ctx)
|
||||
|
||||
func printPrettyJSON(w io.Writer, a any) {
|
||||
bs, err := json.MarshalIndent(a, "", " ")
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "error formatting results to json: %v\n", err)
|
||||
@ -343,10 +334,7 @@ func printPrettyJSON(ctx context.Context, w io.Writer, a any) {
|
||||
|
||||
// Output in the following format:
|
||||
// Bytes Uploaded: 401 kB | Items Uploaded: 59 | Items Skipped: 0 | Errors: 0
|
||||
func outputOneLine(ctx context.Context, w io.Writer, ps []Printable) {
|
||||
// observe bars needs to be flushed before printing
|
||||
observe.Flush(ctx)
|
||||
|
||||
func outputOneLine(w io.Writer, ps []Printable) {
|
||||
headers := ps[0].Headers(true)
|
||||
rows := [][]string{}
|
||||
|
||||
|
||||
@ -2,13 +2,14 @@ package repo
|
||||
|
||||
import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/events"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
@ -72,7 +73,7 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
|
||||
|
||||
overrides[flags.FilesystemPathFN] = abs
|
||||
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderFilesystem,
|
||||
true,
|
||||
@ -109,6 +110,10 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
|
||||
ric := repository.InitConfig{RetentionOpts: retentionOpts}
|
||||
|
||||
if err = r.Initialize(ctx, ric); err != nil {
|
||||
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return Only(ctx, clues.Stack(ErrInitializingRepo, err))
|
||||
}
|
||||
|
||||
@ -158,7 +163,7 @@ func connectFilesystemCmd(cmd *cobra.Command, args []string) error {
|
||||
|
||||
overrides[flags.FilesystemPathFN] = abs
|
||||
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderFilesystem,
|
||||
true,
|
||||
|
||||
@ -5,16 +5,16 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
@ -73,7 +73,7 @@ func (suite *FilesystemE2ESuite) TestInitFilesystemCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "init", "filesystem",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--path", cfg.Path)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -81,9 +81,9 @@ func (suite *FilesystemE2ESuite) TestInitFilesystemCmd() {
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// noop
|
||||
// a second initialization should result in an error
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists, clues.ToCore(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -143,7 +143,7 @@ func (suite *FilesystemE2ESuite) TestConnectFilesystemCmd() {
|
||||
// then test it
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "connect", "filesystem",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--path", cfg.Path)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
|
||||
@ -10,12 +10,11 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/repo"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
|
||||
)
|
||||
@ -77,7 +76,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "init", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--prefix", cfg.Prefix)
|
||||
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -89,7 +88,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
|
||||
// connect with old passphrase
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"repo", "connect", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", cfg.Bucket,
|
||||
"--prefix", cfg.Prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -100,7 +99,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
|
||||
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"repo", "update-passphrase",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--new-passphrase", "newpass")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -111,7 +110,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
|
||||
// connect again with new passphrase
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"repo", "connect", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", cfg.Bucket,
|
||||
"--prefix", cfg.Prefix,
|
||||
"--passphrase", "newpass")
|
||||
@ -124,7 +123,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
|
||||
// connect with old passphrase - it will fail
|
||||
cmd = cliTD.StubRootCmd(
|
||||
"repo", "connect", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", cfg.Bucket,
|
||||
"--prefix", cfg.Prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -4,13 +4,14 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/events"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
)
|
||||
@ -84,7 +85,7 @@ func s3InitCmd() *cobra.Command {
|
||||
func initS3Cmd(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderS3,
|
||||
true,
|
||||
@ -131,6 +132,10 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
|
||||
ric := repository.InitConfig{RetentionOpts: retentionOpts}
|
||||
|
||||
if err = r.Initialize(ctx, ric); err != nil {
|
||||
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return Only(ctx, clues.Stack(ErrInitializingRepo, err))
|
||||
}
|
||||
|
||||
@ -165,7 +170,7 @@ func s3ConnectCmd() *cobra.Command {
|
||||
func connectS3Cmd(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderS3,
|
||||
true,
|
||||
|
||||
@ -11,13 +11,12 @@ import (
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/str"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
@ -80,7 +79,7 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "init", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", test.bucketPrefix+cfg.Bucket,
|
||||
"--prefix", cfg.Prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -89,9 +88,9 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// noop
|
||||
// a second initialization should result in an error
|
||||
err = cmd.ExecuteContext(ctx)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists, clues.ToCore(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -114,9 +113,10 @@ func (suite *S3E2ESuite) TestInitMultipleTimes() {
|
||||
for i := 0; i < 2; i++ {
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "init", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", cfg.Bucket,
|
||||
"--prefix", cfg.Prefix)
|
||||
"--prefix", cfg.Prefix,
|
||||
"--succeed-if-exists")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
// run the command
|
||||
@ -146,7 +146,7 @@ func (suite *S3E2ESuite) TestInitS3Cmd_missingBucket() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "init", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--prefix", cfg.Prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -219,7 +219,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
|
||||
// then test it
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "connect", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", test.bucketPrefix+cfg.Bucket,
|
||||
"--prefix", cfg.Prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -279,7 +279,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd_badInputs() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"repo", "connect", "s3",
|
||||
"--"+flags.ConfigFileFN, configFP,
|
||||
"--config-file", configFP,
|
||||
"--bucket", bucket,
|
||||
"--prefix", prefix)
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -2,6 +2,7 @@ package restore
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
@ -9,14 +10,22 @@ import (
|
||||
|
||||
// called by restore.go to map subcommands to provider-specific handling.
|
||||
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case restoreCommand:
|
||||
c, _ = utils.AddCommand(cmd, exchangeRestoreCmd())
|
||||
c, fs = utils.AddCommand(cmd, exchangeRestoreCmd())
|
||||
|
||||
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
// general flags
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddExchangeDetailsAndRestoreFlags(c, false)
|
||||
flags.AddRestoreConfigFlags(c, true)
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/cli"
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
cliTD "github.com/alcionai/corso/src/cli/testdata"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
@ -18,7 +19,6 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
@ -145,7 +145,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"restore", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.cfgFP,
|
||||
"--config-file", suite.cfgFP,
|
||||
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID))
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -180,7 +180,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd_badTimeFlags() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"restore", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.cfgFP,
|
||||
"--config-file", suite.cfgFP,
|
||||
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID),
|
||||
timeFilter, "smarf")
|
||||
cli.BuildCommandTree(cmd)
|
||||
@ -214,7 +214,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd_badBoolFlags() {
|
||||
|
||||
cmd := cliTD.StubRootCmd(
|
||||
"restore", "exchange",
|
||||
"--"+flags.ConfigFileFN, suite.cfgFP,
|
||||
"--config-file", suite.cfgFP,
|
||||
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID),
|
||||
timeFilter, "wingbat")
|
||||
cli.BuildCommandTree(cmd)
|
||||
|
||||
@ -2,23 +2,30 @@ package restore
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
)
|
||||
|
||||
// called by restore.go to map subcommands to provider-specific handling.
|
||||
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case restoreCommand:
|
||||
c, _ = utils.AddCommand(cmd, groupsRestoreCmd(), utils.MarkPreviewCommand())
|
||||
c, fs = utils.AddCommand(cmd, groupsRestoreCmd(), utils.MarkPreviewCommand())
|
||||
|
||||
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddSiteFlag(c, false)
|
||||
flags.AddSiteIDFlag(c, false)
|
||||
@ -84,10 +91,6 @@ func restoreGroupsCmd(cmd *cobra.Command, args []string) error {
|
||||
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
|
||||
utils.FilterGroupsRestoreInfoSelectors(sel, opts)
|
||||
|
||||
// TODO(pandeyabs): Exclude conversations from restores since they are not
|
||||
// supported yet.
|
||||
sel.Exclude(sel.Conversation(selectors.Any()))
|
||||
|
||||
return runRestore(
|
||||
ctx,
|
||||
cmd,
|
||||
|
||||
@ -60,7 +60,8 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
|
||||
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
|
||||
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
|
||||
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
|
||||
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
|
||||
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
|
||||
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
|
||||
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
|
||||
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
|
||||
"--" + flags.CollisionsFN, flagsTD.Collisions,
|
||||
@ -91,7 +92,6 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
|
||||
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
|
||||
assert.Equal(t, flagsTD.Collisions, opts.RestoreCfg.Collisions)
|
||||
assert.Equal(t, flagsTD.Destination, opts.RestoreCfg.Destination)
|
||||
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
|
||||
// assert.Equal(t, flagsTD.ToResource, opts.RestoreCfg.ProtectedResource)
|
||||
assert.True(t, flags.NoPermissionsFV)
|
||||
flagsTD.AssertProviderFlags(t, cmd)
|
||||
|
||||
@ -2,22 +2,30 @@ package restore
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
)
|
||||
|
||||
// called by restore.go to map subcommands to provider-specific handling.
|
||||
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case restoreCommand:
|
||||
c, _ = utils.AddCommand(cmd, oneDriveRestoreCmd())
|
||||
c, fs = utils.AddCommand(cmd, oneDriveRestoreCmd())
|
||||
|
||||
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --user) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddOneDriveDetailsAndRestoreFlags(c)
|
||||
flags.AddNoPermissionsFlag(c)
|
||||
|
||||
@ -2,22 +2,30 @@ package restore
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
)
|
||||
|
||||
// called by restore.go to map subcommands to provider-specific handling.
|
||||
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
|
||||
var c *cobra.Command
|
||||
var (
|
||||
c *cobra.Command
|
||||
fs *pflag.FlagSet
|
||||
)
|
||||
|
||||
switch cmd.Use {
|
||||
case restoreCommand:
|
||||
c, _ = utils.AddCommand(cmd, sharePointRestoreCmd())
|
||||
c, fs = utils.AddCommand(cmd, sharePointRestoreCmd())
|
||||
|
||||
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
|
||||
|
||||
// Flags addition ordering should follow the order we want them to appear in help and docs:
|
||||
// More generic (ex: --site) and more frequently used flags take precedence.
|
||||
fs.SortFlags = false
|
||||
|
||||
flags.AddBackupIDFlag(c, true)
|
||||
flags.AddSharePointDetailsAndRestoreFlags(c)
|
||||
flags.AddNoPermissionsFlag(c)
|
||||
@ -50,27 +58,7 @@ corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
|
||||
# Restore all files in the "Documents" library.
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--library Documents --folder "Display Templates/Style Sheets"
|
||||
|
||||
# Restore lists by their name(s)
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list "list-name-1,list-name-2"
|
||||
|
||||
# Restore lists created after a given time
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-after 2024-01-01T12:23:34
|
||||
|
||||
# Restore lists created before a given time
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-created-before 2024-01-01T12:23:34
|
||||
|
||||
# Restore lists modified before a given time
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-before 2024-01-01T12:23:34
|
||||
|
||||
# Restore lists modified after a given time
|
||||
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
|
||||
--list-modified-after 2024-01-01T12:23:34`
|
||||
--library Documents --folder "Display Templates/Style Sheets" `
|
||||
)
|
||||
|
||||
// `corso restore sharepoint [<flag>...]`
|
||||
|
||||
@ -59,11 +59,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
|
||||
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
|
||||
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
|
||||
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
|
||||
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
|
||||
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
|
||||
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
|
||||
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
|
||||
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
|
||||
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
|
||||
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
|
||||
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
|
||||
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
|
||||
"--" + flags.CollisionsFN, flagsTD.Collisions,
|
||||
@ -92,11 +89,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
|
||||
assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
|
||||
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
|
||||
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
|
||||
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
|
||||
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
|
||||
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
|
||||
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
|
||||
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
|
||||
assert.ElementsMatch(t, flagsTD.ListItemInput, opts.ListItem)
|
||||
assert.ElementsMatch(t, flagsTD.ListFolderInput, opts.ListFolder)
|
||||
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
|
||||
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
|
||||
assert.Equal(t, flagsTD.Collisions, opts.RestoreCfg.Collisions)
|
||||
|
||||
@ -9,8 +9,8 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
|
||||
@ -1,13 +1,9 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -43,55 +39,3 @@ func trimFolderSlash(folders []string) []string {
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func validateCommonTimeFlags(opts any) error {
|
||||
timeFlags := []string{
|
||||
flags.FileCreatedAfterFN,
|
||||
flags.FileCreatedBeforeFN,
|
||||
flags.FileModifiedAfterFN,
|
||||
flags.FileModifiedBeforeFN,
|
||||
flags.ListCreatedAfterFN,
|
||||
flags.ListCreatedBeforeFN,
|
||||
flags.ListModifiedAfterFN,
|
||||
flags.ListModifiedBeforeFN,
|
||||
}
|
||||
|
||||
isFlagPopulated := func(opts any, flag string) bool {
|
||||
switch opts := opts.(type) {
|
||||
case GroupsOpts:
|
||||
_, ok := opts.Populated[flag]
|
||||
return ok
|
||||
case SharePointOpts:
|
||||
_, ok := opts.Populated[flag]
|
||||
return ok
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
getTimeField := func(opts any, flag string) (string, error) {
|
||||
switch opts := opts.(type) {
|
||||
case GroupsOpts:
|
||||
return opts.GetFileTimeField(flag), nil
|
||||
case SharePointOpts:
|
||||
return opts.GetFileTimeField(flag), nil
|
||||
default:
|
||||
return "", errors.New("unsupported type")
|
||||
}
|
||||
}
|
||||
|
||||
for _, flag := range timeFlags {
|
||||
if populated := isFlagPopulated(opts, flag); populated {
|
||||
timeField, err := getTimeField(opts, flag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !IsValidTimeFormat(timeField) {
|
||||
return clues.New("invalid time format for " + flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -103,6 +103,7 @@ func (suite *FlagUnitSuite) TestAddS3BucketFlags() {
|
||||
assert.Equal(t, "prefix1", flags.PrefixFV, flags.PrefixFN)
|
||||
assert.True(t, flags.DoNotUseTLSFV, flags.DoNotUseTLSFN)
|
||||
assert.True(t, flags.DoNotVerifyTLSFV, flags.DoNotVerifyTLSFN)
|
||||
assert.True(t, flags.SucceedIfExistsFV, flags.SucceedIfExistsFN)
|
||||
},
|
||||
}
|
||||
|
||||
@ -115,6 +116,7 @@ func (suite *FlagUnitSuite) TestAddS3BucketFlags() {
|
||||
"--" + flags.PrefixFN, "prefix1",
|
||||
"--" + flags.DoNotUseTLSFN,
|
||||
"--" + flags.DoNotVerifyTLSFN,
|
||||
"--" + flags.SucceedIfExistsFN,
|
||||
})
|
||||
|
||||
err := cmd.Execute()
|
||||
@ -128,6 +130,7 @@ func (suite *FlagUnitSuite) TestFilesystemFlags() {
|
||||
Use: "test",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
assert.Equal(t, "/tmp/test", flags.FilesystemPathFV, flags.FilesystemPathFN)
|
||||
assert.True(t, flags.SucceedIfExistsFV, flags.SucceedIfExistsFN)
|
||||
assert.Equal(t, "tenantID", flags.AzureClientTenantFV, flags.AzureClientTenantFN)
|
||||
assert.Equal(t, "clientID", flags.AzureClientIDFV, flags.AzureClientIDFN)
|
||||
assert.Equal(t, "secret", flags.AzureClientSecretFV, flags.AzureClientSecretFN)
|
||||
@ -140,6 +143,7 @@ func (suite *FlagUnitSuite) TestFilesystemFlags() {
|
||||
cmd.SetArgs([]string{
|
||||
"test",
|
||||
"--" + flags.FilesystemPathFN, "/tmp/test",
|
||||
"--" + flags.SucceedIfExistsFN,
|
||||
"--" + flags.AzureClientIDFN, "clientID",
|
||||
"--" + flags.AzureClientTenantFN, "tenantID",
|
||||
"--" + flags.AzureClientSecretFN, "secret",
|
||||
|
||||
@ -32,7 +32,8 @@ type GroupsOpts struct {
|
||||
FileModifiedAfter string
|
||||
FileModifiedBefore string
|
||||
|
||||
Lists []string
|
||||
ListFolder []string
|
||||
ListItem []string
|
||||
|
||||
PageFolder []string
|
||||
Page []string
|
||||
@ -43,21 +44,6 @@ type GroupsOpts struct {
|
||||
Populated flags.PopulatedFlags
|
||||
}
|
||||
|
||||
func (g GroupsOpts) GetFileTimeField(flag string) string {
|
||||
switch flag {
|
||||
case flags.FileCreatedAfterFN:
|
||||
return g.FileCreatedAfter
|
||||
case flags.FileCreatedBeforeFN:
|
||||
return g.FileCreatedBefore
|
||||
case flags.FileModifiedAfterFN:
|
||||
return g.FileModifiedAfter
|
||||
case flags.FileModifiedBeforeFN:
|
||||
return g.FileModifiedBefore
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func GroupsAllowedCategories() map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
flags.DataLibraries: {},
|
||||
@ -107,7 +93,8 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
|
||||
MessageLastReplyAfter: flags.MessageLastReplyAfterFV,
|
||||
MessageLastReplyBefore: flags.MessageLastReplyBeforeFV,
|
||||
|
||||
Lists: flags.ListFV,
|
||||
ListFolder: flags.ListFolderFV,
|
||||
ListItem: flags.ListItemFV,
|
||||
|
||||
Page: flags.PageFV,
|
||||
PageFolder: flags.PageFolderFV,
|
||||
@ -139,6 +126,22 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts, isRestore bool
|
||||
}
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
|
||||
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
|
||||
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
|
||||
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
|
||||
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.MessageCreatedAfterFN]; ok && !IsValidTimeFormat(opts.MessageCreatedAfter) {
|
||||
return clues.New("invalid time format for " + flags.MessageCreatedAfterFN)
|
||||
}
|
||||
@ -155,7 +158,7 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts, isRestore bool
|
||||
return clues.New("invalid time format for " + flags.MessageLastReplyBeforeFN)
|
||||
}
|
||||
|
||||
return validateCommonTimeFlags(opts)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddGroupsFilter adds the scope of the provided values to the selector's
|
||||
@ -178,7 +181,7 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
|
||||
var (
|
||||
groups = opts.Groups
|
||||
folderPaths, fileNames = len(opts.FolderPath), len(opts.FileName)
|
||||
lists = len(opts.Lists)
|
||||
listFolders, listItems = len(opts.ListFolder), len(opts.ListItem)
|
||||
pageFolders, pageItems = len(opts.PageFolder), len(opts.Page)
|
||||
chans, chanMsgs = len(opts.Channels), len(opts.Messages)
|
||||
convs, convPosts = len(opts.Conversations), len(opts.Posts)
|
||||
@ -191,7 +194,7 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
|
||||
sel := selectors.NewGroupsRestore(groups)
|
||||
|
||||
if folderPaths+fileNames+
|
||||
lists+
|
||||
listFolders+listItems+
|
||||
pageFolders+pageItems+
|
||||
chans+chanMsgs+
|
||||
convs+convPosts == 0 {
|
||||
@ -201,43 +204,58 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
|
||||
|
||||
// sharepoint site selectors
|
||||
|
||||
if folderPaths+fileNames > 0 {
|
||||
if fileNames == 0 {
|
||||
opts.FileName = selectors.Any()
|
||||
if folderPaths+fileNames+
|
||||
listFolders+listItems+
|
||||
pageFolders+pageItems > 0 {
|
||||
if folderPaths+fileNames > 0 {
|
||||
if fileNames == 0 {
|
||||
opts.FileName = selectors.Any()
|
||||
}
|
||||
|
||||
opts.FolderPath = trimFolderSlash(opts.FolderPath)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.FolderPath)
|
||||
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.LibraryItems(containsFolders, opts.FileName))
|
||||
}
|
||||
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.LibraryItems(prefixFolders, opts.FileName, selectors.PrefixMatch()))
|
||||
}
|
||||
}
|
||||
|
||||
opts.FolderPath = trimFolderSlash(opts.FolderPath)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.FolderPath)
|
||||
if listFolders+listItems > 0 {
|
||||
if listItems == 0 {
|
||||
opts.ListItem = selectors.Any()
|
||||
}
|
||||
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.LibraryItems(containsFolders, opts.FileName))
|
||||
opts.ListFolder = trimFolderSlash(opts.ListFolder)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListFolder)
|
||||
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.ListItems(containsFolders, opts.ListItem))
|
||||
}
|
||||
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch()))
|
||||
}
|
||||
}
|
||||
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.LibraryItems(prefixFolders, opts.FileName, selectors.PrefixMatch()))
|
||||
}
|
||||
}
|
||||
if pageFolders+pageItems > 0 {
|
||||
if pageItems == 0 {
|
||||
opts.Page = selectors.Any()
|
||||
}
|
||||
|
||||
if lists > 0 {
|
||||
opts.Lists = trimFolderSlash(opts.Lists)
|
||||
sel.Include(sel.ListItems(opts.Lists, opts.Lists, selectors.StrictEqualMatch()))
|
||||
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
|
||||
}
|
||||
opts.PageFolder = trimFolderSlash(opts.PageFolder)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.PageFolder)
|
||||
|
||||
if pageFolders+pageItems > 0 {
|
||||
if pageItems == 0 {
|
||||
opts.Page = selectors.Any()
|
||||
}
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.PageItems(containsFolders, opts.Page))
|
||||
}
|
||||
|
||||
opts.PageFolder = trimFolderSlash(opts.PageFolder)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.PageFolder)
|
||||
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.PageItems(containsFolders, opts.Page))
|
||||
}
|
||||
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch()))
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -266,14 +284,9 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
|
||||
opts.Conversations = selectors.Any()
|
||||
}
|
||||
|
||||
// if no post is specified, select all posts in the conversation
|
||||
if convPosts == 0 {
|
||||
opts.Posts = selectors.Any()
|
||||
}
|
||||
|
||||
// if no post is specified, only select conversations;
|
||||
// otherwise, look for conv/post pairs
|
||||
if convs == 0 {
|
||||
// otherwise, look for channel/message pairs
|
||||
if chanMsgs == 0 {
|
||||
sel.Include(sel.Conversation(opts.Conversations))
|
||||
} else {
|
||||
sel.Include(sel.ConversationPosts(opts.Conversations, opts.Posts))
|
||||
|
||||
@ -8,8 +8,8 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -30,7 +30,6 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
containsOnly = []string{"contains"}
|
||||
prefixOnly = []string{"/prefix"}
|
||||
containsAndPrefix = []string{"contains", "/prefix"}
|
||||
listNames = []string{"list-name1"}
|
||||
onlySlash = []string{string(path.PathSeparator)}
|
||||
)
|
||||
|
||||
@ -41,30 +40,34 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
}{
|
||||
// resource
|
||||
{
|
||||
name: "no inputs",
|
||||
opts: utils.GroupsOpts{},
|
||||
expectIncludeLen: 3,
|
||||
name: "no inputs",
|
||||
opts: utils.GroupsOpts{},
|
||||
// TODO: bump to 3 when we release conversations
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: empty,
|
||||
},
|
||||
expectIncludeLen: 3,
|
||||
// TODO: bump to 3 when we release conversations
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "single inputs",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: single,
|
||||
},
|
||||
expectIncludeLen: 3,
|
||||
// TODO: bump to 3 when we release conversations
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "multi inputs",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: multi,
|
||||
},
|
||||
expectIncludeLen: 3,
|
||||
// TODO: bump to 3 when we release conversations
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
// sharepoint
|
||||
{
|
||||
@ -92,12 +95,29 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "list names",
|
||||
name: "list contains",
|
||||
opts: utils.GroupsOpts{
|
||||
Lists: listNames,
|
||||
FileName: empty,
|
||||
FolderPath: empty,
|
||||
ListItem: empty,
|
||||
ListFolder: containsOnly,
|
||||
},
|
||||
expectIncludeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "list prefixes",
|
||||
opts: utils.GroupsOpts{
|
||||
ListFolder: prefixOnly,
|
||||
},
|
||||
expectIncludeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "list prefixes and contains",
|
||||
opts: utils.GroupsOpts{
|
||||
ListFolder: containsAndPrefix,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "Page Folder",
|
||||
opts: utils.GroupsOpts{
|
||||
@ -401,7 +421,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
|
||||
{
|
||||
name: "none",
|
||||
cats: []string{},
|
||||
expectScopeLen: 3,
|
||||
expectScopeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "libraries",
|
||||
@ -423,9 +443,10 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
|
||||
cats: []string{
|
||||
flags.DataLibraries,
|
||||
flags.DataMessages,
|
||||
flags.DataConversations,
|
||||
// flags.DataConversations,
|
||||
},
|
||||
expectScopeLen: 3,
|
||||
// TODO: bump to 3 when we include conversations in all data
|
||||
expectScopeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "bad inputs",
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
)
|
||||
|
||||
@ -26,9 +26,8 @@ func Control() control.Options {
|
||||
opt.ToggleFeatures.ForceItemDataDownload = flags.ForceItemDataDownloadFV
|
||||
opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV
|
||||
opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
||||
opt.ToggleFeatures.DisableLazyItemReader = flags.DisableLazyItemReaderFV
|
||||
opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
||||
opt.ToggleFeatures.UseOldDeltaProcess = flags.UseOldDeltaProcessFV
|
||||
opt.ToggleFeatures.UseDeltaTree = flags.UseDeltaTreeFV
|
||||
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
||||
|
||||
return opt
|
||||
@ -42,27 +41,3 @@ func ControlWithConfig(cfg config.RepoDetails) control.Options {
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
func ParseBackupOptions() control.BackupConfig {
|
||||
opt := control.DefaultBackupConfig()
|
||||
|
||||
if flags.FailFastFV {
|
||||
opt.FailureHandling = control.FailFast
|
||||
}
|
||||
|
||||
dps := int32(flags.DeltaPageSizeFV)
|
||||
if dps > 500 || dps < 1 {
|
||||
dps = 500
|
||||
}
|
||||
|
||||
opt.M365.DeltaPageSize = dps
|
||||
opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
|
||||
opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
|
||||
opt.M365.UseOldDriveDeltaProcess = flags.UseOldDeltaProcessFV
|
||||
opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
|
||||
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
|
||||
opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV
|
||||
opt.Incrementals.ForceItemDataRefresh = flags.ForceItemDataDownloadFV
|
||||
|
||||
return opt
|
||||
}
|
||||
|
||||
@ -9,8 +9,8 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
)
|
||||
|
||||
type RestoreCfgOpts struct {
|
||||
|
||||
@ -25,11 +25,8 @@ type SharePointOpts struct {
|
||||
FileModifiedAfter string
|
||||
FileModifiedBefore string
|
||||
|
||||
Lists []string
|
||||
ListModifiedAfter string
|
||||
ListModifiedBefore string
|
||||
ListCreatedBefore string
|
||||
ListCreatedAfter string
|
||||
ListFolder []string
|
||||
ListItem []string
|
||||
|
||||
PageFolder []string
|
||||
Page []string
|
||||
@ -40,29 +37,6 @@ type SharePointOpts struct {
|
||||
Populated flags.PopulatedFlags
|
||||
}
|
||||
|
||||
func (s SharePointOpts) GetFileTimeField(flag string) string {
|
||||
switch flag {
|
||||
case flags.FileCreatedAfterFN:
|
||||
return s.FileCreatedAfter
|
||||
case flags.FileCreatedBeforeFN:
|
||||
return s.FileCreatedBefore
|
||||
case flags.FileModifiedAfterFN:
|
||||
return s.FileModifiedAfter
|
||||
case flags.FileModifiedBeforeFN:
|
||||
return s.FileModifiedBefore
|
||||
case flags.ListModifiedAfterFN:
|
||||
return s.ListModifiedAfter
|
||||
case flags.ListModifiedBeforeFN:
|
||||
return s.ListModifiedBefore
|
||||
case flags.ListCreatedBeforeFN:
|
||||
return s.ListCreatedBefore
|
||||
case flags.ListCreatedAfterFN:
|
||||
return s.ListCreatedAfter
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
|
||||
return SharePointOpts{
|
||||
SiteID: flags.SiteIDFV,
|
||||
@ -76,11 +50,8 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
|
||||
FileModifiedAfter: flags.FileModifiedAfterFV,
|
||||
FileModifiedBefore: flags.FileModifiedBeforeFV,
|
||||
|
||||
Lists: flags.ListFV,
|
||||
ListModifiedAfter: flags.ListModifiedAfterFV,
|
||||
ListModifiedBefore: flags.ListModifiedBeforeFV,
|
||||
ListCreatedAfter: flags.ListCreatedAfterFV,
|
||||
ListCreatedBefore: flags.ListCreatedBeforeFV,
|
||||
ListFolder: flags.ListFolderFV,
|
||||
ListItem: flags.ListItemFV,
|
||||
|
||||
Page: flags.PageFV,
|
||||
PageFolder: flags.PageFolderFV,
|
||||
@ -95,32 +66,6 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
|
||||
}
|
||||
}
|
||||
|
||||
func SharePointAllowedCategories() map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
flags.DataLibraries: {},
|
||||
flags.DataLists: {},
|
||||
}
|
||||
}
|
||||
|
||||
func AddCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
|
||||
if len(cats) == 0 {
|
||||
// [TODO](hitesh) to enable lists without being invoked explicitly via --data flag
|
||||
// sel.Include(sel.LibraryFolders(selectors.Any()), sel.Lists(selectors.Any()))
|
||||
sel.Include(sel.LibraryFolders(selectors.Any()))
|
||||
}
|
||||
|
||||
for _, d := range cats {
|
||||
switch d {
|
||||
case flags.DataLists:
|
||||
sel.Include(sel.Lists(selectors.Any()))
|
||||
case flags.DataLibraries:
|
||||
sel.Include(sel.LibraryFolders(selectors.Any()))
|
||||
}
|
||||
}
|
||||
|
||||
return sel
|
||||
}
|
||||
|
||||
// ValidateSharePointRestoreFlags checks common flags for correctness and interdependencies
|
||||
func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error {
|
||||
if len(backupID) == 0 {
|
||||
@ -136,7 +81,23 @@ func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error
|
||||
}
|
||||
}
|
||||
|
||||
return validateCommonTimeFlags(opts)
|
||||
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
|
||||
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
|
||||
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
|
||||
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
|
||||
}
|
||||
|
||||
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
|
||||
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddSharePointInfo adds the scope of the provided values to the selector's
|
||||
@ -158,24 +119,24 @@ func AddSharePointInfo(
|
||||
func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointOpts) *selectors.SharePointRestore {
|
||||
sites := opts.SiteID
|
||||
|
||||
folderPaths, fileNames := len(opts.FolderPath), len(opts.FileName)
|
||||
siteIDs, webUrls := len(opts.SiteID), len(opts.WebURL)
|
||||
lists := len(opts.Lists)
|
||||
pageFolders, pageItems := len(opts.PageFolder), len(opts.Page)
|
||||
lfp, lfn := len(opts.FolderPath), len(opts.FileName)
|
||||
ls, lwu := len(opts.SiteID), len(opts.WebURL)
|
||||
slp, sli := len(opts.ListFolder), len(opts.ListItem)
|
||||
pf, pi := len(opts.PageFolder), len(opts.Page)
|
||||
|
||||
if siteIDs == 0 {
|
||||
if ls == 0 {
|
||||
sites = selectors.Any()
|
||||
}
|
||||
|
||||
sel := selectors.NewSharePointRestore(sites)
|
||||
|
||||
if folderPaths+fileNames+webUrls+lists+pageFolders+pageItems == 0 {
|
||||
if lfp+lfn+lwu+slp+sli+pf+pi == 0 {
|
||||
sel.Include(sel.AllData())
|
||||
return sel
|
||||
}
|
||||
|
||||
if folderPaths+fileNames > 0 {
|
||||
if fileNames == 0 {
|
||||
if lfp+lfn > 0 {
|
||||
if lfn == 0 {
|
||||
opts.FileName = selectors.Any()
|
||||
}
|
||||
|
||||
@ -191,14 +152,25 @@ func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointO
|
||||
}
|
||||
}
|
||||
|
||||
if lists > 0 {
|
||||
opts.Lists = trimFolderSlash(opts.Lists)
|
||||
sel.Include(sel.ListItems(opts.Lists, opts.Lists, selectors.StrictEqualMatch()))
|
||||
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
|
||||
if slp+sli > 0 {
|
||||
if sli == 0 {
|
||||
opts.ListItem = selectors.Any()
|
||||
}
|
||||
|
||||
opts.ListFolder = trimFolderSlash(opts.ListFolder)
|
||||
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListFolder)
|
||||
|
||||
if len(containsFolders) > 0 {
|
||||
sel.Include(sel.ListItems(containsFolders, opts.ListItem))
|
||||
}
|
||||
|
||||
if len(prefixFolders) > 0 {
|
||||
sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch()))
|
||||
}
|
||||
}
|
||||
|
||||
if pageFolders+pageItems > 0 {
|
||||
if pageItems == 0 {
|
||||
if pf+pi > 0 {
|
||||
if pi == 0 {
|
||||
opts.Page = selectors.Any()
|
||||
}
|
||||
|
||||
@ -214,7 +186,7 @@ func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointO
|
||||
}
|
||||
}
|
||||
|
||||
if webUrls > 0 {
|
||||
if lwu > 0 {
|
||||
urls := make([]string, 0, len(opts.WebURL))
|
||||
|
||||
for _, wu := range opts.WebURL {
|
||||
@ -253,8 +225,4 @@ func FilterSharePointRestoreInfoSelectors(
|
||||
AddSharePointInfo(sel, opts.FileCreatedBefore, sel.CreatedBefore)
|
||||
AddSharePointInfo(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
|
||||
AddSharePointInfo(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
|
||||
AddSharePointInfo(sel, opts.ListModifiedAfter, sel.ListModifiedAfter)
|
||||
AddSharePointInfo(sel, opts.ListModifiedBefore, sel.ListModifiedBefore)
|
||||
AddSharePointInfo(sel, opts.ListCreatedAfter, sel.ListCreatedAfter)
|
||||
AddSharePointInfo(sel, opts.ListCreatedBefore, sel.ListCreatedBefore)
|
||||
}
|
||||
|
||||
@ -8,8 +8,8 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -31,7 +31,6 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
|
||||
multi = []string{"more", "than", "one"}
|
||||
containsOnly = []string{"contains"}
|
||||
prefixOnly = []string{"/prefix"}
|
||||
listNames = []string{"list-name1"}
|
||||
containsAndPrefix = []string{"contains", "/prefix"}
|
||||
onlySlash = []string{string(path.PathSeparator)}
|
||||
)
|
||||
@ -61,7 +60,8 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
|
||||
opts: utils.SharePointOpts{
|
||||
FileName: single,
|
||||
FolderPath: single,
|
||||
Lists: single,
|
||||
ListItem: single,
|
||||
ListFolder: single,
|
||||
SiteID: single,
|
||||
WebURL: single,
|
||||
},
|
||||
@ -108,12 +108,31 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "list names",
|
||||
name: "list contains",
|
||||
opts: utils.SharePointOpts{
|
||||
Lists: listNames,
|
||||
FileName: empty,
|
||||
FolderPath: empty,
|
||||
ListItem: empty,
|
||||
ListFolder: containsOnly,
|
||||
SiteID: empty,
|
||||
WebURL: empty,
|
||||
},
|
||||
expectIncludeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "list prefixes",
|
||||
opts: utils.SharePointOpts{
|
||||
ListFolder: prefixOnly,
|
||||
},
|
||||
expectIncludeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "list prefixes and contains",
|
||||
opts: utils.SharePointOpts{
|
||||
ListFolder: containsAndPrefix,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "weburl contains",
|
||||
opts: utils.SharePointOpts{
|
||||
@ -279,20 +298,12 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
|
||||
FileCreatedBefore: dttm.Now(),
|
||||
FileModifiedAfter: dttm.Now(),
|
||||
FileModifiedBefore: dttm.Now(),
|
||||
ListCreatedAfter: dttm.Now(),
|
||||
ListCreatedBefore: dttm.Now(),
|
||||
ListModifiedAfter: dttm.Now(),
|
||||
ListModifiedBefore: dttm.Now(),
|
||||
Populated: flags.PopulatedFlags{
|
||||
flags.SiteFN: struct{}{},
|
||||
flags.FileCreatedAfterFN: struct{}{},
|
||||
flags.FileCreatedBeforeFN: struct{}{},
|
||||
flags.FileModifiedAfterFN: struct{}{},
|
||||
flags.FileModifiedBeforeFN: struct{}{},
|
||||
flags.ListCreatedAfterFN: struct{}{},
|
||||
flags.ListCreatedBeforeFN: struct{}{},
|
||||
flags.ListModifiedAfterFN: struct{}{},
|
||||
flags.ListModifiedBeforeFN: struct{}{},
|
||||
},
|
||||
},
|
||||
expect: assert.NoError,
|
||||
@ -358,50 +369,6 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "invalid list created after",
|
||||
backupID: "id",
|
||||
opts: utils.SharePointOpts{
|
||||
ListCreatedAfter: "1235",
|
||||
Populated: flags.PopulatedFlags{
|
||||
flags.ListCreatedAfterFN: struct{}{},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "invalid list created before",
|
||||
backupID: "id",
|
||||
opts: utils.SharePointOpts{
|
||||
ListCreatedBefore: "1235",
|
||||
Populated: flags.PopulatedFlags{
|
||||
flags.ListCreatedBeforeFN: struct{}{},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "invalid list modified after",
|
||||
backupID: "id",
|
||||
opts: utils.SharePointOpts{
|
||||
ListModifiedAfter: "1235",
|
||||
Populated: flags.PopulatedFlags{
|
||||
flags.ListModifiedAfterFN: struct{}{},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "invalid list modified before",
|
||||
backupID: "id",
|
||||
opts: utils.SharePointOpts{
|
||||
ListModifiedBefore: "1235",
|
||||
Populated: flags.PopulatedFlags{
|
||||
flags.ListModifiedBeforeFN: struct{}{},
|
||||
},
|
||||
},
|
||||
expect: assert.Error,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
@ -410,47 +377,3 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *SharePointUtilsSuite) TestAddSharepointCategories() {
|
||||
table := []struct {
|
||||
name string
|
||||
cats []string
|
||||
expectScopeLen int
|
||||
}{
|
||||
{
|
||||
name: "none",
|
||||
cats: []string{},
|
||||
expectScopeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "libraries",
|
||||
cats: []string{flags.DataLibraries},
|
||||
expectScopeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "lists",
|
||||
cats: []string{flags.DataLists},
|
||||
expectScopeLen: 1,
|
||||
},
|
||||
{
|
||||
name: "all allowed",
|
||||
cats: []string{
|
||||
flags.DataLibraries,
|
||||
flags.DataLists,
|
||||
},
|
||||
expectScopeLen: 2,
|
||||
},
|
||||
{
|
||||
name: "bad inputs",
|
||||
cats: []string{"foo"},
|
||||
expectScopeLen: 0,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
sel := utils.AddCategories(selectors.NewSharePointBackup(selectors.Any()), test.cats)
|
||||
scopes := sel.Scopes()
|
||||
assert.Len(suite.T(), scopes, test.expectScopeLen)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,101 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
type TeamsChatsOpts struct {
|
||||
Users []string
|
||||
|
||||
ExportCfg ExportCfgOpts
|
||||
|
||||
Populated flags.PopulatedFlags
|
||||
}
|
||||
|
||||
func TeamsChatsAllowedCategories() map[string]struct{} {
|
||||
return map[string]struct{}{
|
||||
flags.DataChats: {},
|
||||
}
|
||||
}
|
||||
|
||||
func AddTeamsChatsCategories(sel *selectors.TeamsChatsBackup, cats []string) *selectors.TeamsChatsBackup {
|
||||
if len(cats) == 0 {
|
||||
sel.Include(sel.AllData())
|
||||
}
|
||||
|
||||
for _, d := range cats {
|
||||
switch d {
|
||||
case flags.DataChats:
|
||||
sel.Include(sel.Chats(selectors.Any()))
|
||||
}
|
||||
}
|
||||
|
||||
return sel
|
||||
}
|
||||
|
||||
func MakeTeamsChatsOpts(cmd *cobra.Command) TeamsChatsOpts {
|
||||
return TeamsChatsOpts{
|
||||
Users: flags.UserFV,
|
||||
|
||||
ExportCfg: makeExportCfgOpts(cmd),
|
||||
|
||||
// populated contains the list of flags that appear in the
|
||||
// command, according to pflags. Use this to differentiate
|
||||
// between an "empty" and a "missing" value.
|
||||
Populated: flags.GetPopulatedFlags(cmd),
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateTeamsChatsRestoreFlags checks common flags for correctness and interdependencies
|
||||
func ValidateTeamsChatsRestoreFlags(backupID string, opts TeamsChatsOpts, isRestore bool) error {
|
||||
if len(backupID) == 0 {
|
||||
return clues.New("a backup ID is required")
|
||||
}
|
||||
|
||||
// restore isn't currently supported
|
||||
if isRestore {
|
||||
return clues.New("restore not supported")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddTeamsChatsFilter adds the scope of the provided values to the selector's
|
||||
// filter set
|
||||
func AddTeamsChatsFilter(
|
||||
sel *selectors.TeamsChatsRestore,
|
||||
v string,
|
||||
f func(string) []selectors.TeamsChatsScope,
|
||||
) {
|
||||
if len(v) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
sel.Filter(f(v))
|
||||
}
|
||||
|
||||
// IncludeTeamsChatsRestoreDataSelectors builds the common data-selector
|
||||
// inclusions for teamschats commands.
|
||||
func IncludeTeamsChatsRestoreDataSelectors(ctx context.Context, opts TeamsChatsOpts) *selectors.TeamsChatsRestore {
|
||||
users := opts.Users
|
||||
|
||||
if len(opts.Users) == 0 {
|
||||
users = selectors.Any()
|
||||
}
|
||||
|
||||
return selectors.NewTeamsChatsRestore(users)
|
||||
}
|
||||
|
||||
// FilterTeamsChatsRestoreInfoSelectors builds the common info-selector filters.
|
||||
func FilterTeamsChatsRestoreInfoSelectors(
|
||||
sel *selectors.TeamsChatsRestore,
|
||||
opts TeamsChatsOpts,
|
||||
) {
|
||||
// TODO: populate when adding filters
|
||||
}
|
||||
2
src/cli/utils/testdata/opts.go
vendored
2
src/cli/utils/testdata/opts.go
vendored
@ -9,10 +9,10 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
ftd "github.com/alcionai/corso/src/pkg/fault/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
|
||||
@ -10,10 +10,10 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
"github.com/alcionai/corso/src/internal/events"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
@ -51,7 +51,7 @@ func GetAccountAndConnectWithOverrides(
|
||||
provider storage.ProviderType,
|
||||
overrides map[string]string,
|
||||
) (repository.Repositoryer, RepoDetailsAndOpts, error) {
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
provider,
|
||||
true,
|
||||
|
||||
@ -7,8 +7,6 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/converters/eml"
|
||||
"github.com/alcionai/corso/src/internal/converters/ics"
|
||||
"github.com/alcionai/corso/src/internal/converters/vcf"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -29,23 +27,13 @@ func main() {
|
||||
var out string
|
||||
|
||||
switch from {
|
||||
case "json":
|
||||
case "msg":
|
||||
switch to {
|
||||
case "eml":
|
||||
out, err = eml.FromJSON(context.Background(), body)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case "vcf":
|
||||
out, err = vcf.FromJSON(context.Background(), body)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case "ics":
|
||||
out, err = ics.FromJSON(context.Background(), body)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
default:
|
||||
log.Fatal("Unknown target format", to)
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/common/str"
|
||||
@ -18,7 +19,6 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/m365"
|
||||
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
|
||||
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
|
||||
siteMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
|
||||
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
|
||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
@ -28,7 +28,6 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/credentials"
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
@ -59,7 +58,7 @@ func generateAndRestoreItems(
|
||||
service path.ServiceType,
|
||||
cat path.CategoryType,
|
||||
sel selectors.Selector,
|
||||
tenantID, resourceID, destFldr string,
|
||||
tenantID, userID, destFldr string,
|
||||
howMany int,
|
||||
dbf dataBuilderFunc,
|
||||
opts control.Options,
|
||||
@ -74,7 +73,7 @@ func generateAndRestoreItems(
|
||||
nowLegacy = dttm.FormatToLegacy(time.Now())
|
||||
id = uuid.NewString()
|
||||
subject = "automated " + now[:16] + " - " + id[:8]
|
||||
body = "automated " + cat.HumanString() + " generation for " + resourceID + " at " + now + " - " + id
|
||||
body = "automated " + cat.HumanString() + " generation for " + userID + " at " + now + " - " + id
|
||||
)
|
||||
|
||||
items = append(items, item{
|
||||
@ -95,7 +94,7 @@ func generateAndRestoreItems(
|
||||
|
||||
dataColls, err := buildCollections(
|
||||
service,
|
||||
tenantID, resourceID,
|
||||
tenantID, userID,
|
||||
restoreCfg,
|
||||
collections)
|
||||
if err != nil {
|
||||
@ -112,7 +111,7 @@ func generateAndRestoreItems(
|
||||
Selector: sel,
|
||||
}
|
||||
|
||||
handler, err := ctrl.NewServiceHandler(service)
|
||||
handler, err := ctrl.NewServiceHandler(opts, service)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
}
|
||||
@ -193,31 +192,29 @@ type collection struct {
|
||||
|
||||
func buildCollections(
|
||||
service path.ServiceType,
|
||||
tenant, resource string,
|
||||
tenant, user string,
|
||||
restoreCfg control.RestoreConfig,
|
||||
colls []collection,
|
||||
) ([]data.RestoreCollection, error) {
|
||||
var (
|
||||
collections = make([]data.RestoreCollection, 0, len(colls))
|
||||
mc data.Collection
|
||||
)
|
||||
collections := make([]data.RestoreCollection, 0, len(colls))
|
||||
|
||||
for _, c := range colls {
|
||||
switch {
|
||||
case service == path.ExchangeService:
|
||||
emc, err := generateExchangeMockColls(tenant, resource, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pth, err := path.Build(
|
||||
tenant,
|
||||
user,
|
||||
service,
|
||||
c.category,
|
||||
false,
|
||||
c.PathElements...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mc = emc
|
||||
case service == path.SharePointService:
|
||||
smc, err := generateSharepointListsMockColls(tenant, resource, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mc := exchMock.NewCollection(pth, pth, len(c.items))
|
||||
|
||||
mc = smc
|
||||
for i := 0; i < len(c.items); i++ {
|
||||
mc.Names[i] = c.items[i].name
|
||||
mc.Data[i] = c.items[i].data
|
||||
}
|
||||
|
||||
collections = append(collections, data.NoFetchRestoreCollection{Collection: mc})
|
||||
@ -226,49 +223,6 @@ func buildCollections(
|
||||
return collections, nil
|
||||
}
|
||||
|
||||
func generateExchangeMockColls(tenant string, resource string, c collection) (*exchMock.DataCollection, error) {
|
||||
pth, err := path.Build(
|
||||
tenant,
|
||||
resource,
|
||||
path.ExchangeService,
|
||||
c.category,
|
||||
false,
|
||||
c.PathElements...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
emc := exchMock.NewCollection(pth, pth, len(c.items))
|
||||
|
||||
for i := 0; i < len(c.items); i++ {
|
||||
emc.Names[i] = c.items[i].name
|
||||
emc.Data[i] = c.items[i].data
|
||||
}
|
||||
|
||||
return emc, nil
|
||||
}
|
||||
|
||||
func generateSharepointListsMockColls(tenant string, resource string, c collection) (*siteMock.ListCollection, error) {
|
||||
pth, err := path.BuildOrPrefix(
|
||||
tenant,
|
||||
resource,
|
||||
path.SharePointService,
|
||||
c.category,
|
||||
false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
smc := siteMock.NewCollection(pth, pth, len(c.items))
|
||||
|
||||
for i := 0; i < len(c.items); i++ {
|
||||
smc.Names[i] = c.items[i].name
|
||||
smc.Data[i] = c.items[i].data
|
||||
}
|
||||
|
||||
return smc, nil
|
||||
}
|
||||
|
||||
var (
|
||||
folderAName = "folder-a"
|
||||
folderBName = "b"
|
||||
@ -506,7 +460,7 @@ func generateAndRestoreDriveItems(
|
||||
Selector: sel,
|
||||
}
|
||||
|
||||
handler, err := ctrl.NewServiceHandler(service)
|
||||
handler, err := ctrl.NewServiceHandler(opts, service)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
}
|
||||
|
||||
@ -7,8 +7,6 @@ import (
|
||||
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
siteMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
@ -16,23 +14,14 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
var (
|
||||
spFilesCmd = &cobra.Command{
|
||||
Use: "files",
|
||||
Short: "Generate SharePoint files",
|
||||
RunE: handleSharePointLibraryFileFactory,
|
||||
}
|
||||
|
||||
spListsCmd = &cobra.Command{
|
||||
Use: "lists",
|
||||
Short: "Generate SharePoint lists",
|
||||
RunE: handleSharepointListsFactory,
|
||||
}
|
||||
)
|
||||
var spFilesCmd = &cobra.Command{
|
||||
Use: "files",
|
||||
Short: "Generate SharePoint files",
|
||||
RunE: handleSharePointLibraryFileFactory,
|
||||
}
|
||||
|
||||
func AddSharePointCommands(cmd *cobra.Command) {
|
||||
cmd.AddCommand(spFilesCmd)
|
||||
cmd.AddCommand(spListsCmd)
|
||||
}
|
||||
|
||||
func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error {
|
||||
@ -81,52 +70,3 @@ func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleSharepointListsFactory(cmd *cobra.Command, args []string) error {
|
||||
var (
|
||||
ctx = cmd.Context()
|
||||
service = path.SharePointService
|
||||
category = path.ListsCategory
|
||||
errs = fault.New(false)
|
||||
)
|
||||
|
||||
if utils.HasNoFlagsAndShownHelp(cmd) {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctrl, _, ins, err := getControllerAndVerifyResourceOwner(ctx, Site, path.SharePointService)
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
deets, err := generateAndRestoreItems(
|
||||
ctx,
|
||||
ctrl,
|
||||
service,
|
||||
category,
|
||||
selectors.NewSharePointRestore([]string{ins.ID()}).Selector,
|
||||
Tenant, ins.ID(), Destination,
|
||||
Count,
|
||||
func(id, now, subject, body string) []byte {
|
||||
listBytes, err := siteMock.ListBytes(id)
|
||||
if err != nil {
|
||||
logger.CtxErr(ctx, err)
|
||||
return nil
|
||||
}
|
||||
return listBytes
|
||||
},
|
||||
control.DefaultOptions(),
|
||||
errs,
|
||||
count.New())
|
||||
if err != nil {
|
||||
return Only(ctx, err)
|
||||
}
|
||||
|
||||
for _, e := range errs.Recovered() {
|
||||
logger.CtxErr(ctx, err).Error(e.Error())
|
||||
}
|
||||
|
||||
deets.PrintEntries(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -11,8 +11,8 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/repository"
|
||||
@ -90,7 +90,7 @@ func pitrListBackups(
|
||||
|
||||
// TODO(ashmrtn): This may be moved into CLI layer at some point when we add
|
||||
// flags for opening a repo at a point in time.
|
||||
cfg, err := config.ReadCorsoConfig(
|
||||
cfg, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderS3,
|
||||
true,
|
||||
@ -151,7 +151,7 @@ func main() {
|
||||
|
||||
cc.SetContext(context.Background())
|
||||
|
||||
if err := config.InitCmd(&cc, []string{}); err != nil {
|
||||
if err := config.InitFunc(&cc, []string{}); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
44
src/cmd/purge/scripts/onedrivePurge.ps1
Executable file → Normal file
44
src/cmd/purge/scripts/onedrivePurge.ps1
Executable file → Normal file
@ -6,6 +6,12 @@ Param (
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Site for which to delete folders in SharePoint")]
|
||||
[String]$Site,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")]
|
||||
[String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")]
|
||||
[String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Document library root. Can add multiple comma-separated values")]
|
||||
[String[]]$LibraryNameList = @(),
|
||||
|
||||
@ -16,16 +22,7 @@ Param (
|
||||
[String[]]$FolderPrefixPurgeList,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")]
|
||||
[String[]]$LibraryPrefixDeleteList = @(),
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Tenant domain")]
|
||||
[String]$TenantDomain = $ENV:TENANT_DOMAIN,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Azure ClientId")]
|
||||
[String]$ClientId = $ENV:AZURE_CLIENT_ID,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Azure AppCert")]
|
||||
[String]$AppCert = $ENV:AZURE_APP_CERT
|
||||
[String[]]$LibraryPrefixDeleteList = @()
|
||||
)
|
||||
|
||||
Set-StrictMode -Version 2.0
|
||||
@ -40,7 +37,7 @@ function Get-TimestampFromFolderName {
|
||||
|
||||
$name = $folder.Name
|
||||
|
||||
#fallback on folder create time
|
||||
#fallback on folder create time
|
||||
[datetime]$timestamp = $folder.TimeCreated
|
||||
|
||||
try {
|
||||
@ -69,7 +66,7 @@ function Get-TimestampFromListName {
|
||||
|
||||
$name = $list.Title
|
||||
|
||||
#fallback on list create time
|
||||
#fallback on list create time
|
||||
[datetime]$timestamp = $list.LastItemUserModifiedDate
|
||||
|
||||
try {
|
||||
@ -109,9 +106,8 @@ function Purge-Library {
|
||||
Write-Host "`nPurging library: $LibraryName"
|
||||
|
||||
$foldersToPurge = @()
|
||||
$folders = Get-PnPFolderItem -FolderSiteRelativeUrl $LibraryName -ItemType Folder
|
||||
$folders = Get-PnPFolderItem -FolderSiteRelativeUrl $LibraryName -ItemType Folder
|
||||
|
||||
Write-Host "`nFolders: $folders"
|
||||
foreach ($f in $folders) {
|
||||
$folderName = $f.Name
|
||||
$createTime = Get-TimestampFromFolderName -Folder $f
|
||||
@ -163,7 +159,7 @@ function Delete-LibraryByPrefix {
|
||||
Write-Host "`nDeleting library: $LibraryNamePrefix"
|
||||
|
||||
$listsToDelete = @()
|
||||
$lists = Get-PnPList
|
||||
$lists = Get-PnPList
|
||||
|
||||
foreach ($l in $lists) {
|
||||
$listName = $l.Title
|
||||
@ -186,14 +182,6 @@ function Delete-LibraryByPrefix {
|
||||
if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) {
|
||||
Write-Host "Deleting list: "$l.Title
|
||||
try {
|
||||
$listInfo = Get-PnPList -Identity $l.Id | Select-Object -Property Hidden
|
||||
|
||||
# Check if the 'hidden' property is true
|
||||
if ($listInfo.Hidden) {
|
||||
Write-Host "List: $($l.Title) is hidden. Skipping..."
|
||||
continue
|
||||
}
|
||||
|
||||
Remove-PnPList -Identity $l.Id -Force
|
||||
}
|
||||
catch [ System.Management.Automation.ItemNotFoundException ] {
|
||||
@ -213,8 +201,8 @@ if (-not (Get-Module -ListAvailable -Name PnP.PowerShell)) {
|
||||
}
|
||||
|
||||
|
||||
if ([string]::IsNullOrEmpty($ClientId) -or [string]::IsNullOrEmpty($AppCert)) {
|
||||
Write-Host "ClientId and AppCert required as arguments or environment variables."
|
||||
if ([string]::IsNullOrEmpty($AdminUser) -or [string]::IsNullOrEmpty($AdminPwd)) {
|
||||
Write-Host "Admin user name and password required as arguments or environment variables."
|
||||
Exit
|
||||
}
|
||||
|
||||
@ -255,8 +243,12 @@ else {
|
||||
Exit
|
||||
}
|
||||
|
||||
|
||||
$password = convertto-securestring -String $AdminPwd -AsPlainText -Force
|
||||
$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $AdminUser, $password
|
||||
|
||||
Write-Host "`nAuthenticating and connecting to $SiteUrl"
|
||||
Connect-PnPOnline -Url $siteUrl -ClientId $ClientId -CertificateBase64Encoded $AppCert -Tenant $TenantDomain
|
||||
Connect-PnPOnline -Url $siteUrl -Credential $cred
|
||||
Write-Host "Connected to $siteUrl`n"
|
||||
|
||||
# ensure that there are no unexpanded entries in the list of parameters
|
||||
|
||||
@ -12,9 +12,9 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/config"
|
||||
"github.com/alcionai/corso/src/cmd/s3checker/pkg/s3"
|
||||
"github.com/alcionai/corso/src/internal/common/crash"
|
||||
"github.com/alcionai/corso/src/pkg/config"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/storage"
|
||||
)
|
||||
@ -175,7 +175,7 @@ func handleCheckerCommand(cmd *cobra.Command, args []string, f flags) error {
|
||||
|
||||
fmt.Printf("Checking objects with prefix(es) %v\n", f.prefixes)
|
||||
|
||||
if err := config.InitCmd(cmd, args); err != nil {
|
||||
if err := config.InitFunc(cmd, args); err != nil {
|
||||
return clues.Wrap(err, "setting viper")
|
||||
}
|
||||
|
||||
@ -187,7 +187,7 @@ func handleCheckerCommand(cmd *cobra.Command, args []string, f flags) error {
|
||||
storage.Prefix: f.bucketPrefix,
|
||||
}
|
||||
|
||||
repoDetails, err := config.ReadCorsoConfig(
|
||||
repoDetails, err := config.GetConfigRepoDetails(
|
||||
ctx,
|
||||
storage.ProviderS3,
|
||||
false,
|
||||
|
||||
@ -19,40 +19,33 @@ type PermissionInfo struct {
|
||||
}
|
||||
|
||||
const (
|
||||
sanityBackupID = "SANITY_BACKUP_ID"
|
||||
sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER"
|
||||
sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
|
||||
sanityTestRestoreContainerPrefix = "SANITY_TEST_RESTORE_CONTAINER_PREFIX"
|
||||
sanityTestUser = "SANITY_TEST_USER"
|
||||
sanityTestCategory = "SANITY_TEST_CATEGORY"
|
||||
sanityBackupID = "SANITY_BACKUP_ID"
|
||||
sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER"
|
||||
sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
|
||||
sanityTestUser = "SANITY_TEST_USER"
|
||||
)
|
||||
|
||||
type Envs struct {
|
||||
BackupID string
|
||||
SourceContainer string
|
||||
RestoreContainer string
|
||||
// applies for sharepoint lists only
|
||||
RestoreContainerPrefix string
|
||||
Category string
|
||||
GroupID string
|
||||
SiteID string
|
||||
UserID string
|
||||
TeamSiteID string
|
||||
GroupID string
|
||||
SiteID string
|
||||
UserID string
|
||||
TeamSiteID string
|
||||
}
|
||||
|
||||
func EnvVars(ctx context.Context) Envs {
|
||||
folder := strings.TrimSpace(os.Getenv(sanityTestRestoreContainer))
|
||||
|
||||
e := Envs{
|
||||
BackupID: os.Getenv(sanityBackupID),
|
||||
SourceContainer: os.Getenv(sanityTestSourceContainer),
|
||||
RestoreContainer: folder,
|
||||
Category: os.Getenv(sanityTestCategory),
|
||||
RestoreContainerPrefix: os.Getenv(sanityTestRestoreContainerPrefix),
|
||||
GroupID: tconfig.GetM365TeamID(ctx),
|
||||
SiteID: tconfig.GetM365SiteID(ctx),
|
||||
UserID: tconfig.GetM365UserID(ctx),
|
||||
TeamSiteID: tconfig.GetM365TeamSiteID(ctx),
|
||||
BackupID: os.Getenv(sanityBackupID),
|
||||
SourceContainer: os.Getenv(sanityTestSourceContainer),
|
||||
RestoreContainer: folder,
|
||||
GroupID: tconfig.GetM365TeamID(ctx),
|
||||
SiteID: tconfig.GetM365SiteID(ctx),
|
||||
UserID: tconfig.GetM365UserID(ctx),
|
||||
TeamSiteID: tconfig.GetM365TeamSiteID(ctx),
|
||||
}
|
||||
|
||||
if len(os.Getenv(sanityTestUser)) > 0 {
|
||||
|
||||
@ -20,17 +20,30 @@ func BuildFilepathSanitree(
|
||||
info os.FileInfo,
|
||||
err error,
|
||||
) error {
|
||||
if root == nil {
|
||||
root = CreateNewRoot(info, true)
|
||||
return nil
|
||||
if err != nil {
|
||||
Fatal(ctx, "error passed to filepath walker", err)
|
||||
}
|
||||
|
||||
relPath := GetRelativePath(
|
||||
ctx,
|
||||
rootDir,
|
||||
p,
|
||||
info,
|
||||
err)
|
||||
relPath, err := filepath.Rel(rootDir, p)
|
||||
if err != nil {
|
||||
Fatal(ctx, "getting relative filepath", err)
|
||||
}
|
||||
|
||||
if info != nil {
|
||||
Debugf(ctx, "adding: %s", relPath)
|
||||
}
|
||||
|
||||
if root == nil {
|
||||
root = &Sanitree[fs.FileInfo, fs.FileInfo]{
|
||||
Self: info,
|
||||
ID: info.Name(),
|
||||
Name: info.Name(),
|
||||
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
|
||||
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
elems := path.Split(relPath)
|
||||
node := root.NodeAt(ctx, elems[:len(elems)-1])
|
||||
@ -65,41 +78,3 @@ func BuildFilepathSanitree(
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func CreateNewRoot(info fs.FileInfo, initChildren bool) *Sanitree[fs.FileInfo, fs.FileInfo] {
|
||||
root := &Sanitree[fs.FileInfo, fs.FileInfo]{
|
||||
Self: info,
|
||||
ID: info.Name(),
|
||||
Name: info.Name(),
|
||||
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
|
||||
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
|
||||
}
|
||||
|
||||
if initChildren {
|
||||
root.Children = map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}
|
||||
}
|
||||
|
||||
return root
|
||||
}
|
||||
|
||||
func GetRelativePath(
|
||||
ctx context.Context,
|
||||
rootDir, p string,
|
||||
info fs.FileInfo,
|
||||
fileWalkerErr error,
|
||||
) string {
|
||||
if fileWalkerErr != nil {
|
||||
Fatal(ctx, "error passed to filepath walker", fileWalkerErr)
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(rootDir, p)
|
||||
if err != nil {
|
||||
Fatal(ctx, "getting relative filepath", err)
|
||||
}
|
||||
|
||||
if info != nil {
|
||||
Debugf(ctx, "adding: %s", relPath)
|
||||
}
|
||||
|
||||
return relPath
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
|
||||
@ -3,9 +3,7 @@ package driveish
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/cmd/sanity_test/common"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
@ -17,24 +15,10 @@ const (
|
||||
owner = "owner"
|
||||
)
|
||||
|
||||
// sanitree population will grab a superset of data in the drive.
|
||||
// this increases the chance that we'll run into a race collision with
|
||||
// the cleanup script. Sometimes that's okay (deleting old data that
|
||||
// isn't scrutinized in the test), other times it's not. We mark whether
|
||||
// that's okay to do or not by specifying the folders being
|
||||
// scrutinized for the test. Any errors within those folders should cause
|
||||
// a fatal exit. Errors outside of those folders get ignored.
|
||||
//
|
||||
// since we're using folder names, mustPopulateFolders will
|
||||
// work best (ie: have the fewest collisions/side-effects) if the folder
|
||||
// names are very specific. Standard sanity tests should include timestamps,
|
||||
// which should help ensure that. Be warned if you try to use it with
|
||||
// a more generic name: unintended effects could occur.
|
||||
func populateSanitree(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
driveID string,
|
||||
mustPopulateFolders []string,
|
||||
) *common.Sanitree[models.DriveItemable, models.DriveItemable] {
|
||||
common.Infof(ctx, "building sanitree for drive: %s", driveID)
|
||||
|
||||
@ -43,12 +27,10 @@ func populateSanitree(
|
||||
common.Fatal(ctx, "getting drive root folder", err)
|
||||
}
|
||||
|
||||
rootName := ptr.Val(root.GetName())
|
||||
|
||||
stree := &common.Sanitree[models.DriveItemable, models.DriveItemable]{
|
||||
Self: root,
|
||||
ID: ptr.Val(root.GetId()),
|
||||
Name: rootName,
|
||||
Name: ptr.Val(root.GetName()),
|
||||
Leaves: map[string]*common.Sanileaf[models.DriveItemable, models.DriveItemable]{},
|
||||
Children: map[string]*common.Sanitree[models.DriveItemable, models.DriveItemable]{},
|
||||
}
|
||||
@ -58,8 +40,6 @@ func populateSanitree(
|
||||
ac,
|
||||
driveID,
|
||||
stree.Name+"/",
|
||||
mustPopulateFolders,
|
||||
slices.Contains(mustPopulateFolders, rootName),
|
||||
stree)
|
||||
|
||||
return stree
|
||||
@ -68,29 +48,14 @@ func populateSanitree(
|
||||
func recursivelyBuildTree(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
driveID string,
|
||||
location string,
|
||||
mustPopulateFolders []string,
|
||||
isChildOfFolderRequiringNoErrors bool,
|
||||
driveID, location string,
|
||||
stree *common.Sanitree[models.DriveItemable, models.DriveItemable],
|
||||
) {
|
||||
common.Debugf(ctx, "adding: %s", location)
|
||||
|
||||
children, err := ac.Drives().GetFolderChildren(ctx, driveID, stree.ID)
|
||||
if err != nil {
|
||||
if isChildOfFolderRequiringNoErrors {
|
||||
common.Fatal(ctx, "getting drive children by id", err)
|
||||
}
|
||||
|
||||
common.Infof(
|
||||
ctx,
|
||||
"ignoring error getting children in directory %q because it is not within directory set %v\nerror: %s\n%+v",
|
||||
location,
|
||||
mustPopulateFolders,
|
||||
err.Error(),
|
||||
clues.ToCore(err))
|
||||
|
||||
return
|
||||
common.Fatal(ctx, "getting drive children by id", err)
|
||||
}
|
||||
|
||||
for _, driveItem := range children {
|
||||
@ -103,20 +68,17 @@ func recursivelyBuildTree(
|
||||
// currently we don't restore blank folders.
|
||||
// skip permission check for empty folders
|
||||
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
|
||||
common.Infof(ctx, "skipped empty folder: %s%s", location, itemName)
|
||||
common.Infof(ctx, "skipped empty folder: %s/%s", location, itemName)
|
||||
continue
|
||||
}
|
||||
|
||||
cannotAllowErrors := isChildOfFolderRequiringNoErrors ||
|
||||
slices.Contains(mustPopulateFolders, itemName)
|
||||
|
||||
branch := &common.Sanitree[models.DriveItemable, models.DriveItemable]{
|
||||
Parent: stree,
|
||||
Self: driveItem,
|
||||
ID: itemID,
|
||||
Name: itemName,
|
||||
Expand: map[string]any{
|
||||
expandPermissions: permissionIn(ctx, ac, driveID, itemID, cannotAllowErrors),
|
||||
expandPermissions: permissionIn(ctx, ac, driveID, itemID),
|
||||
},
|
||||
Leaves: map[string]*common.Sanileaf[models.DriveItemable, models.DriveItemable]{},
|
||||
Children: map[string]*common.Sanitree[models.DriveItemable, models.DriveItemable]{},
|
||||
@ -129,8 +91,6 @@ func recursivelyBuildTree(
|
||||
ac,
|
||||
driveID,
|
||||
location+branch.Name+"/",
|
||||
mustPopulateFolders,
|
||||
cannotAllowErrors,
|
||||
branch)
|
||||
}
|
||||
|
||||
|
||||
@ -31,8 +31,7 @@ func CheckExport(
|
||||
root := populateSanitree(
|
||||
ctx,
|
||||
ac,
|
||||
driveID,
|
||||
[]string{envs.SourceContainer})
|
||||
driveID)
|
||||
|
||||
sourceTree, ok := root.Children[envs.SourceContainer]
|
||||
common.Assert(
|
||||
|
||||
@ -45,14 +45,7 @@ func CheckRestoration(
|
||||
"drive_id", driveID,
|
||||
"drive_name", driveName)
|
||||
|
||||
root := populateSanitree(
|
||||
ctx,
|
||||
ac,
|
||||
driveID,
|
||||
[]string{
|
||||
envs.SourceContainer,
|
||||
envs.RestoreContainer,
|
||||
})
|
||||
root := populateSanitree(ctx, ac, driveID)
|
||||
|
||||
sourceTree, ok := root.Children[envs.SourceContainer]
|
||||
common.Assert(
|
||||
@ -92,24 +85,12 @@ func permissionIn(
|
||||
ctx context.Context,
|
||||
ac api.Client,
|
||||
driveID, itemID string,
|
||||
cannotAllowErrors bool,
|
||||
) []common.PermissionInfo {
|
||||
pi := []common.PermissionInfo{}
|
||||
|
||||
pcr, err := ac.Drives().GetItemPermission(ctx, driveID, itemID)
|
||||
if err != nil {
|
||||
if cannotAllowErrors {
|
||||
common.Fatal(ctx, "getting permission", err)
|
||||
}
|
||||
|
||||
common.Infof(
|
||||
ctx,
|
||||
"ignoring error getting permissions for %q\nerror: %s,%+v",
|
||||
itemID,
|
||||
err.Error(),
|
||||
clues.ToCore(err))
|
||||
|
||||
return []common.PermissionInfo{}
|
||||
common.Fatal(ctx, "getting permission", err)
|
||||
}
|
||||
|
||||
for _, perm := range pcr.GetValue() {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user