Compare commits

..

No commits in common. "main" and "v0.17.0" have entirely different histories.

478 changed files with 12404 additions and 47928 deletions

View File

@ -1,5 +1,4 @@
name: Backup Restore Test
description: Run various backup/restore/export tests for a service.
inputs:
service:
@ -19,10 +18,6 @@ inputs:
description: Arguments to pass for restore; restore is skipped when missing.
required: false
default: ""
export-args:
description: Arguments to pass for export.
required: false
default: ""
restore-container:
description: Folder to use for testing
required: true
@ -37,9 +32,6 @@ inputs:
description: Runs export tests when true
required: false
default: false
category:
description: category of data for given service
required: false
outputs:
backup-id:
@ -57,9 +49,7 @@ runs:
echo Backup ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-backup-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-backup-${{inputs.kind }}.log
./corso backup create '${{ inputs.service }}' \
--no-stats --hide-progress --json \
${{ inputs.backup-args }} |
@ -78,9 +68,7 @@ runs:
echo Restore ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
./corso restore '${{ inputs.service }}' \
--no-stats \
--hide-progress \
@ -103,17 +91,11 @@ runs:
SANITY_TEST_RESTORE_CONTAINER: ${{ steps.restore.outputs.result }}
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
# lists are not restored to a different folder. they get created adjacent to their originals
# hence SANITY_TEST_RESTORE_CONTAINER_PREFIX is necessary to differentiate restored from original
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
SANITY_TEST_CATEGORY: ${{ inputs.category }}
run: |
echo "---------------------------"
echo Sanity Test Restore ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
./sanity-test restore ${{ inputs.service }}
- name: Export ${{ inputs.service }} ${{ inputs.kind }}
@ -126,11 +108,9 @@ runs:
echo Export ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
./corso export '${{ inputs.service }}' \
/tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }} \
/tmp/export-${{ inputs.service }}-${{inputs.kind }} \
--no-stats \
--hide-progress \
${{ inputs.export-args }} \
@ -143,19 +123,14 @@ runs:
shell: bash
working-directory: src
env:
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{ inputs.kind }}
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
# applies only for sharepoint lists
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
SANITY_TEST_CATEGORY: ${{ inputs.category }}
run: |
echo "---------------------------"
echo Sanity-Test Export ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
./sanity-test export ${{ inputs.service }}
- name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
@ -168,19 +143,17 @@ runs:
echo Export Archive ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-restore-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-restore-${{inputs.kind }}.log
./corso export '${{ inputs.service }}' \
/tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive \
/tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive \
--no-stats \
--hide-progress \
--archive \
${{ inputs.export-args }} \
--backup '${{ steps.backup.outputs.result }}'
unzip /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-archive/*.zip \
-d /tmp/export-${{ inputs.service }}${CATEGORY_SUFFIX}-${{inputs.kind }}-unzipped
unzip /tmp/export-${{ inputs.service }}-${{inputs.kind }}-archive/*.zip \
-d /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
cat /tmp/corsologs
- name: Check archive export ${{ inputs.service }} ${{ inputs.kind }}
@ -188,19 +161,14 @@ runs:
shell: bash
working-directory: src
env:
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}${{ inputs.category && '-' }}${{ inputs.category }}-${{inputs.kind }}-unzipped
SANITY_TEST_RESTORE_CONTAINER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
SANITY_TEST_SOURCE_CONTAINER: ${{ inputs.restore-container }}
SANITY_BACKUP_ID: ${{ inputs.backup-id }}
# applies only for sharepoint lists
SANITY_TEST_RESTORE_CONTAINER_PREFIX: ${{ steps.restore.outputs.result }}
SANITY_TEST_CATEGORY: ${{ inputs.category }}
run: |
echo "---------------------------"
echo Sanity-Test Export Archive ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}${CATEGORY_SUFFIX}-validate-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-${{ inputs.service }}-validate-${{inputs.kind }}.log
./sanity-test export ${{ inputs.service }}
- name: List ${{ inputs.service }} ${{ inputs.kind }}
@ -211,9 +179,7 @@ runs:
echo Backup list ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}${CATEGORY_SUFFIX}-list-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}-list-${{inputs.kind }}.log
./corso backup list ${{ inputs.service }} \
--no-stats \
--hide-progress \
@ -234,10 +200,7 @@ runs:
echo Backup List w/ Backup ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
# Include category in the log file name if present
CATEGORY_SUFFIX=""
[[ -n "${{ inputs.category }}" ]] && CATEGORY_SUFFIX="-${{ inputs.category }}"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}${CATEGORY_SUFFIX}-single-${{inputs.kind }}.log
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}-single-${{inputs.kind }}.log
./corso backup list ${{ inputs.service }} \
--no-stats \
--hide-progress \

View File

@ -1,5 +1,4 @@
name: Setup and Cache Golang
description: Build golang binaries for later use in CI.
# clone of: https://github.com/magnetikonline/action-golang-cache/blob/main/action.yaml
#

View File

@ -1,5 +1,4 @@
name: Publish Binary
description: Publish binary artifacts.
inputs:
version:

View File

@ -1,5 +1,4 @@
name: Publish Website
description: Publish website artifacts.
inputs:
aws-iam-role:

View File

@ -1,5 +1,4 @@
name: Purge M365 User Data
description: Deletes M365 data generated during CI tests.
# Hard deletion of an m365 user's data. Our CI processes create a lot
# of data churn (creation and immediate deletion) of files, the likes
@ -31,19 +30,12 @@ inputs:
description: Secret value of for AZURE_CLIENT_ID
azure-client-secret:
description: Secret value of for AZURE_CLIENT_SECRET
azure-pnp-client-id:
description: Secret value of AZURE_PNP_CLIENT_ID
azure-pnp-client-cert:
description: Base64 encoded private certificate for the azure-pnp-client-id (Secret value of AZURE_PNP_CLIENT_CERT)
azure-tenant-id:
description: Secret value of AZURE_TENANT_ID
description: Secret value of for AZURE_TENANT_ID
m365-admin-user:
description: Secret value of for M365_TENANT_ADMIN_USER
m365-admin-password:
description: Secret value of for M365_TENANT_ADMIN_PASSWORD
tenant-domain:
description: The domain of the tenant (ex. 10rqc2.onmicrosft.com)
required: true
runs:
using: composite
@ -61,13 +53,7 @@ runs:
AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
run: |
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
{
if (./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
break
}
}
run: ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
# TODO(ashmrtn): Re-enable when we figure out errors we're seeing with Get-Mailbox call.
#- name: Reset retention for all mailboxes to 0
@ -88,16 +74,10 @@ runs:
shell: pwsh
working-directory: ./src/cmd/purge/scripts
env:
AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
TENANT_DOMAIN: ${{ inputs.tenant-domain }}
M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
run: |
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
{
if (./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
break
}
}
./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
################################################################################################################
# Sharepoint
@ -108,14 +88,6 @@ runs:
shell: pwsh
working-directory: ./src/cmd/purge/scripts
env:
AZURE_CLIENT_ID: ${{ inputs.azure-pnp-client-id }}
AZURE_APP_CERT: ${{ inputs.azure-pnp-client-cert }}
TENANT_DOMAIN: ${{ inputs.tenant-domain }}
run: |
for ($ATTEMPT_NUM = 1; $ATTEMPT_NUM -le 3; $ATTEMPT_NUM++)
{
if (./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }}) {
break
}
}
M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
run: ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -LibraryPrefixDeleteList ${{ inputs.library-prefix && inputs.library-prefix || '[]' }} -PurgeBeforeTimestamp ${{ inputs.older-than }}

View File

@ -1,5 +1,4 @@
name: Send a message to Teams
description: Send messages to communication apps.
inputs:
msg:

View File

@ -1,5 +1,4 @@
name: Lint Website
description: Lint website content.
inputs:
version:

View File

@ -28,7 +28,7 @@ jobs:
# only run CI tests if the src folder or workflow actions have changed
- name: Check for file changes in src/ or .github/workflows/
uses: dorny/paths-filter@v3
uses: dorny/paths-filter@v2
id: dornycheck
with:
list-files: json

View File

@ -40,5 +40,5 @@ jobs:
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[CORSO FAILED] Publishing Binary"
msg: "[FAILED] Publishing Binary"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

View File

@ -189,7 +189,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ci-test-log
path: src/testlog/*
@ -260,7 +260,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ci-retention-test-log
path: src/testlog/*
@ -315,7 +315,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: unit-test-log
path: src/testlog/*
@ -404,7 +404,7 @@ jobs:
# Upload the original go test log as an artifact for later review.
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: fork-test-log
path: src/testlog/*
@ -463,7 +463,7 @@ jobs:
go-version-file: src/go.mod
- name: Go Lint
uses: golangci/golangci-lint-action@v4
uses: golangci/golangci-lint-action@v3
with:
# Keep pinned to a verson as sometimes updates will add new lint
# failures in unchanged code.
@ -492,8 +492,8 @@ jobs:
# I could not find a way to install tree-grepper without nix
# https://github.com/BrianHicks/tree-grepper/issues/293
- uses: cachix/install-nix-action@v25
- uses: cachix/cachix-action@v14
- uses: cachix/install-nix-action@v24
- uses: cachix/cachix-action@v13
with:
name: tree-grepper
- run: nix-env -if https://github.com/BrianHicks/tree-grepper/archive/refs/heads/main.tar.gz
@ -511,27 +511,6 @@ jobs:
echo "Use len check instead of empty string comparison"
exit 1
fi
- name: Check for cases where errors are not propagated
run: |
# Using `grep .` as the exit codes are always true for correct grammar
if tree-grepper -q go '((if_statement (binary_expression) @_if (block (return_statement (expression_list (call_expression (selector_expression) @_fun ) @ret .)))) (#match? @_if "err != nil") (#match? @_fun "clues.NewWC"))' | grep .; then
echo "Make sure to propagate errors with clues"
exit 1
fi
- name: Check if clues without context are used when context is passed in
run: |
# Using `grep .` as the exit codes are always true for correct grammar
if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue))) . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#match? @clue "WC$"))' | grep .; then
echo "Do not use clues.*WC when context is passed in"
exit 1
fi
- name: Check clues with context is used when context is not passed in
run: |
# Using `grep .` as the exit codes are always true for correct grammar
if tree-grepper -q go '((function_declaration (parameter_list . (parameter_declaration (identifier) @_octx)) body: (block (short_var_declaration left: (expression_list (identifier) @_err . ) right: (expression_list (call_expression (argument_list . (identifier) @_ctx)))) . (if_statement (binary_expression) @_exp consequence: (block (return_statement (expression_list (call_expression (selector_expression (call_expression (selector_expression) @clue))) . )))))) (#eq? @_err "err") (#eq? @_octx "ctx") (#not-eq? @_ctx "ctx") (#eq? @_exp "err != nil") (#match? @clue "^clues\.") (#not-match? @clue "WC$"))' | grep .; then
echo "Use clues.*WC when context is not passed in"
exit 1
fi
# ----------------------------------------------------------------------------------------------------
# --- GitHub Actions Linting -------------------------------------------------------------------------

View File

@ -12,7 +12,7 @@ jobs:
continue-on-error: true
strategy:
matrix:
user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, ""]
user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, '' ]
steps:
- uses: actions/checkout@v4
@ -33,15 +33,12 @@ jobs:
azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
tenant-domain: ${{ vars.TENANT_DOMAIN }}
- name: Notify failure in teams
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[CORSO FAILED] ${{ vars[matrix.user] }} CI Cleanup"
msg: "[FAILED] ${{ vars[matrix.user] }} CI Cleanup"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
Test-Site-Data-Cleanup:
@ -73,13 +70,10 @@ jobs:
azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
tenant-domain: ${{ vars.TENANT_DOMAIN }}
- name: Notify failure in teams
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[CORSO FAILED] ${{ vars[matrix.site] }} CI Cleanup"
msg: "[FAILED] ${{ vars[matrix.site] }} CI Cleanup"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

View File

@ -107,7 +107,7 @@ jobs:
# package all artifacts for later review
- name: Upload Log, Profilers, Traces
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: load-test-profiling
path: ${{ github.workspace }}/testlog/*
@ -155,6 +155,3 @@ jobs:
azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
tenant-domain: ${{ vars.TENANT_DOMAIN }}

View File

@ -6,7 +6,7 @@ on:
workflow_dispatch:
inputs:
user:
description: "User to run longevity test on"
description: 'User to run longevity test on'
permissions:
# required to retrieve AWS credentials
@ -37,7 +37,7 @@ jobs:
CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-longevity.log
RESTORE_DEST_PFX: Corso_Test_Longevity_
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
PREFIX: "longevity"
PREFIX: 'longevity'
# Options for retention.
RETENTION_MODE: GOVERNANCE
@ -113,6 +113,7 @@ jobs:
--extend-retention \
--prefix ${{ env.PREFIX }} \
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
--succeed-if-exists \
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
if grep -q 'Failed to' ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
@ -381,7 +382,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: longevity-test-log
path: src/testlog/*
@ -392,5 +393,5 @@ jobs:
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[CORSO FAILED] Longevity Test"
msg: "[FAILED] Longevity Test"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

View File

@ -107,7 +107,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: nightly-test-log
path: src/testlog/*
@ -118,5 +118,5 @@ jobs:
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[COROS FAILED] Nightly Checks"
msg: "[FAILED] Nightly Checks"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

View File

@ -19,7 +19,7 @@ jobs:
private_key: ${{ secrets.PRIVATE_KEY }}
- name: Slash Command Dispatch
uses: peter-evans/slash-command-dispatch@v4
uses: peter-evans/slash-command-dispatch@v3
env:
TOKEN: ${{ steps.generate_token.outputs.token }}
with:

View File

@ -6,7 +6,7 @@ on:
workflow_dispatch:
inputs:
user:
description: "User to run sanity test on"
description: 'User to run sanity test on'
permissions:
# required to retrieve AWS credentials
@ -48,6 +48,7 @@ jobs:
# setup
steps:
- uses: actions/checkout@v4
- name: Setup Golang with cache
@ -90,9 +91,6 @@ jobs:
azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
tenant-domain: ${{ vars.TENANT_DOMAIN }}
- name: Purge CI-Produced Folders for Sites
timeout-minutes: 30
@ -108,9 +106,6 @@ jobs:
azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
azure-pnp-client-id: ${{ secrets.AZURE_PNP_CLIENT_ID }}
azure-pnp-client-cert: ${{ secrets.AZURE_PNP_CLIENT_CERT }}
tenant-domain: ${{ vars.TENANT_DOMAIN }}
##########################################################################################################################################
@ -198,8 +193,8 @@ jobs:
service: exchange
kind: first-backup
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -211,8 +206,8 @@ jobs:
service: exchange
kind: incremental
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -225,8 +220,8 @@ jobs:
service: exchange
kind: non-delta
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email" --disable-delta'
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -239,12 +234,13 @@ jobs:
service: exchange
kind: non-delta-incremental
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: "--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}"
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
##########################################################################################################################################
# Onedrive
@ -274,8 +270,8 @@ jobs:
service: onedrive
kind: first-backup
backup-args: '--user "${{ env.TEST_USER }}"'
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -299,14 +295,14 @@ jobs:
service: onedrive
kind: incremental
backup-args: '--user "${{ env.TEST_USER }}"'
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}"
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
##########################################################################################################################################
# Sharepoint Library
# Sharepoint
# generate new entries for test
- name: SharePoint - Create new data
@ -333,12 +329,11 @@ jobs:
with:
service: sharepoint
kind: first-backup
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"'
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
category: libraries
# generate some more enteries for incremental check
- name: SharePoint - Create new data (for incremental)
@ -360,103 +355,11 @@ jobs:
with:
service: sharepoint
kind: incremental
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data libraries'
restore-args: "--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}"
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}"'
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
category: libraries
##########################################################################################################################################
# Sharepoint Lists
# generate new entries for test
# The `awk | tr | sed` command chain is used to get a comma separated list of SharePoint list names.
- name: SharePoint Lists - Create new data
id: new-data-creation-sharepoint-lists
timeout-minutes: 30
working-directory: ./src/cmd/factory
run: |
suffix=$(date +"%Y-%m-%d_%H-%M-%S")
go run . sharepoint lists \
--site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
--user ${{ env.TEST_USER }} \
--secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${{ secrets.TENANT_ID }} \
--destination ${{ env.RESTORE_DEST_PFX }}$suffix \
--count 4 |
awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
tee $GITHUB_OUTPUT
# Extracts the common prefix for the Sharepoint list names.
- name: SharePoint Lists - Store restore container
id: sharepoint-lists-store-restore-container
run: |
echo ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} |
cut -d',' -f1 |
cut -d'_' -f1,2,3,4,5 |
sed -e 's/^/result=/' |
tee $GITHUB_OUTPUT
- name: SharePoint Lists - Backup
id: sharepoint-lists-backup
timeout-minutes: 30
uses: ./.github/actions/backup-restore-test
with:
service: sharepoint
kind: first-backup-lists
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
restore-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
export-args: "--list ${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
restore-container: "${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
category: lists
on-collision: copy
# generate some more enteries for incremental check
- name: SharePoint Lists - Create new data (for incremental)
id: inc-data-creation-sharepoint-lists
timeout-minutes: 30
working-directory: ./src/cmd/factory
run: |
suffix=$(date +"%Y-%m-%d_%H-%M-%S")
go run . sharepoint lists \
--site ${{ vars.CORSO_M365_TEST_SITE_URL }} \
--user ${{ env.TEST_USER }} \
--secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${{ secrets.TENANT_ID }} \
--destination ${{ env.RESTORE_DEST_PFX }}$suffix \
--count 4 |
awk 'NR > 1 {print $2}' | tr '\n' ',' | sed -e 's/,$//' -e 's/^/result=/' |
tee $GITHUB_OUTPUT
- name: SharePoint Lists - Store restore container (for incremental)
id: sharepoint-lists-store-restore-container-inc
run: |
echo ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }} |
cut -d',' -f1 |
cut -d'_' -f1,2,3,4,5 |
sed -e 's/^/result=/' |
tee $GITHUB_OUTPUT
- name: SharePoint Lists - Incremental backup
id: sharepoint-lists-incremental
timeout-minutes: 30
uses: ./.github/actions/backup-restore-test
with:
service: sharepoint
kind: incremental-lists
backup-args: '--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" --data lists'
restore-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }} --destination Corso_Test_Sanity_Restore_$(date +'%Y%m%d_%H%M%S')"
export-args: "--list ${{ steps.inc-data-creation-sharepoint-lists.outputs.result }},${{ steps.new-data-creation-sharepoint-lists.outputs.result }}"
restore-container: "${{ steps.sharepoint-lists-store-restore-container-inc.outputs.result }},${{ steps.sharepoint-lists-store-restore-container.outputs.result }}"
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
category: lists
on-collision: copy
##########################################################################################################################################
@ -487,8 +390,8 @@ jobs:
with:
service: groups
kind: first-backup
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}"'
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -512,9 +415,9 @@ jobs:
with:
service: groups
kind: incremental
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}" --data messages,libraries'
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}"'
restore-args: '--site "${{ vars.CORSO_M365_TEST_GROUPS_SITE_URL }}" --folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
restore-container: "${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}"
restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
with-export: true
@ -525,7 +428,7 @@ jobs:
# Upload the original go test output as an artifact for later review.
- name: Upload test log
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: sanity-test-log
path: ${{ env.CORSO_LOG_DIR }}/*
@ -536,5 +439,5 @@ jobs:
if: failure()
uses: ./.github/actions/teams-message
with:
msg: "[CORSO FAILED] Sanity Tests"
msg: "[FAILED] Sanity Tests"
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}

View File

@ -6,71 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased] (beta)
### Fixed
- Handle the case where an email or event cannot be retrieved from Exchange due to an `ErrorCorruptData` error. Corso will skip over the item but report it in the backup summary.
- Emails attached within other emails are now correctly exported
- Gracefully handle email and post attachments without name when exporting to eml
- Use correct timezone for event start and end times in Exchange exports (helps fix issues in relative recurrence patterns)
- Fixed an issue causing exports dealing with calendar data to have high memory usage
## [v0.19.0] (beta) - 2024-02-06
### Added
- Events can now be exported from Exchange backups as .ics files.
- Update repo init configuration to reduce the total number of GET requests sent
to the object store when using corso. This affects repos that have many
backups created in them per day the most.
- Feature Preview: Corso now supports backup, export & restore of SharePoint lists. Lists backup can be initiated using `corso backup create sharepoint --site <site-url> --data lists`.
- Group mailbox(aka conversations) backup and export support is now officially available. Group mailbox posts can be exported as `.eml` files.
### Fixed
- Retry transient 400 "invalidRequest" errors during onedrive & sharepoint backup.
- Backup attachments associated with group mailbox items.
- Groups and Teams backups no longer fail when a resource has no display name.
- Contacts in-place restore failed if the restore destination was empty.
- Link shares with external users are now backed up and restored as expected
- Ensure persistent repo config is populated on repo init if repo init failed partway through during the previous init attempt.
### Changed
- When running `backup details` on an empty backup returns a more helpful error message.
- Backup List additionally shows the data category for each backup.
- Remove hidden `--succeed-if-exists` flag for repo init. Repo init will now succeed without error if run on an existing repo with the same passphrase.
### Known issues
- Backing up a group mailbox item may fail if it has a very large number of attachments (500+).
- Event description for exchange exports might look slightly different for certain events.
- Exchange in-place restore may restore items in well-known folders to different folders if the user has well-known folder names change based on locale and has updated the locale since the backup was created.
- In-place Exchange contacts restore will merge items in folders named "Contacts" or "contacts" into the default folder.
- External users with access through shared links will not receive these links as they are not sent via email during restore.
- Graph API has limited support for certain column types such as `location`, `hyperlink/picture`, and `metadata`. Restoring SharePoint list items containing these columns will result in differences compared to the original items.
- SharePoint list item attachments are not available due to graph API limitations.
- Group mailbox restore is not supported due to limited Graph API support for creating mailbox items.
- Due to Graph API limitations, any group mailbox items present in subfolders other than Inbox aren't backed up.
## [v0.18.0] (beta) - 2024-01-02
### Fixed
- Handle the case where an email cannot be retrieved from Exchange due to an `ErrorInvalidRecipients` error. In
this case, Corso will skip over the item but report this in the backup summary.
- Fix `ErrorItemNotFound` errors when restoring emails with multiple attachments.
- Avoid Graph SDK `Requests must contain extension changes exclusively.` errors by removing server-populated field from restored event items.
- Improve Group mailbox(conversations) backup performance by only downloading new items or items with modified content.
- Handle cases where Exchange backup stored invalid JSON blobs if there were special characters in the user content. These would result in errors during restore.
### Known issues
- Restoring OneDrive, SharePoint, or Teams & Groups items shared with external users while the tenant or site is configured to not allow sharing with external users will not restore permissions.
### Added
- Contacts can now be exported from Exchange backups as .vcf files
## [v0.17.0] (beta) - 2023-12-11
### Changed
- Memory optimizations for large scale OneDrive and Sharepoint backups.
### Fixed
- Resolved a possible deadlock when backing up Teams Channel Messages.
- Fixed an attachment download failure(ErrorTooManyObjectsOpened) during exchange backup.
## [v0.16.0] (beta) - 2023-11-28
@ -502,11 +445,7 @@ this case, Corso will skip over the item but report this in the backup summary.
- Miscellaneous
- Optional usage statistics reporting ([RM-35](https://github.com/alcionai/corso-roadmap/issues/35))
[Unreleased]: https://github.com/alcionai/corso/compare/v0.19.0...HEAD
[v0.19.0]: https://github.com/alcionai/corso/compare/v0.18.0...v0.19.0
[v0.18.0]: https://github.com/alcionai/corso/compare/v0.17.0...v0.18.0
[v0.17.0]: https://github.com/alcionai/corso/compare/v0.16.0...v0.17.0
[v0.16.0]: https://github.com/alcionai/corso/compare/v0.15.0...v0.16.0
[Unreleased]: https://github.com/alcionai/corso/compare/v0.15.0...HEAD
[v0.15.0]: https://github.com/alcionai/corso/compare/v0.14.0...v0.15.0
[v0.14.0]: https://github.com/alcionai/corso/compare/v0.13.0...v0.14.0
[v0.13.0]: https://github.com/alcionai/corso/compare/v0.12.0...v0.13.0

View File

@ -1,6 +1,3 @@
> [!NOTE]
> **The Corso project is no longer actively maintained and has been archived**.
<p align="center">
<img src="https://github.com/alcionai/corso/blob/main/website/static/img/corso_logo.svg?raw=true" alt="Corso Logo" width="100" />
</p>

View File

@ -4,7 +4,6 @@ run:
linters:
enable:
- errcheck
- exhaustive
- forbidigo
- gci
- gofmt
@ -26,11 +25,6 @@ linters:
- staticcheck
linters-settings:
exhaustive:
check:
- switch
default-signifies-exhaustive: false
explicit-exhaustive-switch: true
gci:
sections:
- standard
@ -55,7 +49,7 @@ linters-settings:
# String formatting should be avoided in favor of structured errors (ie: err.With(k, v)).
- '(errors|fmt)\.(New|Stack|Wrap|Error)f?\((# error handling should use clues pkg)?'
# Avoid Warn-level logging in favor of Info or Error.
- 'Warnw?f?\((# logging should use Info or Error)?'
- 'Warn[wf]?\((# logging should use Info or Error)?'
# Prefer suite.Run(name, func() {}) for subtests as testify has it instead
# of suite.T().Run(name, func(t *testing.T) {}).
- '(T\(\)|\st[a-zA-Z0-9]*)\.Run(# prefer testify suite.Run(name, func()) )?'

View File

@ -2,6 +2,7 @@ package backup
import (
"context"
"encoding/json"
"fmt"
"strings"
@ -19,16 +20,14 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/errs/core"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
"github.com/alcionai/corso/src/pkg/store"
)
var ErrEmptyBackup = clues.New("no items in backup")
// ---------------------------------------------------------------------------
// adding commands to cobra
// ---------------------------------------------------------------------------
@ -45,7 +44,6 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{
addOneDriveCommands,
addSharePointCommands,
addGroupsCommands,
addTeamsChatsCommands,
}
// AddCommands attaches all `corso backup * *` commands to the parent.
@ -190,47 +188,45 @@ func genericCreateCommand(
ictx = clues.Add(ctx, "resource_owner_selected", owner)
)
logger.Ctx(ictx).Infof("setting up backup")
bo, err := r.NewBackupWithLookup(ictx, discSel, ins)
if err != nil {
cerr := clues.WrapWC(ictx, err, owner)
errs = append(errs, cerr)
Errf(
ictx,
"%s\nCause: %s",
"Unable to initiate backup",
err.Error())
meta, err := json.Marshal(cerr.Core().Values)
if err != nil {
meta = []byte("Unable to marshal error metadata")
}
Errf(ictx, "%s\nMessage: %v\nMetadata:%s", "Unable to complete backup", err, meta)
continue
}
ictx = clues.Add(
ictx,
ctx,
"resource_owner_id", bo.ResourceOwner.ID(),
"resource_owner_name", clues.Hide(bo.ResourceOwner.Name()))
logger.Ctx(ictx).Infof("running backup")
"resource_owner_name", bo.ResourceOwner.Name())
err = bo.Run(ictx)
if err != nil {
if errors.Is(err, core.ErrServiceNotEnabled) {
logger.Ctx(ictx).Infow("service not enabled",
if errors.Is(err, graph.ErrServiceNotEnabled) {
logger.Ctx(ctx).Infow("service not enabled",
"resource_owner_id", bo.ResourceOwner.ID(),
"service", serviceName)
continue
}
cerr := clues.Wrap(err, owner)
cerr := clues.WrapWC(ictx, err, owner)
errs = append(errs, cerr)
Errf(
ictx,
"%s\nCause: %s",
"Unable to complete backup",
err.Error())
meta, err := json.Marshal(cerr.Core().Values)
if err != nil {
meta = []byte("Unable to marshal error metadata")
}
Errf(ictx, "%s\nMessage: %v\nMetadata:%s", "Unable to complete backup", err, meta)
continue
}
@ -238,10 +234,10 @@ func genericCreateCommand(
bIDs = append(bIDs, string(bo.Results.BackupID))
if !DisplayJSONFormat() {
Infof(ictx, fmt.Sprintf("Backup complete %s %s", observe.Bullet, color.BlueOutput(bo.Results.BackupID)))
printBackupStats(ictx, r, string(bo.Results.BackupID))
Infof(ctx, fmt.Sprintf("Backup complete %s %s", observe.Bullet, color.BlueOutput(bo.Results.BackupID)))
printBackupStats(ctx, r, string(bo.Results.BackupID))
} else {
Infof(ictx, "Backup complete - ID: %v\n", bo.Results.BackupID)
Infof(ctx, "Backup complete - ID: %v\n", bo.Results.BackupID)
}
}
@ -338,7 +334,7 @@ func genericListCommand(
fe.PrintItems(
ctx,
!ifShow(flags.ListAlertsFV),
!ifShow(flags.FailedItemsFV),
!ifShow(flags.ListFailedItemsFV),
!ifShow(flags.ListSkippedItemsFV),
!ifShow(flags.ListRecoveredErrorsFV))
@ -398,10 +394,6 @@ func genericDetailsCore(
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
if len(d.Entries) == 0 {
return nil, ErrEmptyBackup
}
if opts.SkipReduce {
return d, nil
}

View File

@ -5,12 +5,10 @@ import (
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
@ -68,30 +66,3 @@ func (suite *BackupUnitSuite) TestGenericDetailsCore() {
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, expected, output.Entries)
}
func (suite *BackupUnitSuite) TestGenericDetailsCore_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
bg := testdata.VersionedBackupGetter{
Details: &details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.Entry{},
},
},
}
sel := selectors.NewExchangeBackup([]string{"user-id"})
sel.Include(sel.AllData())
_, err := genericDetailsCore(
ctx,
bg,
"backup-ID",
sel.Selector,
control.DefaultOptions())
require.Error(t, err, "has error")
assert.ErrorIs(t, err, ErrEmptyBackup, clues.ToCore(err))
}

View File

@ -3,6 +3,7 @@ package backup
import (
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
@ -61,11 +62,15 @@ corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \
// called by backup.go to map subcommands to provider-specific handling.
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case createCommand:
c, _ = utils.AddCommand(cmd, exchangeCreateCmd())
c, fs = utils.AddCommand(cmd, exchangeCreateCmd())
fs.SortFlags = false
c.Use = c.Use + " " + exchangeServiceCommandCreateUseSuffix
c.Example = exchangeServiceCommandCreateExamples
@ -82,13 +87,15 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
flags.AddDisableSlidingWindowLimiterFlag(c)
case listCommand:
c, _ = utils.AddCommand(cmd, exchangeListCmd())
c, fs = utils.AddCommand(cmd, exchangeListCmd())
fs.SortFlags = false
flags.AddBackupIDFlag(c, false)
flags.AddAllBackupListFlags(c)
case detailsCommand:
c, _ = utils.AddCommand(cmd, exchangeDetailsCmd())
c, fs = utils.AddCommand(cmd, exchangeDetailsCmd())
fs.SortFlags = false
c.Use = c.Use + " " + exchangeServiceCommandDetailsUseSuffix
c.Example = exchangeServiceCommandDetailsExamples
@ -101,7 +108,8 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
flags.AddExchangeDetailsAndRestoreFlags(c, false)
case deleteCommand:
c, _ = utils.AddCommand(cmd, exchangeDeleteCmd())
c, fs = utils.AddCommand(cmd, exchangeDeleteCmd())
fs.SortFlags = false
c.Use = c.Use + " " + exchangeServiceCommandDeleteUseSuffix
c.Example = exchangeServiceCommandDeleteExamples

View File

@ -12,15 +12,14 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/its"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
@ -40,7 +39,7 @@ var (
type NoBackupExchangeE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestNoBackupExchangeE2ESuite(t *testing.T) {
@ -55,7 +54,7 @@ func (suite *NoBackupExchangeE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
}
@ -70,7 +69,7 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_noBackups() {
cmd := cliTD.StubRootCmd(
"backup", "list", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
@ -94,7 +93,7 @@ func (suite *NoBackupExchangeE2ESuite) TestExchangeBackupListCmd_noBackups() {
type BackupExchangeE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestBackupExchangeE2ESuite(t *testing.T) {
@ -109,7 +108,7 @@ func (suite *BackupExchangeE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
}
@ -139,7 +138,7 @@ func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category path.
cmd, ctx := buildExchangeBackupCmd(
ctx,
suite.dpnd.configFilePath,
suite.m365.User.ID,
suite.its.user.ID,
category.String(),
&recorder)
@ -150,11 +149,8 @@ func runExchangeBackupCategoryTest(suite *BackupExchangeE2ESuite, category path.
result := recorder.String()
t.Log("backup results", result)
// As an offhand check: the result should contain the m365 user's email.
assert.Contains(
t,
strings.ToLower(result),
strings.ToLower(suite.m365.User.Provider.Name()))
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.its.user.ID)
}
func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_ServiceNotEnabled_email() {
@ -177,7 +173,7 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
cmd, ctx := buildExchangeBackupCmd(
ctx,
suite.dpnd.configFilePath,
fmt.Sprintf("%s,%s", tconfig.UnlicensedM365UserID(suite.T()), suite.m365.User.ID),
fmt.Sprintf("%s,%s", tconfig.UnlicensedM365UserID(suite.T()), suite.its.user.ID),
category.String(),
&recorder)
err := cmd.ExecuteContext(ctx)
@ -186,11 +182,8 @@ func runExchangeBackupServiceNotEnabledTest(suite *BackupExchangeE2ESuite, categ
result := recorder.String()
t.Log("backup results", result)
// As an offhand check: the result should contain the m365 user's email.
assert.Contains(
t,
strings.ToLower(result),
strings.ToLower(suite.m365.User.Provider.Name()))
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.its.user.ID)
}
func (suite *BackupExchangeE2ESuite) TestExchangeBackupCmd_userNotFound_email() {
@ -229,8 +222,7 @@ func runExchangeBackupUserNotFoundTest(suite *BackupExchangeE2ESuite, category p
assert.Contains(
t,
err.Error(),
"not found",
"error missing user not found")
"not found in tenant", "error missing user not found")
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
t.Logf("backup error message: %s", err.Error())
@ -249,7 +241,7 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAzureClientIDFl
cmd := cliTD.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365.User.ID,
"--user", suite.its.user.ID,
"--azure-client-id", "invalid-value")
cli.BuildCommandTree(cmd)
@ -273,8 +265,8 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_fromConfigFile() {
cmd := cliTD.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365.User.ID,
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--user", suite.its.user.ID,
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -288,11 +280,8 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_fromConfigFile() {
result := suite.dpnd.recorder.String()
t.Log("backup results", result)
// As an offhand check: the result should contain the m365 user's email.
assert.Contains(
t,
strings.ToLower(result),
strings.ToLower(suite.m365.User.Provider.Name()))
// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.its.user.ID)
}
// AWS flags
@ -306,7 +295,7 @@ func (suite *BackupExchangeE2ESuite) TestBackupCreateExchange_badAWSFlags() {
cmd := cliTD.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365.User.ID,
"--user", suite.its.user.ID,
"--aws-access-key", "invalid-value",
"--aws-secret-access-key", "some-invalid-value")
cli.BuildCommandTree(cmd)
@ -329,7 +318,7 @@ type PreparedBackupExchangeE2ESuite struct {
tester.Suite
dpnd dependencies
backupOps map[path.CategoryType]string
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestPreparedBackupExchangeE2ESuite(t *testing.T) {
@ -346,13 +335,13 @@ func (suite *PreparedBackupExchangeE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.ExchangeService)
suite.backupOps = make(map[path.CategoryType]string)
var (
users = []string{suite.m365.User.ID}
ins = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
users = []string{suite.its.user.ID}
ins = idname.NewCache(map[string]string{suite.its.user.ID: suite.its.user.ID})
)
for _, set := range []path.CategoryType{email, contacts, events} {
@ -420,7 +409,7 @@ func runExchangeListCmdTest(suite *PreparedBackupExchangeE2ESuite, category path
cmd := cliTD.StubRootCmd(
"backup", "list", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -461,7 +450,7 @@ func runExchangeListSingleCmdTest(suite *PreparedBackupExchangeE2ESuite, categor
cmd := cliTD.StubRootCmd(
"backup", "list", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(bID))
cli.BuildCommandTree(cmd)
@ -488,7 +477,7 @@ func (suite *PreparedBackupExchangeE2ESuite) TestExchangeListCmd_badID() {
cmd := cliTD.StubRootCmd(
"backup", "list", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", "smarfs")
cli.BuildCommandTree(cmd)
@ -530,7 +519,7 @@ func runExchangeDetailsCmdTest(suite *PreparedBackupExchangeE2ESuite, category p
cmd := cliTD.StubRootCmd(
"backup", "details", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN, string(bID))
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -620,7 +609,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN,
fmt.Sprintf("%s,%s",
string(suite.backupOps[0].Results.BackupID),
@ -634,7 +623,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[0].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -644,7 +633,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd() {
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[1].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -662,7 +651,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_SingleID(
cmd := cliTD.StubRootCmd(
"backup", "delete", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN,
string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -674,7 +663,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_SingleID(
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -692,7 +681,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_UnknownID
cmd := cliTD.StubRootCmd(
"backup", "delete", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN, uuid.NewString())
cli.BuildCommandTree(cmd)
@ -711,7 +700,7 @@ func (suite *BackupDeleteExchangeE2ESuite) TestExchangeBackupDeleteCmd_NoBackupI
cmd := cliTD.StubRootCmd(
"backup", "delete", "exchange",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
// empty backupIDs should error since no data provided

View File

@ -116,20 +116,8 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
opts := utils.MakeExchangeOpts(cmd)
co := utils.Control()
backupOpts := utils.ParseBackupOptions()
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
// restore flags are switched over too and we no longer parse flags beyond
// connection info into control.Options.
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(backupOpts.M365.DeltaPageSize)))
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
assert.True(t, backupOpts.M365.ExchangeImmutableIDs)
assert.True(t, backupOpts.ServiceRateLimiter.DisableSlidingWindowLimiter)
assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(co.DeltaPageSize)))
assert.Equal(t, control.FailFast, co.FailureHandling)
@ -138,8 +126,6 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
assert.True(t, co.ToggleFeatures.DisableDelta)
assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs)
assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter)
assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
flagsTD.AssertGenericBackupFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)

View File

@ -6,6 +6,7 @@ import (
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/flags"
@ -35,12 +36,9 @@ const (
groupsServiceCommandCreateExamples = `# Backup all Groups and Teams data for the Marketing group
corso backup create groups --group Marketing
# Backup only Teams channel messages
# Backup only Teams conversations messages
corso backup create groups --group Marketing --data messages
# Backup only group mailbox posts
corso backup create groups --group Marketing --data conversations
# Backup all Groups and Teams data for all groups
corso backup create groups --group '*'`
@ -53,19 +51,20 @@ corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd
# Explore Marketing messages posted after the start of 2022
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd \
--last-message-reply-after 2022-01-01T00:00:00
# Explore group mailbox posts with conversation subject "hello world"
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"`
--last-message-reply-after 2022-01-01T00:00:00`
)
// called by backup.go to map subcommands to provider-specific handling.
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case createCommand:
c, _ = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsCreateCmd(), utils.MarkPreviewCommand())
fs.SortFlags = false
c.Use = c.Use + " " + groupsServiceCommandCreateUseSuffix
c.Example = groupsServiceCommandCreateExamples
@ -76,16 +75,17 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
flags.AddFetchParallelismFlag(c)
flags.AddDisableDeltaFlag(c)
flags.AddGenericBackupFlags(c)
flags.AddDisableLazyItemReader(c)
case listCommand:
c, _ = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsListCmd(), utils.MarkPreviewCommand())
fs.SortFlags = false
flags.AddBackupIDFlag(c, false)
flags.AddAllBackupListFlags(c)
case detailsCommand:
c, _ = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsDetailsCmd(), utils.MarkPreviewCommand())
fs.SortFlags = false
c.Use = c.Use + " " + groupsServiceCommandDetailsUseSuffix
c.Example = groupsServiceCommandDetailsExamples
@ -99,7 +99,8 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSharePointDetailsAndRestoreFlags(c)
case deleteCommand:
c, _ = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsDeleteCmd(), utils.MarkPreviewCommand())
fs.SortFlags = false
c.Use = c.Use + " " + groupsServiceCommandDeleteUseSuffix
c.Example = groupsServiceCommandDeleteExamples
@ -160,7 +161,7 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Stack(err))
}
ins, err := svcCli.AC.Groups().GetAllIDsAndNames(ctx, errs)
ins, err := svcCli.GroupsMap(ctx, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 groups"))
}
@ -316,7 +317,7 @@ func groupsBackupCreateSelectors(
group, cats []string,
) *selectors.GroupsBackup {
if filters.PathContains(group).Compare(flags.Wildcard) {
return includeAllGroupsWithCategories(ins, cats)
return includeAllGroupWithCategories(ins, cats)
}
sel := selectors.NewGroupsBackup(slices.Clone(group))
@ -324,6 +325,6 @@ func groupsBackupCreateSelectors(
return utils.AddGroupsCategories(sel, cats)
}
func includeAllGroupsWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
return utils.AddGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories)
}

View File

@ -14,15 +14,14 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/its"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
@ -36,7 +35,7 @@ import (
type NoBackupGroupsE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestNoBackupGroupsE2ESuite(t *testing.T) {
@ -51,7 +50,7 @@ func (suite *NoBackupGroupsE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
}
@ -66,7 +65,7 @@ func (suite *NoBackupGroupsE2ESuite) TestGroupsBackupListCmd_noBackups() {
cmd := cliTD.StubRootCmd(
"backup", "list", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
@ -90,7 +89,7 @@ func (suite *NoBackupGroupsE2ESuite) TestGroupsBackupListCmd_noBackups() {
type BackupGroupsE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestBackupGroupsE2ESuite(t *testing.T) {
@ -105,7 +104,7 @@ func (suite *BackupGroupsE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
}
@ -114,8 +113,6 @@ func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_channelMessages() {
}
func (suite *BackupGroupsE2ESuite) TestGroupsBackupCmd_conversations() {
// skip
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
runGroupsBackupCategoryTest(suite, flags.DataConversations)
}
@ -137,7 +134,7 @@ func runGroupsBackupCategoryTest(suite *BackupGroupsE2ESuite, category string) {
cmd, ctx := buildGroupsBackupCmd(
ctx,
suite.dpnd.configFilePath,
suite.m365.Group.ID,
suite.its.group.ID,
category,
&recorder)
@ -185,8 +182,7 @@ func runGroupsBackupGroupNotFoundTest(suite *BackupGroupsE2ESuite, category stri
assert.Contains(
t,
err.Error(),
"not found",
"error missing user not found")
"not found in tenant", "error missing group not found")
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
t.Logf("backup error message: %s", err.Error())
@ -205,7 +201,7 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAzureClientIDFlag()
cmd := cliTD.StubRootCmd(
"backup", "create", "groups",
"--group", suite.m365.Group.ID,
"--group", suite.its.group.ID,
"--azure-client-id", "invalid-value")
cli.BuildCommandTree(cmd)
@ -219,9 +215,6 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAzureClientIDFlag()
}
func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_fromConfigFile() {
// Skip
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
@ -232,8 +225,8 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_fromConfigFile() {
cmd := cliTD.StubRootCmd(
"backup", "create", "groups",
"--group", suite.m365.Group.ID,
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--group", suite.its.group.ID,
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -256,7 +249,7 @@ func (suite *BackupGroupsE2ESuite) TestBackupCreateGroups_badAWSFlags() {
cmd := cliTD.StubRootCmd(
"backup", "create", "groups",
"--group", suite.m365.Group.ID,
"--group", suite.its.group.ID,
"--aws-access-key", "invalid-value",
"--aws-secret-access-key", "some-invalid-value")
cli.BuildCommandTree(cmd)
@ -279,7 +272,7 @@ type PreparedBackupGroupsE2ESuite struct {
tester.Suite
dpnd dependencies
backupOps map[path.CategoryType]string
m365 its.M365IntgTestSetup
its intgTesterSetup
}
func TestPreparedBackupGroupsE2ESuite(t *testing.T) {
@ -296,19 +289,16 @@ func (suite *PreparedBackupGroupsE2ESuite) SetupSuite() {
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.its = newIntegrationTesterSetup(t)
suite.dpnd = prepM365Test(t, ctx, path.GroupsService)
suite.backupOps = make(map[path.CategoryType]string)
var (
groups = []string{suite.m365.Group.ID}
ins = idname.NewCache(map[string]string{suite.m365.Group.ID: suite.m365.Group.ID})
groups = []string{suite.its.group.ID}
ins = idname.NewCache(map[string]string{suite.its.group.ID: suite.its.group.ID})
cats = []path.CategoryType{
path.ChannelMessagesCategory,
// TODO(pandeyabs): CorsoCITeam group mailbox backup is currently broken because of invalid
// odata.NextLink which causes an infinite loop during paging. Disabling conversations tests while
// we go fix the group mailbox.
// path.ConversationPostsCategory,
path.ConversationPostsCategory,
path.LibrariesCategory,
}
)
@ -378,7 +368,7 @@ func runGroupsListCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.Cat
cmd := cliTD.StubRootCmd(
"backup", "list", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -419,7 +409,7 @@ func runGroupsListSingleCmdTest(suite *PreparedBackupGroupsE2ESuite, category pa
cmd := cliTD.StubRootCmd(
"backup", "list", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(bID))
cli.BuildCommandTree(cmd)
@ -446,7 +436,7 @@ func (suite *PreparedBackupGroupsE2ESuite) TestGroupsListCmd_badID() {
cmd := cliTD.StubRootCmd(
"backup", "list", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", "smarfs")
cli.BuildCommandTree(cmd)
@ -462,8 +452,6 @@ func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_channelMessages(
}
func (suite *PreparedBackupGroupsE2ESuite) TestGroupsDetailsCmd_conversations() {
// skip
suite.T().Skip("CorsoCITeam group mailbox backup is broken")
runGroupsDetailsCmdTest(suite, path.ConversationPostsCategory)
}
@ -476,6 +464,10 @@ func runGroupsDetailsCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.
t := suite.T()
if category == path.ConversationPostsCategory {
t.Skip("skipping conversation details test, see issue #4780")
}
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
@ -490,7 +482,7 @@ func runGroupsDetailsCmdTest(suite *PreparedBackupGroupsE2ESuite, category path.
cmd := cliTD.StubRootCmd(
"backup", "details", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN, string(bID))
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
@ -580,7 +572,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN,
fmt.Sprintf("%s,%s",
string(suite.backupOps[0].Results.BackupID),
@ -594,7 +586,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd() {
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backups", string(suite.backupOps[0].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -612,7 +604,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_SingleID() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN,
string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -624,7 +616,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_SingleID() {
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -642,7 +634,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_UnknownID() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN, uuid.NewString())
cli.BuildCommandTree(cmd)
@ -661,7 +653,7 @@ func (suite *BackupDeleteGroupsE2ESuite) TestGroupsBackupDeleteCmd_NoBackupID()
cmd := cliTD.StubRootCmd(
"backup", "delete", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
// empty backupIDs should error since no data provided
@ -680,7 +672,7 @@ func buildGroupsBackupCmd(
) (*cobra.Command, context.Context) {
cmd := cliTD.StubRootCmd(
"backup", "create", "groups",
"--"+flags.ConfigFileFN, configFile,
"--config-file", configFile,
"--"+flags.GroupFN, group,
"--"+flags.CategoryDataFN, category)
cli.BuildCommandTree(cmd)

View File

@ -153,7 +153,6 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.GroupsCategoryDataInput),
"--" + flags.FetchParallelismFN, flagsTD.FetchParallelism,
"--" + flags.DisableDeltaFN,
"--" + flags.DisableLazyItemReaderFN,
},
flagsTD.PreparedGenericBackupFlags(),
flagsTD.PreparedProviderFlags(),
@ -161,25 +160,13 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
opts := utils.MakeGroupsOpts(cmd)
co := utils.Control()
backupOpts := utils.ParseBackupOptions()
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
// restore flags are switched over too and we no longer parse flags beyond
// connection info into control.Options.
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
assert.Equal(t, control.FailFast, co.FailureHandling)
assert.True(t, co.ToggleFeatures.DisableIncrementals)
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
assert.True(t, co.ToggleFeatures.DisableDelta)
assert.True(t, co.ToggleFeatures.DisableLazyItemReader)
assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
flagsTD.AssertGenericBackupFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)

View File

@ -11,19 +11,145 @@ import (
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
gmock "github.com/alcionai/corso/src/pkg/services/m365/api/graph/mock"
"github.com/alcionai/corso/src/pkg/storage"
"github.com/alcionai/corso/src/pkg/storage/testdata"
)
// ---------------------------------------------------------------------------
// Gockable client
// ---------------------------------------------------------------------------
// GockClient produces a new exchange api client that can be
// mocked using gock.
func gockClient(creds account.M365Config, counter *count.Bus) (api.Client, error) {
s, err := gmock.NewService(creds, counter)
if err != nil {
return api.Client{}, err
}
li, err := gmock.NewService(creds, counter, graph.NoTimeout())
if err != nil {
return api.Client{}, err
}
return api.Client{
Credentials: creds,
Stable: s,
LargeItem: li,
}, nil
}
// ---------------------------------------------------------------------------
// Suite Setup
// ---------------------------------------------------------------------------
type ids struct {
ID string
DriveID string
DriveRootFolderID string
}
type intgTesterSetup struct {
acct account.Account
ac api.Client
gockAC api.Client
user ids
site ids
group ids
team ids
}
func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
its := intgTesterSetup{}
ctx, flush := tester.NewContext(t)
defer flush()
graph.InitializeConcurrencyLimiter(ctx, true, 4)
its.acct = tconfig.NewM365Account(t)
creds, err := its.acct.M365Config()
require.NoError(t, err, clues.ToCore(err))
its.ac, err = api.NewClient(
creds,
control.DefaultOptions(),
count.New())
require.NoError(t, err, clues.ToCore(err))
its.gockAC, err = gockClient(creds, count.New())
require.NoError(t, err, clues.ToCore(err))
// user drive
uids := ids{}
uids.ID = tconfig.M365UserID(t)
userDrive, err := its.ac.Users().GetDefaultDrive(ctx, uids.ID)
require.NoError(t, err, clues.ToCore(err))
uids.DriveID = ptr.Val(userDrive.GetId())
userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, uids.DriveID)
require.NoError(t, err, clues.ToCore(err))
uids.DriveRootFolderID = ptr.Val(userDriveRootFolder.GetId())
its.user = uids
// site
sids := ids{}
sids.ID = tconfig.M365SiteID(t)
siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, sids.ID)
require.NoError(t, err, clues.ToCore(err))
sids.DriveID = ptr.Val(siteDrive.GetId())
siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, sids.DriveID)
require.NoError(t, err, clues.ToCore(err))
sids.DriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId())
its.site = sids
// group
gids := ids{}
// use of the TeamID is intentional here, so that we are assured
// the group has full usage of the teams api.
gids.ID = tconfig.M365TeamID(t)
its.group = gids
// team
tids := ids{}
tids.ID = tconfig.M365TeamID(t)
its.team = tids
return its
}
type dependencies struct {
st storage.Storage
repo repository.Repositoryer
@ -88,7 +214,7 @@ func buildExchangeBackupCmd(
) (*cobra.Command, context.Context) {
cmd := cliTD.StubRootCmd(
"backup", "create", "exchange",
"--"+flags.ConfigFileFN, configFile,
"--config-file", configFile,
"--"+flags.UserFN, user,
"--"+flags.CategoryDataFN, category)
cli.BuildCommandTree(cmd)

View File

@ -60,6 +60,7 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
switch cmd.Use {
case createCommand:
c, fs = utils.AddCommand(cmd, oneDriveCreateCmd())
fs.SortFlags = false
c.Use = c.Use + " " + oneDriveServiceCommandCreateUseSuffix
c.Example = oneDriveServiceCommandCreateExamples
@ -67,20 +68,22 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
flags.AddUserFlag(c)
flags.AddGenericBackupFlags(c)
fs.BoolVar(
&flags.UseOldDeltaProcessFV,
flags.UseOldDeltaProcessFN,
&flags.UseDeltaTreeFV,
flags.UseDeltaTreeFN,
false,
"process backups using the old delta processor instead of tree-based enumeration")
cobra.CheckErr(fs.MarkHidden(flags.UseOldDeltaProcessFN))
"process backups using the delta tree instead of standard enumeration")
cobra.CheckErr(fs.MarkHidden(flags.UseDeltaTreeFN))
case listCommand:
c, _ = utils.AddCommand(cmd, oneDriveListCmd())
c, fs = utils.AddCommand(cmd, oneDriveListCmd())
fs.SortFlags = false
flags.AddBackupIDFlag(c, false)
flags.AddAllBackupListFlags(c)
case detailsCommand:
c, _ = utils.AddCommand(cmd, oneDriveDetailsCmd())
c, fs = utils.AddCommand(cmd, oneDriveDetailsCmd())
fs.SortFlags = false
c.Use = c.Use + " " + oneDriveServiceCommandDetailsUseSuffix
c.Example = oneDriveServiceCommandDetailsExamples
@ -90,7 +93,8 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
flags.AddOneDriveDetailsAndRestoreFlags(c)
case deleteCommand:
c, _ = utils.AddCommand(cmd, oneDriveDeleteCmd())
c, fs = utils.AddCommand(cmd, oneDriveDeleteCmd())
fs.SortFlags = false
c.Use = c.Use + " " + oneDriveServiceCommandDeleteUseSuffix
c.Example = oneDriveServiceCommandDeleteExamples

View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
@ -19,7 +20,6 @@ import (
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
@ -64,7 +64,7 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupListCmd_empty() {
cmd := cliTD.StubRootCmd(
"backup", "list", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
@ -93,8 +93,8 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_userNotInTenant() {
cmd := cliTD.StubRootCmd(
"backup", "create", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--"+flags.UserFN, "foo@not-there.com")
"--config-file", suite.dpnd.configFilePath,
"--"+flags.UserFN, "foo@nothere.com")
cli.BuildCommandTree(cmd)
cmd.SetOut(&recorder)
@ -107,8 +107,7 @@ func (suite *NoBackupOneDriveE2ESuite) TestOneDriveBackupCmd_userNotInTenant() {
assert.Contains(
t,
err.Error(),
"not found",
"error missing user not found")
"not found in tenant", "error missing user not found")
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
t.Logf("backup error message: %s", err.Error())
@ -176,7 +175,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN,
fmt.Sprintf("%s,%s",
string(suite.backupOps[0].Results.BackupID),
@ -201,7 +200,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd() {
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backups", string(suite.backupOps[0].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -221,7 +220,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_SingleID(
cmd := cliTD.StubRootCmd(
"backup", "delete", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN,
string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -243,7 +242,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_SingleID(
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[0].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -261,7 +260,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_unknownID
cmd := cliTD.StubRootCmd(
"backup", "delete", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN, uuid.NewString())
cli.BuildCommandTree(cmd)
@ -280,7 +279,7 @@ func (suite *BackupDeleteOneDriveE2ESuite) TestOneDriveBackupDeleteCmd_NoBackupI
cmd := cliTD.StubRootCmd(
"backup", "delete", "onedrive",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
// empty backupIDs should error since no data provided

View File

@ -108,20 +108,11 @@ func (suite *OneDriveUnitSuite) TestBackupCreateFlags() {
opts := utils.MakeOneDriveOpts(cmd)
co := utils.Control()
backupOpts := utils.ParseBackupOptions()
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
// restore flags are switched over too and we no longer parse flags beyond
// connection info into control.Options.
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
assert.Equal(t, control.FailFast, co.FailureHandling)
assert.True(t, co.ToggleFeatures.DisableIncrementals)
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
flagsTD.AssertGenericBackupFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)

View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/flags"
@ -37,11 +38,7 @@ corso backup create sharepoint --site https://example.com/hr
corso backup create sharepoint --site https://example.com/hr,https://example.com/team
# Backup all SharePoint data for all Sites
corso backup create sharepoint --site '*'
# Backup all SharePoint list data for a Site
corso backup create sharepoint --site https://example.com/hr --data lists
`
corso backup create sharepoint --site '*'`
sharePointServiceCommandDeleteExamples = `# Delete SharePoint backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
and 1234abcd-12ab-cd34-56de-1234abce
@ -61,54 +58,39 @@ corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
# Explore all files within the document library "Work Documents"
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--library "Work Documents"
# Explore lists by their name(s)
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list "list-name-1,list-name-2"
# Explore lists created after a given time
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-after 2024-01-01T12:23:34
# Explore lists created before a given time
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-before 2024-01-01T12:23:34
# Explore lists modified before a given time
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-before 2024-01-01T12:23:34
# Explore lists modified after a given time
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-after 2024-01-01T12:23:34`
`
)
// called by backup.go to map subcommands to provider-specific handling.
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case createCommand:
c, _ = utils.AddCommand(cmd, sharePointCreateCmd())
c, fs = utils.AddCommand(cmd, sharePointCreateCmd())
fs.SortFlags = false
c.Use = c.Use + " " + sharePointServiceCommandCreateUseSuffix
c.Example = sharePointServiceCommandCreateExamples
flags.AddSiteFlag(c, true)
flags.AddSiteIDFlag(c, true)
// [TODO](hitesh) to add lists flag to invoke backup for lists
// when explicit invoke is not required anymore
flags.AddDataFlag(c, []string{flags.DataLibraries}, true)
flags.AddGenericBackupFlags(c)
case listCommand:
c, _ = utils.AddCommand(cmd, sharePointListCmd())
c, fs = utils.AddCommand(cmd, sharePointListCmd())
fs.SortFlags = false
flags.AddBackupIDFlag(c, false)
flags.AddAllBackupListFlags(c)
case detailsCommand:
c, _ = utils.AddCommand(cmd, sharePointDetailsCmd())
c, fs = utils.AddCommand(cmd, sharePointDetailsCmd())
fs.SortFlags = false
c.Use = c.Use + " " + sharePointServiceCommandDetailsUseSuffix
c.Example = sharePointServiceCommandDetailsExamples
@ -118,7 +100,8 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
flags.AddSharePointDetailsAndRestoreFlags(c)
case deleteCommand:
c, _ = utils.AddCommand(cmd, sharePointDeleteCmd())
c, fs = utils.AddCommand(cmd, sharePointDeleteCmd())
fs.SortFlags = false
c.Use = c.Use + " " + sharePointServiceCommandDeleteUseSuffix
c.Example = sharePointServiceCommandDeleteExamples
@ -211,12 +194,10 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
flags.SiteFN + " *")
}
allowedCats := utils.SharePointAllowedCategories()
for _, d := range cats {
if _, ok := allowedCats[d]; !ok {
if d != flags.DataLibraries && d != flags.DataPages {
return clues.New(
d + " is an unrecognized data type; only " + flags.DataLibraries + " supported")
d + " is an unrecognized data type; either " + flags.DataLibraries + "or " + flags.DataPages)
}
}
@ -243,11 +224,29 @@ func sharePointBackupCreateSelectors(
sel := selectors.NewSharePointBackup(append(slices.Clone(sites), weburls...))
return utils.AddCategories(sel, cats), nil
return addCategories(sel, cats), nil
}
func includeAllSitesWithCategories(ins idname.Cacher, categories []string) *selectors.SharePointBackup {
return utils.AddCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
return addCategories(selectors.NewSharePointBackup(ins.IDs()), categories)
}
func addCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
// Issue #2631: Libraries are the only supported feature for SharePoint at this time.
if len(cats) == 0 {
sel.Include(sel.LibraryFolders(selectors.Any()))
}
for _, d := range cats {
switch d {
case flags.DataLibraries:
sel.Include(sel.LibraryFolders(selectors.Any()))
case flags.DataPages:
sel.Include(sel.Pages(selectors.Any()))
}
}
return sel
}
// ------------------------------------------------------------------------------------------------

View File

@ -1,29 +1,25 @@
package backup_test
import (
"context"
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/its"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/selectors/testdata"
@ -31,7 +27,7 @@ import (
)
// ---------------------------------------------------------------------------
// tests that require no existing backups
// tests with no prior backup
// ---------------------------------------------------------------------------
type NoBackupSharePointE2ESuite struct {
@ -66,7 +62,7 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
cmd := cliTD.StubRootCmd(
"backup", "list", "sharepoint",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
@ -83,297 +79,6 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
assert.True(t, strings.HasSuffix(result, "No backups available\n"))
}
// ---------------------------------------------------------------------------
// tests with no prior backup
// ---------------------------------------------------------------------------
type BackupSharepointE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
}
func TestBackupSharepointE2ESuite(t *testing.T) {
suite.Run(t, &BackupSharepointE2ESuite{Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
}
func (suite *BackupSharepointE2ESuite) SetupSuite() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
}
func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_lists() {
// Issue: https://github.com/alcionai/corso/issues/4754
suite.T().Skip("unskip when sharepoint lists support is enabled")
runSharepointBackupCategoryTest(suite, flags.DataLists)
}
func runSharepointBackupCategoryTest(suite *BackupSharepointE2ESuite, category string) {
recorder := strings.Builder{}
recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd, ctx := buildSharepointBackupCmd(
ctx,
suite.dpnd.configFilePath,
suite.m365.Site.ID,
category,
&recorder)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := recorder.String()
t.Log("backup results", result)
}
func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_siteNotFound_lists() {
// Issue: https://github.com/alcionai/corso/issues/4754
suite.T().Skip("un-skip test when lists support is enabled")
runSharepointBackupSiteNotFoundTest(suite, flags.DataLists)
}
func runSharepointBackupSiteNotFoundTest(suite *BackupSharepointE2ESuite, category string) {
recorder := strings.Builder{}
recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd, ctx := buildSharepointBackupCmd(
ctx,
suite.dpnd.configFilePath,
uuid.NewString(),
category,
&recorder)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
assert.Contains(
t,
err.Error(),
"Invalid hostname for this tenancy", "error missing site not found")
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
t.Logf("backup error message: %s", err.Error())
result := recorder.String()
t.Log("backup results", result)
}
// ---------------------------------------------------------------------------
// tests prepared with a previous backup
// ---------------------------------------------------------------------------
type PreparedBackupSharepointE2ESuite struct {
tester.Suite
dpnd dependencies
backupOps map[path.CategoryType]string
m365 its.M365IntgTestSetup
}
func TestPreparedBackupSharepointE2ESuite(t *testing.T) {
suite.Run(t, &PreparedBackupSharepointE2ESuite{
Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
})
}
func (suite *PreparedBackupSharepointE2ESuite) SetupSuite() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
suite.backupOps = make(map[path.CategoryType]string)
var (
sites = []string{suite.m365.Site.ID}
ins = idname.NewCache(map[string]string{suite.m365.Site.ID: suite.m365.Site.ID})
cats = []path.CategoryType{
path.ListsCategory,
}
)
for _, set := range cats {
var (
sel = selectors.NewSharePointBackup(sites)
scopes []selectors.SharePointScope
)
switch set {
case path.ListsCategory:
scopes = testdata.SharePointBackupListsScope(sel)
}
sel.Include(scopes)
bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
require.NoError(t, err, clues.ToCore(err))
err = bop.Run(ctx)
require.NoError(t, err, clues.ToCore(err))
bIDs := string(bop.Results.BackupID)
// sanity check, ensure we can find the backup and its details immediately
b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
require.NoError(t, err, "retrieving recent backup by ID")
require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
suite.backupOps[set] = string(b.ID)
}
}
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_lists() {
runSharepointListCmdTest(suite, path.ListsCategory)
}
func runSharepointListCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
suite.dpnd.recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "list", "sharepoint",
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// compare the output
result := suite.dpnd.recorder.String()
assert.Contains(t, result, suite.backupOps[category])
t.Log("backup results", result)
}
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_badID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "list", "sharepoint",
"--config-file", suite.dpnd.configFilePath,
"--backup", uuid.NewString())
cli.BuildCommandTree(cmd)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *PreparedBackupSharepointE2ESuite) TestSharepointDetailsCmd_lists() {
runSharepointDetailsCmdTest(suite, path.ListsCategory)
}
func runSharepointDetailsCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
suite.dpnd.recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
bID := suite.backupOps[category]
// fetch the details from the repo first
deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
require.Empty(t, errs.Recovered())
cmd := cliTD.StubRootCmd(
"backup", "details", "sharepoint",
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupFN, string(bID))
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// compare the output
result := suite.dpnd.recorder.String()
i := 0
findings := make(map[path.CategoryType]int)
incrementor := func(cond bool, cat path.CategoryType) {
if cond {
findings[cat]++
}
}
for _, ent := range deets.Entries {
if ent.SharePoint == nil {
continue
}
isSharePointList := ent.SharePoint.ItemType == details.SharePointList
hasListName := isSharePointList && len(ent.SharePoint.List.Name) > 0
hasItemName := !isSharePointList && len(ent.SharePoint.ItemName) > 0
incrementor(hasListName, category)
incrementor(hasItemName, category)
suite.Run(fmt.Sprintf("detail %d", i), func() {
assert.Contains(suite.T(), result, ent.ShortRef)
})
i++
}
assert.GreaterOrEqual(t, findings[category], 1)
}
// ---------------------------------------------------------------------------
// tests for deleting backups
// ---------------------------------------------------------------------------
@ -441,7 +146,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
cmd := cliTD.StubRootCmd(
"backup", "delete", "sharepoint",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN,
fmt.Sprintf("%s,%s",
string(suite.backupOp.Results.BackupID),
@ -468,7 +173,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd() {
// // a follow-up details call should fail, due to the backup ID being deleted
// cmd = cliTD.StubRootCmd(
// "backup", "details", "sharepoint",
// "--"+flags.ConfigFileFN, suite.cfgFP,
// "--config-file", suite.cfgFP,
// "--backup", string(suite.backupOp.Results.BackupID))
// cli.BuildCommandTree(cmd)
@ -485,7 +190,7 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_unkno
cmd := cliTD.StubRootCmd(
"backup", "delete", "sharepoint",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--config-file", suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN, uuid.NewString())
cli.BuildCommandTree(cmd)
@ -504,30 +209,10 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_NoBac
cmd := cliTD.StubRootCmd(
"backup", "delete", "groups",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
"--config-file", suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
// empty backupIDs should error since no data provided
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
func buildSharepointBackupCmd(
ctx context.Context,
configFile, site, category string,
recorder *strings.Builder,
) (*cobra.Command, context.Context) {
cmd := cliTD.StubRootCmd(
"backup", "create", "sharepoint",
"--config-file", configFile,
"--"+flags.SiteIDFN, site,
"--"+flags.CategoryDataFN, category)
cli.BuildCommandTree(cmd)
cmd.SetOut(recorder)
return cmd, print.SetRootCmd(ctx, cmd)
}

View File

@ -112,21 +112,12 @@ func (suite *SharePointUnitSuite) TestBackupCreateFlags() {
opts := utils.MakeSharePointOpts(cmd)
co := utils.Control()
backupOpts := utils.ParseBackupOptions()
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
// restore flags are switched over too and we no longer parse flags beyond
// connection info into control.Options.
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
assert.Equal(t, control.FailFast, co.FailureHandling)
assert.True(t, co.ToggleFeatures.DisableIncrementals)
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID)
assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL)
assert.Equal(t, control.FailFast, co.FailureHandling)
assert.True(t, co.ToggleFeatures.DisableIncrementals)
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
flagsTD.AssertGenericBackupFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)
@ -218,7 +209,6 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
name string
site []string
weburl []string
cats []string
expect assert.ErrorAssertionFunc
}{
{
@ -226,61 +216,25 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
expect: assert.Error,
},
{
name: "sites but no category",
name: "sites",
site: []string{"smarf"},
expect: assert.NoError,
},
{
name: "web urls but no category",
name: "urls",
weburl: []string{"fnord"},
expect: assert.NoError,
},
{
name: "both web urls and sites but no category",
name: "both",
site: []string{"smarf"},
weburl: []string{"fnord"},
expect: assert.NoError,
},
{
name: "site with libraries category",
site: []string{"smarf"},
cats: []string{flags.DataLibraries},
expect: assert.NoError,
},
{
name: "site with invalid category",
site: []string{"smarf"},
cats: []string{"invalid category"},
expect: assert.Error,
},
{
name: "site with lists category",
site: []string{"smarf"},
cats: []string{flags.DataLists},
expect: assert.NoError,
},
// [TODO]: Uncomment when pages are enabled
// {
// name: "site with pages category",
// site: []string{"smarf"},
// cats: []string{flags.DataPages},
// expect: assert.NoError,
// },
// [TODO]: Uncomment when pages & lists are enabled
// {
// name: "site with all categories",
// site: []string{"smarf"},
// cats: []string{flags.DataLists, flags.DataPages, flags.DataLibraries},
// expect: assert.NoError,
// },
}
for _, test := range table {
suite.Run(test.name, func() {
err := validateSharePointBackupCreateFlags(test.site, test.weburl, test.cats)
err := validateSharePointBackupCreateFlags(test.site, test.weburl, nil)
test.expect(suite.T(), err, clues.ToCore(err))
})
}
@ -366,12 +320,6 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
data: []string{flags.DataPages},
expect: bothIDs,
},
{
name: "Lists",
site: bothIDs,
data: []string{flags.DataLists},
expect: bothIDs,
},
}
for _, test := range table {
suite.Run(test.name, func() {

View File

@ -1,305 +0,0 @@
package backup
import (
"context"
"fmt"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
)
// ------------------------------------------------------------------------------------------------
// setup and globals
// ------------------------------------------------------------------------------------------------
const (
teamschatsServiceCommand = "chats"
teamschatsServiceCommandCreateUseSuffix = "--user <userEmail> | '" + flags.Wildcard + "'"
teamschatsServiceCommandDeleteUseSuffix = "--backups <backupId>"
teamschatsServiceCommandDetailsUseSuffix = "--backup <backupId>"
)
const (
teamschatsServiceCommandCreateExamples = `# Backup all chats with bob@company.hr
corso backup create chats --user bob@company.hr
# Backup all chats for all users
corso backup create chats --user '*'`
teamschatsServiceCommandDeleteExamples = `# Delete chats backup with ID 1234abcd-12ab-cd34-56de-1234abcd \
and 1234abcd-12ab-cd34-56de-1234abce
corso backup delete chats --backups 1234abcd-12ab-cd34-56de-1234abcd,1234abcd-12ab-cd34-56de-1234abce`
teamschatsServiceCommandDetailsExamples = `# Explore chats in Bob's latest backup (1234abcd...)
corso backup details chats --backup 1234abcd-12ab-cd34-56de-1234abcd`
)
// called by backup.go to map subcommands to provider-specific handling.
func addTeamsChatsCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
switch cmd.Use {
case createCommand:
c, _ = utils.AddCommand(cmd, teamschatsCreateCmd(), utils.MarkPreReleaseCommand())
c.Use = c.Use + " " + teamschatsServiceCommandCreateUseSuffix
c.Example = teamschatsServiceCommandCreateExamples
// Flags addition ordering should follow the order we want them to appear in help and docs:
flags.AddUserFlag(c)
flags.AddDataFlag(c, []string{flags.DataChats}, false)
flags.AddGenericBackupFlags(c)
case listCommand:
c, _ = utils.AddCommand(cmd, teamschatsListCmd(), utils.MarkPreReleaseCommand())
flags.AddBackupIDFlag(c, false)
flags.AddAllBackupListFlags(c)
case detailsCommand:
c, _ = utils.AddCommand(cmd, teamschatsDetailsCmd(), utils.MarkPreReleaseCommand())
c.Use = c.Use + " " + teamschatsServiceCommandDetailsUseSuffix
c.Example = teamschatsServiceCommandDetailsExamples
flags.AddSkipReduceFlag(c)
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
flags.AddBackupIDFlag(c, true)
flags.AddTeamsChatsDetailsAndRestoreFlags(c)
case deleteCommand:
c, _ = utils.AddCommand(cmd, teamschatsDeleteCmd(), utils.MarkPreReleaseCommand())
c.Use = c.Use + " " + teamschatsServiceCommandDeleteUseSuffix
c.Example = teamschatsServiceCommandDeleteExamples
flags.AddMultipleBackupIDsFlag(c, false)
flags.AddBackupIDFlag(c, false)
}
return c
}
// ------------------------------------------------------------------------------------------------
// backup create
// ------------------------------------------------------------------------------------------------
// `corso backup create chats [<flag>...]`
func teamschatsCreateCmd() *cobra.Command {
return &cobra.Command{
Use: teamschatsServiceCommand,
Aliases: []string{teamsServiceCommand},
Short: "Backup M365 Chats data",
RunE: createTeamsChatsCmd,
Args: cobra.NoArgs,
}
}
// processes a teamschats backup.
func createTeamsChatsCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
if flags.RunModeFV == flags.RunModeFlagTest {
return nil
}
if err := validateTeamsChatsBackupCreateFlags(flags.UserFV, flags.CategoryDataFV); err != nil {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(
ctx,
cmd,
path.TeamsChatsService)
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
// TODO: log/print recoverable errors
errs := fault.New(false)
svcCli, err := m365.NewM365Client(ctx, *acct)
if err != nil {
return Only(ctx, clues.Stack(err))
}
ins, err := svcCli.AC.Users().GetAllIDsAndNames(ctx, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 teamschats"))
}
sel := teamschatsBackupCreateSelectors(ctx, ins, flags.UserFV, flags.CategoryDataFV)
selectorSet := []selectors.Selector{}
for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) {
selectorSet = append(selectorSet, discSel.Selector)
}
return genericCreateCommand(
ctx,
r,
"Chats",
selectorSet,
ins)
}
// ------------------------------------------------------------------------------------------------
// backup list
// ------------------------------------------------------------------------------------------------
// `corso backup list teamschats [<flag>...]`
func teamschatsListCmd() *cobra.Command {
return &cobra.Command{
Use: teamschatsServiceCommand,
Short: "List the history of M365 Chats backups",
RunE: listTeamsChatsCmd,
Args: cobra.NoArgs,
}
}
// lists the history of backup operations
func listTeamsChatsCmd(cmd *cobra.Command, args []string) error {
return genericListCommand(cmd, flags.BackupIDFV, path.TeamsChatsService, args)
}
// ------------------------------------------------------------------------------------------------
// backup details
// ------------------------------------------------------------------------------------------------
// `corso backup details teamschats [<flag>...]`
func teamschatsDetailsCmd() *cobra.Command {
return &cobra.Command{
Use: teamschatsServiceCommand,
Short: "Shows the details of a M365 Chats backup",
RunE: detailsTeamsChatsCmd,
Args: cobra.NoArgs,
}
}
// processes a teamschats backup.
func detailsTeamsChatsCmd(cmd *cobra.Command, args []string) error {
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
if flags.RunModeFV == flags.RunModeFlagTest {
return nil
}
return runDetailsTeamsChatsCmd(cmd)
}
func runDetailsTeamsChatsCmd(cmd *cobra.Command) error {
ctx := cmd.Context()
opts := utils.MakeTeamsChatsOpts(cmd)
sel := utils.IncludeTeamsChatsRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterTeamsChatsRestoreInfoSelectors(sel, opts)
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) > 0 {
ds.PrintEntries(ctx)
} else {
Info(ctx, selectors.ErrorNoMatchingItems)
}
return nil
}
// ------------------------------------------------------------------------------------------------
// backup delete
// ------------------------------------------------------------------------------------------------
// `corso backup delete teamschats [<flag>...]`
func teamschatsDeleteCmd() *cobra.Command {
return &cobra.Command{
Use: teamschatsServiceCommand,
Short: "Delete backed-up M365 Chats data",
RunE: deleteTeamsChatsCmd,
Args: cobra.NoArgs,
}
}
// deletes an teamschats backup.
func deleteTeamsChatsCmd(cmd *cobra.Command, args []string) error {
backupIDValue := []string{}
if len(flags.BackupIDsFV) > 0 {
backupIDValue = flags.BackupIDsFV
} else if len(flags.BackupIDFV) > 0 {
backupIDValue = append(backupIDValue, flags.BackupIDFV)
} else {
return clues.New("either --backup or --backups flag is required")
}
return genericDeleteCommand(cmd, path.TeamsChatsService, "TeamsChats", backupIDValue, args)
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
func validateTeamsChatsBackupCreateFlags(teamschats, cats []string) error {
if len(teamschats) == 0 {
return clues.New(
"requires one or more --" +
flags.UserFN + " ids, or the wildcard --" +
flags.UserFN + " *")
}
msg := fmt.Sprintf(
" is an unrecognized data type; only %s is supported",
flags.DataChats)
allowedCats := utils.TeamsChatsAllowedCategories()
for _, d := range cats {
if _, ok := allowedCats[d]; !ok {
return clues.New(d + msg)
}
}
return nil
}
func teamschatsBackupCreateSelectors(
ctx context.Context,
ins idname.Cacher,
users, cats []string,
) *selectors.TeamsChatsBackup {
if filters.PathContains(users).Compare(flags.Wildcard) {
return includeAllTeamsChatsWithCategories(ins, cats)
}
sel := selectors.NewTeamsChatsBackup(slices.Clone(users))
return utils.AddTeamsChatsCategories(sel, cats)
}
func includeAllTeamsChatsWithCategories(ins idname.Cacher, categories []string) *selectors.TeamsChatsBackup {
return utils.AddTeamsChatsCategories(selectors.NewTeamsChatsBackup(ins.IDs()), categories)
}

View File

@ -1,636 +0,0 @@
package backup_test
import (
"context"
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/print"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/its"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
)
// ---------------------------------------------------------------------------
// tests that require no existing backups
// ---------------------------------------------------------------------------
type NoBackupTeamsChatsE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
}
func TestNoBackupTeamsChatsE2ESuite(t *testing.T) {
suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
}
func (suite *NoBackupTeamsChatsE2ESuite) SetupSuite() {
t := suite.T()
t.Skip("not fully implemented")
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
}
func (suite *NoBackupTeamsChatsE2ESuite) TestTeamsChatsBackupListCmd_noBackups() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
suite.dpnd.recorder.Reset()
cmd := cliTD.StubRootCmd(
"backup", "list", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := suite.dpnd.recorder.String()
// as an offhand check: the result should contain the m365 teamschat id
assert.True(t, strings.HasSuffix(result, "No backups available\n"))
}
// ---------------------------------------------------------------------------
// tests with no prior backup
// ---------------------------------------------------------------------------
type BackupTeamsChatsE2ESuite struct {
tester.Suite
dpnd dependencies
m365 its.M365IntgTestSetup
}
func TestBackupTeamsChatsE2ESuite(t *testing.T) {
suite.Run(t, &BackupTeamsChatsE2ESuite{Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
}
func (suite *BackupTeamsChatsE2ESuite) SetupSuite() {
t := suite.T()
t.Skip("not fully implemented")
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
}
func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_chats() {
runTeamsChatsBackupCategoryTest(suite, flags.DataChats)
}
func runTeamsChatsBackupCategoryTest(suite *BackupTeamsChatsE2ESuite, category string) {
recorder := strings.Builder{}
recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd, ctx := buildTeamsChatsBackupCmd(
ctx,
suite.dpnd.configFilePath,
suite.m365.User.ID,
category,
&recorder)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
result := recorder.String()
t.Log("backup results", result)
}
func (suite *BackupTeamsChatsE2ESuite) TestTeamsChatsBackupCmd_teamschatNotFound_chats() {
runTeamsChatsBackupTeamsChatNotFoundTest(suite, flags.DataChats)
}
func runTeamsChatsBackupTeamsChatNotFoundTest(suite *BackupTeamsChatsE2ESuite, category string) {
recorder := strings.Builder{}
recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd, ctx := buildTeamsChatsBackupCmd(
ctx,
suite.dpnd.configFilePath,
"foo@not-there.com",
category,
&recorder)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
assert.Contains(
t,
err.Error(),
"not found",
"error missing user not found")
assert.NotContains(t, err.Error(), "runtime error", "panic happened")
t.Logf("backup error message: %s", err.Error())
result := recorder.String()
t.Log("backup results", result)
}
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAzureClientIDFlag() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.dpnd.recorder.Reset()
cmd := cliTD.StubRootCmd(
"backup", "create", "chats",
"--teamschat", suite.m365.User.ID,
"--azure-client-id", "invalid-value")
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_fromConfigFile() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
suite.dpnd.recorder.Reset()
cmd := cliTD.StubRootCmd(
"backup", "create", "chats",
"--teamschat", suite.m365.User.ID,
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
}
// AWS flags
func (suite *BackupTeamsChatsE2ESuite) TestBackupCreateTeamsChats_badAWSFlags() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
suite.dpnd.recorder.Reset()
cmd := cliTD.StubRootCmd(
"backup", "create", "chats",
"--teamschat", suite.m365.User.ID,
"--aws-access-key", "invalid-value",
"--aws-secret-access-key", "some-invalid-value")
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
// since invalid aws creds are explicitly set, should see a failure
require.Error(t, err, clues.ToCore(err))
}
// ---------------------------------------------------------------------------
// tests prepared with a previous backup
// ---------------------------------------------------------------------------
type PreparedBackupTeamsChatsE2ESuite struct {
tester.Suite
dpnd dependencies
backupOps map[path.CategoryType]string
m365 its.M365IntgTestSetup
}
func TestPreparedBackupTeamsChatsE2ESuite(t *testing.T) {
suite.Run(t, &PreparedBackupTeamsChatsE2ESuite{
Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
})
}
func (suite *PreparedBackupTeamsChatsE2ESuite) SetupSuite() {
t := suite.T()
t.Skip("not fully implemented")
ctx, flush := tester.NewContext(t)
defer flush()
suite.m365 = its.GetM365(t)
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
suite.backupOps = make(map[path.CategoryType]string)
var (
teamschats = []string{suite.m365.User.ID}
ins = idname.NewCache(map[string]string{suite.m365.User.ID: suite.m365.User.ID})
cats = []path.CategoryType{
path.ChatsCategory,
}
)
for _, set := range cats {
var (
sel = selectors.NewTeamsChatsBackup(teamschats)
scopes []selectors.TeamsChatsScope
)
switch set {
case path.ChatsCategory:
scopes = selTD.TeamsChatsBackupChatScope(sel)
}
sel.Include(scopes)
bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
require.NoError(t, err, clues.ToCore(err))
err = bop.Run(ctx)
require.NoError(t, err, clues.ToCore(err))
bIDs := string(bop.Results.BackupID)
// sanity check, ensure we can find the backup and its details immediately
b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
require.NoError(t, err, "retrieving recent backup by ID")
require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
suite.backupOps[set] = string(b.ID)
}
}
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_chats() {
runTeamsChatsListCmdTest(suite, path.ChatsCategory)
}
func runTeamsChatsListCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
suite.dpnd.recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "list", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// compare the output
result := suite.dpnd.recorder.String()
assert.Contains(t, result, suite.backupOps[category])
}
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_singleID_chats() {
runTeamsChatsListSingleCmdTest(suite, path.ChatsCategory)
}
func runTeamsChatsListSingleCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
suite.dpnd.recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
bID := suite.backupOps[category]
cmd := cliTD.StubRootCmd(
"backup", "list", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--backup", string(bID))
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// compare the output
result := suite.dpnd.recorder.String()
assert.Contains(t, result, bID)
}
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsListCmd_badID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "list", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--backup", "smarfs")
cli.BuildCommandTree(cmd)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *PreparedBackupTeamsChatsE2ESuite) TestTeamsChatsDetailsCmd_chats() {
runTeamsChatsDetailsCmdTest(suite, path.ChatsCategory)
}
func runTeamsChatsDetailsCmdTest(suite *PreparedBackupTeamsChatsE2ESuite, category path.CategoryType) {
suite.dpnd.recorder.Reset()
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
bID := suite.backupOps[category]
// fetch the details from the repo first
deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
require.Empty(t, errs.Recovered())
cmd := cliTD.StubRootCmd(
"backup", "details", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--"+flags.BackupFN, string(bID))
cli.BuildCommandTree(cmd)
cmd.SetOut(&suite.dpnd.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// compare the output
result := suite.dpnd.recorder.String()
i := 0
foundFolders := 0
for _, ent := range deets.Entries {
// Skip folders as they don't mean anything to the end teamschat.
if ent.Folder != nil {
foundFolders++
continue
}
suite.Run(fmt.Sprintf("detail %d", i), func() {
assert.Contains(suite.T(), result, ent.ShortRef)
})
i++
}
// We only backup the default folder for each category so there should be at
// least that folder (we don't make details entries for prefix folders).
assert.GreaterOrEqual(t, foundFolders, 1)
}
// ---------------------------------------------------------------------------
// tests for deleting backups
// ---------------------------------------------------------------------------
type BackupDeleteTeamsChatsE2ESuite struct {
tester.Suite
dpnd dependencies
backupOps [3]operations.BackupOperation
}
func TestBackupDeleteTeamsChatsE2ESuite(t *testing.T) {
suite.Run(t, &BackupDeleteTeamsChatsE2ESuite{
Suite: tester.NewE2ESuite(
t,
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
})
}
func (suite *BackupDeleteTeamsChatsE2ESuite) SetupSuite() {
t := suite.T()
t.Skip("not fully implemented")
ctx, flush := tester.NewContext(t)
defer flush()
suite.dpnd = prepM365Test(t, ctx, path.TeamsChatsService)
m365TeamsChatID := tconfig.M365TeamID(t)
teamschats := []string{m365TeamsChatID}
// some tests require an existing backup
sel := selectors.NewTeamsChatsBackup(teamschats)
sel.Include(selTD.TeamsChatsBackupChatScope(sel))
for i := 0; i < cap(suite.backupOps); i++ {
backupOp, err := suite.dpnd.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, err, clues.ToCore(err))
suite.backupOps[i] = backupOp
err = suite.backupOps[i].Run(ctx)
require.NoError(t, err, clues.ToCore(err))
}
}
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "delete", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN,
fmt.Sprintf("%s,%s",
string(suite.backupOps[0].Results.BackupID),
string(suite.backupOps[1].Results.BackupID)))
cli.BuildCommandTree(cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--backups", string(suite.backupOps[0].Results.BackupID))
cli.BuildCommandTree(cmd)
err = cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_SingleID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "delete", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--"+flags.BackupFN,
string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// a follow-up details call should fail, due to the backup ID being deleted
cmd = cliTD.StubRootCmd(
"backup", "details", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--backup", string(suite.backupOps[2].Results.BackupID))
cli.BuildCommandTree(cmd)
err = cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_UnknownID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "delete", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath,
"--"+flags.BackupIDsFN, uuid.NewString())
cli.BuildCommandTree(cmd)
// unknown backupIDs should error since the modelStore can't find the backup
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
func (suite *BackupDeleteTeamsChatsE2ESuite) TestTeamsChatsBackupDeleteCmd_NoBackupID() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.dpnd.vpr)
defer flush()
cmd := cliTD.StubRootCmd(
"backup", "delete", "chats",
"--"+flags.ConfigFileFN, suite.dpnd.configFilePath)
cli.BuildCommandTree(cmd)
// empty backupIDs should error since no data provided
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
func buildTeamsChatsBackupCmd(
ctx context.Context,
configFile, resource, category string,
recorder *strings.Builder,
) (*cobra.Command, context.Context) {
cmd := cliTD.StubRootCmd(
"backup", "create", "chats",
"--"+flags.ConfigFileFN, configFile,
"--"+flags.UserFN, resource,
"--"+flags.CategoryDataFN, category)
cli.BuildCommandTree(cmd)
cmd.SetOut(recorder)
return cmd, print.SetRootCmd(ctx, cmd)
}

View File

@ -1,248 +0,0 @@
package backup
import (
"testing"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli/flags"
flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
)
type TeamsChatsUnitSuite struct {
tester.Suite
}
func TestTeamsChatsUnitSuite(t *testing.T) {
suite.Run(t, &TeamsChatsUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *TeamsChatsUnitSuite) TestAddTeamsChatsCommands() {
expectUse := teamschatsServiceCommand
table := []struct {
name string
use string
expectUse string
expectShort string
expectRunE func(*cobra.Command, []string) error
}{
{
name: "create teamschats",
use: createCommand,
expectUse: expectUse + " " + teamschatsServiceCommandCreateUseSuffix,
expectShort: teamschatsCreateCmd().Short,
expectRunE: createTeamsChatsCmd,
},
{
name: "list teamschats",
use: listCommand,
expectUse: expectUse,
expectShort: teamschatsListCmd().Short,
expectRunE: listTeamsChatsCmd,
},
{
name: "details teamschats",
use: detailsCommand,
expectUse: expectUse + " " + teamschatsServiceCommandDetailsUseSuffix,
expectShort: teamschatsDetailsCmd().Short,
expectRunE: detailsTeamsChatsCmd,
},
{
name: "delete teamschats",
use: deleteCommand,
expectUse: expectUse + " " + teamschatsServiceCommandDeleteUseSuffix,
expectShort: teamschatsDeleteCmd().Short,
expectRunE: deleteTeamsChatsCmd,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
cmd := &cobra.Command{Use: test.use}
c := addTeamsChatsCommands(cmd)
require.NotNil(t, c)
cmds := cmd.Commands()
require.Len(t, cmds, 1)
child := cmds[0]
assert.Equal(t, test.expectUse, child.Use)
assert.Equal(t, test.expectShort, child.Short)
tester.AreSameFunc(t, test.expectRunE, child.RunE)
})
}
}
func (suite *TeamsChatsUnitSuite) TestValidateTeamsChatsBackupCreateFlags() {
table := []struct {
name string
cats []string
expect assert.ErrorAssertionFunc
}{
{
name: "none",
cats: []string{},
expect: assert.NoError,
},
{
name: "chats",
cats: []string{flags.DataChats},
expect: assert.NoError,
},
{
name: "all allowed",
cats: []string{
flags.DataChats,
},
expect: assert.NoError,
},
{
name: "bad inputs",
cats: []string{"foo"},
expect: assert.Error,
},
}
for _, test := range table {
suite.Run(test.name, func() {
err := validateTeamsChatsBackupCreateFlags([]string{"*"}, test.cats)
test.expect(suite.T(), err, clues.ToCore(err))
})
}
}
func (suite *TeamsChatsUnitSuite) TestBackupCreateFlags() {
t := suite.T()
cmd := cliTD.SetUpCmdHasFlags(
t,
&cobra.Command{Use: createCommand},
addTeamsChatsCommands,
[]cliTD.UseCobraCommandFn{
flags.AddAllProviderFlags,
flags.AddAllStorageFlags,
},
flagsTD.WithFlags(
teamschatsServiceCommand,
[]string{
"--" + flags.RunModeFN, flags.RunModeFlagTest,
"--" + flags.UserFN, flagsTD.FlgInputs(flagsTD.UsersInput),
"--" + flags.CategoryDataFN, flagsTD.FlgInputs(flagsTD.TeamsChatsCategoryDataInput),
},
flagsTD.PreparedGenericBackupFlags(),
flagsTD.PreparedProviderFlags(),
flagsTD.PreparedStorageFlags()))
opts := utils.MakeTeamsChatsOpts(cmd)
co := utils.Control()
backupOpts := utils.ParseBackupOptions()
// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
// restore flags are switched over too and we no longer parse flags beyond
// connection info into control.Options.
assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
assert.Equal(t, control.FailFast, co.FailureHandling)
assert.True(t, co.ToggleFeatures.DisableIncrementals)
assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
flagsTD.AssertGenericBackupFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)
}
func (suite *TeamsChatsUnitSuite) TestBackupListFlags() {
t := suite.T()
cmd := cliTD.SetUpCmdHasFlags(
t,
&cobra.Command{Use: listCommand},
addTeamsChatsCommands,
[]cliTD.UseCobraCommandFn{
flags.AddAllProviderFlags,
flags.AddAllStorageFlags,
},
flagsTD.WithFlags(
teamschatsServiceCommand,
[]string{
"--" + flags.RunModeFN, flags.RunModeFlagTest,
"--" + flags.BackupFN, flagsTD.BackupInput,
},
flagsTD.PreparedBackupListFlags(),
flagsTD.PreparedProviderFlags(),
flagsTD.PreparedStorageFlags()))
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
flagsTD.AssertBackupListFlags(t, cmd)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)
}
func (suite *TeamsChatsUnitSuite) TestBackupDetailsFlags() {
t := suite.T()
cmd := cliTD.SetUpCmdHasFlags(
t,
&cobra.Command{Use: detailsCommand},
addTeamsChatsCommands,
[]cliTD.UseCobraCommandFn{
flags.AddAllProviderFlags,
flags.AddAllStorageFlags,
},
flagsTD.WithFlags(
teamschatsServiceCommand,
[]string{
"--" + flags.RunModeFN, flags.RunModeFlagTest,
"--" + flags.BackupFN, flagsTD.BackupInput,
"--" + flags.SkipReduceFN,
},
flagsTD.PreparedTeamsChatsFlags(),
flagsTD.PreparedProviderFlags(),
flagsTD.PreparedStorageFlags()))
co := utils.Control()
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
assert.True(t, co.SkipReduce)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)
flagsTD.AssertTeamsChatsFlags(t, cmd)
}
func (suite *TeamsChatsUnitSuite) TestBackupDeleteFlags() {
t := suite.T()
cmd := cliTD.SetUpCmdHasFlags(
t,
&cobra.Command{Use: deleteCommand},
addTeamsChatsCommands,
[]cliTD.UseCobraCommandFn{
flags.AddAllProviderFlags,
flags.AddAllStorageFlags,
},
flagsTD.WithFlags(
teamschatsServiceCommand,
[]string{
"--" + flags.RunModeFN, flags.RunModeFlagTest,
"--" + flags.BackupFN, flagsTD.BackupInput,
},
flagsTD.PreparedProviderFlags(),
flagsTD.PreparedStorageFlags()))
assert.Equal(t, flagsTD.BackupInput, flags.BackupIDFV)
flagsTD.AssertProviderFlags(t, cmd)
flagsTD.AssertStorageFlags(t, cmd)
}

View File

@ -10,6 +10,7 @@ import (
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/backup"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/debug"
"github.com/alcionai/corso/src/cli/export"
"github.com/alcionai/corso/src/cli/flags"
@ -19,7 +20,6 @@ import (
"github.com/alcionai/corso/src/cli/restore"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/logger"
)
@ -38,7 +38,7 @@ var corsoCmd = &cobra.Command{
}
func preRun(cc *cobra.Command, args []string) error {
if err := config.InitCmd(cc, args); err != nil {
if err := config.InitFunc(cc, args); err != nil {
return err
}

View File

@ -33,7 +33,8 @@ const (
)
var (
defaultConfigFilePath string
configFilePath string
configFilePathFlag string
configDir string
displayDefaultFP = filepath.Join("$HOME", ".corso.toml")
)
@ -57,7 +58,7 @@ func init() {
Infof(context.Background(), "cannot stat CORSO_CONFIG_DIR [%s]: %v", envDir, err)
} else {
configDir = envDir
defaultConfigFilePath = filepath.Join(configDir, ".corso.toml")
configFilePath = filepath.Join(configDir, ".corso.toml")
}
}
@ -68,7 +69,7 @@ func init() {
if len(configDir) == 0 {
configDir = homeDir
defaultConfigFilePath = filepath.Join(configDir, ".corso.toml")
configFilePath = filepath.Join(configDir, ".corso.toml")
}
}
@ -76,63 +77,40 @@ func init() {
func AddConfigFlags(cmd *cobra.Command) {
pf := cmd.PersistentFlags()
pf.StringVar(
&flags.ConfigFileFV,
flags.ConfigFileFN, displayDefaultFP, "config file location")
&configFilePathFlag,
"config-file", displayDefaultFP, "config file location")
}
// ---------------------------------------------------------------------------------------------------------
// Initialization & Storage
// ---------------------------------------------------------------------------------------------------------
// InitCmd provides a func that lazily initializes viper and
// InitFunc provides a func that lazily initializes viper and
// verifies that the configuration was able to read a file.
func InitCmd(cmd *cobra.Command, args []string) error {
_, err := commonInit(cmd.Context(), flags.ConfigFileFV)
return clues.Stack(err).OrNil()
}
func InitFunc(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
// InitConfig allows sdk consumers to initialize viper.
func InitConfig(
ctx context.Context,
userDefinedConfigFile string,
) (context.Context, error) {
return commonInit(ctx, userDefinedConfigFile)
}
func commonInit(
ctx context.Context,
userDefinedConfigFile string,
) (context.Context, error) {
fp := userDefinedConfigFile
fp := configFilePathFlag
if len(fp) == 0 || fp == displayDefaultFP {
fp = defaultConfigFilePath
fp = configFilePath
}
vpr := GetViper(ctx)
if err := initWithViper(ctx, vpr, fp); err != nil {
return ctx, err
if err := initWithViper(vpr, fp); err != nil {
return err
}
return SetViper(ctx, vpr), clues.Stack(Read(ctx)).OrNil()
ctx = SetViper(ctx, vpr)
return Read(ctx)
}
// initWithViper implements InitConfig, but takes in a viper
// struct for testing.
func initWithViper(
ctx context.Context,
vpr *viper.Viper,
configFP string,
) error {
logger.Ctx(ctx).Debugw("initializing viper", "config_file_path", configFP)
defer func() {
logger.Ctx(ctx).Debugw("initialized config", "config_file_path", configFP)
}()
func initWithViper(vpr *viper.Viper, configFP string) error {
// Configure default config file location
if len(configFP) == 0 || configFP == displayDefaultFP {
configFP = defaultConfigFilePath
// Find home directory.
_, err := os.Stat(configDir)
if err != nil {
@ -264,17 +242,19 @@ func writeRepoConfigWithViper(
return nil
}
// ReadCorsoConfig creates a storage and account instance by mediating all the possible
// GetStorageAndAccount creates a storage and account instance by mediating all the possible
// data sources (config file, env vars, flag overrides) and the config file.
func ReadCorsoConfig(
func GetConfigRepoDetails(
ctx context.Context,
provider storage.ProviderType,
readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string,
) (RepoDetails, error) {
) (
RepoDetails,
error,
) {
config, err := getStorageAndAccountWithViper(
ctx,
GetViper(ctx),
provider,
readFromFile,
@ -287,13 +267,15 @@ func ReadCorsoConfig(
// getSorageAndAccountWithViper implements GetSorageAndAccount, but takes in a viper
// struct for testing.
func getStorageAndAccountWithViper(
ctx context.Context,
vpr *viper.Viper,
provider storage.ProviderType,
readFromFile bool,
mustMatchFromConfig bool,
overrides map[string]string,
) (RepoDetails, error) {
) (
RepoDetails,
error,
) {
var (
config RepoDetails
err error
@ -303,9 +285,6 @@ func getStorageAndAccountWithViper(
// possibly read the prior config from a .corso file
if readFromFile {
ctx = clues.Add(ctx, "viper_config_file", vpr.ConfigFileUsed())
logger.Ctx(ctx).Debug("reading config from file")
if err := vpr.ReadInConfig(); err != nil {
configNotSet := errors.As(err, &viper.ConfigFileNotFoundError{})
configNotFound := errors.Is(err, fs.ErrNotExist)
@ -314,8 +293,6 @@ func getStorageAndAccountWithViper(
return config, clues.Wrap(err, "reading corso config file: "+vpr.ConfigFileUsed())
}
logger.Ctx(ctx).Info("config file not found")
readConfigFromViper = false
}

View File

@ -16,7 +16,6 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/its"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control/repository"
@ -150,9 +149,6 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
)
ctx, flush := tester.NewContext(t)
defer flush()
const (
bkt = "write-read-config-bucket"
tid = "3c0748d2-470e-444c-9064-1268e52609d5"
@ -161,7 +157,7 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
host = "some-host"
)
err := initWithViper(ctx, vpr, testConfigFilePath)
err := initWithViper(vpr, testConfigFilePath)
require.NoError(t, err, "initializing repo config", clues.ToCore(err))
s3Cfg := &storage.S3Config{
@ -209,15 +205,12 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
testConfigFilePath = filepath.Join(t.TempDir(), "corso.toml")
)
ctx, flush := tester.NewContext(t)
defer flush()
const (
bkt = "must-match-config-bucket"
tid = "dfb12063-7598-458b-85ab-42352c5c25e2"
)
err := initWithViper(ctx, vpr, testConfigFilePath)
err := initWithViper(vpr, testConfigFilePath)
require.NoError(t, err, "initializing repo config")
s3Cfg := &storage.S3Config{Bucket: bkt}
@ -292,9 +285,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
vpr = viper.New()
)
ctx, flush := tester.NewContext(t)
defer flush()
const (
b = "read-repo-config-basic-bucket"
tID = "6f34ac30-8196-469b-bf8f-d83deadbbbba"
@ -359,7 +349,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
flags.PassphraseFV = "passphrase-flags"
repoDetails, err := getStorageAndAccountWithViper(
ctx,
vpr,
storage.ProviderS3,
true,
@ -400,7 +389,6 @@ func (suite *ConfigSuite) TestReadFromFlags() {
type ConfigIntegrationSuite struct {
tester.Suite
m365 its.M365IntgTestSetup
}
func TestConfigIntegrationSuite(t *testing.T) {
@ -409,27 +397,21 @@ func TestConfigIntegrationSuite(t *testing.T) {
[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
}
func (suite *ConfigIntegrationSuite) SetupSuite() {
suite.m365 = its.GetM365(suite.T())
}
func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
t := suite.T()
vpr := viper.New()
ctx, flush := tester.NewContext(t)
defer flush()
const (
bkt = "get-storage-and-account-bucket"
end = "https://get-storage-and-account.com"
pfx = "get-storage-and-account-prefix/"
tid = "3a2faa4e-a882-445c-9d27-f552ef189381"
)
// Configure viper to read test config file
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
err := initWithViper(ctx, vpr, testConfigFilePath)
err := initWithViper(vpr, testConfigFilePath)
require.NoError(t, err, "initializing repo config", clues.ToCore(err))
s3Cfg := &storage.S3Config{
@ -439,10 +421,9 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
DoNotVerifyTLS: true,
DoNotUseTLS: true,
}
m365 := account.M365Config{AzureTenantID: tid}
creds := suite.m365.Creds
err = writeRepoConfigWithViper(vpr, s3Cfg, creds, repository.Options{}, "repoid")
err = writeRepoConfigWithViper(vpr, s3Cfg, m365, repository.Options{}, "repoid")
require.NoError(t, err, "writing repo config", clues.ToCore(err))
require.Equal(
@ -454,7 +435,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
err = vpr.ReadInConfig()
require.NoError(t, err, "reading repo config", clues.ToCore(err))
cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, true, true, nil)
cfg, err := getStorageAndAccountWithViper(vpr, storage.ProviderS3, true, true, nil)
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
readS3Cfg, err := cfg.Storage.ToS3Config()
@ -483,19 +464,17 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
t := suite.T()
vpr := viper.New()
ctx, flush := tester.NewContext(t)
defer flush()
const (
bkt = "get-storage-and-account-no-file-bucket"
end = "https://get-storage-and-account.com/no-file"
pfx = "get-storage-and-account-no-file-prefix/"
tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1"
)
creds := suite.m365.Creds
m365 := account.M365Config{AzureTenantID: tid}
overrides := map[string]string{
account.AzureTenantID: suite.m365.TenantID,
account.AzureTenantID: tid,
account.AccountProviderTypeKey: account.ProviderM365.String(),
storage.Bucket: bkt,
storage.Endpoint: end,
@ -505,7 +484,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
storage.StorageProviderTypeKey: storage.ProviderS3.String(),
}
cfg, err := getStorageAndAccountWithViper(ctx, vpr, storage.ProviderS3, false, true, overrides)
cfg, err := getStorageAndAccountWithViper(vpr, storage.ProviderS3, false, true, overrides)
require.NoError(t, err, "getting storage and account from config", clues.ToCore(err))
readS3Cfg, err := cfg.Storage.ToS3Config()
@ -524,7 +503,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
readM365, err := cfg.Account.M365Config()
require.NoError(t, err, "reading m365 config from account", clues.ToCore(err))
assert.Equal(t, readM365.AzureTenantID, creds.AzureTenantID)
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
}

View File

@ -2,6 +2,7 @@ package debug
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,13 +11,21 @@ import (
// called by debug.go to map subcommands to provider-specific handling.
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case metadataFilesCommand:
c, _ = utils.AddCommand(cmd, exchangeMetadataFilesCmd(), utils.MarkDebugCommand())
c, fs = utils.AddCommand(cmd, exchangeMetadataFilesCmd(), utils.MarkDebugCommand())
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
}

View File

@ -2,6 +2,7 @@ package debug
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,14 +11,21 @@ import (
// called by debug.go to map subcommands to provider-specific handling.
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case metadataFilesCommand:
c, _ = utils.AddCommand(cmd, groupsMetadataFilesCmd(), utils.MarkDebugCommand())
c, fs = utils.AddCommand(cmd, groupsMetadataFilesCmd(), utils.MarkDebugCommand())
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
}

View File

@ -2,6 +2,7 @@ package debug
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,13 +11,21 @@ import (
// called by debug.go to map subcommands to provider-specific handling.
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case metadataFilesCommand:
c, _ = utils.AddCommand(cmd, oneDriveMetadataFilesCmd(), utils.MarkDebugCommand())
c, fs = utils.AddCommand(cmd, oneDriveMetadataFilesCmd(), utils.MarkDebugCommand())
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
}

View File

@ -2,6 +2,7 @@ package debug
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,13 +11,21 @@ import (
// called by debug.go to map subcommands to provider-specific handling.
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case metadataFilesCommand:
c, _ = utils.AddCommand(cmd, sharePointMetadataFilesCmd(), utils.MarkDebugCommand())
c, fs = utils.AddCommand(cmd, sharePointMetadataFilesCmd(), utils.MarkDebugCommand())
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
}

View File

@ -3,6 +3,7 @@ package export
import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,14 +11,21 @@ import (
// called by export.go to map subcommands to provider-specific handling.
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case exportCommand:
c, _ = utils.AddCommand(cmd, exchangeExportCmd())
c, fs = utils.AddCommand(cmd, exchangeExportCmd())
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddExchangeDetailsAndRestoreFlags(c, true)
flags.AddExportConfigFlags(c)

View File

@ -11,11 +11,10 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -97,7 +96,7 @@ func runExport(
return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export"))
}
collections, err := eo.Run(ctx)
expColl, err := eo.Run(ctx)
if err != nil {
if errors.Is(err, data.ErrNotFound) {
return Only(ctx, clues.New("Backup or backup details missing for id "+backupID))
@ -106,18 +105,20 @@ func runExport(
return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export"))
}
if err = showExportProgress(ctx, eo, collections, exportLocation); err != nil {
return err
}
// It would be better to give a progressbar than a spinner, but we
// have any way of knowing how many files are available as of now.
diskWriteComplete := observe.MessageWithCompletion(ctx, observe.ProgressCfg{}, "Writing data to disk")
if len(eo.Errors.Recovered()) > 0 {
Infof(ctx, "\nExport failures")
err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors)
for _, i := range eo.Errors.Recovered() {
Err(ctx, i.Error())
}
// The progressbar has to be closed before we move on as the Infof
// below flushes progressbar to prevent clobbering the output and
// that causes the entire export operation to stall indefinitely.
// https://github.com/alcionai/corso/blob/8102523dc62c001b301cd2ab4e799f86146ab1a0/src/cli/print/print.go#L151
close(diskWriteComplete)
return Only(ctx, clues.New("Incomplete export of "+serviceName+" data"))
if err != nil {
return Only(ctx, err)
}
stats := eo.GetStats()
@ -131,23 +132,3 @@ func runExport(
return nil
}
// slim wrapper that allows us to defer the progress bar closure with the expected scope.
func showExportProgress(
ctx context.Context,
op operations.ExportOperation,
collections []export.Collectioner,
exportLocation string,
) error {
// It would be better to give a progressbar than a spinner, but we
// have any way of knowing how many files are available as of now.
progressMessage := observe.MessageWithCompletion(ctx, observe.DefaultCfg(), "Writing data to disk")
defer close(progressMessage)
err := export.ConsumeExportCollections(ctx, exportLocation, collections, op.Errors)
if err != nil {
return Only(ctx, err)
}
return nil
}

View File

@ -3,6 +3,7 @@ package export
import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -11,14 +12,21 @@ import (
// called by export.go to map subcommands to provider-specific handling.
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case exportCommand:
c, _ = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsExportCmd(), utils.MarkPreviewCommand())
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddSiteFlag(c, false)
flags.AddSiteIDFlag(c, false)
@ -50,13 +58,7 @@ corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd
# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to /my-exports
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd \
--folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00
# Export all posts from a conversation with topic "hello world" from group mailbox's last backup to /my-exports
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world"
# Export post with ID 98765abcdef from a conversation from group mailbox's last backup to /my-exports
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --conversation "hello world" --post 98765abcdef`
--folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00`
)
// `corso export groups [<flag>...] <destination>`

View File

@ -3,6 +3,7 @@ package export
import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,14 +11,21 @@ import (
// called by export.go to map subcommands to provider-specific handling.
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case exportCommand:
c, _ = utils.AddCommand(cmd, oneDriveExportCmd())
c, fs = utils.AddCommand(cmd, oneDriveExportCmd())
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddOneDriveDetailsAndRestoreFlags(c)
flags.AddExportConfigFlags(c)

View File

@ -3,6 +3,7 @@ package export
import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -10,14 +11,21 @@ import (
// called by export.go to map subcommands to provider-specific handling.
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case exportCommand:
c, _ = utils.AddCommand(cmd, sharePointExportCmd())
c, fs = utils.AddCommand(cmd, sharePointExportCmd())
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddSharePointDetailsAndRestoreFlags(c)
flags.AddExportConfigFlags(c)
@ -45,27 +53,7 @@ corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
# Export all files in the "Documents" library to the current directory.
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--library Documents --folder "Display Templates/Style Sheets" .
# Export lists by their name(s)
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list "list-name-1,list-name-2" .
# Export lists created after a given time
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-after 2024-01-01T12:23:34 .
# Export lists created before a given time
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-before 2024-01-01T12:23:34 .
# Export lists modified before a given time
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-before 2024-01-01T12:23:34 .
# Export lists modified after a given time
corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-after 2024-01-01T12:23:34 .`
--library Documents --folder "Display Templates/Style Sheets" .`
)
// `corso export sharepoint [<flag>...] <destination>`

View File

@ -60,11 +60,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
"--" + flags.FormatFN, flagsTD.FormatType,
@ -91,11 +88,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListItemInput, opts.ListItem)
assert.ElementsMatch(t, flagsTD.ListFolderInput, opts.ListFolder)
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
assert.Equal(t, flagsTD.Archive, opts.ExportCfg.Archive)

View File

@ -12,11 +12,9 @@ func AddAllBackupListFlags(cmd *cobra.Command) {
}
func AddFailedItemsFN(cmd *cobra.Command) {
fs := cmd.Flags()
fs.StringVar(
&FailedItemsFV, FailedItemsFN, Show,
cmd.Flags().StringVar(
&ListFailedItemsFV, FailedItemsFN, Show,
"Toggles showing or hiding the list of items that failed.")
cobra.CheckErr(fs.MarkHidden(FailedItemsFN))
}
func AddSkippedItemsFN(cmd *cobra.Command) {

View File

@ -28,6 +28,13 @@ func AddFilesystemFlags(cmd *cobra.Command) {
"",
"path to local or network storage")
cobra.CheckErr(cmd.MarkFlagRequired(FilesystemPathFN))
fs.BoolVar(
&SucceedIfExistsFV,
SucceedIfExistsFN,
false,
"Exit with success if the repo has already been initialized.")
cobra.CheckErr(fs.MarkHidden("succeed-if-exists"))
}
func FilesystemFlagOverrides(cmd *cobra.Command) map[string]string {

View File

@ -13,7 +13,7 @@ const (
FileModifiedAfterFN = "file-modified-after"
FileModifiedBeforeFN = "file-modified-before"
UseOldDeltaProcessFN = "use-old-delta-process"
UseDeltaTreeFN = "use-delta-tree"
)
var (
@ -25,7 +25,7 @@ var (
FileModifiedAfterFV string
FileModifiedBeforeFV string
UseOldDeltaProcessFV bool
UseDeltaTreeFV bool
)
// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the

View File

@ -6,45 +6,41 @@ import (
const (
AlertsFN = "alerts"
ConfigFileFN = "config-file"
DeltaPageSizeFN = "delta-page-size"
DisableDeltaFN = "disable-delta"
DisableIncrementalsFN = "disable-incrementals"
DisableLazyItemReaderFN = "disable-lazy-item-reader"
DisableSlidingWindowLimiterFN = "disable-sliding-window-limiter"
ForceItemDataDownloadFN = "force-item-data-download"
EnableImmutableIDFN = "enable-immutable-id"
FailFastFN = "fail-fast"
FailedItemsFN = "failed-items"
FetchParallelismFN = "fetch-parallelism"
NoPermissionsFN = "no-permissions"
NoStatsFN = "no-stats"
RecoveredErrorsFN = "recovered-errors"
NoPermissionsFN = "no-permissions"
RunModeFN = "run-mode"
SkippedItemsFN = "skipped-items"
SkipReduceFN = "skip-reduce"
)
var (
ConfigFileFV string
DeltaPageSizeFV int
DisableDeltaFV bool
DisableIncrementalsFV bool
DisableLazyItemReaderFV bool
DisableSlidingWindowLimiterFV bool
ForceItemDataDownloadFV bool
EnableImmutableIDFV bool
FailFastFV bool
FailedItemsFV string
FetchParallelismFV int
ListAlertsFV string
ListFailedItemsFV string
ListSkippedItemsFV string
ListRecoveredErrorsFV string
NoPermissionsFV bool
NoStatsFV bool
// RunMode describes the type of run, such as:
// flagtest, dry, run. Should default to 'run'.
RunModeFV string
NoPermissionsFV bool
SkipReduceFV bool
)
@ -180,19 +176,3 @@ func AddDisableSlidingWindowLimiterFlag(cmd *cobra.Command) {
"Disable sliding window rate limiter.")
cobra.CheckErr(fs.MarkHidden(DisableSlidingWindowLimiterFN))
}
// AddDisableLazyItemReader disables lazy item reader, such that we fall back to
// prefetch reader. This flag is currently only meant for groups conversations
// backup. Although it can be utilized for other services in future.
//
// This flag should only be used if lazy item reader is the default choice and
// we want to fallback to prefetch reader.
func AddDisableLazyItemReader(cmd *cobra.Command) {
fs := cmd.Flags()
fs.BoolVar(
&DisableLazyItemReaderFV,
DisableLazyItemReaderFN,
false,
"Disable lazy item reader.")
cobra.CheckErr(fs.MarkHidden(DisableLazyItemReaderFN))
}

View File

@ -14,6 +14,7 @@ const (
// Corso Flags
PassphraseFN = "passphrase"
NewPassphraseFN = "new-passphrase"
SucceedIfExistsFN = "succeed-if-exists"
)
var (
@ -24,6 +25,7 @@ var (
AWSSessionTokenFV string
PassphraseFV string
NewPhasephraseFV string
SucceedIfExistsFV bool
)
// AddMultipleBackupIDsFlag adds the --backups flag.

View File

@ -38,6 +38,11 @@ func AddS3BucketFlags(cmd *cobra.Command) {
fs.StringVar(&EndpointFV, EndpointFN, "", "S3 service endpoint.")
fs.BoolVar(&DoNotUseTLSFV, DoNotUseTLSFN, false, "Disable TLS (HTTPS)")
fs.BoolVar(&DoNotVerifyTLSFV, DoNotVerifyTLSFN, false, "Disable TLS (HTTPS) certificate verification.")
// In general, we don't want to expose this flag to users and have them mistake it
// for a broad-scale idempotency solution. We can un-hide it later the need arises.
fs.BoolVar(&SucceedIfExistsFV, SucceedIfExistsFN, false, "Exit with success if the repo has already been initialized.")
cobra.CheckErr(fs.MarkHidden("succeed-if-exists"))
}
func S3FlagOverrides(cmd *cobra.Command) map[string]string {

View File

@ -7,37 +7,24 @@ import (
const (
DataLibraries = "libraries"
DataPages = "pages"
DataLists = "lists"
)
const (
LibraryFN = "library"
ListFN = "list"
ListModifiedAfterFN = "list-modified-after"
ListModifiedBeforeFN = "list-modified-before"
ListCreatedAfterFN = "list-created-after"
ListCreatedBeforeFN = "list-created-before"
ListFolderFN = "list"
ListItemFN = "list-item"
PageFolderFN = "page-folder"
PageFN = "page"
SiteFN = "site" // site only accepts WebURL values
SiteIDFN = "site-id" // site-id accepts actual site ids
)
var (
LibraryFV string
ListFV []string
ListModifiedAfterFV string
ListModifiedBeforeFV string
ListCreatedAfterFV string
ListCreatedBeforeFV string
ListFolderFV []string
ListItemFV []string
PageFolderFV []string
PageFV []string
SiteIDFV []string
WebURLFV []string
)
@ -79,26 +66,17 @@ func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) {
"Select files modified before this datetime.")
// lists
fs.StringSliceVar(
&ListFV,
ListFN, nil,
"Select lists by name.")
fs.StringVar(
&ListModifiedAfterFV,
ListModifiedAfterFN, "",
"Select lists modified after this datetime.")
fs.StringVar(
&ListModifiedBeforeFV,
ListModifiedBeforeFN, "",
"Select lists modified before this datetime.")
fs.StringVar(
&ListCreatedAfterFV,
ListCreatedAfterFN, "",
"Select lists created after this datetime.")
fs.StringVar(
&ListCreatedBeforeFV,
ListCreatedBeforeFN, "",
"Select lists created before this datetime.")
&ListFolderFV,
ListFolderFN, nil,
"Select lists by name; accepts '"+Wildcard+"' to select all lists.")
cobra.CheckErr(fs.MarkHidden(ListFolderFN))
fs.StringSliceVar(
&ListItemFV,
ListItemFN, nil,
"Select lists by item name; accepts '"+Wildcard+"' to select all lists.")
cobra.CheckErr(fs.MarkHidden(ListItemFN))
// pages

View File

@ -1,13 +0,0 @@
package flags
import (
"github.com/spf13/cobra"
)
const (
DataChats = "chats"
)
func AddTeamsChatsDetailsAndRestoreFlags(cmd *cobra.Command) {
// TODO: add details flags
}

View File

@ -20,7 +20,7 @@ func PreparedBackupListFlags() []string {
func AssertBackupListFlags(t *testing.T, cmd *cobra.Command) {
assert.Equal(t, flags.Show, flags.ListAlertsFV)
assert.Equal(t, flags.Show, flags.FailedItemsFV)
assert.Equal(t, flags.Show, flags.ListFailedItemsFV)
assert.Equal(t, flags.Show, flags.ListSkippedItemsFV)
assert.Equal(t, flags.Show, flags.ListRecoveredErrorsFV)
}

View File

@ -21,7 +21,6 @@ var (
ExchangeCategoryDataInput = []string{"email", "events", "contacts"}
SharepointCategoryDataInput = []string{"files", "lists", "pages"}
GroupsCategoryDataInput = []string{"files", "lists", "pages", "messages"}
TeamsChatsCategoryDataInput = []string{"chats"}
ChannelInput = []string{"channel1", "channel2"}
MessageInput = []string{"message1", "message2"}
@ -60,11 +59,8 @@ var (
FileModifiedAfterInput = "fileModifiedAfter"
FileModifiedBeforeInput = "fileModifiedBefore"
ListsInput = []string{"listName1", "listName2"}
ListCreatedAfterInput = "listCreatedAfter"
ListCreatedBeforeInput = "listCreatedBefore"
ListModifiedAfterInput = "listModifiedAfter"
ListModifiedBeforeInput = "listModifiedBefore"
ListFolderInput = []string{"listFolder1", "listFolder2"}
ListItemInput = []string{"listItem1", "listItem2"}
PageFolderInput = []string{"pageFolder1", "pageFolder2"}
PageInput = []string{"page1", "page2"}

View File

@ -1,25 +0,0 @@
package testdata
import (
"testing"
"github.com/spf13/cobra"
)
func PreparedTeamsChatsFlags() []string {
return []string{
// FIXME: populate when adding filters
// "--" + flags.ChatCreatedAfterFN, ChatCreatedAfterInput,
// "--" + flags.ChatCreatedBeforeFN, ChatCreatedBeforeInput,
// "--" + flags.ChatLastMessageAfterFN, ChatLastMessageAfterInput,
// "--" + flags.ChatLastMessageBeforeFN, ChatLastMessageBeforeInput,
}
}
func AssertTeamsChatsFlags(t *testing.T, cmd *cobra.Command) {
// FIXME: populate when adding filters
// assert.Equal(t, ChatCreatedAfterInput, flags.ChatCreatedAfterFV)
// assert.Equal(t, ChatCreatedBeforeInput, flags.ChatCreatedBeforeFV)
// assert.Equal(t, ChatLastMessageAfterInput, flags.ChatLastMessageAfterFV)
// assert.Equal(t, ChatLastMessageBeforeInput, flags.ChatLastMessageBeforeFV)
}

View File

@ -133,7 +133,7 @@ func Pretty(ctx context.Context, a any) {
return
}
printPrettyJSON(ctx, getRootCmd(ctx).ErrOrStderr(), a)
printPrettyJSON(getRootCmd(ctx).ErrOrStderr(), a)
}
// PrettyJSON prettifies and prints the value.
@ -143,7 +143,7 @@ func PrettyJSON(ctx context.Context, p minimumPrintabler) {
return
}
outputJSON(ctx, getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
outputJSON(getRootCmd(ctx).ErrOrStderr(), p, outputAsJSONDebug)
}
// out is the testable core of exported print funcs
@ -193,56 +193,56 @@ type minimumPrintabler interface {
// Item prints the printable, according to the caller's requested format.
func Item(ctx context.Context, p Printable) {
printItem(ctx, getRootCmd(ctx).OutOrStdout(), p)
printItem(getRootCmd(ctx).OutOrStdout(), p)
}
// print prints the printable items,
// according to the caller's requested format.
func printItem(ctx context.Context, w io.Writer, p Printable) {
func printItem(w io.Writer, p Printable) {
if outputAsJSON || outputAsJSONDebug {
outputJSON(ctx, w, p, outputAsJSONDebug)
outputJSON(w, p, outputAsJSONDebug)
return
}
outputTable(ctx, w, []Printable{p})
outputTable(w, []Printable{p})
}
// ItemProperties prints the printable either as in a single line or a json
// The difference between this and Item is that this one does not print the ID
func ItemProperties(ctx context.Context, p Printable) {
printItemProperties(ctx, getRootCmd(ctx).OutOrStdout(), p)
printItemProperties(getRootCmd(ctx).OutOrStdout(), p)
}
// print prints the printable items,
// according to the caller's requested format.
func printItemProperties(ctx context.Context, w io.Writer, p Printable) {
func printItemProperties(w io.Writer, p Printable) {
if outputAsJSON || outputAsJSONDebug {
outputJSON(ctx, w, p, outputAsJSONDebug)
outputJSON(w, p, outputAsJSONDebug)
return
}
outputOneLine(ctx, w, []Printable{p})
outputOneLine(w, []Printable{p})
}
// All prints the slice of printable items,
// according to the caller's requested format.
func All(ctx context.Context, ps ...Printable) {
printAll(ctx, getRootCmd(ctx).OutOrStdout(), ps)
printAll(getRootCmd(ctx).OutOrStdout(), ps)
}
// printAll prints the slice of printable items,
// according to the caller's requested format.
func printAll(ctx context.Context, w io.Writer, ps []Printable) {
func printAll(w io.Writer, ps []Printable) {
if len(ps) == 0 {
return
}
if outputAsJSON || outputAsJSONDebug {
outputJSONArr(ctx, w, ps, outputAsJSONDebug)
outputJSONArr(w, ps, outputAsJSONDebug)
return
}
outputTable(ctx, w, ps)
outputTable(w, ps)
}
// ------------------------------------------------------------------------------------------
@ -252,11 +252,11 @@ func printAll(ctx context.Context, w io.Writer, ps []Printable) {
// Table writes the printables in a tabular format. Takes headers from
// the 0th printable only.
func Table(ctx context.Context, ps []Printable) {
outputTable(ctx, getRootCmd(ctx).OutOrStdout(), ps)
outputTable(getRootCmd(ctx).OutOrStdout(), ps)
}
// output to stdout the list of printable structs in a table
func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
func outputTable(w io.Writer, ps []Printable) {
t := table.Table{
Headers: ps[0].Headers(false),
Rows: [][]string{},
@ -266,9 +266,6 @@ func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
t.Rows = append(t.Rows, p.Values(false))
}
// observe bars needs to be flushed before printing
observe.Flush(ctx)
_ = t.WriteTable(
w,
&table.Config{
@ -282,20 +279,20 @@ func outputTable(ctx context.Context, w io.Writer, ps []Printable) {
// JSON
// ------------------------------------------------------------------------------------------
func outputJSON(ctx context.Context, w io.Writer, p minimumPrintabler, debug bool) {
func outputJSON(w io.Writer, p minimumPrintabler, debug bool) {
if debug {
printJSON(ctx, w, p)
printJSON(w, p)
return
}
if debug {
printJSON(ctx, w, p)
printJSON(w, p)
} else {
printJSON(ctx, w, p.MinimumPrintable())
printJSON(w, p.MinimumPrintable())
}
}
func outputJSONArr(ctx context.Context, w io.Writer, ps []Printable, debug bool) {
func outputJSONArr(w io.Writer, ps []Printable, debug bool) {
sl := make([]any, 0, len(ps))
for _, p := range ps {
@ -306,14 +303,11 @@ func outputJSONArr(ctx context.Context, w io.Writer, ps []Printable, debug bool)
}
}
printJSON(ctx, w, sl)
printJSON(w, sl)
}
// output to stdout the list of printable structs as json.
func printJSON(ctx context.Context, w io.Writer, a any) {
// observe bars needs to be flushed before printing
observe.Flush(ctx)
func printJSON(w io.Writer, a any) {
bs, err := json.Marshal(a)
if err != nil {
fmt.Fprintf(w, "error formatting results to json: %v\n", err)
@ -324,10 +318,7 @@ func printJSON(ctx context.Context, w io.Writer, a any) {
}
// output to stdout the list of printable structs as prettified json.
func printPrettyJSON(ctx context.Context, w io.Writer, a any) {
// observe bars needs to be flushed before printing
observe.Flush(ctx)
func printPrettyJSON(w io.Writer, a any) {
bs, err := json.MarshalIndent(a, "", " ")
if err != nil {
fmt.Fprintf(w, "error formatting results to json: %v\n", err)
@ -343,10 +334,7 @@ func printPrettyJSON(ctx context.Context, w io.Writer, a any) {
// Output in the following format:
// Bytes Uploaded: 401 kB | Items Uploaded: 59 | Items Skipped: 0 | Errors: 0
func outputOneLine(ctx context.Context, w io.Writer, ps []Printable) {
// observe bars needs to be flushed before printing
observe.Flush(ctx)
func outputOneLine(w io.Writer, ps []Printable) {
headers := ps[0].Headers(true)
rows := [][]string{}

View File

@ -2,13 +2,14 @@ package repo
import (
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/config"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -72,7 +73,7 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
overrides[flags.FilesystemPathFN] = abs
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderFilesystem,
true,
@ -109,6 +110,10 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
ric := repository.InitConfig{RetentionOpts: retentionOpts}
if err = r.Initialize(ctx, ric); err != nil {
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
return nil
}
return Only(ctx, clues.Stack(ErrInitializingRepo, err))
}
@ -158,7 +163,7 @@ func connectFilesystemCmd(cmd *cobra.Command, args []string) error {
overrides[flags.FilesystemPathFN] = abs
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderFilesystem,
true,

View File

@ -5,16 +5,16 @@ import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/config"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -73,7 +73,7 @@ func (suite *FilesystemE2ESuite) TestInitFilesystemCmd() {
cmd := cliTD.StubRootCmd(
"repo", "init", "filesystem",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--path", cfg.Path)
cli.BuildCommandTree(cmd)
@ -81,9 +81,9 @@ func (suite *FilesystemE2ESuite) TestInitFilesystemCmd() {
err = cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// noop
// a second initialization should result in an error
err = cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists, clues.ToCore(err))
})
}
}
@ -143,7 +143,7 @@ func (suite *FilesystemE2ESuite) TestConnectFilesystemCmd() {
// then test it
cmd := cliTD.StubRootCmd(
"repo", "connect", "filesystem",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--path", cfg.Path)
cli.BuildCommandTree(cmd)

View File

@ -10,12 +10,11 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/repo"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/storage"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
)
@ -77,7 +76,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
cmd := cliTD.StubRootCmd(
"repo", "init", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)
@ -89,7 +88,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
// connect with old passphrase
cmd = cliTD.StubRootCmd(
"repo", "connect", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", cfg.Bucket,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)
@ -100,7 +99,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
cmd = cliTD.StubRootCmd(
"repo", "update-passphrase",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--new-passphrase", "newpass")
cli.BuildCommandTree(cmd)
@ -111,7 +110,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
// connect again with new passphrase
cmd = cliTD.StubRootCmd(
"repo", "connect", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", cfg.Bucket,
"--prefix", cfg.Prefix,
"--passphrase", "newpass")
@ -124,7 +123,7 @@ func (suite *RepoE2ESuite) TestUpdatePassphraseCmd() {
// connect with old passphrase - it will fail
cmd = cliTD.StubRootCmd(
"repo", "connect", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", cfg.Bucket,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)

View File

@ -4,13 +4,14 @@ import (
"strings"
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -84,7 +85,7 @@ func s3InitCmd() *cobra.Command {
func initS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderS3,
true,
@ -131,6 +132,10 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
ric := repository.InitConfig{RetentionOpts: retentionOpts}
if err = r.Initialize(ctx, ric); err != nil {
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
return nil
}
return Only(ctx, clues.Stack(ErrInitializingRepo, err))
}
@ -165,7 +170,7 @@ func s3ConnectCmd() *cobra.Command {
func connectS3Cmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderS3,
true,

View File

@ -11,13 +11,12 @@ import (
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/config"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -80,7 +79,7 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
cmd := cliTD.StubRootCmd(
"repo", "init", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", test.bucketPrefix+cfg.Bucket,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)
@ -89,9 +88,9 @@ func (suite *S3E2ESuite) TestInitS3Cmd() {
err = cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
// noop
// a second initialization should result in an error
err = cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))
assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists, clues.ToCore(err))
})
}
}
@ -114,9 +113,10 @@ func (suite *S3E2ESuite) TestInitMultipleTimes() {
for i := 0; i < 2; i++ {
cmd := cliTD.StubRootCmd(
"repo", "init", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", cfg.Bucket,
"--prefix", cfg.Prefix)
"--prefix", cfg.Prefix,
"--succeed-if-exists")
cli.BuildCommandTree(cmd)
// run the command
@ -146,7 +146,7 @@ func (suite *S3E2ESuite) TestInitS3Cmd_missingBucket() {
cmd := cliTD.StubRootCmd(
"repo", "init", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)
@ -219,7 +219,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
// then test it
cmd := cliTD.StubRootCmd(
"repo", "connect", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", test.bucketPrefix+cfg.Bucket,
"--prefix", cfg.Prefix)
cli.BuildCommandTree(cmd)
@ -279,7 +279,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd_badInputs() {
cmd := cliTD.StubRootCmd(
"repo", "connect", "s3",
"--"+flags.ConfigFileFN, configFP,
"--config-file", configFP,
"--bucket", bucket,
"--prefix", prefix)
cli.BuildCommandTree(cmd)

View File

@ -2,6 +2,7 @@ package restore
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
@ -9,14 +10,22 @@ import (
// called by restore.go to map subcommands to provider-specific handling.
func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case restoreCommand:
c, _ = utils.AddCommand(cmd, exchangeRestoreCmd())
c, fs = utils.AddCommand(cmd, exchangeRestoreCmd())
c.Use = c.Use + " " + exchangeServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
// general flags
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddExchangeDetailsAndRestoreFlags(c, false)
flags.AddRestoreConfigFlags(c, true)

View File

@ -11,6 +11,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
@ -18,7 +19,6 @@ import (
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
@ -145,7 +145,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd() {
cmd := cliTD.StubRootCmd(
"restore", "exchange",
"--"+flags.ConfigFileFN, suite.cfgFP,
"--config-file", suite.cfgFP,
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID))
cli.BuildCommandTree(cmd)
@ -180,7 +180,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd_badTimeFlags() {
cmd := cliTD.StubRootCmd(
"restore", "exchange",
"--"+flags.ConfigFileFN, suite.cfgFP,
"--config-file", suite.cfgFP,
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID),
timeFilter, "smarf")
cli.BuildCommandTree(cmd)
@ -214,7 +214,7 @@ func (suite *RestoreExchangeE2ESuite) TestExchangeRestoreCmd_badBoolFlags() {
cmd := cliTD.StubRootCmd(
"restore", "exchange",
"--"+flags.ConfigFileFN, suite.cfgFP,
"--config-file", suite.cfgFP,
"--"+flags.BackupFN, string(suite.backupOps[set].Results.BackupID),
timeFilter, "wingbat")
cli.BuildCommandTree(cmd)

View File

@ -2,23 +2,30 @@ package restore
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/internal/common/dttm"
)
// called by restore.go to map subcommands to provider-specific handling.
func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case restoreCommand:
c, _ = utils.AddCommand(cmd, groupsRestoreCmd(), utils.MarkPreviewCommand())
c, fs = utils.AddCommand(cmd, groupsRestoreCmd(), utils.MarkPreviewCommand())
c.Use = c.Use + " " + groupsServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddSiteFlag(c, false)
flags.AddSiteIDFlag(c, false)
@ -84,10 +91,6 @@ func restoreGroupsCmd(cmd *cobra.Command, args []string) error {
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
utils.FilterGroupsRestoreInfoSelectors(sel, opts)
// TODO(pandeyabs): Exclude conversations from restores since they are not
// supported yet.
sel.Exclude(sel.Conversation(selectors.Any()))
return runRestore(
ctx,
cmd,

View File

@ -60,7 +60,8 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
"--" + flags.CollisionsFN, flagsTD.Collisions,
@ -91,7 +92,6 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.Equal(t, flagsTD.Collisions, opts.RestoreCfg.Collisions)
assert.Equal(t, flagsTD.Destination, opts.RestoreCfg.Destination)
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
// assert.Equal(t, flagsTD.ToResource, opts.RestoreCfg.ProtectedResource)
assert.True(t, flags.NoPermissionsFV)
flagsTD.AssertProviderFlags(t, cmd)

View File

@ -2,22 +2,30 @@ package restore
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/internal/common/dttm"
)
// called by restore.go to map subcommands to provider-specific handling.
func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case restoreCommand:
c, _ = utils.AddCommand(cmd, oneDriveRestoreCmd())
c, fs = utils.AddCommand(cmd, oneDriveRestoreCmd())
c.Use = c.Use + " " + oneDriveServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddOneDriveDetailsAndRestoreFlags(c)
flags.AddNoPermissionsFlag(c)

View File

@ -2,22 +2,30 @@ package restore
import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/internal/common/dttm"
)
// called by restore.go to map subcommands to provider-specific handling.
func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
var c *cobra.Command
var (
c *cobra.Command
fs *pflag.FlagSet
)
switch cmd.Use {
case restoreCommand:
c, _ = utils.AddCommand(cmd, sharePointRestoreCmd())
c, fs = utils.AddCommand(cmd, sharePointRestoreCmd())
c.Use = c.Use + " " + sharePointServiceCommandUseSuffix
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --site) and more frequently used flags take precedence.
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddSharePointDetailsAndRestoreFlags(c)
flags.AddNoPermissionsFlag(c)
@ -50,27 +58,7 @@ corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
# Restore all files in the "Documents" library.
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--library Documents --folder "Display Templates/Style Sheets"
# Restore lists by their name(s)
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list "list-name-1,list-name-2"
# Restore lists created after a given time
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-after 2024-01-01T12:23:34
# Restore lists created before a given time
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-created-before 2024-01-01T12:23:34
# Restore lists modified before a given time
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-before 2024-01-01T12:23:34
# Restore lists modified after a given time
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--list-modified-after 2024-01-01T12:23:34`
--library Documents --folder "Display Templates/Style Sheets" `
)
// `corso restore sharepoint [<flag>...]`

View File

@ -59,11 +59,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.FileCreatedBeforeFN, flagsTD.FileCreatedBeforeInput,
"--" + flags.FileModifiedAfterFN, flagsTD.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, flagsTD.FileModifiedBeforeInput,
"--" + flags.ListFN, flagsTD.FlgInputs(flagsTD.ListsInput),
"--" + flags.ListCreatedAfterFN, flagsTD.ListCreatedAfterInput,
"--" + flags.ListCreatedBeforeFN, flagsTD.ListCreatedBeforeInput,
"--" + flags.ListModifiedAfterFN, flagsTD.ListModifiedAfterInput,
"--" + flags.ListModifiedBeforeFN, flagsTD.ListModifiedBeforeInput,
"--" + flags.ListItemFN, flagsTD.FlgInputs(flagsTD.ListItemInput),
"--" + flags.ListFolderFN, flagsTD.FlgInputs(flagsTD.ListFolderInput),
"--" + flags.PageFN, flagsTD.FlgInputs(flagsTD.PageInput),
"--" + flags.PageFolderFN, flagsTD.FlgInputs(flagsTD.PageFolderInput),
"--" + flags.CollisionsFN, flagsTD.Collisions,
@ -92,11 +89,8 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, flagsTD.FileCreatedBeforeInput, opts.FileCreatedBefore)
assert.Equal(t, flagsTD.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, flagsTD.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListsInput, opts.Lists)
assert.Equal(t, flagsTD.ListCreatedAfterInput, opts.ListCreatedAfter)
assert.Equal(t, flagsTD.ListCreatedBeforeInput, opts.ListCreatedBefore)
assert.Equal(t, flagsTD.ListModifiedAfterInput, opts.ListModifiedAfter)
assert.Equal(t, flagsTD.ListModifiedBeforeInput, opts.ListModifiedBefore)
assert.ElementsMatch(t, flagsTD.ListItemInput, opts.ListItem)
assert.ElementsMatch(t, flagsTD.ListFolderInput, opts.ListFolder)
assert.ElementsMatch(t, flagsTD.PageInput, opts.Page)
assert.ElementsMatch(t, flagsTD.PageFolderInput, opts.PageFolder)
assert.Equal(t, flagsTD.Collisions, opts.RestoreCfg.Collisions)

View File

@ -9,8 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/selectors"
)

View File

@ -1,13 +1,9 @@
package utils
import (
"errors"
"strconv"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -43,55 +39,3 @@ func trimFolderSlash(folders []string) []string {
return res
}
func validateCommonTimeFlags(opts any) error {
timeFlags := []string{
flags.FileCreatedAfterFN,
flags.FileCreatedBeforeFN,
flags.FileModifiedAfterFN,
flags.FileModifiedBeforeFN,
flags.ListCreatedAfterFN,
flags.ListCreatedBeforeFN,
flags.ListModifiedAfterFN,
flags.ListModifiedBeforeFN,
}
isFlagPopulated := func(opts any, flag string) bool {
switch opts := opts.(type) {
case GroupsOpts:
_, ok := opts.Populated[flag]
return ok
case SharePointOpts:
_, ok := opts.Populated[flag]
return ok
default:
return false
}
}
getTimeField := func(opts any, flag string) (string, error) {
switch opts := opts.(type) {
case GroupsOpts:
return opts.GetFileTimeField(flag), nil
case SharePointOpts:
return opts.GetFileTimeField(flag), nil
default:
return "", errors.New("unsupported type")
}
}
for _, flag := range timeFlags {
if populated := isFlagPopulated(opts, flag); populated {
timeField, err := getTimeField(opts, flag)
if err != nil {
return err
}
if !IsValidTimeFormat(timeField) {
return clues.New("invalid time format for " + flag)
}
}
}
return nil
}

View File

@ -103,6 +103,7 @@ func (suite *FlagUnitSuite) TestAddS3BucketFlags() {
assert.Equal(t, "prefix1", flags.PrefixFV, flags.PrefixFN)
assert.True(t, flags.DoNotUseTLSFV, flags.DoNotUseTLSFN)
assert.True(t, flags.DoNotVerifyTLSFV, flags.DoNotVerifyTLSFN)
assert.True(t, flags.SucceedIfExistsFV, flags.SucceedIfExistsFN)
},
}
@ -115,6 +116,7 @@ func (suite *FlagUnitSuite) TestAddS3BucketFlags() {
"--" + flags.PrefixFN, "prefix1",
"--" + flags.DoNotUseTLSFN,
"--" + flags.DoNotVerifyTLSFN,
"--" + flags.SucceedIfExistsFN,
})
err := cmd.Execute()
@ -128,6 +130,7 @@ func (suite *FlagUnitSuite) TestFilesystemFlags() {
Use: "test",
Run: func(cmd *cobra.Command, args []string) {
assert.Equal(t, "/tmp/test", flags.FilesystemPathFV, flags.FilesystemPathFN)
assert.True(t, flags.SucceedIfExistsFV, flags.SucceedIfExistsFN)
assert.Equal(t, "tenantID", flags.AzureClientTenantFV, flags.AzureClientTenantFN)
assert.Equal(t, "clientID", flags.AzureClientIDFV, flags.AzureClientIDFN)
assert.Equal(t, "secret", flags.AzureClientSecretFV, flags.AzureClientSecretFN)
@ -140,6 +143,7 @@ func (suite *FlagUnitSuite) TestFilesystemFlags() {
cmd.SetArgs([]string{
"test",
"--" + flags.FilesystemPathFN, "/tmp/test",
"--" + flags.SucceedIfExistsFN,
"--" + flags.AzureClientIDFN, "clientID",
"--" + flags.AzureClientTenantFN, "tenantID",
"--" + flags.AzureClientSecretFN, "secret",

View File

@ -32,7 +32,8 @@ type GroupsOpts struct {
FileModifiedAfter string
FileModifiedBefore string
Lists []string
ListFolder []string
ListItem []string
PageFolder []string
Page []string
@ -43,21 +44,6 @@ type GroupsOpts struct {
Populated flags.PopulatedFlags
}
func (g GroupsOpts) GetFileTimeField(flag string) string {
switch flag {
case flags.FileCreatedAfterFN:
return g.FileCreatedAfter
case flags.FileCreatedBeforeFN:
return g.FileCreatedBefore
case flags.FileModifiedAfterFN:
return g.FileModifiedAfter
case flags.FileModifiedBeforeFN:
return g.FileModifiedBefore
default:
return ""
}
}
func GroupsAllowedCategories() map[string]struct{} {
return map[string]struct{}{
flags.DataLibraries: {},
@ -107,7 +93,8 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
MessageLastReplyAfter: flags.MessageLastReplyAfterFV,
MessageLastReplyBefore: flags.MessageLastReplyBeforeFV,
Lists: flags.ListFV,
ListFolder: flags.ListFolderFV,
ListItem: flags.ListItemFV,
Page: flags.PageFV,
PageFolder: flags.PageFolderFV,
@ -139,6 +126,22 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts, isRestore bool
}
}
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
}
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
}
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
}
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
}
if _, ok := opts.Populated[flags.MessageCreatedAfterFN]; ok && !IsValidTimeFormat(opts.MessageCreatedAfter) {
return clues.New("invalid time format for " + flags.MessageCreatedAfterFN)
}
@ -155,7 +158,7 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts, isRestore bool
return clues.New("invalid time format for " + flags.MessageLastReplyBeforeFN)
}
return validateCommonTimeFlags(opts)
return nil
}
// AddGroupsFilter adds the scope of the provided values to the selector's
@ -178,7 +181,7 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
var (
groups = opts.Groups
folderPaths, fileNames = len(opts.FolderPath), len(opts.FileName)
lists = len(opts.Lists)
listFolders, listItems = len(opts.ListFolder), len(opts.ListItem)
pageFolders, pageItems = len(opts.PageFolder), len(opts.Page)
chans, chanMsgs = len(opts.Channels), len(opts.Messages)
convs, convPosts = len(opts.Conversations), len(opts.Posts)
@ -191,7 +194,7 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
sel := selectors.NewGroupsRestore(groups)
if folderPaths+fileNames+
lists+
listFolders+listItems+
pageFolders+pageItems+
chans+chanMsgs+
convs+convPosts == 0 {
@ -201,6 +204,9 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
// sharepoint site selectors
if folderPaths+fileNames+
listFolders+listItems+
pageFolders+pageItems > 0 {
if folderPaths+fileNames > 0 {
if fileNames == 0 {
opts.FileName = selectors.Any()
@ -218,10 +224,21 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
}
}
if lists > 0 {
opts.Lists = trimFolderSlash(opts.Lists)
sel.Include(sel.ListItems(opts.Lists, opts.Lists, selectors.StrictEqualMatch()))
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
if listFolders+listItems > 0 {
if listItems == 0 {
opts.ListItem = selectors.Any()
}
opts.ListFolder = trimFolderSlash(opts.ListFolder)
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListFolder)
if len(containsFolders) > 0 {
sel.Include(sel.ListItems(containsFolders, opts.ListItem))
}
if len(prefixFolders) > 0 {
sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch()))
}
}
if pageFolders+pageItems > 0 {
@ -240,6 +257,7 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch()))
}
}
}
// channel and message selectors
@ -266,14 +284,9 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
opts.Conversations = selectors.Any()
}
// if no post is specified, select all posts in the conversation
if convPosts == 0 {
opts.Posts = selectors.Any()
}
// if no post is specified, only select conversations;
// otherwise, look for conv/post pairs
if convs == 0 {
// otherwise, look for channel/message pairs
if chanMsgs == 0 {
sel.Include(sel.Conversation(opts.Conversations))
} else {
sel.Include(sel.ConversationPosts(opts.Conversations, opts.Posts))

View File

@ -8,8 +8,8 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -30,7 +30,6 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
containsOnly = []string{"contains"}
prefixOnly = []string{"/prefix"}
containsAndPrefix = []string{"contains", "/prefix"}
listNames = []string{"list-name1"}
onlySlash = []string{string(path.PathSeparator)}
)
@ -43,28 +42,32 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
{
name: "no inputs",
opts: utils.GroupsOpts{},
expectIncludeLen: 3,
// TODO: bump to 3 when we release conversations
expectIncludeLen: 2,
},
{
name: "empty",
opts: utils.GroupsOpts{
Groups: empty,
},
expectIncludeLen: 3,
// TODO: bump to 3 when we release conversations
expectIncludeLen: 2,
},
{
name: "single inputs",
opts: utils.GroupsOpts{
Groups: single,
},
expectIncludeLen: 3,
// TODO: bump to 3 when we release conversations
expectIncludeLen: 2,
},
{
name: "multi inputs",
opts: utils.GroupsOpts{
Groups: multi,
},
expectIncludeLen: 3,
// TODO: bump to 3 when we release conversations
expectIncludeLen: 2,
},
// sharepoint
{
@ -92,12 +95,29 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
expectIncludeLen: 2,
},
{
name: "list names",
name: "list contains",
opts: utils.GroupsOpts{
Lists: listNames,
FileName: empty,
FolderPath: empty,
ListItem: empty,
ListFolder: containsOnly,
},
expectIncludeLen: 1,
},
{
name: "list prefixes",
opts: utils.GroupsOpts{
ListFolder: prefixOnly,
},
expectIncludeLen: 1,
},
{
name: "list prefixes and contains",
opts: utils.GroupsOpts{
ListFolder: containsAndPrefix,
},
expectIncludeLen: 2,
},
{
name: "Page Folder",
opts: utils.GroupsOpts{
@ -401,7 +421,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
{
name: "none",
cats: []string{},
expectScopeLen: 3,
expectScopeLen: 2,
},
{
name: "libraries",
@ -423,9 +443,10 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
cats: []string{
flags.DataLibraries,
flags.DataMessages,
flags.DataConversations,
// flags.DataConversations,
},
expectScopeLen: 3,
// TODO: bump to 3 when we include conversations in all data
expectScopeLen: 2,
},
{
name: "bad inputs",

View File

@ -1,8 +1,8 @@
package utils
import (
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
)
@ -26,9 +26,8 @@ func Control() control.Options {
opt.ToggleFeatures.ForceItemDataDownload = flags.ForceItemDataDownloadFV
opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV
opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
opt.ToggleFeatures.DisableLazyItemReader = flags.DisableLazyItemReaderFV
opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV
opt.ToggleFeatures.UseOldDeltaProcess = flags.UseOldDeltaProcessFV
opt.ToggleFeatures.UseDeltaTree = flags.UseDeltaTreeFV
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
return opt
@ -42,27 +41,3 @@ func ControlWithConfig(cfg config.RepoDetails) control.Options {
return opt
}
func ParseBackupOptions() control.BackupConfig {
opt := control.DefaultBackupConfig()
if flags.FailFastFV {
opt.FailureHandling = control.FailFast
}
dps := int32(flags.DeltaPageSizeFV)
if dps > 500 || dps < 1 {
dps = 500
}
opt.M365.DeltaPageSize = dps
opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
opt.M365.UseOldDriveDeltaProcess = flags.UseOldDeltaProcessFV
opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
opt.Parallelism.ItemFetch = flags.FetchParallelismFV
opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV
opt.Incrementals.ForceItemDataRefresh = flags.ForceItemDataDownloadFV
return opt
}

View File

@ -9,8 +9,8 @@ import (
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/dttm"
)
type RestoreCfgOpts struct {

View File

@ -25,11 +25,8 @@ type SharePointOpts struct {
FileModifiedAfter string
FileModifiedBefore string
Lists []string
ListModifiedAfter string
ListModifiedBefore string
ListCreatedBefore string
ListCreatedAfter string
ListFolder []string
ListItem []string
PageFolder []string
Page []string
@ -40,29 +37,6 @@ type SharePointOpts struct {
Populated flags.PopulatedFlags
}
func (s SharePointOpts) GetFileTimeField(flag string) string {
switch flag {
case flags.FileCreatedAfterFN:
return s.FileCreatedAfter
case flags.FileCreatedBeforeFN:
return s.FileCreatedBefore
case flags.FileModifiedAfterFN:
return s.FileModifiedAfter
case flags.FileModifiedBeforeFN:
return s.FileModifiedBefore
case flags.ListModifiedAfterFN:
return s.ListModifiedAfter
case flags.ListModifiedBeforeFN:
return s.ListModifiedBefore
case flags.ListCreatedBeforeFN:
return s.ListCreatedBefore
case flags.ListCreatedAfterFN:
return s.ListCreatedAfter
default:
return ""
}
}
func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
return SharePointOpts{
SiteID: flags.SiteIDFV,
@ -76,11 +50,8 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
FileModifiedAfter: flags.FileModifiedAfterFV,
FileModifiedBefore: flags.FileModifiedBeforeFV,
Lists: flags.ListFV,
ListModifiedAfter: flags.ListModifiedAfterFV,
ListModifiedBefore: flags.ListModifiedBeforeFV,
ListCreatedAfter: flags.ListCreatedAfterFV,
ListCreatedBefore: flags.ListCreatedBeforeFV,
ListFolder: flags.ListFolderFV,
ListItem: flags.ListItemFV,
Page: flags.PageFV,
PageFolder: flags.PageFolderFV,
@ -95,32 +66,6 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
}
}
func SharePointAllowedCategories() map[string]struct{} {
return map[string]struct{}{
flags.DataLibraries: {},
flags.DataLists: {},
}
}
func AddCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
if len(cats) == 0 {
// [TODO](hitesh) to enable lists without being invoked explicitly via --data flag
// sel.Include(sel.LibraryFolders(selectors.Any()), sel.Lists(selectors.Any()))
sel.Include(sel.LibraryFolders(selectors.Any()))
}
for _, d := range cats {
switch d {
case flags.DataLists:
sel.Include(sel.Lists(selectors.Any()))
case flags.DataLibraries:
sel.Include(sel.LibraryFolders(selectors.Any()))
}
}
return sel
}
// ValidateSharePointRestoreFlags checks common flags for correctness and interdependencies
func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error {
if len(backupID) == 0 {
@ -136,7 +81,23 @@ func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error
}
}
return validateCommonTimeFlags(opts)
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
}
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
}
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
}
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
}
return nil
}
// AddSharePointInfo adds the scope of the provided values to the selector's
@ -158,24 +119,24 @@ func AddSharePointInfo(
func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointOpts) *selectors.SharePointRestore {
sites := opts.SiteID
folderPaths, fileNames := len(opts.FolderPath), len(opts.FileName)
siteIDs, webUrls := len(opts.SiteID), len(opts.WebURL)
lists := len(opts.Lists)
pageFolders, pageItems := len(opts.PageFolder), len(opts.Page)
lfp, lfn := len(opts.FolderPath), len(opts.FileName)
ls, lwu := len(opts.SiteID), len(opts.WebURL)
slp, sli := len(opts.ListFolder), len(opts.ListItem)
pf, pi := len(opts.PageFolder), len(opts.Page)
if siteIDs == 0 {
if ls == 0 {
sites = selectors.Any()
}
sel := selectors.NewSharePointRestore(sites)
if folderPaths+fileNames+webUrls+lists+pageFolders+pageItems == 0 {
if lfp+lfn+lwu+slp+sli+pf+pi == 0 {
sel.Include(sel.AllData())
return sel
}
if folderPaths+fileNames > 0 {
if fileNames == 0 {
if lfp+lfn > 0 {
if lfn == 0 {
opts.FileName = selectors.Any()
}
@ -191,14 +152,25 @@ func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointO
}
}
if lists > 0 {
opts.Lists = trimFolderSlash(opts.Lists)
sel.Include(sel.ListItems(opts.Lists, opts.Lists, selectors.StrictEqualMatch()))
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
if slp+sli > 0 {
if sli == 0 {
opts.ListItem = selectors.Any()
}
if pageFolders+pageItems > 0 {
if pageItems == 0 {
opts.ListFolder = trimFolderSlash(opts.ListFolder)
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListFolder)
if len(containsFolders) > 0 {
sel.Include(sel.ListItems(containsFolders, opts.ListItem))
}
if len(prefixFolders) > 0 {
sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch()))
}
}
if pf+pi > 0 {
if pi == 0 {
opts.Page = selectors.Any()
}
@ -214,7 +186,7 @@ func IncludeSharePointRestoreDataSelectors(ctx context.Context, opts SharePointO
}
}
if webUrls > 0 {
if lwu > 0 {
urls := make([]string, 0, len(opts.WebURL))
for _, wu := range opts.WebURL {
@ -253,8 +225,4 @@ func FilterSharePointRestoreInfoSelectors(
AddSharePointInfo(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddSharePointInfo(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddSharePointInfo(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
AddSharePointInfo(sel, opts.ListModifiedAfter, sel.ListModifiedAfter)
AddSharePointInfo(sel, opts.ListModifiedBefore, sel.ListModifiedBefore)
AddSharePointInfo(sel, opts.ListCreatedAfter, sel.ListCreatedAfter)
AddSharePointInfo(sel, opts.ListCreatedBefore, sel.ListCreatedBefore)
}

View File

@ -8,8 +8,8 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -31,7 +31,6 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
multi = []string{"more", "than", "one"}
containsOnly = []string{"contains"}
prefixOnly = []string{"/prefix"}
listNames = []string{"list-name1"}
containsAndPrefix = []string{"contains", "/prefix"}
onlySlash = []string{string(path.PathSeparator)}
)
@ -61,7 +60,8 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
opts: utils.SharePointOpts{
FileName: single,
FolderPath: single,
Lists: single,
ListItem: single,
ListFolder: single,
SiteID: single,
WebURL: single,
},
@ -108,12 +108,31 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() {
expectIncludeLen: 2,
},
{
name: "list names",
name: "list contains",
opts: utils.SharePointOpts{
Lists: listNames,
FileName: empty,
FolderPath: empty,
ListItem: empty,
ListFolder: containsOnly,
SiteID: empty,
WebURL: empty,
},
expectIncludeLen: 1,
},
{
name: "list prefixes",
opts: utils.SharePointOpts{
ListFolder: prefixOnly,
},
expectIncludeLen: 1,
},
{
name: "list prefixes and contains",
opts: utils.SharePointOpts{
ListFolder: containsAndPrefix,
},
expectIncludeLen: 2,
},
{
name: "weburl contains",
opts: utils.SharePointOpts{
@ -279,20 +298,12 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
FileCreatedBefore: dttm.Now(),
FileModifiedAfter: dttm.Now(),
FileModifiedBefore: dttm.Now(),
ListCreatedAfter: dttm.Now(),
ListCreatedBefore: dttm.Now(),
ListModifiedAfter: dttm.Now(),
ListModifiedBefore: dttm.Now(),
Populated: flags.PopulatedFlags{
flags.SiteFN: struct{}{},
flags.FileCreatedAfterFN: struct{}{},
flags.FileCreatedBeforeFN: struct{}{},
flags.FileModifiedAfterFN: struct{}{},
flags.FileModifiedBeforeFN: struct{}{},
flags.ListCreatedAfterFN: struct{}{},
flags.ListCreatedBeforeFN: struct{}{},
flags.ListModifiedAfterFN: struct{}{},
flags.ListModifiedBeforeFN: struct{}{},
},
},
expect: assert.NoError,
@ -358,50 +369,6 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
},
expect: assert.Error,
},
{
name: "invalid list created after",
backupID: "id",
opts: utils.SharePointOpts{
ListCreatedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.ListCreatedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list created before",
backupID: "id",
opts: utils.SharePointOpts{
ListCreatedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.ListCreatedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list modified after",
backupID: "id",
opts: utils.SharePointOpts{
ListModifiedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.ListModifiedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid list modified before",
backupID: "id",
opts: utils.SharePointOpts{
ListModifiedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.ListModifiedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -410,47 +377,3 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
})
}
}
func (suite *SharePointUtilsSuite) TestAddSharepointCategories() {
table := []struct {
name string
cats []string
expectScopeLen int
}{
{
name: "none",
cats: []string{},
expectScopeLen: 1,
},
{
name: "libraries",
cats: []string{flags.DataLibraries},
expectScopeLen: 1,
},
{
name: "lists",
cats: []string{flags.DataLists},
expectScopeLen: 1,
},
{
name: "all allowed",
cats: []string{
flags.DataLibraries,
flags.DataLists,
},
expectScopeLen: 2,
},
{
name: "bad inputs",
cats: []string{"foo"},
expectScopeLen: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
sel := utils.AddCategories(selectors.NewSharePointBackup(selectors.Any()), test.cats)
scopes := sel.Scopes()
assert.Len(suite.T(), scopes, test.expectScopeLen)
})
}
}

View File

@ -1,101 +0,0 @@
package utils
import (
"context"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/pkg/selectors"
)
type TeamsChatsOpts struct {
Users []string
ExportCfg ExportCfgOpts
Populated flags.PopulatedFlags
}
func TeamsChatsAllowedCategories() map[string]struct{} {
return map[string]struct{}{
flags.DataChats: {},
}
}
func AddTeamsChatsCategories(sel *selectors.TeamsChatsBackup, cats []string) *selectors.TeamsChatsBackup {
if len(cats) == 0 {
sel.Include(sel.AllData())
}
for _, d := range cats {
switch d {
case flags.DataChats:
sel.Include(sel.Chats(selectors.Any()))
}
}
return sel
}
func MakeTeamsChatsOpts(cmd *cobra.Command) TeamsChatsOpts {
return TeamsChatsOpts{
Users: flags.UserFV,
ExportCfg: makeExportCfgOpts(cmd),
// populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate
// between an "empty" and a "missing" value.
Populated: flags.GetPopulatedFlags(cmd),
}
}
// ValidateTeamsChatsRestoreFlags checks common flags for correctness and interdependencies
func ValidateTeamsChatsRestoreFlags(backupID string, opts TeamsChatsOpts, isRestore bool) error {
if len(backupID) == 0 {
return clues.New("a backup ID is required")
}
// restore isn't currently supported
if isRestore {
return clues.New("restore not supported")
}
return nil
}
// AddTeamsChatsFilter adds the scope of the provided values to the selector's
// filter set
func AddTeamsChatsFilter(
sel *selectors.TeamsChatsRestore,
v string,
f func(string) []selectors.TeamsChatsScope,
) {
if len(v) == 0 {
return
}
sel.Filter(f(v))
}
// IncludeTeamsChatsRestoreDataSelectors builds the common data-selector
// inclusions for teamschats commands.
func IncludeTeamsChatsRestoreDataSelectors(ctx context.Context, opts TeamsChatsOpts) *selectors.TeamsChatsRestore {
users := opts.Users
if len(opts.Users) == 0 {
users = selectors.Any()
}
return selectors.NewTeamsChatsRestore(users)
}
// FilterTeamsChatsRestoreInfoSelectors builds the common info-selector filters.
func FilterTeamsChatsRestoreInfoSelectors(
sel *selectors.TeamsChatsRestore,
opts TeamsChatsOpts,
) {
// TODO: populate when adding filters
}

View File

@ -9,10 +9,10 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/fault"
ftd "github.com/alcionai/corso/src/pkg/fault/testdata"
"github.com/alcionai/corso/src/pkg/path"

View File

@ -10,10 +10,10 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -51,7 +51,7 @@ func GetAccountAndConnectWithOverrides(
provider storage.ProviderType,
overrides map[string]string,
) (repository.Repositoryer, RepoDetailsAndOpts, error) {
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
provider,
true,

View File

@ -7,8 +7,6 @@ import (
"os"
"github.com/alcionai/corso/src/internal/converters/eml"
"github.com/alcionai/corso/src/internal/converters/ics"
"github.com/alcionai/corso/src/internal/converters/vcf"
)
func main() {
@ -29,23 +27,13 @@ func main() {
var out string
switch from {
case "json":
case "msg":
switch to {
case "eml":
out, err = eml.FromJSON(context.Background(), body)
if err != nil {
log.Fatal(err)
}
case "vcf":
out, err = vcf.FromJSON(context.Background(), body)
if err != nil {
log.Fatal(err)
}
case "ics":
out, err = ics.FromJSON(context.Background(), body)
if err != nil {
log.Fatal(err)
}
default:
log.Fatal("Unknown target format", to)
}

View File

@ -11,6 +11,7 @@ import (
"github.com/google/uuid"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
@ -18,7 +19,6 @@ import (
"github.com/alcionai/corso/src/internal/m365"
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
siteMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester"
@ -28,7 +28,6 @@ import (
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
@ -59,7 +58,7 @@ func generateAndRestoreItems(
service path.ServiceType,
cat path.CategoryType,
sel selectors.Selector,
tenantID, resourceID, destFldr string,
tenantID, userID, destFldr string,
howMany int,
dbf dataBuilderFunc,
opts control.Options,
@ -74,7 +73,7 @@ func generateAndRestoreItems(
nowLegacy = dttm.FormatToLegacy(time.Now())
id = uuid.NewString()
subject = "automated " + now[:16] + " - " + id[:8]
body = "automated " + cat.HumanString() + " generation for " + resourceID + " at " + now + " - " + id
body = "automated " + cat.HumanString() + " generation for " + userID + " at " + now + " - " + id
)
items = append(items, item{
@ -95,7 +94,7 @@ func generateAndRestoreItems(
dataColls, err := buildCollections(
service,
tenantID, resourceID,
tenantID, userID,
restoreCfg,
collections)
if err != nil {
@ -112,7 +111,7 @@ func generateAndRestoreItems(
Selector: sel,
}
handler, err := ctrl.NewServiceHandler(service)
handler, err := ctrl.NewServiceHandler(opts, service)
if err != nil {
return nil, clues.Stack(err)
}
@ -193,44 +192,17 @@ type collection struct {
func buildCollections(
service path.ServiceType,
tenant, resource string,
tenant, user string,
restoreCfg control.RestoreConfig,
colls []collection,
) ([]data.RestoreCollection, error) {
var (
collections = make([]data.RestoreCollection, 0, len(colls))
mc data.Collection
)
collections := make([]data.RestoreCollection, 0, len(colls))
for _, c := range colls {
switch {
case service == path.ExchangeService:
emc, err := generateExchangeMockColls(tenant, resource, c)
if err != nil {
return nil, err
}
mc = emc
case service == path.SharePointService:
smc, err := generateSharepointListsMockColls(tenant, resource, c)
if err != nil {
return nil, err
}
mc = smc
}
collections = append(collections, data.NoFetchRestoreCollection{Collection: mc})
}
return collections, nil
}
func generateExchangeMockColls(tenant string, resource string, c collection) (*exchMock.DataCollection, error) {
pth, err := path.Build(
tenant,
resource,
path.ExchangeService,
user,
service,
c.category,
false,
c.PathElements...)
@ -238,35 +210,17 @@ func generateExchangeMockColls(tenant string, resource string, c collection) (*e
return nil, err
}
emc := exchMock.NewCollection(pth, pth, len(c.items))
mc := exchMock.NewCollection(pth, pth, len(c.items))
for i := 0; i < len(c.items); i++ {
emc.Names[i] = c.items[i].name
emc.Data[i] = c.items[i].data
mc.Names[i] = c.items[i].name
mc.Data[i] = c.items[i].data
}
return emc, nil
collections = append(collections, data.NoFetchRestoreCollection{Collection: mc})
}
func generateSharepointListsMockColls(tenant string, resource string, c collection) (*siteMock.ListCollection, error) {
pth, err := path.BuildOrPrefix(
tenant,
resource,
path.SharePointService,
c.category,
false)
if err != nil {
return nil, err
}
smc := siteMock.NewCollection(pth, pth, len(c.items))
for i := 0; i < len(c.items); i++ {
smc.Names[i] = c.items[i].name
smc.Data[i] = c.items[i].data
}
return smc, nil
return collections, nil
}
var (
@ -506,7 +460,7 @@ func generateAndRestoreDriveItems(
Selector: sel,
}
handler, err := ctrl.NewServiceHandler(service)
handler, err := ctrl.NewServiceHandler(opts, service)
if err != nil {
return nil, clues.Stack(err)
}

View File

@ -7,8 +7,6 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
siteMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -16,23 +14,14 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
)
var (
spFilesCmd = &cobra.Command{
var spFilesCmd = &cobra.Command{
Use: "files",
Short: "Generate SharePoint files",
RunE: handleSharePointLibraryFileFactory,
}
spListsCmd = &cobra.Command{
Use: "lists",
Short: "Generate SharePoint lists",
RunE: handleSharepointListsFactory,
}
)
func AddSharePointCommands(cmd *cobra.Command) {
cmd.AddCommand(spFilesCmd)
cmd.AddCommand(spListsCmd)
}
func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error {
@ -81,52 +70,3 @@ func handleSharePointLibraryFileFactory(cmd *cobra.Command, args []string) error
return nil
}
func handleSharepointListsFactory(cmd *cobra.Command, args []string) error {
var (
ctx = cmd.Context()
service = path.SharePointService
category = path.ListsCategory
errs = fault.New(false)
)
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
ctrl, _, ins, err := getControllerAndVerifyResourceOwner(ctx, Site, path.SharePointService)
if err != nil {
return Only(ctx, err)
}
deets, err := generateAndRestoreItems(
ctx,
ctrl,
service,
category,
selectors.NewSharePointRestore([]string{ins.ID()}).Selector,
Tenant, ins.ID(), Destination,
Count,
func(id, now, subject, body string) []byte {
listBytes, err := siteMock.ListBytes(id)
if err != nil {
logger.CtxErr(ctx, err)
return nil
}
return listBytes
},
control.DefaultOptions(),
errs,
count.New())
if err != nil {
return Only(ctx, err)
}
for _, e := range errs.Recovered() {
logger.CtxErr(ctx, err).Error(e.Error())
}
deets.PrintEntries(ctx)
return nil
}

View File

@ -11,8 +11,8 @@ import (
"github.com/spf13/cobra"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
@ -90,7 +90,7 @@ func pitrListBackups(
// TODO(ashmrtn): This may be moved into CLI layer at some point when we add
// flags for opening a repo at a point in time.
cfg, err := config.ReadCorsoConfig(
cfg, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderS3,
true,
@ -151,7 +151,7 @@ func main() {
cc.SetContext(context.Background())
if err := config.InitCmd(&cc, []string{}); err != nil {
if err := config.InitFunc(&cc, []string{}); err != nil {
return
}

36
src/cmd/purge/scripts/onedrivePurge.ps1 Executable file → Normal file
View File

@ -6,6 +6,12 @@ Param (
[Parameter(Mandatory = $False, HelpMessage = "Site for which to delete folders in SharePoint")]
[String]$Site,
[Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")]
[String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER,
[Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")]
[String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD,
[Parameter(Mandatory = $False, HelpMessage = "Document library root. Can add multiple comma-separated values")]
[String[]]$LibraryNameList = @(),
@ -16,16 +22,7 @@ Param (
[String[]]$FolderPrefixPurgeList,
[Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")]
[String[]]$LibraryPrefixDeleteList = @(),
[Parameter(Mandatory = $False, HelpMessage = "Tenant domain")]
[String]$TenantDomain = $ENV:TENANT_DOMAIN,
[Parameter(Mandatory = $False, HelpMessage = "Azure ClientId")]
[String]$ClientId = $ENV:AZURE_CLIENT_ID,
[Parameter(Mandatory = $False, HelpMessage = "Azure AppCert")]
[String]$AppCert = $ENV:AZURE_APP_CERT
[String[]]$LibraryPrefixDeleteList = @()
)
Set-StrictMode -Version 2.0
@ -111,7 +108,6 @@ function Purge-Library {
$foldersToPurge = @()
$folders = Get-PnPFolderItem -FolderSiteRelativeUrl $LibraryName -ItemType Folder
Write-Host "`nFolders: $folders"
foreach ($f in $folders) {
$folderName = $f.Name
$createTime = Get-TimestampFromFolderName -Folder $f
@ -186,14 +182,6 @@ function Delete-LibraryByPrefix {
if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) {
Write-Host "Deleting list: "$l.Title
try {
$listInfo = Get-PnPList -Identity $l.Id | Select-Object -Property Hidden
# Check if the 'hidden' property is true
if ($listInfo.Hidden) {
Write-Host "List: $($l.Title) is hidden. Skipping..."
continue
}
Remove-PnPList -Identity $l.Id -Force
}
catch [ System.Management.Automation.ItemNotFoundException ] {
@ -213,8 +201,8 @@ if (-not (Get-Module -ListAvailable -Name PnP.PowerShell)) {
}
if ([string]::IsNullOrEmpty($ClientId) -or [string]::IsNullOrEmpty($AppCert)) {
Write-Host "ClientId and AppCert required as arguments or environment variables."
if ([string]::IsNullOrEmpty($AdminUser) -or [string]::IsNullOrEmpty($AdminPwd)) {
Write-Host "Admin user name and password required as arguments or environment variables."
Exit
}
@ -255,8 +243,12 @@ else {
Exit
}
$password = convertto-securestring -String $AdminPwd -AsPlainText -Force
$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $AdminUser, $password
Write-Host "`nAuthenticating and connecting to $SiteUrl"
Connect-PnPOnline -Url $siteUrl -ClientId $ClientId -CertificateBase64Encoded $AppCert -Tenant $TenantDomain
Connect-PnPOnline -Url $siteUrl -Credential $cred
Write-Host "Connected to $siteUrl`n"
# ensure that there are no unexpanded entries in the list of parameters

View File

@ -12,9 +12,9 @@ import (
"github.com/spf13/cobra"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cmd/s3checker/pkg/s3"
"github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/pkg/config"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/storage"
)
@ -175,7 +175,7 @@ func handleCheckerCommand(cmd *cobra.Command, args []string, f flags) error {
fmt.Printf("Checking objects with prefix(es) %v\n", f.prefixes)
if err := config.InitCmd(cmd, args); err != nil {
if err := config.InitFunc(cmd, args); err != nil {
return clues.Wrap(err, "setting viper")
}
@ -187,7 +187,7 @@ func handleCheckerCommand(cmd *cobra.Command, args []string, f flags) error {
storage.Prefix: f.bucketPrefix,
}
repoDetails, err := config.ReadCorsoConfig(
repoDetails, err := config.GetConfigRepoDetails(
ctx,
storage.ProviderS3,
false,

View File

@ -22,18 +22,13 @@ const (
sanityBackupID = "SANITY_BACKUP_ID"
sanityTestSourceContainer = "SANITY_TEST_SOURCE_CONTAINER"
sanityTestRestoreContainer = "SANITY_TEST_RESTORE_CONTAINER"
sanityTestRestoreContainerPrefix = "SANITY_TEST_RESTORE_CONTAINER_PREFIX"
sanityTestUser = "SANITY_TEST_USER"
sanityTestCategory = "SANITY_TEST_CATEGORY"
)
type Envs struct {
BackupID string
SourceContainer string
RestoreContainer string
// applies for sharepoint lists only
RestoreContainerPrefix string
Category string
GroupID string
SiteID string
UserID string
@ -47,8 +42,6 @@ func EnvVars(ctx context.Context) Envs {
BackupID: os.Getenv(sanityBackupID),
SourceContainer: os.Getenv(sanityTestSourceContainer),
RestoreContainer: folder,
Category: os.Getenv(sanityTestCategory),
RestoreContainerPrefix: os.Getenv(sanityTestRestoreContainerPrefix),
GroupID: tconfig.GetM365TeamID(ctx),
SiteID: tconfig.GetM365SiteID(ctx),
UserID: tconfig.GetM365UserID(ctx),

View File

@ -20,17 +20,30 @@ func BuildFilepathSanitree(
info os.FileInfo,
err error,
) error {
if root == nil {
root = CreateNewRoot(info, true)
return nil
if err != nil {
Fatal(ctx, "error passed to filepath walker", err)
}
relPath := GetRelativePath(
ctx,
rootDir,
p,
info,
err)
relPath, err := filepath.Rel(rootDir, p)
if err != nil {
Fatal(ctx, "getting relative filepath", err)
}
if info != nil {
Debugf(ctx, "adding: %s", relPath)
}
if root == nil {
root = &Sanitree[fs.FileInfo, fs.FileInfo]{
Self: info,
ID: info.Name(),
Name: info.Name(),
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
}
return nil
}
elems := path.Split(relPath)
node := root.NodeAt(ctx, elems[:len(elems)-1])
@ -65,41 +78,3 @@ func BuildFilepathSanitree(
return root
}
func CreateNewRoot(info fs.FileInfo, initChildren bool) *Sanitree[fs.FileInfo, fs.FileInfo] {
root := &Sanitree[fs.FileInfo, fs.FileInfo]{
Self: info,
ID: info.Name(),
Name: info.Name(),
Leaves: map[string]*Sanileaf[fs.FileInfo, fs.FileInfo]{},
Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
}
if initChildren {
root.Children = map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}
}
return root
}
func GetRelativePath(
ctx context.Context,
rootDir, p string,
info fs.FileInfo,
fileWalkerErr error,
) string {
if fileWalkerErr != nil {
Fatal(ctx, "error passed to filepath walker", fileWalkerErr)
}
relPath, err := filepath.Rel(rootDir, p)
if err != nil {
Fatal(ctx, "getting relative filepath", err)
}
if info != nil {
Debugf(ctx, "adding: %s", relPath)
}
return relPath
}

View File

@ -10,7 +10,7 @@ import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/logger"
)

View File

@ -3,9 +3,7 @@ package driveish
import (
"context"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/internal/common/ptr"
@ -17,24 +15,10 @@ const (
owner = "owner"
)
// sanitree population will grab a superset of data in the drive.
// this increases the chance that we'll run into a race collision with
// the cleanup script. Sometimes that's okay (deleting old data that
// isn't scrutinized in the test), other times it's not. We mark whether
// that's okay to do or not by specifying the folders being
// scrutinized for the test. Any errors within those folders should cause
// a fatal exit. Errors outside of those folders get ignored.
//
// since we're using folder names, mustPopulateFolders will
// work best (ie: have the fewest collisions/side-effects) if the folder
// names are very specific. Standard sanity tests should include timestamps,
// which should help ensure that. Be warned if you try to use it with
// a more generic name: unintended effects could occur.
func populateSanitree(
ctx context.Context,
ac api.Client,
driveID string,
mustPopulateFolders []string,
) *common.Sanitree[models.DriveItemable, models.DriveItemable] {
common.Infof(ctx, "building sanitree for drive: %s", driveID)
@ -43,12 +27,10 @@ func populateSanitree(
common.Fatal(ctx, "getting drive root folder", err)
}
rootName := ptr.Val(root.GetName())
stree := &common.Sanitree[models.DriveItemable, models.DriveItemable]{
Self: root,
ID: ptr.Val(root.GetId()),
Name: rootName,
Name: ptr.Val(root.GetName()),
Leaves: map[string]*common.Sanileaf[models.DriveItemable, models.DriveItemable]{},
Children: map[string]*common.Sanitree[models.DriveItemable, models.DriveItemable]{},
}
@ -58,8 +40,6 @@ func populateSanitree(
ac,
driveID,
stree.Name+"/",
mustPopulateFolders,
slices.Contains(mustPopulateFolders, rootName),
stree)
return stree
@ -68,31 +48,16 @@ func populateSanitree(
func recursivelyBuildTree(
ctx context.Context,
ac api.Client,
driveID string,
location string,
mustPopulateFolders []string,
isChildOfFolderRequiringNoErrors bool,
driveID, location string,
stree *common.Sanitree[models.DriveItemable, models.DriveItemable],
) {
common.Debugf(ctx, "adding: %s", location)
children, err := ac.Drives().GetFolderChildren(ctx, driveID, stree.ID)
if err != nil {
if isChildOfFolderRequiringNoErrors {
common.Fatal(ctx, "getting drive children by id", err)
}
common.Infof(
ctx,
"ignoring error getting children in directory %q because it is not within directory set %v\nerror: %s\n%+v",
location,
mustPopulateFolders,
err.Error(),
clues.ToCore(err))
return
}
for _, driveItem := range children {
var (
itemID = ptr.Val(driveItem.GetId())
@ -103,20 +68,17 @@ func recursivelyBuildTree(
// currently we don't restore blank folders.
// skip permission check for empty folders
if ptr.Val(driveItem.GetFolder().GetChildCount()) == 0 {
common.Infof(ctx, "skipped empty folder: %s%s", location, itemName)
common.Infof(ctx, "skipped empty folder: %s/%s", location, itemName)
continue
}
cannotAllowErrors := isChildOfFolderRequiringNoErrors ||
slices.Contains(mustPopulateFolders, itemName)
branch := &common.Sanitree[models.DriveItemable, models.DriveItemable]{
Parent: stree,
Self: driveItem,
ID: itemID,
Name: itemName,
Expand: map[string]any{
expandPermissions: permissionIn(ctx, ac, driveID, itemID, cannotAllowErrors),
expandPermissions: permissionIn(ctx, ac, driveID, itemID),
},
Leaves: map[string]*common.Sanileaf[models.DriveItemable, models.DriveItemable]{},
Children: map[string]*common.Sanitree[models.DriveItemable, models.DriveItemable]{},
@ -129,8 +91,6 @@ func recursivelyBuildTree(
ac,
driveID,
location+branch.Name+"/",
mustPopulateFolders,
cannotAllowErrors,
branch)
}

View File

@ -31,8 +31,7 @@ func CheckExport(
root := populateSanitree(
ctx,
ac,
driveID,
[]string{envs.SourceContainer})
driveID)
sourceTree, ok := root.Children[envs.SourceContainer]
common.Assert(

View File

@ -45,14 +45,7 @@ func CheckRestoration(
"drive_id", driveID,
"drive_name", driveName)
root := populateSanitree(
ctx,
ac,
driveID,
[]string{
envs.SourceContainer,
envs.RestoreContainer,
})
root := populateSanitree(ctx, ac, driveID)
sourceTree, ok := root.Children[envs.SourceContainer]
common.Assert(
@ -92,26 +85,14 @@ func permissionIn(
ctx context.Context,
ac api.Client,
driveID, itemID string,
cannotAllowErrors bool,
) []common.PermissionInfo {
pi := []common.PermissionInfo{}
pcr, err := ac.Drives().GetItemPermission(ctx, driveID, itemID)
if err != nil {
if cannotAllowErrors {
common.Fatal(ctx, "getting permission", err)
}
common.Infof(
ctx,
"ignoring error getting permissions for %q\nerror: %s,%+v",
itemID,
err.Error(),
clues.ToCore(err))
return []common.PermissionInfo{}
}
for _, perm := range pcr.GetValue() {
if perm.GetGrantedToV2() == nil {
continue

Some files were not shown because too many files have changed in this diff Show More