larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sanity test yml refs (#3442)

Browse Source 
#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🤖 Supportability/Tests

#### Test Plan

- [x] 💪 Manual
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-05-18 10:07:59 -06:00 committed by GitHub
parent 53f734db2e
commit 8cd5541244
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 79 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.github/actions/backup-restore-test/action.yml vendored
View File

@ -50,9 +50,11 @@ runs:
run: | run: |
set -euo pipefail set -euo pipefail
./corso restore '${{ inputs.service }}' \ ./corso restore '${{ inputs.service }}' \
--no-stats --hide-progress \ --no-stats \
--hide-progress \
${{ inputs.restore-args }} \ ${{ inputs.restore-args }} \
--backup '${{ steps.backup.outputs.result }}' 2>&1 | --backup '${{ steps.backup.outputs.result }}' \
2>&1 |
tee /tmp/corsologs | tee /tmp/corsologs |
grep -i -e 'Restoring to folder ' | grep -i -e 'Restoring to folder ' |
sed "s/Restoring to folder /result=/" | sed "s/Restoring to folder /result=/" |
@ -77,7 +79,9 @@ runs:
run: | run: |
set -euo pipefail set -euo pipefail
./corso backup list ${{ inputs.service }} \ ./corso backup list ${{ inputs.service }} \
--no-stats --hide-progress 2>&1 | --no-stats \
--hide-progress \
2>&1 |
tee /tmp/corso-backup-list.log tee /tmp/corso-backup-list.log
if ! grep -q ${{ steps.backup.outputs.result }} /tmp/corso-backup-list.log if ! grep -q ${{ steps.backup.outputs.result }} /tmp/corso-backup-list.log
@ -92,8 +96,10 @@ runs:
run: | run: |
set -euo pipefail set -euo pipefail
./corso backup list ${{ inputs.service }} \ ./corso backup list ${{ inputs.service }} \
--no-stats --hide-progress \ --no-stats \
--backup "${{ steps.backup.outputs.result }}" 2>&1 | --hide-progress \
--backup "${{ steps.backup.outputs.result }}" \
2>&1 |
tee /tmp/corso-backup-list-item.log tee /tmp/corso-backup-list-item.log
if ! grep -q ${{ steps.backup.outputs.result }} /tmp/corso-backup-list-item.log if ! grep -q ${{ steps.backup.outputs.result }} /tmp/corso-backup-list-item.log

130
.github/workflows/sanity-test.yaml vendored
View File

@ -27,20 +27,20 @@ jobs:
environment: Testing environment: Testing
runs-on: ubuntu-latest runs-on: ubuntu-latest
env: env:
# Need these in the local env so that corso can read them
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }} AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }} AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
# re-used values
# don't forget: return to Corso_Test_Sanity_
CORSO_LOG_DIR: testlog CORSO_LOG_DIR: testlog
CORSO_LOG_FILE: testlog/testlogging.log CORSO_LOG_FILE: testlog/testlogging.log
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
RESTORE_DEST_PFX: Corso_Test_Sanity_ RESTORE_DEST_PFX: Corso_Test_Sanity_
TEST_RESULT: test_results TEST_RESULT: test_results
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }} TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
TEST_SITE: ${{ secrets.CORSO_M365_TEST_SITE_URL }}
SECONDARY_TEST_USER : ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
# The default working directory doesn't seem to apply to things without # The default working directory doesn't seem to apply to things without
# the 'run' directive. https://stackoverflow.com/a/67845456 # the 'run' directive. https://stackoverflow.com/a/67845456
WORKING_DIR: src WORKING_DIR: src
@ -87,9 +87,9 @@ jobs:
user: ${{ env.TEST_USER }} user: ${{ env.TEST_USER }}
folder-prefix: ${{ env.RESTORE_DEST_PFX }} folder-prefix: ${{ env.RESTORE_DEST_PFX }}
older-than: ${{ env.NOW }} older-than: ${{ env.NOW }}
azure-client-id: ${{ env.AZURE_CLIENT_ID }} azure-client-id: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
azure-client-secret: ${{ env.AZURE_CLIENT_SECRET }} azure-client-secret: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
azure-tenant-id: ${{ env.AZURE_TENANT_ID }} azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
@ -97,13 +97,13 @@ jobs:
if: always() if: always()
uses: ./.github/actions/purge-m365-data uses: ./.github/actions/purge-m365-data
with: with:
site: ${{ env.TEST_SITE }} site: ${{ secrets.CORSO_M365_TEST_SITE_URL }}
folder-prefix: ${{ env.RESTORE_DEST_PFX }} folder-prefix: ${{ env.RESTORE_DEST_PFX }}
libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }}
older-than: ${{ env.NOW }} older-than: ${{ env.NOW }}
azure-client-id: ${{ env.AZURE_CLIENT_ID }} azure-client-id: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
azure-client-secret: ${{ env.AZURE_CLIENT_SECRET }} azure-client-secret: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
azure-tenant-id: ${{ env.AZURE_TENANT_ID }} azure-tenant-id: ${{ secrets.TENANT_ID }}
m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
@ -122,10 +122,13 @@ jobs:
run: | run: |
set -euo pipefail set -euo pipefail
prefix=$(date +"%Y-%m-%d-%T") prefix=$(date +"%Y-%m-%d-%T")
echo -e "\nRepo init test\n" >> ${CORSO_LOG_FILE} echo -e "\nRepo init test\n" >> ${{ env.CORSO_LOG_FILE }}
./corso repo init s3 \ ./corso repo init s3 \
--no-stats --hide-progress --prefix $prefix \ --no-stats \
--bucket ${CORSO_BUCKET} 2>&1 | tee $TEST_RESULT/initrepo.txt --hide-progress \
--prefix $prefix \
--bucket ${{ secrets.CI_TESTS_S3_BUCKET }} \
2>&1 | tee $TEST_RESULT/initrepo.txt
if ! grep -q 'Initialized a S3 repository within bucket' $TEST_RESULT/initrepo.txt if ! grep -q 'Initialized a S3 repository within bucket' $TEST_RESULT/initrepo.txt
then then
@ -138,10 +141,13 @@ jobs:
- name: Repo connect test - name: Repo connect test
run: | run: |
set -euo pipefail set -euo pipefail
echo -e "\nRepo connect test\n" >> ${CORSO_LOG_FILE} echo -e "\nRepo connect test\n" >> ${{ env.CORSO_LOG_FILE }}
./corso repo connect s3 \ ./corso repo connect s3 \
--no-stats --hide-progress --prefix ${{ steps.repo-init.outputs.result }} \ --no-stats \
--bucket ${CORSO_BUCKET} 2>&1 | tee $TEST_RESULT/connect.txt --hide-progress \
--prefix ${{ steps.repo-init.outputs.result }} \
--bucket ${{ secrets.CI_TESTS_S3_BUCKET }} \
2>&1 | tee $TEST_RESULT/connect.txt
if ! grep -q 'Connected to S3 bucket' $TEST_RESULT/connect.txt if ! grep -q 'Connected to S3 bucket' $TEST_RESULT/connect.txt
then then
@ -159,9 +165,9 @@ jobs:
working-directory: ./src/cmd/factory working-directory: ./src/cmd/factory
run: | run: |
go run . exchange emails \ go run . exchange emails \
--user ${TEST_USER} \ --user ${{ env.TEST_USER }} \
--tenant ${AZURE_TENANT_ID} \ --tenant ${{ secrets.TENANT_ID }} \
--destination ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }} \ --destination ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }} \
--count 4 --count 4
- name: Exchange - Backup - name: Exchange - Backup
@ -170,9 +176,9 @@ jobs:
with: with:
service: exchange service: exchange
kind: backup kind: backup
backup-args: '--mailbox "${TEST_USER}" --data "email"' backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
- name: Exchange - Incremental backup - name: Exchange - Incremental backup
id: exchange-backup-incremental id: exchange-backup-incremental
@ -180,9 +186,9 @@ jobs:
with: with:
service: exchange service: exchange
kind: backup-incremental kind: backup-incremental
backup-args: '--mailbox "${TEST_USER}" --data "email"' backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
base-backup: ${{ steps.exchange-backup.outputs.backup-id }} base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
- name: Exchange - Non delta backup - name: Exchange - Non delta backup
@ -191,9 +197,9 @@ jobs:
with: with:
service: exchange service: exchange
kind: backup-non-delta kind: backup-non-delta
backup-args: '--mailbox "${TEST_USER}" --data "email" --disable-delta' backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email" --disable-delta'
restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
base-backup: ${{ steps.exchange-backup.outputs.backup-id }} base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
- name: Exchange - Incremental backup after non-delta - name: Exchange - Incremental backup after non-delta
@ -202,9 +208,9 @@ jobs:
with: with:
service: exchange service: exchange
kind: backup-incremental-after-non-delta kind: backup-incremental-after-non-delta
backup-args: '--mailbox "${TEST_USER}" --data "email"' backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
base-backup: ${{ steps.exchange-backup.outputs.backup-id }} base-backup: ${{ steps.exchange-backup.outputs.backup-id }}
@ -220,10 +226,10 @@ jobs:
suffix=$(date +"%Y-%m-%d_%H-%M-%S") suffix=$(date +"%Y-%m-%d_%H-%M-%S")
go run . onedrive files \ go run . onedrive files \
--user ${TEST_USER} \ --user ${{ env.TEST_USER }} \
--secondaryuser ${SECONDARY_TEST_USER} \ --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${AZURE_TENANT_ID} \ --tenant ${{ secrets.TENANT_ID }} \
--destination ${RESTORE_DEST_PFX}$suffix \ --destination ${{ env.RESTORE_DEST_PFX }}$suffix \
--count 4 --count 4
echo result="${suffix}" >> $GITHUB_OUTPUT echo result="${suffix}" >> $GITHUB_OUTPUT
@ -234,19 +240,19 @@ jobs:
with: with:
service: onedrive service: onedrive
kind: backup kind: backup
backup-args: '--user "${TEST_USER}"' backup-args: '--user "${{ env.TEST_USER }}"'
restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
# generate some more enteries for incremental check # generate some more enteries for incremental check
- name: OneDrive - Create new data (for incremental) - name: OneDrive - Create new data (for incremental)
working-directory: ./src/cmd/factory working-directory: ./src/cmd/factory
run: | run: |
go run . onedrive files \ go run . onedrive files \
--user ${TEST_USER} \ --user ${{ env.TEST_USER }} \
--secondaryuser ${SECONDARY_TEST_USER} \ --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${AZURE_TENANT_ID} \ --tenant ${{ secrets.TENANT_ID }} \
--destination ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} \ --destination ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }} \
--count 4 --count 4
- name: OneDrive - Incremental backup - name: OneDrive - Incremental backup
@ -255,9 +261,9 @@ jobs:
with: with:
service: onedrive service: onedrive
kind: incremental kind: incremental
backup-args: '--user "${TEST_USER}"' backup-args: '--user "${{ env.TEST_USER }}"'
restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
########################################################################################################################################## ##########################################################################################################################################
@ -271,11 +277,11 @@ jobs:
suffix=$(date +"%Y-%m-%d_%H-%M-%S") suffix=$(date +"%Y-%m-%d_%H-%M-%S")
go run . sharepoint files \ go run . sharepoint files \
--site ${TEST_SITE} \ --site ${{ secrets.CORSO_M365_TEST_SITE_URL }} \
--user ${TEST_USER} \ --user ${{ env.TEST_USER }} \
--secondaryuser ${SECONDARY_TEST_USER} \ --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${AZURE_TENANT_ID} \ --tenant ${{ secrets.TENANT_ID }} \
--destination ${RESTORE_DEST_PFX}$suffix \ --destination ${{ env.RESTORE_DEST_PFX }}$suffix \
--count 4 --count 4
echo result="${suffix}" >> $GITHUB_OUTPUT echo result="${suffix}" >> $GITHUB_OUTPUT
@ -286,20 +292,20 @@ jobs:
with: with:
service: sharepoint service: sharepoint
kind: backup kind: backup
backup-args: '--site "${TEST_SITE}"' backup-args: '--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}"'
restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
# generate some more enteries for incremental check # generate some more enteries for incremental check
- name: SharePoint - Create new data (for incremental) - name: SharePoint - Create new data (for incremental)
working-directory: ./src/cmd/factory working-directory: ./src/cmd/factory
run: | run: |
go run . sharepoint files \ go run . sharepoint files \
--site ${TEST_SITE} \ --site ${{ secrets.CORSO_M365_TEST_SITE_URL }} \
--user ${TEST_USER} \ --user ${{ env.TEST_USER }} \
--secondaryuser ${SECONDARY_TEST_USER} \ --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
--tenant ${AZURE_TENANT_ID} \ --tenant ${{ secrets.TENANT_ID }} \
--destination ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} \ --destination ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }} \
--count 4 --count 4
- name: SharePoint - Incremental backup - name: SharePoint - Incremental backup
@ -308,9 +314,9 @@ jobs:
with: with:
service: sharepoint service: sharepoint
kind: incremental kind: incremental
backup-args: '--site "${TEST_SITE}"' backup-args: '--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}"'
restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions' restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }}' test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
########################################################################################################################################## ##########################################################################################################################################