b086f8c3ff
PowerShell switched to requiring certificate credentials so the existing cleanup jobs have been failing since the switch --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change - [ ] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [x] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Test Plan - [ ] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
397 lines
14 KiB
YAML
397 lines
14 KiB
YAML
name: Longevity Testing
|
|
on:
|
|
schedule:
|
|
# Run every day at 04:00 GMT (roughly 8pm PST)
|
|
- cron: "0 4 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
user:
|
|
description: "User to run longevity test on"
|
|
|
|
permissions:
|
|
# required to retrieve AWS credentials
|
|
id-token: write
|
|
contents: write
|
|
|
|
# cancel currently running jobs if a new version of the branch is pushed
|
|
concurrency:
|
|
group: longevity_testing-${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
SetM365App:
|
|
uses: alcionai/corso/.github/workflows/accSelector.yaml@main
|
|
|
|
Longevity-Tests:
|
|
needs: [SetM365App]
|
|
environment: Testing
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
# Need these in the local env so that corso can read them
|
|
AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
|
|
AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
|
|
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
|
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
|
|
# re-used values
|
|
CORSO_LOG_DIR: ${{ github.workspace }}/src/testlog
|
|
CORSO_LOG_FILE: ${{ github.workspace }}/src/testlog/run-longevity.log
|
|
RESTORE_DEST_PFX: Corso_Test_Longevity_
|
|
TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || vars.CORSO_M365_TEST_USER_ID }}
|
|
PREFIX: "longevity"
|
|
|
|
# Options for retention.
|
|
RETENTION_MODE: GOVERNANCE
|
|
# Time to retain blobs for in hours.
|
|
RETENTION_DURATION: 216
|
|
defaults:
|
|
run:
|
|
working-directory: src
|
|
|
|
############################################################################
|
|
# setup
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0 # needed to get latest tag
|
|
|
|
- name: Setup Golang with cache
|
|
uses: magnetikonline/action-golang-cache@v4
|
|
with:
|
|
go-version-file: src/go.mod
|
|
|
|
- run: |
|
|
go build -o longevity-test ./cmd/longevity_test
|
|
go build -o s3checker ./cmd/s3checker
|
|
|
|
- name: Get version string
|
|
id: version
|
|
run: |
|
|
echo version=$(git describe --tags --abbrev=0) | tee -a $GITHUB_OUTPUT
|
|
|
|
# Checkout the .github directory at the original branch's ref so we have a
|
|
# stable view of the actions.
|
|
- name: Code Checkout
|
|
working-directory: ${{ github.workspace }}
|
|
run: |
|
|
git checkout ${{ steps.version.outputs.version }}
|
|
git checkout ${{ github.ref }} -- .github
|
|
|
|
- run: go build -o corso
|
|
timeout-minutes: 10
|
|
|
|
- run: mkdir ${CORSO_LOG_DIR}
|
|
|
|
# Use shorter-lived credentials obtained from assume-role since these
|
|
# runs haven't been taking long.
|
|
- name: Configure AWS credentials from Test account
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
timeout-minutes: 10
|
|
with:
|
|
role-to-assume: ${{ secrets.AWS_IAM_ROLE }}
|
|
role-session-name: integration-testing
|
|
aws-region: us-east-1
|
|
|
|
##########################################################################
|
|
# Repository commands
|
|
|
|
- name: Version Test
|
|
timeout-minutes: 10
|
|
run: |
|
|
./corso --version | grep -c 'Corso version:'
|
|
|
|
- name: Repo init test
|
|
id: repo-init
|
|
timeout-minutes: 10
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nRepo init test\n" >> ${{ env.CORSO_LOG_FILE }}
|
|
./corso repo init s3 \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--retention-mode $(echo "${{ env.RETENTION_MODE }}" | tr '[:upper:]' '[:lower:]') \
|
|
--retention-duration "${{ env.RETENTION_DURATION }}h" \
|
|
--extend-retention \
|
|
--prefix ${{ env.PREFIX }} \
|
|
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
|
|
|
|
if grep -q 'Failed to' ${{ env.CORSO_LOG_DIR }}/gotest-repo-init.log
|
|
then
|
|
echo "Repo could not be initialized"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Repo connect test
|
|
timeout-minutes: 10
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nRepo connect test\n" >> ${{ env.CORSO_LOG_FILE }}
|
|
./corso repo connect s3 \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--prefix ${{ env.PREFIX }} \
|
|
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
|
|
|
|
if ! grep -q 'Connected to S3 bucket' ${{ env.CORSO_LOG_DIR }}/gotest-repo-connect.log
|
|
then
|
|
echo "Repo could not be connected"
|
|
exit 1
|
|
fi
|
|
|
|
##########################################################################
|
|
# Exchange
|
|
|
|
- name: Backup exchange test
|
|
id: exchange-test
|
|
timeout-minutes: 30
|
|
run: |
|
|
echo -e "\nBackup Exchange test\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup create exchange \
|
|
--no-stats \
|
|
--mailbox "${TEST_USER}" \
|
|
--hide-progress \
|
|
--json \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt
|
|
|
|
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_exchange.txt )
|
|
|
|
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
|
|
echo "backup was not successful"
|
|
exit 1
|
|
fi
|
|
|
|
data=$( echo $resultjson | jq -r '.[0] | .id' )
|
|
echo result=$data >> $GITHUB_OUTPUT
|
|
|
|
##########################################################################
|
|
# Onedrive
|
|
|
|
- name: Backup onedrive test
|
|
id: onedrive-test
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nBackup OneDrive test\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup create onedrive \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--user "${TEST_USER}" \
|
|
--json \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt
|
|
|
|
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_onedrive.txt )
|
|
|
|
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
|
|
echo "backup was not successful"
|
|
exit 1
|
|
fi
|
|
|
|
data=$( echo $resultjson | jq -r '.[0] | .id' )
|
|
echo result=$data >> $GITHUB_OUTPUT
|
|
|
|
##########################################################################
|
|
# Sharepoint test
|
|
- name: Backup sharepoint test
|
|
id: sharepoint-test
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nBackup SharePoint test\n" >> ${CORSO_LOG_FILE}
|
|
|
|
./corso backup create sharepoint \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--site "${{ vars.CORSO_M365_TEST_SITE_URL }}" \
|
|
--json \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt
|
|
|
|
resultjson=$(sed -e '1,/Completed Backups/d' ${{ env.CORSO_LOG_DIR }}/backup_sharepoint.txt )
|
|
|
|
if [[ $( echo $resultjson | jq -r '.[0] | .stats.errorCount') -ne 0 ]]; then
|
|
echo "backup was not successful"
|
|
exit 1
|
|
fi
|
|
|
|
data=$( echo $resultjson | jq -r '.[0] | .id' )
|
|
echo result=$data >> $GITHUB_OUTPUT
|
|
|
|
##########################################################################
|
|
# Backup Exchange Deletion test
|
|
- name: Backup Delete exchange test
|
|
id: delete-exchange-test
|
|
timeout-minutes: 30
|
|
env:
|
|
SERVICE: "exchange"
|
|
DELETION_DAYS: 10
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nDelete Backup exchange \n" >> ${CORSO_LOG_FILE}
|
|
./longevity-test
|
|
|
|
##########################################################################
|
|
# Backup Onedrive Deletion test
|
|
- name: Backup Delete onedrive test
|
|
id: delete-onedrive-test
|
|
timeout-minutes: 30
|
|
env:
|
|
SERVICE: "onedrive"
|
|
DELETION_DAYS: 10
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nDelete Backup onedrive \n" >> ${CORSO_LOG_FILE}
|
|
./longevity-test
|
|
|
|
##########################################################################
|
|
# Backup Sharepoint Deletion test
|
|
- name: Backup Delete Sharepoint test
|
|
id: delete-sharepoint-test
|
|
timeout-minutes: 30
|
|
env:
|
|
SERVICE: "sharepoint"
|
|
DELETION_DAYS: 5
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\nDelete Backup sharepoint \n" >> ${CORSO_LOG_FILE}
|
|
./longevity-test
|
|
|
|
##########################################################################
|
|
# Export OneDrive Test
|
|
- name: OneDrive Export test
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\Export OneDrive test\n" >> ${CORSO_LOG_FILE}
|
|
|
|
echo -e "\Export OneDrive test - first entry\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup list onedrive 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
|
|
while read -r line; do
|
|
./corso export onedrive \
|
|
"/tmp/corso-export--$line" \
|
|
--no-stats \
|
|
--backup "$line" \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_first.txt
|
|
done
|
|
|
|
echo -e "\Export OneDrive test - last entry\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup list onedrive 2>/dev/null | tail -n1 | awk '{print $1}' |
|
|
while read -r line; do
|
|
./corso export onedrive \
|
|
"/tmp/corso-export--$line" \
|
|
--no-stats \
|
|
--backup "$line" \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_onedrive_last.txt
|
|
done
|
|
|
|
##########################################################################
|
|
# Export SharePoint Test
|
|
- name: SharePoint Export test
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\Export SharePoint test\n" >> ${CORSO_LOG_FILE}
|
|
|
|
echo -e "\Export SharePoint test - first entry\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup list sharepoint 2>/dev/null | tail -n+2 | head -n1 | awk '{print $1}' |
|
|
while read -r line; do
|
|
./corso export sharepoint \
|
|
"/tmp/corso-export--$line" \
|
|
--no-stats \
|
|
--backup "$line" \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_first.txt
|
|
done
|
|
|
|
echo -e "\Export SharePoint test - last entry\n" >> ${CORSO_LOG_FILE}
|
|
./corso backup list sharepoint 2>/dev/null | tail -n1 | awk '{print $1}' |
|
|
while read -r line; do
|
|
./corso export sharepoint \
|
|
"/tmp/corso-export--$line" \
|
|
--no-stats \
|
|
--backup "$line" \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/export_sharepoint_last.txt
|
|
done
|
|
|
|
##########################################################################
|
|
# Maintenance test
|
|
- name: Maintenance test Daily
|
|
id: maintenance-test-daily
|
|
timeout-minutes: 30
|
|
run: |
|
|
set -euo pipefail
|
|
echo -e "\n Maintenance test Daily\n" >> ${CORSO_LOG_FILE}
|
|
|
|
# Run with the force flag so it doesn't fail if the github runner
|
|
# hostname isn't what's expected. This is only safe because we can
|
|
# guarantee only one runner will be executing maintenance at a time.
|
|
./corso repo maintenance --mode metadata \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--force \
|
|
--json \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_metadata.txt
|
|
|
|
- name: Maintenance test Weekly
|
|
id: maintenance-test-weekly
|
|
timeout-minutes: 30
|
|
run: |
|
|
if [[ $(date +%A) == "Saturday" ]]; then
|
|
set -euo pipefail
|
|
echo -e "\n Maintenance test Weekly\n" >> ${CORSO_LOG_FILE}
|
|
|
|
./corso repo maintenance --mode complete \
|
|
--no-stats \
|
|
--hide-progress \
|
|
--force \
|
|
--json \
|
|
2>&1 | tee ${{ env.CORSO_LOG_DIR }}/maintenance_complete.txt
|
|
|
|
# TODO(ashmrtn): We can also check that non-current versions of
|
|
# blobs don't have their retention extended if we want.
|
|
#
|
|
# Assuming no failures during full maintenance, current versions of
|
|
# objects with the below versions should have retention times that
|
|
# are roughly (now + RETENTION_DURATION). We can explicitly check
|
|
# for this, but leave a little breathing room since maintenance may
|
|
# take some time to run.
|
|
#
|
|
# If we pick a live-retention-duration that is too small then we'll
|
|
# start seeing failures. The check for live objects is a lower bound
|
|
# check.
|
|
#
|
|
# Blob prefixes are as follows:
|
|
# - kopia.blobcfg - repo-wide config
|
|
# - kopia.repository - repo-wide config
|
|
# - p - data pack blobs (i.e. file data)
|
|
# - q - metadata pack blobs (i.e. manifests, directory listings, etc)
|
|
# - x - index blobs
|
|
./s3checker \
|
|
--bucket ${{ secrets.CI_RETENTION_TESTS_S3_BUCKET }} \
|
|
--prefix ${{ env.PREFIX }} \
|
|
--retention-mode ${{ env.RETENTION_MODE }} \
|
|
--live-retention-duration "$((${{ env.RETENTION_DURATION }}-1))h" \
|
|
--object-prefix "kopia.blobcfg" \
|
|
--object-prefix "kopia.repository" \
|
|
--object-prefix "p" \
|
|
--object-prefix "q" \
|
|
--object-prefix "x"
|
|
fi
|
|
|
|
##########################################################################
|
|
# Logging & Notifications
|
|
# Upload the original go test output as an artifact for later review.
|
|
- name: Upload test log
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: longevity-test-log
|
|
path: src/testlog/*
|
|
if-no-files-found: error
|
|
retention-days: 14
|
|
|
|
- name: Notify failure in teams
|
|
if: failure()
|
|
uses: ./.github/actions/teams-message
|
|
with:
|
|
msg: "[CORSO FAILED] Longevity Test"
|
|
teams_url: ${{ secrets.TEAMS_CORSO_CI_WEBHOOK_URL }}
|