corso/.github/workflows/sanity-test.yaml

name: Sanity Testing
on:
  push:
    branches:
      - main
      - 3135-logic
  workflow_dispatch:
    inputs:
      user:
        description: 'User to run sanity test on'

permissions:
  # required to retrieve AWS credentials
  id-token: write
  contents: write

# cancel currently running jobs if a new version of the branch is pushed
concurrency:
  group: sanity_testing-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  SetM365App:
    uses: alcionai/corso/.github/workflows/accSelector.yaml@main

  Sanity-Tests:
    needs: [ SetM365App ]
    environment: Testing
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
      AZURE_CLIENT_ID: ${{ secrets[needs.SetM365App.outputs.client_id_env] }}
      AZURE_CLIENT_SECRET: ${{ secrets[needs.SetM365App.outputs.client_secret_env] }}
      AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
      CORSO_BUCKET: ${{ secrets.CI_TESTS_S3_BUCKET }}
      CORSO_LOG_DIR: testlog
      CORSO_LOG_FILE: testlog/testlogging.log
      CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
      RESTORE_DEST_PFX: Corso_Test_Sanity_
      TEST_RESULT: test_results
      TEST_USER: ${{ github.event.inputs.user != '' && github.event.inputs.user || secrets.CORSO_M365_TEST_USER_ID }}
      TEST_SITE: ${{ secrets.CORSO_M365_TEST_SITE_URL }}
      SECONDARY_TEST_USER : ${{  secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
      # The default working directory doesn't seem to apply to things without
      # the 'run' directive. https://stackoverflow.com/a/67845456
      WORKING_DIR: src

    defaults:
      run:
        working-directory: ${{ env.WORKING_DIR }}
    steps:
      - uses: actions/checkout@v3

      - name: Setup Golang with cache
        uses: magnetikonline/action-golang-cache@v4
        with:
          go-version-file: src/go.mod

      - run: make build
      - run: go build -o sanity-test ./cmd/sanity_test

      - run: mkdir ${TEST_RESULT}
      - run: mkdir ${CORSO_LOG_DIR}

      - name: Version Test
        run: |
          ./corso --version | grep -c 'Corso version:'

      - name: Repo init test
        id: repo-init
        env:
          TEST_RESULT: "test_results"
        run: |
          set -euo pipefail
          prefix=$(date +"%Y-%m-%d-%T")
          echo -e "\nRepo init test\n" >> ${CORSO_LOG_FILE}
          ./corso repo init s3 \
            --no-stats --hide-progress --prefix $prefix \
            --bucket  ${CORSO_BUCKET}  2>&1 | tee $TEST_RESULT/initrepo.txt

          if ! grep -q 'Initialized a S3 repository within bucket' $TEST_RESULT/initrepo.txt
          then
            echo "Repo could not be initialized"
            exit 1
          fi

          echo result="$prefix" >> $GITHUB_OUTPUT

      - name: Repo connect test
        run: |
          set -euo pipefail
          echo -e "\nRepo connect test\n" >> ${CORSO_LOG_FILE}
          ./corso repo connect s3 \
            --no-stats --hide-progress --prefix ${{ steps.repo-init.outputs.result }} \
            --bucket  ${CORSO_BUCKET} 2>&1 | tee $TEST_RESULT/connect.txt

          if ! grep -q 'Connected to S3 bucket' $TEST_RESULT/connect.txt
          then
            echo "Repo could not be connected"
            exit 1
          fi

##########################################################################################################################################

# Exchange

      # generate new entries to roll into the next load test
      # only runs if the test was successful
      - name: Exchange - Create new data
        working-directory: ./src/cmd/factory
        run: |
          go run . exchange emails \
          --user ${TEST_USER} \
          --tenant ${AZURE_TENANT_ID} \
          --destination ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }} \
          --count 4

      - name: Exchange - Backup
        id: exchange-backup
        uses: ./.github/actions/backup-restore-test
        with:
          service: exchange
          kind: backup
          backup-args: '--mailbox "${TEST_USER}" --data "email"'
          restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'

      - name: Exchange - Incremental backup
        id: exchange-backup-incremental
        uses: ./.github/actions/backup-restore-test
        with:
          service: exchange
          kind: backup-incremental
          backup-args: '--mailbox "${TEST_USER}" --data "email"'
          restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}

      - name: Exchange - Non delta backup
        id: exchange-backup-non-delta
        uses: ./.github/actions/backup-restore-test
        with:
          service: exchange
          kind: backup-non-delta
          backup-args: '--mailbox "${TEST_USER}" --data "email" --disable-delta'
          restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}

      - name: Exchange - Incremental backup after non-delta
        id: exchange-backup-incremental-after-non-delta
        uses: ./.github/actions/backup-restore-test
        with:
          service: exchange
          kind: backup-incremental-after-non-delta
          backup-args: '--mailbox "${TEST_USER}" --data "email"'
          restore-args: '--email-folder ${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.repo-init.outputs.result }}'
          base-backup: ${{ steps.exchange-backup.outputs.backup-id }}


##########################################################################################################################################

# Onedrive

      # generate new entries for test
      - name: OneDrive - Create new data
        id: new-data-creation-onedrive
        working-directory: ./src/cmd/factory
        run: |
          suffix=$(date +"%Y-%m-%d_%H-%M-%S")

          go run . onedrive files  \
            --user ${TEST_USER} \
            --secondaryuser  ${SECONDARY_TEST_USER} \
            --tenant ${AZURE_TENANT_ID} \
            --destination ${RESTORE_DEST_PFX}$suffix \
            --count 4

          echo result="${suffix}" >> $GITHUB_OUTPUT

      - name: OneDrive - Backup
        id: onedrive-backup
        uses: ./.github/actions/backup-restore-test
        with:
          service: onedrive
          kind: backup
          backup-args: '--user "${TEST_USER}"'
          restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }}'

      # generate some more enteries for incremental check
      - name: OneDrive - Create new data (for incremental)
        working-directory: ./src/cmd/factory
        run: |
          go run . onedrive files  \
            --user ${TEST_USER} \
            --secondaryuser  ${SECONDARY_TEST_USER} \
            --tenant ${AZURE_TENANT_ID} \
            --destination ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} \
            --count 4

      - name: OneDrive - Incremental backup
        id: onedrive-incremental
        uses: ./.github/actions/backup-restore-test
        with:
          service: onedrive
          kind: incremental
          backup-args: '--user "${TEST_USER}"'
          restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }} --restore-permissions'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-onedrive.outputs.result }}'

##########################################################################################################################################

# Sharepoint

      # generate new entries for test
      - name: SharePoint - Create new data
        id: new-data-creation-sharepoint
        working-directory: ./src/cmd/factory
        run: |
          suffix=$(date +"%Y-%m-%d_%H-%M-%S")

          go run . sharepoint files  \
            --site ${TEST_SITE} \
            --user ${TEST_USER} \
            --secondaryuser  ${SECONDARY_TEST_USER} \
            --tenant ${AZURE_TENANT_ID} \
            --destination ${RESTORE_DEST_PFX}$suffix \
            --count 4

          echo result="${suffix}" >> $GITHUB_OUTPUT

      - name: SharePoint - Backup
        id: sharepoint-backup
        uses: ./.github/actions/backup-restore-test
        with:
          service: sharepoint
          kind: backup
          backup-args: '--site "${TEST_SITE}"'
          restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }}'

      # generate some more enteries for incremental check
      - name: SharePoint - Create new data (for incremental)
        working-directory: ./src/cmd/factory
        run: |
          go run . sharepoint files  \
            --site ${TEST_SITE} \
            --user ${TEST_USER} \
            --secondaryuser  ${SECONDARY_TEST_USER} \
            --tenant ${AZURE_TENANT_ID} \
            --destination ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} \
            --count 4

      - name: SharePoint - Incremental backup
        id: sharepoint-incremental
        uses: ./.github/actions/backup-restore-test
        with:
          service: sharepoint
          kind: incremental
          backup-args: '--site "${TEST_SITE}"'
          restore-args: '--folder ${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }} --restore-permissions'
          test-folder: '${RESTORE_DEST_PFX}${{ steps.new-data-creation-sharepoint.outputs.result }}'

##########################################################################################################################################

# Logging & Notify

      # Upload the original go test output as an artifact for later review.
      - name: Upload test log
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: "test-logs"
          path: ${{ env.WORKING_DIR }}/${{ env.CORSO_LOG_DIR }}/
          if-no-files-found: error
          retention-days: 14

      - name: SHA info
        id: sha-info
        if: failure()
        run: |
          echo ${GITHUB_REF#refs/heads/}-${GITHUB_SHA}
          echo SHA=${GITHUB_REF#refs/heads/}-${GITHUB_SHA} >> $GITHUB_OUTPUT
          echo RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}  >> $GITHUB_OUTPUT
          echo COMMIT_URL=${{ github.server_url }}/${{ github.repository }}/commit/${GITHUB_SHA} >> $GITHUB_OUTPUT

      - name: Send Github Action failure to Slack
        id: slack-notification
        if: failure()
        uses: slackapi/slack-github-action@v1.23.0
        with:
          payload: |
            {
              "text": "GitHub Action build result: ${{ job.status }} on SHA: ${{  steps.sha-info.outputs.SHA }}",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "[FAILED] Sanity Checks :: <${{  steps.sha-info.outputs.RUN_URL }}|[Logs]> <${{ github.event.pull_request.html_url || github.event.head_commit.url }}|[Base]>\nCommit: <${{  steps.sha-info.outputs.COMMIT_URL }}|${{  steps.sha-info.outputs.SHA }}>"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK