corso/.github/workflows/load_test.yml

name: Nightly Load Testing
on:
  schedule:
    # every day at 01:59 (01:59am) UTC
    # - cron: "59 1 * * *"
    # temp, for testing: every 4 hours
    - cron: "0 */4 * * *"

permissions:
  # required to retrieve AWS credentials
  id-token: write
  contents: write

# cancel currently running jobs if a new version of the branch is pushed
concurrency:
  group: load_testing-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  Load-Tests:
    environment: Load Testing
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: src
    steps:
      - uses: actions/checkout@v3

      - name: Build the otel-daemon
        run: make build-otel-daemon

      # TODO: write logs to a file in src/test_results for archival
      - name: Run the otel-daemon
        run: make otel-daemon

      - name: Setup Golang with cache
        uses: magnetikonline/action-golang-cache@v4
        with:
          go-version-file: src/go.mod

      - run: mkdir test_results

      # Install gotestfmt
      - name: Set up gotestfmt
        run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest

      # run the tests
      - name: Integration Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY_SECRET }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          CORSO_LOAD_TESTS: true 
          CORSO_LOG_FILE: ./src/test_results/testlogging.log
          CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
          CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
          CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}
          IGNORE_LOAD_TEST_USER_ID: ${{ secrets.EXT_SDK_TEST_USER_ID }}
          LOG_GRAPH_REQUESTS: true
        run: |
          set -euo pipefail
          go test       \
          -count=1      \
          -json         \
          -v            \
          --timeout 12h \
          -blockprofile=block.prof \
          -cpuprofile=cpu.prof     \
          -memprofile=mem.prof     \
          -mutexprofile=mutex.prof \
          -trace=trace.out         \
          -outputdir=test_results  \
          ./pkg/repository/repository_load_test.go \
          2>&1 | tee ./test_results/goloadtest.log | gotestfmt -hide successful-tests

      # generate new entries to roll into the next load test
      # only runs if the test was successful
      - name: New Data Creation
        working-directory: ./src/cmd/factory
        env:
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
        run: |
          go run . exchange emails \
          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
          --destination lt_${{ env.NOW }} \
          --count 10
          go run . exchange contacts \
          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
          --destination lt_${{ env.NOW }} \
          --count 10
          go run . exchange events \
          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
          --destination lt_${{ env.NOW }} \
          --count 10

      - name: Put Down the Daemons Arisen
        if: always()
        run: docker kill otel-daemon

      # package all artifacts for later review
      - name: Upload Log, Profilers, Traces
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: load-test-profiling
          path: src/test_results/*
          if-no-files-found: error
          retention-days: 14
  
  setup:
    environment: Load Testing
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.build.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
      - id: build
        run: |
          u=$(echo "${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}" | sed 's/\[/["/g' | sed 's/\]/"]/g' | sed 's/|/","/g')
          marr=$(cat <<-end
            { "user": $u }
          end
          )
          m=$(echo $marr | jq -c .)
          echo "matrix=$m" >> $GITHUB_OUTPUT

  purge-load-test-user-data:
    needs: [setup, Load-Tests]
    if: always()
    environment: Load Testing
    runs-on: ubuntu-latest
    strategy:
      matrix:
        user: ${{ fromJson(needs.setup.outputs.matrix).user }}
        folder: [Corso_Restore_, '']
    steps:
      - uses: actions/checkout@v3
      - name: Set folder boundary datetime
        run: |
          echo "NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
      - name: Purge Load-Test-Produced Folders
        uses: ./.github/actions/purge-m365-user-data
        with:
          older-than: ${{ env.NOW }}
          folder-prefix: ${{ matrix.folder }}
          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
          azure-client-secret: ${{ secrets.AZURE_CLIENT_SECRET }}
          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
          user: ${{ matrix.user }}