From 011e24583f7e5f073e9614538953c3be22f46c5b Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 28 Sep 2022 20:01:15 -0600 Subject: [PATCH] add profiling, tracing to load testing (#971) ## Description add diagnostic records to load test runs ## Type of change - [x] :robot: Test - [x] :computer: CI/Deployment ## Issue(s) * #902 ## Test Plan - [x] :green_heart: E2E --- .github/workflows/ci.yml | 14 +++++---- .github/workflows/load_test.yml | 34 ++++++++++++++++------ src/pkg/repository/repository_load_test.go | 33 ++++++++++++++++----- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d62889178..3dc920698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: permissions: # required to retrieve AWS credentials id-token: write - contents: read + contents: write pull-requests: read # cancel currently running jobs if a new version of the branch is pushed @@ -62,6 +62,8 @@ jobs: with: go-version-file: src/go.mod + - run: mkdir testlog + # Install gotestfmt - name: Set up gotestfmt if: needs.precheck.outputs.fileschanged == 'true' @@ -85,12 +87,13 @@ jobs: CORSO_CI_TESTS: true CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} - RUDDERSTACK_CORSO_WRITE_KEY: ${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }} - RUDDERSTACK_CORSO_DATA_PLANE_URL: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }} TENANT_ID: ${{ secrets.TENANT_ID }} run: | set -euo pipefail - go test -json -v ./... 2>&1 | tee /tmp/gotest.log | gotestfmt -hide successful-tests + go test \ + -json \ + -v \ + ./... 2>&1 | tee ./testlog/gotest.log | gotestfmt -hide successful-tests # Upload the original go test log as an artifact for later review. - name: Upload test log @@ -98,8 +101,9 @@ jobs: uses: actions/upload-artifact@v3 with: name: test-log - path: /tmp/gotest.log + path: testlog/gotest.log if-no-files-found: error + retention-days: 14 # ---------------------------------------------------------------------------------------------------- # --- Source Code Linting ---------------------------------------------------------------------------- diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index f68eda14e..274d7e448 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -7,7 +7,12 @@ on: permissions: # required to retrieve AWS credentials id-token: write - contents: read + contents: write + +# cancel currently running jobs if a new version of the branch is pushed +concurrency: + group: load_testing-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: Load-Tests: @@ -24,6 +29,8 @@ jobs: with: go-version-file: src/go.mod + - run: mkdir test_results + # Install gotestfmt - name: Set up gotestfmt run: go install github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest @@ -47,16 +54,25 @@ jobs: run: | set -euo pipefail go test \ + -count=1 \ -json \ -v \ - -count=1 \ --timeout 12h \ - ./... 2>&1 | tee /tmp/gotest.log | gotestfmt -hide successful-tests + -blockprofile=block.prof \ + -cpuprofile=cpu.prof \ + -memprofile=mem.prof \ + -mutexprofile=mutex.prof \ + -trace=trace.out \ + -outputdir=test_results \ + ./pkg/repository/repository_load_test.go \ + 2>&1 | tee ./test_results/goloadtest.log | gotestfmt -hide successful-tests - # Upload the original go test log as an artifact for later review. - - name: Upload test log - uses: actions/upload-artifact@v2 + # package all artifacts for later review + - name: Upload Log, Profilers, Traces + if: always() + uses: actions/upload-artifact@v3 with: - name: test-log - path: /tmp/gotest.log - if-no-files-found: error \ No newline at end of file + name: load-test-profiling + path: src/test_results/* + if-no-files-found: error + retention-days: 14 diff --git a/src/pkg/repository/repository_load_test.go b/src/pkg/repository/repository_load_test.go index 8a68f8dcf..3fadaaf34 100644 --- a/src/pkg/repository/repository_load_test.go +++ b/src/pkg/repository/repository_load_test.go @@ -9,9 +9,11 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common" + "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/repository" @@ -105,9 +107,20 @@ func runBackupDetailsLoadTest( t.Run("backup_details_"+name, func(t *testing.T) { ds, b, err := r.BackupDetails(ctx, backupID) require.NoError(t, err, "retrieving details in backup "+backupID) - require.NotNil(t, ds, "backup details") - require.NotNil(t, b, "backup") - assert.Equal(t, b.ItemsWritten, len(ds.Entries)) + require.NotNil(t, ds, "backup details must exist") + require.NotNil(t, b, "backup must exist") + + sansfldr := []details.DetailsEntry{} + + for _, ent := range ds.Entries { + if ent.Folder == nil { + sansfldr = append(sansfldr, ent) + } + } + + assert.Equal(t, + b.ItemsWritten, len(sansfldr), + "items written to backup must match the count of entries, minus folder entries") }) } @@ -156,6 +169,7 @@ func TestRepositoryLoadTestExchangeSuite(t *testing.T) { func (suite *RepositoryLoadTestExchangeSuite) SetupSuite() { t := suite.T() + t.Parallel() suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) } @@ -179,13 +193,12 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() { service = "exchange" ) - t.Parallel() - m356User := tester.M365UserID(t) // backup bsel := selectors.NewExchangeBackup() - bsel.Include(bsel.Users([]string{m356User})) + bsel.Include(bsel.MailFolders([]string{m356User}, []string{exchange.DefaultMailFolder})) + // bsel.Include(bsel.Users([]string{m356User})) // bsel.Include(bsel.Users(selectors.Any())) b, err := r.NewBackup(ctx, bsel.Selector) @@ -231,6 +244,7 @@ func TestRepositoryLoadTestOneDriveSuite(t *testing.T) { func (suite *RepositoryLoadTestOneDriveSuite) SetupSuite() { t := suite.T() + t.Parallel() suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) } @@ -254,11 +268,14 @@ func (suite *RepositoryLoadTestOneDriveSuite) TestOneDrive() { service = "one_drive" ) - t.Parallel() + t.Skip("temp issue-902-live") + + m356User := tester.M365UserID(t) // backup bsel := selectors.NewOneDriveBackup() - bsel.Include(bsel.Users(selectors.Any())) + bsel.Include(bsel.Users([]string{m356User})) + // bsel.Include(bsel.Users(selectors.Any())) b, err := r.NewBackup(ctx, bsel.Selector) require.NoError(t, err)