From 88812dc70accddc4017b65f4d1b7cd99f8d05f2d Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 9 May 2023 15:21:37 -0600 Subject: [PATCH] run cli e2e on the nightly schedule, not CI (#3367) e2e tests are large, slow, and lack configuration control that prevents data restoration explosions. This moves those tests out of the standard CI and into the nightly test suite to be run on a less frequent cadence. The goal is to improve CI test speed and test stability. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :robot: Supportability/Tests #### Test Plan - [x] :green_heart: E2E --- .github/workflows/load_test.yml | 10 ++++++---- .github/workflows/nightly_test.yml | 5 +++-- src/cli/backup/exchange_e2e_test.go | 4 ---- src/cli/backup/onedrive_e2e_test.go | 8 ++------ src/cli/backup/sharepoint_e2e_test.go | 5 +---- src/cli/repo/s3_e2e_test.go | 1 - src/cli/restore/exchange_e2e_test.go | 4 +--- 7 files changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index 5cc1e3c05..9241b3d8f 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -1,10 +1,8 @@ name: Nightly Load Testing on: schedule: - # every day at 01:59 (01:59am) UTC - # - cron: "59 1 * * *" - # temp, for testing: every 4 hours - - cron: "0 */4 * * *" + # every day at 03:59 GMT (roughly 8pm PST) + - cron: "59 3 * * *" permissions: # required to retrieve AWS credentials @@ -20,6 +18,10 @@ jobs: Load-Tests: environment: Load Testing runs-on: ubuntu-latest + # Skipping load testing for now. They need some love to get up and + # running properly, and it's better to not fight for resources with + # tests that are guaranteed to fail. + if: false defaults: run: working-directory: src diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index ccc93fdce..2ab0b7b8d 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -3,8 +3,8 @@ on: workflow_dispatch: schedule: - # Run every day at 0 minutes and 0 hours (midnight GMT) - - cron: "0 0 * * *" + # Run every day at 04:00 GMT (roughly 8pm PST) + - cron: "0 4 * * *" permissions: # required to retrieve AWS credentials @@ -122,6 +122,7 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets[env.AZURE_CLIENT_SECRET_NAME] }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} CORSO_NIGHTLY_TESTS: true + CORSO_E2E_TESTS: true CORSO_M365_TEST_USER_ID: ${{ vars.CORSO_M365_TEST_USER_ID }} CORSO_SECONDARY_M365_TEST_USER_ID: ${{ vars.CORSO_SECONDARY_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} diff --git a/src/cli/backup/exchange_e2e_test.go b/src/cli/backup/exchange_e2e_test.go index d135c8747..e5c60df2b 100644 --- a/src/cli/backup/exchange_e2e_test.go +++ b/src/cli/backup/exchange_e2e_test.go @@ -54,7 +54,6 @@ func TestNoBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &NoBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -120,7 +119,6 @@ func TestBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &BackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -235,7 +233,6 @@ func TestPreparedBackupExchangeE2ESuite(t *testing.T) { suite.Run(t, &PreparedBackupExchangeE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -490,7 +487,6 @@ func TestBackupDeleteExchangeE2ESuite(t *testing.T) { Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, ), }) } diff --git a/src/cli/backup/onedrive_e2e_test.go b/src/cli/backup/onedrive_e2e_test.go index d41bbc1aa..9e6c134bc 100644 --- a/src/cli/backup/onedrive_e2e_test.go +++ b/src/cli/backup/onedrive_e2e_test.go @@ -44,9 +44,7 @@ func TestNoBackupOneDriveE2ESuite(t *testing.T) { suite.Run(t, &NoBackupOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } @@ -148,9 +146,7 @@ func TestBackupDeleteOneDriveE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteOneDriveE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/backup/sharepoint_e2e_test.go b/src/cli/backup/sharepoint_e2e_test.go index 4471e9755..09d65d90e 100644 --- a/src/cli/backup/sharepoint_e2e_test.go +++ b/src/cli/backup/sharepoint_e2e_test.go @@ -45,7 +45,6 @@ func TestNoBackupSharePointE2ESuite(t *testing.T) { suite.Run(t, &NoBackupSharePointE2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } @@ -112,9 +111,7 @@ func TestBackupDeleteSharePointE2ESuite(t *testing.T) { suite.Run(t, &BackupDeleteSharePointE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index d5e6c992e..388b687e2 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -25,7 +25,6 @@ func TestS3E2ESuite(t *testing.T) { suite.Run(t, &S3E2ESuite{Suite: tester.NewE2ESuite( t, [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, )}) } diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 0d9bf7b58..30114aa4f 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -48,9 +48,7 @@ func TestRestoreExchangeE2ESuite(t *testing.T) { suite.Run(t, &RestoreExchangeE2ESuite{ Suite: tester.NewE2ESuite( t, - [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}, - tester.CorsoCITests, - ), + [][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs}), }) }