diff --git a/.github/actions/purge-m365-user-data/action.yml b/.github/actions/purge-m365-user-data/action.yml index 8f2114178..975c0852b 100644 --- a/.github/actions/purge-m365-user-data/action.yml +++ b/.github/actions/purge-m365-user-data/action.yml @@ -18,7 +18,7 @@ inputs: user: description: User whose data is to be purged. folder-prefix: - description: Name of the folder to be purged. If false, will purge the set of static, well known folders instead. + description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: description: Minimum-age of folders to be deleted. azure-client-id: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07c4fd592..b20cdf61e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,8 +173,8 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} - CORSO_CI_TESTS: true - CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }} + CORSO_CI_TESTS: true + CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} run: | set -euo pipefail diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 3e16272a4..461b90e6a 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -12,12 +12,10 @@ jobs: strategy: matrix: folder: [Corso_Restore_, TestRestore, ''] - user: [CORSO_M356_TEST_USER_ID, CORSO_SECONDARY_M356_TEST_USER_ID] + user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID] steps: - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} # sets the maximimum time to now-30m. # CI test have a 10 minute timeout. diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index 3b0bdc488..f7bffe014 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -3,8 +3,8 @@ on: schedule: # every day at 01:59 (01:59am) UTC # - cron: "59 1 * * *" - # temp, for testing: every 4 hours - - cron: "0 */4 * * *" + # temp, for testing: every 6 hours + - cron: "0 */6 * * *" permissions: # required to retrieve AWS credentials @@ -59,7 +59,8 @@ jobs: AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} - CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }} + CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }} + CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }} CORSO_LOAD_TESTS: true run: | set -euo pipefail @@ -76,14 +77,6 @@ jobs: -outputdir=test_results \ ./pkg/repository/repository_load_test.go \ 2>&1 | tee ./test_results/goloadtest.log | gotestfmt -hide successful-tests - - # currently, we're required to generate a unique folder for each factory - # production. Whenever possible, this should be reverted to increasing the - # item count of a single folder instead, to prevent overproduction of folders - # during restore. - - name: Set folder destination date - run: | - echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV # generate new entries to roll into the next load test # only runs if the test was successful @@ -93,35 +86,21 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} - CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }} + CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }} run: | go run . exchange emails \ - --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \ + --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \ --destination lt_${{ env.NOW }} \ --count 10 go run . exchange contacts \ - --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \ + --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \ --destination lt_${{ env.NOW }} \ --count 10 go run . exchange events \ - --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \ + --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \ --destination lt_${{ env.NOW }} \ --count 10 - # cleanup folders produced by load test - - name: Restored Folder Purge - if: always() - working-directory: ./src - env: - AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} - AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} - DELETE_FOLDER_PREFIX: "Corso_Restore_" - run: > - go run ./cmd/purge/purge.go - --user '*' - --prefix ${{ env.DELETE_FOLDER_PREFIX }} - - name: Put Down the Daemons Arisen if: always() run: docker kill otel-daemon @@ -135,3 +114,44 @@ jobs: path: src/test_results/* if-no-files-found: error retention-days: 14 + + setup: + environment: Testing + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - id: build + run: | + u=$(echo "${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}" | sed 's/\[/["/g' | sed 's/\]/"]/g' | sed 's/|/","/g') + marr=$(cat <<-end + { "user": $u } + end + ) + m=$(echo $marr | jq -c .) + echo "matrix=$m" >> $GITHUB_OUTPUT + + purge-load-test-user-data: + needs: [setup, Load-Tests] + if: always() + environment: Testing + runs-on: ubuntu-latest + strategy: + matrix: + user: ${{ fromJson(needs.setup.outputs.matrix).user }} + folder: [Corso_Restore_,''] + steps: + - uses: actions/checkout@v3 + - name: Set folder boundary datetime + run: | + echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV + - name: Purge Load-Test-Produced Folders + uses: ./.github/actions/purge-m365-user-data + with: + older-than: ${{ env.NOW }} + folder-prefix: ${{ matrix.folder }} + azure-client-id: ${{ secrets.CLIENT_ID }} + azure-client-secret: ${{ secrets.CLIENT_SECRET }} + azure-tenant-id: ${{ secrets.TENANT_ID }} + user: ${{ matrix.user }} diff --git a/docs/docs/developers/testing.md b/docs/docs/developers/testing.md index 1071a7562..f0e8ce7a3 100644 --- a/docs/docs/developers/testing.md +++ b/docs/docs/developers/testing.md @@ -51,8 +51,8 @@ The complete list of environment constants is available at ## Advanced options -- To override the M365 user for tests, use `CORSO_M356_TEST_USER_ID` +- To override the M365 user for tests, use `CORSO_M365_TEST_USER_ID` ```bash - export CORSO_M356_TEST_USER_ID="..." + export CORSO_M365_TEST_USER_ID="..." ``` diff --git a/src/.golangci.yml b/src/.golangci.yml index 0d65a6d9d..55cae6fd6 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -28,6 +28,8 @@ linters-settings: # Don't allow use of path as it hardcodes separator to `/`. # Use filepath instead. - '\bpath\.(Ext|Base|Dir|Join)' + # Don't allow the typo m356 to be used in place of m365. + - '[Mm]356' lll: line-length: 120 revive: diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 6cbe44c13..01b12bf53 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -94,7 +94,7 @@ func s3InitCmd() *cobra.Command { return &cobra.Command{ Use: s3ProviderCommand, Short: "Initialize a S3 repository", - Long: `Bootstraps a new S3 repository and connects it to your m356 account.`, + Long: `Bootstraps a new S3 repository and connects it to your m365 account.`, RunE: initS3Cmd, Args: cobra.NoArgs, Example: s3ProviderCommandInitExamples, diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index feabd63a7..cc98e13b0 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -75,7 +75,7 @@ func (gs graphService) ErrPolicy() bool { func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnector, error) { m365, err := acct.M365Config() if err != nil { - return nil, errors.Wrap(err, "retrieving m356 account configuration") + return nil, errors.Wrap(err, "retrieving m365 account configuration") } gc := GraphConnector{ diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 0a7e874dc..6c4b9cce5 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -20,18 +20,20 @@ const ( TestCfgStorageProvider = "provider" // M365 config - TestCfgAzureTenantID = "azure_tenantid" - TestCfgUserID = "m365userid" - TestCfgSecondaryUserID = "secondarym365userid" - TestCfgLoadTestUserID = "loadttestm365userid" - TestCfgAccountProvider = "account_provider" + TestCfgAzureTenantID = "azure_tenantid" + TestCfgUserID = "m365userid" + TestCfgSecondaryUserID = "secondarym365userid" + TestCfgLoadTestUserID = "loadtestm365userid" + TestCfgLoadTestOrgUsers = "loadtestm365orgusers" + TestCfgAccountProvider = "account_provider" ) // test specific env vars const ( - EnvCorsoM365TestUserID = "CORSO_M356_TEST_USER_ID" - EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M356_TEST_USER_ID" - EnvCorsoM365LoadTestUserID = "CORSO_M356_LOAD_TEST_USER_ID" + EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" + EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" + EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" + EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS" EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE" ) @@ -128,6 +130,13 @@ func readTestConfig() (map[string]string, error) { vpr.GetString(TestCfgLoadTestUserID), "leeg@8qzvrj.onmicrosoft.com", ) + fallbackTo( + testEnv, + TestCfgLoadTestOrgUsers, + os.Getenv(EnvCorsoM365LoadTestOrgUsers), + vpr.GetString(TestCfgLoadTestOrgUsers), + "lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com", + ) testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath) testConfig = testEnv diff --git a/src/internal/tester/users.go b/src/internal/tester/users.go index e5ccfe8f9..7e1ff41f1 100644 --- a/src/internal/tester/users.go +++ b/src/internal/tester/users.go @@ -1,13 +1,14 @@ package tester import ( + "strings" "testing" "github.com/stretchr/testify/require" ) // M365UserID returns an userID string representing the m365UserID described -// by either the env var CORSO_M356_TEST_USER_ID, the corso_test.toml config +// by either the env var CORSO_M365_TEST_USER_ID, the corso_test.toml config // file or the default value (in that order of priority). The default is a // last-attempt fallback that will only work on alcion's testing org. func M365UserID(t *testing.T) string { @@ -18,7 +19,7 @@ func M365UserID(t *testing.T) string { } // SecondaryM365UserID returns an userID string representing the m365UserID -// described by either the env var CORSO_SECONDARY_M356_TEST_USER_ID, the +// described by either the env var CORSO_SECONDARY_M365_TEST_USER_ID, the // corso_test.toml config file or the default value (in that order of priority). // The default is a last-attempt fallback that will only work on alcion's // testing org. @@ -30,7 +31,7 @@ func SecondaryM365UserID(t *testing.T) string { } // LoadTestM365UserID returns an userID string representing the m365UserID -// described by either the env var CORSO_M356_LOAD_TEST_USER_ID, the +// described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the // corso_test.toml config file or the default value (in that order of priority). // The default is a last-attempt fallback that will only work on alcion's // testing org. @@ -40,3 +41,19 @@ func LoadTestM365UserID(t *testing.T) string { return cfg[TestCfgLoadTestUserID] } + +// expects cfg value to be a string representing an array like: +// "['foo@example.com','bar@example.com']" +func LoadTestM365OrgUsers(t *testing.T) []string { + cfg, err := readTestConfig() + require.NoError(t, err, "retrieving load test m365 org users from test configuration") + + users := cfg[TestCfgLoadTestOrgUsers] + users = strings.TrimPrefix(users, "[") + users = strings.TrimSuffix(users, "]") + users = strings.ReplaceAll(users, `"`, "") + users = strings.ReplaceAll(users, `'`, "") + users = strings.ReplaceAll(users, "|", ",") + + return strings.Split(users, ",") +} diff --git a/src/pkg/repository/repository_load_test.go b/src/pkg/repository/repository_load_test.go index 1c858acf7..629d6049e 100644 --- a/src/pkg/repository/repository_load_test.go +++ b/src/pkg/repository/repository_load_test.go @@ -25,34 +25,8 @@ import ( "github.com/alcionai/corso/src/pkg/storage" ) -func userSet(t *testing.T) []string { - // avoid adding the following users - // they are reserved for other purposes - // "LeeG@8qzvrj.onmicrosoft.com", - // "ntoja@8qzvrj.onmicrosoft.com", - return []string{ - "AdeleV@8qzvrj.onmicrosoft.com", - "AlexW@8qzvrj.onmicrosoft.com", - "ashmarks@8qzvrj.onmicrosoft.com", - "DiegoS@8qzvrj.onmicrosoft.com", - "dustina@8qzvrj.onmicrosoft.com", - "george.martinez@8qzvrj.onmicrosoft.com", - "GradyA@8qzvrj.onmicrosoft.com", - "HenriettaM@8qzvrj.onmicrosoft.com", - "IsaiahL@8qzvrj.onmicrosoft.com", - "JohannaL@8qzvrj.onmicrosoft.com", - "JoniS@8qzvrj.onmicrosoft.com", - "LidiaH@8qzvrj.onmicrosoft.com", - "LynneR@8qzvrj.onmicrosoft.com", - "MeganB@8qzvrj.onmicrosoft.com", - "MiriamG@8qzvrj.onmicrosoft.com", - "NestorW@8qzvrj.onmicrosoft.com", - "PattiF@8qzvrj.onmicrosoft.com", - "PradeepG@8qzvrj.onmicrosoft.com", - "Rfinders@8qzvrj.onmicrosoft.com", - "vkarma@8qzvrj.onmicrosoft.com", - "greg.sanders@8qzvrj.onmicrosoft.com", - } +func orgUserSet(t *testing.T) []string { + return tester.LoadTestM365OrgUsers(t) } func singleUserSet(t *testing.T) []string { @@ -403,7 +377,7 @@ func (suite *RepositoryLoadTestExchangeSuite) SetupSuite() { t := suite.T() t.Parallel() suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) - suite.usersUnderTest = userSet(t) + suite.usersUnderTest = orgUserSet(t) } func (suite *RepositoryLoadTestExchangeSuite) TeardownSuite() { @@ -506,7 +480,7 @@ func (suite *RepositoryLoadTestOneDriveSuite) SetupSuite() { t.Skip("temp issue-902-live") t.Parallel() suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) - suite.usersUnderTest = userSet(t) + suite.usersUnderTest = orgUserSet(t) } func (suite *RepositoryLoadTestOneDriveSuite) TeardownSuite() {