replace golang purge script with powershell script (#1407)

## Description

Three primary changes:
1. move the list of load test users into a secret in the github test domain.
2. purge all data that was generated for users in the load test
3. fix all `m356` typos.

## Type of change

- [x] 🤖 Test
- [x] 💻 CI/Deployment

## Issue(s)

* #1266

## Test Plan

- [x] 💪 Manual
- [x] 💚 E2E

.github/actions/purge-m365-user-data/action.yml

@@ -18,7 +18,7 @@ inputs:
   user:
     description: User whose data is to be purged.
   folder-prefix:
-    description: Name of the folder to be purged.  If false, will purge the set of static, well known folders instead.
+    description: Name of the folder to be purged.  If falsy, will purge the set of static, well known folders instead.
   older-than:
     description: Minimum-age of folders to be deleted.
   azure-client-id:

.github/workflows/ci.yml

@@ -173,8 +173,8 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
-          CORSO_CI_TESTS: true
+          CORSO_CI_TESTS: true 
-          CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }}
+          CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
         run: |
           set -euo pipefail

.github/workflows/ci_test_cleanup.yml

@@ -12,12 +12,10 @@ jobs:
     strategy:
       matrix:
         folder: [Corso_Restore_, TestRestore, '']
-        user: [CORSO_M356_TEST_USER_ID, CORSO_SECONDARY_M356_TEST_USER_ID]
+        user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID]
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
 
       # sets the maximimum time to now-30m.
       # CI test have a 10 minute timeout.

.github/workflows/load_test.yml

@@ -3,8 +3,8 @@ on:
   schedule:
     # every day at 01:59 (01:59am) UTC
     # - cron: "59 1 * * *"
-    # temp, for testing: every 4 hours
+    # temp, for testing: every 6 hours
-    - cron: "0 */4 * * *"
+    - cron: "0 */6 * * *"
 
 permissions:
   # required to retrieve AWS credentials
@@ -59,7 +59,8 @@ jobs:
           AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
-          CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
+          CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
+          CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
           CORSO_LOAD_TESTS: true 
         run: |
           set -euo pipefail
@@ -76,14 +77,6 @@ jobs:
           -outputdir=test_results  \
           ./pkg/repository/repository_load_test.go \
           2>&1 | tee ./test_results/goloadtest.log | gotestfmt -hide successful-tests
-        
-      # currently, we're required to generate a unique folder for each factory
-      # production.  Whenever possible, this should be reverted to increasing the
-      # item count of a single folder instead, to prevent overproduction of folders
-      # during restore.
-      - name: Set folder destination date
-        run: |
-          echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV
 
       # generate new entries to roll into the next load test
       # only runs if the test was successful
@@ -93,35 +86,21 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
-          CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
+          CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
         run: |
           go run . exchange emails \
-          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
           --destination lt_${{ env.NOW }} \
           --count 10
           go run . exchange contacts \
-          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
           --destination lt_${{ env.NOW }} \
           --count 10
           go run . exchange events \
-          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
           --destination lt_${{ env.NOW }} \
           --count 10
 
-      # cleanup folders produced by load test
-      - name: Restored Folder Purge
-        if: always()
-        working-directory: ./src
-        env:
-          AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
-          AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
-          AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
-          DELETE_FOLDER_PREFIX: "Corso_Restore_"
-        run: >
-          go run ./cmd/purge/purge.go
-          --user '*'
-          --prefix ${{ env.DELETE_FOLDER_PREFIX }}
-
       - name: Put Down the Daemons Arisen
         if: always()
         run: docker kill otel-daemon
@@ -135,3 +114,44 @@ jobs:
           path: src/test_results/*
           if-no-files-found: error
           retention-days: 14
+  
+  setup:
+    environment: Testing
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.build.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - id: build
+        run: |
+          u=$(echo "${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}" | sed 's/\[/["/g' | sed 's/\]/"]/g' | sed 's/|/","/g')
+          marr=$(cat <<-end
+            { "user": $u }
+          end
+          )
+          m=$(echo $marr | jq -c .)
+          echo "matrix=$m" >> $GITHUB_OUTPUT
+
+  purge-load-test-user-data:
+    needs: [setup, Load-Tests]
+    if: always()
+    environment: Testing
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        user: ${{ fromJson(needs.setup.outputs.matrix).user }}
+        folder: [Corso_Restore_,'']
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set folder boundary datetime
+        run: |
+          echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV
+      - name: Purge Load-Test-Produced Folders
+        uses: ./.github/actions/purge-m365-user-data
+        with:
+          older-than: ${{ env.NOW }}
+          folder-prefix: ${{ matrix.folder }}
+          azure-client-id: ${{ secrets.CLIENT_ID }}
+          azure-client-secret: ${{ secrets.CLIENT_SECRET }}
+          azure-tenant-id: ${{ secrets.TENANT_ID }}
+          user: ${{ matrix.user }}

docs/docs/developers/testing.md

@@ -51,8 +51,8 @@ The complete list of environment constants is available at
 
 ## Advanced options
 
-- To override the M365 user for tests, use `CORSO_M356_TEST_USER_ID`
+- To override the M365 user for tests, use `CORSO_M365_TEST_USER_ID`
 
     ```bash
-    export CORSO_M356_TEST_USER_ID="..."
+    export CORSO_M365_TEST_USER_ID="..."
     ```

src/.golangci.yml

@@ -28,6 +28,8 @@ linters-settings:
       # Don't allow use of path as it hardcodes separator to `/`.
       # Use filepath instead.
       - '\bpath\.(Ext|Base|Dir|Join)'
+      # Don't allow the typo m356 to be used in place of m365.
+      - '[Mm]356'
   lll:
     line-length: 120
   revive:

src/cli/repo/s3.go

@@ -94,7 +94,7 @@ func s3InitCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:     s3ProviderCommand,
 		Short:   "Initialize a S3 repository",
-		Long:    `Bootstraps a new S3 repository and connects it to your m356 account.`,
+		Long:    `Bootstraps a new S3 repository and connects it to your m365 account.`,
 		RunE:    initS3Cmd,
 		Args:    cobra.NoArgs,
 		Example: s3ProviderCommandInitExamples,

src/internal/connector/graph_connector.go

@@ -75,7 +75,7 @@ func (gs graphService) ErrPolicy() bool {
 func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnector, error) {
 	m365, err := acct.M365Config()
 	if err != nil {
-		return nil, errors.Wrap(err, "retrieving m356 account configuration")
+		return nil, errors.Wrap(err, "retrieving m365 account configuration")
 	}
 
 	gc := GraphConnector{

src/internal/tester/config.go

@@ -20,18 +20,20 @@ const (
 	TestCfgStorageProvider = "provider"
 
 	// M365 config
-	TestCfgAzureTenantID   = "azure_tenantid"
+	TestCfgAzureTenantID    = "azure_tenantid"
-	TestCfgUserID          = "m365userid"
+	TestCfgUserID           = "m365userid"
-	TestCfgSecondaryUserID = "secondarym365userid"
+	TestCfgSecondaryUserID  = "secondarym365userid"
-	TestCfgLoadTestUserID  = "loadttestm365userid"
+	TestCfgLoadTestUserID   = "loadtestm365userid"
-	TestCfgAccountProvider = "account_provider"
+	TestCfgLoadTestOrgUsers = "loadtestm365orgusers"
+	TestCfgAccountProvider  = "account_provider"
 )
 
 // test specific env vars
 const (
-	EnvCorsoM365TestUserID          = "CORSO_M356_TEST_USER_ID"
+	EnvCorsoM365TestUserID          = "CORSO_M365_TEST_USER_ID"
-	EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M356_TEST_USER_ID"
+	EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID"
-	EnvCorsoM365LoadTestUserID      = "CORSO_M356_LOAD_TEST_USER_ID"
+	EnvCorsoM365LoadTestUserID      = "CORSO_M365_LOAD_TEST_USER_ID"
+	EnvCorsoM365LoadTestOrgUsers    = "CORSO_M365_LOAD_TEST_ORG_USERS"
 	EnvCorsoTestConfigFilePath      = "CORSO_TEST_CONFIG_FILE"
 )
 
@@ -128,6 +130,13 @@ func readTestConfig() (map[string]string, error) {
 		vpr.GetString(TestCfgLoadTestUserID),
 		"leeg@8qzvrj.onmicrosoft.com",
 	)
+	fallbackTo(
+		testEnv,
+		TestCfgLoadTestOrgUsers,
+		os.Getenv(EnvCorsoM365LoadTestOrgUsers),
+		vpr.GetString(TestCfgLoadTestOrgUsers),
+		"lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com",
+	)
 
 	testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
 	testConfig = testEnv

src/internal/tester/users.go

@@ -1,13 +1,14 @@
 package tester
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
 // M365UserID returns an userID string representing the m365UserID described
-// by either the env var CORSO_M356_TEST_USER_ID, the corso_test.toml config
+// by either the env var CORSO_M365_TEST_USER_ID, the corso_test.toml config
 // file or the default value (in that order of priority).  The default is a
 // last-attempt fallback that will only work on alcion's testing org.
 func M365UserID(t *testing.T) string {
@@ -18,7 +19,7 @@ func M365UserID(t *testing.T) string {
 }
 
 // SecondaryM365UserID returns an userID string representing the m365UserID
-// described by either the env var CORSO_SECONDARY_M356_TEST_USER_ID, the
+// described by either the env var CORSO_SECONDARY_M365_TEST_USER_ID, the
 // corso_test.toml config file or the default value (in that order of priority).
 // The default is a last-attempt fallback that will only work on alcion's
 // testing org.
@@ -30,7 +31,7 @@ func SecondaryM365UserID(t *testing.T) string {
 }
 
 // LoadTestM365UserID returns an userID string representing the m365UserID
-// described by either the env var CORSO_M356_LOAD_TEST_USER_ID, the
+// described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the
 // corso_test.toml config file or the default value (in that order of priority).
 // The default is a last-attempt fallback that will only work on alcion's
 // testing org.
@@ -40,3 +41,19 @@ func LoadTestM365UserID(t *testing.T) string {
 
 	return cfg[TestCfgLoadTestUserID]
 }
+
+// expects cfg value to be a string representing an array like:
+// "['foo@example.com','bar@example.com']"
+func LoadTestM365OrgUsers(t *testing.T) []string {
+	cfg, err := readTestConfig()
+	require.NoError(t, err, "retrieving load test m365 org users from test configuration")
+
+	users := cfg[TestCfgLoadTestOrgUsers]
+	users = strings.TrimPrefix(users, "[")
+	users = strings.TrimSuffix(users, "]")
+	users = strings.ReplaceAll(users, `"`, "")
+	users = strings.ReplaceAll(users, `'`, "")
+	users = strings.ReplaceAll(users, "|", ",")
+
+	return strings.Split(users, ",")
+}

src/pkg/repository/repository_load_test.go

@@ -25,34 +25,8 @@ import (
 	"github.com/alcionai/corso/src/pkg/storage"
 )
 
-func userSet(t *testing.T) []string {
+func orgUserSet(t *testing.T) []string {
-	// avoid adding the following users
+	return tester.LoadTestM365OrgUsers(t)
-	// they are reserved for other purposes
-	// "LeeG@8qzvrj.onmicrosoft.com",
-	// "ntoja@8qzvrj.onmicrosoft.com",
-	return []string{
-		"AdeleV@8qzvrj.onmicrosoft.com",
-		"AlexW@8qzvrj.onmicrosoft.com",
-		"ashmarks@8qzvrj.onmicrosoft.com",
-		"DiegoS@8qzvrj.onmicrosoft.com",
-		"dustina@8qzvrj.onmicrosoft.com",
-		"george.martinez@8qzvrj.onmicrosoft.com",
-		"GradyA@8qzvrj.onmicrosoft.com",
-		"HenriettaM@8qzvrj.onmicrosoft.com",
-		"IsaiahL@8qzvrj.onmicrosoft.com",
-		"JohannaL@8qzvrj.onmicrosoft.com",
-		"JoniS@8qzvrj.onmicrosoft.com",
-		"LidiaH@8qzvrj.onmicrosoft.com",
-		"LynneR@8qzvrj.onmicrosoft.com",
-		"MeganB@8qzvrj.onmicrosoft.com",
-		"MiriamG@8qzvrj.onmicrosoft.com",
-		"NestorW@8qzvrj.onmicrosoft.com",
-		"PattiF@8qzvrj.onmicrosoft.com",
-		"PradeepG@8qzvrj.onmicrosoft.com",
-		"Rfinders@8qzvrj.onmicrosoft.com",
-		"vkarma@8qzvrj.onmicrosoft.com",
-		"greg.sanders@8qzvrj.onmicrosoft.com",
-	}
 }
 
 func singleUserSet(t *testing.T) []string {
@@ -403,7 +377,7 @@ func (suite *RepositoryLoadTestExchangeSuite) SetupSuite() {
 	t := suite.T()
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
-	suite.usersUnderTest = userSet(t)
+	suite.usersUnderTest = orgUserSet(t)
 }
 
 func (suite *RepositoryLoadTestExchangeSuite) TeardownSuite() {
@@ -506,7 +480,7 @@ func (suite *RepositoryLoadTestOneDriveSuite) SetupSuite() {
 	t.Skip("temp issue-902-live")
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
-	suite.usersUnderTest = userSet(t)
+	suite.usersUnderTest = orgUserSet(t)
 }
 
 func (suite *RepositoryLoadTestOneDriveSuite) TeardownSuite() {