add rolling data production to load test (#1339)

## Description

Adds production of 10 exchange items of each type, only for the load_test user, every time the load tests succeed.  Also migrates that user ID out to a secret value stored in github.

## Type of change

- [x] 🤖 Test

## Issue(s)

* #902

## Test Plan

- [x] 💚 E2E

.github/workflows/load_test.yml

@@ -52,6 +52,7 @@ jobs:
           AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+          CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
           CORSO_LOAD_TESTS: true 
         run: |
           set -euo pipefail
@@ -78,6 +79,37 @@ jobs:
           path: src/test_results/*
           if-no-files-found: error
           retention-days: 14
+        
+      # currently, we're required to generate a unique folder for each factory
+      # production.  Whenever possible, this should be reverted to increasing the
+      # item count of a single folder instead, to prevent overproduction of folders
+      # during restore.
+      - name: Set folder destination date
+        run: |
+          echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV
+
+      # generate new entries to roll into the next load test
+      # only runs if the test was successful
+      - name: New Data Creation
+        working-directory: ./src/cmd/factory
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+          AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
+          CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
+        run: |
+          go run . exchange emails \
+          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --destination lt_${{ env.NOW }} \
+          --count 10
+          go run . exchange contacts \
+          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --destination lt_${{ env.NOW }} \
+          --count 10
+          go run . exchange events \
+          --user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
+          --destination lt_${{ env.NOW }} \
+          --count 10
 
       # cleanup folders produced by load test
       - name: Restored Folder Purge
@@ -92,4 +124,3 @@ jobs:
           go run ./cmd/purge/purge.go
           --user '*'
           --prefix ${{ env.DELETE_FOLDER_PREFIX }}
-        

src/internal/tester/config.go

@@ -23,6 +23,7 @@ const (
 	TestCfgAzureTenantID   = "azure_tenantid"
 	TestCfgUserID          = "m365userid"
 	TestCfgSecondaryUserID = "secondarym365userid"
+	TestCfgLoadTestUserID  = "loadttestm365userid"
 	TestCfgAccountProvider = "account_provider"
 )
 
@@ -30,6 +31,7 @@ const (
 const (
 	EnvCorsoM365TestUserID          = "CORSO_M356_TEST_USER_ID"
 	EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M356_TEST_USER_ID"
+	EnvCorsoM365LoadTestUserID      = "CORSO_M356_LOAD_TEST_USER_ID"
 	EnvCorsoTestConfigFilePath      = "CORSO_TEST_CONFIG_FILE"
 )
 
@@ -119,6 +121,13 @@ func readTestConfig() (map[string]string, error) {
 		"lidiah@8qzvrj.onmicrosoft.com",
 		//"lynner@8qzvrj.onmicrosoft.com",
 	)
+	fallbackTo(
+		testEnv,
+		TestCfgLoadTestUserID,
+		os.Getenv(EnvCorsoM365LoadTestUserID),
+		vpr.GetString(TestCfgLoadTestUserID),
+		"leeg@8qzvrj.onmicrosoft.com",
+	)
 
 	testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
 	testConfig = testEnv

src/internal/tester/users.go

@@ -28,3 +28,15 @@ func SecondaryM365UserID(t *testing.T) string {
 
 	return cfg[TestCfgSecondaryUserID]
 }
+
+// LoadTestM365UserID returns an userID string representing the m365UserID
+// described by either the env var CORSO_M356_LOAD_TEST_USER_ID, the
+// corso_test.toml config file or the default value (in that order of priority).
+// The default is a last-attempt fallback that will only work on alcion's
+// testing org.
+func LoadTestM365UserID(t *testing.T) string {
+	cfg, err := readTestConfig()
+	require.NoError(t, err, "retrieving load test m365 user id from test configuration")
+
+	return cfg[TestCfgLoadTestUserID]
+}

src/pkg/repository/repository_load_test.go

@@ -22,37 +22,39 @@ import (
 	"github.com/alcionai/corso/src/pkg/storage"
 )
 
-var alcUsers = []string{
+func userSet(t *testing.T) []string {
-	"AdeleV@8qzvrj.onmicrosoft.com",
-	"AlexW@8qzvrj.onmicrosoft.com",
-	"ashmarks@8qzvrj.onmicrosoft.com",
-	"DiegoS@8qzvrj.onmicrosoft.com",
-	"dustina@8qzvrj.onmicrosoft.com",
-	"george.martinez@8qzvrj.onmicrosoft.com",
-	"GradyA@8qzvrj.onmicrosoft.com",
-	"HenriettaM@8qzvrj.onmicrosoft.com",
-	"IsaiahL@8qzvrj.onmicrosoft.com",
-	"JohannaL@8qzvrj.onmicrosoft.com",
-	"JoniS@8qzvrj.onmicrosoft.com",
-	"LidiaH@8qzvrj.onmicrosoft.com",
-	"LynneR@8qzvrj.onmicrosoft.com",
-	"MeganB@8qzvrj.onmicrosoft.com",
-	"MiriamG@8qzvrj.onmicrosoft.com",
-	"NestorW@8qzvrj.onmicrosoft.com",
-	"PattiF@8qzvrj.onmicrosoft.com",
-	"PradeepG@8qzvrj.onmicrosoft.com",
-	"Rfinders@8qzvrj.onmicrosoft.com",
-	"vkarma@8qzvrj.onmicrosoft.com",
-	"greg.sanders@8qzvrj.onmicrosoft.com",
-
 	// avoid adding the following users
 	// they are reserved for other purposes
-
 	// "LeeG@8qzvrj.onmicrosoft.com",
 	// "ntoja@8qzvrj.onmicrosoft.com",
+	return []string{
+		"AdeleV@8qzvrj.onmicrosoft.com",
+		"AlexW@8qzvrj.onmicrosoft.com",
+		"ashmarks@8qzvrj.onmicrosoft.com",
+		"DiegoS@8qzvrj.onmicrosoft.com",
+		"dustina@8qzvrj.onmicrosoft.com",
+		"george.martinez@8qzvrj.onmicrosoft.com",
+		"GradyA@8qzvrj.onmicrosoft.com",
+		"HenriettaM@8qzvrj.onmicrosoft.com",
+		"IsaiahL@8qzvrj.onmicrosoft.com",
+		"JohannaL@8qzvrj.onmicrosoft.com",
+		"JoniS@8qzvrj.onmicrosoft.com",
+		"LidiaH@8qzvrj.onmicrosoft.com",
+		"LynneR@8qzvrj.onmicrosoft.com",
+		"MeganB@8qzvrj.onmicrosoft.com",
+		"MiriamG@8qzvrj.onmicrosoft.com",
+		"NestorW@8qzvrj.onmicrosoft.com",
+		"PattiF@8qzvrj.onmicrosoft.com",
+		"PradeepG@8qzvrj.onmicrosoft.com",
+		"Rfinders@8qzvrj.onmicrosoft.com",
+		"vkarma@8qzvrj.onmicrosoft.com",
+		"greg.sanders@8qzvrj.onmicrosoft.com",
+	}
 }
 
-var largeDatasetUser = []string{"LeeG@8qzvrj.onmicrosoft.com"}
+func singleUserSet(t *testing.T) []string {
+	return []string{tester.LoadTestM365UserID(t)}
+}
 
 func initM365Repo(t *testing.T) (
 	context.Context, repository.Repository, account.Account, storage.Storage,
@@ -88,12 +90,12 @@ func runLoadTest(
 	t *testing.T,
 	ctx context.Context,
 	r repository.Repository,
-	service string,
+	prefix, service string,
 	usersUnderTest []string,
 	bupSel, restSel selectors.Selector,
 ) {
 	//revive:enable:context-as-argument
-	t.Run("load_test_main", func(t *testing.T) {
+	t.Run(prefix+"_load_test_main", func(t *testing.T) {
 		b, err := r.NewBackup(ctx, bupSel)
 		require.NoError(t, err)
 
@@ -345,10 +347,11 @@ func normalizeCategorySet(t *testing.T, cats map[string]struct{}) []string {
 
 type RepositoryLoadTestExchangeSuite struct {
 	suite.Suite
-	ctx  context.Context
+	ctx            context.Context
-	repo repository.Repository
+	repo           repository.Repository
-	acct account.Account
+	acct           account.Account
-	st   storage.Storage
+	st             storage.Storage
+	usersUnderTest []string
 }
 
 func TestRepositoryLoadTestExchangeSuite(t *testing.T) {
@@ -363,6 +366,7 @@ func (suite *RepositoryLoadTestExchangeSuite) SetupSuite() {
 	t := suite.T()
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.usersUnderTest = userSet(t)
 }
 
 func (suite *RepositoryLoadTestExchangeSuite) TeardownSuite() {
@@ -373,20 +377,18 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	usersUnderTest := alcUsers
-
 	bsel := selectors.NewExchangeBackup()
-	bsel.Include(bsel.MailFolders(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.MailFolders(suite.usersUnderTest, selectors.Any()))
-	bsel.Include(bsel.ContactFolders(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.ContactFolders(suite.usersUnderTest, selectors.Any()))
-	bsel.Include(bsel.EventCalendars(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.EventCalendars(suite.usersUnderTest, selectors.Any()))
 	sel := bsel.Selector
 
 	runLoadTest(
 		suite.T(),
 		ctx,
 		suite.repo,
-		"exchange",
+		"all_users", "exchange",
-		usersUnderTest,
+		suite.usersUnderTest,
 		sel, sel, // same selection for backup and restore
 	)
 }
@@ -395,10 +397,11 @@ func (suite *RepositoryLoadTestExchangeSuite) TestExchange() {
 
 type RepositoryIndividualLoadTestExchangeSuite struct {
 	suite.Suite
-	ctx  context.Context
+	ctx            context.Context
-	repo repository.Repository
+	repo           repository.Repository
-	acct account.Account
+	acct           account.Account
-	st   storage.Storage
+	st             storage.Storage
+	usersUnderTest []string
 }
 
 func TestRepositoryIndividualLoadTestExchangeSuite(t *testing.T) {
@@ -413,6 +416,7 @@ func (suite *RepositoryIndividualLoadTestExchangeSuite) SetupSuite() {
 	t := suite.T()
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.usersUnderTest = singleUserSet(t)
 }
 
 func (suite *RepositoryIndividualLoadTestExchangeSuite) TeardownSuite() {
@@ -423,20 +427,18 @@ func (suite *RepositoryIndividualLoadTestExchangeSuite) TestExchange() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	usersUnderTest := largeDatasetUser
-
 	bsel := selectors.NewExchangeBackup()
-	bsel.Include(bsel.MailFolders(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.MailFolders(suite.usersUnderTest, selectors.Any()))
-	bsel.Include(bsel.ContactFolders(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.ContactFolders(suite.usersUnderTest, selectors.Any()))
-	bsel.Include(bsel.EventCalendars(usersUnderTest, selectors.Any()))
+	bsel.Include(bsel.EventCalendars(suite.usersUnderTest, selectors.Any()))
 	sel := bsel.Selector
 
 	runLoadTest(
 		suite.T(),
 		ctx,
 		suite.repo,
-		"exchange",
+		"single_user", "exchange",
-		usersUnderTest,
+		suite.usersUnderTest,
 		sel, sel, // same selection for backup and restore
 	)
 }
@@ -447,10 +449,11 @@ func (suite *RepositoryIndividualLoadTestExchangeSuite) TestExchange() {
 
 type RepositoryLoadTestOneDriveSuite struct {
 	suite.Suite
-	ctx  context.Context
+	ctx            context.Context
-	repo repository.Repository
+	repo           repository.Repository
-	acct account.Account
+	acct           account.Account
-	st   storage.Storage
+	st             storage.Storage
+	usersUnderTest []string
 }
 
 func TestRepositoryLoadTestOneDriveSuite(t *testing.T) {
@@ -466,6 +469,7 @@ func (suite *RepositoryLoadTestOneDriveSuite) SetupSuite() {
 	t.Skip("temp issue-902-live")
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.usersUnderTest = userSet(t)
 }
 
 func (suite *RepositoryLoadTestOneDriveSuite) TeardownSuite() {
@@ -476,28 +480,27 @@ func (suite *RepositoryLoadTestOneDriveSuite) TestOneDrive() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	usersUnderTest := alcUsers
-
 	bsel := selectors.NewOneDriveBackup()
-	bsel.Include(bsel.Users(usersUnderTest))
+	bsel.Include(bsel.Users(suite.usersUnderTest))
 	sel := bsel.Selector
 
 	runLoadTest(
 		suite.T(),
 		ctx,
 		suite.repo,
-		"one_drive",
+		"all_users", "one_drive",
-		usersUnderTest,
+		suite.usersUnderTest,
 		sel, sel, // same selection for backup and restore
 	)
 }
 
 type RepositoryIndividualLoadTestOneDriveSuite struct {
 	suite.Suite
-	ctx  context.Context
+	ctx            context.Context
-	repo repository.Repository
+	repo           repository.Repository
-	acct account.Account
+	acct           account.Account
-	st   storage.Storage
+	st             storage.Storage
+	usersUnderTest []string
 }
 
 func TestRepositoryIndividualLoadTestOneDriveSuite(t *testing.T) {
@@ -513,6 +516,7 @@ func (suite *RepositoryIndividualLoadTestOneDriveSuite) SetupSuite() {
 	t.Skip("temp issue-902-live")
 	t.Parallel()
 	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.usersUnderTest = singleUserSet(t)
 }
 
 func (suite *RepositoryIndividualLoadTestOneDriveSuite) TeardownSuite() {
@@ -523,18 +527,16 @@ func (suite *RepositoryIndividualLoadTestOneDriveSuite) TestOneDrive() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	usersUnderTest := largeDatasetUser
-
 	bsel := selectors.NewOneDriveBackup()
-	bsel.Include(bsel.Users(usersUnderTest))
+	bsel.Include(bsel.Users(suite.usersUnderTest))
 	sel := bsel.Selector
 
 	runLoadTest(
 		suite.T(),
 		ctx,
 		suite.repo,
-		"one_drive",
+		"single_user", "one_drive",
-		usersUnderTest,
+		suite.usersUnderTest,
 		sel, sel, // same selection for backup and restore
 	)
 }