larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace golang purge script with powershell script (#1407)

Browse Source 
## Description

Three primary changes:
1. move the list of load test users into a secret in the github test domain.
2. purge all data that was generated for users in the load test
3. fix all `m356` typos.

## Type of change

- [x] 🤖 Test
- [x] 💻 CI/Deployment

## Issue(s)

* #1266

## Test Plan

- [x] 💪 Manual
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-11-04 09:41:36 -06:00 committed by GitHub
parent c4632cb152
commit 2d66dcbd7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 100 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/purge-m365-user-data/action.yml vendored
View File

@ -18,7 +18,7 @@ inputs:
user:
description: User whose data is to be purged.
folder-prefix:
description: Name of the folder to be purged. If false, will purge the set of static, well known folders instead.
description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead.
older-than:
description: Minimum-age of folders to be deleted.
azure-client-id:

2
.github/workflows/ci.yml vendored
View File

@ -174,7 +174,7 @@ jobs:
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
CORSO_CI_TESTS: true
CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }}
CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
run: |
set -euo pipefail

4
.github/workflows/ci_test_cleanup.yml vendored
View File

@ -12,12 +12,10 @@ jobs:
strategy:
matrix:
folder: [Corso_Restore_, TestRestore, '']
user: [CORSO_M356_TEST_USER_ID, CORSO_SECONDARY_M356_TEST_USER_ID]
user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID]
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
# sets the maximimum time to now-30m.
# CI test have a 10 minute timeout.

78
.github/workflows/load_test.yml vendored
View File

@ -3,8 +3,8 @@ on:
schedule:
# every day at 01:59 (01:59am) UTC
# - cron: "59 1 * * *"
# temp, for testing: every 4 hours
- cron: "0 */4 * * *"
# temp, for testing: every 6 hours
- cron: "0 */6 * * *"
permissions:
# required to retrieve AWS credentials
@ -59,7 +59,8 @@ jobs:
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
CORSO_LOAD_TESTS: true
run: |
set -euo pipefail
@ -77,14 +78,6 @@ jobs:
./pkg/repository/repository_load_test.go \
2>&1 | tee ./test_results/goloadtest.log | gotestfmt -hide successful-tests
# currently, we're required to generate a unique folder for each factory
# production. Whenever possible, this should be reverted to increasing the
# item count of a single folder instead, to prevent overproduction of folders
# during restore.
- name: Set folder destination date
run: |
echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV
# generate new entries to roll into the next load test
# only runs if the test was successful
- name: New Data Creation
@ -93,35 +86,21 @@ jobs:
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
CORSO_M356_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M356_LOAD_TEST_USER_ID }}
CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
run: |
go run . exchange emails \
--user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
--user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
--destination lt_${{ env.NOW }} \
--count 10
go run . exchange contacts \
--user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
--user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
--destination lt_${{ env.NOW }} \
--count 10
go run . exchange events \
--user ${{ env.CORSO_M356_LOAD_TEST_USER_ID }} \
--user ${{ env.CORSO_M365_LOAD_TEST_USER_ID }} \
--destination lt_${{ env.NOW }} \
--count 10
# cleanup folders produced by load test
- name: Restored Folder Purge
if: always()
working-directory: ./src
env:
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
DELETE_FOLDER_PREFIX: "Corso_Restore_"
run: >
go run ./cmd/purge/purge.go
--user '*'
--prefix ${{ env.DELETE_FOLDER_PREFIX }}
- name: Put Down the Daemons Arisen
if: always()
run: docker kill otel-daemon
@ -135,3 +114,44 @@ jobs:
path: src/test_results/*
if-no-files-found: error
retention-days: 14
setup:
environment: Testing
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.build.outputs.matrix }}
steps:
- uses: actions/checkout@v3
- id: build
run: |
u=$(echo "${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}" | sed 's/\[/["/g' | sed 's/\]/"]/g' | sed 's/|/","/g')
marr=$(cat <<-end
{ "user": $u }
end
)
m=$(echo $marr | jq -c .)
echo "matrix=$m" >> $GITHUB_OUTPUT
purge-load-test-user-data:
needs: [setup, Load-Tests]
if: always()
environment: Testing
runs-on: ubuntu-latest
strategy:
matrix:
user: ${{ fromJson(needs.setup.outputs.matrix).user }}
folder: [Corso_Restore_,'']
steps:
- uses: actions/checkout@v3
- name: Set folder boundary datetime
run: |
echo "NOW=$(date -u +"%Y-%m-%d_%H-%M-%S")" >> $GITHUB_ENV
- name: Purge Load-Test-Produced Folders
uses: ./.github/actions/purge-m365-user-data
with:
older-than: ${{ env.NOW }}
folder-prefix: ${{ matrix.folder }}
azure-client-id: ${{ secrets.CLIENT_ID }}
azure-client-secret: ${{ secrets.CLIENT_SECRET }}
azure-tenant-id: ${{ secrets.TENANT_ID }}
user: ${{ matrix.user }}

4
docs/docs/developers/testing.md
View File

@ -51,8 +51,8 @@ The complete list of environment constants is available at
## Advanced options
- To override the M365 user for tests, use `CORSO_M356_TEST_USER_ID`
- To override the M365 user for tests, use `CORSO_M365_TEST_USER_ID`
```bash
export CORSO_M356_TEST_USER_ID="..."
export CORSO_M365_TEST_USER_ID="..."
```

2
src/.golangci.yml
View File

@ -28,6 +28,8 @@ linters-settings:
# Don't allow use of path as it hardcodes separator to `/`.
# Use filepath instead.
- '\bpath\.(Ext|Base|Dir|Join)'
# Don't allow the typo m356 to be used in place of m365.
- '[Mm]356'
lll:
line-length: 120
revive:

2
src/cli/repo/s3.go
View File

@ -94,7 +94,7 @@ func s3InitCmd() *cobra.Command {
return &cobra.Command{
Use: s3ProviderCommand,
Short: "Initialize a S3 repository",
Long: `Bootstraps a new S3 repository and connects it to your m356 account.`,
Long: `Bootstraps a new S3 repository and connects it to your m365 account.`,
RunE: initS3Cmd,
Args: cobra.NoArgs,
Example: s3ProviderCommandInitExamples,

2
src/internal/connector/graph_connector.go
View File

@ -75,7 +75,7 @@ func (gs graphService) ErrPolicy() bool {
func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnector, error) {
m365, err := acct.M365Config()
if err != nil {
return nil, errors.Wrap(err, "retrieving m356 account configuration")
return nil, errors.Wrap(err, "retrieving m365 account configuration")
}
gc := GraphConnector{

25
src/internal/tester/config.go
View File

@ -20,18 +20,20 @@ const (
TestCfgStorageProvider = "provider"
// M365 config
TestCfgAzureTenantID = "azure_tenantid"
TestCfgUserID = "m365userid"
TestCfgSecondaryUserID = "secondarym365userid"
TestCfgLoadTestUserID = "loadttestm365userid"
TestCfgAccountProvider = "account_provider"
TestCfgAzureTenantID = "azure_tenantid"
TestCfgUserID = "m365userid"
TestCfgSecondaryUserID = "secondarym365userid"
TestCfgLoadTestUserID = "loadtestm365userid"
TestCfgLoadTestOrgUsers = "loadtestm365orgusers"
TestCfgAccountProvider = "account_provider"
)
// test specific env vars
const (
EnvCorsoM365TestUserID = "CORSO_M356_TEST_USER_ID"
EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M356_TEST_USER_ID"
EnvCorsoM365LoadTestUserID = "CORSO_M356_LOAD_TEST_USER_ID"
EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID"
EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID"
EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID"
EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS"
EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE"
)
@ -128,6 +130,13 @@ func readTestConfig() (map[string]string, error) {
vpr.GetString(TestCfgLoadTestUserID),
"leeg@8qzvrj.onmicrosoft.com",
)
fallbackTo(
testEnv,
TestCfgLoadTestOrgUsers,
os.Getenv(EnvCorsoM365LoadTestOrgUsers),
vpr.GetString(TestCfgLoadTestOrgUsers),
"lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com",
)
testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
testConfig = testEnv

23
src/internal/tester/users.go
View File

@ -1,13 +1,14 @@
package tester
import (
"strings"
"testing"
"github.com/stretchr/testify/require"
)
// M365UserID returns an userID string representing the m365UserID described
// by either the env var CORSO_M356_TEST_USER_ID, the corso_test.toml config
// by either the env var CORSO_M365_TEST_USER_ID, the corso_test.toml config
// file or the default value (in that order of priority). The default is a
// last-attempt fallback that will only work on alcion's testing org.
func M365UserID(t *testing.T) string {
@ -18,7 +19,7 @@ func M365UserID(t *testing.T) string {
}
// SecondaryM365UserID returns an userID string representing the m365UserID
// described by either the env var CORSO_SECONDARY_M356_TEST_USER_ID, the
// described by either the env var CORSO_SECONDARY_M365_TEST_USER_ID, the
// corso_test.toml config file or the default value (in that order of priority).
// The default is a last-attempt fallback that will only work on alcion's
// testing org.
@ -30,7 +31,7 @@ func SecondaryM365UserID(t *testing.T) string {
}
// LoadTestM365UserID returns an userID string representing the m365UserID
// described by either the env var CORSO_M356_LOAD_TEST_USER_ID, the
// described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the
// corso_test.toml config file or the default value (in that order of priority).
// The default is a last-attempt fallback that will only work on alcion's
// testing org.
@ -40,3 +41,19 @@ func LoadTestM365UserID(t *testing.T) string {
return cfg[TestCfgLoadTestUserID]
}
// expects cfg value to be a string representing an array like:
// "['foo@example.com','bar@example.com']"
func LoadTestM365OrgUsers(t *testing.T) []string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving load test m365 org users from test configuration")
users := cfg[TestCfgLoadTestOrgUsers]
users = strings.TrimPrefix(users, "[")
users = strings.TrimSuffix(users, "]")
users = strings.ReplaceAll(users, `"`, "")
users = strings.ReplaceAll(users, `'`, "")
users = strings.ReplaceAll(users, "|", ",")
return strings.Split(users, ",")
}

34
src/pkg/repository/repository_load_test.go
View File

@ -25,34 +25,8 @@ import (
"github.com/alcionai/corso/src/pkg/storage"
)
func userSet(t *testing.T) []string {
// avoid adding the following users
// they are reserved for other purposes
// "LeeG@8qzvrj.onmicrosoft.com",
// "ntoja@8qzvrj.onmicrosoft.com",
return []string{
"AdeleV@8qzvrj.onmicrosoft.com",
"AlexW@8qzvrj.onmicrosoft.com",
"ashmarks@8qzvrj.onmicrosoft.com",
"DiegoS@8qzvrj.onmicrosoft.com",
"dustina@8qzvrj.onmicrosoft.com",
"george.martinez@8qzvrj.onmicrosoft.com",
"GradyA@8qzvrj.onmicrosoft.com",
"HenriettaM@8qzvrj.onmicrosoft.com",
"IsaiahL@8qzvrj.onmicrosoft.com",
"JohannaL@8qzvrj.onmicrosoft.com",
"JoniS@8qzvrj.onmicrosoft.com",
"LidiaH@8qzvrj.onmicrosoft.com",
"LynneR@8qzvrj.onmicrosoft.com",
"MeganB@8qzvrj.onmicrosoft.com",
"MiriamG@8qzvrj.onmicrosoft.com",
"NestorW@8qzvrj.onmicrosoft.com",
"PattiF@8qzvrj.onmicrosoft.com",
"PradeepG@8qzvrj.onmicrosoft.com",
"Rfinders@8qzvrj.onmicrosoft.com",
"vkarma@8qzvrj.onmicrosoft.com",
"greg.sanders@8qzvrj.onmicrosoft.com",
}
func orgUserSet(t *testing.T) []string {
return tester.LoadTestM365OrgUsers(t)
}
func singleUserSet(t *testing.T) []string {
@ -403,7 +377,7 @@ func (suite *RepositoryLoadTestExchangeSuite) SetupSuite() {
t := suite.T()
t.Parallel()
suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
suite.usersUnderTest = userSet(t)
suite.usersUnderTest = orgUserSet(t)
}
func (suite *RepositoryLoadTestExchangeSuite) TeardownSuite() {
@ -506,7 +480,7 @@ func (suite *RepositoryLoadTestOneDriveSuite) SetupSuite() {
t.Skip("temp issue-902-live")
t.Parallel()
suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
suite.usersUnderTest = userSet(t)
suite.usersUnderTest = orgUserSet(t)
}
func (suite *RepositoryLoadTestOneDriveSuite) TeardownSuite() {