From d1f4cd872fc318cac8c4bc3dbd5f50b1cbb918fb Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 11 Apr 2023 16:38:00 -0600 Subject: [PATCH] separate matrix and job for site cleanup (#3095) #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :bug: Bugfix #### Test Plan - [x] :green_heart: E2E --- .../action.yml | 35 +++++++++++++------ .github/workflows/ci_test_cleanup.yml | 35 +++++++++++++++++-- .github/workflows/load_test.yml | 19 +++++----- .github/workflows/sanity-test.yaml | 8 ++--- 4 files changed, 70 insertions(+), 27 deletions(-) rename .github/actions/{purge-m365-user-data => purge-m365-data}/action.yml (88%) diff --git a/.github/actions/purge-m365-user-data/action.yml b/.github/actions/purge-m365-data/action.yml similarity index 88% rename from .github/actions/purge-m365-user-data/action.yml rename to .github/actions/purge-m365-data/action.yml index fa246ce82..d7681e55b 100644 --- a/.github/actions/purge-m365-user-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -38,6 +38,11 @@ inputs: runs: using: composite steps: + + ################################################################################################################ + # Exchange + # + - name: Run the Exchange purge scripts for user if: ${{ inputs.user != '' }} shell: pwsh @@ -48,6 +53,20 @@ runs: AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} run: | ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + + - name: Reset retention for all mailboxes to 0 + if: ${{ inputs.user == '' }} + shell: pwsh + working-directory: ./src/cmd/purge/scripts + env: + M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} + M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} + run: | + ./exchangeRetention.ps1 + + ################################################################################################################ + # OneDrive + # - name: Run the OneDrive purge scripts for user if: ${{ inputs.user != '' }} @@ -59,8 +78,12 @@ runs: run: | ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + ################################################################################################################ + # Sharepoint + # + - name: Run SharePoint purge script - if: ${{ inputs.user == '' }} + if: ${{ inputs.site != '' }} shell: pwsh working-directory: ./src/cmd/purge/scripts env: @@ -68,13 +91,3 @@ runs: M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} - - - name: Reset retention for all mailboxes to 0 - if: ${{ inputs.user == '' }} - shell: pwsh - working-directory: ./src/cmd/purge/scripts - env: - M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} - M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} - run: | - ./exchangeRetention.ps1 diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index b7aae8aae..c82fc343f 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -12,7 +12,6 @@ jobs: strategy: matrix: user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ] - site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] steps: - uses: actions/checkout@v3 @@ -25,10 +24,40 @@ jobs: run: | echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - - name: Purge CI-Produced Folders - uses: ./.github/actions/purge-m365-user-data + - name: Purge CI-Produced Folders for Users + uses: ./.github/actions/purge-m365-data with: user: ${{ secrets[matrix.user] }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + older-than: ${{ env.HALF_HOUR_AGO }} + azure-client-id: ${{ secrets.CLIENT_ID }} + azure-client-secret: ${{ secrets.CLIENT_SECRET }} + azure-tenant-id: ${{ secrets.TENANT_ID }} + m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} + m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} + + Test-Site-Data-Cleanup: + environment: Testing + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] + + steps: + - uses: actions/checkout@v3 + + # sets the maximum time to now-30m. + # CI test have a 10 minute timeout. + # At 20 minutes ago, we should be safe from conflicts. + # The additional 10 minutes is just to be good citizens. + - name: Set purge boundary + run: | + echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV + + - name: Purge CI-Produced Folders for Sites + uses: ./.github/actions/purge-m365-data + with: site: ${{ secrets[matrix.site] }} folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index c75b29db2..ec428c42a 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -136,19 +136,20 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - user: ${{ fromJson(needs.setup.outputs.matrix).user }} - folder: [Corso_Restore_, ''] + user: [ CORSO_M365_LOAD_TEST_USER_ID, '' ] steps: - uses: actions/checkout@v3 - name: Set folder boundary datetime run: | echo "NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - name: Purge Load-Test-Produced Folders - uses: ./.github/actions/purge-m365-user-data + uses: ./.github/actions/purge-m365-data with: - older-than: ${{ env.NOW }} - folder-prefix: ${{ matrix.folder }} - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-client-secret: ${{ secrets.AZURE_CLIENT_SECRET }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - user: ${{ matrix.user }} + user: ${{ secrets[matrix.user] }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + older-than: ${{ env.HALF_HOUR_AGO }} + azure-client-id: ${{ secrets.CLIENT_ID }} + azure-client-secret: ${{ secrets.CLIENT_SECRET }} + azure-tenant-id: ${{ secrets.TENANT_ID }} + m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }} + m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }} diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 4def40f29..dfefd8c04 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -122,10 +122,10 @@ jobs: ./corso backup create exchange \ --no-stats \ --user "${CORSO_M365_TEST_USER_ID}" \ - --hide-progress \ - --data 'email' \ - --json \ - 2>&1 | tee $TEST_RESULT/backup_exchange.txt + --hide-progress \ + --data 'email' \ + --json \ + 2>&1 | tee $TEST_RESULT/backup_exchange.txt resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_exchange.txt )