separate matrix and job for site cleanup (#3095)

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🐛 Bugfix

#### Test Plan

- [x] 💚 E2E

.github/actions/purge-m365-data/action.yml

@@ -38,6 +38,11 @@ inputs:
 runs:
   using: composite
   steps:
+
+    ################################################################################################################
+    # Exchange
+    #
+
     - name: Run the Exchange purge scripts for user
       if: ${{ inputs.user != '' }}
       shell: pwsh
@@ -49,6 +54,20 @@ runs:
       run: |
         ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
 
+    - name: Reset retention for all mailboxes to 0
+      if: ${{ inputs.user == '' }}
+      shell: pwsh
+      working-directory: ./src/cmd/purge/scripts
+      env:
+        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
+        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+      run: |
+        ./exchangeRetention.ps1
+
+    ################################################################################################################
+    # OneDrive
+    #
+    
     - name: Run the OneDrive purge scripts for user
       if: ${{ inputs.user != '' }}
       shell: pwsh
@@ -59,8 +78,12 @@ runs:
       run: |
         ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }}
 
+    ################################################################################################################
+    # Sharepoint
+    #
+
     - name: Run SharePoint purge script
-      if: ${{ inputs.user == '' }}
+      if: ${{ inputs.site != '' }}
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
       env:
@@ -68,13 +91,3 @@ runs:
         M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
       run: |
         ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
-
-    - name: Reset retention for all mailboxes to 0
-      if: ${{ inputs.user == '' }}
-      shell: pwsh
-      working-directory: ./src/cmd/purge/scripts
-      env:
-        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
-        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
-      run: |
-        ./exchangeRetention.ps1

.github/workflows/ci_test_cleanup.yml

@@ -12,6 +12,36 @@ jobs:
     strategy:
       matrix:
         user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # sets the maximum time to now-30m.
+      # CI test have a 10 minute timeout.
+      # At 20 minutes ago, we should be safe from conflicts.
+      # The additional 10 minutes is just to be good citizens.
+      - name: Set purge boundary
+        run: |
+          echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
+
+      - name: Purge CI-Produced Folders for Users
+        uses: ./.github/actions/purge-m365-data
+        with:
+          user: ${{ secrets[matrix.user] }}
+          folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }}
+          older-than: ${{ env.HALF_HOUR_AGO }}
+          azure-client-id: ${{ secrets.CLIENT_ID }}
+          azure-client-secret: ${{ secrets.CLIENT_SECRET }}
+          azure-tenant-id: ${{ secrets.TENANT_ID }}
+          m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
+          m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}
+
+  Test-Site-Data-Cleanup:
+    environment: Testing
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
         site: [ CORSO_M365_TEST_SITE_URL, EXT_SDK_TEST_SITE_URL ] 
 
     steps:
@@ -25,10 +55,9 @@ jobs:
         run: |
           echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
 
-      - name: Purge CI-Produced Folders
+      - name: Purge CI-Produced Folders for Sites
-        uses: ./.github/actions/purge-m365-user-data
+        uses: ./.github/actions/purge-m365-data
         with:
-          user: ${{ secrets[matrix.user] }}
           site: ${{ secrets[matrix.site] }}
           folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} 
           libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} 

.github/workflows/load_test.yml

@@ -136,19 +136,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        user: ${{ fromJson(needs.setup.outputs.matrix).user }}
+        user: [ CORSO_M365_LOAD_TEST_USER_ID, '' ]
-        folder: [Corso_Restore_, '']
     steps:
       - uses: actions/checkout@v3
       - name: Set folder boundary datetime
         run: |
           echo "NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
       - name: Purge Load-Test-Produced Folders
-        uses: ./.github/actions/purge-m365-user-data
+        uses: ./.github/actions/purge-m365-data
         with:
-          older-than: ${{ env.NOW }}
+          user: ${{ secrets[matrix.user] }}
-          folder-prefix: ${{ matrix.folder }}
+          folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }}
-          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          older-than: ${{ env.HALF_HOUR_AGO }}
-          azure-client-secret: ${{ secrets.AZURE_CLIENT_SECRET }}
+          azure-client-id: ${{ secrets.CLIENT_ID }}
-          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          azure-client-secret: ${{ secrets.CLIENT_SECRET }}
-          user: ${{ matrix.user }}
+          azure-tenant-id: ${{ secrets.TENANT_ID }}
+          m365-admin-user: ${{ secrets.M365_TENANT_ADMIN_USER }}
+          m365-admin-password: ${{ secrets.M365_TENANT_ADMIN_PASSWORD }}

.github/workflows/sanity-test.yaml

@@ -122,10 +122,10 @@ jobs:
           ./corso backup create exchange \
           --no-stats \
           --user "${CORSO_M365_TEST_USER_ID}" \
-           --hide-progress \
+          --hide-progress \
-           --data 'email' \
+          --data 'email' \
-            --json \
+          --json \
-            2>&1 | tee $TEST_RESULT/backup_exchange.txt 
+          2>&1 | tee $TEST_RESULT/backup_exchange.txt 
 
           resultjson=$(sed -e '1,/Completed Backups/d' $TEST_RESULT/backup_exchange.txt )