Improved CI cleanup script (#2851)

Improved this significantly based on some learnings from trying to get this in better shape
* Reduce folder enumerations significantly
* Address issue when cleanup gets backlogged and cannot enumerate all items to clean in a single job
* Remove the need to launch one job per prefix or folder name
* The only way to clean /Audits reliably seems to be emptying (with hard delete) of /Revoverable Items (parent of Audits)
* Eliminated need to control concurrency and overhead of job spin up
* Attempt to set explicit network timeout for SOAP calls to prevent long hangs being observed

NOTE: This change affect both cleanup script and the corresponding GHA workflow
---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [x] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E

.github/actions/purge-m365-user-data/action.yml

@@ -9,10 +9,9 @@ name: Purge M365 User Data
 #
 # The script focuses on the cleaning up the following:
 # * All folders, descending from the exchange root, of a given prefix.
-# * All folders in PersonMetadata 
+# * All folders in PersonMetadata
 # * All already soft-deleted items
-# * All recoverable items in Audits
+# * All folders under recoverable items
-# * All recoverable items in Purges
 
 inputs:
   user:
@@ -35,20 +34,19 @@ inputs:
 runs:
   using: composite
   steps:
-
+    - name: Run the all purge scripts for user
-    - name: Run the folder-matrix purge script set
+      if: ${{ inputs.user != '' }}
-      if: ${{ inputs.folder-prefix != '' }}
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
-      env: 
+      env:
         AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
         AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
         AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
       run: |
-        ./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderPrefixPurge ${{ inputs.folder-prefix }} -FolderBeforePurge ${{ inputs.older-than }}
+        ./foldersAndItems.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata --FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
 
     - name: Reset retention for all mailboxes to 0
-      if: ${{ inputs.folder-prefix == '' && inputs.user == '' }}
+      if: ${{ inputs.user == '' }}
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
       env:
@@ -57,25 +55,9 @@ runs:
       run: |
         ./setRetention.ps1
 
-    - name: Run the static purge script set
-      if: ${{ inputs.folder-prefix == '' && inputs.user != '' }}
-      shell: pwsh
-      working-directory: ./src/cmd/purge/scripts
-      env:
-        AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
-        AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
-        AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
-      # powershell doesn't like multiline commands, each of these must be on a single line
-      run: |
-        ./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderNamePurge PersonMetadata 
-        ./foldersAndItems.ps1 -WellKnownRoot deleteditems -User ${{ inputs.user }} 
-        ./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Audits 
-        ./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Purges
-        ./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Deletions
-
     - name: Run the old purge script to clear out onedrive buildup
       working-directory: ./src
-      if: ${{ inputs.folder-prefix != '' }}
+      if: ${{ inputs.folder-prefix != '' && inputs.user != ''}}
       shell: sh
       env:
         AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}

.github/workflows/ci_test_cleanup.yml

@@ -10,14 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     continue-on-error: true
     strategy:
-      # jobs are expanded per user. Try to run 2 jobs per active user at a time
-      max-parallel: 6
       matrix:
-        folder: [Corso_Restore_, TestRestore, testfolder, incrementals_ci_, Alcion_Restore_, '']
+        folder: 'Corso_Restore_, TestRestore, testfolder, incrementals_ci_, Alcion_Restore_'
-        user: [CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID]
+        user: [ CORSO_M365_TEST_USER_ID, CORSO_SECONDARY_M365_TEST_USER_ID, EXT_SDK_TEST_USER_ID, '' ]
-        include:
-          - folder: ""
-            user: ""
 
     steps:
       - uses: actions/checkout@v3

src/cmd/purge/scripts/foldersAndItems.ps1

@@ -4,17 +4,14 @@ Param (
     [Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")]
     [String]$User,
 
-    [Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")]
+    [Parameter(Mandatory = $True, HelpMessage = "Purge folders or contacts before this date time (UTC)")]
-    [String]$WellKnownRoot = "deleteditems",
+    [datetime]$PurgeBeforeTimestamp,
 
-    [Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")]
+    [Parameter(Mandatory = $True, HelpMessage = "Name of specific folder to purge under root")]
-    [datetime]$FolderBeforePurge,
+    [String[]]$FolderNamePurgeList,
 
-    [Parameter(Mandatory = $False, HelpMessage = "Name of specific folder to purge under root")]
+    [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")]
-    [String]$FolderNamePurge,
+    [String[]]$FolderPrefixPurgeList,
-
-    [Parameter(Mandatory = $False, HelpMessage = "Purge folders with this prefix")]
-    [String]$FolderPrefixPurge,
 
     [Parameter(Mandatory = $False, HelpMessage = "Azure TenantId")]
     [String]$TenantId = $ENV:AZURE_TENANT_ID,
@@ -26,6 +23,10 @@ Param (
     [String]$ClientSecret = $ENV:AZURE_CLIENT_SECRET
 )
 
+Set-StrictMode -Version 2.0
+# Attempt to set network timeout to 10min
+[System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000
+
 function Get-AccessToken {
     [CmdletBinding()]
     Param()
@@ -34,11 +35,11 @@ function Get-AccessToken {
         Write-Host "`nNeed to specify TenantId, ClientId, and ClientSecret as parameters or ENVs"
     }
 
-    $body=@{
+    $body = @{
-        client_id=$ClientId
+        client_id     = $ClientId
-        client_secret=$ClientSecret
+        client_secret = $ClientSecret
-        scope="https://outlook.office365.com/.default"
+        scope         = "https://outlook.office365.com/.default"
-        grant_type="client_credentials"
+        grant_type    = "client_credentials"
     }
     
     $res = Invoke-WebRequest -Uri "https://login.microsoftonline.com/$TenantId/oauth2/v2.0/token" -ContentType "application/x-www-form-urlencoded" -Body $body -Method Post
@@ -102,20 +103,62 @@ function Invoke-SOAPRequest {
     return $xmlResponse
 }
 
-function Remove-Folder {
+function IsNameMatch {
-    [CmdletBinding(SupportsShouldProcess)]
     Param(
-        [Parameter(Mandatory = $True, HelpMessage = "OAuth token to connect to Exchange Online.")]
+        [Parameter(Mandatory = $True, HelpMessage = "Folder name to evaluate for match against a list of targets")]
-        [Securestring]$Token,
+        [string]$FolderName,
 
-        [Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")]
+        [Parameter(Mandatory = $True, HelpMessage = "Folder names to evaluate for match")]
-        [String]$User,
+        [string[]]$FolderNamePurgeList = @()
+    )
 
-        [Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")]
+    return ($FolderName -in $FolderNamePurgeList)
-        [String]$WellKnownRoot = "deleteditems"
+}
-    )    
 
-    # SOAP message for getting the folder id
+function IsPrefixAndAgeMatch {
+    Param(
+        [Parameter(Mandatory = $True, HelpMessage = "Folder name to evaluate for match against a list of targets")]
+        [string]$FolderName,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Folder creation times")]
+        [string]$FolderCreateTime,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Folder name prefixes to evaluate for match")]
+        [string[]]$FolderPrefixPurgeList,
+
+        [Parameter(Mandatory = $TRUE, HelpMessage = "Purge folders before this date time (UTC)")]
+        [datetime]$PurgeBeforeTimestamp
+    )
+
+    if ($PurgeBeforeTimestamp -gt $folderCreateTime ) {
+        foreach ($prefix in $FolderPrefixPurgeList) {
+            if ($FolderName -like "$prefix*") {
+                return $true
+            }
+        }
+    }
+
+    return $false
+}
+
+function Get-FoldersToPurge {
+    Param(
+        [Parameter(Mandatory = $True, HelpMessage = "Folder under which to look for items matching removal criteria")]
+        [String]$WellKnownRoot,
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")]
+        [datetime]$PurgeBeforeTimestamp,
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders with these names")]
+        [string[]]$FolderNamePurgeList = @(),
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders with these prefixes")]
+        [string[]]$FolderPrefixPurgeList = @()
+    )
+
+    $foldersToDelete = @()
+    
+    # SOAP message for getting the folders
     $body = @"
 <FindFolder Traversal="Deep" xmlns="http://schemas.microsoft.com/exchange/services/2006/messages">
     <FolderShape>
@@ -130,78 +173,360 @@ function Remove-Folder {
 </FindFolder>
 "@
 
-    Write-Host "`nLooking for folders under well-known folder: $WellKnownRoot & matching folder: $FolderNamePurge$FolderNamePrefixPurge & for user: $User"
+    Write-Host "`nLooking for folders under well-known folder: $WellKnownRoot matching folders: $FolderNamePurgeList or prefixes: $FolderPrefixPurgeList for user: $User"
     $getFolderIdMsg = Initialize-SOAPMessage -User $User -Body $body
     $response = Invoke-SOAPRequest -Token $Token -Message $getFolderIdMsg
 
     # Get the folders from the response
-    $folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types"} | 
+    $folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types" } | 
-                    Select-Object -ExpandProperty Node
+    Select-Object -ExpandProperty Node
 
-    $folderId = $null
+    # Are there more folders to list
-    $changeKey = $null
+    $rootFolder = $response | Select-Xml -XPath "//m:RootFolder" -Namespace @{m = "http://schemas.microsoft.com/exchange/services/2006/messages" } | 
-    $totalCount = $null
+    Select-Object -ExpandProperty Node
+    $moreToList = ![System.Convert]::ToBoolean($rootFolder.IncludesLastItemInRange)
 
     # Loop through folders
     foreach ($folder in $folders) {
-        $folderId = $folder.FolderId.Id
-        $changeKey = $folder.FolderId.Changekey
-        $totalCount = $folder.TotalCount
         $folderName = $folder.DisplayName
         $folderCreateTime = $folder.ExtendedProperty
-            | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" }
+        | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" }
-            | Select-Object -ExpandProperty Value
+        | Select-Object -ExpandProperty Value
-            | Get-Date
+        | Get-Date
 
-        if ((![String]::IsNullOrEmpty($FolderNamePurge) -and $folderName -ne $FolderNamePurge) -or
+        $IsNameMatchParams = @{
-            (![String]::IsNullOrEmpty($FolderPrefixPurge) -and $folderName -notlike "$FolderPrefixPurge*") -or
+            'FolderName'          = $folderName;
-            (![String]::IsNullOrEmpty($FolderBeforePurge) -and $folderCreateTime -gt $FolderBeforePurge)) {
+            'FolderNamePurgeList' = $FolderNamePurgeList
+        } 
+
+        $IsPrefixAndAgeMatchParams = @{
+            'FolderName'            = $folderName;
+            'FolderCreateTime'      = $folderCreateTime;
+            'FolderPrefixPurgeList' = $FolderPrefixPurgeList;
+            'PurgeBeforeTimestamp'  = $PurgeBeforeTimestamp;
+        }
+
+        if ((IsNameMatch @IsNameMatchParams) -or (IsPrefixAndAgeMatch @IsPrefixAndAgeMatchParams)) {
+            Write-Host "`nFound desired folder to purge: $folderName ($folderCreateTime)"
+            $foldersToDelete += $folder
+        }
+    }
+
+    # powershel does not do well when returning empty arrays
+    return $foldersToDelete, $moreToList
+}
+
+function Empty-Folder {
+    [CmdletBinding(SupportsShouldProcess)]
+    Param(
+        [Parameter(Mandatory = $False, HelpMessage = "List of well-known folders to empty ")]
+        [String[]]$WellKnownRootList = @(),
+
+        [Parameter(Mandatory = $False, HelpMessage = "List of folderIds to empty ")]
+        [string[]]$FolderIdList = @(),
+
+        [Parameter(Mandatory = $False, HelpMessage = "List of folder names to empty ")]
+        [string[]]$FolderNameList = @()
+    )
+
+    $folderIdsBody = ""
+    $foldersToEmptyCount = $FolderIdList.count + $WellKnownRootList.count
+
+    foreach ($wnr in $WellKnownRootList) {
+        $folderIdsBody += "<t:DistinguishedFolderId Id='$wnr'/>"
+    }
+
+    foreach ($fid in $FolderIdList) {
+        $folderIdsBody += "<t:FolderId Id='$fid'/>"
+    }
+
+    if ($PSCmdlet.ShouldProcess("Emptying $foldersToEmptyCount folders ($WellKnownRootList $FolderNameList)", "$foldersToEmptyCount folders ($WellKnownRootList $FolderNameList)", "Empty folders")) {
+        Write-Host "`nEmptying $foldersToEmptyCount folders ($WellKnownRootList $FolderNameList)"
+
+        # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
+        $body = @"
+<m:EmptyFolder DeleteType="HardDelete" DeleteSubFolders="true">
+    <m:FolderIds>
+    $folderIdsBody
+    </m:FolderIds>
+</m:EmptyFolder>
+"@
+
+        $emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body
+        $response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg
+    }
+}
+
+function Delete-Folder {
+    [CmdletBinding(SupportsShouldProcess)]
+    Param(
+        [Parameter(Mandatory = $True, HelpMessage = "List of folderIds to remove ")]
+        [String[]]$FolderIdList,
+
+        [Parameter(Mandatory = $False, HelpMessage = "List of folder names to remove ")]
+        [String[]]$FolderNameList = @()
+    )
+
+    $folderIdsBody = ""
+    $foldersToRemoveCount = $FolderIdList.count
+
+    foreach ($fid in $FolderIdList) {
+        $folderIdsBody += "<t:FolderId Id='$fid'/>"
+    }
+
+    if ($PSCmdlet.ShouldProcess("Removing $foldersToRemoveCount folders ($FolderNameList)", "$foldersToRemoveCount folders ($FolderNameList)", "Delete folders")) {
+        Write-Host "`nRemoving $foldersToRemoveCount folders ($FolderNameList)"
+
+        # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
+        $body = @"
+<m:DeleteFolder DeleteType="HardDelete" DeleteSubFolders="true">
+    <m:FolderIds>
+    $folderIdsBody
+    </m:FolderIds>
+</m:DeleteFolder>
+"@
+
+        $emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body
+        $response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg
+    }
+}
+
+function Purge-Folders {
+    [CmdletBinding(SupportsShouldProcess)]
+    Param(
+        [Parameter(Mandatory = $True, HelpMessage = "Folder under which to look for items matching removal criteria")]
+        [String]$WellKnownRoot,
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders with these names")]
+        [string[]]$FolderNamePurgeList = @(),
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders with these prefixes")]
+        [string[]]$FolderPrefixPurgeList = @(),
+
+        [Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")]
+        [datetime]$PurgeBeforeTimestamp
+    )  
+
+    if (($FolderNamePurgeList.count -eq 0) -and 
+        ($FolderPrefixPurgeList.count -eq 0 -or $PurgeBeforeTimestamp -eq $null )) {
+        Write-Host "Either a list of specific folders or a list of prefixes and purge timestamp is required"
+        Exit
+    }
+
+    Write-Host "`nPurging CI-produced folders..."
+    Write-Host "--------------------------------"
+
+    if ($FolderNamePurgeList.count -gt 0) {
+        Write-Host "Folders with names: $FolderNamePurgeList"
+    }
+
+    if ($FolderPrefixPurgeList.count -gt 0 -and $PurgeBeforeTimestamp -ne $null) {
+        Write-Host "Folders older than $PurgeBeforeTimestamp with prefix: $FolderPrefixPurgeList"
+    }
+
+    $foldersToDeleteParams = @{
+        'WellKnownRoot'         = $WellKnownRoot;
+        'FolderNamePurgeList'   = $FolderNamePurgeList;
+        'FolderPrefixPurgeList' = $FolderPrefixPurgeList;
+        'PurgeBeforeTimestamp'  = $PurgeBeforeTimestamp
+    }
+
+    $moreToList = $True
+    # only get max of 1000 results so we may need to iterate over eligible folders  
+    while ($moreToList) {
+        $foldersToDelete, $moreToList = Get-FoldersToPurge @foldersToDeleteParams
+        $foldersToDeleteCount = $foldersToDelete.count
+        $foldersToDeleteIds = @()
+        $folderNames = @()
+
+        if ($foldersToDeleteCount -eq 0) {
+            Write-Host "`nNo folders to purge matching the criteria"
+            break
+        }
+
+        foreach ($folder in $foldersToDelete) {
+            $foldersToDeleteIds += $folder.FolderId.Id
+            $folderNames += $folder.DisplayName
+        }
+        
+        Empty-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames
+        Delete-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames
+    }
+}
+
+function Create-Contact {
+    $now = (Get-Date (Get-Date).ToUniversalTime() -Format "o")
+    #used to create a recent seed contact that will be shielded from cleanup. CI tests rely on this    
+    $body = @"
+<CreateItem xmlns="http://schemas.microsoft.com/exchange/services/2006/messages" >
+    <SavedItemFolderId>
+        <t:DistinguishedFolderId Id="contacts"/>
+    </SavedItemFolderId>
+    <Items>
+        <t:Contact>
+            <t:GivenName>Sanitago</t:GivenName>
+            <t:Surname>TestContact - $now</t:Surname>
+            <t:CompanyName>Corso test enterprises</t:CompanyName>
+            <t:EmailAddresses>
+                <t:Entry Key="EmailAddress1">sanitago@example.com</t:Entry>
+            </t:EmailAddresses>
+            <t:PhoneNumbers>
+                <t:Entry Key="BusinessPhone">4255550199</t:Entry>
+            </t:PhoneNumbers>
+            <t:Birthday>2000-01-01T11:59:00Z</t:Birthday>
+            <t:JobTitle>Tester</t:JobTitle>
+            <t:Surname>Plate</t:Surname>
+        </t:Contact>
+    </Items>
+</CreateItem>
+"@
+
+    $createContactMsg = Initialize-SOAPMessage -User $User -Body $body
+    $response = Invoke-SOAPRequest -Token $Token -Message $createContactMsg
+}
+
+function Get-ItemsToPurge {
+    Param(
+        [Parameter(Mandatory = $True, HelpMessage = "Folder under which to look for items matching removal criteria")]
+        [String]$WellKnownRoot,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Purge items before this date time (UTC)")]
+        [datetime]$PurgeBeforeTimestamp
+    )
+
+    $itemsToDelete = @()
+
+    # SOAP message for getting the folder id
+    $body = @"
+<FindItem Traversal="Shallow" xmlns="http://schemas.microsoft.com/exchange/services/2006/messages">
+    <ItemShape>
+        <t:BaseShape>Default</t:BaseShape>
+        <t:AdditionalProperties>
+            <t:ExtendedFieldURI PropertyTag="0x3007" PropertyType="SystemTime"/>
+        </t:AdditionalProperties>
+    </ItemShape>
+    <ParentFolderIds>
+        <t:DistinguishedFolderId Id="$WellKnownRoot"/>
+    </ParentFolderIds>
+</FindItem>
+"@
+
+    Write-Host "`nLooking for items under well-known folder: $WellKnownRoot older than $PurgeBeforeTimestamp for user: $User"
+    $getItemsMsg = Initialize-SOAPMessage -User $User -Body $body
+    $response = Invoke-SOAPRequest -Token $Token -Message $getItemsMsg
+
+    # Get the contacts from the response
+    $items = $response | Select-Xml -XPath "//t:Items/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types" } | 
+    Select-Object -ExpandProperty Node
+
+    # Are there more folders to list
+    $rootFolder = $response | Select-Xml -XPath "//m:RootFolder" -Namespace @{m = "http://schemas.microsoft.com/exchange/services/2006/messages" } | 
+    Select-Object -ExpandProperty Node
+    $moreToList = ![System.Convert]::ToBoolean($rootFolder.IncludesLastItemInRange)
+
+    foreach ($item in $items) {
+        $itemId = $item.ItemId.Id
+        $changeKey = $item.ItemId.Changekey
+        $itemName = $item.DisplayName
+        $itemCreateTime = $item.ExtendedProperty
+        | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" }
+        | Select-Object -ExpandProperty Value
+        | Get-Date
+
+        if ([String]::IsNullOrEmpty($itemId) -or [String]::IsNullOrEmpty($changeKey)) {
             continue
         }
 
-        if (![String]::IsNullOrEmpty($FolderNamePurge)) {
+        if (![String]::IsNullOrEmpty($PurgeBeforeTimestamp) -and $itemCreateTime -gt $PurgeBeforeTimestamp) {
-            Write-Host "`nFound desired folder to purge: $FolderNamePurge"
+            continue
         }
 
-        Write-Verbose "`nFolder Id and ChangeKey for ""$folderName"": $folderId, $changeKey"
+        Write-Verbose "Item Id and ChangeKey for ""$itemName"": $itemId, $changeKey"
+        $itemsToDelete += $item
+    }
 
-        # Empty and delete the folder if found
+    return $itemsToDelete, $moreToList
-        if (![String]::IsNullOrEmpty($folderId) -and ![String]::IsNullOrEmpty($changeKey)) {
+}
-            if ($PSCmdlet.ShouldProcess("$folderName ($totalCount items) created $folderCreateTime", "Emptying folder")) {
-                Write-Host "Emptying folder $folderName ($totalCount items)..."
 
-                # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
+function Purge-Contacts {
-                $body = @"
+    [CmdletBinding(SupportsShouldProcess)]
-        <m:EmptyFolder DeleteType="HardDelete" DeleteSubFolders="true">
+    Param(
-            <m:FolderIds>
+        [Parameter(Mandatory = $True, HelpMessage = "Purge items before this date time (UTC)")]
-                <t:FolderId Id="$folderId" ChangeKey="$changeKey" />
+        [datetime]$PurgeBeforeTimestamp
-            </m:FolderIds>
+    )
-        </m:EmptyFolder>
+
+    Write-Host "`nCleaning up contacts older than $PurgeBeforeTimestamp" 
+    Write-Host "-------------------------------------------------------" 
+
+    # Create one seed contact which will have recent create date and will not be sweapt
+    # This is needed since tests rely on some contact data being present
+    Write-Host "`nCreating seed contact" 
+    Create-Contact
+
+    $moreToList = $True
+    # only get max of 1000 results so we may need to iterate over eligible contacts  
+    while ($moreToList) {
+        $itemsToDelete, $moreToList = Get-ItemsToPurge -WellKnownRoot "contacts" -PurgeBeforeTimestamp $PurgeBeforeTimestamp
+        $itemsToDeleteCount = $itemsToDelete.count
+        $itemsToDeleteBody = ""
+
+        if ($itemsToDeleteCount -eq 0) {
+            Write-Host "`nNo more contacts to delete matching criteria"
+            break
+        }
+
+        Write-Host "`nQueueing $itemsToDeleteCount items to delete"
+        foreach ($item in $itemsToDelete) {
+            $itemId = $item.ItemId.Id
+            $changeKey = $item.ItemId.Changekey
+            $itemsToDeleteBody += "<t:ItemId Id='$itemId' ChangeKey='$changeKey' />`n"
+        }
+
+        # Do the actual deletion in a batch request 
+        # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
+        $body = @"
+<m:DeleteItem DeleteType="HardDelete">
+    <m:ItemIds>
+        $itemsToDeleteBody
+    </m:ItemIds>
+</m:DeleteItem>
 "@
-                $emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body
+        
-                $response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg
+        if ($PSCmdlet.ShouldProcess("Deleting $itemsToDeleteCount items...", "$itemsToDeleteCount items", "Delete items")) {
-            }
+            Write-Host "`nDeleting $itemsToDeleteCount items..."
-
+        
-            if ($PSCmdlet.ShouldProcess($folderName, "Deleting folder")) {
+            $emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body
-                Write-Host "Deleting folder $folderName..."
+            $response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg
-
+        
-                # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
+            Write-Host "`nDeleted $itemsToDeleteCount items..."
-                $body = @"
+        
-            <m:DeleteFolder  DeleteType="HardDelete" DeleteSubFolders="true">
-                <m:FolderIds>
-                    <t:FolderId Id="$folderId" ChangeKey="$changeKey" />
-                </m:FolderIds>
-            </m:DeleteFolder>
-"@
-                $deleteFolderMsg = Initialize-SOAPMessage -User $User -Body $body
-                $response = Invoke-SOAPRequest -Token $Token -Message $deleteFolderMsg
-            }
-
-            Write-Host "Deleted folder $folderName ($totalCount items)"
         }
     }
 }
 
-$token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force 
+Write-Host 'Authenticating with Exchange Web Services ...'
+$global:Token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force 
 
-Remove-Folder -Token $token  -User $User -WellKnownRoot $WellKnownRoot
+$purgeFolderParams = @{
+    'WellKnownRoot'         = "root";
+    'FolderNamePurgeList'   = $FolderNamePurgeList;
+    'FolderPrefixPurgeList' = $FolderPrefixPurgeList;
+    'PurgeBeforeTimestamp'  = $PurgeBeforeTimestamp
+}
+
+#purge older prefix folders
+Purge-Folders @purgeFolderParams
+
+#purge older contacts 
+Purge-Contacts -PurgeBeforeTimestamp $PurgeBeforeTimestamp
+
+# Empty Deleted Items and then purge all recoverable items. Deletes the following
+# -/Recoverable Items/Audits
+# -/Recoverable Items/Deletion
+# -/Recoverable Items/Purges
+# -/Recoverable Items/Versions
+# -/Recoverable Items/Calendar Logging
+# -/Recoverable Items/SubstrateHolds
+Write-Host "`nProcess well-known folders that are always purged" 
+Write-Host "---------------------------------------------------" 
+Empty-Folder -WellKnownRoot "deleteditems", "recoverableitemsroot"