From aa271fe09bda2455860a1e187785989859c48a4a Mon Sep 17 00:00:00 2001 From: Georgi Matev Date: Thu, 1 Jun 2023 10:46:12 -0700 Subject: [PATCH] Exchange cleanup improvements (#3551) Exchange cleanup improvements: * Explicitly clean-up folders in DeletedItems - Empty folder does not seem to work reliably * Cleanup any folders in DeletedItems - helps get rid of random manual deletions * Harden paging - issues where if no items were deleted initial page, we'd never make progress * More focused search on `msgfolderroot` --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/cmd/purge/scripts/exchangePurge.ps1 | 194 +++++++++++++++--------- 1 file changed, 122 insertions(+), 72 deletions(-) diff --git a/src/cmd/purge/scripts/exchangePurge.ps1 b/src/cmd/purge/scripts/exchangePurge.ps1 index 240364b8d..c6ca7228a 100644 --- a/src/cmd/purge/scripts/exchangePurge.ps1 +++ b/src/cmd/purge/scripts/exchangePurge.ps1 @@ -185,77 +185,109 @@ function Get-FoldersToPurge { [string[]]$FolderNamePurgeList = @(), [Parameter(Mandatory = $False, HelpMessage = "Purge folders with these prefixes")] - [string[]]$FolderPrefixPurgeList = @() + [string[]]$FolderPrefixPurgeList = @(), + + [Parameter(Mandatory = $False, HelpMessage = "Perform shallow traversal only")] + [bool]$PurgeTraversalShallow = $false ) - $foldersToDelete = @() + Write-Host "`nLooking for folders under well-known folder: $WellKnownRoot matching folders: $FolderNamePurgeList or prefixes: $FolderPrefixPurgeList for user: $User" - # SOAP message for getting the folders - $body = @" - + $foldersToDelete = @() + $traversal = "Deep" + if ($PurgeTraversalShallow) { + $traversal = "Shallow" + } + + $offset = 0 + $moreToList = $true + + # get all folder pages + while ($moreToList) { + # SOAP message for getting the folders + $body = @" + Default + "@ - Write-Host "`nLooking for folders under well-known folder: $WellKnownRoot matching folders: $FolderNamePurgeList or prefixes: $FolderPrefixPurgeList for user: $User" - $getFolderIdMsg = Initialize-SOAPMessage -User $User -Body $body - $response = Invoke-SOAPRequest -Token $Token -Message $getFolderIdMsg + try { + Write-Host "`nRetrieving folders starting from offset: $offset" - # Get the folders from the response - $folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types" } | - Select-Object -ExpandProperty Node + $getFolderIdMsg = Initialize-SOAPMessage -User $User -Body $body + $response = Invoke-SOAPRequest -Token $Token -Message $getFolderIdMsg - # Are there more folders to list - $rootFolder = $response | Select-Xml -XPath "//m:RootFolder" -Namespace @{m = "http://schemas.microsoft.com/exchange/services/2006/messages" } | - Select-Object -ExpandProperty Node - $moreToList = ![System.Convert]::ToBoolean($rootFolder.IncludesLastItemInRange) + # Are there more folders to list + $rootFolder = $response | Select-Xml -XPath "//m:RootFolder" -Namespace @{m = "http://schemas.microsoft.com/exchange/services/2006/messages" } | + Select-Object -ExpandProperty Node + $moreToList = ![System.Convert]::ToBoolean($rootFolder.IncludesLastItemInRange) + } + catch { + Write-Host "Error retrieving folders" - # Loop through folders - foreach ($folder in $folders) { - $folderName = $folder.DisplayName - $folderCreateTime = $folder.ExtendedProperty - | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" } - | Select-Object -ExpandProperty Value - | Get-Date + Write-Host $response.OuterXml + Exit + } + + # Get the folders from the response + $folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types" } | + Select-Object -ExpandProperty Node - if ($FolderNamePurgeList.count -gt 0) { - $IsNameMatchParams = @{ - 'FolderName' = $folderName; - 'FolderNamePurgeList' = $FolderNamePurgeList - } + # Loop through folders + foreach ($folder in $folders) { + $folderName = $folder.DisplayName + $folderCreateTime = $folder.ExtendedProperty + | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" } + | Select-Object -ExpandProperty Value + | Get-Date - if ((IsNameMatch @IsNameMatchParams)) { - Write-Host "• Found name match: $folderName ($folderCreateTime)" - $foldersToDelete += $folder - continue + if ($FolderNamePurgeList.count -gt 0) { + $IsNameMatchParams = @{ + 'FolderName' = $folderName; + 'FolderNamePurgeList' = $FolderNamePurgeList + } + + if ((IsNameMatch @IsNameMatchParams)) { + Write-Host "• Found name match: $folderName ($folderCreateTime)" + $foldersToDelete += $folder + continue + } + } + + if ($FolderPrefixPurgeList.count -gt 0) { + $IsPrefixAndAgeMatchParams = @{ + 'FolderName' = $folderName; + 'FolderCreateTime' = $folderCreateTime; + 'FolderPrefixPurgeList' = $FolderPrefixPurgeList; + 'PurgeBeforeTimestamp' = $PurgeBeforeTimestamp; + } + + if ((IsPrefixAndAgeMatch @IsPrefixAndAgeMatchParams)) { + Write-Host "• Found prefix match: $folderName ($folderCreateTime)" + $foldersToDelete += $folder + } } } - if ($FolderPrefixPurgeList.count -gt 0) { - $IsPrefixAndAgeMatchParams = @{ - 'FolderName' = $folderName; - 'FolderCreateTime' = $folderCreateTime; - 'FolderPrefixPurgeList' = $FolderPrefixPurgeList; - 'PurgeBeforeTimestamp' = $PurgeBeforeTimestamp; - } - - if ((IsPrefixAndAgeMatch @IsPrefixAndAgeMatchParams)) { - Write-Host "• Found prefix match: $folderName ($folderCreateTime)" - $foldersToDelete += $folder - } + if (!$moreToList -or $null -eq $folders) { + Write-Host "Retrieved all folders." + } + else { + $offset += $folders.count } } # powershel does not do well when returning empty arrays - return $foldersToDelete, $moreToList + return , $foldersToDelete } function Empty-Folder { @@ -355,7 +387,10 @@ function Purge-Folders { [string[]]$FolderPrefixPurgeList = @(), [Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")] - [datetime]$PurgeBeforeTimestamp + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $False, HelpMessage = "Perform shallow traversal only")] + [bool]$PurgeTraversalShallow = $false ) if (($FolderNamePurgeList.count -eq 0) -and @@ -364,9 +399,6 @@ function Purge-Folders { Exit } - Write-Host "`nPurging CI-produced folders..." - Write-Host "--------------------------------" - if ($FolderNamePurgeList.count -gt 0) { Write-Host "Folders with names: $FolderNamePurgeList" } @@ -382,30 +414,27 @@ function Purge-Folders { 'WellKnownRoot' = $WellKnownRoot; 'FolderNamePurgeList' = $FolderNamePurgeList; 'FolderPrefixPurgeList' = $FolderPrefixPurgeList; - 'PurgeBeforeTimestamp' = $PurgeBeforeTimestamp + 'PurgeBeforeTimestamp' = $PurgeBeforeTimestamp; + 'PurgeTraversalShallow' = $PurgeTraversalShallow } - $moreToList = $True - # only get max of 1000 results so we may need to iterate over eligible folders - while ($moreToList) { - $foldersToDelete, $moreToList = Get-FoldersToPurge @foldersToDeleteParams - $foldersToDeleteCount = $foldersToDelete.count - $foldersToDeleteIds = @() - $folderNames = @() + $foldersToDelete = Get-FoldersToPurge @foldersToDeleteParams + $foldersToDeleteCount = $foldersToDelete.count + $foldersToDeleteIds = @() + $folderNames = @() - if ($foldersToDeleteCount -eq 0) { - Write-Host "`nNo folders to purge matching the criteria" - break - } - - foreach ($folder in $foldersToDelete) { - $foldersToDeleteIds += $folder.FolderId.Id - $folderNames += $folder.DisplayName - } - - Empty-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames - Delete-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames + if ($foldersToDeleteCount -eq 0) { + Write-Host "`nNo folders to purge matching the criteria" + return } + + foreach ($folder in $foldersToDelete) { + $foldersToDeleteIds += $folder.FolderId.Id + $folderNames += $folder.DisplayName + } + + Empty-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames + Delete-Folder -FolderIdList $foldersToDeleteIds -FolderNameList $folderNames } function Create-Contact { @@ -459,7 +488,7 @@ function Get-ItemsToPurge { $foldersToSearchBody = "" if (![String]::IsNullOrEmpty($SubFolderName)) { - $subFolders, $moreToList = Get-FoldersToPurge -WellKnownRoot $WellKnownRoot -FolderNamePurgeList $SubFolderName -PurgeBeforeTimestamp $PurgeBeforeTimestamp + $subFolders = Get-FoldersToPurge -WellKnownRoot $WellKnownRoot -FolderNamePurgeList $SubFolderName -PurgeBeforeTimestamp $PurgeBeforeTimestamp if ($subFolders.count -gt 0 ) { $foldersToSearchBody = "" @@ -615,6 +644,8 @@ function Purge-Items { } } +### MAIN #### + Write-Host 'Authenticating with Exchange Web Services ...' $global:Token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force @@ -622,14 +653,17 @@ $global:Token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force $FolderNamePurgeList = $FolderNamePurgeList | ForEach-Object { @($_.Split(',').Trim()) } $FolderPrefixPurgeList = $FolderPrefixPurgeList | ForEach-Object { @($_.Split(',').Trim()) } +Write-Host "`nPurging CI-produced folders under 'msgfolderroot' ..." +Write-Host "--------------------------------------------------------" + $purgeFolderParams = @{ - 'WellKnownRoot' = "root"; + 'WellKnownRoot' = "msgfolderroot"; 'FolderNamePurgeList' = $FolderNamePurgeList; 'FolderPrefixPurgeList' = $FolderPrefixPurgeList; 'PurgeBeforeTimestamp' = $PurgeBeforeTimestamp } -#purge older prefix folders +#purge older prefix folders from msgfolderroot Purge-Folders @purgeFolderParams #purge older contacts @@ -647,4 +681,20 @@ Purge-Items -ItemsFolder "calendar" -ItemsSubFolder "Birthdays" -PurgeBeforeTime # -/Recoverable Items/SubstrateHolds Write-Host "`nProcess well-known folders that are always purged" Write-Host "---------------------------------------------------" -Empty-Folder -WellKnownRoot "deleteditems", "recoverableitemsroot" \ No newline at end of file + +# We explicitly also clean direct folders under Deleted Items since there is some evidence +# that suggests that emptying alone may not be reliable +Write-Host "`nExplicit delete of all folders under 'DeletedItems' ..." +Write-Host "----------------------------------------------------------" + +$purgeFolderParams = @{ + 'WellKnownRoot' = "deleteditems"; + 'FolderNamePurgeList' = $FolderNamePurgeList; + 'FolderPrefixPurgeList' = @('*'); + 'PurgeBeforeTimestamp' = (Get-Date); + 'PurgeTraversalShallow' = $true +} + +Purge-Folders @purgeFolderParams + +Empty-Folder -WellKnownRootList "deleteditems", "recoverableitemsroot"