diff --git a/.github/actions/purge-m365-user-data/action.yml b/.github/actions/purge-m365-user-data/action.yml new file mode 100644 index 000000000..8f2114178 --- /dev/null +++ b/.github/actions/purge-m365-user-data/action.yml @@ -0,0 +1,59 @@ +name: Purge M365 User Data + +# Hard deletion of an m365 user's data. Our CI processes create a lot +# of data churn (creation and immediate deletion) of files, the likes +# of which wouldn't otherwise be seen by users of the system. Standard +# APIs don't have the tooling to gut out all the cruft which we accrue +# in microsoft's hidden nooks and secret crannies. A manual, SOAPy +# exorcism is the only way. +# +# The script focuses on the cleaning up the following: +# * All folders, descending from the exchange root, of a given prefix. +# * All folders in PersonMetadata +# * All already soft-deleted items +# * All recoverable items in Audits +# * All recoverable items in Purges + +inputs: + user: + description: User whose data is to be purged. + folder-prefix: + description: Name of the folder to be purged. If false, will purge the set of static, well known folders instead. + older-than: + description: Minimum-age of folders to be deleted. + azure-client-id: + description: Secret value of for AZURE_CLIENT_ID + azure-client-secret: + description: Secret value of for AZURE_CLIENT_SECRET + azure-tenant-id: + description: Secret value of for AZURE_TENANT_ID + +runs: + using: composite + steps: + + - name: Run the folder-matrix purge script set + if: ${{ inputs.folder-prefix != '' }} + shell: pwsh + working-directory: ./src/cmd/purge/scripts + env: + AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} + AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} + AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} + run: | + ./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderPrefixPurge ${{ inputs.folder-prefix }} -FolderBeforePurge ${{ inputs.older-than }} + + - name: Run the static purge script set + if: ${{ inputs.folder-prefix == '' }} + shell: pwsh + working-directory: ./src/cmd/purge/scripts + env: + AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} + AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} + AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} + # powershell doesn't like multiline commands, each of these must be on a single line + run: | + ./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderNamePurge PersonMetadata + ./foldersAndItems.ps1 -WellKnownRoot deleteditems -User ${{ inputs.user }} + ./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Audits + ./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Purges \ No newline at end of file diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 94e597a95..3e16272a4 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -8,17 +8,17 @@ jobs: Test-User-Data-Cleanup: environment: Testing runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + folder: [Corso_Restore_, TestRestore, ''] + user: [CORSO_M356_TEST_USER_ID, CORSO_SECONDARY_M356_TEST_USER_ID] steps: - uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} - - name: Setup Golang with cache - uses: magnetikonline/action-golang-cache@v3 - with: - go-version-file: src/go.mod - # sets the maximimum time to now-30m. # CI test have a 10 minute timeout. # At 20 minutes ago, we should be safe from conflicts. @@ -27,31 +27,12 @@ jobs: run: | echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV - # run the folder purge - - name: Purge primary user folders - working-directory: ./src - env: - AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} - AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} - CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }} - DELETE_FOLDER_PREFIX: "Corso_Restore_" - run: > - go run ./cmd/purge/purge.go - --user ${{ secrets.CORSO_M356_TEST_USER_ID }} - --prefix ${{ env.DELETE_FOLDER_PREFIX }} - --before ${{ env.HALF_HOUR_AGO }} - - - name: Purge secondary user folders - working-directory: ./src - env: - AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} - AZURE_TENANT_ID: ${{ secrets.TENANT_ID }} - CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M356_TEST_USER_ID }} - DELETE_FOLDER_PREFIX: "Corso_Restore_" - run: > - go run ./cmd/purge/purge.go - --user ${{ secrets.CORSO_SECONDARY_M356_TEST_USER_ID }} - --prefix ${{ env.DELETE_FOLDER_PREFIX }} - --before ${{ env.HALF_HOUR_AGO }} + - name: Purge CI-Produced Folders + uses: ./.github/actions/purge-m365-user-data + with: + user: ${{ secrets[matrix.user] }} + folder-prefix: ${{ matrix.folder }} + older-than: ${{ env.HALF_HOUR_AGO }} + azure-client-id: ${{ secrets.CLIENT_ID }} + azure-client-secret: ${{ secrets.CLIENT_SECRET }} + azure-tenant-id: ${{ secrets.TENANT_ID }} diff --git a/.gitignore b/.gitignore index f3bb16eb2..46f5189b8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.dll *.so *.dylib +.DS_Store # Test binary, built with `go test -c` *.test diff --git a/src/cmd/purge/scripts/foldersAndItems.ps1 b/src/cmd/purge/scripts/foldersAndItems.ps1 new file mode 100644 index 000000000..c086a44d9 --- /dev/null +++ b/src/cmd/purge/scripts/foldersAndItems.ps1 @@ -0,0 +1,207 @@ +[CmdletBinding(SupportsShouldProcess)] + +Param ( + [Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")] + [String]$User, + + [Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")] + [String]$WellKnownRoot = "deleteditems", + + [Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$FolderBeforePurge, + + [Parameter(Mandatory = $False, HelpMessage = "Name of specific folder to purge under root")] + [String]$FolderNamePurge, + + [Parameter(Mandatory = $False, HelpMessage = "Purge folders with this prefix")] + [String]$FolderPrefixPurge, + + [Parameter(Mandatory = $False, HelpMessage = "Azure TenantId")] + [String]$TenantId = $ENV:AZURE_TENANT_ID, + + [Parameter(Mandatory = $False, HelpMessage = "Azure ClientId")] + [String]$ClientId = $ENV:AZURE_CLIENT_ID, + + [Parameter(Mandatory = $False, HelpMessage = "Azure ClientSecret")] + [String]$ClientSecret = $ENV:AZURE_CLIENT_SECRET +) + +function Get-AccessToken { + [CmdletBinding()] + Param() + + if ([String]::IsNullOrEmpty($TenantId) -or [String]::IsNullOrEmpty($ClientId) -or [String]::IsNullOrEmpty($ClientSecret)) { + Write-Host "Need to specify TenantId, ClientId, and ClientSecret as parameters or ENVs" + } + + $body=@{ + client_id=$ClientId + client_secret=$ClientSecret + scope="https://outlook.office365.com/.default" + grant_type="client_credentials" + } + + $res = Invoke-WebRequest -Uri "https://login.microsoftonline.com/$TenantId/oauth2/v2.0/token" -ContentType "application/x-www-form-urlencoded" -Body $body -Method Post + + return $res.content | ConvertFrom-Json | Select-Object -ExpandProperty access_token +} + +function Initialize-SOAPMessage { + [CmdletBinding()] + Param( + [Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")] + [String]$User, + + [Parameter(Mandatory = $True, HelpMessage = "The message body")] + [String]$Body + ) + + $Message = @" + + + + + + + $User + + + + + + $Body + + +"@ + + return $Message +} + +function Invoke-SOAPRequest { + [CmdletBinding()] + Param( + [Parameter(Mandatory = $True, HelpMessage = "OAuth token to connect to Exchange Online.")] + [securestring]$Token, + + [Parameter(Mandatory = $True, HelpMessage = "The message ")] + [String]$Message, + + [Parameter(Mandatory = $False, HelpMessage = "The http method")] + [String]$Method = "Post" + ) + + # EWS service url for Exchange Online + $webServiceUrl = "https://outlook.office365.com/EWS/Exchange.asmx" + + $Response = Invoke-WebRequest -Uri $webServiceUrl -Authentication Bearer -Token $Token -Body $Message -Method $Method + [xml]$xmlResponse = $Response.Content + + return $xmlResponse +} + +function Remove-Folder { + [CmdletBinding(SupportsShouldProcess)] + Param( + [Parameter(Mandatory = $True, HelpMessage = "OAuth token to connect to Exchange Online.")] + [Securestring]$Token, + + [Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")] + [String]$User, + + [Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")] + [String]$WellKnownRoot = "deleteditems" + ) + + # SOAP message for getting the folder id + $body = @" + + + Default + + + + + + + + +"@ + + Write-Host "Looking for folders under well-known folder: $WellKnownRoot & matching folder: $FolderNamePurge$FolderNamePrefixPurge & for user: $User" + $getFolderIdMsg = Initialize-SOAPMessage -User $User -Body $body + $response = Invoke-SOAPRequest -Token $Token -Message $getFolderIdMsg + + # Get the folders from the response + $folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types"} | + Select-Object -ExpandProperty Node + + $folderId = $null + $changeKey = $null + $totalCount = $null + + # Loop through folders + foreach ($folder in $folders) { + $folderId = $folder.FolderId.Id + $changeKey = $folder.FolderId.Changekey + $totalCount = $folder.TotalCount + $folderName = $folder.DisplayName + $folderCreateTime = $folder.ExtendedProperty + | Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" } + | Select-Object -ExpandProperty Value + | Get-Date + + if ((![String]::IsNullOrEmpty($FolderNamePurge) -and $folderName -ne $FolderNamePurge) -or + (![String]::IsNullOrEmpty($FolderPrefixPurge) -and $folderName -notlike "$FolderPrefixPurge*") -or + (![String]::IsNullOrEmpty($FolderBeforePurge) -and $folderCreateTime -gt $FolderBeforePurge)) { + continue + } + + if (![String]::IsNullOrEmpty($FolderNamePurge)) { + Write-Host "Found desired folder to purge: $FolderNamePurge" + } + + Write-Verbose "Folder Id and ChangeKey for ""$folderName"": $folderId, $changeKey" + + # Empty and delete the folder if found + if (![String]::IsNullOrEmpty($folderId) -and ![String]::IsNullOrEmpty($changeKey)) { + if ($PSCmdlet.ShouldProcess("$folderName ($totalCount items) created $folderCreateTime", "Emptying folder")) { + Write-Host "Emptying folder $folderName ($totalCount items)..." + + # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete + $body = @" + + + + + +"@ + $emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body + $response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg + } + + if ($PSCmdlet.ShouldProcess($folderName, "Deleting folder")) { + Write-Host "Deleting folder $folderName..." + + # DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete + $body = @" + + + + + +"@ + $deleteFolderMsg = Initialize-SOAPMessage -User $User -Body $body + $response = Invoke-SOAPRequest -Token $Token -Message $deleteFolderMsg + } + + Write-Host "Deleted folder $folderName ($totalCount items)" + } + } +} + +$token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force + +Remove-Folder -Token $token -User $User -WellKnownRoot $WellKnownRoot \ No newline at end of file