replace golang purge script with powershell script (#1364)
## Description Swap from GraphAPI-based data cleanup (which doesn't allow us to access hidden folders like Audit retention or delete-restoration) to powershell script-based cleanup (which allows us to make SOAP requests against legacy exchange apis). ## Type of change - [x] 🐛 Bugfix - [x] 🤖 Test ## Issue(s) * #1266 ## Test Plan - [x] 💪 Manual - [x] 💚 E2E
This commit is contained in:
Keepers
GitHub
parent
0e29655645
commit
e6191f017b
59
.github/actions/purge-m365-user-data/action.yml
vendored
Normal file
59
.github/actions/purge-m365-user-data/action.yml
vendored
Normal file
@ -0,0 +1,59 @@
|
||||
name: Purge M365 User Data
|
||||
|
||||
# Hard deletion of an m365 user's data. Our CI processes create a lot
|
||||
# of data churn (creation and immediate deletion) of files, the likes
|
||||
# of which wouldn't otherwise be seen by users of the system. Standard
|
||||
# APIs don't have the tooling to gut out all the cruft which we accrue
|
||||
# in microsoft's hidden nooks and secret crannies. A manual, SOAPy
|
||||
# exorcism is the only way.
|
||||
#
|
||||
# The script focuses on the cleaning up the following:
|
||||
# * All folders, descending from the exchange root, of a given prefix.
|
||||
# * All folders in PersonMetadata
|
||||
# * All already soft-deleted items
|
||||
# * All recoverable items in Audits
|
||||
# * All recoverable items in Purges
|
||||
|
||||
inputs:
|
||||
user:
|
||||
description: User whose data is to be purged.
|
||||
folder-prefix:
|
||||
description: Name of the folder to be purged. If false, will purge the set of static, well known folders instead.
|
||||
older-than:
|
||||
description: Minimum-age of folders to be deleted.
|
||||
azure-client-id:
|
||||
description: Secret value of for AZURE_CLIENT_ID
|
||||
azure-client-secret:
|
||||
description: Secret value of for AZURE_CLIENT_SECRET
|
||||
azure-tenant-id:
|
||||
description: Secret value of for AZURE_TENANT_ID
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
|
||||
- name: Run the folder-matrix purge script set
|
||||
if: ${{ inputs.folder-prefix != '' }}
|
||||
shell: pwsh
|
||||
working-directory: ./src/cmd/purge/scripts
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
|
||||
AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
|
||||
AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
|
||||
run: |
|
||||
./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderPrefixPurge ${{ inputs.folder-prefix }} -FolderBeforePurge ${{ inputs.older-than }}
|
||||
|
||||
- name: Run the static purge script set
|
||||
if: ${{ inputs.folder-prefix == '' }}
|
||||
shell: pwsh
|
||||
working-directory: ./src/cmd/purge/scripts
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
|
||||
AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
|
||||
AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
|
||||
# powershell doesn't like multiline commands, each of these must be on a single line
|
||||
run: |
|
||||
./foldersAndItems.ps1 -WellKnownRoot root -User ${{ inputs.user }} -FolderNamePurge PersonMetadata
|
||||
./foldersAndItems.ps1 -WellKnownRoot deleteditems -User ${{ inputs.user }}
|
||||
./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Audits
|
||||
./foldersAndItems.ps1 -WellKnownRoot recoverableitemsroot -User ${{ inputs.user }} -FolderNamePurge Purges
|
||||
47
.github/workflows/ci_test_cleanup.yml
vendored
47
.github/workflows/ci_test_cleanup.yml
vendored
@ -8,17 +8,17 @@ jobs:
|
||||
Test-User-Data-Cleanup:
|
||||
environment: Testing
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
matrix:
|
||||
folder: [Corso_Restore_, TestRestore, '']
|
||||
user: [CORSO_M356_TEST_USER_ID, CORSO_SECONDARY_M356_TEST_USER_ID]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.head_ref }}
|
||||
|
||||
- name: Setup Golang with cache
|
||||
uses: magnetikonline/action-golang-cache@v3
|
||||
with:
|
||||
go-version-file: src/go.mod
|
||||
|
||||
# sets the maximimum time to now-30m.
|
||||
# CI test have a 10 minute timeout.
|
||||
# At 20 minutes ago, we should be safe from conflicts.
|
||||
@ -27,31 +27,12 @@ jobs:
|
||||
run: |
|
||||
echo "HALF_HOUR_AGO=$(date -d '30 minutes ago' -u +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_ENV
|
||||
|
||||
# run the folder purge
|
||||
- name: Purge primary user folders
|
||||
working-directory: ./src
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
|
||||
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }}
|
||||
DELETE_FOLDER_PREFIX: "Corso_Restore_"
|
||||
run: >
|
||||
go run ./cmd/purge/purge.go
|
||||
--user ${{ secrets.CORSO_M356_TEST_USER_ID }}
|
||||
--prefix ${{ env.DELETE_FOLDER_PREFIX }}
|
||||
--before ${{ env.HALF_HOUR_AGO }}
|
||||
|
||||
- name: Purge secondary user folders
|
||||
working-directory: ./src
|
||||
env:
|
||||
AZURE_CLIENT_ID: ${{ secrets.CLIENT_ID }}
|
||||
AZURE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
|
||||
AZURE_TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M356_TEST_USER_ID }}
|
||||
DELETE_FOLDER_PREFIX: "Corso_Restore_"
|
||||
run: >
|
||||
go run ./cmd/purge/purge.go
|
||||
--user ${{ secrets.CORSO_SECONDARY_M356_TEST_USER_ID }}
|
||||
--prefix ${{ env.DELETE_FOLDER_PREFIX }}
|
||||
--before ${{ env.HALF_HOUR_AGO }}
|
||||
- name: Purge CI-Produced Folders
|
||||
uses: ./.github/actions/purge-m365-user-data
|
||||
with:
|
||||
user: ${{ secrets[matrix.user] }}
|
||||
folder-prefix: ${{ matrix.folder }}
|
||||
older-than: ${{ env.HALF_HOUR_AGO }}
|
||||
azure-client-id: ${{ secrets.CLIENT_ID }}
|
||||
azure-client-secret: ${{ secrets.CLIENT_SECRET }}
|
||||
azure-tenant-id: ${{ secrets.TENANT_ID }}
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -4,6 +4,7 @@
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
.DS_Store
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
207
src/cmd/purge/scripts/foldersAndItems.ps1
Normal file
207
src/cmd/purge/scripts/foldersAndItems.ps1
Normal file
@ -0,0 +1,207 @@
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
|
||||
Param (
|
||||
[Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")]
|
||||
[String]$User,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")]
|
||||
[String]$WellKnownRoot = "deleteditems",
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Purge folders before this date time (UTC)")]
|
||||
[datetime]$FolderBeforePurge,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Name of specific folder to purge under root")]
|
||||
[String]$FolderNamePurge,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Purge folders with this prefix")]
|
||||
[String]$FolderPrefixPurge,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Azure TenantId")]
|
||||
[String]$TenantId = $ENV:AZURE_TENANT_ID,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Azure ClientId")]
|
||||
[String]$ClientId = $ENV:AZURE_CLIENT_ID,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Azure ClientSecret")]
|
||||
[String]$ClientSecret = $ENV:AZURE_CLIENT_SECRET
|
||||
)
|
||||
|
||||
function Get-AccessToken {
|
||||
[CmdletBinding()]
|
||||
Param()
|
||||
|
||||
if ([String]::IsNullOrEmpty($TenantId) -or [String]::IsNullOrEmpty($ClientId) -or [String]::IsNullOrEmpty($ClientSecret)) {
|
||||
Write-Host "Need to specify TenantId, ClientId, and ClientSecret as parameters or ENVs"
|
||||
}
|
||||
|
||||
$body=@{
|
||||
client_id=$ClientId
|
||||
client_secret=$ClientSecret
|
||||
scope="https://outlook.office365.com/.default"
|
||||
grant_type="client_credentials"
|
||||
}
|
||||
|
||||
$res = Invoke-WebRequest -Uri "https://login.microsoftonline.com/$TenantId/oauth2/v2.0/token" -ContentType "application/x-www-form-urlencoded" -Body $body -Method Post
|
||||
|
||||
return $res.content | ConvertFrom-Json | Select-Object -ExpandProperty access_token
|
||||
}
|
||||
|
||||
function Initialize-SOAPMessage {
|
||||
[CmdletBinding()]
|
||||
Param(
|
||||
[Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")]
|
||||
[String]$User,
|
||||
|
||||
[Parameter(Mandatory = $True, HelpMessage = "The message body")]
|
||||
[String]$Body
|
||||
)
|
||||
|
||||
$Message = @"
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:m="http://schemas.microsoft.com/exchange/services/2006/messages"
|
||||
xmlns:t="http://schemas.microsoft.com/exchange/services/2006/types"
|
||||
xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
|
||||
|
||||
<soap:Header>
|
||||
<t:ExchangeImpersonation>
|
||||
<t:ConnectingSID>
|
||||
<t:PrimarySmtpAddress>$User</t:PrimarySmtpAddress>
|
||||
</t:ConnectingSID>
|
||||
</t:ExchangeImpersonation>
|
||||
</soap:Header>
|
||||
|
||||
<soap:Body>
|
||||
$Body
|
||||
</soap:Body>
|
||||
</soap:Envelope>
|
||||
"@
|
||||
|
||||
return $Message
|
||||
}
|
||||
|
||||
function Invoke-SOAPRequest {
|
||||
[CmdletBinding()]
|
||||
Param(
|
||||
[Parameter(Mandatory = $True, HelpMessage = "OAuth token to connect to Exchange Online.")]
|
||||
[securestring]$Token,
|
||||
|
||||
[Parameter(Mandatory = $True, HelpMessage = "The message ")]
|
||||
[String]$Message,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "The http method")]
|
||||
[String]$Method = "Post"
|
||||
)
|
||||
|
||||
# EWS service url for Exchange Online
|
||||
$webServiceUrl = "https://outlook.office365.com/EWS/Exchange.asmx"
|
||||
|
||||
$Response = Invoke-WebRequest -Uri $webServiceUrl -Authentication Bearer -Token $Token -Body $Message -Method $Method
|
||||
[xml]$xmlResponse = $Response.Content
|
||||
|
||||
return $xmlResponse
|
||||
}
|
||||
|
||||
function Remove-Folder {
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
Param(
|
||||
[Parameter(Mandatory = $True, HelpMessage = "OAuth token to connect to Exchange Online.")]
|
||||
[Securestring]$Token,
|
||||
|
||||
[Parameter(Mandatory = $True, HelpMessage = "User for which to delete folders")]
|
||||
[String]$User,
|
||||
|
||||
[Parameter(Mandatory = $False, HelpMessage = "Well-known name of folder under which to clean")]
|
||||
[String]$WellKnownRoot = "deleteditems"
|
||||
)
|
||||
|
||||
# SOAP message for getting the folder id
|
||||
$body = @"
|
||||
<FindFolder Traversal="Deep" xmlns="http://schemas.microsoft.com/exchange/services/2006/messages">
|
||||
<FolderShape>
|
||||
<t:BaseShape>Default</t:BaseShape>
|
||||
<t:AdditionalProperties>
|
||||
<t:ExtendedFieldURI PropertyTag="0x3007" PropertyType="SystemTime"/>
|
||||
</t:AdditionalProperties>
|
||||
</FolderShape>
|
||||
<ParentFolderIds>
|
||||
<t:DistinguishedFolderId Id="$WellKnownRoot"/>
|
||||
</ParentFolderIds>
|
||||
</FindFolder>
|
||||
"@
|
||||
|
||||
Write-Host "Looking for folders under well-known folder: $WellKnownRoot & matching folder: $FolderNamePurge$FolderNamePrefixPurge & for user: $User"
|
||||
$getFolderIdMsg = Initialize-SOAPMessage -User $User -Body $body
|
||||
$response = Invoke-SOAPRequest -Token $Token -Message $getFolderIdMsg
|
||||
|
||||
# Get the folders from the response
|
||||
$folders = $response | Select-Xml -XPath "//t:Folders/*" -Namespace @{t = "http://schemas.microsoft.com/exchange/services/2006/types"} |
|
||||
Select-Object -ExpandProperty Node
|
||||
|
||||
$folderId = $null
|
||||
$changeKey = $null
|
||||
$totalCount = $null
|
||||
|
||||
# Loop through folders
|
||||
foreach ($folder in $folders) {
|
||||
$folderId = $folder.FolderId.Id
|
||||
$changeKey = $folder.FolderId.Changekey
|
||||
$totalCount = $folder.TotalCount
|
||||
$folderName = $folder.DisplayName
|
||||
$folderCreateTime = $folder.ExtendedProperty
|
||||
| Where-Object { $_.ExtendedFieldURI.PropertyTag -eq "0x3007" }
|
||||
| Select-Object -ExpandProperty Value
|
||||
| Get-Date
|
||||
|
||||
if ((![String]::IsNullOrEmpty($FolderNamePurge) -and $folderName -ne $FolderNamePurge) -or
|
||||
(![String]::IsNullOrEmpty($FolderPrefixPurge) -and $folderName -notlike "$FolderPrefixPurge*") -or
|
||||
(![String]::IsNullOrEmpty($FolderBeforePurge) -and $folderCreateTime -gt $FolderBeforePurge)) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (![String]::IsNullOrEmpty($FolderNamePurge)) {
|
||||
Write-Host "Found desired folder to purge: $FolderNamePurge"
|
||||
}
|
||||
|
||||
Write-Verbose "Folder Id and ChangeKey for ""$folderName"": $folderId, $changeKey"
|
||||
|
||||
# Empty and delete the folder if found
|
||||
if (![String]::IsNullOrEmpty($folderId) -and ![String]::IsNullOrEmpty($changeKey)) {
|
||||
if ($PSCmdlet.ShouldProcess("$folderName ($totalCount items) created $folderCreateTime", "Emptying folder")) {
|
||||
Write-Host "Emptying folder $folderName ($totalCount items)..."
|
||||
|
||||
# DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
|
||||
$body = @"
|
||||
<m:EmptyFolder DeleteType="HardDelete" DeleteSubFolders="true">
|
||||
<m:FolderIds>
|
||||
<t:FolderId Id="$folderId" ChangeKey="$changeKey" />
|
||||
</m:FolderIds>
|
||||
</m:EmptyFolder>
|
||||
"@
|
||||
$emptyFolderMsg = Initialize-SOAPMessage -User $User -Body $body
|
||||
$response = Invoke-SOAPRequest -Token $Token -Message $emptyFolderMsg
|
||||
}
|
||||
|
||||
if ($PSCmdlet.ShouldProcess($folderName, "Deleting folder")) {
|
||||
Write-Host "Deleting folder $folderName..."
|
||||
|
||||
# DeleteType = HardDelete, MoveToDeletedItems, or SoftDelete
|
||||
$body = @"
|
||||
<m:DeleteFolder DeleteType="HardDelete" DeleteSubFolders="true">
|
||||
<m:FolderIds>
|
||||
<t:FolderId Id="$folderId" ChangeKey="$changeKey" />
|
||||
</m:FolderIds>
|
||||
</m:DeleteFolder>
|
||||
"@
|
||||
$deleteFolderMsg = Initialize-SOAPMessage -User $User -Body $body
|
||||
$response = Invoke-SOAPRequest -Token $Token -Message $deleteFolderMsg
|
||||
}
|
||||
|
||||
Write-Host "Deleted folder $folderName ($totalCount items)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$token = Get-AccessToken | ConvertTo-SecureString -AsPlainText -Force
|
||||
|
||||
Remove-Folder -Token $token -User $User -WellKnownRoot $WellKnownRoot
|
||||
Loading…
x
Reference in New Issue
Block a user