From 6c989dbd780fc4999bc0d720a56f77b861903db7 Mon Sep 17 00:00:00 2001 From: Georgi Matev Date: Mon, 20 Mar 2023 13:36:27 -0700 Subject: [PATCH] Adding new OneDrive and Sharepoint CI cleanup (#2870) New script for OneDrive and Sharepoint cleanup Commit by commit review is encouraged for readability since this involves renames of older scripts. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E Tested this with `nektos/act `so action logic should be mostly sound. There is one issue with the script installed for exchange that I believe is just a difference the container image used by GHA and the local I could use for `act` --- .../actions/purge-m365-user-data/action.yml | 44 ++--- .github/workflows/ci_test_cleanup.yml | 3 +- ...{foldersAndItems.ps1 => exchangePurge.ps1} | 0 ...setRetention.ps1 => exchangeRetention.ps1} | 6 +- src/cmd/purge/scripts/onedrivePurge.ps1 | 172 ++++++++++++++++++ 5 files changed, 200 insertions(+), 25 deletions(-) rename src/cmd/purge/scripts/{foldersAndItems.ps1 => exchangePurge.ps1} (100%) rename src/cmd/purge/scripts/{setRetention.ps1 => exchangeRetention.ps1} (87%) mode change 100644 => 100755 create mode 100644 src/cmd/purge/scripts/onedrivePurge.ps1 diff --git a/.github/actions/purge-m365-user-data/action.yml b/.github/actions/purge-m365-user-data/action.yml index 21ed51f50..822d62bc2 100644 --- a/.github/actions/purge-m365-user-data/action.yml +++ b/.github/actions/purge-m365-user-data/action.yml @@ -16,6 +16,8 @@ name: Purge M365 User Data inputs: user: description: User whose data is to be purged. + site: + description: Sharepoint site where data is to be purged. folder-prefix: description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: @@ -27,14 +29,14 @@ inputs: azure-tenant-id: description: Secret value of for AZURE_TENANT_ID m365-admin-user: - description: Secret value of for M365TENANT_ADMIN_USER + description: Secret value of for M365_TENANT_ADMIN_USER m365-admin-password: - description: Secret value of for M365TENANT_ADMIN_PASSWORD + description: Secret value of for M365_TENANT_ADMIN_PASSWORD runs: using: composite steps: - - name: Run the all purge scripts for user + - name: Run the all Exchange amd OneDrive purge scripts for user if: ${{ inputs.user != '' }} shell: pwsh working-directory: ./src/cmd/purge/scripts @@ -42,30 +44,28 @@ runs: AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} + M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} + M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./foldersAndItems.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList Corso_Restore_,TestRestore,testfolder,incrementals_ci_,Alcion_Restore_ -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + + - name: Run SharePoint purge script + if: ${{ inputs.user == '' }} + shell: pwsh + working-directory: ./src/cmd/purge/scripts + env: + M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} + M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} + run: | + ./onedrivePurge.ps1 -Site ${{ inputs.site }} -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Reset retention for all mailboxes to 0 if: ${{ inputs.user == '' }} shell: pwsh working-directory: ./src/cmd/purge/scripts env: - M365TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} - M365TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} + M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} + M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./setRetention.ps1 - - - name: Run the old purge script to clear out onedrive buildup - working-directory: ./src - if: ${{ inputs.folder-prefix != '' && inputs.user != ''}} - shell: sh - env: - AZURE_CLIENT_ID: ${{ inputs.azure-client-id }} - AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} - AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} - run: | - go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix Corso_Restore_ - go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix TestRestore - go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix testfolder - go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix incrementals_ci_ - go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix Alcion_Restore_ + ./exchangeRetention.ps1 diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 279583008..7b85bd5d7 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -31,7 +31,8 @@ jobs: uses: ./.github/actions/purge-m365-user-data with: user: ${{ secrets[matrix.user] }} - folder-prefix: ${{ matrix.folder }} + site: ${{ secrets.CORSO_M365_TEST_SITE_URL}} + folder-prefix: "Corso_Restore_, TestRestore, testfolder, incrementals_ci_, Alcion_Restore_" older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} diff --git a/src/cmd/purge/scripts/foldersAndItems.ps1 b/src/cmd/purge/scripts/exchangePurge.ps1 similarity index 100% rename from src/cmd/purge/scripts/foldersAndItems.ps1 rename to src/cmd/purge/scripts/exchangePurge.ps1 diff --git a/src/cmd/purge/scripts/setRetention.ps1 b/src/cmd/purge/scripts/exchangeRetention.ps1 old mode 100644 new mode 100755 similarity index 87% rename from src/cmd/purge/scripts/setRetention.ps1 rename to src/cmd/purge/scripts/exchangeRetention.ps1 index 752652879..d093472cc --- a/src/cmd/purge/scripts/setRetention.ps1 +++ b/src/cmd/purge/scripts/exchangeRetention.ps1 @@ -4,15 +4,17 @@ # -w /usr/reset-retnention m365pnp/powershell pwsh -c "./setRetention.ps1" Param ( [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")] - [String]$AdminUser = $ENV:M365TENANT_ADMIN_USER, + [String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER, [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")] - [String]$AdminPwd = $ENV:M365TENANT_ADMIN_PASSWORD + [String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD ) # Setup ExchangeOnline if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) { + $ProgressPreference = 'SilentlyContinue' Install-Module -Name ExchangeOnlineManagement -MinimumVersion 3.0.0 -Force + $ProgressPreference = 'Continue' } Write-Host "`nConnecting to Exchange..." diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 new file mode 100644 index 000000000..0389f8857 --- /dev/null +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -0,0 +1,172 @@ +[CmdletBinding(SupportsShouldProcess)] +Param ( + [Parameter(Mandatory = $False, HelpMessage = "User for which to delete folders in OneDrive")] + [String]$User, + + [Parameter(Mandatory = $False, HelpMessage = "Site for which to delete folders in SharePoint")] + [String]$Site, + + [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")] + [String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER, + + [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")] + [String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD, + + [Parameter(Mandatory = $False, HelpMessage = "Document library root. Can add multiple comma-separated values")] + [String[]]$LibraryNameList = @(), + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] + [String[]]$FolderPrefixPurgeList +) + +Set-StrictMode -Version 2.0 +# Attempt to set network timeout to 10min +[System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000 + +function Get-TimestampFromName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "Folder ")] + [Microsoft.SharePoint.Client.Folder]$folder + ) + + $name = $folder.Name + + #fallback on folder create time + [datetime]$timestamp = $folder.TimeCreated + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "Folder: $name, create timestamp: $timestamp" + + return $timestamp +} +function Purge-Library { + [CmdletBinding(SupportsShouldProcess)] + Param ( + [Parameter(Mandatory = $True, HelpMessage = "Document library root")] + [String]$LibraryName, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] + [String[]]$FolderPrefixPurgeList, + + [Parameter(Mandatory = $True, HelpMessage = "Site suffix")] + [String[]]$SiteSuffix + ) + + $foldersToPurge = @() + $folders = Get-PnPFolderItem -FolderSiteRelativeUrl $LibraryName -ItemType Folder + + foreach ($f in $folders) { + $folderName = $f.Name + $createTime = Get-TimestampFromName -Folder $f + + if ($PurgeBeforeTimestamp -gt $createTime) { + foreach ($p in $FolderPrefixPurgeList) { + if ($folderName -like "$p*") { + $foldersToPurge += $f + } + } + } + } + + Write-Host "Found"$foldersToPurge.count"folders to purge" + + foreach ($f in $foldersToPurge) { + $folderName = $f.Name + $siteRelativeParentPath = "" + + if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") { + $siteRelativeParentPath = $Matches.1 + } + + if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) { + Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath" + try { + Remove-PnPFolder -Name $f.Name -Folder $siteRelativeParentPath -Force + } + catch [ System.Management.Automation.ItemNotFoundException ] { + Write-Host "Folder: "$f.Name" with parent: $siteRelativeParentPath is already deleted. Skipping..." + } + } + } +} + +######## MAIN ######### + +# Setup SharePointPnP +if (-not (Get-Module -ListAvailable -Name PnP.PowerShell)) { + $ProgressPreference = 'SilentlyContinue' + Install-Module -Name PnP.PowerShell -Force + $ProgressPreference = 'Continue' +} + + +if ([string]::IsNullOrEmpty($AdminUser) -or [string]::IsNullOrEmpty($AdminPwd)) { + Write-Host "Admin user name and password required as arguments or environment variables." + Exit +} + +# Connet to OneDrive or Sharepoint +$siteUrl = $null +if (![string]::IsNullOrEmpty($User)) { + # Works for dev domains where format is @.onmicrosoft.com + $domain = $User.Split('@')[1].Split('.')[0] + $userNameEscaped = $User.Replace('.', '_').Replace('@', '_') + $siteUrl = "https://$domain-my.sharepoint.com/personal/$userNameEscaped/" + + if ($LibraryNameList.count -eq 0) { + $LibraryNameList = @("Documents") + Write-Host "`nUsing default OneDrive library: $LibraryNameList" + } +} +elseif (![string]::IsNullOrEmpty($Site)) { + $siteUrl = $Site + + if ($LibraryNameList.count -eq 0) { + $LibraryNameList = @("Shared Documents") + Write-Host "`nUsing default SharePoint library: $LibraryNameList" + } +} +else { + Write-Host "User (for OneDrvie) or Site (for Sharpeoint) is required" + Exit +} + +#extract the suffix after the domain +$siteSiffix = "" +if ($siteUrl -imatch "^.*?(?<=sharepoint.com)(.*?$)") { + $siteSiffix = $Matches.1 +} +else { + Write-Host "Site url appears to be malformed" + Exit +} + + +$password = convertto-securestring -String $AdminPwd -AsPlainText -Force +$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $AdminUser, $password + +Write-Host "`nAuthenticating and connecting to $SiteUrl" +Connect-PnPOnline -Url $siteUrl -Credential $cred +Write-Host "Connected to $siteUrl`n" + +foreach ($library in $LibraryNameList) { + Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSiffix +} \ No newline at end of file