Adding new OneDrive and Sharepoint CI cleanup (#2870)

New script for OneDrive and Sharepoint cleanup 

Commit by commit review is encouraged for readability since this involves renames of older scripts.


---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [x] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)


#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E

Tested this with `nektos/act `so action logic should be mostly sound. There is one issue with the script installed for exchange that I believe is just a difference the container image used by GHA and the local I could use for `act`

.github/actions/purge-m365-user-data/action.yml

@@ -16,6 +16,8 @@ name: Purge M365 User Data
 inputs:
   user:
     description: User whose data is to be purged.
+  site:
+    description: Sharepoint site where data is to be purged.
   folder-prefix:
     description: Name of the folder to be purged.  If falsy, will purge the set of static, well known folders instead.
   older-than:
@@ -27,14 +29,14 @@ inputs:
   azure-tenant-id:
     description: Secret value of for AZURE_TENANT_ID
   m365-admin-user:
-    description: Secret value of for M365TENANT_ADMIN_USER
+    description: Secret value of for M365_TENANT_ADMIN_USER
   m365-admin-password:
-    description: Secret value of for M365TENANT_ADMIN_PASSWORD
+    description: Secret value of for M365_TENANT_ADMIN_PASSWORD
 
 runs:
   using: composite
   steps:
-    - name: Run the all purge scripts for user
+    - name: Run the all Exchange amd OneDrive purge scripts for user
       if: ${{ inputs.user != '' }}
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
@@ -42,30 +44,28 @@ runs:
         AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
         AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
         AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
+        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
+        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
       run: |
-        ./foldersAndItems.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList Corso_Restore_,TestRestore,testfolder,incrementals_ci_,Alcion_Restore_ -PurgeBeforeTimestamp ${{ inputs.older-than }}
+        ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
+        ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
+
+    - name: Run SharePoint purge script
+      if: ${{ inputs.user == '' }}
+      shell: pwsh
+      working-directory: ./src/cmd/purge/scripts
+      env:
+        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
+        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+      run: |
+        ./onedrivePurge.ps1 -Site ${{ inputs.site }} -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }}
 
     - name: Reset retention for all mailboxes to 0
       if: ${{ inputs.user == '' }}
       shell: pwsh
       working-directory: ./src/cmd/purge/scripts
       env:
-        M365TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
+        M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }}
-        M365TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
+        M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }}
       run: |
-        ./setRetention.ps1
+        ./exchangeRetention.ps1
-
-    - name: Run the old purge script to clear out onedrive buildup
-      working-directory: ./src
-      if: ${{ inputs.folder-prefix != '' && inputs.user != ''}}
-      shell: sh
-      env:
-        AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
-        AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
-        AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
-      run: |
-        go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix Corso_Restore_
-        go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix TestRestore
-        go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix testfolder
-        go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix incrementals_ci_
-        go run ./cmd/purge/purge.go onedrive --user ${{ inputs.user }} --before ${{ inputs.older-than }} --prefix Alcion_Restore_

.github/workflows/ci_test_cleanup.yml

@@ -31,7 +31,8 @@ jobs:
         uses: ./.github/actions/purge-m365-user-data
         with:
           user: ${{ secrets[matrix.user] }}
-          folder-prefix: ${{ matrix.folder }}
+          site: ${{ secrets.CORSO_M365_TEST_SITE_URL}}
+          folder-prefix: "Corso_Restore_, TestRestore, testfolder, incrementals_ci_, Alcion_Restore_"
           older-than: ${{ env.HALF_HOUR_AGO }}
           azure-client-id: ${{ secrets.CLIENT_ID }}
           azure-client-secret: ${{ secrets.CLIENT_SECRET }}

src/cmd/purge/scripts/exchangeRetention.ps1

@@ -4,15 +4,17 @@
 #                     -w /usr/reset-retnention m365pnp/powershell pwsh -c "./setRetention.ps1"
 Param (
     [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")]
-    [String]$AdminUser = $ENV:M365TENANT_ADMIN_USER,
+    [String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER,
 
     [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")]
-    [String]$AdminPwd = $ENV:M365TENANT_ADMIN_PASSWORD
+    [String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD
 )
 
 # Setup ExchangeOnline
 if (-not (Get-Module -ListAvailable -Name ExchangeOnlineManagement)) {
+    $ProgressPreference = 'SilentlyContinue'
     Install-Module -Name ExchangeOnlineManagement -MinimumVersion 3.0.0 -Force
+    $ProgressPreference = 'Continue'
 }
 
 Write-Host "`nConnecting to Exchange..."

src/cmd/purge/scripts/onedrivePurge.ps1

@@ -0,0 +1,172 @@
+[CmdletBinding(SupportsShouldProcess)]
+Param (
+    [Parameter(Mandatory = $False, HelpMessage = "User for which to delete folders in OneDrive")]
+    [String]$User,
+
+    [Parameter(Mandatory = $False, HelpMessage = "Site for which to delete folders in SharePoint")]
+    [String]$Site,
+
+    [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin email")]
+    [String]$AdminUser = $ENV:M365_TENANT_ADMIN_USER,
+
+    [Parameter(Mandatory = $False, HelpMessage = "Exchange Admin password")]
+    [String]$AdminPwd = $ENV:M365_TENANT_ADMIN_PASSWORD,
+
+    [Parameter(Mandatory = $False, HelpMessage = "Document library root. Can add multiple comma-separated values")]
+    [String[]]$LibraryNameList = @(),
+
+    [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")]
+    [datetime]$PurgeBeforeTimestamp,
+
+    [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")]
+    [String[]]$FolderPrefixPurgeList
+)
+
+Set-StrictMode -Version 2.0
+# Attempt to set network timeout to 10min
+[System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000
+
+function Get-TimestampFromName {
+    param (
+        [Parameter(Mandatory = $True, HelpMessage = "Folder ")]
+        [Microsoft.SharePoint.Client.Folder]$folder
+    )
+
+    $name = $folder.Name
+
+    #fallback on folder create time 
+    [datetime]$timestamp = $folder.TimeCreated
+
+    try {
+        # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601
+        if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") {
+            $timestamp = [System.Convert]::ToDatetime($Matches.0)
+        }
+
+        # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS
+        if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") {
+            $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal")
+        }
+    }
+    catch {}
+
+    Write-Verbose "Folder: $name, create timestamp: $timestamp"
+
+    return $timestamp
+}
+function Purge-Library {
+    [CmdletBinding(SupportsShouldProcess)]
+    Param (
+        [Parameter(Mandatory = $True, HelpMessage = "Document library root")]
+        [String]$LibraryName,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")]
+        [datetime]$PurgeBeforeTimestamp,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")]
+        [String[]]$FolderPrefixPurgeList,
+
+        [Parameter(Mandatory = $True, HelpMessage = "Site suffix")]
+        [String[]]$SiteSuffix
+    )
+
+    $foldersToPurge = @()
+    $folders = Get-PnPFolderItem -FolderSiteRelativeUrl $LibraryName -ItemType Folder 
+
+    foreach ($f in $folders) {
+        $folderName = $f.Name
+        $createTime = Get-TimestampFromName -Folder $f
+
+        if ($PurgeBeforeTimestamp -gt $createTime) {
+            foreach ($p in $FolderPrefixPurgeList) {
+                if ($folderName -like "$p*") {
+                    $foldersToPurge += $f
+                }
+            }
+        }
+    }
+
+    Write-Host "Found"$foldersToPurge.count"folders to purge"
+
+    foreach ($f in $foldersToPurge) {
+        $folderName = $f.Name
+        $siteRelativeParentPath = ""
+
+        if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") {
+            $siteRelativeParentPath = $Matches.1
+        }
+ 
+        if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) {
+            Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath"
+            try {
+                Remove-PnPFolder -Name $f.Name -Folder $siteRelativeParentPath -Force
+            }
+            catch [ System.Management.Automation.ItemNotFoundException ] {
+                Write-Host "Folder: "$f.Name" with parent: $siteRelativeParentPath is already deleted. Skipping..."
+            }
+        }
+    }
+}
+
+######## MAIN #########
+
+# Setup SharePointPnP
+if (-not (Get-Module -ListAvailable -Name PnP.PowerShell)) {
+    $ProgressPreference = 'SilentlyContinue'
+    Install-Module -Name PnP.PowerShell -Force
+    $ProgressPreference = 'Continue'
+}
+
+
+if ([string]::IsNullOrEmpty($AdminUser) -or [string]::IsNullOrEmpty($AdminPwd)) {
+    Write-Host "Admin user name and password required as arguments or environment variables."
+    Exit
+}
+
+# Connet to OneDrive or Sharepoint
+$siteUrl = $null
+if (![string]::IsNullOrEmpty($User)) {
+    # Works for dev domains where format is <user name>@<domain>.onmicrosoft.com
+    $domain = $User.Split('@')[1].Split('.')[0]
+    $userNameEscaped = $User.Replace('.', '_').Replace('@', '_')
+    $siteUrl = "https://$domain-my.sharepoint.com/personal/$userNameEscaped/"
+
+    if ($LibraryNameList.count -eq 0) {
+        $LibraryNameList = @("Documents")
+        Write-Host "`nUsing default OneDrive library: $LibraryNameList"
+    }
+}
+elseif (![string]::IsNullOrEmpty($Site)) {
+    $siteUrl = $Site
+
+    if ($LibraryNameList.count -eq 0) {
+        $LibraryNameList = @("Shared Documents")
+        Write-Host "`nUsing default SharePoint library: $LibraryNameList"
+    }
+}
+else {
+    Write-Host "User (for OneDrvie) or Site (for Sharpeoint) is required"
+    Exit
+}
+
+#extract the suffix after the domain
+$siteSiffix = ""
+if ($siteUrl -imatch "^.*?(?<=sharepoint.com)(.*?$)") {
+    $siteSiffix = $Matches.1
+}
+else {
+    Write-Host "Site url appears to be malformed"
+    Exit
+}
+
+
+$password = convertto-securestring -String $AdminPwd -AsPlainText -Force
+$cred = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $AdminUser, $password
+
+Write-Host "`nAuthenticating and connecting to $SiteUrl"
+Connect-PnPOnline -Url $siteUrl -Credential $cred
+Write-Host "Connected to $siteUrl`n"
+
+foreach ($library in $LibraryNameList) {
+    Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSiffix
+}