diff --git a/.github/actions/purge-m365-data/action.yml b/.github/actions/purge-m365-data/action.yml index d7681e55b..725fc711f 100644 --- a/.github/actions/purge-m365-data/action.yml +++ b/.github/actions/purge-m365-data/action.yml @@ -19,7 +19,9 @@ inputs: site: description: Sharepoint site where data is to be purged. libraries: - description: List of library names within site where data is to be purged. + description: List of library names within the site where data is to be purged. + library-prefix: + description: List of library names within the site where the library will get deleted entirely. folder-prefix: description: Name of the folder to be purged. If falsy, will purge the set of static, well known folders instead. older-than: @@ -76,7 +78,10 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 \ + -User ${{ inputs.user }} \ + -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} ################################################################################################################ # Sharepoint @@ -90,4 +95,8 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -Site ${{ inputs.site }} -LibraryNameList "${{ inputs.libraries }}".split(",") -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -Site ${{ inputs.site }} \ + -LibraryNameList "${{ inputs.libraries }}".split(",") \ + -FolderPrefixPurgeList ${{ inputs.folder-prefix }} \ + -LibraryPrefixDeleteList ${{ inputs.library-prefix }} \ + -PurgeBeforeTimestamp ${{ inputs.older-than }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 65e678e4b..9f51d3e14 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -62,6 +62,7 @@ jobs: site: ${{ vars[matrix.site] }} folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} + library-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} diff --git a/src/cmd/purge/scripts/onedrivePurge.ps1 b/src/cmd/purge/scripts/onedrivePurge.ps1 index 7372245aa..b4717d35e 100644 --- a/src/cmd/purge/scripts/onedrivePurge.ps1 +++ b/src/cmd/purge/scripts/onedrivePurge.ps1 @@ -19,14 +19,17 @@ Param ( [datetime]$PurgeBeforeTimestamp, [Parameter(Mandatory = $True, HelpMessage = "Purge folders with this prefix")] - [String[]]$FolderPrefixPurgeList + [String[]]$FolderPrefixPurgeList, + + [Parameter(Mandatory = $False, HelpMessage = "Delete document libraries with this prefix")] + [String[]]$LibraryPrefixDeleteList ) Set-StrictMode -Version 2.0 # Attempt to set network timeout to 10min [System.Net.ServicePointManager]::MaxServicePointIdleTime = 600000 -function Get-TimestampFromName { +function Get-TimestampFromFolderName { param ( [Parameter(Mandatory = $True, HelpMessage = "Folder ")] [Microsoft.SharePoint.Client.Folder]$folder @@ -54,6 +57,36 @@ function Get-TimestampFromName { return $timestamp } + +function Get-TimestampFromListName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "List ")] + [Microsoft.SharePoint.Client.List]$list + ) + + $name = $list.Title + + #fallback on list create time + [datetime]$timestamp = $list.LastItemUserModifiedDate + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "List: $name, create timestamp: $timestamp" + + return $timestamp +} + function Purge-Library { [CmdletBinding(SupportsShouldProcess)] Param ( @@ -77,7 +110,7 @@ function Purge-Library { foreach ($f in $folders) { $folderName = $f.Name - $createTime = Get-TimestampFromName -Folder $f + $createTime = Get-TimestampFromFolderName -Folder $f if ($PurgeBeforeTimestamp -gt $createTime) { foreach ($p in $FolderPrefixPurgeList) { @@ -97,7 +130,7 @@ function Purge-Library { if ($f.ServerRelativeUrl -imatch "$SiteSuffix/{0,1}(.+?)/{0,1}$folderName$") { $siteRelativeParentPath = $Matches.1 } - + if ($PSCmdlet.ShouldProcess("Name: " + $f.Name + " Parent: " + $siteRelativeParentPath, "Remove folder")) { Write-Host "Deleting folder: "$f.Name" with parent: $siteRelativeParentPath" try { @@ -110,6 +143,54 @@ function Purge-Library { } } +function Delete-LibraryByPrefix { + [CmdletBinding(SupportsShouldProcess)] + Param ( + [Parameter(Mandatory = $True, HelpMessage = "Document library root")] + [String]$LibraryNamePrefix, + + [Parameter(Mandatory = $True, HelpMessage = "Purge folders before this date time (UTC)")] + [datetime]$PurgeBeforeTimestamp, + + [Parameter(Mandatory = $True, HelpMessage = "Site suffix")] + [String[]]$SiteSuffix + ) + + Write-Host "`nDeleting library: $LibraryNamePrefix" + + $listsToDelete = @() + $lists = Get-PnPList + + foreach ($l in $lists) { + $listName = $l.Title + $createTime = Get-TimestampFromListName -List $l + + if ($PurgeBeforeTimestamp -gt $createTime) { + foreach ($p in $FolderPrefixPurgeList) { + if ($listName -like "$p*") { + $listsToDelete += $l + } + } + } + } + + Write-Host "Found"$listsToDelete.count"lists to delete" + + foreach ($l in $listsToDelete) { + $listName = $l.Title + + if ($PSCmdlet.ShouldProcess("Name: " + $l.Title + "Remove folder")) { + Write-Host "Deleting list: "$l.Title + try { + Remove-PnPList -Identity $l.Id -Force + } + catch [ System.Management.Automation.ItemNotFoundException ] { + Write-Host "List: "$f.Name" is already deleted. Skipping..." + } + } + } +} + ######## MAIN ######### # Setup SharePointPnP @@ -176,4 +257,8 @@ $FolderPrefixPurgeList = $FolderPrefixPurgeList | ForEach-Object { @($_.Split(', foreach ($library in $LibraryNameList) { Purge-Library -LibraryName $library -PurgeBeforeTimestamp $PurgeBeforeTimestamp -FolderPrefixPurgeList $FolderPrefixPurgeList -SiteSuffix $siteSuffix -} \ No newline at end of file +} + +foreach ($libraryPfx in $LibraryPrefixDeleteList) { + Delete-LibraryByPrefix -LibraryNamePrefix $libraryPfx -PurgeBeforeTimestamp $PurgeBeforeTimestamp -SiteSuffix $siteSuffix +}