From b23faa0fac6fcf97ac8eb46ea2642c01c6a05fa9 Mon Sep 17 00:00:00 2001 From: Georgi Matev Date: Thu, 30 Mar 2023 16:31:15 -0700 Subject: [PATCH] CI Cleanup improvements (#2992) * Allow exchange timestamps from item/folder names based with create time as backup * Parametrize folder prefixes and move to var from secrets for ease of management. There is nothing sensitive --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [x] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../actions/purge-m365-user-data/action.yml | 4 +- .github/workflows/ci_test_cleanup.yml | 4 +- src/cmd/purge/scripts/exchangePurge.ps1 | 42 +++++++++++++++++-- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/.github/actions/purge-m365-user-data/action.yml b/.github/actions/purge-m365-user-data/action.yml index 573943c5d..fa246ce82 100644 --- a/.github/actions/purge-m365-user-data/action.yml +++ b/.github/actions/purge-m365-user-data/action.yml @@ -47,7 +47,7 @@ runs: AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }} AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }} run: | - ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./exchangePurge.ps1 -User ${{ inputs.user }} -FolderNamePurgeList PersonMetadata -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Run the OneDrive purge scripts for user if: ${{ inputs.user != '' }} @@ -57,7 +57,7 @@ runs: M365_TENANT_ADMIN_USER: ${{ inputs.m365-admin-user }} M365_TENANT_ADMIN_PASSWORD: ${{ inputs.m365-admin-password }} run: | - ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList ${{ inputs.folder-prefix }} -PurgeBeforeTimestamp ${{ inputs.older-than }} + ./onedrivePurge.ps1 -User ${{ inputs.user }} -FolderPrefixPurgeList "${{ inputs.folder-prefix }}".Split(",") -PurgeBeforeTimestamp ${{ inputs.older-than }} - name: Run SharePoint purge script if: ${{ inputs.user == '' }} diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml index 549692ba2..6b3ebc2e7 100644 --- a/.github/workflows/ci_test_cleanup.yml +++ b/.github/workflows/ci_test_cleanup.yml @@ -29,8 +29,8 @@ jobs: with: user: ${{ secrets[matrix.user] }} site: ${{ secrets.CORSO_M365_TEST_SITE_URL}} - folder-prefix: "Corso_Restore_, TestRestore, testfolder, incrementals_ci_, Alcion_Restore_" - libraries: ${{ secrets.CORSO_M365_TEST_SITE_LIBRARIES }} + folder-prefix: ${{ vars.CORSO_M365_TEST_PREFIXES }} + libraries: ${{ vars.CORSO_M365_TEST_SITE_LIBRARIES }} older-than: ${{ env.HALF_HOUR_AGO }} azure-client-id: ${{ secrets.CLIENT_ID }} azure-client-secret: ${{ secrets.CLIENT_SECRET }} diff --git a/src/cmd/purge/scripts/exchangePurge.ps1 b/src/cmd/purge/scripts/exchangePurge.ps1 index 0233b4c88..63775d90a 100644 --- a/src/cmd/purge/scripts/exchangePurge.ps1 +++ b/src/cmd/purge/scripts/exchangePurge.ps1 @@ -115,13 +115,43 @@ function IsNameMatch { return ($FolderName -in $FolderNamePurgeList) } +function Get-TimestampFromName { + param ( + [Parameter(Mandatory = $True, HelpMessage = "name")] + [String]$name, + + [Parameter(Mandatory = $True, HelpMessage = "Default timestamp if not found in name")] + [datetime]$defaultTimestamp + ) + + #fallback on folder create time + [datetime]$timestamp = $defaultTimestamp + + try { + # Assumes that the timestamp is at the end and starts with yyyy-mm-ddT and is ISO8601 + if ($name -imatch "(\d{4}}-\d{2}-\d{2}T.*)") { + $timestamp = [System.Convert]::ToDatetime($Matches.0) + } + + # Assumes that the timestamp is at the end and starts with dd-MMM-yyyy_HH-MM-SS + if ($name -imatch "(\d{2}-[a-zA-Z]{3}-\d{4}_\d{2}-\d{2}-\d{2})") { + $timestamp = [datetime]::ParseExact($Matches.0, "dd-MMM-yyyy_HH-mm-ss", [CultureInfo]::InvariantCulture, "AssumeUniversal") + } + } + catch {} + + Write-Verbose "Folder: $name, create timestamp: $timestamp" + + return $timestamp +} + function IsPrefixAndAgeMatch { Param( [Parameter(Mandatory = $True, HelpMessage = "Folder name to evaluate for match against a list of targets")] [string]$FolderName, [Parameter(Mandatory = $True, HelpMessage = "Folder creation times")] - [string]$FolderCreateTime, + [datetime]$FolderCreateTime, [Parameter(Mandatory = $True, HelpMessage = "Folder name prefixes to evaluate for match")] [string[]]$FolderPrefixPurgeList, @@ -129,8 +159,10 @@ function IsPrefixAndAgeMatch { [Parameter(Mandatory = $TRUE, HelpMessage = "Purge folders before this date time (UTC)")] [datetime]$PurgeBeforeTimestamp ) - - if ($PurgeBeforeTimestamp -gt $folderCreateTime ) { + + $folderTimestamp = Get-TimestampFromName -name $FolderName -defaultTimestamp $FolderCreateTime + + if ($PurgeBeforeTimestamp -gt $folderTimestamp ) { foreach ($prefix in $FolderPrefixPurgeList) { if ($FolderName -like "$prefix*") { return $true @@ -437,7 +469,9 @@ function Get-ItemsToPurge { continue } - if (![String]::IsNullOrEmpty($PurgeBeforeTimestamp) -and $itemCreateTime -gt $PurgeBeforeTimestamp) { + $itemTimestamp = Get-TimestampFromName -name $itemName -defaultTimestamp $itemCreateTime + + if (![String]::IsNullOrEmpty($PurgeBeforeTimestamp) -and $itemTimestamp -gt $PurgeBeforeTimestamp) { continue }