From 20aa3d3a289be608b9deb96d262eda037678c0d3 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 5 Aug 2022 12:24:58 -0600 Subject: [PATCH] purge test user folders every half hour (#491) --- .github/workflows/ci.yml | 2 +- .github/workflows/ci_test_cleanup.yml | 58 +++++++++++++++++++ .github/workflows/docgen.yml | 4 +- src/internal/common/time.go | 11 +++- .../connector/exchange/service_functions.go | 4 +- .../mockconnector/mock_data_collection.go | 2 +- 6 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/ci_test_cleanup.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9fb8f111..e8799f6bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ permissions: contents: read jobs: - Run-All: + Test-Suite: environment: Testing runs-on: ubuntu-latest defaults: diff --git a/.github/workflows/ci_test_cleanup.yml b/.github/workflows/ci_test_cleanup.yml new file mode 100644 index 000000000..0429da50d --- /dev/null +++ b/.github/workflows/ci_test_cleanup.yml @@ -0,0 +1,58 @@ +name: CI Test Cleanup +on: + schedule: + # every half hour + - cron: "*/30 * * * *" + +jobs: + Test-User-Data-Cleanup: + runs-on: ubuntu-latest + + # check out the repo + - name: Repo Code Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # Get values for cache paths to be used in later steps + - id: go-cache-paths + working-directory: ./src + run: | + echo "::set-output name=go-build::$(go env GOCACHE)" + echo "::set-output name=go-mod::$(go env GOMODCACHE)" + - name: Golang Setup + uses: actions/setup-go@v3 + with: + go-version: 1.18 + cache: true + cache-dependency-path: src/go.sum + + # download packages + - name: Cache Go Mod + uses: actions/cache@v3 + id: cache + with: + path: ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + + - name: Run go mod download + if: steps.cache.outputs.cache-hit != 'true' + working-directory: ./src + run: go mod download + + # sets the maximimum time to now-30m. + # CI test have a 10 minute timeout. + # At 20 minutes ago, we should be safe from conflicts. + # The additional 10 minutes is just to be good citizens. + - name: Set purge boundary + run: | + echo "HALF_HOUR_AGO=${date -u -v-30M --iso-8601=seconds}" >> $GITHUB_ENV + + # run the folder purge + - name: Purge folders + working-directory: ./src + run: | + go run ./cmd/purge/purge.go \ + --user ${{ secrets.CORSO_M356_TEST_USER_ID }} \ + --prefix "Corso_Restore_" \ + --before ${{ HALF_HOUR_AGO }} diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index fea56188b..ff0a4c1e2 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -4,7 +4,7 @@ on: branches: [main] jobs: - Run-All: + Generate-Markdown: environment: Testing runs-on: ubuntu-latest steps: @@ -54,7 +54,7 @@ jobs: mkdir -p ./docs/docs/cli mv ./src/cmd/mdgen/cli_markdown/* ./docs/docs/cli/ rm -R ./src/cmd/mdgen/cli_markdown/ - + # make a commit for any file changes - name: Commit the Auto-Generated Files uses: stefanzweifel/git-auto-commit-action@v4 diff --git a/src/internal/common/time.go b/src/internal/common/time.go index 091e9fd63..aa6777f68 100644 --- a/src/internal/common/time.go +++ b/src/internal/common/time.go @@ -6,13 +6,20 @@ import ( ) const ( + StandardTimeFormat = time.RFC3339Nano SimpleDateTimeFormat = "02-Jan-2006_15:04:05" ) +// FormatNow produces the current time in UTC using the provided +// time format. +func FormatNow(fmt string) string { + return time.Now().UTC().Format(fmt) +} + // FormatTime produces the standard format for corso time values. // Always formats into the UTC timezone. func FormatTime(t time.Time) string { - return t.UTC().Format(time.RFC3339Nano) + return t.UTC().Format(StandardTimeFormat) } // FormatSimpleDateTime produces a simple datetime of the format @@ -27,7 +34,7 @@ func ParseTime(s string) (time.Time, error) { if len(s) == 0 { return time.Time{}, errors.New("cannot interpret an empty string as time.Time") } - t, err := time.Parse(time.RFC3339Nano, s) + t, err := time.Parse(StandardTimeFormat, s) if err == nil { return t.UTC(), nil } diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 5e4a38237..b6150598c 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" absser "github.com/microsoft/kiota-abstractions-go/serialization" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" @@ -200,8 +199,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) ( // GetCopyRestoreFolder utility function to create an unique folder for the restore process func GetCopyRestoreFolder(service graph.Service, user string) (*string, error) { - now := time.Now().UTC() - newFolder := fmt.Sprintf("Corso_Restore_%s", common.FormatSimpleDateTime(now)) + newFolder := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat)) isFolder, err := GetMailFolderID(service, newFolder, user) if err != nil { // Verify unique folder was not found diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index 9326a571d..441dbe10f 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -95,7 +95,7 @@ func GetMockMessageBytes(subject string) []byte { } message := "{\n \"@odata.etag\": \"W/\\\"CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AAAZ0f0I\\\"\",\n \"id\": \"AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AAAZ3wG3AAA=\",\n \"createdDateTime\": \"2022-04-08T18:08:02Z\",\n \"lastModifiedDateTime\": \"2022-05-17T13:46:55Z\",\n \"changeKey\": \"CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AAAZ0f0I\",\n \"categories\": [],\n \"receivedDateTime\": \"2022-04-08T18:08:02Z\",\n \"sentDateTime\": \"2022-04-08T18:07:53Z\",\n \"hasAttachments\": false,\n \"internetMessageId\": \"\",\n \"subject\": \"" + - subject + " " + common.FormatSimpleDateTime(time.Now()) + " Different\",\n \"bodyPreview\": \"Who is coming to next week's party? I cannot imagine it is July soon\",\n \"importance\": \"normal\",\n \"parentFolderId\": \"AQMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4ADVkZWQwNmNlMTgALgAAAw_9XBStqZdPuOVIalVTz7sBAHzBhzS2FNNNiXdGkRghSr4AAAIBDAAAAA==\",\n \"conversationId\": \"AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAI7SSzmEPaRJsY-TWIALn1g=\",\n \"conversationIndex\": \"AQHYS3N3jtJLOYQ9pEmxj9NYgAufWA==\",\n \"isDeliveryReceiptRequested\": null,\n \"isReadReceiptRequested\": false,\n \"isRead\": true,\n \"isDraft\": false,\n \"webLink\": \"https://outlook.office365.com/owa/?ItemID=AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8%2B7BwB8wYc0thTTTYl3RpEYIUq%2BAAAAAAEMAAB8wYc0thTTTYl3RpEYIUq%2BAAAZ3wG3AAA%3D&exvsurl=1&viewmodel=ReadMessageItem\",\n \"inferenceClassification\": \"focused\",\n \"body\": {\n \"contentType\": \"html\",\n \"content\": \"

I've been going through with the changing of messages. It shouldn't have the same calls, right? Call Me?

 

We want to be able to send multiple messages and we want to be able to respond and do other things that make sense for our users. In this case. Let’s consider a Mailbox

\"\n },\n \"sender\": {\n \"emailAddress\": {\n \"name\": \"Lidia Holloway\",\n \"address\": \"" + userID + "\"\n }\n },\n \"from\": {\n \"emailAddress\": {\n \"name\": \"Lidia Holloway\",\n \"address\": \"lidiah@8qzvrj.onmicrosoft.com\"\n }\n },\n \"toRecipients\": [\n {\n \"emailAddress\": {\n \"name\": \"Dustin Abbot\",\n \"address\": \"dustina@8qzvrj.onmicrosoft.com\"\n }\n }\n ],\n \"ccRecipients\": [],\n \"bccRecipients\": [],\n \"replyTo\": [],\n \"flag\": {\n \"flagStatus\": \"notFlagged\"\n }\n}\n" + subject + " " + common.FormatNow(common.SimpleDateTimeFormat) + " Different\",\n \"bodyPreview\": \"Who is coming to next week's party? I cannot imagine it is July soon\",\n \"importance\": \"normal\",\n \"parentFolderId\": \"AQMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4ADVkZWQwNmNlMTgALgAAAw_9XBStqZdPuOVIalVTz7sBAHzBhzS2FNNNiXdGkRghSr4AAAIBDAAAAA==\",\n \"conversationId\": \"AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAI7SSzmEPaRJsY-TWIALn1g=\",\n \"conversationIndex\": \"AQHYS3N3jtJLOYQ9pEmxj9NYgAufWA==\",\n \"isDeliveryReceiptRequested\": null,\n \"isReadReceiptRequested\": false,\n \"isRead\": true,\n \"isDraft\": false,\n \"webLink\": \"https://outlook.office365.com/owa/?ItemID=AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8%2B7BwB8wYc0thTTTYl3RpEYIUq%2BAAAAAAEMAAB8wYc0thTTTYl3RpEYIUq%2BAAAZ3wG3AAA%3D&exvsurl=1&viewmodel=ReadMessageItem\",\n \"inferenceClassification\": \"focused\",\n \"body\": {\n \"contentType\": \"html\",\n \"content\": \"

I've been going through with the changing of messages. It shouldn't have the same calls, right? Call Me?

 

We want to be able to send multiple messages and we want to be able to respond and do other things that make sense for our users. In this case. Let’s consider a Mailbox

\"\n },\n \"sender\": {\n \"emailAddress\": {\n \"name\": \"Lidia Holloway\",\n \"address\": \"" + userID + "\"\n }\n },\n \"from\": {\n \"emailAddress\": {\n \"name\": \"Lidia Holloway\",\n \"address\": \"lidiah@8qzvrj.onmicrosoft.com\"\n }\n },\n \"toRecipients\": [\n {\n \"emailAddress\": {\n \"name\": \"Dustin Abbot\",\n \"address\": \"dustina@8qzvrj.onmicrosoft.com\"\n }\n }\n ],\n \"ccRecipients\": [],\n \"bccRecipients\": [],\n \"replyTo\": [],\n \"flag\": {\n \"flagStatus\": \"notFlagged\"\n }\n}\n" return []byte(message) }