From 35d89427ce577f9b4ebb229a6c8ec2e0f1c47425 Mon Sep 17 00:00:00 2001 From: Danny Date: Fri, 3 Feb 2023 17:02:50 -0500 Subject: [PATCH] GC: Restore: `event.item` attachment support (#2355) ## Description `Item.Attachments` of OdataType `Event` require special transformations prior to being uploaded. ## Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included ## Type of change - [x] :sunflower: Feature ## Issue(s) *closes #2353 ## Test Plan - [x] :zap: Unit test --- src/internal/connector/exchange/attachment.go | 37 ++-- .../connector/exchange/restore_test.go | 16 +- .../connector/exchange/service_restore.go | 38 ++-- .../mockconnector/mock_data_message.go | 186 ++++++++++++++++++ .../connector/support/m365Transform.go | 90 +++++++++ 5 files changed, 337 insertions(+), 30 deletions(-) diff --git a/src/internal/connector/exchange/attachment.go b/src/internal/connector/exchange/attachment.go index 6ed05b5df..94e6dbc6a 100644 --- a/src/internal/connector/exchange/attachment.go +++ b/src/internal/connector/exchange/attachment.go @@ -8,6 +8,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/uploadsession" "github.com/alcionai/corso/src/pkg/logger" ) @@ -44,8 +45,11 @@ func uploadAttachment( attachment models.Attachmentable, ) error { logger.Ctx(ctx).Debugf("uploading attachment with size %d", *attachment.GetSize()) - attachmentType := attachmentType(attachment) + var ( + attachmentType = attachmentType(attachment) + err error + ) // Reference attachments that are inline() do not need to be recreated. The contents are part of the body. if attachmentType == models.REFERENCE_ATTACHMENTTYPE && attachment.GetIsInline() != nil && *attachment.GetIsInline() { @@ -55,19 +59,26 @@ func uploadAttachment( // item Attachments to be skipped until the completion of Issue #2353 if attachmentType == models.ITEM_ATTACHMENTTYPE { - name := "" - if attachment.GetName() != nil { - name = *attachment.GetName() + prev := attachment + + attachment, err = support.ToItemAttachment(attachment) + if err != nil { + name := "" + if prev.GetName() != nil { + name = *prev.GetName() + } + + // TODO: Update to support PII protection + logger.Ctx(ctx).Infow("item attachment uploads are not supported ", + "err", err, + "attachment_name", name, + "attachment_type", attachmentType, + "internal_item_type", getItemAttachmentItemType(prev), + "attachment_id", *prev.GetId(), + ) + + return nil } - - logger.Ctx(ctx).Infow("item attachment uploads are not supported ", - "attachment_name", name, // TODO: Update to support PII protection - "attachment_type", attachmentType, - "internal_item_type", getItemAttachmentItemType(attachment), - "attachment_id", *attachment.GetId(), - ) - - return nil } // For Item/Reference attachments *or* file attachments < 3MB, use the attachments endpoint diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 187d0c127..360d15266 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -176,11 +176,23 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { }, }, { - name: "Test Mail: Item Attachment", + name: "Test Mail: Item Attachment_Event", bytes: mockconnector.GetMockMessageWithItemAttachmentEvent("Event Item Attachment"), category: path.EmailCategory, destination: func(t *testing.T, ctx context.Context) string { - folderName := "TestRestoreMailItemAttachment: " + common.FormatSimpleDateTime(now) + folderName := "TestRestoreEventItemAttachment: " + common.FormatSimpleDateTime(now) + folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) + require.NoError(t, err) + + return *folder.GetId() + }, + }, + { // Restore will upload the Message without uploading the attachment + name: "Test Mail: Item Attachment_NestedEvent", + bytes: mockconnector.GetMockMessageWithNestedItemAttachmentEvent("Nested Item Attachment"), + category: path.EmailCategory, + destination: func(t *testing.T, ctx context.Context) string { + folderName := "TestRestoreNestedEventItemAttachment: " + common.FormatSimpleDateTime(now) folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index e1144249a..45e2ff1c4 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -189,23 +189,32 @@ func RestoreMailMessage( // 1st: No transmission // 2nd: Send Date // 3rd: Recv Date + svlep := make([]models.SingleValueLegacyExtendedPropertyable, 0) sv1 := models.NewSingleValueLegacyExtendedProperty() sv1.SetId(&valueID) sv1.SetValue(&enableValue) + svlep = append(svlep, sv1) - sv2 := models.NewSingleValueLegacyExtendedProperty() - sendPropertyValue := common.FormatLegacyTime(*clone.GetSentDateTime()) - sendPropertyTag := MailSendDateTimeOverrideProperty - sv2.SetId(&sendPropertyTag) - sv2.SetValue(&sendPropertyValue) + if clone.GetSentDateTime() != nil { + sv2 := models.NewSingleValueLegacyExtendedProperty() + sendPropertyValue := common.FormatLegacyTime(*clone.GetSentDateTime()) + sendPropertyTag := MailSendDateTimeOverrideProperty + sv2.SetId(&sendPropertyTag) + sv2.SetValue(&sendPropertyValue) - sv3 := models.NewSingleValueLegacyExtendedProperty() - recvPropertyValue := common.FormatLegacyTime(*clone.GetReceivedDateTime()) - recvPropertyTag := MailReceiveDateTimeOverriveProperty - sv3.SetId(&recvPropertyTag) - sv3.SetValue(&recvPropertyValue) + svlep = append(svlep, sv2) + } + + if clone.GetReceivedDateTime() != nil { + sv3 := models.NewSingleValueLegacyExtendedProperty() + recvPropertyValue := common.FormatLegacyTime(*clone.GetReceivedDateTime()) + recvPropertyTag := MailReceiveDateTimeOverriveProperty + sv3.SetId(&recvPropertyTag) + sv3.SetValue(&recvPropertyValue) + + svlep = append(svlep, sv3) + } - svlep := []models.SingleValueLegacyExtendedPropertyable{sv1, sv2, sv3} clone.SetSingleValueExtendedProperties(svlep) // Switch workflow based on collision policy @@ -248,10 +257,9 @@ func SendMailToBackStore( errs error ) - if *message.GetHasAttachments() { - attached = message.GetAttachments() - message.SetAttachments([]models.Attachmentable{}) - } + // Item.Attachments --> HasAttachments doesn't always have a value populated when deserialized + attached = message.GetAttachments() + message.SetAttachments([]models.Attachmentable{}) sentMessage, err := service.Client().UsersById(user).MailFoldersById(destination).Messages().Post(ctx, message, nil) if err != nil { diff --git a/src/internal/connector/mockconnector/mock_data_message.go b/src/internal/connector/mockconnector/mock_data_message.go index da06cbaab..4c2e84235 100644 --- a/src/internal/connector/mockconnector/mock_data_message.go +++ b/src/internal/connector/mockconnector/mock_data_message.go @@ -359,3 +359,189 @@ func GetMockMessageWithItemAttachmentEvent(subject string) []byte { return []byte(message) } + +func GetMockMessageWithNestedItemAttachmentEvent(subject string) []byte { + //nolint:lll + // Order of fields: + // 1. subject + // 2. alias + // 3. sender address + // 4. from address + // 5. toRecipients email address + template := `{ + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages(attachments())/$entity", + "@odata.etag": "W/\"CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AADFK782\"", + "id": "AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AADFfThSAAA=", + "createdDateTime": "2023-02-02T21:38:27Z", + "lastModifiedDateTime": "2023-02-02T22:42:49Z", + "changeKey": "CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AADFK782", + "categories": [], + "receivedDateTime": "2023-02-02T21:38:27Z", + "sentDateTime": "2023-02-02T21:38:24Z", + "hasAttachments": true, + "internetMessageId": "", + "subject": "%[1]v", + "bodyPreview": "Dustin,\r\n\r\nI'm here to see if we are still able to discover our object.", + "importance": "normal", + "parentFolderId": "AQMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4ADVkZWQwNmNlMTgALgAAAw_9XBStqZdPuOVIalVTz7sBAHzBhzS2FNNNiXdGkRghSr4AAAIBDAAAAA==", + "conversationId": "AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAB13OyMdkNJJqEaIrGi3Yjc=", + "conversationIndex": "AQHZN06dHXc7Ix2Q0kmoRoisaLdiNw==", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": false, + "isDraft": false, + "webLink": "https://outlook.office365.com/owa/?ItemID=AAMkAGQ1NzTruncated", + "inferenceClassification": "focused", + "body": { + "contentType": "html", + "content": "\r\n
Dustin,

I'm here to see if we are still able to discover our object. 
" + }, + "sender": { + "emailAddress": { + "name": "%[2]s", + "address": "%[3]s" + } + }, + "from": { + "emailAddress": { + "name": "%[2]s", + "address": "%[4]s" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "%[2]s", + "address": "%[5]s" + } + } + ], + "ccRecipients": [], + "bccRecipients": [], + "replyTo": [], + "flag": { + "flagStatus": "notFlagged" + }, + "attachments": [ + { + "@odata.type": "#microsoft.graph.itemAttachment", + "id": "AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AADFfThSAAABEgAQAIyAgT1ZccRCjKKyF7VZ3dA=", + "lastModifiedDateTime": "2023-02-02T21:38:27Z", + "name": "Mail Item Attachment", + "contentType": null, + "size": 5362, + "isInline": false, + "item@odata.associationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages('')/$ref", + "item@odata.navigationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages('')", + "item": { + "@odata.type": "#microsoft.graph.message", + "id": "", + "createdDateTime": "2023-02-02T21:38:27Z", + "lastModifiedDateTime": "2023-02-02T21:38:27Z", + "receivedDateTime": "2023-02-01T13:48:47Z", + "sentDateTime": "2023-02-01T13:48:46Z", + "hasAttachments": true, + "internetMessageId": "", + "subject": "Mail Item Attachment", + "bodyPreview": "Lookingtodothis", + "importance": "normal", + "conversationId": "AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAMNK0NU7Kx5GhAaHdzhfSRU=", + "conversationIndex": "AQHZN02pw0rQ1TsrHkaEBod3OF9JFQ==", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": true, + "isDraft": false, + "webLink": "https://outlook.office365.com/owa/?AttachmentItemID=AAMkAGQ1NzViZTdhLTEwMTM", + "body": { + "contentType": "html", + "content": "\r\nLookingtodothis 
" + }, + "sender": { + "emailAddress": { + "name": "A Stranger", + "address": "foobar@8qzvrj.onmicrosoft.com" + } + }, + "from": { + "emailAddress": { + "name": "A Stranger", + "address": "foobar@8qzvrj.onmicrosoft.com" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "Direct Report", + "address": "notAvailable@8qzvrj.onmicrosoft.com" + } + } + ], + "flag": { + "flagStatus": "notFlagged" + }, + "attachments": [ + { + "@odata.type": "#microsoft.graph.itemAttachment", + "id": "AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AADFfThSAAACEgAQAIyAgT1ZccRCjKKyF7VZ3dASABAAuYCb3N2YZ02RpJrZPzCBFQ==", + "lastModifiedDateTime": "2023-02-02T21:38:27Z", + "name": "Holidayevent", + "contentType": null, + "size": 2331, + "isInline": false, + "item@odata.associationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/events('')/$ref", + "item@odata.navigationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/events('')", + "item": { + "@odata.type": "#microsoft.graph.event", + "id": "", + "createdDateTime": "2023-02-02T21:38:27Z", + "lastModifiedDateTime": "2023-02-02T21:38:27Z", + "originalStartTimeZone": "tzone://Microsoft/Utc", + "originalEndTimeZone": "tzone://Microsoft/Utc", + "reminderMinutesBeforeStart": 0, + "isReminderOn": false, + "hasAttachments": false, + "subject": "Discuss Gifts for Children", + "isAllDay": false, + "isCancelled": false, + "isOrganizer": true, + "responseRequested": true, + "type": "singleInstance", + "isOnlineMeeting": false, + "isDraft": true, + "body": { + "contentType": "html", + "content": "\r\nLet'slookforfunding! " + }, + "start": { + "dateTime": "2016-12-02T18:00:00.0000000Z", + "timeZone": "UTC" + }, + "end": { + "dateTime": "2016-12-02T19:00:00.0000000Z", + "timeZone": "UTC" + }, + "organizer": { + "emailAddress": { + "name": "Event Manager", + "address": "philonis@8qzvrj.onmicrosoft.com" + } + } + } + } + ] + } + } + ] + }` + + message := fmt.Sprintf( + template, + subject, + defaultAlias, + defaultMessageSender, + defaultMessageFrom, + defaultMessageTo, + ) + + return []byte(message) +} diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 651689430..7fa207c9e 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -1,11 +1,14 @@ package support import ( + "fmt" "strings" "github.com/microsoftgraph/msgraph-sdk-go/models" ) +const itemAttachment = "#microsoft.graph.itemAttachment" + // CloneMessageableFields places data from original data into new message object. // SingleLegacyValueProperty is not populated during this operation func CloneMessageableFields(orig, message models.Messageable) models.Messageable { @@ -278,3 +281,90 @@ func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDe return newColumn } + +// ToItemAttachment transforms internal item, OutlookItemables, into +// objects that are able to be uploaded into M365. +// Supported Internal Items: +// - Events +func ToItemAttachment(orig models.Attachmentable) (models.Attachmentable, error) { + transform, ok := orig.(models.ItemAttachmentable) + supported := "#microsoft.graph.event" + + if !ok { // Shouldn't ever happen + return nil, fmt.Errorf("transforming attachment to item attachment") + } + + item := transform.GetItem() + itemType := item.GetOdataType() + + switch *itemType { + case supported: + event := item.(models.Eventable) + + newEvent, err := sanitizeEvent(event) + if err != nil { + return nil, err + } + + transform.SetItem(newEvent) + + return transform, nil + default: + return nil, fmt.Errorf("exiting ToItemAttachment: %s not supported", *itemType) + } +} + +// sanitizeEvent transfers data into event object and +// removes unique IDs from the M365 object +func sanitizeEvent(orig models.Eventable) (models.Eventable, error) { + newEvent := models.NewEvent() + newEvent.SetAttendees(orig.GetAttendees()) + newEvent.SetBody(orig.GetBody()) + newEvent.SetBodyPreview(orig.GetBodyPreview()) + newEvent.SetCalendar(orig.GetCalendar()) + newEvent.SetCreatedDateTime(orig.GetCreatedDateTime()) + newEvent.SetEnd(orig.GetEnd()) + newEvent.SetHasAttachments(orig.GetHasAttachments()) + newEvent.SetHideAttendees(orig.GetHideAttendees()) + newEvent.SetImportance(orig.GetImportance()) + newEvent.SetIsAllDay(orig.GetIsAllDay()) + newEvent.SetIsOnlineMeeting(orig.GetIsOnlineMeeting()) + newEvent.SetLocation(orig.GetLocation()) + newEvent.SetLocations(orig.GetLocations()) + newEvent.SetSensitivity(orig.GetSensitivity()) + newEvent.SetReminderMinutesBeforeStart(orig.GetReminderMinutesBeforeStart()) + newEvent.SetStart(orig.GetStart()) + newEvent.SetSubject(orig.GetSubject()) + newEvent.SetType(orig.GetType()) + + // Sanitation + // isDraft and isOrganizer *bool ptr's have to be removed completely + // from JSON in order for POST method to succeed. + // Current as of 2/2/2023 + + newEvent.SetIsOrganizer(nil) + newEvent.SetIsDraft(nil) + newEvent.SetAdditionalData(orig.GetAdditionalData()) + + attached := orig.GetAttachments() + attachments := make([]models.Attachmentable, len(attached)) + + for _, ax := range attached { + if *ax.GetOdataType() == itemAttachment { + newAttachment, err := ToItemAttachment(ax) + if err != nil { + return nil, err + } + + attachments = append(attachments, newAttachment) + + continue + } + + attachments = append(attachments, ax) + } + + newEvent.SetAttachments(attachments) + + return newEvent, nil +}