From a4b50a1ec05cd44bb361a861b0203dd8dc9a1c92 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 8 Feb 2023 08:58:51 -0500 Subject: [PATCH] GC: Item mail attachment Handling (#2422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Support for `itemAttachment.Mail` added to GC restore pipeline. Nested attachments within items disabled due to Kiota bug. Issue #2428 created to re-enable `itemAttachment.Item.Attachments` when the bug is patched. ## Does this PR need a docs update or release note? - [x] 🏢 : Yes. Known issues and ChangeLog updates required. ## Type of change - [x] :sunflower: Feature - [x] :bug: Bugfix ## Issue(s) * related to https://github.com/microsoft/kiota-serialization-json-go/issues/61 * closes #2372 ## Test Plan - [x] :zap: Unit test --- CHANGELOG.md | 9 + src/internal/connector/exchange/attachment.go | 5 +- .../connector/exchange/restore_test.go | 50 ++++- .../connector/exchange/service_restore.go | 14 ++ .../mockconnector/mock_data_message.go | 186 ++++++++++++++++++ .../connector/support/m365Transform.go | 123 +++++++++--- website/docs/support/known-issues.md | 2 + 7 files changed, 357 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d75a352..cccc745a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] (alpha) +### Added + +### Fixed +- Support for item.Attachment:Mail restore + +### Changed + +### Known Issues +- Nested attachments are currently not restored due to an [issue](https://github.com/microsoft/kiota-serialization-json-go/issues/61) discovered in the Graph APIs ## [v0.3.0] (alpha) - 2023-2-07 diff --git a/src/internal/connector/exchange/attachment.go b/src/internal/connector/exchange/attachment.go index 94e6dbc6a..075ab09a6 100644 --- a/src/internal/connector/exchange/attachment.go +++ b/src/internal/connector/exchange/attachment.go @@ -68,8 +68,9 @@ func uploadAttachment( name = *prev.GetName() } - // TODO: Update to support PII protection - logger.Ctx(ctx).Infow("item attachment uploads are not supported ", + // TODO: (rkeepers) Update to support PII protection + msg := "item attachment restore not supported for this type. skipping upload." + logger.Ctx(ctx).Infow(msg, "err", err, "attachment_name", name, "attachment_type", attachmentType, diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 360d15266..e6db75129 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -130,12 +130,13 @@ type containerDeleter interface { // TestRestoreExchangeObject verifies path.Category usage for restored objects func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { - a := tester.NewM365Account(suite.T()) + t := suite.T() + a := tester.NewM365Account(t) m365, err := a.M365Config() - require.NoError(suite.T(), err) + require.NoError(t, err) service, err := createService(m365) - require.NoError(suite.T(), err) + require.NoError(t, err) deleters := map[path.CategoryType]containerDeleter{ path.EmailCategory: suite.ac.Mail(), @@ -187,6 +188,48 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { return *folder.GetId() }, }, + { + name: "Test Mail: Item Attachment_Mail", + bytes: mockconnector.GetMockMessageWithItemAttachmentMail("Mail Item Attachment"), + category: path.EmailCategory, + destination: func(t *testing.T, ctx context.Context) string { + folderName := "TestRestoreMailItemAttachment: " + common.FormatSimpleDateTime(now) + folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) + require.NoError(t, err) + + return *folder.GetId() + }, + }, + { + name: "Test Mail: Hydrated Item Attachment Mail", + bytes: mockconnector.GetMockMessageWithNestedItemAttachmentMail(t, + mockconnector.GetMockMessageBytes("Basic Item Attachment"), + "Mail Item Attachment", + ), + category: path.EmailCategory, + destination: func(t *testing.T, ctx context.Context) string { + folderName := "TestRestoreMailBasicItemAttachment: " + common.FormatSimpleDateTime(now) + folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) + require.NoError(t, err) + + return *folder.GetId() + }, + }, + { + name: "Test Mail: Hydrated Item Attachment Mail One Attach", + bytes: mockconnector.GetMockMessageWithNestedItemAttachmentMail(t, + mockconnector.GetMockMessageWithDirectAttachment("Item Attachment Included"), + "Mail Item Attachment", + ), + category: path.EmailCategory, + destination: func(t *testing.T, ctx context.Context) string { + folderName := "ItemMailAttachmentwAttachment " + common.FormatSimpleDateTime(now) + folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName) + require.NoError(t, err) + + return *folder.GetId() + }, + }, { // Restore will upload the Message without uploading the attachment name: "Test Mail: Item Attachment_NestedEvent", bytes: mockconnector.GetMockMessageWithNestedItemAttachmentEvent("Nested Item Attachment"), @@ -291,6 +334,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { ) assert.NoError(t, err, support.ConnectorStackErrorTrace(err)) assert.NotNil(t, info, "item info was not populated") + assert.NotNil(t, deleters) assert.NoError(t, deleters[test.category].DeleteContainer(ctx, userID, destination)) }) } diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 6da25450d..bb0179c76 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -283,6 +283,20 @@ func SendMailToBackStore( for _, attachment := range attached { if err := uploadAttachment(ctx, uploader, attachment); err != nil { + if attachment.GetOdataType() != nil && + *attachment.GetOdataType() == "#microsoft.graph.itemAttachment" { + var name string + if attachment.GetName() != nil { + name = *attachment.GetName() + } + + logger.Ctx(ctx).Infow( + "item attachment upload not successful. content not accepted by M365 server", + "Attachment Name", name) + + continue + } + errs = support.WrapAndAppend( fmt.Sprintf("uploading attachment for message %s: %s", id, support.ConnectorStackErrorTrace(err)), diff --git a/src/internal/connector/mockconnector/mock_data_message.go b/src/internal/connector/mockconnector/mock_data_message.go index 4c2e84235..50ff3345c 100644 --- a/src/internal/connector/mockconnector/mock_data_message.go +++ b/src/internal/connector/mockconnector/mock_data_message.go @@ -3,6 +3,12 @@ package mockconnector import ( "encoding/base64" "fmt" + "testing" + + js "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/common" ) @@ -360,6 +366,143 @@ func GetMockMessageWithItemAttachmentEvent(subject string) []byte { return []byte(message) } +func GetMockMessageWithItemAttachmentMail(subject string) []byte { + //nolint:lll + // Order of fields: + // 1. subject + // 2. alias + // 3. sender address + // 4. from address + // 5. toRecipients email address + template := `{ + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages(attachments())/$entity", + "@odata.etag": "W/\"CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AADKTqr3\"", + "id": "AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AADKo35SAAA=", + "createdDateTime": "2023-02-06T20:03:40Z", + "lastModifiedDateTime": "2023-02-06T20:03:42Z", + "changeKey": "CQAAABYAAAB8wYc0thTTTYl3RpEYIUq+AADKTqr3", + "categories": [], + "receivedDateTime": "2023-02-06T20:03:40Z", + "sentDateTime": "2023-02-06T20:03:37Z", + "hasAttachments": true, + "internetMessageId": "", + "subject": "%[1]s", + "bodyPreview": "Nested Items are not encapsulated in a trivial manner. Review the findings.\r\n\r\nBest,\r\n\r\nYour Test Case", + "importance": "normal", + "parentFolderId": "AQMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4ADVkZWQwNmNlMTgALgAAAw_9XBStqZdPuOVIalVTz7sBAHzBhzS2FNNNiXdGkRghSr4AAAIBDAAAAA==", + "conversationId": "AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAPe8pEQOrBxLvFNhfDtMyEI=", + "conversationIndex": "AQHZOmYA97ykRA6sHEu8U2F8O0zIQg==", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": false, + "isDraft": false, + "webLink": "https://outlook.office365.com/owal=ReadMessageItem", + "inferenceClassification": "focused", + "body": { + "contentType": "html", + "content": "\r\n
Nested Items are not encapsulated in a trivial manner. Review the findings.

Best, 

Your Test Case
" + }, + "sender": { + "emailAddress": { + "name": "%[2]s", + "address": "%[3]s" + } + }, + "from": { + "emailAddress": { + "name": "%[2]s", + "address": "%[4]s" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "%[2]s", + "address": "%[5]s" + } + } + ], + "ccRecipients": [], + "bccRecipients": [], + "replyTo": [], + "flag": { + "flagStatus": "notFlagged" + }, + "attachments": [ + { + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#/attachments(microsoft.graph.itemAttachment/item())/$entity", + "@odata.type": "#microsoft.graph.itemAttachment", + "id": "AAMkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOABGAAAAAAAPvVwUramXT7jlSGpVU8_7BwB8wYc0thTTTYl3RpEYIUq_AAAAAAEMAAB8wYc0thTTTYl3RpEYIUq_AADKo35SAAABEgAQABv3spWM8g5IriSvYJe5kO8=", + "lastModifiedDateTime": "2023-02-06T20:03:40Z", + "name": "Not Something Small. 28-Jul-2022_20:53:33 Different", + "contentType": null, + "size": 10959, + "isInline": false, + "item@odata.associationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages('')/$ref", + "item@odata.navigationLink": "https://graph.microsoft.com/v1.0/users('f435c656-f8b2-4d71-93c3-6e092f52a167')/messages('')", + "item": { + "@odata.type": "#microsoft.graph.message", + "id": "", + "createdDateTime": "2023-02-06T20:03:40Z", + "lastModifiedDateTime": "2023-02-06T20:03:40Z", + "receivedDateTime": "2022-07-28T20:53:33Z", + "sentDateTime": "2022-07-28T20:53:33Z", + "hasAttachments": false, + "internetMessageId": "", + "subject": "Not Something Small. 28-Jul-2022_20:53:33 Different", + "bodyPreview": "I've been going through with the changing of messages. It shouldn't have the same calls, right? Call Me?\r\n\r\nWe want to be able to send multiple messages and we want to be able to respond and do other things that make sense for our users. In this case. Let", + "importance": "normal", + "conversationId": "AAQkAGQ1NzViZTdhLTEwMTMtNGJjNi05YWI2LTg4NWRlZDA2Y2UxOAAQAOlAM0OrVQlHkhUZeZMPxgg=", + "conversationIndex": "AQHYosQZ6UAzQ6tVCUeSFRl5kw/GCA==", + "isDeliveryReceiptRequested": false, + "isReadReceiptRequested": false, + "isRead": true, + "isDraft": false, + "webLink": "https://outlook.office365.com/owa/?AttachmentItemID=Aviewmodel=ItemAttachment", + "body": { + "contentType": "html", + "content": "\r\n

I've been going through with the changing of messages. It shouldn't have the same calls, right? Call Me?

 

We want to be able to send multiple messages and we want to be able to respond and do other things that make sense for our users. In this case. Let’s consider a Mailbox

" + }, + "sender": { + "emailAddress": { + "name": "%[2]s", + "address": "%[3]s" + } + }, + "from": { + "emailAddress": { + "name": "%[2]s", + "address": "%[4]s" + } + }, + "toRecipients": [ + { + "emailAddress": { + "name": "Direct Report", + "address": "notAvailable@8qzvrj.onmicrosoft.com" + } + } + ], + "flag": { + "flagStatus": "notFlagged" + } + } + } + ] + }` + + message := fmt.Sprintf( + template, + subject, + defaultAlias, + defaultMessageSender, + defaultMessageFrom, + defaultMessageTo, + ) + + return []byte(message) +} + func GetMockMessageWithNestedItemAttachmentEvent(subject string) []byte { //nolint:lll // Order of fields: @@ -545,3 +688,46 @@ func GetMockMessageWithNestedItemAttachmentEvent(subject string) []byte { return []byte(message) } + +func GetMockMessageWithNestedItemAttachmentMail(t *testing.T, nested []byte, subject string) []byte { + base := GetMockMessageBytes(subject) + message, err := hydrateMessage(base) + require.NoError(t, err) + + nestedMessage, err := hydrateMessage(nested) + require.NoError(t, err) + + iaNode := models.NewItemAttachment() + attachmentSize := int32(len(nested)) + iaNode.SetSize(&attachmentSize) + + internalName := "Nested Message" + iaNode.SetName(&internalName) + iaNode.SetItem(nestedMessage) + message.SetAttachments([]models.Attachmentable{iaNode}) + + wtr := js.NewJsonSerializationWriter() + err = wtr.WriteObjectValue("", message) + require.NoError(t, err) + + byteArray, err := wtr.GetSerializedContent() + require.NoError(t, err) + + return byteArray +} + +func hydrateMessage(byteArray []byte) (models.Messageable, error) { + parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", byteArray) + if err != nil { + return nil, errors.Wrap(err, "deserializing bytes into base m365 object") + } + + anObject, err := parseNode.GetObjectValue(models.CreateMessageFromDiscriminatorValue) + if err != nil { + return nil, errors.Wrap(err, "parsing m365 object factory") + } + + message := anObject.(models.Messageable) + + return message, nil +} diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 7fa207c9e..4f8227a29 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -7,7 +7,11 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" ) -const itemAttachment = "#microsoft.graph.itemAttachment" +//========================================================== +// m365Transform.go contains utility functions that +// either add, modify, or remove fields from M365 +// objects for interacton with M365 services +//========================================================= // CloneMessageableFields places data from original data into new message object. // SingleLegacyValueProperty is not populated during this operation @@ -282,14 +286,35 @@ func cloneColumnDefinitionable(orig models.ColumnDefinitionable) models.ColumnDe return newColumn } +// =============================================================================================== +// Sanitization section +// Set of functions that support ItemAttachemtable object restoration. +// These attachments can be nested as well as possess one of the other +// reference types. To ensure proper upload, each interior`item` requires +// that certain fields be modified. +// ItemAttachment: +// https://learn.microsoft.com/en-us/graph/api/resources/itemattachment?view=graph-rest-1.0 +// https://learn.microsoft.com/en-us/exchange/client-developer/exchange-web-services/attachments-and-ews-in-exchange +// https://learn.microsoft.com/en-us/exchange/client-developer/exchange-web-services/folders-and-items-in-ews-in-exchange +// =============================================================================================== +// M365 Models possess a field, OData.Type which indicate +// the represent the intended model in string format. +// The constants listed here identify the supported itemAttachments +// currently supported for Restore operations. +// itemAttachments +// support ODataType values +// +//nolint:lll +const ( + itemAttachment = "#microsoft.graph.itemAttachment" + eventItemType = "#microsoft.graph.event" + mailItemType = "#microsoft.graph.message" +) + // ToItemAttachment transforms internal item, OutlookItemables, into // objects that are able to be uploaded into M365. -// Supported Internal Items: -// - Events func ToItemAttachment(orig models.Attachmentable) (models.Attachmentable, error) { transform, ok := orig.(models.ItemAttachmentable) - supported := "#microsoft.graph.event" - if !ok { // Shouldn't ever happen return nil, fmt.Errorf("transforming attachment to item attachment") } @@ -298,7 +323,7 @@ func ToItemAttachment(orig models.Attachmentable) (models.Attachmentable, error) itemType := item.GetOdataType() switch *itemType { - case supported: + case eventItemType: event := item.(models.Eventable) newEvent, err := sanitizeEvent(event) @@ -308,12 +333,45 @@ func ToItemAttachment(orig models.Attachmentable) (models.Attachmentable, error) transform.SetItem(newEvent) + return transform, nil + case mailItemType: + message := item.(models.Messageable) + + newMessage, err := sanitizeMessage(message) + if err != nil { + return nil, err + } + + transform.SetItem(newMessage) + return transform, nil default: return nil, fmt.Errorf("exiting ToItemAttachment: %s not supported", *itemType) } } +// TODO #2428 (dadam39): re-apply nested attachments for itemAttachments +// func sanitizeAttachments(attached []models.Attachmentable) ([]models.Attachmentable, error) { +// attachments := make([]models.Attachmentable, len(attached)) + +// for _, ax := range attached { +// if *ax.GetOdataType() == itemAttachment { +// newAttachment, err := ToItemAttachment(ax) +// if err != nil { +// return nil, err +// } + +// attachments = append(attachments, newAttachment) + +// continue +// } + +// attachments = append(attachments, ax) +// } + +// return attachments, nil +// } + // sanitizeEvent transfers data into event object and // removes unique IDs from the M365 object func sanitizeEvent(orig models.Eventable) (models.Eventable, error) { @@ -324,7 +382,9 @@ func sanitizeEvent(orig models.Eventable) (models.Eventable, error) { newEvent.SetCalendar(orig.GetCalendar()) newEvent.SetCreatedDateTime(orig.GetCreatedDateTime()) newEvent.SetEnd(orig.GetEnd()) - newEvent.SetHasAttachments(orig.GetHasAttachments()) + // TODO: dadams39 Nested attachments not supported + // Upstream: https://github.com/microsoft/kiota-serialization-json-go/issues/61 + newEvent.SetHasAttachments(nil) newEvent.SetHideAttendees(orig.GetHideAttendees()) newEvent.SetImportance(orig.GetImportance()) newEvent.SetIsAllDay(orig.GetIsAllDay()) @@ -337,7 +397,7 @@ func sanitizeEvent(orig models.Eventable) (models.Eventable, error) { newEvent.SetSubject(orig.GetSubject()) newEvent.SetType(orig.GetType()) - // Sanitation + // Sanitation NOTE // isDraft and isOrganizer *bool ptr's have to be removed completely // from JSON in order for POST method to succeed. // Current as of 2/2/2023 @@ -346,25 +406,34 @@ func sanitizeEvent(orig models.Eventable) (models.Eventable, error) { newEvent.SetIsDraft(nil) newEvent.SetAdditionalData(orig.GetAdditionalData()) - attached := orig.GetAttachments() - attachments := make([]models.Attachmentable, len(attached)) - - for _, ax := range attached { - if *ax.GetOdataType() == itemAttachment { - newAttachment, err := ToItemAttachment(ax) - if err != nil { - return nil, err - } - - attachments = append(attachments, newAttachment) - - continue - } - - attachments = append(attachments, ax) - } - - newEvent.SetAttachments(attachments) + // TODO #2428 (dadam39): re-apply nested attachments for itemAttachments + // Upstream: https://github.com/microsoft/kiota-serialization-json-go/issues/61 + // attachments, err := sanitizeAttachments(message.GetAttachments()) + // if err != nil { + // return nil, err + // } + newEvent.SetAttachments(nil) return newEvent, nil } + +func sanitizeMessage(orig models.Messageable) (models.Messageable, error) { + message := ToMessage(orig) + + // TODO #2428 (dadam39): re-apply nested attachments for itemAttachments + // Upstream: https://github.com/microsoft/kiota-serialization-json-go/issues/61 + // attachments, err := sanitizeAttachments(message.GetAttachments()) + // if err != nil { + // return nil, err + // } + message.SetAttachments(nil) + + // The following fields are set to nil to + // not interfere with M365 guard checks. + message.SetHasAttachments(nil) + message.SetParentFolderId(nil) + message.SetInternetMessageHeaders(nil) + message.SetIsDraft(nil) + + return message, nil +} diff --git a/website/docs/support/known-issues.md b/website/docs/support/known-issues.md index af2bc2db9..7b9c56052 100644 --- a/website/docs/support/known-issues.md +++ b/website/docs/support/known-issues.md @@ -12,6 +12,8 @@ Below is a list of known Corso issues and limitations: Advanced restore options such as in-place restore, or restore to a specific folder or to a different account aren't yet supported. +* Restoration of Nested attachments within Exchange Mail or Calendars aren't yet supported. + * Provides no guarantees about whether data moved, added, or deleted in M365 while a backup is being created will be included in the running backup. Future backups run when the data isn't modified will include the data.