From 3c0179986bccd4764c9af0e785340f2dc3098352 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Wed, 28 Sep 2022 13:23:54 -0700 Subject: [PATCH] Restore and backup tests for GraphConnector (#980) ## Description Add framework and tests for some exchange mail restore and backup situations. Framework can be used to test other situations in the future, this is just the starting point. Some logic in the test can be further generalized/factored out once we know more about how paths will be transformed during restore ## Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Test - [ ] :computer: CI/Deployment - [ ] :hamster: Trivial/Minor ## Issue(s) * #913 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/graph_connector_helper_test.go | 316 ++++++++++++++++++ .../connector/graph_connector_test.go | 113 +++++++ .../mockconnector/mock_data_collection.go | 72 +++- 3 files changed, 485 insertions(+), 16 deletions(-) create mode 100644 src/internal/connector/graph_connector_helper_test.go diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go new file mode 100644 index 000000000..69ed6a377 --- /dev/null +++ b/src/internal/connector/graph_connector_helper_test.go @@ -0,0 +1,316 @@ +package connector + +import ( + "io" + "testing" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/internal/connector/mockconnector" + "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" +) + +func mustToDataLayerPath( + t *testing.T, + service path.ServiceType, + tenant, user string, + category path.CategoryType, + elements []string, + isItem bool, +) path.Path { + var ( + err error + res path.Path + ) + + pb := path.Builder{}.Append(elements...) + + switch service { + case path.ExchangeService: + res, err = pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem) + case path.OneDriveService: + require.Equal(t, path.FilesCategory, category) + + res, err = pb.ToDataLayerOneDrivePath(tenant, user, isItem) + + default: + err = errors.Errorf("bad service type %s", service.String()) + } + + require.NoError(t, err) + + return res +} + +func notNilAndEq[T any](t *testing.T, expected *T, got *T, msg string) { + t.Helper() + + if assert.NotNil(t, expected, "expected "+msg) && assert.NotNil(t, got, "got "+msg) { + assert.Equal(t, *expected, *got, msg) + } +} + +type itemInfo struct { + // lookupKey is a string that can be used to find this data from a set of + // other data in the same collection. This key should be something that will + // be the same before and after restoring the item in M365 and may not be + // the M365 ID. When restoring items out of place, the item is assigned a + // new ID making it unsuitable for a lookup key. + lookupKey string + name string + data []byte +} + +type colInfo struct { + // Elements (in order) for the path representing this collection. Should + // only contain elements after the prefix that corso uses for the path. For + // example, a collection for the Inbox folder in exchange mail would just be + // "Inbox". + pathElements []string + category path.CategoryType + items []itemInfo +} + +func checkMessage( + t *testing.T, + expected models.Messageable, + got models.Messageable, +) { + assert.Equal(t, expected.GetBccRecipients(), got.GetBccRecipients(), "BccRecipients") + + notNilAndEq(t, expected.GetBody().GetContentType(), got.GetBody().GetContentType(), "Body.ContentType") + + // Skip Body.Content as there may be display formatting that changes. + + // Skip BodyPreview as it is auto-generated on the server side and isn't + // always just the first 255 characters if the message is HTML and has + // multiple paragraphs. + + assert.Equal(t, expected.GetCategories(), got.GetCategories(), "Categories") + + assert.Equal(t, expected.GetCcRecipients(), got.GetCcRecipients(), "CcRecipients") + + // Skip ChangeKey as it's tied to this specific instance of the item. + + // Skip ConversationId as it's tied to this specific instance of the item. + + // Skip ConversationIndex as it's tied to this specific instance of the item. + + // Skip CreatedDateTime as it's tied to this specific instance of the item. + + assert.Equal(t, expected.GetFlag(), got.GetFlag(), "Flag") + + assert.Equal(t, expected.GetFrom(), got.GetFrom(), "From") + + notNilAndEq(t, expected.GetHasAttachments(), got.GetHasAttachments(), "HasAttachments") + + // Skip Id as it's tied to this specific instance of the item. + + notNilAndEq(t, expected.GetImportance(), got.GetImportance(), "Importance") + + notNilAndEq(t, expected.GetInferenceClassification(), got.GetInferenceClassification(), "InferenceClassification") + + assert.Equal(t, expected.GetInternetMessageHeaders(), got.GetInternetMessageHeaders(), "InternetMessageHeaders") + + notNilAndEq(t, expected.GetInternetMessageId(), got.GetInternetMessageId(), "InternetMessageId") + + notNilAndEq( + t, + expected.GetIsDeliveryReceiptRequested(), + got.GetIsDeliveryReceiptRequested(), + "IsDeliverReceiptRequested", + ) + + notNilAndEq(t, expected.GetIsDraft(), got.GetIsDraft(), "IsDraft") + + notNilAndEq(t, expected.GetIsRead(), got.GetIsRead(), "IsRead") + + notNilAndEq(t, expected.GetIsReadReceiptRequested(), got.GetIsReadReceiptRequested(), "IsReadReceiptRequested") + + // Skip LastModifiedDateTime as it's tied to this specific instance of the item. + + // Skip ParentFolderId as we restore to a different folder by default. + + notNilAndEq(t, expected.GetReceivedDateTime(), got.GetReceivedDateTime(), "ReceivedDateTime") + + assert.Equal(t, expected.GetReplyTo(), got.GetReplyTo(), "ReplyTo") + + assert.Equal(t, expected.GetSender(), got.GetSender(), "Sender") + + notNilAndEq(t, expected.GetSentDateTime(), got.GetSentDateTime(), "SentDateTime") + + notNilAndEq(t, expected.GetSubject(), got.GetSubject(), "Subject") + + assert.Equal(t, expected.GetToRecipients(), got.GetToRecipients(), "ToRecipients") + + // Skip WebLink as it's tied to this specific instance of the item. + + assert.Equal(t, expected.GetUniqueBody(), got.GetUniqueBody(), "UniqueBody") +} + +func compareExchangeEmail( + t *testing.T, + expected map[string][]byte, + item data.Stream, +) { + itemData, err := io.ReadAll(item.ToReader()) + if !assert.NoError(t, err, "reading collection item: %s", item.UUID()) { + return + } + + itemMessageParsable, err := support.CreateMessageFromBytes(itemData) + if !assert.NoError(t, err, "deserializing backed up message") { + return + } + + itemMessage := itemMessageParsable + + expectedBytes, ok := expected[*itemMessage.GetSubject()] + if !assert.True(t, ok, "unexpected item with Subject %q", *itemMessage.GetSubject()) { + return + } + + expectedMessageParsable, err := support.CreateMessageFromBytes(expectedBytes) + assert.NoError(t, err, "deserializing source message") + + checkMessage(t, expectedMessageParsable, itemMessage) +} + +func compareItem( + t *testing.T, + expected map[string][]byte, + service path.ServiceType, + category path.CategoryType, + item data.Stream, +) { + switch service { + case path.ExchangeService: + switch category { + case path.EmailCategory: + compareExchangeEmail(t, expected, item) + default: + assert.FailNowf(t, "unexpected Exchange category: %s", category.String()) + } + default: + assert.FailNowf(t, "unexpected service: %s", service.String()) + } +} + +func checkHasCollections( + t *testing.T, + expected map[string]map[string][]byte, + got []data.Collection, +) { + t.Helper() + + expectedNames := make([]string, 0, len(expected)) + gotNames := make([]string, 0, len(got)) + + for e := range expected { + expectedNames = append(expectedNames, e) + } + + for _, g := range got { + gotNames = append(gotNames, g.FullPath().String()) + } + + assert.ElementsMatch(t, expectedNames, gotNames) +} + +func checkCollections( + t *testing.T, + expected map[string]map[string][]byte, + got []data.Collection, +) { + checkHasCollections(t, expected, got) + + for _, returned := range got { + service := returned.FullPath().Service() + category := returned.FullPath().Category() + expectedColData := expected[returned.FullPath().String()] + + if expectedColData == nil { + // Missing/extra collections will be reported in the above `ElementsMatch` + // call. + continue + } + + for item := range returned.Items() { + compareItem(t, expectedColData, service, category, item) + } + } +} + +func collectionsForInfo( + t *testing.T, + service path.ServiceType, + tenant, user string, + dest control.RestoreDestination, + allInfo []colInfo, +) (int, []data.Collection, map[string]map[string][]byte) { + collections := make([]data.Collection, 0, len(allInfo)) + expectedData := make(map[string]map[string][]byte, len(allInfo)) + totalItems := 0 + + for _, info := range allInfo { + pth := mustToDataLayerPath( + t, + service, + tenant, + user, + info.category, + info.pathElements, + false, + ) + c := mockconnector.NewMockExchangeCollection(pth, len(info.items)) + + // TODO(ashmrtn): This will need expanded/broken up by service/category + // depending on how restore for that service/category places data back in + // M365. + baseDestPath := mustToDataLayerPath( + t, + service, + tenant, + user, + info.category, + []string{dest.ContainerName}, + false, + ) + + expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.items)) + + for i := 0; i < len(info.items); i++ { + c.Names[i] = info.items[i].name + c.Data[i] = info.items[i].data + + expectedData[baseDestPath.String()][info.items[i].lookupKey] = info.items[i].data + } + + collections = append(collections, c) + totalItems += len(info.items) + } + + return totalItems, collections, expectedData +} + +func getSelectorWith(service path.ServiceType) selectors.Selector { + s := selectors.ServiceUnknown + + switch service { + case path.ExchangeService: + s = selectors.ServiceExchange + case path.OneDriveService: + s = selectors.ServiceOneDrive + } + + return selectors.Selector{ + Service: s, + } +} diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index e228e6cb4..b26b86d4e 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -367,6 +367,8 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteCalendar() { } } +// TODO(ashmrtn): Merge this with the below once we get comparison logic for +// contacts. func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() { t := suite.T() sel := selectors.NewExchangeRestore() @@ -400,3 +402,114 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() { assert.Equal(t, value.FolderCount, 1) suite.T().Log(value.String()) } + +func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { + bodyText := "This email has some text. However, all the text is on the same line." + subjectText := "Test message for restore" + + table := []struct { + name string + service path.ServiceType + collections []colInfo + backupSelFunc func(dest control.RestoreDestination, backupUser string) selectors.Selector + }{ + { + name: "MultipleEmailsSingleFolder", + service: path.ExchangeService, + collections: []colInfo{ + { + pathElements: []string{"Inbox"}, + category: path.EmailCategory, + items: []itemInfo{ + { + name: "someencodeditemID", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-1", + bodyText+" 1.", + ), + lookupKey: subjectText + "-1", + }, + { + name: "someencodeditemID2", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-2", + bodyText+" 2.", + ), + lookupKey: subjectText + "-2", + }, + { + name: "someencodeditemID3", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-3", + bodyText+" 3.", + ), + lookupKey: subjectText + "-3", + }, + }, + }, + }, + // TODO(ashmrtn): Generalize this once we know the path transforms that + // occur during restore. + backupSelFunc: func(dest control.RestoreDestination, backupUser string) selectors.Selector { + backupSel := selectors.NewExchangeBackup() + backupSel.Include(backupSel.MailFolders( + []string{backupUser}, + []string{dest.ContainerName}, + )) + + return backupSel.Selector + }, + }, + } + + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + ctx := context.Background() + // Get a dest per test so they're independent. + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive) + + totalItems, collections, expectedData := collectionsForInfo( + t, + test.service, + suite.connector.tenant, + suite.user, + dest, + test.collections, + ) + + t.Logf("Restoring collections to %s\n", dest.ContainerName) + + restoreGC := loadConnector(ctx, t) + restoreSel := getSelectorWith(test.service) + err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + require.NoError(t, err) + + status := restoreGC.AwaitStatus() + assert.Equal(t, len(test.collections), status.FolderCount, "status.FolderCount") + assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") + assert.Equal(t, totalItems, status.Successful, "status.Successful") + + t.Logf("Restore complete\n") + + // Run a backup and compare its output with what we put in. + + backupGC := loadConnector(ctx, t) + backupSel := test.backupSelFunc(dest, suite.user) + t.Logf("Selective backup of %s\n", backupSel) + + dcs, err := backupGC.DataCollections(ctx, backupSel) + require.NoError(t, err) + + t.Logf("Backup enumeration complete\n") + + // Pull the data prior to waiting for the status as otherwise it will + // deadlock. + checkCollections(t, expectedData, dcs) + + status = backupGC.AwaitStatus() + assert.Equal(t, len(test.collections), status.FolderCount, "status.FolderCount") + assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") + assert.Equal(t, totalItems, status.Successful, "status.Successful") + }) + } +} diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index eaa7fc82a..388cbda1b 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -2,6 +2,7 @@ package mockconnector import ( "bytes" + "fmt" "io" "math/rand" "strconv" @@ -16,6 +17,35 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +//nolint:lll +const ( + defaultMessageBody = "Lidia,

We have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.
" + + "

Thanking you in advance,
" + + "

Dustin
" + defaultMessagePreview = "Lidia,\\n\\nWe have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.\\n" + + "\\nThanking you in adv" + + // Order of fields to fill in: + // 1. message body + // 2. message preview + // 3. sender user ID + // 4. subject + messageTmpl = "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," + + "\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"" + + "\\n
%s" + + "
\",\"contentType\":\"html\"}," + + "\"bodyPreview\":\"%s\"," + + "\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," + + "\"from\":{\"emailAddress\":{\"address\":\"%s\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"\"," + + "\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," + + "\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," + + "\"subject\":\"%s\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," + + "\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem\"}" +) + // MockExchangeDataCollection represents a mock exchange mailbox type MockExchangeDataCollection struct { fullPath path.Path @@ -150,22 +180,32 @@ func GetMockMessageBytes(subject string) []byte { userID := "foobar@8qzvrj.onmicrosoft.com" timestamp := " " + common.FormatNow(common.SimpleDateTimeFormat) - //nolint:lll - message := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," + - "\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"" + - "\\n
Lidia,

We have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.
" + - "

Thanking you in advance,
" + - "

Dustin
\",\"contentType\":\"html\"}," + - "\"bodyPreview\":\"Lidia,\\n\\nWe have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.\\n" + - "\\nThanking you in adv\",\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," + - "\"from\":{\"emailAddress\":{\"address\":\"" + userID + "\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"\"," + - "\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," + - "\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," + - "\"subject\":\"TPS Report " + subject + timestamp + "\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," + - "\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%3D&exvsurl=1&viewmodel=ReadMessageItem\"}" + message := fmt.Sprintf( + messageTmpl, + defaultMessageBody, + defaultMessagePreview, + userID, + "TPS Report "+subject+timestamp, + ) + + return []byte(message) +} + +func GetMockMessageWithBodyBytes(subject, body string) []byte { + userID := "foobar@8qzvrj.onmicrosoft.com" + preview := body + + if len(preview) > 255 { + preview = preview[:256] + } + + message := fmt.Sprintf( + messageTmpl, + body, + preview, + userID, + subject, + ) return []byte(message) }