diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go new file mode 100644 index 000000000..69ed6a377 --- /dev/null +++ b/src/internal/connector/graph_connector_helper_test.go @@ -0,0 +1,316 @@ +package connector + +import ( + "io" + "testing" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/alcionai/corso/src/internal/connector/mockconnector" + "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" +) + +func mustToDataLayerPath( + t *testing.T, + service path.ServiceType, + tenant, user string, + category path.CategoryType, + elements []string, + isItem bool, +) path.Path { + var ( + err error + res path.Path + ) + + pb := path.Builder{}.Append(elements...) + + switch service { + case path.ExchangeService: + res, err = pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem) + case path.OneDriveService: + require.Equal(t, path.FilesCategory, category) + + res, err = pb.ToDataLayerOneDrivePath(tenant, user, isItem) + + default: + err = errors.Errorf("bad service type %s", service.String()) + } + + require.NoError(t, err) + + return res +} + +func notNilAndEq[T any](t *testing.T, expected *T, got *T, msg string) { + t.Helper() + + if assert.NotNil(t, expected, "expected "+msg) && assert.NotNil(t, got, "got "+msg) { + assert.Equal(t, *expected, *got, msg) + } +} + +type itemInfo struct { + // lookupKey is a string that can be used to find this data from a set of + // other data in the same collection. This key should be something that will + // be the same before and after restoring the item in M365 and may not be + // the M365 ID. When restoring items out of place, the item is assigned a + // new ID making it unsuitable for a lookup key. + lookupKey string + name string + data []byte +} + +type colInfo struct { + // Elements (in order) for the path representing this collection. Should + // only contain elements after the prefix that corso uses for the path. For + // example, a collection for the Inbox folder in exchange mail would just be + // "Inbox". + pathElements []string + category path.CategoryType + items []itemInfo +} + +func checkMessage( + t *testing.T, + expected models.Messageable, + got models.Messageable, +) { + assert.Equal(t, expected.GetBccRecipients(), got.GetBccRecipients(), "BccRecipients") + + notNilAndEq(t, expected.GetBody().GetContentType(), got.GetBody().GetContentType(), "Body.ContentType") + + // Skip Body.Content as there may be display formatting that changes. + + // Skip BodyPreview as it is auto-generated on the server side and isn't + // always just the first 255 characters if the message is HTML and has + // multiple paragraphs. + + assert.Equal(t, expected.GetCategories(), got.GetCategories(), "Categories") + + assert.Equal(t, expected.GetCcRecipients(), got.GetCcRecipients(), "CcRecipients") + + // Skip ChangeKey as it's tied to this specific instance of the item. + + // Skip ConversationId as it's tied to this specific instance of the item. + + // Skip ConversationIndex as it's tied to this specific instance of the item. + + // Skip CreatedDateTime as it's tied to this specific instance of the item. + + assert.Equal(t, expected.GetFlag(), got.GetFlag(), "Flag") + + assert.Equal(t, expected.GetFrom(), got.GetFrom(), "From") + + notNilAndEq(t, expected.GetHasAttachments(), got.GetHasAttachments(), "HasAttachments") + + // Skip Id as it's tied to this specific instance of the item. + + notNilAndEq(t, expected.GetImportance(), got.GetImportance(), "Importance") + + notNilAndEq(t, expected.GetInferenceClassification(), got.GetInferenceClassification(), "InferenceClassification") + + assert.Equal(t, expected.GetInternetMessageHeaders(), got.GetInternetMessageHeaders(), "InternetMessageHeaders") + + notNilAndEq(t, expected.GetInternetMessageId(), got.GetInternetMessageId(), "InternetMessageId") + + notNilAndEq( + t, + expected.GetIsDeliveryReceiptRequested(), + got.GetIsDeliveryReceiptRequested(), + "IsDeliverReceiptRequested", + ) + + notNilAndEq(t, expected.GetIsDraft(), got.GetIsDraft(), "IsDraft") + + notNilAndEq(t, expected.GetIsRead(), got.GetIsRead(), "IsRead") + + notNilAndEq(t, expected.GetIsReadReceiptRequested(), got.GetIsReadReceiptRequested(), "IsReadReceiptRequested") + + // Skip LastModifiedDateTime as it's tied to this specific instance of the item. + + // Skip ParentFolderId as we restore to a different folder by default. + + notNilAndEq(t, expected.GetReceivedDateTime(), got.GetReceivedDateTime(), "ReceivedDateTime") + + assert.Equal(t, expected.GetReplyTo(), got.GetReplyTo(), "ReplyTo") + + assert.Equal(t, expected.GetSender(), got.GetSender(), "Sender") + + notNilAndEq(t, expected.GetSentDateTime(), got.GetSentDateTime(), "SentDateTime") + + notNilAndEq(t, expected.GetSubject(), got.GetSubject(), "Subject") + + assert.Equal(t, expected.GetToRecipients(), got.GetToRecipients(), "ToRecipients") + + // Skip WebLink as it's tied to this specific instance of the item. + + assert.Equal(t, expected.GetUniqueBody(), got.GetUniqueBody(), "UniqueBody") +} + +func compareExchangeEmail( + t *testing.T, + expected map[string][]byte, + item data.Stream, +) { + itemData, err := io.ReadAll(item.ToReader()) + if !assert.NoError(t, err, "reading collection item: %s", item.UUID()) { + return + } + + itemMessageParsable, err := support.CreateMessageFromBytes(itemData) + if !assert.NoError(t, err, "deserializing backed up message") { + return + } + + itemMessage := itemMessageParsable + + expectedBytes, ok := expected[*itemMessage.GetSubject()] + if !assert.True(t, ok, "unexpected item with Subject %q", *itemMessage.GetSubject()) { + return + } + + expectedMessageParsable, err := support.CreateMessageFromBytes(expectedBytes) + assert.NoError(t, err, "deserializing source message") + + checkMessage(t, expectedMessageParsable, itemMessage) +} + +func compareItem( + t *testing.T, + expected map[string][]byte, + service path.ServiceType, + category path.CategoryType, + item data.Stream, +) { + switch service { + case path.ExchangeService: + switch category { + case path.EmailCategory: + compareExchangeEmail(t, expected, item) + default: + assert.FailNowf(t, "unexpected Exchange category: %s", category.String()) + } + default: + assert.FailNowf(t, "unexpected service: %s", service.String()) + } +} + +func checkHasCollections( + t *testing.T, + expected map[string]map[string][]byte, + got []data.Collection, +) { + t.Helper() + + expectedNames := make([]string, 0, len(expected)) + gotNames := make([]string, 0, len(got)) + + for e := range expected { + expectedNames = append(expectedNames, e) + } + + for _, g := range got { + gotNames = append(gotNames, g.FullPath().String()) + } + + assert.ElementsMatch(t, expectedNames, gotNames) +} + +func checkCollections( + t *testing.T, + expected map[string]map[string][]byte, + got []data.Collection, +) { + checkHasCollections(t, expected, got) + + for _, returned := range got { + service := returned.FullPath().Service() + category := returned.FullPath().Category() + expectedColData := expected[returned.FullPath().String()] + + if expectedColData == nil { + // Missing/extra collections will be reported in the above `ElementsMatch` + // call. + continue + } + + for item := range returned.Items() { + compareItem(t, expectedColData, service, category, item) + } + } +} + +func collectionsForInfo( + t *testing.T, + service path.ServiceType, + tenant, user string, + dest control.RestoreDestination, + allInfo []colInfo, +) (int, []data.Collection, map[string]map[string][]byte) { + collections := make([]data.Collection, 0, len(allInfo)) + expectedData := make(map[string]map[string][]byte, len(allInfo)) + totalItems := 0 + + for _, info := range allInfo { + pth := mustToDataLayerPath( + t, + service, + tenant, + user, + info.category, + info.pathElements, + false, + ) + c := mockconnector.NewMockExchangeCollection(pth, len(info.items)) + + // TODO(ashmrtn): This will need expanded/broken up by service/category + // depending on how restore for that service/category places data back in + // M365. + baseDestPath := mustToDataLayerPath( + t, + service, + tenant, + user, + info.category, + []string{dest.ContainerName}, + false, + ) + + expectedData[baseDestPath.String()] = make(map[string][]byte, len(info.items)) + + for i := 0; i < len(info.items); i++ { + c.Names[i] = info.items[i].name + c.Data[i] = info.items[i].data + + expectedData[baseDestPath.String()][info.items[i].lookupKey] = info.items[i].data + } + + collections = append(collections, c) + totalItems += len(info.items) + } + + return totalItems, collections, expectedData +} + +func getSelectorWith(service path.ServiceType) selectors.Selector { + s := selectors.ServiceUnknown + + switch service { + case path.ExchangeService: + s = selectors.ServiceExchange + case path.OneDriveService: + s = selectors.ServiceOneDrive + } + + return selectors.Selector{ + Service: s, + } +} diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index e228e6cb4..b26b86d4e 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -367,6 +367,8 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteCalendar() { } } +// TODO(ashmrtn): Merge this with the below once we get comparison logic for +// contacts. func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() { t := suite.T() sel := selectors.NewExchangeRestore() @@ -400,3 +402,114 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() { assert.Equal(t, value.FolderCount, 1) suite.T().Log(value.String()) } + +func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { + bodyText := "This email has some text. However, all the text is on the same line." + subjectText := "Test message for restore" + + table := []struct { + name string + service path.ServiceType + collections []colInfo + backupSelFunc func(dest control.RestoreDestination, backupUser string) selectors.Selector + }{ + { + name: "MultipleEmailsSingleFolder", + service: path.ExchangeService, + collections: []colInfo{ + { + pathElements: []string{"Inbox"}, + category: path.EmailCategory, + items: []itemInfo{ + { + name: "someencodeditemID", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-1", + bodyText+" 1.", + ), + lookupKey: subjectText + "-1", + }, + { + name: "someencodeditemID2", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-2", + bodyText+" 2.", + ), + lookupKey: subjectText + "-2", + }, + { + name: "someencodeditemID3", + data: mockconnector.GetMockMessageWithBodyBytes( + subjectText+"-3", + bodyText+" 3.", + ), + lookupKey: subjectText + "-3", + }, + }, + }, + }, + // TODO(ashmrtn): Generalize this once we know the path transforms that + // occur during restore. + backupSelFunc: func(dest control.RestoreDestination, backupUser string) selectors.Selector { + backupSel := selectors.NewExchangeBackup() + backupSel.Include(backupSel.MailFolders( + []string{backupUser}, + []string{dest.ContainerName}, + )) + + return backupSel.Selector + }, + }, + } + + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + ctx := context.Background() + // Get a dest per test so they're independent. + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive) + + totalItems, collections, expectedData := collectionsForInfo( + t, + test.service, + suite.connector.tenant, + suite.user, + dest, + test.collections, + ) + + t.Logf("Restoring collections to %s\n", dest.ContainerName) + + restoreGC := loadConnector(ctx, t) + restoreSel := getSelectorWith(test.service) + err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + require.NoError(t, err) + + status := restoreGC.AwaitStatus() + assert.Equal(t, len(test.collections), status.FolderCount, "status.FolderCount") + assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") + assert.Equal(t, totalItems, status.Successful, "status.Successful") + + t.Logf("Restore complete\n") + + // Run a backup and compare its output with what we put in. + + backupGC := loadConnector(ctx, t) + backupSel := test.backupSelFunc(dest, suite.user) + t.Logf("Selective backup of %s\n", backupSel) + + dcs, err := backupGC.DataCollections(ctx, backupSel) + require.NoError(t, err) + + t.Logf("Backup enumeration complete\n") + + // Pull the data prior to waiting for the status as otherwise it will + // deadlock. + checkCollections(t, expectedData, dcs) + + status = backupGC.AwaitStatus() + assert.Equal(t, len(test.collections), status.FolderCount, "status.FolderCount") + assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") + assert.Equal(t, totalItems, status.Successful, "status.Successful") + }) + } +} diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index eaa7fc82a..388cbda1b 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -2,6 +2,7 @@ package mockconnector import ( "bytes" + "fmt" "io" "math/rand" "strconv" @@ -16,6 +17,35 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +//nolint:lll +const ( + defaultMessageBody = "Lidia,

We have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.
" + + "

Thanking you in advance,
" + + "

Dustin
" + defaultMessagePreview = "Lidia,\\n\\nWe have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.\\n" + + "\\nThanking you in adv" + + // Order of fields to fill in: + // 1. message body + // 2. message preview + // 3. sender user ID + // 4. subject + messageTmpl = "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," + + "\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"" + + "\\n
%s" + + "
\",\"contentType\":\"html\"}," + + "\"bodyPreview\":\"%s\"," + + "\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," + + "\"from\":{\"emailAddress\":{\"address\":\"%s\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"\"," + + "\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," + + "\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," + + "\"subject\":\"%s\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," + + "\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem\"}" +) + // MockExchangeDataCollection represents a mock exchange mailbox type MockExchangeDataCollection struct { fullPath path.Path @@ -150,22 +180,32 @@ func GetMockMessageBytes(subject string) []byte { userID := "foobar@8qzvrj.onmicrosoft.com" timestamp := " " + common.FormatNow(common.SimpleDateTimeFormat) - //nolint:lll - message := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," + - "\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"" + - "\\n
Lidia,

We have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.
" + - "

Thanking you in advance,
" + - "

Dustin
\",\"contentType\":\"html\"}," + - "\"bodyPreview\":\"Lidia,\\n\\nWe have not received any reports on the development during Q2. It is in our best interest to have a new TPS Report by next Thursday prior to the retreat. If you have any questions, please let me know so I can address them.\\n" + - "\\nThanking you in adv\",\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," + - "\"from\":{\"emailAddress\":{\"address\":\"" + userID + "\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"\"," + - "\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," + - "\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," + - "\"subject\":\"TPS Report " + subject + timestamp + "\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," + - "\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%3D&exvsurl=1&viewmodel=ReadMessageItem\"}" + message := fmt.Sprintf( + messageTmpl, + defaultMessageBody, + defaultMessagePreview, + userID, + "TPS Report "+subject+timestamp, + ) + + return []byte(message) +} + +func GetMockMessageWithBodyBytes(subject, body string) []byte { + userID := "foobar@8qzvrj.onmicrosoft.com" + preview := body + + if len(preview) > 255 { + preview = preview[:256] + } + + message := fmt.Sprintf( + messageTmpl, + body, + preview, + userID, + subject, + ) return []byte(message) }