diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 72efe9e38..e6b3fc44b 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -93,6 +93,8 @@ func getPopulateFunction(optID optionIdentifier) populater { return PopulateForMailCollection case contacts: return PopulateForContactCollection + case events: + return PopulateForEventCollection default: return nil } @@ -208,6 +210,95 @@ func PopulateForMailCollection( statusChannel <- status } +func PopulateForEventCollection( + ctx context.Context, + service graph.Service, + user string, + jobs []string, + dataChannel chan<- data.Stream, + statusChannel chan<- *support.ConnectorOperationStatus, +) { + var ( + errs error + attemptedItems, success int + ) + objectWriter := kw.NewJsonSerializationWriter() + + for _, task := range jobs { + response, err := service.Client().UsersById(user).EventsById(task).Get() + if err != nil { + trace := support.ConnectorStackErrorTrace(err) + errs = support.WrapAndAppend( + user, + errors.Wrapf(err, "unable to retrieve items %s; details: %s", task, trace), + errs, + ) + continue + } + err = eventToDataCollection(ctx, service.Client(), objectWriter, dataChannel, response, user) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + + if service.ErrPolicy() { + break + } + continue + } + success++ + } + close(dataChannel) + attemptedItems += len(jobs) + status := support.CreateStatus(ctx, support.Backup, attemptedItems, success, 1, errs) + logger.Ctx(ctx).Debug(status.String()) + statusChannel <- status +} + +func eventToDataCollection( + ctx context.Context, + client *msgraphsdk.GraphServiceClient, + objectWriter *kw.JsonSerializationWriter, + dataChannel chan<- data.Stream, + event models.Eventable, + user string, +) error { + var err error + defer objectWriter.Close() + if *event.GetHasAttachments() { + var retriesErr error + for count := 0; count < numberOfRetries; count++ { + attached, err := client. + UsersById(user). + EventsById(*event.GetId()). + Attachments(). + Get() + retriesErr = err + if err == nil && attached != nil { + event.SetAttachments(attached.GetValue()) + break + } + } + if retriesErr != nil { + logger.Ctx(ctx).Debug("exceeded maximum retries") + return support.WrapAndAppend( + *event.GetId(), + errors.Wrap(retriesErr, "attachment failed"), + nil) + } + } + err = objectWriter.WriteObjectValue("", event) + if err != nil { + return support.SetNonRecoverableError(errors.Wrap(err, *event.GetId())) + } + byteArray, err := objectWriter.GetSerializedContent() + if err != nil { + return support.WrapAndAppend(*event.GetId(), errors.Wrap(err, "serializing content"), nil) + } + if byteArray != nil { + dataChannel <- &Stream{id: *event.GetId(), message: byteArray, info: nil} + } + return nil +} + func contactToDataCollection( ctx context.Context, client *msgraphsdk.GraphServiceClient, diff --git a/src/internal/connector/exchange/exchange_service_test.go b/src/internal/connector/exchange/exchange_service_test.go index 43d21457f..8f9fac5b9 100644 --- a/src/internal/connector/exchange/exchange_service_test.go +++ b/src/internal/connector/exchange/exchange_service_test.go @@ -190,9 +190,11 @@ func (suite *ExchangeServiceSuite) TestOptionsForContacts() { } } -// TestSetupExchangeCollection ensures that the helper -// function SetupExchangeCollectionVars returns a non-nil variable for returns -// in regards to the selector.ExchangeScope. +// TestSetupExchangeCollection ensures SetupExchangeCollectionVars returns a non-nil variable for +// the following selector types: +// - Mail +// - Contacts +// - Events func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() { userID := tester.M365UserID(suite.T()) sel := selectors.NewExchangeBackup() @@ -204,13 +206,10 @@ func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() { for _, test := range scopes { suite.T().Run(test.Category().String(), func(t *testing.T) { discriminateFunc, graphQuery, iterFunc, err := SetupExchangeCollectionVars(test) - if test.Category() == selectors.ExchangeMailFolder || - test.Category() == selectors.ExchangeContactFolder { - assert.NoError(t, err) - assert.NotNil(t, discriminateFunc) - assert.NotNil(t, graphQuery) - assert.NotNil(t, iterFunc) - } + assert.NoError(t, err) + assert.NotNil(t, discriminateFunc) + assert.NotNil(t, graphQuery) + assert.NotNil(t, iterFunc) }) } } diff --git a/src/internal/connector/exchange/optionidentifier_string.go b/src/internal/connector/exchange/optionidentifier_string.go index 08344aa92..88d1c7ac3 100644 --- a/src/internal/connector/exchange/optionidentifier_string.go +++ b/src/internal/connector/exchange/optionidentifier_string.go @@ -10,13 +10,14 @@ func _() { var x [1]struct{} _ = x[unknown-0] _ = x[folders-1] - _ = x[messages-2] - _ = x[users-3] + _ = x[events-2] + _ = x[messages-3] + _ = x[users-4] } -const _optionIdentifier_name = "unknownfoldersmessagesusers" +const _optionIdentifier_name = "unknownfolderseventsmessagesusers" -var _optionIdentifier_index = [...]uint8{0, 7, 14, 22, 27} +var _optionIdentifier_index = [...]uint8{0, 7, 14, 20, 28, 33} func (i optionIdentifier) String() string { if i < 0 || i >= optionIdentifier(len(_optionIdentifier_index)-1) { diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index a7fe18a5d..a431942b2 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -194,6 +194,12 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) ( IterateAndFilterMessagesForCollections, nil } + if scope.IncludesCategory(selectors.ExchangeEvent) { + return models.CreateEventCollectionResponseFromDiscriminatorValue, + GetAllEventsForUser, + IterateSelectAllEventsForCollections, + nil + } if scope.IncludesCategory(selectors.ExchangeContactFolder) { return models.CreateContactFromDiscriminatorValue, diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index 29a37d2e6..d91aa4382 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -7,6 +7,7 @@ import ( msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" mscontacts "github.com/microsoftgraph/msgraph-sdk-go/users/item/contacts" + msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events" msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders" msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" msitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages/item" @@ -18,17 +19,74 @@ import ( "github.com/alcionai/corso/pkg/selectors" ) +var ( + fieldsForEvents = map[string]int{ + "calendar": 1, + "end": 2, + "id": 3, + "isOnlineMeeting": 4, + "isReminderOn": 5, + "responseStatus": 6, + "responseRequested": 7, + "showAs": 8, + "subject": 9, + } + + fieldsForFolders = map[string]int{ + "childFolderCount": 1, + "displayName": 2, + "id": 3, + "isHidden": 4, + "parentFolderId": 5, + "totalItemCount": 6, + "unreadItemCount": 7, + } + + fieldsForUsers = map[string]int{ + "birthday": 1, + "businessPhones": 2, + "city": 3, + "companyName": 4, + "department": 5, + "displayName": 6, + "employeeId": 7, + "id": 8, + } + + fieldsForMessages = map[string]int{ + "conservationId": 1, + "conversationIndex": 2, + "parentFolderId": 3, + "subject": 4, + "webLink": 5, + "id": 6, + } + + fieldsForContacts = map[string]int{ + "id": 1, + "companyName": 2, + "department": 3, + "displayName": 4, + "fileAs": 5, + "givenName": 6, + "manager": 7, + "parentFolderId": 8, + } +) + type optionIdentifier int const ( mailCategory = "mail" contactsCategory = "contacts" + eventsCategory = "events" ) //go:generate stringer -type=optionIdentifier const ( unknown optionIdentifier = iota folders + events messages users contacts @@ -73,6 +131,17 @@ func GetAllFolderNamesForUser(gs graph.Service, user string) (absser.Parsable, e return gs.Client().UsersById(user).MailFolders().GetWithRequestConfigurationAndResponseHandler(options, nil) } +// GetAllEvents for User. Default returns EventResponseCollection for events in the future +// of the time that the call was made. There a +func GetAllEventsForUser(gs graph.Service, user string) (absser.Parsable, error) { + options, err := optionsForEvents([]string{"id", "calendar"}) + if err != nil { + return nil, err + } + + return gs.Client().UsersById(user).Events().GetWithRequestConfigurationAndResponseHandler(options, nil) +} + // GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator) // @returns a callback func that works with msgraphgocore.PageIterator.Iterate function type GraphIterateFunc func( @@ -130,6 +199,51 @@ func IterateSelectAllMessagesForCollections( } } +func IterateSelectAllEventsForCollections( + tenant string, + scope selectors.ExchangeScope, + errs error, + failFast bool, + credentials account.M365Config, + collections map[string]*Collection, + statusCh chan<- *support.ConnectorOperationStatus, +) func(any) bool { + var isDirectorySet bool + return func(eventItem any) bool { + eventFolder := "Events" + user := scope.Get(selectors.ExchangeUser)[0] + if !isDirectorySet { + service, err := createService(credentials, failFast) + if err != nil { + errs = support.WrapAndAppend(user, err, errs) + return true + } + edc := NewCollection( + user, + []string{tenant, user, eventsCategory, eventFolder}, + events, + service, + statusCh, + ) + collections[eventFolder] = &edc + isDirectorySet = true + } + + event, ok := eventItem.(models.Eventable) + if !ok { + errs = support.WrapAndAppend( + user, + errors.New("event iteration failure"), + errs, + ) + return true + } + + collections[eventFolder].AddJob(*event.GetId()) + return true + } +} + // IterateAllContactsForCollection GraphIterateFunc for moving through // a ContactsCollectionsResponse using the msgraphgocore paging interface. // Contacts Ids are placed into a collection based upon the parent folder @@ -375,6 +489,22 @@ func optionsForMailFolders(moreOps []string) (*msfolder.MailFoldersRequestBuilde return options, nil } +// optionsForEvents ensures valid option inputs for exchange.Events +// @return is first call in Events().GetWithRequestConfigurationAndResponseHandler(options, handler) +func optionsForEvents(moreOps []string) (*msevents.EventsRequestBuilderGetRequestConfiguration, error) { + selecting, err := buildOptions(moreOps, events) + if err != nil { + return nil, err + } + requestParameters := &msevents.EventsRequestBuilderGetQueryParameters{ + Select: selecting, + } + options := &msevents.EventsRequestBuilderGetRequestConfiguration{ + QueryParameters: requestParameters, + } + return options, nil +} + // optionsForContacts transforms options into select query for MailContacts // @return is the first call in Contacts().GetWithRequestConfigurationAndResponseHandler(options, handler) func optionsForContacts(moreOps []string) (*mscontacts.ContactsRequestBuilderGetRequestConfiguration, error) { @@ -396,47 +526,11 @@ func optionsForContacts(moreOps []string) (*mscontacts.ContactsRequestBuilderGet // the second is an error. An error is returned if an unsupported option or optionIdentifier was used func buildOptions(options []string, optID optionIdentifier) ([]string, error) { var allowedOptions map[string]int - - fieldsForFolders := map[string]int{ - "displayName": 1, - "isHidden": 2, - "parentFolderId": 3, - "id": 4, - } - - fieldsForUsers := map[string]int{ - "birthday": 1, - "businessPhones": 2, - "city": 3, - "companyName": 4, - "department": 5, - "displayName": 6, - "employeeId": 7, - "id": 8, - } - - fieldsForMessages := map[string]int{ - "conservationId": 1, - "conversationIndex": 2, - "parentFolderId": 3, - "subject": 4, - "webLink": 5, - "id": 6, - } - - fieldsForContacts := map[string]int{ - "id": 1, - "companyName": 2, - "department": 3, - "displayName": 4, - "fileAs": 5, - "givenName": 6, - "manager": 7, - "parentFolderId": 8, - } returnedOptions := []string{"id"} switch optID { + case events: + allowedOptions = fieldsForEvents case contacts: allowedOptions = fieldsForContacts case folders: diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 0d8946c71..61329b0e9 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -328,6 +328,7 @@ func (gc *GraphConnector) createCollections( "user %s M365 query: %s", user, support.ConnectorStackErrorTrace(err)) } + pageIterator, err := msgraphgocore.NewPageIterator(response, &gc.graphService.adapter, transformer) if err != nil { return nil, err diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 812719a5d..27df4a82b 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -50,20 +50,34 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() { suite.connector, err = NewGraphConnector(a) suite.NoError(err) suite.user = "lidiah@8qzvrj.onmicrosoft.com" -} - -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector() { tester.LogTimeOfTest(suite.T()) - suite.NotNil(suite.connector) } -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() { - err := suite.connector.setTenantUsers() +// TestSetTenantUsers verifies GraphConnector's ability to query +// the users associated with the credentials +func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { + newConnector := GraphConnector{ + tenant: "test_tenant", + Users: make(map[string]string, 0), + status: nil, + statusCh: make(chan *support.ConnectorOperationStatus), + credentials: suite.connector.credentials, + } + service, err := newConnector.createService(false) + require.NoError(suite.T(), err) + newConnector.graphService = *service + + suite.Equal(len(newConnector.Users), 0) + err = newConnector.setTenantUsers() assert.NoError(suite.T(), err) - suite.Greater(len(suite.connector.Users), 0) + suite.Greater(len(newConnector.Users), 0) } -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataCollection() { +// TestExchangeDataCollection verifies interface between operation and +// GraphConnector remains stable to receive a non-zero amount of Collections +// for the Exchange Package. Enabled exchange applications: +// - mail +func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { userID := tester.M365UserID(suite.T()) sel := selectors.NewExchangeBackup() sel.Include(sel.Users([]string{userID})) @@ -86,7 +100,10 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataColl suite.Greater(len(exchangeData.FullPath()), 2) } -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_MailRegressionTest() { +// TestMailSerializationRegression verifies that all mail data stored in the +// test account can be successfully downloaded into bytes and restored into +// M365 mail objects +func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() { t := suite.T() user := tester.M365UserID(suite.T()) sel := selectors.NewExchangeBackup() @@ -123,8 +140,10 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_MailRegressionTe } } -// TestGraphConnector_TestContactSequence verifies retrieval sequence -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSequence() { +// TestContactBackupSequence verifies ability to query contact items +// and to store contact within Collection. Downloaded contacts are run through +// a regression test to ensure that downloaded items can be uploaded. +func (suite *GraphConnectorIntegrationSuite) TestContactBackupSequence() { userID := tester.M365UserID(suite.T()) sel := selectors.NewExchangeBackup() sel.Include(sel.Users([]string{userID})) @@ -149,9 +168,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSeque read, err := buf.ReadFrom(stream.ToReader()) suite.NoError(err) suite.NotZero(read) - message, err := support.CreateMessageFromBytes(buf.Bytes()) - suite.NotNil(message) - suite.NoError(err) + contact, err := support.CreateContactFromBytes(buf.Bytes()) + assert.NotNil(t, contact) + assert.NoError(t, err) } number++ @@ -160,9 +179,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSeque suite.Greater(len(collections), 0) } -// TestGraphConnector_restoreMessages uses mock data to ensure GraphConnector -// is able to restore a messageable item to a Mailbox. -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages() { +// TestRestoreMessages uses mock data to ensure GraphConnector +// is able to restore a single messageable item to a Mailbox. +func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() { user := tester.M365UserID(suite.T()) if len(user) == 0 { suite.T().Skip("Environment not configured: missing m365 test user") @@ -172,8 +191,8 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages( assert.NoError(suite.T(), err) } -// TestGraphConnector_SingleMailFolderCollectionQuery verifies that single folder support -// enabled createCollections +// TestGraphConnector_SingleMailFolderCollectionQuery verifies single folder support +// for Backup operation func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolderCollectionQuery() { t := suite.T() sel := selectors.NewExchangeBackup() @@ -184,18 +203,51 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder require.NoError(t, err) suite.Equal(len(collections), 1) for _, edc := range collections { + number := 0 streamChannel := edc.Items() // Verify that each message can be restored for stream := range streamChannel { + testName := fmt.Sprintf("%s_InboxMessage_%d", edc.FullPath()[1], number) + suite.T().Run(testName, func(t *testing.T) { + buf := &bytes.Buffer{} + read, err := buf.ReadFrom(stream.ToReader()) + suite.NoError(err) + suite.NotZero(read) + message, err := support.CreateMessageFromBytes(buf.Bytes()) + suite.NotNil(message) + suite.NoError(err) + number++ + }) + } + } + } +} + +// TestEventsBackupSequence ensures functionality of createCollections +// to be able to successfully query, download and restore event objects +func (suite *GraphConnectorIntegrationSuite) TestEventsBackupSequence() { + t := suite.T() + sel := selectors.NewExchangeBackup() + sel.Include(sel.Events([]string{suite.user}, []string{selectors.AnyTgt})) + scopes := sel.Scopes() + assert.Greater(t, len(scopes), 0) + collections, err := suite.connector.createCollections(context.Background(), scopes[0]) + require.NoError(t, err) + suite.Greater(len(collections), 0) + for _, edc := range collections { + streamChannel := edc.Items() + number := 0 + for stream := range streamChannel { + testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[1], number) + suite.T().Run(testName, func(t *testing.T) { buf := &bytes.Buffer{} read, err := buf.ReadFrom(stream.ToReader()) suite.NoError(err) suite.NotZero(read) - message, err := support.CreateMessageFromBytes(buf.Bytes()) - suite.NotNil(message) - suite.NoError(err) - - } + event, err := support.CreateEventFromBytes(buf.Bytes()) + assert.NotNil(t, event) + assert.NoError(t, err) + }) } } } @@ -204,9 +256,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder // Exchange Functions //------------------------------------------------------- -// TestGraphConnector_CreateAndDeleteFolder ensures msgraph application has the ability +// TestCreateAndDeleteFolder ensures GraphConnector has the ability // to create and remove folders within the tenant -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_CreateAndDeleteFolder() { +func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteFolder() { userID := tester.M365UserID(suite.T()) now := time.Now() folderName := "TestFolder: " + common.FormatSimpleDateTime(now) @@ -218,9 +270,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_CreateAndDeleteF } } -// TestGraphConnector_GetMailFolderID verifies the ability to retrieve folder ID of folders +// TestGetMailFolderID verifies the ability to retrieve folder ID of folders // at the top level of the file tree -func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_GetMailFolderID() { +func (suite *GraphConnectorIntegrationSuite) TestGetMailFolderID() { userID := tester.M365UserID(suite.T()) folderName := "Inbox" folderID, err := exchange.GetMailFolderID(&suite.connector.graphService, folderName, userID) diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index 724d4894f..51c448865 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -41,3 +41,13 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { contact := parsable.(models.Contactable) return contact, nil } + +// CreateEventFromBytes transforms given bytes into models.Eventable object +func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { + parsable, err := CreateFromBytes(bytes, models.CreateEventFromDiscriminatorValue) + if err != nil { + return nil, err + } + event := parsable.(models.Eventable) + return event, nil +}