GC: Events Support added. Not connected within ExchangeDataCollection (#516)

m365 events added for querying and iteration. Test suite added. Mock data is in a different PR.

src/internal/connector/exchange/exchange_data_collection.go

@@ -93,6 +93,8 @@ func getPopulateFunction(optID optionIdentifier) populater {
 		return PopulateForMailCollection
 	case contacts:
 		return PopulateForContactCollection
+	case events:
+		return PopulateForEventCollection
 	default:
 		return nil
 	}
@@ -208,6 +210,95 @@ func PopulateForMailCollection(
 	statusChannel <- status
 }
 
+func PopulateForEventCollection(
+	ctx context.Context,
+	service graph.Service,
+	user string,
+	jobs []string,
+	dataChannel chan<- data.Stream,
+	statusChannel chan<- *support.ConnectorOperationStatus,
+) {
+	var (
+		errs                    error
+		attemptedItems, success int
+	)
+	objectWriter := kw.NewJsonSerializationWriter()
+
+	for _, task := range jobs {
+		response, err := service.Client().UsersById(user).EventsById(task).Get()
+		if err != nil {
+			trace := support.ConnectorStackErrorTrace(err)
+			errs = support.WrapAndAppend(
+				user,
+				errors.Wrapf(err, "unable to retrieve items %s; details: %s", task, trace),
+				errs,
+			)
+			continue
+		}
+		err = eventToDataCollection(ctx, service.Client(), objectWriter, dataChannel, response, user)
+		if err != nil {
+			errs = support.WrapAndAppend(user, err, errs)
+
+			if service.ErrPolicy() {
+				break
+			}
+			continue
+		}
+		success++
+	}
+	close(dataChannel)
+	attemptedItems += len(jobs)
+	status := support.CreateStatus(ctx, support.Backup, attemptedItems, success, 1, errs)
+	logger.Ctx(ctx).Debug(status.String())
+	statusChannel <- status
+}
+
+func eventToDataCollection(
+	ctx context.Context,
+	client *msgraphsdk.GraphServiceClient,
+	objectWriter *kw.JsonSerializationWriter,
+	dataChannel chan<- data.Stream,
+	event models.Eventable,
+	user string,
+) error {
+	var err error
+	defer objectWriter.Close()
+	if *event.GetHasAttachments() {
+		var retriesErr error
+		for count := 0; count < numberOfRetries; count++ {
+			attached, err := client.
+				UsersById(user).
+				EventsById(*event.GetId()).
+				Attachments().
+				Get()
+			retriesErr = err
+			if err == nil && attached != nil {
+				event.SetAttachments(attached.GetValue())
+				break
+			}
+		}
+		if retriesErr != nil {
+			logger.Ctx(ctx).Debug("exceeded maximum retries")
+			return support.WrapAndAppend(
+				*event.GetId(),
+				errors.Wrap(retriesErr, "attachment failed"),
+				nil)
+		}
+	}
+	err = objectWriter.WriteObjectValue("", event)
+	if err != nil {
+		return support.SetNonRecoverableError(errors.Wrap(err, *event.GetId()))
+	}
+	byteArray, err := objectWriter.GetSerializedContent()
+	if err != nil {
+		return support.WrapAndAppend(*event.GetId(), errors.Wrap(err, "serializing content"), nil)
+	}
+	if byteArray != nil {
+		dataChannel <- &Stream{id: *event.GetId(), message: byteArray, info: nil}
+	}
+	return nil
+}
+
 func contactToDataCollection(
 	ctx context.Context,
 	client *msgraphsdk.GraphServiceClient,

src/internal/connector/exchange/exchange_service_test.go

@@ -190,9 +190,11 @@ func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
 	}
 }
 
-// TestSetupExchangeCollection ensures that the helper
-// function SetupExchangeCollectionVars returns a non-nil variable for returns
-// in regards to the selector.ExchangeScope.
+// TestSetupExchangeCollection ensures SetupExchangeCollectionVars returns a non-nil variable for
+// the following selector types:
+// - Mail
+// - Contacts
+// - Events
 func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() {
 	userID := tester.M365UserID(suite.T())
 	sel := selectors.NewExchangeBackup()
@@ -204,13 +206,10 @@ func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() {
 	for _, test := range scopes {
 		suite.T().Run(test.Category().String(), func(t *testing.T) {
 			discriminateFunc, graphQuery, iterFunc, err := SetupExchangeCollectionVars(test)
-			if test.Category() == selectors.ExchangeMailFolder ||
-				test.Category() == selectors.ExchangeContactFolder {
 			assert.NoError(t, err)
 			assert.NotNil(t, discriminateFunc)
 			assert.NotNil(t, graphQuery)
 			assert.NotNil(t, iterFunc)
-			}
 		})
 	}
 }

src/internal/connector/exchange/optionidentifier_string.go

@@ -10,13 +10,14 @@ func _() {
 	var x [1]struct{}
 	_ = x[unknown-0]
 	_ = x[folders-1]
-	_ = x[messages-2]
-	_ = x[users-3]
+	_ = x[events-2]
+	_ = x[messages-3]
+	_ = x[users-4]
 }
 
-const _optionIdentifier_name = "unknownfoldersmessagesusers"
+const _optionIdentifier_name = "unknownfolderseventsmessagesusers"
 
-var _optionIdentifier_index = [...]uint8{0, 7, 14, 22, 27}
+var _optionIdentifier_index = [...]uint8{0, 7, 14, 20, 28, 33}
 
 func (i optionIdentifier) String() string {
 	if i < 0 || i >= optionIdentifier(len(_optionIdentifier_index)-1) {

src/internal/connector/exchange/service_functions.go

@@ -194,6 +194,12 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
 			IterateAndFilterMessagesForCollections,
 			nil
 	}
+	if scope.IncludesCategory(selectors.ExchangeEvent) {
+		return models.CreateEventCollectionResponseFromDiscriminatorValue,
+			GetAllEventsForUser,
+			IterateSelectAllEventsForCollections,
+			nil
+	}
 
 	if scope.IncludesCategory(selectors.ExchangeContactFolder) {
 		return models.CreateContactFromDiscriminatorValue,

src/internal/connector/exchange/service_query.go

@@ -7,6 +7,7 @@ import (
 	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	mscontacts "github.com/microsoftgraph/msgraph-sdk-go/users/item/contacts"
+	msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events"
 	msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders"
 	msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages"
 	msitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages/item"
@@ -18,17 +19,74 @@ import (
 	"github.com/alcionai/corso/pkg/selectors"
 )
 
+var (
+	fieldsForEvents = map[string]int{
+		"calendar":          1,
+		"end":               2,
+		"id":                3,
+		"isOnlineMeeting":   4,
+		"isReminderOn":      5,
+		"responseStatus":    6,
+		"responseRequested": 7,
+		"showAs":            8,
+		"subject":           9,
+	}
+
+	fieldsForFolders = map[string]int{
+		"childFolderCount": 1,
+		"displayName":      2,
+		"id":               3,
+		"isHidden":         4,
+		"parentFolderId":   5,
+		"totalItemCount":   6,
+		"unreadItemCount":  7,
+	}
+
+	fieldsForUsers = map[string]int{
+		"birthday":       1,
+		"businessPhones": 2,
+		"city":           3,
+		"companyName":    4,
+		"department":     5,
+		"displayName":    6,
+		"employeeId":     7,
+		"id":             8,
+	}
+
+	fieldsForMessages = map[string]int{
+		"conservationId":    1,
+		"conversationIndex": 2,
+		"parentFolderId":    3,
+		"subject":           4,
+		"webLink":           5,
+		"id":                6,
+	}
+
+	fieldsForContacts = map[string]int{
+		"id":             1,
+		"companyName":    2,
+		"department":     3,
+		"displayName":    4,
+		"fileAs":         5,
+		"givenName":      6,
+		"manager":        7,
+		"parentFolderId": 8,
+	}
+)
+
 type optionIdentifier int
 
 const (
 	mailCategory     = "mail"
 	contactsCategory = "contacts"
+	eventsCategory   = "events"
 )
 
 //go:generate stringer -type=optionIdentifier
 const (
 	unknown optionIdentifier = iota
 	folders
+	events
 	messages
 	users
 	contacts
@@ -73,6 +131,17 @@ func GetAllFolderNamesForUser(gs graph.Service, user string) (absser.Parsable, e
 	return gs.Client().UsersById(user).MailFolders().GetWithRequestConfigurationAndResponseHandler(options, nil)
 }
 
+// GetAllEvents for User. Default returns EventResponseCollection for events in the future
+// of the time that the call was made. There a
+func GetAllEventsForUser(gs graph.Service, user string) (absser.Parsable, error) {
+	options, err := optionsForEvents([]string{"id", "calendar"})
+	if err != nil {
+		return nil, err
+	}
+
+	return gs.Client().UsersById(user).Events().GetWithRequestConfigurationAndResponseHandler(options, nil)
+}
+
 // GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator)
 // @returns a callback func that works with msgraphgocore.PageIterator.Iterate function
 type GraphIterateFunc func(
@@ -130,6 +199,51 @@ func IterateSelectAllMessagesForCollections(
 	}
 }
 
+func IterateSelectAllEventsForCollections(
+	tenant string,
+	scope selectors.ExchangeScope,
+	errs error,
+	failFast bool,
+	credentials account.M365Config,
+	collections map[string]*Collection,
+	statusCh chan<- *support.ConnectorOperationStatus,
+) func(any) bool {
+	var isDirectorySet bool
+	return func(eventItem any) bool {
+		eventFolder := "Events"
+		user := scope.Get(selectors.ExchangeUser)[0]
+		if !isDirectorySet {
+			service, err := createService(credentials, failFast)
+			if err != nil {
+				errs = support.WrapAndAppend(user, err, errs)
+				return true
+			}
+			edc := NewCollection(
+				user,
+				[]string{tenant, user, eventsCategory, eventFolder},
+				events,
+				service,
+				statusCh,
+			)
+			collections[eventFolder] = &edc
+			isDirectorySet = true
+		}
+
+		event, ok := eventItem.(models.Eventable)
+		if !ok {
+			errs = support.WrapAndAppend(
+				user,
+				errors.New("event iteration failure"),
+				errs,
+			)
+			return true
+		}
+
+		collections[eventFolder].AddJob(*event.GetId())
+		return true
+	}
+}
+
 // IterateAllContactsForCollection GraphIterateFunc for moving through
 // a ContactsCollectionsResponse using the msgraphgocore paging interface.
 // Contacts Ids are placed into a collection based upon the parent folder
@@ -375,6 +489,22 @@ func optionsForMailFolders(moreOps []string) (*msfolder.MailFoldersRequestBuilde
 	return options, nil
 }
 
+// optionsForEvents ensures valid option inputs for exchange.Events
+// @return is first call in Events().GetWithRequestConfigurationAndResponseHandler(options, handler)
+func optionsForEvents(moreOps []string) (*msevents.EventsRequestBuilderGetRequestConfiguration, error) {
+	selecting, err := buildOptions(moreOps, events)
+	if err != nil {
+		return nil, err
+	}
+	requestParameters := &msevents.EventsRequestBuilderGetQueryParameters{
+		Select: selecting,
+	}
+	options := &msevents.EventsRequestBuilderGetRequestConfiguration{
+		QueryParameters: requestParameters,
+	}
+	return options, nil
+}
+
 // optionsForContacts transforms options into select query for MailContacts
 // @return is the first call in Contacts().GetWithRequestConfigurationAndResponseHandler(options, handler)
 func optionsForContacts(moreOps []string) (*mscontacts.ContactsRequestBuilderGetRequestConfiguration, error) {
@@ -396,47 +526,11 @@ func optionsForContacts(moreOps []string) (*mscontacts.ContactsRequestBuilderGet
 // the second is an error. An error is returned if an unsupported option or optionIdentifier was used
 func buildOptions(options []string, optID optionIdentifier) ([]string, error) {
 	var allowedOptions map[string]int
-
-	fieldsForFolders := map[string]int{
-		"displayName":    1,
-		"isHidden":       2,
-		"parentFolderId": 3,
-		"id":             4,
-	}
-
-	fieldsForUsers := map[string]int{
-		"birthday":       1,
-		"businessPhones": 2,
-		"city":           3,
-		"companyName":    4,
-		"department":     5,
-		"displayName":    6,
-		"employeeId":     7,
-		"id":             8,
-	}
-
-	fieldsForMessages := map[string]int{
-		"conservationId":    1,
-		"conversationIndex": 2,
-		"parentFolderId":    3,
-		"subject":           4,
-		"webLink":           5,
-		"id":                6,
-	}
-
-	fieldsForContacts := map[string]int{
-		"id":             1,
-		"companyName":    2,
-		"department":     3,
-		"displayName":    4,
-		"fileAs":         5,
-		"givenName":      6,
-		"manager":        7,
-		"parentFolderId": 8,
-	}
 	returnedOptions := []string{"id"}
 
 	switch optID {
+	case events:
+		allowedOptions = fieldsForEvents
 	case contacts:
 		allowedOptions = fieldsForContacts
 	case folders:

src/internal/connector/graph_connector.go

@@ -328,6 +328,7 @@ func (gc *GraphConnector) createCollections(
 			"user %s M365 query: %s",
 			user, support.ConnectorStackErrorTrace(err))
 	}
+
 	pageIterator, err := msgraphgocore.NewPageIterator(response, &gc.graphService.adapter, transformer)
 	if err != nil {
 		return nil, err

src/internal/connector/graph_connector_test.go

@@ -50,20 +50,34 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
 	suite.connector, err = NewGraphConnector(a)
 	suite.NoError(err)
 	suite.user = "lidiah@8qzvrj.onmicrosoft.com"
-}
-
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector() {
 	tester.LogTimeOfTest(suite.T())
-	suite.NotNil(suite.connector)
 }
 
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_setTenantUsers() {
-	err := suite.connector.setTenantUsers()
+// TestSetTenantUsers verifies GraphConnector's ability to query
+// the users associated with the credentials
+func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
+	newConnector := GraphConnector{
+		tenant:      "test_tenant",
+		Users:       make(map[string]string, 0),
+		status:      nil,
+		statusCh:    make(chan *support.ConnectorOperationStatus),
+		credentials: suite.connector.credentials,
+	}
+	service, err := newConnector.createService(false)
+	require.NoError(suite.T(), err)
+	newConnector.graphService = *service
+
+	suite.Equal(len(newConnector.Users), 0)
+	err = newConnector.setTenantUsers()
 	assert.NoError(suite.T(), err)
-	suite.Greater(len(suite.connector.Users), 0)
+	suite.Greater(len(newConnector.Users), 0)
 }
 
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataCollection() {
+// TestExchangeDataCollection verifies interface between operation and
+// GraphConnector remains stable to receive a non-zero amount of Collections
+// for the Exchange Package. Enabled exchange applications:
+// - mail
+func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
 	userID := tester.M365UserID(suite.T())
 	sel := selectors.NewExchangeBackup()
 	sel.Include(sel.Users([]string{userID}))
@@ -86,7 +100,10 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataColl
 	suite.Greater(len(exchangeData.FullPath()), 2)
 }
 
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_MailRegressionTest() {
+// TestMailSerializationRegression verifies that all mail data stored in the
+// test account can be successfully downloaded into bytes and restored into
+// M365 mail objects
+func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
 	t := suite.T()
 	user := tester.M365UserID(suite.T())
 	sel := selectors.NewExchangeBackup()
@@ -123,8 +140,10 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_MailRegressionTe
 	}
 }
 
-// TestGraphConnector_TestContactSequence verifies retrieval sequence
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSequence() {
+// TestContactBackupSequence verifies ability to query contact items
+// and to store contact within Collection. Downloaded contacts are run through
+// a regression test to ensure that downloaded items can be uploaded.
+func (suite *GraphConnectorIntegrationSuite) TestContactBackupSequence() {
 	userID := tester.M365UserID(suite.T())
 	sel := selectors.NewExchangeBackup()
 	sel.Include(sel.Users([]string{userID}))
@@ -149,9 +168,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSeque
 				read, err := buf.ReadFrom(stream.ToReader())
 				suite.NoError(err)
 				suite.NotZero(read)
-				message, err := support.CreateMessageFromBytes(buf.Bytes())
-				suite.NotNil(message)
-				suite.NoError(err)
+				contact, err := support.CreateContactFromBytes(buf.Bytes())
+				assert.NotNil(t, contact)
+				assert.NoError(t, err)
 
 			}
 			number++
@@ -160,9 +179,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_TestContactSeque
 	suite.Greater(len(collections), 0)
 }
 
-// TestGraphConnector_restoreMessages uses mock data to ensure GraphConnector
-// is able to restore a messageable item to a Mailbox.
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages() {
+// TestRestoreMessages uses mock data to ensure GraphConnector
+// is able to restore a single messageable item to a Mailbox.
+func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() {
 	user := tester.M365UserID(suite.T())
 	if len(user) == 0 {
 		suite.T().Skip("Environment not configured: missing m365 test user")
@@ -172,8 +191,8 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages(
 	assert.NoError(suite.T(), err)
 }
 
-// TestGraphConnector_SingleMailFolderCollectionQuery verifies that single folder support
-// enabled createCollections
+// TestGraphConnector_SingleMailFolderCollectionQuery verifies single folder support
+// for Backup operation
 func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolderCollectionQuery() {
 	t := suite.T()
 	sel := selectors.NewExchangeBackup()
@@ -184,9 +203,12 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder
 		require.NoError(t, err)
 		suite.Equal(len(collections), 1)
 		for _, edc := range collections {
+			number := 0
 			streamChannel := edc.Items()
 			// Verify that each message can be restored
 			for stream := range streamChannel {
+				testName := fmt.Sprintf("%s_InboxMessage_%d", edc.FullPath()[1], number)
+				suite.T().Run(testName, func(t *testing.T) {
 					buf := &bytes.Buffer{}
 					read, err := buf.ReadFrom(stream.ToReader())
 					suite.NoError(err)
@@ -194,19 +216,49 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder
 					message, err := support.CreateMessageFromBytes(buf.Bytes())
 					suite.NotNil(message)
 					suite.NoError(err)
-
+					number++
+				})
 			}
 		}
 	}
 }
 
+// TestEventsBackupSequence ensures functionality of createCollections
+// to be able to successfully query, download and restore event objects
+func (suite *GraphConnectorIntegrationSuite) TestEventsBackupSequence() {
+	t := suite.T()
+	sel := selectors.NewExchangeBackup()
+	sel.Include(sel.Events([]string{suite.user}, []string{selectors.AnyTgt}))
+	scopes := sel.Scopes()
+	assert.Greater(t, len(scopes), 0)
+	collections, err := suite.connector.createCollections(context.Background(), scopes[0])
+	require.NoError(t, err)
+	suite.Greater(len(collections), 0)
+	for _, edc := range collections {
+		streamChannel := edc.Items()
+		number := 0
+		for stream := range streamChannel {
+			testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[1], number)
+			suite.T().Run(testName, func(t *testing.T) {
+				buf := &bytes.Buffer{}
+				read, err := buf.ReadFrom(stream.ToReader())
+				suite.NoError(err)
+				suite.NotZero(read)
+				event, err := support.CreateEventFromBytes(buf.Bytes())
+				assert.NotNil(t, event)
+				assert.NoError(t, err)
+			})
+		}
+	}
+}
+
 ///------------------------------------------------------------
 // Exchange Functions
 //-------------------------------------------------------
 
-//  TestGraphConnector_CreateAndDeleteFolder ensures msgraph application has the ability
+//  TestCreateAndDeleteFolder ensures GraphConnector has the ability
 //  to create and remove folders within the tenant
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_CreateAndDeleteFolder() {
+func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteFolder() {
 	userID := tester.M365UserID(suite.T())
 	now := time.Now()
 	folderName := "TestFolder: " + common.FormatSimpleDateTime(now)
@@ -218,9 +270,9 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_CreateAndDeleteF
 	}
 }
 
-// TestGraphConnector_GetMailFolderID verifies the ability to retrieve folder ID of folders
+// TestGetMailFolderID verifies the ability to retrieve folder ID of folders
 // at the top level of the file tree
-func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_GetMailFolderID() {
+func (suite *GraphConnectorIntegrationSuite) TestGetMailFolderID() {
 	userID := tester.M365UserID(suite.T())
 	folderName := "Inbox"
 	folderID, err := exchange.GetMailFolderID(&suite.connector.graphService, folderName, userID)

src/internal/connector/support/m365Support.go

@@ -41,3 +41,13 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) {
 	contact := parsable.(models.Contactable)
 	return contact, nil
 }
+
+// CreateEventFromBytes transforms given bytes into models.Eventable object
+func CreateEventFromBytes(bytes []byte) (models.Eventable, error) {
+	parsable, err := CreateFromBytes(bytes, models.CreateEventFromDiscriminatorValue)
+	if err != nil {
+		return nil, err
+	}
+	event := parsable.(models.Eventable)
+	return event, nil
+}