larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first pass migrate graph api to subfolder (#2003)

Browse Source 
## Description

In order to establish a standard api around our
graph client usage, and thus be able to mock
for testing, we need to migrate graph client
usage into another pacakge.  This is the first
step in that process of refactoring.

All changes are code relocation importing and
exporting may change as needed.  No logic
was altered.

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🤖 Test
- [x] 🐹 Trivial/Minor

## Issue(s)

* #1967

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-01-03 18:35:32 -07:00 committed by GitHub
parent 753ede5a1a
commit e5edbfd77c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1301 additions and 1178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/cmd/getM365/getItem.go
View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -94,7 +95,7 @@ func runDisplayM365JSON(
gs graph.Servicer,
) error {
var (
get exchange.GraphRetrievalFunc
get api.GraphRetrievalFunc
serializeFunc exchange.GraphSerializeFunc
cat = graph.StringToPathCategory(category)
)

50
src/internal/connector/exchange/api/api.go Normal file
View File

@ -0,0 +1,50 @@
package api
import (
"context"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/internal/connector/graph"
)
// ---------------------------------------------------------------------------
// common types
// ---------------------------------------------------------------------------
// DeltaUpdate holds the results of a current delta token. It normally
// gets produced when aggregating the addition and removal of items in
// a delta-queriable folder.
type DeltaUpdate struct {
// the deltaLink itself
URL string
// true if the old delta was marked as invalid
Reset bool
}
// GraphQuery represents functions which perform exchange-specific queries
// into M365 backstore. Responses -> returned items will only contain the information
// that is included in the options
// TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, gs graph.Servicer, userID string) (serialization.Parsable, error)
// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve
// the default associated data of a M365 object. This varies by object. Additional
// Queries must be run to obtain the omitted fields.
type GraphRetrievalFunc func(
ctx context.Context,
gs graph.Servicer,
user, m365ID string,
) (serialization.Parsable, error)
// ---------------------------------------------------------------------------
// interfaces
// ---------------------------------------------------------------------------
// API is a struct used to fulfill the interface for exchange
// queries that are traditionally backed by GraphAPI. A
// struct is used in this case, instead of deferring to
// pure function wrappers, so that the boundary separates the
// granular implementation of the graphAPI and kiota away
// from the exchange package's broader intents.
// type API struct{}

189
src/internal/connector/exchange/api/api_test.go Normal file
View File

@ -0,0 +1,189 @@
package api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
)
type ExchangeServiceSuite struct {
suite.Suite
gs graph.Servicer
credentials account.M365Config
}
func TestExchangeServiceSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
tester.CorsoGraphConnectorExchangeTests)
suite.Run(t, new(ExchangeServiceSuite))
}
func (suite *ExchangeServiceSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.credentials = m365
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
}
func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := optionsForCalendars(test.params)
test.checkError(t, err)
})
}
}
// TestOptionsForFolders ensures that approved query options
// are added to the RequestBuildConfiguration. Expected will always be +1
// on than the input as "id" are always included within the select parameters
func (suite *ExchangeServiceSuite) TestOptionsForFolders() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Folder Option",
params: []string{"parentFolderId"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Folder Options: Valid",
params: []string{"displayName", "isHidden"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Folder option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
config, err := optionsForMailFolders(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(config.QueryParameters.Select))
}
})
}
}
// TestOptionsForContacts similar to TestExchangeService_optionsForFolders
func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Contact Option",
params: []string{"displayName"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Contact Options: Valid",
params: []string{"displayName", "parentFolderId"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Contact Option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
options, err := optionsForContacts(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(options.QueryParameters.Select))
}
})
}
}
// TestGraphQueryFunctions verifies if Query functions APIs
// through Microsoft Graph are functional
func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
ctx, flush := tester.NewContext()
defer flush()
userID := tester.M365UserID(suite.T())
tests := []struct {
name string
function GraphQuery
}{
{
name: "GraphQuery: Get All ContactFolders",
function: GetAllContactFolderNamesForUser,
},
{
name: "GraphQuery: Get All Calendars for User",
function: GetAllCalendarNamesForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
response, err := test.function(ctx, suite.gs, userID)
assert.NoError(t, err)
assert.NotNil(t, response)
})
}
}

203
src/internal/connector/exchange/api/contacts.go Normal file
View File

@ -0,0 +1,203 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateContactFolder makes a contact folder with the displayName of folderName.
// If successful, returns the created folder object.
func CreateContactFolder(
ctx context.Context,
gs graph.Servicer,
user, folderName string,
) (models.ContactFolderable, error) {
requestBody := models.NewContactFolder()
temp := folderName
requestBody.SetDisplayName(&temp)
return gs.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
}
// DeleteContactFolder deletes the ContactFolder associated with the M365 ID if permissions are valid.
// Errors returned if the function call was not successful.
func DeleteContactFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
}
// RetrieveContactDataForUser is a GraphRetrievalFun that returns all associated fields.
func RetrieveContactDataForUser(
ctx context.Context,
gs graph.Servicer,
user, m365ID string,
) (serialization.Parsable, error) {
return gs.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil)
}
// GetAllContactFolderNamesForUser is a GraphQuery function for getting
// ContactFolderId and display names for contacts. All other information is omitted.
// Does not return the default Contact Folder
func GetAllContactFolderNamesForUser(
ctx context.Context,
gs graph.Servicer,
user string,
) (serialization.Parsable, error) {
options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).ContactFolders().Get(ctx, options)
}
func GetContactFolderByID(
ctx context.Context,
gs graph.Servicer,
userID, dirID string,
optionalFields ...string,
) (models.ContactFolderable, error) {
fields := append([]string{"displayName", "parentFolderId"}, optionalFields...)
ofcf, err := optionsForContactFolderByID(fields)
if err != nil {
return nil, errors.Wrapf(err, "options for contact folder: %v", fields)
}
return gs.Client().
UsersById(userID).
ContactFoldersById(dirID).
Get(ctx, ofcf)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func GetContactChildFoldersBuilder(
ctx context.Context,
gs graph.Servicer,
userID, baseDirID string,
optionalFields ...string,
) (
*users.ItemContactFoldersItemChildFoldersRequestBuilder,
*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration,
error,
) {
fields := append([]string{"displayName", "parentFolderId"}, optionalFields...)
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return nil, nil, errors.Wrapf(err, "options for contact child folders: %v", fields)
}
builder := gs.Client().
UsersById(userID).
ContactFoldersById(baseDirID).
ChildFolders()
return builder, ofcf, nil
}
// FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts
// of the targeted directory
func FetchContactIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter()))
// note: happy path, not the error condition
if err == nil {
return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, DeltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
ContactFoldersById(directoryID).
Contacts().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return ids, removedIDs, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

129
src/internal/connector/exchange/api/events.go Normal file
View File

@ -0,0 +1,129 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go
func CreateCalendar(ctx context.Context, gs graph.Servicer, user, calendarName string) (models.Calendarable, error) {
requestbody := models.NewCalendar()
requestbody.SetName(&calendarName)
return gs.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
}
// DeleteCalendar removes calendar from user's M365 account
// Reference: https://docs.microsoft.com/en-us/graph/api/calendar-delete?view=graph-rest-1.0&tabs=go
func DeleteCalendar(ctx context.Context, gs graph.Servicer, user, calendarID string) error {
return gs.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
}
// RetrieveEventDataForUser is a GraphRetrievalFunc that returns event data.
// Calendarable and attachment fields are omitted due to size
func RetrieveEventDataForUser(
ctx context.Context,
gs graph.Servicer,
user, m365ID string,
) (serialization.Parsable, error) {
return gs.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil)
}
func GetAllCalendarNamesForUser(ctx context.Context, gs graph.Servicer, user string) (serialization.Parsable, error) {
options, err := optionsForCalendars([]string{"name", "owner"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Calendars().Get(ctx, options)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func GetCalendarsBuilder(
ctx context.Context,
gs graph.Servicer,
userID string,
optionalFields ...string,
) (
*users.ItemCalendarsRequestBuilder,
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
error,
) {
ofcf, err := optionsForCalendars(optionalFields)
if err != nil {
return nil, nil, errors.Wrapf(err, "options for event calendars: %v", optionalFields)
}
builder := gs.Client().
UsersById(userID).
Calendars()
return builder, ofcf, nil
}
// FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
func FetchEventIDsFromCalendar(
ctx context.Context,
gs graph.Servicer,
user, calendarID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
var (
errs *multierror.Error
ids []string
)
options, err := optionsForEventsByCalendar([]string{"id"})
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
builder := gs.Client().
UsersById(user).
CalendarsById(calendarID).
Events()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return nil, nil, DeltaUpdate{}, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("event with nil ID in calendar %s", calendarID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
ids = append(ids, *item.GetId())
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemCalendarsItemEventsRequestBuilder(*nextLink, gs.Adapter())
}
// Events don't have a delta endpoint so just return an empty string.
return ids, nil, DeltaUpdate{}, errs.ErrorOrNil()
}

189
src/internal/connector/exchange/api/mail.go Normal file
View File

@ -0,0 +1,189 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateMailFolder makes a mail folder iff a folder of the same name does not exist
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http
func CreateMailFolder(ctx context.Context, gs graph.Servicer, user, folder string) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
}
func CreateMailFolderWithParent(
ctx context.Context,
gs graph.Servicer,
user, folder, parentID string,
) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().
UsersById(user).
MailFoldersById(parentID).
ChildFolders().
Post(ctx, requestBody, nil)
}
// DeleteMailFolder removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/mailfolder-delete?view=graph-rest-1.0&tabs=http
func DeleteMailFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
}
// RetrieveMessageDataForUser is a GraphRetrievalFunc that returns message data.
// Attachment field is omitted due to size.
func RetrieveMessageDataForUser(
ctx context.Context,
gs graph.Servicer,
user, m365ID string,
) (serialization.Parsable, error) {
return gs.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil)
}
// GetMailFoldersBuilder retrieves all of the users current mail folders.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func GetAllMailFoldersBuilder(
ctx context.Context,
gs graph.Servicer,
userID string,
) *users.ItemMailFoldersDeltaRequestBuilder {
return gs.Client().
UsersById(userID).
MailFolders().
Delta()
}
func GetMailFolderByID(
ctx context.Context,
gs graph.Servicer,
userID, dirID string,
optionalFields ...string,
) (models.MailFolderable, error) {
ofmf, err := optionsForMailFoldersItem(optionalFields)
if err != nil {
return nil, errors.Wrapf(err, "options for mail folder: %v", optionalFields)
}
return gs.Client().
UsersById(userID).
MailFoldersById(dirID).
Get(ctx, ofmf)
}
// FetchMessageIDsFromDirectory function that returns a list of all the m365IDs of the exchange.Mail
// of the targeted directory
func FetchMessageIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()))
// note: happy path, not the error condition
if err == nil {
return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, DeltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
MailFoldersById(directoryID).
Messages().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return ids, removedIDs, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

82
src/internal/connector/exchange/query_options.go → src/internal/connector/exchange/api/options.go
View File

@ -1,9 +1,9 @@
package exchange
package api
import (
"fmt"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/microsoftgraph/msgraph-sdk-go/users"
)
// -----------------------------------------------------------------------
@ -74,16 +74,16 @@ var (
func optionsForFolderMessagesDelta(
moreOps []string,
) (*msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForMessages)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -94,7 +94,7 @@ func optionsForFolderMessagesDelta(
// @param moreOps should reflect elements from fieldsForCalendars
// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler
func optionsForCalendars(moreOps []string) (
*msuser.ItemCalendarsRequestBuilderGetRequestConfiguration,
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForCalendars)
@ -102,10 +102,10 @@ func optionsForCalendars(moreOps []string) (
return nil, err
}
// should be a CalendarsRequestBuilderGetRequestConfiguration
requestParams := &msuser.ItemCalendarsRequestBuilderGetQueryParameters{
requestParams := &users.ItemCalendarsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemCalendarsRequestBuilderGetRequestConfiguration{
options := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParams,
}
@ -115,7 +115,7 @@ func optionsForCalendars(moreOps []string) (
// optionsForContactFolders places allowed options for exchange.ContactFolder object
// @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler
func optionsForContactFolders(moreOps []string) (
*msuser.ItemContactFoldersRequestBuilderGetRequestConfiguration,
*users.ItemContactFoldersRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
@ -123,10 +123,10 @@ func optionsForContactFolders(moreOps []string) (
return nil, err
}
requestParameters := &msuser.ItemContactFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -134,7 +134,7 @@ func optionsForContactFolders(moreOps []string) (
}
func optionsForContactFolderByID(moreOps []string) (
*msuser.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration,
*users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
@ -142,10 +142,10 @@ func optionsForContactFolderByID(moreOps []string) (
return nil, err
}
requestParameters := &msuser.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -157,16 +157,16 @@ func optionsForContactFolderByID(moreOps []string) (
// @return is first call in MailFolders().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForMailFolders(
moreOps []string,
) (*msuser.ItemMailFoldersRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -178,16 +178,16 @@ func optionsForMailFolders(
// Returns first call in MailFoldersById().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForMailFoldersItem(
moreOps []string,
) (*msuser.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -196,35 +196,17 @@ func optionsForMailFoldersItem(
func optionsForContactFoldersItemDelta(
moreOps []string,
) (*msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForEvents ensures valid option inputs for exchange.Events
// @return is first call in Events().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForEvents(moreOps []string) (*msuser.ItemEventsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForEvents)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemEventsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemEventsRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -234,17 +216,17 @@ func optionsForEvents(moreOps []string) (*msuser.ItemEventsRequestBuilderGetRequ
// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar
func optionsForEventsByCalendar(
moreOps []string,
) (*msuser.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForEvents)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{
requestParameters := &users.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -254,16 +236,16 @@ func optionsForEventsByCalendar(
// optionsForContactChildFolders builds a contacts child folders request.
func optionsForContactChildFolders(
moreOps []string,
) (*msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -272,16 +254,16 @@ func optionsForContactChildFolders(
// optionsForContacts transforms options into select query for MailContacts
// @return is the first call in Contacts().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForContacts(moreOps []string) (*msuser.ItemContactsRequestBuilderGetRequestConfiguration, error) {
func optionsForContacts(moreOps []string) (*users.ItemContactsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactsRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactsRequestBuilderGetRequestConfiguration{
options := &users.ItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}

31
src/internal/connector/exchange/contact_folder_cache.go
View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -24,19 +25,7 @@ func (cfc *contactFolderCache) populateContactRoot(
directoryID string,
baseContainerPath []string,
) error {
wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForContactFolderByID(wantedOpts)
if err != nil {
return errors.Wrapf(err, "getting options for contact folder cache: %v", wantedOpts)
}
f, err := cfc.
gs.
Client().
UsersById(cfc.userID).
ContactFoldersById(directoryID).
Get(ctx, opts)
f, err := api.GetContactFolderByID(ctx, cfc.gs, cfc.userID, directoryID)
if err != nil {
return errors.Wrapf(
err,
@ -65,21 +54,17 @@ func (cfc *contactFolderCache) Populate(
return err
}
var (
errs error
options, err = optionsForContactChildFolders([]string{"displayName", "parentFolderId"})
)
var errs error
builder, options, err := api.GetContactChildFoldersBuilder(
ctx,
cfc.gs,
cfc.userID,
baseID)
if err != nil {
return errors.Wrap(err, "contact cache resolver option")
}
builder := cfc.
gs.Client().
UsersById(cfc.userID).
ContactFoldersById(baseID).
ChildFolders()
for {
resp, err := builder.Get(ctx, options)
if err != nil {

192
src/internal/connector/exchange/container_resolver_test.go
View File

@ -11,9 +11,14 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// mocks and helpers
// ---------------------------------------------------------------------------
type mockContainer struct {
id *string
name *string
@ -34,6 +39,10 @@ func (m mockContainer) GetParentFolderId() *string {
return m.parentID
}
// ---------------------------------------------------------------------------
// unit suite
// ---------------------------------------------------------------------------
type FolderCacheUnitSuite struct {
suite.Suite
}
@ -284,6 +293,10 @@ func resolverWithContainers(numContainers int) (*containerResolver, []*mockCache
return resolver, containers
}
// ---------------------------------------------------------------------------
// configured unit suite
// ---------------------------------------------------------------------------
// TestConfiguredFolderCacheUnitSuite cannot run its tests in parallel.
type ConfiguredFolderCacheUnitSuite struct {
suite.Suite
@ -431,3 +444,182 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestAddToCache() {
require.NoError(t, err)
assert.Equal(t, m.expectedPath, p.String())
}
// ---------------------------------------------------------------------------
// integration suite
// ---------------------------------------------------------------------------
type FolderCacheIntegrationSuite struct {
suite.Suite
credentials account.M365Config
gs graph.Servicer
}
func TestFolderCacheIntegrationSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoConnectorExchangeFolderCacheTests)
suite.Run(t, new(FolderCacheIntegrationSuite))
}
func (suite *FolderCacheIntegrationSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.credentials = m365
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
}
// Testing to ensure that cache system works for in multiple different environments
func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
ctx, flush := tester.NewContext()
defer flush()
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
connector, err := createService(m365)
require.NoError(suite.T(), err)
var (
user = tester.M365UserID(suite.T())
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string
pathFunc1 func(t *testing.T) path.Path
pathFunc2 func(t *testing.T) path.Path
category path.CategoryType
}{
{
name: "Mail Cache Test",
category: path.EmailCategory,
pathFunc1: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Croix").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
pathFunc2: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Felicius").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
},
{
name: "Contact Cache Test",
category: path.ContactsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("HufflePuff").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Ravenclaw").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
{
name: "Event Cache Test",
category: path.EventsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Durmstrang").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Beauxbatons").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
}
)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc1(t),
folderName,
directoryCaches,
)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc2(t),
folderName,
directoryCaches,
)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})
}
}

5
src/internal/connector/exchange/event_calendar_cache.go
View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -31,7 +32,7 @@ func (ecc *eventCalendarCache) Populate(
ecc.containerResolver = newContainerResolver()
}
options, err := optionsForCalendars([]string{"name"})
builder, options, err := api.GetCalendarsBuilder(ctx, ecc.gs, ecc.userID, "name")
if err != nil {
return err
}
@ -41,8 +42,6 @@ func (ecc *eventCalendarCache) Populate(
directories = make([]graph.Container, 0)
)
builder := ecc.gs.Client().UsersById(ecc.userID).Calendars()
for {
resp, err := builder.Get(ctx, options)
if err != nil {

9
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -18,6 +18,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -135,14 +136,14 @@ func (col *Collection) Items() <-chan data.Stream {
// GetQueryAndSerializeFunc helper function that returns the two functions functions
// required to convert M365 identifier into a byte array filled with the serialized data
func GetQueryAndSerializeFunc(category path.CategoryType) (GraphRetrievalFunc, GraphSerializeFunc) {
func GetQueryAndSerializeFunc(category path.CategoryType) (api.GraphRetrievalFunc, GraphSerializeFunc) {
switch category {
case path.ContactsCategory:
return RetrieveContactDataForUser, serializeAndStreamContact
return api.RetrieveContactDataForUser, serializeAndStreamContact
case path.EventsCategory:
return RetrieveEventDataForUser, serializeAndStreamEvent
return api.RetrieveEventDataForUser, serializeAndStreamEvent
case path.EmailCategory:
return RetrieveMessageDataForUser, serializeAndStreamMessage
return api.RetrieveMessageDataForUser, serializeAndStreamMessage
// Unsupported options returns nil, nil
default:
return nil, nil

599
src/internal/connector/exchange/exchange_service_test.go
View File

@ -1,599 +0,0 @@
package exchange
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
)
type ExchangeServiceSuite struct {
suite.Suite
credentials account.M365Config
gs graph.Servicer
}
func TestExchangeServiceSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
tester.CorsoGraphConnectorExchangeTests)
suite.Run(t, new(ExchangeServiceSuite))
}
func (suite *ExchangeServiceSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365)
require.NoError(t, err)
suite.credentials = m365
suite.gs = service
}
// TestCreateService verifies that services are created
// when called with the correct range of params. NOTE:
// incorrect tenant or password information will NOT generate
// an error.
func (suite *ExchangeServiceSuite) TestCreateService() {
creds := suite.credentials
invalidCredentials := suite.credentials
invalidCredentials.AzureClientSecret = ""
tests := []struct {
name string
credentials account.M365Config
checkErr assert.ErrorAssertionFunc
}{
{
name: "Valid Service Creation",
credentials: creds,
checkErr: assert.NoError,
},
{
name: "Invalid Service Creation",
credentials: invalidCredentials,
checkErr: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
t.Log(test.credentials.AzureClientSecret)
_, err := createService(test.credentials)
test.checkErr(t, err)
})
}
}
func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := optionsForCalendars(test.params)
test.checkError(t, err)
})
}
}
// TestOptionsForFolders ensures that approved query options
// are added to the RequestBuildConfiguration. Expected will always be +1
// on than the input as "id" are always included within the select parameters
func (suite *ExchangeServiceSuite) TestOptionsForFolders() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Folder Option",
params: []string{"parentFolderId"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Folder Options: Valid",
params: []string{"displayName", "isHidden"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Folder option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
config, err := optionsForMailFolders(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(config.QueryParameters.Select))
}
})
}
}
// TestOptionsForContacts similar to TestExchangeService_optionsForFolders
func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Contact Option",
params: []string{"displayName"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Contact Options: Valid",
params: []string{"displayName", "parentFolderId"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Contact Option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
options, err := optionsForContacts(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(options.QueryParameters.Select))
}
})
}
}
// TestGraphQueryFunctions verifies if Query functions APIs
// through Microsoft Graph are functional
func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
ctx, flush := tester.NewContext()
defer flush()
userID := tester.M365UserID(suite.T())
tests := []struct {
name string
function GraphQuery
}{
{
name: "GraphQuery: Get All Contacts For User",
function: GetAllContactsForUser,
},
{
name: "GraphQuery: Get All Folders",
function: GetAllFolderNamesForUser,
},
{
name: "GraphQuery: Get All ContactFolders",
function: GetAllContactFolderNamesForUser,
},
{
name: "GraphQuery: Get Default ContactFolder",
function: GetDefaultContactFolderForUser,
},
{
name: "GraphQuery: Get All Events for User",
function: GetAllEventsForUser,
},
{
name: "GraphQuery: Get All Calendars for User",
function: GetAllCalendarNamesForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
response, err := test.function(ctx, suite.gs, userID)
assert.NoError(t, err)
assert.NotNil(t, response)
})
}
}
//==========================
// Restore Functions
//==========================
// TestRestoreContact ensures contact object can be created, placed into
// the Corso Folder. The function handles test clean-up.
func (suite *ExchangeServiceSuite) TestRestoreContact() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
now = time.Now()
folderName = "TestRestoreContact: " + common.FormatSimpleDateTime(now)
)
aFolder, err := CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
folderID := *aFolder.GetId()
defer func() {
// Remove the folder containing contact prior to exiting test
err = DeleteContactFolder(ctx, suite.gs, userID, folderID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeContact(ctx,
mockconnector.GetMockContactBytes("Corso TestContact"),
suite.gs,
control.Copy,
folderID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "contact item info")
}
// TestRestoreEvent verifies that event object is able to created
// and sent into the test account of the Corso user in the newly created Corso Calendar
func (suite *ExchangeServiceSuite) TestRestoreEvent() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
name = "TestRestoreEvent: " + common.FormatSimpleDateTime(time.Now())
)
calendar, err := CreateCalendar(ctx, suite.gs, userID, name)
require.NoError(t, err)
calendarID := *calendar.GetId()
defer func() {
// Removes calendar containing events created during the test
err = DeleteCalendar(ctx, suite.gs, userID, calendarID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeEvent(ctx,
mockconnector.GetMockEventWithAttendeesBytes(name),
suite.gs,
control.Copy,
calendarID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "event item info")
}
// TestRestoreExchangeObject verifies path.Category usage for restored objects
func (suite *ExchangeServiceSuite) TestRestoreExchangeObject() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
service, err := createService(m365)
require.NoError(suite.T(), err)
userID := tester.M365UserID(suite.T())
now := time.Now()
tests := []struct {
name string
bytes []byte
category path.CategoryType
cleanupFunc func(context.Context, graph.Servicer, string, string) error
destination func(*testing.T, context.Context) string
}{
{
name: "Test Mail",
bytes: mockconnector.GetMockMessageBytes("Restore Exchange Object"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Direct Attachment",
bytes: mockconnector.GetMockMessageWithDirectAttachment("Restore 1 Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Large Attachment",
bytes: mockconnector.GetMockMessageWithLargeAttachment("Restore Large Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Two Attachments",
bytes: mockconnector.GetMockMessageWithTwoAttachments("Restore 2 Attachments"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Reference(OneDrive) Attachment",
bytes: mockconnector.GetMessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
// TODO: #884 - reinstate when able to specify root folder by name
{
name: "Test Contact",
bytes: mockconnector.GetMockContactBytes("Test_Omega"),
category: path.ContactsCategory,
cleanupFunc: DeleteContactFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now)
folder, err := CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Events",
bytes: mockconnector.GetDefaultMockEventBytes("Restored Event Object"),
category: path.EventsCategory,
cleanupFunc: DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now)
calendar, err := CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
{
name: "Test Event with Attachment",
bytes: mockconnector.GetMockEventWithAttachment("Restored Event Attachment"),
category: path.EventsCategory,
cleanupFunc: DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now)
calendar, err := CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
destination := test.destination(t, ctx)
info, err := RestoreExchangeObject(
ctx,
test.bytes,
test.category,
control.Copy,
service,
destination,
userID,
)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "item info is populated")
cleanupError := test.cleanupFunc(ctx, service, userID, destination)
assert.NoError(t, cleanupError)
})
}
}
// Testing to ensure that cache system works for in multiple different environments
func (suite *ExchangeServiceSuite) TestGetContainerIDFromCache() {
ctx, flush := tester.NewContext()
defer flush()
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
connector, err := createService(m365)
require.NoError(suite.T(), err)
var (
user = tester.M365UserID(suite.T())
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string
pathFunc1 func(t *testing.T) path.Path
pathFunc2 func(t *testing.T) path.Path
category path.CategoryType
}{
{
name: "Mail Cache Test",
category: path.EmailCategory,
pathFunc1: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Croix").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
pathFunc2: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Felicius").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
},
{
name: "Contact Cache Test",
category: path.ContactsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("HufflePuff").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Ravenclaw").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
{
name: "Event Cache Test",
category: path.EventsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Durmstrang").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Beauxbatons").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
}
)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc1(t),
folderName,
directoryCaches)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc2(t),
folderName,
directoryCaches)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})
}
}

7
src/internal/connector/exchange/iterators_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
@ -83,7 +84,7 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
tests := []struct {
name string
queryFunc GraphQuery
queryFunc api.GraphQuery
scope selectors.ExchangeScope
iterativeFunction func(
container map[string]graph.Container,
@ -93,13 +94,13 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
}{
{
name: "Contacts Iterative Check",
queryFunc: GetAllContactFolderNamesForUser,
queryFunc: api.GetAllContactFolderNamesForUser,
transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectContactContainers,
},
{
name: "Events Iterative Check",
queryFunc: GetAllCalendarNamesForUser,
queryFunc: api.GetAllCalendarNamesForUser,
transformer: models.CreateCalendarCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectCalendarContainers,
},

26
src/internal/connector/exchange/mail_folder_cache.go
View File

@ -7,6 +7,7 @@ import (
msfolderdelta "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -31,22 +32,10 @@ type mailFolderCache struct {
func (mc *mailFolderCache) populateMailRoot(
ctx context.Context,
) error {
wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForMailFoldersItem(wantedOpts)
if err != nil {
return errors.Wrapf(err, "getting options for mail folders %v", wantedOpts)
}
for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} {
var directory string
f, err := mc.
gs.
Client().
UsersById(mc.userID).
MailFoldersById(fldr).
Get(ctx, opts)
f, err := api.GetMailFolderByID(ctx, mc.gs, mc.userID, fldr, "displayName", "parentFolderId")
if err != nil {
return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err))
}
@ -56,7 +45,6 @@ func (mc *mailFolderCache) populateMailRoot(
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(directory))
if err := mc.addFolder(temp); err != nil {
return errors.Wrap(err, "initializing mail resolver")
}
@ -79,15 +67,7 @@ func (mc *mailFolderCache) Populate(
return err
}
// Even though this uses the `Delta` query, we do no store or re-use
// the delta-link tokens like with other queries. The goal is always
// to retrieve the complete history of folders.
query := mc.
gs.
Client().
UsersById(mc.userID).
MailFolders().
Delta()
query := api.GetAllMailFoldersBuilder(ctx, mc.gs, mc.userID)
var errs *multierror.Error

269
src/internal/connector/exchange/restore_test.go Normal file
View File

@ -0,0 +1,269 @@
package exchange
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
)
type ExchangeRestoreSuite struct {
suite.Suite
gs graph.Servicer
}
func TestExchangeRestoreSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoConnectorRestoreExchangeCollectionTests)
suite.Run(t, new(ExchangeRestoreSuite))
}
func (suite *ExchangeRestoreSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
}
// TestRestoreContact ensures contact object can be created, placed into
// the Corso Folder. The function handles test clean-up.
func (suite *ExchangeRestoreSuite) TestRestoreContact() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
now = time.Now()
folderName = "TestRestoreContact: " + common.FormatSimpleDateTime(now)
)
aFolder, err := api.CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
folderID := *aFolder.GetId()
defer func() {
// Remove the folder containing contact prior to exiting test
err = api.DeleteContactFolder(ctx, suite.gs, userID, folderID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeContact(ctx,
mockconnector.GetMockContactBytes("Corso TestContact"),
suite.gs,
control.Copy,
folderID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "contact item info")
}
// TestRestoreEvent verifies that event object is able to created
// and sent into the test account of the Corso user in the newly created Corso Calendar
func (suite *ExchangeRestoreSuite) TestRestoreEvent() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
name = "TestRestoreEvent: " + common.FormatSimpleDateTime(time.Now())
)
calendar, err := api.CreateCalendar(ctx, suite.gs, userID, name)
require.NoError(t, err)
calendarID := *calendar.GetId()
defer func() {
// Removes calendar containing events created during the test
err = api.DeleteCalendar(ctx, suite.gs, userID, calendarID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeEvent(ctx,
mockconnector.GetMockEventWithAttendeesBytes(name),
suite.gs,
control.Copy,
calendarID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "event item info")
}
// TestRestoreExchangeObject verifies path.Category usage for restored objects
func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
service, err := createService(m365)
require.NoError(suite.T(), err)
userID := tester.M365UserID(suite.T())
now := time.Now()
tests := []struct {
name string
bytes []byte
category path.CategoryType
cleanupFunc func(context.Context, graph.Servicer, string, string) error
destination func(*testing.T, context.Context) string
}{
{
name: "Test Mail",
bytes: mockconnector.GetMockMessageBytes("Restore Exchange Object"),
category: path.EmailCategory,
cleanupFunc: api.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Direct Attachment",
bytes: mockconnector.GetMockMessageWithDirectAttachment("Restore 1 Attachment"),
category: path.EmailCategory,
cleanupFunc: api.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Large Attachment",
bytes: mockconnector.GetMockMessageWithLargeAttachment("Restore Large Attachment"),
category: path.EmailCategory,
cleanupFunc: api.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Two Attachments",
bytes: mockconnector.GetMockMessageWithTwoAttachments("Restore 2 Attachments"),
category: path.EmailCategory,
cleanupFunc: api.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Reference(OneDrive) Attachment",
bytes: mockconnector.GetMessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"),
category: path.EmailCategory,
cleanupFunc: api.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
// TODO: #884 - reinstate when able to specify root folder by name
{
name: "Test Contact",
bytes: mockconnector.GetMockContactBytes("Test_Omega"),
category: path.ContactsCategory,
cleanupFunc: api.DeleteContactFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now)
folder, err := api.CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Events",
bytes: mockconnector.GetDefaultMockEventBytes("Restored Event Object"),
category: path.EventsCategory,
cleanupFunc: api.DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now)
calendar, err := api.CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
{
name: "Test Event with Attachment",
bytes: mockconnector.GetMockEventWithAttachment("Restored Event Attachment"),
category: path.EventsCategory,
cleanupFunc: api.DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now)
calendar, err := api.CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
destination := test.destination(t, ctx)
info, err := RestoreExchangeObject(
ctx,
test.bytes,
test.category,
control.Copy,
service,
destination,
userID,
)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "item info is populated")
cleanupError := test.cleanupFunc(ctx, service, userID, destination)
assert.NoError(t, cleanupError)
})
}
}

75
src/internal/connector/exchange/service_functions.go
View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
@ -15,9 +14,6 @@ import (
var ErrFolderNotFound = errors.New("folder not found")
// createService internal constructor for exchangeService struct returns an error
// iff the params for the entry are incorrect (e.g. len(TenantID) == 0, etc.)
// NOTE: Incorrect account information will result in errors on subsequent queries.
func createService(credentials account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
@ -31,76 +27,7 @@ func createService(credentials account.M365Config) (*graph.Service, error) {
return graph.NewService(adapter), nil
}
// CreateMailFolder makes a mail folder iff a folder of the same name does not exist
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http
func CreateMailFolder(ctx context.Context, gs graph.Servicer, user, folder string) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
}
func CreateMailFolderWithParent(
ctx context.Context,
gs graph.Servicer,
user, folder, parentID string,
) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().
UsersById(user).
MailFoldersById(parentID).
ChildFolders().
Post(ctx, requestBody, nil)
}
// DeleteMailFolder removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/mailfolder-delete?view=graph-rest-1.0&tabs=http
func DeleteMailFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
}
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go
func CreateCalendar(ctx context.Context, gs graph.Servicer, user, calendarName string) (models.Calendarable, error) {
requestbody := models.NewCalendar()
requestbody.SetName(&calendarName)
return gs.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
}
// DeleteCalendar removes calendar from user's M365 account
// Reference: https://docs.microsoft.com/en-us/graph/api/calendar-delete?view=graph-rest-1.0&tabs=go
func DeleteCalendar(ctx context.Context, gs graph.Servicer, user, calendarID string) error {
return gs.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
}
// CreateContactFolder makes a contact folder with the displayName of folderName.
// If successful, returns the created folder object.
func CreateContactFolder(
ctx context.Context,
gs graph.Servicer,
user, folderName string,
) (models.ContactFolderable, error) {
requestBody := models.NewContactFolder()
temp := folderName
requestBody.SetDisplayName(&temp)
return gs.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
}
// DeleteContactFolder deletes the ContactFolder associated with the M365 ID if permissions are valid.
// Errors returned if the function call was not successful.
func DeleteContactFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
}
// PopulateExchangeContainerResolver gets a folder resolver if one is available for
// populateExchangeContainerResolver gets a folder resolver if one is available for
// this category of data. If one is not available, returns nil so that other
// logic in the caller can complete as long as they check if the resolver is not
// nil. If an error occurs populating the resolver, returns an error.

292
src/internal/connector/exchange/service_iterators.go
View File

@ -5,11 +5,10 @@ import (
"fmt"
"strings"
multierror "github.com/hashicorp/go-multierror"
"github.com/microsoftgraph/msgraph-sdk-go/models"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -19,14 +18,6 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
)
// carries details about delta retrieval in aggregators
type deltaUpdate struct {
// the deltaLink itself
url string
// true if the old delta was marked as invalid
reset bool
}
// filterContainersAndFillCollections is a utility function
// that places the M365 object ids belonging to specific directories
// into a Collection. Messages outside of those directories are omitted.
@ -104,14 +95,14 @@ func filterContainersAndFillCollections(
// to reset which will prevent any old items from being retained in
// storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned.
newDelta = deltaUpdate{reset: true}
newDelta = api.DeltaUpdate{Reset: true}
}
continue
}
if len(newDelta.url) > 0 {
deltaURLs[cID] = newDelta.url
if len(newDelta.URL) > 0 {
deltaURLs[cID] = newDelta.URL
}
edc := NewCollection(
@ -122,7 +113,7 @@ func filterContainersAndFillCollections(
service,
statusUpdater,
ctrlOpts,
newDelta.reset,
newDelta.Reset,
)
collections[cID] = &edc
@ -278,282 +269,17 @@ type FetchIDFunc func(
ctx context.Context,
gs graph.Servicer,
user, containerID, oldDeltaToken string,
) ([]string, []string, deltaUpdate, error)
) ([]string, []string, api.DeltaUpdate, error)
func getFetchIDFunc(category path.CategoryType) (FetchIDFunc, error) {
switch category {
case path.EmailCategory:
return FetchMessageIDsFromDirectory, nil
return api.FetchMessageIDsFromDirectory, nil
case path.EventsCategory:
return FetchEventIDsFromCalendar, nil
return api.FetchEventIDsFromCalendar, nil
case path.ContactsCategory:
return FetchContactIDsFromDirectory, nil
return api.FetchContactIDsFromDirectory, nil
default:
return nil, fmt.Errorf("category %s not supported by getFetchIDFunc", category)
}
}
// ---------------------------------------------------------------------------
// events
// ---------------------------------------------------------------------------
// FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
func FetchEventIDsFromCalendar(
ctx context.Context,
gs graph.Servicer,
user, calendarID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
)
options, err := optionsForEventsByCalendar([]string{"id"})
if err != nil {
return nil, nil, deltaUpdate{}, err
}
builder := gs.Client().
UsersById(user).
CalendarsById(calendarID).
Events()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return nil, nil, deltaUpdate{}, err
}
return nil, nil, deltaUpdate{}, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("event with nil ID in calendar %s", calendarID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
ids = append(ids, *item.GetId())
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemCalendarsItemEventsRequestBuilder(*nextLink, gs.Adapter())
}
// Events don't have a delta endpoint so just return an empty string.
return ids, nil, deltaUpdate{}, errs.ErrorOrNil()
}
// ---------------------------------------------------------------------------
// contacts
// ---------------------------------------------------------------------------
// FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts
// of the targeted directory
func FetchContactIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, deltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *msuser.ItemContactFoldersItemContactsDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter()))
// happy path
if err == nil {
return ids, removedIDs, deltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, deltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
ContactFoldersById(directoryID).
Contacts().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, deltaUpdate{}, err
}
return ids, removedIDs, deltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}
// ---------------------------------------------------------------------------
// messages
// ---------------------------------------------------------------------------
// FetchMessageIDsFromDirectory function that returns a list of all the m365IDs of the exchange.Mail
// of the targeted directory
func FetchMessageIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, deltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *msuser.ItemMailFoldersItemMessagesDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()))
// happy path
if err == nil {
return ids, removedIDs, deltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, deltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
MailFoldersById(directoryID).
Messages().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, deltaUpdate{}, err
}
return ids, removedIDs, deltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

109
src/internal/connector/exchange/service_query.go
View File

@ -1,109 +0,0 @@
package exchange
import (
"context"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/internal/connector/graph"
)
// GraphQuery represents functions which perform exchange-specific queries
// into M365 backstore. Responses -> returned items will only contain the information
// that is included in the options
// TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, gs graph.Servicer, userID string) (absser.Parsable, error)
// GetAllContactsForUser is a GraphQuery function for querying all the contacts in a user's account
func GetAllContactsForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
selecting := []string{"parentFolderId"}
options, err := optionsForContacts(selecting)
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Contacts().Get(ctx, options)
}
// GetAllFolderDisplayNamesForUser is a GraphQuery function for getting FolderId and display
// names for Mail Folder. All other information for the MailFolder object is omitted.
func GetAllFolderNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForMailFolders([]string{"displayName"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).MailFolders().Get(ctx, options)
}
func GetAllCalendarNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForCalendars([]string{"name", "owner"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Calendars().Get(ctx, options)
}
// GetDefaultContactFolderForUser is a GraphQuery function for getting the ContactFolderId
// and display names for the default "Contacts" folder.
// Only returns the default Contact Folder
func GetDefaultContactFolderForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForContactChildFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return gs.Client().
UsersById(user).
ContactFoldersById(rootFolderAlias).
ChildFolders().
Get(ctx, options)
}
// GetAllContactFolderNamesForUser is a GraphQuery function for getting ContactFolderId
// and display names for contacts. All other information is omitted.
// Does not return the default Contact Folder
func GetAllContactFolderNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).ContactFolders().Get(ctx, options)
}
// GetAllEvents for User. Default returns EventResponseCollection for future events.
// of the time that the call was made. 'calendar' option must be present to gain
// access to additional data map in future calls.
func GetAllEventsForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForEvents([]string{"id", "calendar"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Events().Get(ctx, options)
}
// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve
// the default associated data of a M365 object. This varies by object. Additional
// Queries must be run to obtain the omitted fields.
type GraphRetrievalFunc func(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error)
// RetrieveContactDataForUser is a GraphRetrievalFun that returns all associated fields.
func RetrieveContactDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil)
}
// RetrieveEventDataForUser is a GraphRetrievalFunc that returns event data.
// Calendarable and attachment fields are omitted due to size
func RetrieveEventDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil)
}
// RetrieveMessageDataForUser is a GraphRetrievalFunc that returns message data.
// Attachment field is omitted due to size.
func RetrieveMessageDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil)
}

13
src/internal/connector/exchange/service_restore.go
View File

@ -11,6 +11,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -532,8 +533,12 @@ func establishMailRestoreLocation(
continue
}
temp, err := CreateMailFolderWithParent(ctx,
service, user, folder, folderID)
temp, err := api.CreateMailFolderWithParent(
ctx,
service,
user,
folder,
folderID)
if err != nil {
// Should only error if cache malfunctions or incorrect parameters
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
@ -580,7 +585,7 @@ func establishContactsRestoreLocation(
return cached, nil
}
temp, err := CreateContactFolder(ctx, gs, user, folders[0])
temp, err := api.CreateContactFolder(ctx, gs, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
@ -613,7 +618,7 @@ func establishEventsRestoreLocation(
return cached, nil
}
temp, err := CreateCalendar(ctx, gs, user, folders[0])
temp, err := api.CreateCalendar(ctx, gs, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}

5
src/internal/operations/backup_integration_test.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
@ -924,7 +925,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
switch category {
case path.EmailCategory:
ids, _, _, err := exchange.FetchMessageIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "")
ids, _, _, err := api.FetchMessageIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "")
require.NoError(t, err, "getting message ids")
require.NotEmpty(t, ids, "message ids in folder")
@ -932,7 +933,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
require.NoError(t, err, "deleting email item: %s", support.ConnectorStackErrorTrace(err))
case path.ContactsCategory:
ids, _, _, err := exchange.FetchContactIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "")
ids, _, _, err := api.FetchContactIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "")
require.NoError(t, err, "getting contact ids")
require.NotEmpty(t, ids, "contact ids in folder")

2
src/internal/tester/integration_runners.go
View File

@ -22,6 +22,8 @@ const (
CorsoConnectorCreateExchangeCollectionTests = "CORSO_CONNECTOR_CREATE_EXCHANGE_COLLECTION_TESTS"
CorsoConnectorCreateSharePointCollectionTests = "CORSO_CONNECTOR_CREATE_SHAREPOINT_COLLECTION_TESTS"
CorsoConnectorDataCollectionTests = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS"
CorsoConnectorExchangeFolderCacheTests = "CORSO_CONNECTOR_EXCHANGE_FOLDER_CACHE_TESTS"
CorsoConnectorRestoreExchangeCollectionTests = "CORSO_CONNECTOR_RESTORE_EXCHANGE_COLLECTION_TESTS"
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"