Compare commits

...

7 Commits
main ... rbme

Author SHA1 Message Date
ryanfkeepers
a4b6309661 testing 2023-01-04 09:14:18 -07:00
ryanfkeepers
c895fca1e1 fix messy rebase issues 2023-01-03 17:45:40 -07:00
ryanfkeepers
a6e244e963 fix tes panics 2023-01-03 17:43:58 -07:00
ryanfkeepers
2c96d06de5 linter and error fixes 2023-01-03 17:43:58 -07:00
ryanfkeepers
3246f6135e add newClient constructor in api 2023-01-03 17:43:58 -07:00
ryanfkeepers
5ebbdd1b4e methodize all api funcs to the Client
In order to use the api layer as an interface, we
need the functions therein to be methods, so
that callers can leverage local interfaces.  This
change introduces the api.Client, and begins
to spread it throughout the exchange package,
largely in place of graph servicers.
2023-01-03 17:43:56 -07:00
ryanfkeepers
94134bf013 first pass migrate graph api to subfolder
In order to establish a standard api around our
graph client usage, and thus be able to mock
for testing, we need to migrate graph client
usage into another pacakge.  This is the first
step in that process of refactoring.
2023-01-03 17:32:32 -07:00
30 changed files with 1540 additions and 1266 deletions

View File

@ -47,7 +47,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
return nil
}
gc, tenantID, err := getGCAndVerifyUser(ctx, user)
gc, acct, err := getGCAndVerifyUser(ctx, user)
if err != nil {
return Only(ctx, err)
}
@ -55,10 +55,11 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
deets, err := generateAndRestoreItems(
ctx,
gc,
acct,
service,
category,
selectors.NewExchangeRestore([]string{user}).Selector,
tenantID, user, destination,
user, destination,
count,
func(id, now, subject, body string) []byte {
return mockconnector.GetMockMessageWith(
@ -87,7 +88,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
return nil
}
gc, tenantID, err := getGCAndVerifyUser(ctx, user)
gc, acct, err := getGCAndVerifyUser(ctx, user)
if err != nil {
return Only(ctx, err)
}
@ -95,10 +96,11 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
deets, err := generateAndRestoreItems(
ctx,
gc,
acct,
service,
category,
selectors.NewExchangeRestore([]string{user}).Selector,
tenantID, user, destination,
user, destination,
count,
func(id, now, subject, body string) []byte {
return mockconnector.GetMockEventWith(
@ -126,7 +128,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
return nil
}
gc, tenantID, err := getGCAndVerifyUser(ctx, user)
gc, acct, err := getGCAndVerifyUser(ctx, user)
if err != nil {
return Only(ctx, err)
}
@ -134,10 +136,11 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
deets, err := generateAndRestoreItems(
ctx,
gc,
acct,
service,
category,
selectors.NewExchangeRestore([]string{user}).Selector,
tenantID, user, destination,
user, destination,
count,
func(id, now, subject, body string) []byte {
given, mid, sur := id[:8], id[9:13], id[len(id)-12:]

View File

@ -108,10 +108,11 @@ type dataBuilderFunc func(id, now, subject, body string) []byte
func generateAndRestoreItems(
ctx context.Context,
gc *connector.GraphConnector,
acct account.Account,
service path.ServiceType,
cat path.CategoryType,
sel selectors.Selector,
tenantID, userID, destFldr string,
userID, destFldr string,
howMany int,
dbf dataBuilderFunc,
) (*details.Details, error) {
@ -144,7 +145,7 @@ func generateAndRestoreItems(
dataColls, err := buildCollections(
service,
tenantID, userID,
acct.ID(), userID,
dest,
collections,
)
@ -154,14 +155,14 @@ func generateAndRestoreItems(
Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, destination)
return gc.RestoreDataCollections(ctx, sel, dest, dataColls)
return gc.RestoreDataCollections(ctx, acct, sel, dest, dataColls)
}
// ------------------------------------------------------------------------------------------
// Common Helpers
// ------------------------------------------------------------------------------------------
func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, string, error) {
func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, account.Account, error) {
tid := common.First(tenant, os.Getenv(account.AzureTenantID))
// get account info
@ -172,13 +173,13 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil {
return nil, "", errors.Wrap(err, "finding m365 account details")
return nil, account.Account{}, errors.Wrap(err, "finding m365 account details")
}
// build a graph connector
gc, err := connector.NewGraphConnector(ctx, acct, connector.Users)
if err != nil {
return nil, "", errors.Wrap(err, "connecting to graph api")
return nil, account.Account{}, errors.Wrap(err, "connecting to graph api")
}
normUsers := map[string]struct{}{}
@ -188,10 +189,10 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
}
if _, ok := normUsers[strings.ToLower(user)]; !ok {
return nil, "", errors.New("user not found within tenant")
return nil, account.Account{}, errors.New("user not found within tenant")
}
return gc, tid, nil
return gc, acct, nil
}
type item struct {

View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -76,12 +77,12 @@ func handleGetCommand(cmd *cobra.Command, args []string) error {
return nil
}
gc, err := getGC(ctx)
gc, creds, err := getGC(ctx)
if err != nil {
return err
}
err = runDisplayM365JSON(ctx, gc.Service)
err = runDisplayM365JSON(ctx, gc.Service, creds)
if err != nil {
return Only(ctx, errors.Wrapf(err, "unable to create mock from M365: %s", m365ID))
}
@ -92,16 +93,22 @@ func handleGetCommand(cmd *cobra.Command, args []string) error {
func runDisplayM365JSON(
ctx context.Context,
gs graph.Servicer,
creds account.M365Config,
) error {
var (
get exchange.GraphRetrievalFunc
get api.GraphRetrievalFunc
serializeFunc exchange.GraphSerializeFunc
cat = graph.StringToPathCategory(category)
)
ac, err := api.NewClient(creds)
if err != nil {
return err
}
switch cat {
case path.EmailCategory, path.EventsCategory, path.ContactsCategory:
get, serializeFunc = exchange.GetQueryAndSerializeFunc(cat)
get, serializeFunc = exchange.GetQueryAndSerializeFunc(ac, cat)
default:
return fmt.Errorf("unable to process category: %s", cat)
}
@ -110,7 +117,7 @@ func runDisplayM365JSON(
sw := kw.NewJsonSerializationWriter()
response, err := get(ctx, gs, user, m365ID)
response, err := get(ctx, user, m365ID)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
@ -158,7 +165,7 @@ func runDisplayM365JSON(
// Helpers
//-------------------------------------------------------------------------------
func getGC(ctx context.Context) (*connector.GraphConnector, error) {
func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config, error) {
// get account info
m365Cfg := account.M365Config{
M365: credentials.GetM365(),
@ -167,13 +174,13 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) {
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil {
return nil, Only(ctx, errors.Wrap(err, "finding m365 account details"))
return nil, m365Cfg, Only(ctx, errors.Wrap(err, "finding m365 account details"))
}
gc, err := connector.NewGraphConnector(ctx, acct, connector.Users)
if err != nil {
return nil, Only(ctx, errors.Wrap(err, "connecting to graph API"))
return nil, m365Cfg, Only(ctx, errors.Wrap(err, "connecting to graph API"))
}
return gc, nil
return gc, m365Cfg, nil
}

View File

@ -0,0 +1,88 @@
package api
import (
"context"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
)
// ---------------------------------------------------------------------------
// common types
// ---------------------------------------------------------------------------
// DeltaUpdate holds the results of a current delta token. It normally
// gets produced when aggregating the addition and removal of items in
// a delta-queriable folder.
type DeltaUpdate struct {
// the deltaLink itself
URL string
// true if the old delta was marked as invalid
Reset bool
}
// GraphQuery represents functions which perform exchange-specific queries
// into M365 backstore. Responses -> returned items will only contain the information
// that is included in the options
// TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error)
// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve
// the default associated data of a M365 object. This varies by object. Additional
// Queries must be run to obtain the omitted fields.
type GraphRetrievalFunc func(
ctx context.Context,
user, m365ID string,
) (serialization.Parsable, error)
// ---------------------------------------------------------------------------
// interfaces
// ---------------------------------------------------------------------------
// Client is used to fulfill the interface for exchange
// queries that are traditionally backed by GraphAPI. A
// struct is used in this case, instead of deferring to
// pure function wrappers, so that the boundary separates the
// granular implementation of the graphAPI and kiota away
// from the exchange package's broader intents.
type Client struct {
Credentials account.M365Config
// The stable service is re-usable for any non-paged request.
// This allows us to maintain performance across async requests.
stable graph.Servicer
}
// NewClient produces a new exchange api client. Must be used in
// place of creating an ad-hoc client struct.
func NewClient(creds account.M365Config) (Client, error) {
s, err := newService(creds)
if err != nil {
return Client{}, err
}
return Client{creds, s}, nil
}
// service generates a new service. Used for paged and other long-running
// requests instead of the client's stable service, so that in-flight state
// within the adapter doesn't get clobbered
func (c Client) service() (*graph.Service, error) {
return newService(c.Credentials)
}
func newService(creds account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
creds.AzureTenantID,
creds.AzureClientID,
creds.AzureClientSecret,
)
if err != nil {
return nil, errors.Wrap(err, "generating graph api service client")
}
return graph.NewService(adapter), nil
}

View File

@ -0,0 +1,192 @@
package api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
)
type ExchangeServiceSuite struct {
suite.Suite
gs graph.Servicer
credentials account.M365Config
}
func TestExchangeServiceSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
tester.CorsoGraphConnectorExchangeTests)
suite.Run(t, new(ExchangeServiceSuite))
}
func (suite *ExchangeServiceSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.credentials = m365
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
}
func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := optionsForCalendars(test.params)
test.checkError(t, err)
})
}
}
// TestOptionsForFolders ensures that approved query options
// are added to the RequestBuildConfiguration. Expected will always be +1
// on than the input as "id" are always included within the select parameters
func (suite *ExchangeServiceSuite) TestOptionsForFolders() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Folder Option",
params: []string{"parentFolderId"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Folder Options: Valid",
params: []string{"displayName", "isHidden"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Folder option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
config, err := optionsForMailFolders(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(config.QueryParameters.Select))
}
})
}
}
// TestOptionsForContacts similar to TestExchangeService_optionsForFolders
func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Contact Option",
params: []string{"displayName"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Contact Options: Valid",
params: []string{"displayName", "parentFolderId"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Contact Option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
options, err := optionsForContacts(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(options.QueryParameters.Select))
}
})
}
}
// TestGraphQueryFunctions verifies if Query functions APIs
// through Microsoft Graph are functional
func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
ctx, flush := tester.NewContext()
defer flush()
c, err := NewClient(suite.credentials)
require.NoError(suite.T(), err)
userID := tester.M365UserID(suite.T())
tests := []struct {
name string
function GraphQuery
}{
{
name: "GraphQuery: Get All ContactFolders",
function: c.GetAllContactFolderNamesForUser,
},
{
name: "GraphQuery: Get All Calendars for User",
function: c.GetAllCalendarNamesForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
response, err := test.function(ctx, userID)
assert.NoError(t, err)
assert.NotNil(t, response)
})
}
}

View File

@ -0,0 +1,211 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateContactFolder makes a contact folder with the displayName of folderName.
// If successful, returns the created folder object.
func (c Client) CreateContactFolder(
ctx context.Context,
user, folderName string,
) (models.ContactFolderable, error) {
requestBody := models.NewContactFolder()
temp := folderName
requestBody.SetDisplayName(&temp)
return c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
}
// DeleteContactFolder deletes the ContactFolder associated with the M365 ID if permissions are valid.
// Errors returned if the function call was not successful.
func (c Client) DeleteContactFolder(
ctx context.Context,
user, folderID string,
) error {
return c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
}
// RetrieveContactDataForUser is a GraphRetrievalFun that returns all associated fields.
func (c Client) RetrieveContactDataForUser(
ctx context.Context,
user, m365ID string,
) (serialization.Parsable, error) {
return c.stable.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil)
}
// GetAllContactFolderNamesForUser is a GraphQuery function for getting
// ContactFolderId and display names for contacts. All other information is omitted.
// Does not return the default Contact Folder
func (c Client) GetAllContactFolderNamesForUser(
ctx context.Context,
user string,
) (serialization.Parsable, error) {
options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return c.stable.Client().UsersById(user).ContactFolders().Get(ctx, options)
}
func (c Client) GetContactFolderByID(
ctx context.Context,
userID, dirID string,
optionalFields ...string,
) (models.ContactFolderable, error) {
fields := append([]string{"displayName", "parentFolderId"}, optionalFields...)
ofcf, err := optionsForContactFolderByID(fields)
if err != nil {
return nil, errors.Wrapf(err, "options for contact folder: %v", fields)
}
return c.stable.Client().
UsersById(userID).
ContactFoldersById(dirID).
Get(ctx, ofcf)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetContactChildFoldersBuilder(
ctx context.Context,
userID, baseDirID string,
optionalFields ...string,
) (
*users.ItemContactFoldersItemChildFoldersRequestBuilder,
*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration,
*graph.Service,
error,
) {
service, err := c.service()
if err != nil {
return nil, nil, nil, err
}
fields := append([]string{"displayName", "parentFolderId"}, optionalFields...)
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "options for contact child folders: %v", fields)
}
builder := service.Client().
UsersById(userID).
ContactFoldersById(baseDirID).
ChildFolders()
return builder, ofcf, service, nil
}
// FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts
// of the targeted directory
func (c Client) FetchContactIDsFromDirectory(
ctx context.Context,
user, directoryID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
service, err := c.service()
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, service.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter()))
// note: happy path, not the error condition
if err == nil {
return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, DeltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := service.Client().
UsersById(user).
ContactFoldersById(directoryID).
Contacts().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return ids, removedIDs, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

View File

@ -0,0 +1,141 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go
func (c Client) CreateCalendar(
ctx context.Context,
user, calendarName string,
) (models.Calendarable, error) {
requestbody := models.NewCalendar()
requestbody.SetName(&calendarName)
return c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
}
// DeleteCalendar removes calendar from user's M365 account
// Reference: https://docs.microsoft.com/en-us/graph/api/calendar-delete?view=graph-rest-1.0&tabs=go
func (c Client) DeleteCalendar(
ctx context.Context,
user, calendarID string,
) error {
return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
}
// RetrieveEventDataForUser is a GraphRetrievalFunc that returns event data.
// Calendarable and attachment fields are omitted due to size
func (c Client) RetrieveEventDataForUser(
ctx context.Context,
user, m365ID string,
) (serialization.Parsable, error) {
return c.stable.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil)
}
func (c Client) GetAllCalendarNamesForUser(
ctx context.Context,
user string,
) (serialization.Parsable, error) {
options, err := optionsForCalendars([]string{"name", "owner"})
if err != nil {
return nil, err
}
return c.stable.Client().UsersById(user).Calendars().Get(ctx, options)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetCalendarsBuilder(
ctx context.Context,
userID string,
optionalFields ...string,
) (
*users.ItemCalendarsRequestBuilder,
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
*graph.Service,
error,
) {
service, err := c.service()
if err != nil {
return nil, nil, nil, err
}
ofcf, err := optionsForCalendars(optionalFields)
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "options for event calendars: %v", optionalFields)
}
builder := service.Client().UsersById(userID).Calendars()
return builder, ofcf, service, nil
}
// FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
func (c Client) FetchEventIDsFromCalendar(
ctx context.Context,
user, calendarID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
service, err := c.service()
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
var (
errs *multierror.Error
ids []string
)
options, err := optionsForEventsByCalendar([]string{"id"})
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
builder := service.Client().UsersById(user).CalendarsById(calendarID).Events()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return nil, nil, DeltaUpdate{}, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("event with nil ID in calendar %s", calendarID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
ids = append(ids, *item.GetId())
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemCalendarsItemEventsRequestBuilder(*nextLink, service.Adapter())
}
// Events don't have a delta endpoint so just return an empty string.
return ids, nil, DeltaUpdate{}, errs.ErrorOrNil()
}

View File

@ -0,0 +1,205 @@
package api
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
)
// CreateMailFolder makes a mail folder iff a folder of the same name does not exist
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http
func (c Client) NewName(
ctx context.Context,
user, folder string,
) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
}
func (c Client) CreateMailFolderWithParent(
ctx context.Context,
user, folder, parentID string,
) (models.MailFolderable, error) {
service, err := c.service()
if err != nil {
return nil, err
}
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return service.
Client().
UsersById(user).
MailFoldersById(parentID).
ChildFolders().
Post(ctx, requestBody, nil)
}
// DeleteMailFolder removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/mailfolder-delete?view=graph-rest-1.0&tabs=http
func (c Client) DeleteMailFolder(
ctx context.Context,
user, folderID string,
) error {
return c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
}
// RetrieveMessageDataForUser is a GraphRetrievalFunc that returns message data.
// Attachment field is omitted due to size.
func (c Client) RetrieveMessageDataForUser(
ctx context.Context,
user, m365ID string,
) (serialization.Parsable, error) {
return c.stable.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil)
}
// GetMailFoldersBuilder retrieves all of the users current mail folders.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetAllMailFoldersBuilder(
ctx context.Context,
userID string,
) (
*users.ItemMailFoldersDeltaRequestBuilder,
*graph.Service,
error,
) {
service, err := c.service()
if err != nil {
return nil, nil, err
}
builder := service.Client().
UsersById(userID).
MailFolders().
Delta()
return builder, service, nil
}
func (c Client) GetMailFolderByID(
ctx context.Context,
userID, dirID string,
optionalFields ...string,
) (models.MailFolderable, error) {
ofmf, err := optionsForMailFoldersItem(optionalFields)
if err != nil {
return nil, errors.Wrapf(err, "options for mail folder: %v", optionalFields)
}
return c.stable.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf)
}
// FetchMessageIDsFromDirectory function that returns a list of all the m365IDs of the exchange.Mail
// of the targeted directory
func (c Client) FetchMessageIDsFromDirectory(
ctx context.Context,
user, directoryID, oldDelta string,
) ([]string, []string, DeltaUpdate, error) {
service, err := c.service()
if err != nil {
return nil, nil, DeltaUpdate{}, err
}
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, service.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter()))
// note: happy path, not the error condition
if err == nil {
return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, DeltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := service.Client().UsersById(user).MailFoldersById(directoryID).Messages().Delta()
if err := getIDs(builder); err != nil {
return nil, nil, DeltaUpdate{}, err
}
return ids, removedIDs, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

View File

@ -1,9 +1,9 @@
package exchange
package api
import (
"fmt"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/microsoftgraph/msgraph-sdk-go/users"
)
// -----------------------------------------------------------------------
@ -74,16 +74,16 @@ var (
func optionsForFolderMessagesDelta(
moreOps []string,
) (*msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForMessages)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -94,7 +94,7 @@ func optionsForFolderMessagesDelta(
// @param moreOps should reflect elements from fieldsForCalendars
// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler
func optionsForCalendars(moreOps []string) (
*msuser.ItemCalendarsRequestBuilderGetRequestConfiguration,
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForCalendars)
@ -102,10 +102,10 @@ func optionsForCalendars(moreOps []string) (
return nil, err
}
// should be a CalendarsRequestBuilderGetRequestConfiguration
requestParams := &msuser.ItemCalendarsRequestBuilderGetQueryParameters{
requestParams := &users.ItemCalendarsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemCalendarsRequestBuilderGetRequestConfiguration{
options := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParams,
}
@ -115,7 +115,7 @@ func optionsForCalendars(moreOps []string) (
// optionsForContactFolders places allowed options for exchange.ContactFolder object
// @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler
func optionsForContactFolders(moreOps []string) (
*msuser.ItemContactFoldersRequestBuilderGetRequestConfiguration,
*users.ItemContactFoldersRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
@ -123,10 +123,10 @@ func optionsForContactFolders(moreOps []string) (
return nil, err
}
requestParameters := &msuser.ItemContactFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -134,7 +134,7 @@ func optionsForContactFolders(moreOps []string) (
}
func optionsForContactFolderByID(moreOps []string) (
*msuser.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration,
*users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
@ -142,10 +142,10 @@ func optionsForContactFolderByID(moreOps []string) (
return nil, err
}
requestParameters := &msuser.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -157,16 +157,16 @@ func optionsForContactFolderByID(moreOps []string) (
// @return is first call in MailFolders().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForMailFolders(
moreOps []string,
) (*msuser.ItemMailFoldersRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -178,16 +178,16 @@ func optionsForMailFolders(
// Returns first call in MailFoldersById().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForMailFoldersItem(
moreOps []string,
) (*msuser.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForFolders)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
requestParameters := &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
options := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -196,35 +196,17 @@ func optionsForMailFoldersItem(
func optionsForContactFoldersItemDelta(
moreOps []string,
) (*msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForEvents ensures valid option inputs for exchange.Events
// @return is first call in Events().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForEvents(moreOps []string) (*msuser.ItemEventsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForEvents)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemEventsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemEventsRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -234,17 +216,17 @@ func optionsForEvents(moreOps []string) (*msuser.ItemEventsRequestBuilderGetRequ
// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar
func optionsForEventsByCalendar(
moreOps []string,
) (*msuser.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForEvents)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{
requestParameters := &users.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -254,16 +236,16 @@ func optionsForEventsByCalendar(
// optionsForContactChildFolders builds a contacts child folders request.
func optionsForContactChildFolders(
moreOps []string,
) (*msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) {
) (*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
options := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
@ -272,16 +254,16 @@ func optionsForContactChildFolders(
// optionsForContacts transforms options into select query for MailContacts
// @return is the first call in Contacts().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForContacts(moreOps []string) (*msuser.ItemContactsRequestBuilderGetRequestConfiguration, error) {
func optionsForContacts(moreOps []string) (*users.ItemContactsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, fieldsForContacts)
if err != nil {
return nil, err
}
requestParameters := &msuser.ItemContactsRequestBuilderGetQueryParameters{
requestParameters := &users.ItemContactsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &msuser.ItemContactsRequestBuilderGetRequestConfiguration{
options := &users.ItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}

View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -15,7 +16,8 @@ var _ graph.ContainerResolver = &contactFolderCache{}
type contactFolderCache struct {
*containerResolver
gs graph.Servicer
ac api.Client
// gs graph.Servicer
userID string
}
@ -24,19 +26,7 @@ func (cfc *contactFolderCache) populateContactRoot(
directoryID string,
baseContainerPath []string,
) error {
wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForContactFolderByID(wantedOpts)
if err != nil {
return errors.Wrapf(err, "getting options for contact folder cache: %v", wantedOpts)
}
f, err := cfc.
gs.
Client().
UsersById(cfc.userID).
ContactFoldersById(directoryID).
Get(ctx, opts)
f, err := cfc.ac.GetContactFolderByID(ctx, cfc.userID, directoryID)
if err != nil {
return errors.Wrapf(
err,
@ -65,21 +55,16 @@ func (cfc *contactFolderCache) Populate(
return err
}
var (
errs error
options, err = optionsForContactChildFolders([]string{"displayName", "parentFolderId"})
)
var errs error
builder, options, servicer, err := cfc.ac.GetContactChildFoldersBuilder(
ctx,
cfc.userID,
baseID)
if err != nil {
return errors.Wrap(err, "contact cache resolver option")
}
builder := cfc.
gs.Client().
UsersById(cfc.userID).
ContactFoldersById(baseID).
ChildFolders()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
@ -112,7 +97,7 @@ func (cfc *contactFolderCache) Populate(
break
}
builder = msuser.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), cfc.gs.Adapter())
builder = msuser.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter())
}
if err := cfc.populatePaths(ctx); err != nil {

View File

@ -11,9 +11,14 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// mocks and helpers
// ---------------------------------------------------------------------------
type mockContainer struct {
id *string
name *string
@ -34,6 +39,10 @@ func (m mockContainer) GetParentFolderId() *string {
return m.parentID
}
// ---------------------------------------------------------------------------
// unit suite
// ---------------------------------------------------------------------------
type FolderCacheUnitSuite struct {
suite.Suite
}
@ -284,6 +293,10 @@ func resolverWithContainers(numContainers int) (*containerResolver, []*mockCache
return resolver, containers
}
// ---------------------------------------------------------------------------
// configured unit suite
// ---------------------------------------------------------------------------
// TestConfiguredFolderCacheUnitSuite cannot run its tests in parallel.
type ConfiguredFolderCacheUnitSuite struct {
suite.Suite
@ -431,3 +444,177 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestAddToCache() {
require.NoError(t, err)
assert.Equal(t, m.expectedPath, p.String())
}
// ---------------------------------------------------------------------------
// integration suite
// ---------------------------------------------------------------------------
type FolderCacheIntegrationSuite struct {
suite.Suite
credentials account.M365Config
gs graph.Servicer
}
func TestFolderCacheIntegrationSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoConnectorExchangeFolderCacheTests)
suite.Run(t, new(FolderCacheIntegrationSuite))
}
func (suite *FolderCacheIntegrationSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.credentials = m365
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
}
// Testing to ensure that cache system works for in multiple different environments
func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
ctx, flush := tester.NewContext()
defer flush()
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
var (
user = tester.M365UserID(suite.T())
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string
pathFunc1 func(t *testing.T) path.Path
pathFunc2 func(t *testing.T) path.Path
category path.CategoryType
}{
{
name: "Mail Cache Test",
category: path.EmailCategory,
pathFunc1: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Croix").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
pathFunc2: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Felicius").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
},
{
name: "Contact Cache Test",
category: path.ContactsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("HufflePuff").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Ravenclaw").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
{
name: "Event Cache Test",
category: path.EventsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Durmstrang").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Beauxbatons").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
}
)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := CreateContainerDestinaion(
ctx,
m365,
test.pathFunc1(t),
folderName,
directoryCaches)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := CreateContainerDestinaion(
ctx,
m365,
test.pathFunc2(t),
folderName,
directoryCaches)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})
}
}

View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -15,7 +16,8 @@ var _ graph.ContainerResolver = &eventCalendarCache{}
type eventCalendarCache struct {
*containerResolver
gs graph.Servicer
// gs graph.Servicer
ac api.Client
userID string
}
@ -31,7 +33,7 @@ func (ecc *eventCalendarCache) Populate(
ecc.containerResolver = newContainerResolver()
}
options, err := optionsForCalendars([]string{"name"})
builder, options, servicer, err := ecc.ac.GetCalendarsBuilder(ctx, ecc.userID, "name")
if err != nil {
return err
}
@ -41,8 +43,6 @@ func (ecc *eventCalendarCache) Populate(
directories = make([]graph.Container, 0)
)
builder := ecc.gs.Client().UsersById(ecc.userID).Calendars()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
@ -68,7 +68,7 @@ func (ecc *eventCalendarCache) Populate(
break
}
builder = msuser.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), ecc.gs.Adapter())
builder = msuser.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter())
}
for _, container := range directories {

View File

@ -18,6 +18,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -58,6 +59,7 @@ type Collection struct {
// service - client/adapter pair used to access M365 back store
service graph.Servicer
ac api.Client
category path.CategoryType
statusUpdater support.StatusUpdater
@ -87,12 +89,14 @@ func NewCollection(
user string,
curr, prev path.Path,
category path.CategoryType,
ac api.Client,
service graph.Servicer,
statusUpdater support.StatusUpdater,
ctrlOpts control.Options,
doNotMergeItems bool,
) Collection {
collection := Collection{
ac: ac,
category: category,
ctrl: ctrlOpts,
data: make(chan data.Stream, collectionChannelBufferSize),
@ -135,14 +139,14 @@ func (col *Collection) Items() <-chan data.Stream {
// GetQueryAndSerializeFunc helper function that returns the two functions functions
// required to convert M365 identifier into a byte array filled with the serialized data
func GetQueryAndSerializeFunc(category path.CategoryType) (GraphRetrievalFunc, GraphSerializeFunc) {
func GetQueryAndSerializeFunc(ac api.Client, category path.CategoryType) (api.GraphRetrievalFunc, GraphSerializeFunc) {
switch category {
case path.ContactsCategory:
return RetrieveContactDataForUser, serializeAndStreamContact
return ac.RetrieveContactDataForUser, serializeAndStreamContact
case path.EventsCategory:
return RetrieveEventDataForUser, serializeAndStreamEvent
return ac.RetrieveEventDataForUser, serializeAndStreamEvent
case path.EmailCategory:
return RetrieveMessageDataForUser, serializeAndStreamMessage
return ac.RetrieveMessageDataForUser, serializeAndStreamMessage
// Unsupported options returns nil, nil
default:
return nil, nil
@ -203,7 +207,7 @@ func (col *Collection) streamItems(ctx context.Context) {
// get QueryBasedonIdentifier
// verify that it is the correct type in called function
// serializationFunction
query, serializeFunc := GetQueryAndSerializeFunc(col.category)
query, serializeFunc := GetQueryAndSerializeFunc(col.ac, col.category)
if query == nil {
errs = fmt.Errorf("unrecognized collection type: %s", col.category)
return
@ -262,7 +266,7 @@ func (col *Collection) streamItems(ctx context.Context) {
)
for i := 1; i <= numberOfRetries; i++ {
response, err = query(ctx, col.service, user, id)
response, err = query(ctx, user, id)
if err == nil {
break
}

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
@ -136,7 +137,7 @@ func (suite *ExchangeDataCollectionSuite) TestNewCollection_state() {
c := NewCollection(
"u",
test.curr, test.prev,
0, nil, nil, control.Options{},
0, api.Client{}, nil, nil, control.Options{},
false)
assert.Equal(t, test.expect, c.State())
})

View File

@ -1,599 +0,0 @@
package exchange
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
)
type ExchangeServiceSuite struct {
suite.Suite
credentials account.M365Config
gs graph.Servicer
}
func TestExchangeServiceSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
tester.CorsoGraphConnectorExchangeTests)
suite.Run(t, new(ExchangeServiceSuite))
}
func (suite *ExchangeServiceSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365)
require.NoError(t, err)
suite.credentials = m365
suite.gs = service
}
// TestCreateService verifies that services are created
// when called with the correct range of params. NOTE:
// incorrect tenant or password information will NOT generate
// an error.
func (suite *ExchangeServiceSuite) TestCreateService() {
creds := suite.credentials
invalidCredentials := suite.credentials
invalidCredentials.AzureClientSecret = ""
tests := []struct {
name string
credentials account.M365Config
checkErr assert.ErrorAssertionFunc
}{
{
name: "Valid Service Creation",
credentials: creds,
checkErr: assert.NoError,
},
{
name: "Invalid Service Creation",
credentials: invalidCredentials,
checkErr: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
t.Log(test.credentials.AzureClientSecret)
_, err := createService(test.credentials)
test.checkErr(t, err)
})
}
}
func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := optionsForCalendars(test.params)
test.checkError(t, err)
})
}
}
// TestOptionsForFolders ensures that approved query options
// are added to the RequestBuildConfiguration. Expected will always be +1
// on than the input as "id" are always included within the select parameters
func (suite *ExchangeServiceSuite) TestOptionsForFolders() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Folder Option",
params: []string{"parentFolderId"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Folder Options: Valid",
params: []string{"displayName", "isHidden"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Folder option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
config, err := optionsForMailFolders(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(config.QueryParameters.Select))
}
})
}
}
// TestOptionsForContacts similar to TestExchangeService_optionsForFolders
func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
tests := []struct {
name string
params []string
checkError assert.ErrorAssertionFunc
expected int
}{
{
name: "Valid Contact Option",
params: []string{"displayName"},
checkError: assert.NoError,
expected: 2,
},
{
name: "Multiple Contact Options: Valid",
params: []string{"displayName", "parentFolderId"},
checkError: assert.NoError,
expected: 3,
},
{
name: "Invalid Contact Option param",
params: []string{"status"},
checkError: assert.Error,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
options, err := optionsForContacts(test.params)
test.checkError(t, err)
if err == nil {
suite.Equal(test.expected, len(options.QueryParameters.Select))
}
})
}
}
// TestGraphQueryFunctions verifies if Query functions APIs
// through Microsoft Graph are functional
func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
ctx, flush := tester.NewContext()
defer flush()
userID := tester.M365UserID(suite.T())
tests := []struct {
name string
function GraphQuery
}{
{
name: "GraphQuery: Get All Contacts For User",
function: GetAllContactsForUser,
},
{
name: "GraphQuery: Get All Folders",
function: GetAllFolderNamesForUser,
},
{
name: "GraphQuery: Get All ContactFolders",
function: GetAllContactFolderNamesForUser,
},
{
name: "GraphQuery: Get Default ContactFolder",
function: GetDefaultContactFolderForUser,
},
{
name: "GraphQuery: Get All Events for User",
function: GetAllEventsForUser,
},
{
name: "GraphQuery: Get All Calendars for User",
function: GetAllCalendarNamesForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
response, err := test.function(ctx, suite.gs, userID)
assert.NoError(t, err)
assert.NotNil(t, response)
})
}
}
//==========================
// Restore Functions
//==========================
// TestRestoreContact ensures contact object can be created, placed into
// the Corso Folder. The function handles test clean-up.
func (suite *ExchangeServiceSuite) TestRestoreContact() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
now = time.Now()
folderName = "TestRestoreContact: " + common.FormatSimpleDateTime(now)
)
aFolder, err := CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
folderID := *aFolder.GetId()
defer func() {
// Remove the folder containing contact prior to exiting test
err = DeleteContactFolder(ctx, suite.gs, userID, folderID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeContact(ctx,
mockconnector.GetMockContactBytes("Corso TestContact"),
suite.gs,
control.Copy,
folderID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "contact item info")
}
// TestRestoreEvent verifies that event object is able to created
// and sent into the test account of the Corso user in the newly created Corso Calendar
func (suite *ExchangeServiceSuite) TestRestoreEvent() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
name = "TestRestoreEvent: " + common.FormatSimpleDateTime(time.Now())
)
calendar, err := CreateCalendar(ctx, suite.gs, userID, name)
require.NoError(t, err)
calendarID := *calendar.GetId()
defer func() {
// Removes calendar containing events created during the test
err = DeleteCalendar(ctx, suite.gs, userID, calendarID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeEvent(ctx,
mockconnector.GetMockEventWithAttendeesBytes(name),
suite.gs,
control.Copy,
calendarID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "event item info")
}
// TestRestoreExchangeObject verifies path.Category usage for restored objects
func (suite *ExchangeServiceSuite) TestRestoreExchangeObject() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
service, err := createService(m365)
require.NoError(suite.T(), err)
userID := tester.M365UserID(suite.T())
now := time.Now()
tests := []struct {
name string
bytes []byte
category path.CategoryType
cleanupFunc func(context.Context, graph.Servicer, string, string) error
destination func(*testing.T, context.Context) string
}{
{
name: "Test Mail",
bytes: mockconnector.GetMockMessageBytes("Restore Exchange Object"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Direct Attachment",
bytes: mockconnector.GetMockMessageWithDirectAttachment("Restore 1 Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Large Attachment",
bytes: mockconnector.GetMockMessageWithLargeAttachment("Restore Large Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Two Attachments",
bytes: mockconnector.GetMockMessageWithTwoAttachments("Restore 2 Attachments"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Reference(OneDrive) Attachment",
bytes: mockconnector.GetMessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"),
category: path.EmailCategory,
cleanupFunc: DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now)
folder, err := CreateMailFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
// TODO: #884 - reinstate when able to specify root folder by name
{
name: "Test Contact",
bytes: mockconnector.GetMockContactBytes("Test_Omega"),
category: path.ContactsCategory,
cleanupFunc: DeleteContactFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now)
folder, err := CreateContactFolder(ctx, suite.gs, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Events",
bytes: mockconnector.GetDefaultMockEventBytes("Restored Event Object"),
category: path.EventsCategory,
cleanupFunc: DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now)
calendar, err := CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
{
name: "Test Event with Attachment",
bytes: mockconnector.GetMockEventWithAttachment("Restored Event Attachment"),
category: path.EventsCategory,
cleanupFunc: DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now)
calendar, err := CreateCalendar(ctx, suite.gs, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
destination := test.destination(t, ctx)
info, err := RestoreExchangeObject(
ctx,
test.bytes,
test.category,
control.Copy,
service,
destination,
userID,
)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "item info is populated")
cleanupError := test.cleanupFunc(ctx, service, userID, destination)
assert.NoError(t, cleanupError)
})
}
}
// Testing to ensure that cache system works for in multiple different environments
func (suite *ExchangeServiceSuite) TestGetContainerIDFromCache() {
ctx, flush := tester.NewContext()
defer flush()
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
connector, err := createService(m365)
require.NoError(suite.T(), err)
var (
user = tester.M365UserID(suite.T())
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string
pathFunc1 func(t *testing.T) path.Path
pathFunc2 func(t *testing.T) path.Path
category path.CategoryType
}{
{
name: "Mail Cache Test",
category: path.EmailCategory,
pathFunc1: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Croix").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
pathFunc2: func(t *testing.T) path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Felicius").ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EmailCategory,
false,
)
require.NoError(t, err)
return pth
},
},
{
name: "Contact Cache Test",
category: path.ContactsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("HufflePuff").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Ravenclaw").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
{
name: "Event Cache Test",
category: path.EventsCategory,
pathFunc1: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Durmstrang").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
pathFunc2: func(t *testing.T) path.Path {
aPath, err := path.Builder{}.Append("Beauxbatons").
ToDataLayerExchangePathForCategory(
suite.credentials.AzureTenantID,
user,
path.EventsCategory,
false,
)
require.NoError(t, err)
return aPath
},
},
}
)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc1(t),
folderName,
directoryCaches)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := CreateContainerDestinaion(
ctx,
connector,
test.pathFunc2(t),
folderName,
directoryCaches)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})
}
}

View File

@ -7,13 +7,15 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
)
type CacheResolverSuite struct {
suite.Suite
gs graph.Servicer
credentials account.M365Config
}
func TestCacheResolverIntegrationSuite(t *testing.T) {
@ -35,27 +37,27 @@ func (suite *CacheResolverSuite) SetupSuite() {
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365)
require.NoError(t, err)
suite.gs = service
suite.credentials = m365
}
func (suite *CacheResolverSuite) TestPopulate() {
ctx, flush := tester.NewContext()
defer flush()
ac, err := api.NewClient(suite.credentials)
require.NoError(suite.T(), err)
eventFunc := func(t *testing.T) graph.ContainerResolver {
return &eventCalendarCache{
userID: tester.M365UserID(t),
gs: suite.gs,
ac: ac,
}
}
contactFunc := func(t *testing.T) graph.ContainerResolver {
return &contactFolderCache{
userID: tester.M365UserID(t),
gs: suite.gs,
ac: ac,
}
}

View File

@ -10,10 +10,12 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -83,7 +85,7 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
tests := []struct {
name string
queryFunc GraphQuery
queryFunc func(*testing.T, account.M365Config) api.GraphQuery
scope selectors.ExchangeScope
iterativeFunction func(
container map[string]graph.Container,
@ -93,13 +95,21 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
}{
{
name: "Contacts Iterative Check",
queryFunc: GetAllContactFolderNamesForUser,
queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery {
ac, err := api.NewClient(amc)
require.NoError(t, err)
return ac.GetAllContactFolderNamesForUser
},
transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectContactContainers,
},
{
name: "Events Iterative Check",
queryFunc: GetAllCalendarNamesForUser,
queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery {
ac, err := api.NewClient(amc)
require.NoError(t, err)
return ac.GetAllCalendarNamesForUser
},
transformer: models.CreateCalendarCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectCalendarContainers,
},
@ -113,7 +123,7 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
service, err := createService(m365)
require.NoError(t, err)
response, err := test.queryFunc(ctx, service, userID)
response, err := test.queryFunc(t, m365)(ctx, userID)
require.NoError(t, err)
// Iterator Creation

View File

@ -7,6 +7,7 @@ import (
msfolderdelta "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
@ -19,7 +20,8 @@ var _ graph.ContainerResolver = &mailFolderCache{}
// nameLookup map: Key: DisplayName Value: ID
type mailFolderCache struct {
*containerResolver
gs graph.Servicer
// gs graph.Servicer
ac api.Client
userID string
}
@ -31,22 +33,10 @@ type mailFolderCache struct {
func (mc *mailFolderCache) populateMailRoot(
ctx context.Context,
) error {
wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForMailFoldersItem(wantedOpts)
if err != nil {
return errors.Wrapf(err, "getting options for mail folders %v", wantedOpts)
}
for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} {
var directory string
f, err := mc.
gs.
Client().
UsersById(mc.userID).
MailFoldersById(fldr).
Get(ctx, opts)
f, err := mc.ac.GetMailFolderByID(ctx, mc.userID, fldr, "displayName", "parentFolderId")
if err != nil {
return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err))
}
@ -56,7 +46,6 @@ func (mc *mailFolderCache) populateMailRoot(
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(directory))
if err := mc.addFolder(temp); err != nil {
return errors.Wrap(err, "initializing mail resolver")
}
@ -79,15 +68,10 @@ func (mc *mailFolderCache) Populate(
return err
}
// Even though this uses the `Delta` query, we do no store or re-use
// the delta-link tokens like with other queries. The goal is always
// to retrieve the complete history of folders.
query := mc.
gs.
Client().
UsersById(mc.userID).
MailFolders().
Delta()
query, servicer, err := mc.ac.GetAllMailFoldersBuilder(ctx, mc.userID)
if err != nil {
return err
}
var errs *multierror.Error
@ -114,7 +98,7 @@ func (mc *mailFolderCache) Populate(
break
}
query = msfolderdelta.NewItemMailFoldersDeltaRequestBuilder(*link, mc.gs.Adapter())
query = msfolderdelta.NewItemMailFoldersDeltaRequestBuilder(*link, servicer.Adapter())
}
if err := mc.populatePaths(ctx); err != nil {

View File

@ -8,8 +8,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
)
const (
@ -26,7 +27,7 @@ const (
type MailFolderCacheIntegrationSuite struct {
suite.Suite
gs graph.Servicer
credentials account.M365Config
}
func TestMailFolderCacheIntegrationSuite(t *testing.T) {
@ -48,10 +49,7 @@ func (suite *MailFolderCacheIntegrationSuite) SetupSuite() {
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365)
require.NoError(t, err)
suite.gs = service
suite.credentials = m365
}
func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
@ -83,9 +81,12 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ac, err := api.NewClient(suite.credentials)
require.NoError(t, err)
mfc := mailFolderCache{
userID: userID,
gs: suite.gs,
ac: ac,
}
require.NoError(t, mfc.Populate(ctx, test.root, test.path...))

View File

@ -0,0 +1,274 @@
package exchange
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
)
type ExchangeRestoreSuite struct {
suite.Suite
gs graph.Servicer
credentials account.M365Config
ac api.Client
}
func TestExchangeRestoreSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoConnectorRestoreExchangeCollectionTests)
suite.Run(t, new(ExchangeRestoreSuite))
}
func (suite *ExchangeRestoreSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.credentials = m365
suite.ac, err = api.NewClient(m365)
require.NoError(t, err)
adpt, err := graph.CreateAdapter(m365.AzureTenantID, m365.AzureClientID, m365.AzureClientSecret)
require.NoError(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
}
// TestRestoreContact ensures contact object can be created, placed into
// the Corso Folder. The function handles test clean-up.
func (suite *ExchangeRestoreSuite) TestRestoreContact() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
now = time.Now()
folderName = "TestRestoreContact: " + common.FormatSimpleDateTime(now)
)
aFolder, err := suite.ac.CreateContactFolder(ctx, userID, folderName)
require.NoError(t, err)
folderID := *aFolder.GetId()
defer func() {
// Remove the folder containing contact prior to exiting test
err = suite.ac.DeleteContactFolder(ctx, userID, folderID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeContact(
ctx,
mockconnector.GetMockContactBytes("Corso TestContact"),
suite.gs,
control.Copy,
folderID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "contact item info")
}
// TestRestoreEvent verifies that event object is able to created
// and sent into the test account of the Corso user in the newly created Corso Calendar
func (suite *ExchangeRestoreSuite) TestRestoreEvent() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
userID = tester.M365UserID(t)
name = "TestRestoreEvent: " + common.FormatSimpleDateTime(time.Now())
)
calendar, err := suite.ac.CreateCalendar(ctx, userID, name)
require.NoError(t, err)
calendarID := *calendar.GetId()
defer func() {
// Removes calendar containing events created during the test
err = suite.ac.DeleteCalendar(ctx, userID, calendarID)
assert.NoError(t, err)
}()
info, err := RestoreExchangeEvent(ctx,
mockconnector.GetMockEventWithAttendeesBytes(name),
suite.gs,
control.Copy,
calendarID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "event item info")
}
// TestRestoreExchangeObject verifies path.Category usage for restored objects
func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
service, err := createService(m365)
require.NoError(suite.T(), err)
userID := tester.M365UserID(suite.T())
now := time.Now()
tests := []struct {
name string
bytes []byte
category path.CategoryType
cleanupFunc func(context.Context, string, string) error
destination func(*testing.T, context.Context) string
}{
{
name: "Test Mail",
bytes: mockconnector.GetMockMessageBytes("Restore Exchange Object"),
category: path.EmailCategory,
cleanupFunc: suite.ac.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.NewName(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Direct Attachment",
bytes: mockconnector.GetMockMessageWithDirectAttachment("Restore 1 Attachment"),
category: path.EmailCategory,
cleanupFunc: suite.ac.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.NewName(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: One Large Attachment",
bytes: mockconnector.GetMockMessageWithLargeAttachment("Restore Large Attachment"),
category: path.EmailCategory,
cleanupFunc: suite.ac.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.NewName(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Two Attachments",
bytes: mockconnector.GetMockMessageWithTwoAttachments("Restore 2 Attachments"),
category: path.EmailCategory,
cleanupFunc: suite.ac.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.NewName(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Mail: Reference(OneDrive) Attachment",
bytes: mockconnector.GetMessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"),
category: path.EmailCategory,
cleanupFunc: suite.ac.DeleteMailFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.NewName(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
// TODO: #884 - reinstate when able to specify root folder by name
{
name: "Test Contact",
bytes: mockconnector.GetMockContactBytes("Test_Omega"),
category: path.ContactsCategory,
cleanupFunc: suite.ac.DeleteContactFolder,
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.CreateContactFolder(ctx, userID, folderName)
require.NoError(t, err)
return *folder.GetId()
},
},
{
name: "Test Events",
bytes: mockconnector.GetDefaultMockEventBytes("Restored Event Object"),
category: path.EventsCategory,
cleanupFunc: suite.ac.DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now)
calendar, err := suite.ac.CreateCalendar(ctx, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
{
name: "Test Event with Attachment",
bytes: mockconnector.GetMockEventWithAttachment("Restored Event Attachment"),
category: path.EventsCategory,
cleanupFunc: suite.ac.DeleteCalendar,
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now)
calendar, err := suite.ac.CreateCalendar(ctx, userID, calendarName)
require.NoError(t, err)
return *calendar.GetId()
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
destination := test.destination(t, ctx)
info, err := RestoreExchangeObject(
ctx,
test.bytes,
test.category,
control.Copy,
service,
destination,
userID,
)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, info, "item info is populated")
cleanupError := test.cleanupFunc(ctx, userID, destination)
assert.NoError(t, cleanupError)
})
}
}

View File

@ -4,9 +4,9 @@ import (
"context"
"fmt"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/path"
@ -15,9 +15,6 @@ import (
var ErrFolderNotFound = errors.New("folder not found")
// createService internal constructor for exchangeService struct returns an error
// iff the params for the entry are incorrect (e.g. len(TenantID) == 0, etc.)
// NOTE: Incorrect account information will result in errors on subsequent queries.
func createService(credentials account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
@ -31,76 +28,7 @@ func createService(credentials account.M365Config) (*graph.Service, error) {
return graph.NewService(adapter), nil
}
// CreateMailFolder makes a mail folder iff a folder of the same name does not exist
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http
func CreateMailFolder(ctx context.Context, gs graph.Servicer, user, folder string) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
}
func CreateMailFolderWithParent(
ctx context.Context,
gs graph.Servicer,
user, folder, parentID string,
) (models.MailFolderable, error) {
isHidden := false
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return gs.Client().
UsersById(user).
MailFoldersById(parentID).
ChildFolders().
Post(ctx, requestBody, nil)
}
// DeleteMailFolder removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/mailfolder-delete?view=graph-rest-1.0&tabs=http
func DeleteMailFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
}
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go
func CreateCalendar(ctx context.Context, gs graph.Servicer, user, calendarName string) (models.Calendarable, error) {
requestbody := models.NewCalendar()
requestbody.SetName(&calendarName)
return gs.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
}
// DeleteCalendar removes calendar from user's M365 account
// Reference: https://docs.microsoft.com/en-us/graph/api/calendar-delete?view=graph-rest-1.0&tabs=go
func DeleteCalendar(ctx context.Context, gs graph.Servicer, user, calendarID string) error {
return gs.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
}
// CreateContactFolder makes a contact folder with the displayName of folderName.
// If successful, returns the created folder object.
func CreateContactFolder(
ctx context.Context,
gs graph.Servicer,
user, folderName string,
) (models.ContactFolderable, error) {
requestBody := models.NewContactFolder()
temp := folderName
requestBody.SetDisplayName(&temp)
return gs.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
}
// DeleteContactFolder deletes the ContactFolder associated with the M365 ID if permissions are valid.
// Errors returned if the function call was not successful.
func DeleteContactFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error {
return gs.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
}
// PopulateExchangeContainerResolver gets a folder resolver if one is available for
// populateExchangeContainerResolver gets a folder resolver if one is available for
// this category of data. If one is not available, returns nil so that other
// logic in the caller can complete as long as they check if the resolver is not
// nil. If an error occurs populating the resolver, returns an error.
@ -111,9 +39,9 @@ func PopulateExchangeContainerResolver(
var (
res graph.ContainerResolver
cacheRoot string
service, err = createService(qp.Credentials)
)
ac, err := api.NewClient(qp.Credentials)
if err != nil {
return nil, err
}
@ -122,21 +50,21 @@ func PopulateExchangeContainerResolver(
case path.EmailCategory:
res = &mailFolderCache{
userID: qp.ResourceOwner,
gs: service,
ac: ac,
}
cacheRoot = rootFolderAlias
case path.ContactsCategory:
res = &contactFolderCache{
userID: qp.ResourceOwner,
gs: service,
ac: ac,
}
cacheRoot = DefaultContactFolder
case path.EventsCategory:
res = &eventCalendarCache{
userID: qp.ResourceOwner,
gs: service,
ac: ac,
}
cacheRoot = DefaultCalendar

View File

@ -5,11 +5,10 @@ import (
"fmt"
"strings"
multierror "github.com/hashicorp/go-multierror"
"github.com/microsoftgraph/msgraph-sdk-go/models"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -19,14 +18,6 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
)
// carries details about delta retrieval in aggregators
type deltaUpdate struct {
// the deltaLink itself
url string
// true if the old delta was marked as invalid
reset bool
}
// filterContainersAndFillCollections is a utility function
// that places the M365 object ids belonging to specific directories
// into a Collection. Messages outside of those directories are omitted.
@ -52,7 +43,13 @@ func filterContainersAndFillCollections(
tombstones = makeTombstones(dps)
)
getJobs, err := getFetchIDFunc(qp.Category)
// TODO(rkeepers): pass in the api client instead of generating it here.
ac, err := api.NewClient(qp.Credentials)
if err != nil {
return err
}
getJobs, err := getFetchIDFunc(ac, qp.Category)
if err != nil {
return support.WrapAndAppend(qp.ResourceOwner, err, errs)
}
@ -94,7 +91,7 @@ func filterContainersAndFillCollections(
}
}
added, removed, newDelta, err := getJobs(ctx, service, qp.ResourceOwner, cID, prevDelta)
added, removed, newDelta, err := getJobs(ctx, qp.ResourceOwner, cID, prevDelta)
if err != nil {
if graph.IsErrDeletedInFlight(err) == nil {
errs = support.WrapAndAppend(qp.ResourceOwner, err, errs)
@ -104,14 +101,14 @@ func filterContainersAndFillCollections(
// to reset which will prevent any old items from being retained in
// storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned.
newDelta = deltaUpdate{reset: true}
newDelta = api.DeltaUpdate{Reset: true}
}
continue
}
if len(newDelta.url) > 0 {
deltaURLs[cID] = newDelta.url
if len(newDelta.URL) > 0 {
deltaURLs[cID] = newDelta.URL
}
edc := NewCollection(
@ -119,11 +116,11 @@ func filterContainersAndFillCollections(
currPath,
prevPath,
scope.Category().PathType(),
ac,
service,
statusUpdater,
ctrlOpts,
newDelta.reset,
)
newDelta.Reset)
collections[cID] = &edc
edc.added = append(edc.added, added...)
@ -169,11 +166,11 @@ func filterContainersAndFillCollections(
nil, // marks the collection as deleted
prevPath,
scope.Category().PathType(),
ac,
service,
statusUpdater,
ctrlOpts,
false,
)
false)
collections[id] = &edc
}
@ -276,284 +273,18 @@ func IterativeCollectCalendarContainers(
// container supports fetching delta records.
type FetchIDFunc func(
ctx context.Context,
gs graph.Servicer,
user, containerID, oldDeltaToken string,
) ([]string, []string, deltaUpdate, error)
) ([]string, []string, api.DeltaUpdate, error)
func getFetchIDFunc(category path.CategoryType) (FetchIDFunc, error) {
func getFetchIDFunc(ac api.Client, category path.CategoryType) (FetchIDFunc, error) {
switch category {
case path.EmailCategory:
return FetchMessageIDsFromDirectory, nil
return ac.FetchMessageIDsFromDirectory, nil
case path.EventsCategory:
return FetchEventIDsFromCalendar, nil
return ac.FetchEventIDsFromCalendar, nil
case path.ContactsCategory:
return FetchContactIDsFromDirectory, nil
return ac.FetchContactIDsFromDirectory, nil
default:
return nil, fmt.Errorf("category %s not supported by getFetchIDFunc", category)
}
}
// ---------------------------------------------------------------------------
// events
// ---------------------------------------------------------------------------
// FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
func FetchEventIDsFromCalendar(
ctx context.Context,
gs graph.Servicer,
user, calendarID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
)
options, err := optionsForEventsByCalendar([]string{"id"})
if err != nil {
return nil, nil, deltaUpdate{}, err
}
builder := gs.Client().
UsersById(user).
CalendarsById(calendarID).
Events()
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return nil, nil, deltaUpdate{}, err
}
return nil, nil, deltaUpdate{}, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("event with nil ID in calendar %s", calendarID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
ids = append(ids, *item.GetId())
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemCalendarsItemEventsRequestBuilder(*nextLink, gs.Adapter())
}
// Events don't have a delta endpoint so just return an empty string.
return ids, nil, deltaUpdate{}, errs.ErrorOrNil()
}
// ---------------------------------------------------------------------------
// contacts
// ---------------------------------------------------------------------------
// FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts
// of the targeted directory
func FetchContactIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, deltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *msuser.ItemContactFoldersItemContactsDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter()))
// happy path
if err == nil {
return ids, removedIDs, deltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, deltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
ContactFoldersById(directoryID).
Contacts().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, deltaUpdate{}, err
}
return ids, removedIDs, deltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}
// ---------------------------------------------------------------------------
// messages
// ---------------------------------------------------------------------------
// FetchMessageIDsFromDirectory function that returns a list of all the m365IDs of the exchange.Mail
// of the targeted directory
func FetchMessageIDsFromDirectory(
ctx context.Context,
gs graph.Servicer,
user, directoryID, oldDelta string,
) ([]string, []string, deltaUpdate, error) {
var (
errs *multierror.Error
ids []string
removedIDs []string
deltaURL string
resetDelta bool
)
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, deltaUpdate{}, errors.Wrap(err, "getting query options")
}
getIDs := func(builder *msuser.ItemMailFoldersItemMessagesDeltaRequestBuilder) error {
for {
resp, err := builder.Get(ctx, options)
if err != nil {
if err := graph.IsErrDeletedInFlight(err); err != nil {
return err
}
if err := graph.IsErrInvalidDelta(err); err != nil {
return err
}
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, item := range resp.GetValue() {
if item.GetId() == nil {
errs = multierror.Append(
errs,
errors.Errorf("item with nil ID in folder %s", directoryID),
)
// TODO(ashmrtn): Handle fail-fast.
continue
}
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
ids = append(ids, *item.GetId())
} else {
removedIDs = append(removedIDs, *item.GetId())
}
}
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
}
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
break
}
builder = msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, gs.Adapter())
}
return nil
}
if len(oldDelta) > 0 {
err := getIDs(msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()))
// happy path
if err == nil {
return ids, removedIDs, deltaUpdate{deltaURL, false}, errs.ErrorOrNil()
}
// only return on error if it is NOT a delta issue.
// otherwise we'll retry the call with the regular builder
if graph.IsErrInvalidDelta(err) == nil {
return nil, nil, deltaUpdate{}, err
}
resetDelta = true
errs = nil
}
builder := gs.Client().
UsersById(user).
MailFoldersById(directoryID).
Messages().
Delta()
if err := getIDs(builder); err != nil {
return nil, nil, deltaUpdate{}, err
}
return ids, removedIDs, deltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
}

View File

@ -1,109 +0,0 @@
package exchange
import (
"context"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/internal/connector/graph"
)
// GraphQuery represents functions which perform exchange-specific queries
// into M365 backstore. Responses -> returned items will only contain the information
// that is included in the options
// TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, gs graph.Servicer, userID string) (absser.Parsable, error)
// GetAllContactsForUser is a GraphQuery function for querying all the contacts in a user's account
func GetAllContactsForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
selecting := []string{"parentFolderId"}
options, err := optionsForContacts(selecting)
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Contacts().Get(ctx, options)
}
// GetAllFolderDisplayNamesForUser is a GraphQuery function for getting FolderId and display
// names for Mail Folder. All other information for the MailFolder object is omitted.
func GetAllFolderNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForMailFolders([]string{"displayName"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).MailFolders().Get(ctx, options)
}
func GetAllCalendarNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForCalendars([]string{"name", "owner"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Calendars().Get(ctx, options)
}
// GetDefaultContactFolderForUser is a GraphQuery function for getting the ContactFolderId
// and display names for the default "Contacts" folder.
// Only returns the default Contact Folder
func GetDefaultContactFolderForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForContactChildFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return gs.Client().
UsersById(user).
ContactFoldersById(rootFolderAlias).
ChildFolders().
Get(ctx, options)
}
// GetAllContactFolderNamesForUser is a GraphQuery function for getting ContactFolderId
// and display names for contacts. All other information is omitted.
// Does not return the default Contact Folder
func GetAllContactFolderNamesForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).ContactFolders().Get(ctx, options)
}
// GetAllEvents for User. Default returns EventResponseCollection for future events.
// of the time that the call was made. 'calendar' option must be present to gain
// access to additional data map in future calls.
func GetAllEventsForUser(ctx context.Context, gs graph.Servicer, user string) (absser.Parsable, error) {
options, err := optionsForEvents([]string{"id", "calendar"})
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Events().Get(ctx, options)
}
// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve
// the default associated data of a M365 object. This varies by object. Additional
// Queries must be run to obtain the omitted fields.
type GraphRetrievalFunc func(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error)
// RetrieveContactDataForUser is a GraphRetrievalFun that returns all associated fields.
func RetrieveContactDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil)
}
// RetrieveEventDataForUser is a GraphRetrievalFunc that returns event data.
// Calendarable and attachment fields are omitted due to size
func RetrieveEventDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil)
}
// RetrieveMessageDataForUser is a GraphRetrievalFunc that returns message data.
// Attachment field is omitted due to size.
func RetrieveMessageDataForUser(ctx context.Context, gs graph.Servicer, user, m365ID string) (absser.Parsable, error) {
return gs.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil)
}

View File

@ -11,11 +11,13 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
D "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
@ -283,6 +285,7 @@ func SendMailToBackStore(
// @param dest: container destination to M365
func RestoreExchangeDataCollections(
ctx context.Context,
creds account.M365Config,
gs graph.Servicer,
dest control.RestoreDestination,
dcs []data.Collection,
@ -312,7 +315,7 @@ func RestoreExchangeDataCollections(
containerID, err := CreateContainerDestinaion(
ctx,
gs,
creds,
dc.FullPath(),
dest.ContainerName,
userCaches)
@ -430,7 +433,7 @@ func restoreCollection(
// @ returns the container ID of the new destination container.
func CreateContainerDestinaion(
ctx context.Context,
gs graph.Servicer,
creds account.M365Config,
directory path.Path,
destination string,
caches map[path.CategoryType]graph.ContainerResolver,
@ -443,12 +446,18 @@ func CreateContainerDestinaion(
newPathFolders = append([]string{destination}, directory.Folders()...)
)
// TODO(rkeepers): pass the api client into this func, rather than generating one.
ac, err := api.NewClient(creds)
if err != nil {
return "", err
}
switch category {
case path.EmailCategory:
if directoryCache == nil {
mfc := &mailFolderCache{
userID: user,
gs: gs,
ac: ac,
}
caches[category] = mfc
@ -458,16 +467,17 @@ func CreateContainerDestinaion(
return establishMailRestoreLocation(
ctx,
ac,
newPathFolders,
directoryCache,
user,
gs,
newCache)
case path.ContactsCategory:
if directoryCache == nil {
cfc := &contactFolderCache{
userID: user,
gs: gs,
ac: ac,
}
caches[category] = cfc
newCache = true
@ -476,16 +486,17 @@ func CreateContainerDestinaion(
return establishContactsRestoreLocation(
ctx,
ac,
newPathFolders,
directoryCache,
user,
gs,
newCache)
case path.EventsCategory:
if directoryCache == nil {
ecc := &eventCalendarCache{
userID: user,
gs: gs,
ac: ac,
}
caches[category] = ecc
newCache = true
@ -494,10 +505,10 @@ func CreateContainerDestinaion(
return establishEventsRestoreLocation(
ctx,
ac,
newPathFolders,
directoryCache,
user,
gs,
newCache,
)
default:
@ -512,10 +523,10 @@ func CreateContainerDestinaion(
// @param isNewCache identifies if the cache is created and not populated
func establishMailRestoreLocation(
ctx context.Context,
ac api.Client,
folders []string,
mfc graph.ContainerResolver,
user string,
service graph.Servicer,
isNewCache bool,
) (string, error) {
// Process starts with the root folder in order to recreate
@ -532,8 +543,7 @@ func establishMailRestoreLocation(
continue
}
temp, err := CreateMailFolderWithParent(ctx,
service, user, folder, folderID)
temp, err := ac.CreateMailFolderWithParent(ctx, user, folder, folderID)
if err != nil {
// Should only error if cache malfunctions or incorrect parameters
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
@ -569,10 +579,10 @@ func establishMailRestoreLocation(
// @param isNewCache bool representation of whether Populate function needs to be run
func establishContactsRestoreLocation(
ctx context.Context,
ac api.Client,
folders []string,
cfc graph.ContainerResolver,
user string,
gs graph.Servicer,
isNewCache bool,
) (string, error) {
cached, ok := cfc.PathInCache(folders[0])
@ -580,7 +590,7 @@ func establishContactsRestoreLocation(
return cached, nil
}
temp, err := CreateContactFolder(ctx, gs, user, folders[0])
temp, err := ac.CreateContactFolder(ctx, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
@ -602,10 +612,10 @@ func establishContactsRestoreLocation(
func establishEventsRestoreLocation(
ctx context.Context,
ac api.Client,
folders []string,
ecc graph.ContainerResolver, // eventCalendarCache
user string,
gs graph.Servicer,
isNewCache bool,
) (string, error) {
cached, ok := ecc.PathInCache(folders[0])
@ -613,7 +623,7 @@ func establishEventsRestoreLocation(
return cached, nil
}
temp, err := CreateCalendar(ctx, gs, user, folders[0])
temp, err := ac.CreateCalendar(ctx, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}

View File

@ -252,6 +252,7 @@ func (gc *GraphConnector) UnionSiteIDsAndWebURLs(ctx context.Context, ids, urls
// SideEffect: gc.status is updated at the completion of operation
func (gc *GraphConnector) RestoreDataCollections(
ctx context.Context,
acct account.Account,
selector selectors.Selector,
dest control.RestoreDestination,
dcs []data.Collection,
@ -265,9 +266,14 @@ func (gc *GraphConnector) RestoreDataCollections(
deets = &details.Builder{}
)
creds, err := acct.M365Config()
if err != nil {
return nil, errors.Wrap(err, "malformed azure credentials")
}
switch selector.Service {
case selectors.ServiceExchange:
status, err = exchange.RestoreExchangeDataCollections(ctx, gc.Service, dest, dcs, deets)
status, err = exchange.RestoreExchangeDataCollections(ctx, creds, gc.Service, dest, dcs, deets)
case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(ctx, gc.Service, dest, dcs, deets)
case selectors.ServiceSharePoint:

View File

@ -168,18 +168,20 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_ErrorChecking()
}
func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
gc := GraphConnector{wg: &sync.WaitGroup{}}
sel := selectors.Selector{
var (
t = suite.T()
acct = tester.NewM365Account(t)
dest = tester.DefaultTestRestoreDestination()
gc = GraphConnector{wg: &sync.WaitGroup{}}
sel = selectors.Selector{
Service: selectors.ServiceUnknown,
}
dest := tester.DefaultTestRestoreDestination()
)
deets, err := gc.RestoreDataCollections(ctx, sel, dest, nil)
deets, err := gc.RestoreDataCollections(ctx, acct, sel, dest, nil)
assert.Error(t, err)
assert.NotNil(t, deets)

View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
@ -135,6 +136,7 @@ type GraphConnectorIntegrationSuite struct {
suite.Suite
connector *GraphConnector
user string
acct account.Account
}
func TestGraphConnectorIntegrationSuite(t *testing.T) {
@ -155,6 +157,7 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() {
suite.connector = loadConnector(ctx, suite.T(), Users)
suite.user = tester.M365UserID(suite.T())
suite.acct = tester.NewM365Account(suite.T())
tester.LogTimeOfTest(suite.T())
}
@ -265,7 +268,12 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
ctx, flush := tester.NewContext()
defer flush()
deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col)
deets, err := suite.connector.RestoreDataCollections(
ctx,
suite.acct,
test.sel,
dest,
test.col)
require.NoError(t, err)
assert.NotNil(t, deets)
@ -308,6 +316,7 @@ func mustGetDefaultDriveID(
func runRestoreBackupTest(
t *testing.T,
acct account.Account,
test restoreBackupInfo,
tenant string,
users []string,
@ -349,7 +358,12 @@ func runRestoreBackupTest(
restoreGC := loadConnector(ctx, t, test.resource)
restoreSel := getSelectorWith(test.service)
deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
deets, err := restoreGC.RestoreDataCollections(
ctx,
acct,
restoreSel,
dest,
collections)
require.NoError(t, err)
assert.NotNil(t, deets)
@ -724,7 +738,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
runRestoreBackupTest(t, test, suite.connector.tenant, []string{suite.user})
runRestoreBackupTest(t, suite.acct, test, suite.connector.tenant, []string{suite.user})
})
}
}
@ -833,7 +847,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
)
restoreGC := loadConnector(ctx, t, test.resource)
deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
deets, err := restoreGC.RestoreDataCollections(ctx, suite.acct, restoreSel, dest, collections)
require.NoError(t, err)
require.NotNil(t, deets)
@ -977,7 +991,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiuserRestoreAndBackup() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
runRestoreBackupTest(t, test, suite.connector.tenant, users)
runRestoreBackupTest(t, suite.acct, test, suite.connector.tenant, users)
})
}
}

View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
@ -293,6 +294,7 @@ func generateContainerOfItems(
ctx context.Context,
gc *connector.GraphConnector,
service path.ServiceType,
acct account.Account,
cat path.CategoryType,
sel selectors.Selector,
tenantID, userID, destFldr string,
@ -329,7 +331,7 @@ func generateContainerOfItems(
dest,
collections)
deets, err := gc.RestoreDataCollections(ctx, sel, dest, dataColls)
deets, err := gc.RestoreDataCollections(ctx, acct, sel, dest, dataColls)
require.NoError(t, err)
return deets
@ -650,6 +652,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
gc, err := connector.NewGraphConnector(ctx, acct, connector.Users)
require.NoError(t, err)
ac, err := api.NewClient(m365)
require.NoError(t, err)
// generate 3 new folders with two items each.
// Only the first two folders will be part of the initial backup and
// incrementals. The third folder will be introduced partway through
@ -708,6 +713,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
ctx,
gc,
path.ExchangeService,
acct,
category,
selectors.NewExchangeRestore(users).Selector,
m365.AzureTenantID, suite.user, destName,
@ -897,7 +903,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
switch category {
case path.EmailCategory:
ids, _, _, err := exchange.FetchMessageIDsFromDirectory(ctx, gc.Service, suite.user, folderID, "")
ids, _, _, err := ac.FetchMessageIDsFromDirectory(ctx, suite.user, folderID, "")
require.NoError(t, err, "getting message ids")
require.NotEmpty(t, ids, "message ids in folder")
@ -905,7 +911,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
require.NoError(t, err, "deleting email item: %s", support.ConnectorStackErrorTrace(err))
case path.ContactsCategory:
ids, _, _, err := exchange.FetchContactIDsFromDirectory(ctx, gc.Service, suite.user, folderID, "")
ids, _, _, err := ac.FetchContactIDsFromDirectory(ctx, suite.user, folderID, "")
require.NoError(t, err, "getting contact ids")
require.NotEmpty(t, ids, "contact ids in folder")

View File

@ -188,7 +188,12 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De
defer closer()
defer close(restoreComplete)
restoreDetails, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
restoreDetails, err = gc.RestoreDataCollections(
ctx,
op.account,
op.Selectors,
op.Destination,
dcs)
if err != nil {
err = errors.Wrap(err, "restoring service data")
opStats.writeErr = err

View File

@ -22,6 +22,8 @@ const (
CorsoConnectorCreateExchangeCollectionTests = "CORSO_CONNECTOR_CREATE_EXCHANGE_COLLECTION_TESTS"
CorsoConnectorCreateSharePointCollectionTests = "CORSO_CONNECTOR_CREATE_SHAREPOINT_COLLECTION_TESTS"
CorsoConnectorDataCollectionTests = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS"
CorsoConnectorExchangeFolderCacheTests = "CORSO_CONNECTOR_EXCHANGE_FOLDER_CACHE_TESTS"
CorsoConnectorRestoreExchangeCollectionTests = "CORSO_CONNECTOR_RESTORE_EXCHANGE_COLLECTION_TESTS"
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"