migrate resolver iterators (#2009)

## Description

Previous changes held out on migrating the
resolver iterators into the api package because
they embedded function calls from the resolvers
themselves.  This change migrates that code
into the api package, and accepts a callback
function to hook in the resolver updates.  This
change sets up resolvers to better utilize an
interface in the next PR.

## Does this PR need a docs update or release note?

- [x]  No 

## Type of change

- [x] 🤖 Test

## Issue(s)

* #1967

## Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-01-04 13:02:47 -07:00 committed by GitHub
parent 0d0a7516f0
commit 672fe54c41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 190 additions and 369 deletions

View File

@ -86,3 +86,23 @@ func newService(creds account.M365Config) (*graph.Service, error) {
return graph.NewService(adapter), nil
}
// ---------------------------------------------------------------------------
// helper funcs
// ---------------------------------------------------------------------------
// checkIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func checkIDAndName(c graph.Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
}
ptr := c.GetDisplayName()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without display name", *idPtr)
}
return nil
}

View File

@ -76,28 +76,30 @@ func (c Client) GetContactFolderByID(
Get(ctx, ofcf)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetContactChildFoldersBuilder(
// EnumerateContactsFolders iterates through all of the users current
// contacts folders, converting each to a graph.CacheFolder, and calling
// fn(cf) on each one. If fn(cf) errors, the error is aggregated
// into a multierror that gets returned to the caller.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
func (c Client) EnumerateContactsFolders(
ctx context.Context,
userID, baseDirID string,
optionalFields ...string,
) (
*users.ItemContactFoldersItemChildFoldersRequestBuilder,
*users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration,
*graph.Service,
error,
) {
fn func(graph.CacheFolder) error,
) error {
service, err := c.service()
if err != nil {
return nil, nil, nil, err
return err
}
fields := append([]string{"displayName", "parentFolderId"}, optionalFields...)
var (
errs *multierror.Error
fields = []string{"displayName", "parentFolderId"}
)
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "options for contact child folders: %v", fields)
return errors.Wrapf(err, "options for contact child folders: %v", fields)
}
builder := service.Client().
@ -105,7 +107,35 @@ func (c Client) GetContactChildFoldersBuilder(
ContactFoldersById(baseDirID).
ChildFolders()
return builder, ofcf, service, nil
for {
resp, err := builder.Get(ctx, ofcf)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, fold := range resp.GetValue() {
if err := checkIDAndName(fold); err != nil {
errs = multierror.Append(err, errs)
continue
}
temp := graph.NewCacheFolder(fold, nil)
err = fn(temp)
if err != nil {
errs = multierror.Append(err, errs)
continue
}
}
if resp.GetOdataNextLink() == nil {
break
}
builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), service.Adapter())
}
return errs.ErrorOrNil()
}
// FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
@ -54,31 +55,61 @@ func (c Client) GetAllCalendarNamesForUser(
return c.stable.Client().UsersById(user).Calendars().Get(ctx, options)
}
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetCalendarsBuilder(
// EnumerateCalendars iterates through all of the users current
// contacts folders, converting each to a graph.CacheFolder, and
// calling fn(cf) on each one. If fn(cf) errors, the error is
// aggregated into a multierror that gets returned to the caller.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
func (c Client) EnumerateCalendars(
ctx context.Context,
userID string,
optionalFields ...string,
) (
*users.ItemCalendarsRequestBuilder,
*users.ItemCalendarsRequestBuilderGetRequestConfiguration,
*graph.Service,
error,
) {
fn func(graph.CacheFolder) error,
) error {
service, err := c.service()
if err != nil {
return nil, nil, nil, err
return err
}
ofcf, err := optionsForCalendars(optionalFields)
var errs *multierror.Error
ofc, err := optionsForCalendars([]string{"name"})
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "options for event calendars: %v", optionalFields)
return errors.Wrapf(err, "options for event calendars")
}
builder := service.Client().UsersById(userID).Calendars()
return builder, ofcf, service, nil
for {
resp, err := builder.Get(ctx, ofc)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, cal := range resp.GetValue() {
cd := CalendarDisplayable{Calendarable: cal}
if err := checkIDAndName(cd); err != nil {
errs = multierror.Append(err, errs)
continue
}
temp := graph.NewCacheFolder(cd, path.Builder{}.Append(*cd.GetDisplayName()))
err = fn(temp)
if err != nil {
errs = multierror.Append(err, errs)
continue
}
}
if resp.GetOdataNextLink() == nil {
break
}
builder = users.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), service.Adapter())
}
return errs.ErrorOrNil()
}
// FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
@ -138,3 +169,30 @@ func (c Client) FetchEventIDsFromCalendar(
// Events don't have a delta endpoint so just return an empty string.
return ids, nil, DeltaUpdate{}, errs.ErrorOrNil()
}
// ---------------------------------------------------------------------------
// helper funcs
// ---------------------------------------------------------------------------
// CalendarDisplayable is a wrapper that complies with the
// models.Calendarable interface with the graph.Container
// interfaces. Calendars do not have a parentFolderID.
// Therefore, that value will always return nil.
type CalendarDisplayable struct {
models.Calendarable
}
// GetDisplayName returns the *string of the models.Calendable
// variant: calendar.GetName()
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// GetParentFolderId returns the default calendar name address
// EventCalendars have a flat hierarchy and Calendars are rooted
// at the default
//
//nolint:revive
func (c CalendarDisplayable) GetParentFolderId() *string {
return nil
}

View File

@ -66,30 +66,54 @@ func (c Client) RetrieveMessageDataForUser(
return c.stable.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil)
}
// GetMailFoldersBuilder retrieves all of the users current mail folders.
// EnumeratetMailFolders iterates through all of the users current
// mail folders, converting each to a graph.CacheFolder, and calling
// fn(cf) on each one. If fn(cf) errors, the error is aggregated
// into a multierror that gets returned to the caller.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
// TODO: we want this to be the full handler, not only the builder.
// but this halfway point minimizes changes for now.
func (c Client) GetAllMailFoldersBuilder(
func (c Client) EnumerateMailFolders(
ctx context.Context,
userID string,
) (
*users.ItemMailFoldersDeltaRequestBuilder,
*graph.Service,
error,
) {
fn func(graph.CacheFolder) error,
) error {
service, err := c.service()
if err != nil {
return nil, nil, err
return err
}
builder := service.Client().
var (
errs *multierror.Error
builder = service.Client().
UsersById(userID).
MailFolders().
Delta()
)
return builder, service, nil
for {
resp, err := builder.Get(ctx, nil)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, v := range resp.GetValue() {
temp := graph.NewCacheFolder(v, nil)
if err := fn(temp); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "iterating mail folders delta"))
continue
}
}
link := resp.GetOdataNextLink()
if link == nil {
break
}
builder = users.NewItemMailFoldersDeltaRequestBuilder(*link, service.Adapter())
}
return errs.ErrorOrNil()
}
func (c Client) GetMailFolderByID(

View File

@ -1,7 +1,6 @@
package exchange
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
@ -37,42 +36,3 @@ func checkRequiredValues(c graph.Container) error {
return nil
}
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the container interface.
// Calendars do not have a parentFolderID. Therefore,
// the call will always return nil
type CalendarDisplayable struct {
models.Calendarable
}
// GetDisplayName returns the *string of the models.Calendable
// variant: calendar.GetName()
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// GetParentFolderId returns the default calendar name address
// EventCalendars have a flat hierarchy and Calendars are rooted
// at the default
//
//nolint:revive
func (c CalendarDisplayable) GetParentFolderId() *string {
return nil
}
// CreateCalendarDisplayable helper function to create the
// calendarDisplayable during msgraph-sdk-go iterative process
// @param entry is the input supplied by pageIterator.Iterate()
// @param parentID of Calendar sets. Only populate when used with
// EventCalendarCache
func CreateCalendarDisplayable(entry any) *CalendarDisplayable {
calendar, ok := entry.(models.Calendarable)
if !ok {
return nil
}
return &CalendarDisplayable{
Calendarable: calendar,
}
}

View File

@ -3,7 +3,6 @@ package exchange
import (
"context"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
@ -54,60 +53,16 @@ func (cfc *contactFolderCache) Populate(
return err
}
var errs error
builder, options, servicer, err := cfc.ac.GetContactChildFoldersBuilder(
ctx,
cfc.userID,
baseID)
err := cfc.ac.EnumerateContactsFolders(ctx, cfc.userID, baseID, cfc.addFolder)
if err != nil {
return errors.Wrap(err, "contact cache resolver option")
}
for {
resp, err := builder.Get(ctx, options)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, fold := range resp.GetValue() {
if err := checkIDAndName(fold); err != nil {
errs = support.WrapAndAppend(
"adding folder to contact resolver",
err,
errs,
)
continue
}
temp := graph.NewCacheFolder(fold, nil)
err = cfc.addFolder(temp)
if err != nil {
errs = support.WrapAndAppend(
"cache build in cfc.Populate",
err,
errs)
}
}
if resp.GetOdataNextLink() == nil {
break
}
builder = msuser.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter())
return err
}
if err := cfc.populatePaths(ctx); err != nil {
errs = support.WrapAndAppend(
"contacts resolver",
err,
errs,
)
return errors.Wrap(err, "contacts resolver")
}
return errs
return nil
}
func (cfc *contactFolderCache) init(

View File

@ -597,9 +597,10 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
test.pathFunc1(t),
folderName,
directoryCaches)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
@ -609,10 +610,11 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
test.pathFunc2(t),
folderName,
directoryCaches)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})

View File

@ -3,12 +3,10 @@ package exchange
import (
"context"
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
@ -32,56 +30,12 @@ func (ecc *eventCalendarCache) Populate(
ecc.containerResolver = newContainerResolver()
}
builder, options, servicer, err := ecc.ac.GetCalendarsBuilder(ctx, ecc.userID, "name")
err := ecc.ac.EnumerateCalendars(ctx, ecc.userID, ecc.addFolder)
if err != nil {
return err
}
var (
errs error
directories = make([]graph.Container, 0)
)
for {
resp, err := builder.Get(ctx, options)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, cal := range resp.GetValue() {
temp := CreateCalendarDisplayable(cal)
if err := checkIDAndName(temp); err != nil {
errs = support.WrapAndAppend(
"adding folder to cache",
err,
errs,
)
continue
}
directories = append(directories, temp)
}
if resp.GetOdataNextLink() == nil {
break
}
builder = msuser.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter())
}
for _, container := range directories {
temp := graph.NewCacheFolder(container, path.Builder{}.Append(*container.GetDisplayName()))
if err := ecc.addFolder(temp); err != nil {
errs = support.WrapAndAppend(
"failure adding "+*container.GetDisplayName(),
err,
errs)
}
}
return errs
return nil
}
// AddToCache adds container to map in field 'cache'

View File

@ -3,20 +3,14 @@ package exchange
import (
"testing"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/selectors"
)
type ExchangeIteratorSuite struct {
@ -56,99 +50,3 @@ func (suite *ExchangeIteratorSuite) TestDescendable() {
assert.NotNil(t, aDescendable.GetId())
assert.NotNil(t, aDescendable.GetParentFolderId())
}
// TestCollectionFunctions verifies ability to gather
// containers functions are valid for current versioning of msgraph-go-sdk.
// Tests for mail have been moved to graph_connector_test.go.
// exchange.Mail uses a sequential delta function.
// TODO: Add exchange.Mail when delta iterator functionality implemented
func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
mailScope, contactScope, eventScope []selectors.ExchangeScope
userID = tester.M365UserID(t)
users = []string{userID}
sel = selectors.NewExchangeBackup(users)
)
eb, err := sel.ToExchangeBackup()
require.NoError(suite.T(), err)
contactScope = sel.ContactFolders(users, []string{DefaultContactFolder}, selectors.PrefixMatch())
eventScope = sel.EventCalendars(users, []string{DefaultCalendar}, selectors.PrefixMatch())
mailScope = sel.MailFolders(users, []string{DefaultMailFolder}, selectors.PrefixMatch())
eb.Include(contactScope, eventScope, mailScope)
tests := []struct {
name string
queryFunc func(*testing.T, account.M365Config) api.GraphQuery
scope selectors.ExchangeScope
iterativeFunction func(
container map[string]graph.Container,
aFilter string,
errUpdater func(string, error)) func(any) bool
transformer absser.ParsableFactory
}{
{
name: "Contacts Iterative Check",
queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery {
ac, err := api.NewClient(amc)
require.NoError(t, err)
return ac.GetAllContactFolderNamesForUser
},
transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectContactContainers,
},
{
name: "Events Iterative Check",
queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery {
ac, err := api.NewClient(amc)
require.NoError(t, err)
return ac.GetAllCalendarNamesForUser
},
transformer: models.CreateCalendarCollectionResponseFromDiscriminatorValue,
iterativeFunction: IterativeCollectCalendarContainers,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365)
require.NoError(t, err)
response, err := test.queryFunc(t, m365)(ctx, userID)
require.NoError(t, err)
// Iterator Creation
pageIterator, err := msgraphgocore.NewPageIterator(
response,
service.Adapter(),
test.transformer)
require.NoError(t, err)
// Create collection for iterate test
collections := make(map[string]graph.Container)
var errs error
errUpdater := func(id string, err error) {
errs = support.WrapAndAppend(id, err, errs)
}
// callbackFunc iterates through all models.Messageable and fills exchange.Collection.added[]
// with corresponding item IDs. New collections are created for each directory
callbackFunc := test.iterativeFunction(collections, "", errUpdater)
iterateError := pageIterator.Iterate(ctx, callbackFunc)
assert.NoError(t, iterateError)
assert.NoError(t, errs)
})
}
}

View File

@ -3,8 +3,6 @@ package exchange
import (
"context"
multierror "github.com/hashicorp/go-multierror"
msfolderdelta "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
@ -67,44 +65,16 @@ func (mc *mailFolderCache) Populate(
return err
}
query, servicer, err := mc.ac.GetAllMailFoldersBuilder(ctx, mc.userID)
err := mc.ac.EnumerateMailFolders(ctx, mc.userID, mc.addFolder)
if err != nil {
return err
}
var errs *multierror.Error
for {
resp, err := query.Get(ctx, nil)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
for _, f := range resp.GetValue() {
temp := graph.NewCacheFolder(f, nil)
// Use addFolder instead of AddToCache to be conservative about path
// population. The fetch order of the folders could cause failures while
// trying to resolve paths, so put it off until we've gotten all folders.
if err := mc.addFolder(temp); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "delta fetch"))
continue
}
}
link := resp.GetOdataNextLink()
if link == nil {
break
}
query = msfolderdelta.NewItemMailFoldersDeltaRequestBuilder(*link, servicer.Adapter())
}
if err := mc.populatePaths(ctx); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "mail resolver"))
return errors.Wrap(err, "mail resolver")
}
return errs.ErrorOrNil()
return nil
}
// init ensures that the structure's fields are initialized.

View File

@ -3,9 +3,7 @@ package exchange
import (
"context"
"fmt"
"strings"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
@ -220,54 +218,6 @@ func pathFromPrevString(ps string) (path.Path, error) {
return p, nil
}
func IterativeCollectContactContainers(
containers map[string]graph.Container,
nameContains string,
errUpdater func(string, error),
) func(any) bool {
return func(entry any) bool {
folder, ok := entry.(models.ContactFolderable)
if !ok {
errUpdater("iterateCollectContactContainers",
errors.New("casting item to models.ContactFolderable"))
return false
}
include := len(nameContains) == 0 ||
strings.Contains(*folder.GetDisplayName(), nameContains)
if include {
containers[*folder.GetDisplayName()] = folder
}
return true
}
}
func IterativeCollectCalendarContainers(
containers map[string]graph.Container,
nameContains string,
errUpdater func(string, error),
) func(any) bool {
return func(entry any) bool {
cal, ok := entry.(models.Calendarable)
if !ok {
errUpdater("iterativeCollectCalendarContainers",
errors.New("casting item to models.Calendarable"))
return false
}
include := len(nameContains) == 0 ||
strings.Contains(*cal.GetName(), nameContains)
if include {
temp := CreateCalendarDisplayable(cal)
containers[*temp.GetDisplayName()] = temp
}
return true
}
}
// FetchIDFunc collection of helper functions which return a list of all item
// IDs in the given container and a delta token for future requests if the
// container supports fetching delta records.

View File

@ -635,8 +635,8 @@ func establishEventsRestoreLocation(
return "", errors.Wrap(err, "populating event cache")
}
transform := CreateCalendarDisplayable(temp)
if err = ecc.AddToCache(ctx, transform); err != nil {
displayable := api.CalendarDisplayable{Calendarable: temp}
if err = ecc.AddToCache(ctx, displayable); err != nil {
return "", errors.Wrap(err, "adding new calendar to cache")
}
}