From d707c22205c3675aaa7728437a34ba32f415703f Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 2 Sep 2022 16:18:49 -0700 Subject: [PATCH] Resolve mail exchange folders to their real path in the hierarchy (#752) ## Description Creates and uses a `ContainerResolver` interface to fetch container paths for items of different categories (when other resolvers are implemented). If the resolver is not available or fails to resolve a folder, defaults to the old implementation of using the folder's ID as its path. ## Type of change Please check the type of change your PR introduces: - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :hamster: Trivial/Minor ## Issue(s) * #456 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/exchange/iterators_test.go | 29 +++- .../connector/exchange/service_iterators.go | 135 +++++++++++++++++- src/internal/connector/graph/service.go | 15 ++ 3 files changed, 176 insertions(+), 3 deletions(-) diff --git a/src/internal/connector/exchange/iterators_test.go b/src/internal/connector/exchange/iterators_test.go index 50c2c3d47..366c2a201 100644 --- a/src/internal/connector/exchange/iterators_test.go +++ b/src/internal/connector/exchange/iterators_test.go @@ -158,7 +158,34 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() { nil) iterateError := pageIterator.Iterate(callbackFunc) - require.NoError(t, iterateError) + assert.NoError(t, iterateError) + assert.NoError(t, errs) + + // TODO(ashmrtn): Only check Exchange Mail folder names right now because + // other resolvers aren't implemented. Once they are we can expand these + // checks, potentially by breaking things out into separate tests per + // category. + if !test.scope.IncludesCategory(selectors.ExchangeMail) { + return + } + + expectedFolderNames := map[string]struct{}{ + "Inbox": {}, + "Sent Items": {}, + "Deleted Items": {}, + } + + for _, c := range collections { + // TODO(ashmrtn): Update these checks when collections support path.Path. + require.Greater(t, len(c.FullPath()), 4) + + folder := c.FullPath()[4] + if _, ok := expectedFolderNames[folder]; ok { + delete(expectedFolderNames, folder) + } + } + + assert.Empty(t, expectedFolderNames) }) } } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 13aa95d3e..ead4e775c 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -3,6 +3,7 @@ package exchange import ( "context" "fmt" + "strings" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" @@ -13,6 +14,8 @@ import ( "github.com/alcionai/corso/pkg/selectors" ) +var errNilResolver = errors.New("nil resolver") + // descendable represents objects that implement msgraph-sdk-go/models.entityable // and have the concept of a "parent folder". type descendable interface { @@ -37,6 +40,71 @@ type GraphIterateFunc func( statusUpdater support.StatusUpdater, ) func(any) bool +// maybeGetAndPopulateFolderResolver gets a folder resolver if one is available for +// this category of data. If one is not available, returns nil so that other +// logic in the caller can complete as long as they check if the resolver is not +// nil. If an error occurs populating the resolver, returns an error. +func maybeGetAndPopulateFolderResolver( + ctx context.Context, + qp graph.QueryParams, + category path.CategoryType, +) (graph.ContainerResolver, error) { + var res graph.ContainerResolver + + switch category { + case path.EmailCategory: + service, err := createService(qp.Credentials, qp.FailFast) + if err != nil { + return nil, err + } + + res = &mailFolderCache{ + userID: qp.User, + gs: service, + } + + default: + return nil, nil + } + + if err := res.Populate(ctx); err != nil { + return nil, errors.Wrap(err, "populating directory resolver") + } + + return res, nil +} + +func resolveCollectionPath( + ctx context.Context, + resolver graph.ContainerResolver, + tenantID, user, folderID string, + category path.CategoryType, +) ([]string, error) { + if resolver == nil { + // Allows caller to default to old-style path. + return nil, errors.WithStack(errNilResolver) + } + + p, err := resolver.IDToPath(ctx, folderID) + if err != nil { + return nil, errors.Wrap(err, "resolving folder ID") + } + + fullPath, err := p.ToDataLayerExchangePathForCategory( + tenantID, + user, + category, + false, + ) + if err != nil { + return nil, errors.Wrap(err, "converting to canonical path") + } + + // TODO(ashmrtn): This can return the path directly when Collections take + // path.Path. + return strings.Split(fullPath.String(), "/"), nil +} + // IterateSelectAllDescendablesForCollection utility function for // Iterating through MessagesCollectionResponse or ContactsCollectionResponse, // objects belonging to any folder are @@ -52,6 +120,7 @@ func IterateSelectAllDescendablesForCollections( isCategorySet bool collectionType optionIdentifier category path.CategoryType + resolver graph.ContainerResolver ) return func(pageItem any) bool { @@ -67,6 +136,16 @@ func IterateSelectAllDescendablesForCollections( category = path.ContactsCategory } + if r, err := maybeGetAndPopulateFolderResolver(ctx, qp, category); err != nil { + errs = support.WrapAndAppend( + "getting folder resolver for category "+category.String(), + err, + errs, + ) + } else { + resolver = r + } + isCategorySet = true } @@ -75,9 +154,29 @@ func IterateSelectAllDescendablesForCollections( errs = support.WrapAndAppendf(qp.User, errors.New("descendable conversion failure"), errs) return true } + // Saving to messages to list. Indexed by folder directory := *entry.GetParentFolderId() + dirPath := []string{qp.Credentials.TenantID, qp.User, category.String(), directory} + if _, ok = collections[directory]; !ok { + newPath, err := resolveCollectionPath( + ctx, + resolver, + qp.Credentials.TenantID, + qp.User, + directory, + category, + ) + + if err != nil { + if !errors.Is(err, errNilResolver) { + errs = support.WrapAndAppend("", err, errs) + } + } else { + dirPath = newPath + } + service, err := createService(qp.Credentials, qp.FailFast) if err != nil { errs = support.WrapAndAppend(qp.User, err, errs) @@ -86,7 +185,7 @@ func IterateSelectAllDescendablesForCollections( edc := NewCollection( qp.User, - []string{qp.Credentials.TenantID, qp.User, category.String(), directory}, + dirPath, collectionType, service, statusUpdater, @@ -239,6 +338,15 @@ func IterateFilterFolderDirectoriesForCollections( err error ) + resolver, err := maybeGetAndPopulateFolderResolver(ctx, qp, path.EmailCategory) + if err != nil { + errs = support.WrapAndAppend( + "getting folder resolver for category email", + err, + errs, + ) + } + return func(folderItem any) bool { folder, ok := folderItem.(displayable) if !ok { @@ -260,6 +368,29 @@ func IterateFilterFolderDirectoriesForCollections( } directory := *folder.GetId() + dirPath := []string{ + qp.Credentials.TenantID, + qp.User, + path.EmailCategory.String(), + directory, + } + + p, err := resolveCollectionPath( + ctx, + resolver, + qp.Credentials.TenantID, + qp.User, + directory, + path.EmailCategory, + ) + + if err != nil { + if !errors.Is(err, errNilResolver) { + errs = support.WrapAndAppend("", err, errs) + } + } else { + dirPath = p + } service, err = createService(qp.Credentials, qp.FailFast) if err != nil { @@ -277,7 +408,7 @@ func IterateFilterFolderDirectoriesForCollections( temp := NewCollection( qp.User, - []string{qp.Credentials.TenantID, qp.User, path.EmailCategory.String(), directory}, + dirPath, messages, service, statusUpdater, diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 9b28cc6e6..d75388ba5 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -1,8 +1,11 @@ package graph import ( + "context" + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/alcionai/corso/internal/path" "github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/selectors" ) @@ -24,3 +27,15 @@ type Service interface { // ErrPolicy returns if the service is implementing a Fast-Fail policy or not ErrPolicy() bool } + +// ContainerResolver houses functions for getting information about containers +// from remote APIs (i.e. resolve folder paths with Graph API). Resolvers may +// cache information about containers. +type ContainerResolver interface { + // IDToPath takes an m365 container ID and converts it to a hierarchical path + // to that container. The path has a similar format to paths on the local + // file system. + IDToPath(ctx context.Context, m365ID string) (*path.Builder, error) + // Populate performs any setup logic the resolver may need. + Populate(context.Context) error +}