From 93ad16dc304fe8ff67746fb630d9daf13eb66045 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 17 Nov 2022 15:08:37 -0700 Subject: [PATCH] extract scope from qp (#1510) ## Description Since scopes are service specific, we cannot easily house them within the graph QueryParam struct, unless we bloat the struct with all types. Alternatively, we could add a generic "scope" with parsers, much like the Selector itself. But really, the most simple solution is to only pass the scope within the tree of service funcs that use it. ## Type of change - [x] :hamster: Refactor ## Issue(s) * #1506 ## Test Plan - [x] :zap: Unit test --- src/internal/connector/data_collections.go | 36 +++++++------------ .../connector/exchange/service_functions.go | 30 +++++++++------- .../connector/exchange/service_iterators.go | 19 +++++----- src/internal/connector/graph/service.go | 9 +++-- .../connector/sharepoint/service_iterators.go | 2 ++ src/internal/tester/integration_runners.go | 1 + 6 files changed, 47 insertions(+), 50 deletions(-) diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index fa4087015..833c1f977 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -114,23 +114,17 @@ func (gc *GraphConnector) createExchangeCollections( collections := make(map[string]*exchange.Collection) qp := graph.QueryParams{ - User: user, - Scope: scope, - FailFast: gc.failFast, - Credentials: gc.credentials, + Category: scope.Category().PathType(), + ResourceOwner: user, + FailFast: gc.failFast, + Credentials: gc.credentials, } - itemCategory := qp.Scope.Category().PathType() - - foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", itemCategory.String(), user)) + foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, user)) defer closer() defer close(foldersComplete) - resolver, err := exchange.PopulateExchangeContainerResolver( - ctx, - qp, - qp.Scope.Category().PathType(), - ) + resolver, err := exchange.PopulateExchangeContainerResolver(ctx, qp) if err != nil { return nil, errors.Wrap(err, "getting folder cache") } @@ -140,7 +134,8 @@ func (gc *GraphConnector) createExchangeCollections( qp, collections, gc.UpdateStatus, - resolver) + resolver, + scope) if err != nil { return nil, errors.Wrap(err, "filling collections") @@ -264,18 +259,13 @@ func (gc *GraphConnector) createSharePointCollections( collections := make(map[string]*sharepoint.Collection) qp := graph.QueryParams{ - // TODO: Resource owner, not user/site. - User: site, - // TODO: generic scope handling in query params. - // - or, break scope out of QP. - // Scope: scope, - FailFast: gc.failFast, - Credentials: gc.credentials, + Category: scope.Category().PathType(), + ResourceOwner: site, + FailFast: gc.failFast, + Credentials: gc.credentials, } - itemCategory := qp.Scope.Category().PathType() - - foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", itemCategory.String(), site)) + foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, site)) defer closer() defer close(foldersComplete) diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 8aa7ae505..b8fcec817 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -23,9 +23,9 @@ type exchangeService struct { credentials account.M365Config } -///------------------------------------------------------------ +// /------------------------------------------------------------ // Functions to comply with graph.Service Interface -//------------------------------------------------------- +// ------------------------------------------------------- func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient { return &es.client } @@ -137,7 +137,6 @@ func DeleteContactFolder(ctx context.Context, gs graph.Service, user, folderID s func PopulateExchangeContainerResolver( ctx context.Context, qp graph.QueryParams, - category path.CategoryType, ) (graph.ContainerResolver, error) { var ( res graph.ContainerResolver @@ -149,30 +148,30 @@ func PopulateExchangeContainerResolver( return nil, err } - switch category { + switch qp.Category { case path.EmailCategory: res = &mailFolderCache{ - userID: qp.User, + userID: qp.ResourceOwner, gs: service, } cacheRoot = rootFolderAlias case path.ContactsCategory: res = &contactFolderCache{ - userID: qp.User, + userID: qp.ResourceOwner, gs: service, } cacheRoot = DefaultContactFolder case path.EventsCategory: res = &eventCalendarCache{ - userID: qp.User, + userID: qp.ResourceOwner, gs: service, } cacheRoot = DefaultCalendar default: - return nil, fmt.Errorf("ContainerResolver not present for %s type", category) + return nil, fmt.Errorf("ContainerResolver not present for %s type", qp.Category) } if err := res.Populate(ctx, cacheRoot); err != nil { @@ -182,8 +181,13 @@ func PopulateExchangeContainerResolver( return res, nil } -func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.CachedContainer) (path.Path, bool) { +func pathAndMatch( + qp graph.QueryParams, + c graph.CachedContainer, + scope selectors.ExchangeScope, +) (path.Path, bool) { var ( + category = scope.Category().PathType() directory string pb = c.Path() ) @@ -195,7 +199,7 @@ func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.Cach dirPath, err := pb.ToDataLayerExchangePathForCategory( qp.Credentials.AzureTenantID, - qp.User, + qp.ResourceOwner, category, false, ) @@ -211,11 +215,11 @@ func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.Cach switch category { case path.EmailCategory: - return dirPath, qp.Scope.Matches(selectors.ExchangeMailFolder, directory) + return dirPath, scope.Matches(selectors.ExchangeMailFolder, directory) case path.ContactsCategory: - return dirPath, qp.Scope.Matches(selectors.ExchangeContactFolder, directory) + return dirPath, scope.Matches(selectors.ExchangeContactFolder, directory) case path.EventsCategory: - return dirPath, qp.Scope.Matches(selectors.ExchangeEventCalendar, directory) + return dirPath, scope.Matches(selectors.ExchangeEventCalendar, directory) default: return nil, false } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 5ce694ba3..f3d8f0b41 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" ) // FilterContainersAndFillCollections is a utility function @@ -26,21 +27,21 @@ func FilterContainersAndFillCollections( collections map[string]*Collection, statusUpdater support.StatusUpdater, resolver graph.ContainerResolver, + scope selectors.ExchangeScope, ) error { var ( - category = qp.Scope.Category().PathType() - collectionType = CategoryToOptionIdentifier(category) + collectionType = CategoryToOptionIdentifier(scope.Category().PathType()) errs error ) for _, c := range resolver.Items() { - dirPath, ok := pathAndMatch(qp, category, c) + dirPath, ok := pathAndMatch(qp, c, scope) if ok { // Create only those that match service, err := createService(qp.Credentials, qp.FailFast) if err != nil { errs = support.WrapAndAppend( - qp.User+" FilterContainerAndFillCollection", + qp.ResourceOwner+" FilterContainerAndFillCollection", err, errs) @@ -50,7 +51,7 @@ func FilterContainersAndFillCollections( } edc := NewCollection( - qp.User, + qp.ResourceOwner, dirPath, collectionType, service, @@ -61,10 +62,10 @@ func FilterContainersAndFillCollections( } for directoryID, col := range collections { - fetchFunc, err := getFetchIDFunc(category) + fetchFunc, err := getFetchIDFunc(scope.Category().PathType()) if err != nil { errs = support.WrapAndAppend( - qp.User, + qp.ResourceOwner, err, errs) @@ -75,10 +76,10 @@ func FilterContainersAndFillCollections( continue } - jobs, err := fetchFunc(ctx, col.service, qp.User, directoryID) + jobs, err := fetchFunc(ctx, col.service, qp.ResourceOwner, directoryID) if err != nil { errs = support.WrapAndAppend( - qp.User, + qp.ResourceOwner, err, errs, ) diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 0ecdfb7af..7281dbe94 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -7,14 +7,13 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" ) type QueryParams struct { - User string - Scope selectors.ExchangeScope - Credentials account.M365Config - FailFast bool + Category path.CategoryType + ResourceOwner string + Credentials account.M365Config + FailFast bool } type Service interface { diff --git a/src/internal/connector/sharepoint/service_iterators.go b/src/internal/connector/sharepoint/service_iterators.go index a1f682c26..d7fa23484 100644 --- a/src/internal/connector/sharepoint/service_iterators.go +++ b/src/internal/connector/sharepoint/service_iterators.go @@ -5,6 +5,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/pkg/selectors" ) // FilterContainersAndFillCollections is a utility function @@ -17,6 +18,7 @@ func FilterContainersAndFillCollections( collections map[string]*Collection, statusUpdater support.StatusUpdater, resolver graph.ContainerResolver, + scope selectors.SharePointScope, ) error { return nil } diff --git a/src/internal/tester/integration_runners.go b/src/internal/tester/integration_runners.go index cfc2b2623..0215e0a9c 100644 --- a/src/internal/tester/integration_runners.go +++ b/src/internal/tester/integration_runners.go @@ -16,6 +16,7 @@ const ( CorsoCLIRepoTests = "CORSO_COMMAND_LINE_REPO_TESTS" CorsoCLIRestoreTests = "CORSO_COMMAND_LINE_RESTORE_TESTS" CorsoCLITests = "CORSO_COMMAND_LINE_TESTS" + CorsoConnectorCreateCollectionTests = "CORSO_CONNECTOR_CREATE_COLLECTION_TESTS" CorsoConnectorCreateExchangeCollectionTests = "CORSO_CONNECTOR_CREATE_EXCHANGE_COLLECTION_TESTS" CorsoConnectorCreateSharePointCollectionTests = "CORSO_CONNECTOR_CREATE_SHAREPOINT_COLLECTION_TESTS" CorsoConnectorDataCollectionTests = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS"