From 5cd258346696afc5d341c9a183302ae670965519 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 19 Oct 2022 10:08:03 -0400 Subject: [PATCH] GC: Purge: Updates to use graph.ContainerResolver (#1190) ## Description purge.go to use the ContainerResolvers to collect containers from the Graph Connector. NOTE: CollectFolders in service_query.go unchanged. `CollectFolderrs()` is stubbed for another PR. This will reduce the amount of line changes to be used in the future. ## Type of change - [x] :bug: Bugfix ## Issue(s) *closes #1189 ## Test Plan - [x] :zap: Unit test --- src/cmd/purge/purge.go | 70 ++++++- .../connector/exchange/service_functions.go | 144 +++++--------- .../exchange/service_functions_test.go | 179 +++++++++++++++--- .../connector/graph/service_helper.go | 19 ++ 4 files changed, 283 insertions(+), 129 deletions(-) diff --git a/src/cmd/purge/purge.go b/src/cmd/purge/purge.go index 101ed4a4d..cdcfdcce5 100644 --- a/src/cmd/purge/purge.go +++ b/src/cmd/purge/purge.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os" "time" @@ -18,7 +19,10 @@ import ( "github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" + "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" ) var purgeCmd = &cobra.Command{ @@ -231,11 +235,17 @@ func purgeMailFolders( uid string, ) error { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { - mfs, err := exchange.GetAllMailFolders(ctx, gs, uid, prefix) + params, err := exchangeQueryParamFactory(uid, path.EmailCategory) if err != nil { return nil, err } + allFolders, err := exchange.GetAllMailFolders(ctx, *params, gs) + if err != nil { + return nil, err + } + + mfs := containerFilter(prefix, allFolders) purgables := make([]purgable, len(mfs)) for i, v := range mfs { @@ -261,11 +271,17 @@ func purgeCalendarFolders( uid string, ) error { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { - cfs, err := exchange.GetAllCalendars(ctx, gs, uid, prefix) + params, err := exchangeQueryParamFactory(uid, path.EventsCategory) if err != nil { return nil, err } + allCalendars, err := exchange.GetAllCalendars(ctx, *params, gs) + if err != nil { + return nil, err + } + + cfs := containerFilter(prefix, allCalendars) purgables := make([]purgable, len(cfs)) for i, v := range cfs { @@ -291,11 +307,17 @@ func purgeContactFolders( uid string, ) error { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { - cfs, err := exchange.GetAllContactFolders(ctx, gs, uid, prefix) + params, err := exchangeQueryParamFactory(uid, path.ContactsCategory) if err != nil { return nil, err } + allContainers, err := exchange.GetAllContactFolders(ctx, *params, gs) + if err != nil { + return nil, err + } + + cfs := containerFilter(prefix, allContainers) purgables := make([]purgable, len(cfs)) for i, v := range cfs { @@ -475,3 +497,45 @@ func userOrUsers(u string, us map[string]string) map[string]string { return map[string]string{u: u} } + +// containerFilter filters container list based on prefix +// @returns cachedContainers that meet the requirements for purging. +func containerFilter(nameContains string, containers []graph.CachedContainer) []graph.CachedContainer { + f := filters.In(nameContains) + result := make([]graph.CachedContainer, 0) + + for _, folder := range containers { + if f.Compare(*folder.GetDisplayName()) { + result = append(result, folder) + } + } + + return result +} + +func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.QueryParams, error) { + var scope selectors.ExchangeScope + + switch category { + case path.ContactsCategory: + scope = selectors.NewExchangeBackup().ContactFolders([]string{user}, selectors.Any())[0] + case path.EmailCategory: + scope = selectors.NewExchangeBackup().MailFolders([]string{user}, selectors.Any())[0] + case path.EventsCategory: + scope = selectors.NewExchangeBackup().EventCalendars([]string{user}, selectors.Any())[0] + default: + return nil, fmt.Errorf("category %s not supported", category) + } + + params := &graph.QueryParams{ + User: user, + Scope: scope, + FailFast: false, + Credentials: account.M365Config{ + M365: credentials.GetM365(), + AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)), + }, + } + + return params, nil +} diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index c184224c5..218cb0063 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -3,7 +3,6 @@ package exchange import ( "context" "fmt" - "strings" "github.com/hashicorp/go-multierror" absser "github.com/microsoft/kiota-abstractions-go/serialization" @@ -140,88 +139,51 @@ func DeleteContactFolder(ctx context.Context, gs graph.Service, user, folderID s // Returns a slice of {ID, DisplayName} tuples. func GetAllMailFolders( ctx context.Context, + qp graph.QueryParams, gs graph.Service, - user, nameContains string, -) ([]models.MailFolderable, error) { - var ( - mfs = []models.MailFolderable{} - err error - ) +) ([]graph.CachedContainer, error) { + containers := make([]graph.CachedContainer, 0) - resp, err := GetAllFolderNamesForUser(ctx, gs, user) + resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EmailCategory) if err != nil { return nil, err } - iter, err := msgraphgocore.NewPageIterator( - resp, gs.Adapter(), models.CreateMailFolderCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, err - } - - cb := func(item any) bool { - folder, ok := item.(models.MailFolderable) - if !ok { - err = errors.New("casting item to models.MailFolderable") - return false + for _, c := range resolver.Items() { + directories := c.Path().Elements() + if len(directories) == 0 { + continue } - include := len(nameContains) == 0 || - (len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains)) - if include { - mfs = append(mfs, folder) + if qp.Scope.Matches(selectors.ExchangeMailFolder, directories[len(directories)-1]) { + containers = append(containers, c) } - - return true } - if err := iter.Iterate(ctx, cb); err != nil { - return nil, err - } - - return mfs, err + return containers, nil } // GetAllCalendars retrieves all event calendars for the specified user. // If nameContains is populated, only returns calendars matching that property. // Returns a slice of {ID, DisplayName} tuples. -func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains string) ([]graph.Container, error) { - var ( - cs = make(map[string]graph.Container) - containers = make([]graph.Container, 0) - err, errs error - errUpdater = func(s string, e error) { - errs = support.WrapAndAppend(s, e, errs) +func GetAllCalendars( + ctx context.Context, + qp graph.QueryParams, + gs graph.Service, +) ([]graph.CachedContainer, error) { + containers := make([]graph.CachedContainer, 0) + + resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EventsCategory) + if err != nil { + return nil, err + } + + for _, c := range resolver.Items() { + directories := c.Path().Elements() + + if qp.Scope.Matches(selectors.ExchangeEventCalendar, directories[len(directories)-1]) { + containers = append(containers, c) } - ) - - resp, err := GetAllCalendarNamesForUser(ctx, gs, user) - if err != nil { - return nil, err - } - - iter, err := msgraphgocore.NewPageIterator( - resp, gs.Adapter(), models.CreateCalendarCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, err - } - - cb := IterativeCollectCalendarContainers( - cs, - nameContains, - errUpdater, - ) - - if err := iter.Iterate(ctx, cb); err != nil { - return nil, err - } - - if errs != nil { - return nil, errs - } - - for _, calendar := range cs { - containers = append(containers, calendar) } return containers, err @@ -233,39 +195,31 @@ func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains s // https://github.com/alcionai/corso/issues/1122 func GetAllContactFolders( ctx context.Context, + qp graph.QueryParams, gs graph.Service, - user, nameContains string, -) ([]graph.Container, error) { +) ([]graph.CachedContainer, error) { var ( - cs = make(map[string]graph.Container) - containers = make([]graph.Container, 0) - err, errs error - errUpdater = func(s string, e error) { - errs = support.WrapAndAppend(s, e, errs) + query string + containers = make([]graph.CachedContainer, 0) + ) + + resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.ContactsCategory) + if err != nil { + return nil, err + } + + for _, c := range resolver.Items() { + directories := c.Path().Elements() + + if len(directories) == 0 { + query = DefaultContactFolder + } else { + query = directories[len(directories)-1] } - ) - resp, err := GetAllContactFolderNamesForUser(ctx, gs, user) - if err != nil { - return nil, err - } - - iter, err := msgraphgocore.NewPageIterator( - resp, gs.Adapter(), models.CreateContactFolderCollectionResponseFromDiscriminatorValue) - if err != nil { - return nil, err - } - - cb := IterativeCollectContactContainers( - cs, nameContains, errUpdater, - ) - - if err := iter.Iterate(ctx, cb); err != nil { - return nil, err - } - - for _, entry := range cs { - containers = append(containers, entry) + if qp.Scope.Matches(selectors.ExchangeContactFolder, query) { + containers = append(containers, c) + } } return containers, err diff --git a/src/internal/connector/exchange/service_functions_test.go b/src/internal/connector/exchange/service_functions_test.go index f7752d3c9..7e9976611 100644 --- a/src/internal/connector/exchange/service_functions_test.go +++ b/src/internal/connector/exchange/service_functions_test.go @@ -9,14 +9,21 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/selectors" ) type ServiceFunctionsIntegrationSuite struct { suite.Suite m365UserID string + creds account.M365Config } +const ( + invalidUser = "fnords_mc_snarfens" + nonExistantLookup = "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√" +) + func TestServiceFunctionsIntegrationSuite(t *testing.T) { if err := tester.RunOnAny( tester.CorsoCITests, @@ -30,7 +37,13 @@ func TestServiceFunctionsIntegrationSuite(t *testing.T) { } func (suite *ServiceFunctionsIntegrationSuite) SetupSuite() { - suite.m365UserID = tester.M365UserID(suite.T()) + t := suite.T() + suite.m365UserID = tester.M365UserID(t) + a := tester.NewM365Account(t) + m365, err := a.M365Config() + require.NoError(t, err) + + suite.creds = m365 } func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { @@ -38,42 +51,73 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { defer flush() gs := loadService(suite.T()) + userID := tester.M365UserID(suite.T()) table := []struct { - name, contains, user string - expectCount assert.ComparisonAssertionFunc - expectErr assert.ErrorAssertionFunc + name, user string + expectCount assert.ComparisonAssertionFunc + getScope func(t *testing.T) selectors.ExchangeScope + expectErr assert.ErrorAssertionFunc }{ { name: "plain lookup", - user: suite.m365UserID, + user: userID, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors.NewExchangeBackup().EventCalendars([]string{userID}, selectors.Any())[0] + }, }, { - name: "root calendar", - contains: DefaultCalendar, - user: suite.m365UserID, + name: "Get Default Calendar", + user: userID, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{DefaultCalendar})[0] + }, + }, + { + name: "non-root calendar", + user: userID, + expectCount: assert.Greater, + expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{"Birthdays"})[0] + }, }, { name: "nonsense user", - user: "fnords_mc_snarfens", + user: invalidUser, expectCount: assert.Equal, expectErr: assert.Error, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + EventCalendars([]string{invalidUser}, []string{DefaultContactFolder})[0] + }, }, { name: "nonsense matcher", - contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√", - user: suite.m365UserID, + user: userID, expectCount: assert.Equal, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + EventCalendars([]string{userID}, []string{nonExistantLookup})[0] + }, }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - cals, err := GetAllCalendars(ctx, gs, test.user, test.contains) + params := graph.QueryParams{ + User: test.user, + Scope: test.getScope(t), + FailFast: false, + Credentials: suite.creds, + } + cals, err := GetAllCalendars(ctx, params, gs) test.expectErr(t, err) test.expectCount(t, len(cals), 0) }) @@ -88,40 +132,76 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() { user := tester.M365UserID(suite.T()) table := []struct { - name, contains, user string - expectCount assert.ComparisonAssertionFunc - expectErr assert.ErrorAssertionFunc + name, user string + expectCount assert.ComparisonAssertionFunc + getScope func(t *testing.T) selectors.ExchangeScope + expectErr assert.ErrorAssertionFunc }{ { name: "plain lookup", user: user, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + ContactFolders([]string{user}, selectors.Any())[0] + }, }, { - name: "root folder", - contains: "Contact", // DefaultContactFolder doesn't work here? + name: "default contact folder", user: user, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + ContactFolders([]string{user}, []string{DefaultContactFolder})[0] + }, + }, + { + name: "Trial folder lookup", + user: user, + expectCount: assert.Greater, + expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + ContactFolders([]string{user}, []string{"TrialFolder"})[0] + }, }, { name: "nonsense user", - user: "fnords_mc_snarfens", + user: invalidUser, expectCount: assert.Equal, expectErr: assert.Error, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + ContactFolders([]string{invalidUser}, []string{DefaultContactFolder})[0] + }, }, { name: "nonsense matcher", - contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√", user: user, expectCount: assert.Equal, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + ContactFolders([]string{user}, []string{nonExistantLookup})[0] + }, }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - cals, err := GetAllContactFolders(ctx, gs, test.user, test.contains) + params := graph.QueryParams{ + User: test.user, + Scope: test.getScope(t), + FailFast: false, + Credentials: suite.creds, + } + cals, err := GetAllContactFolders(ctx, params, gs) test.expectErr(t, err) test.expectCount(t, len(cals), 0) }) @@ -133,42 +213,79 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() { defer flush() gs := loadService(suite.T()) + userID := tester.M365UserID(suite.T()) table := []struct { - name, contains, user string - expectCount assert.ComparisonAssertionFunc - expectErr assert.ErrorAssertionFunc + name, user string + expectCount assert.ComparisonAssertionFunc + getScope func(t *testing.T) selectors.ExchangeScope + expectErr assert.ErrorAssertionFunc }{ { name: "plain lookup", - user: suite.m365UserID, + user: userID, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + MailFolders([]string{userID}, selectors.Any())[0] + }, }, { - name: "Root folder", - contains: DefaultMailFolder, - user: suite.m365UserID, + name: "root folder", + user: userID, expectCount: assert.Greater, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + MailFolders([]string{userID}, []string{DefaultMailFolder})[0] + }, + }, + { + name: "Trial folder lookup", + user: userID, + expectCount: assert.Greater, + expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + MailFolders([]string{userID}, []string{"Drafts"})[0] + }, }, { name: "nonsense user", - user: "fnords_mc_snarfens", + user: invalidUser, expectCount: assert.Equal, expectErr: assert.Error, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + MailFolders([]string{invalidUser}, []string{DefaultMailFolder})[0] + }, }, { name: "nonsense matcher", - contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√", - user: suite.m365UserID, + user: userID, expectCount: assert.Equal, expectErr: assert.NoError, + getScope: func(t *testing.T) selectors.ExchangeScope { + return selectors. + NewExchangeBackup(). + MailFolders([]string{userID}, []string{nonExistantLookup})[0] + }, }, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - cals, err := GetAllMailFolders(ctx, gs, test.user, test.contains) + params := graph.QueryParams{ + User: test.user, + Scope: test.getScope(t), + FailFast: false, + Credentials: suite.creds, + } + cals, err := GetAllMailFolders(ctx, params, gs) test.expectErr(t, err) test.expectCount(t, len(cals), 0) }) diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go index 658c8026e..ecd292f59 100644 --- a/src/internal/connector/graph/service_helper.go +++ b/src/internal/connector/graph/service_helper.go @@ -14,6 +14,8 @@ import ( msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" ) const ( @@ -65,3 +67,20 @@ func (handler *LoggingMiddleware) Intercept( return pipeline.Next(req, middlewareIndex) } + +// ScopeToPathCategory helper function that maps selectors.ExchangeScope to path.CategoryType +func ScopeToPathCategory(scope selectors.ExchangeScope) path.CategoryType { + if scope.IncludesCategory(selectors.ExchangeMail) { + return path.EmailCategory + } + + if scope.IncludesCategory(selectors.ExchangeContact) { + return path.ContactsCategory + } + + if scope.IncludesCategory(selectors.ExchangeEvent) { + return path.EventsCategory + } + + return path.UnknownCategory +}