GC: Purge: Updates to use graph.ContainerResolver (#1190)

## Description
purge.go to use the ContainerResolvers to collect containers from the Graph Connector.

NOTE: CollectFolders in service_query.go unchanged. `CollectFolderrs()` is stubbed for another PR. This will reduce the amount of line changes to be used in the future. 
<!-- Insert PR description-->

## Type of change

- [x] 🐛 Bugfix

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
*closes  #1189<issue>

## Test Plan

- [x]  Unit test
This commit is contained in:
Danny 2022-10-19 10:08:03 -04:00 committed by GitHub
parent 5ff890b760
commit 5cd2583466
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 283 additions and 129 deletions

View File

@ -2,6 +2,7 @@ package main
import ( import (
"context" "context"
"fmt"
"os" "os"
"time" "time"
@ -18,7 +19,10 @@ import (
"github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
) )
var purgeCmd = &cobra.Command{ var purgeCmd = &cobra.Command{
@ -231,11 +235,17 @@ func purgeMailFolders(
uid string, uid string,
) error { ) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
mfs, err := exchange.GetAllMailFolders(ctx, gs, uid, prefix) params, err := exchangeQueryParamFactory(uid, path.EmailCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
allFolders, err := exchange.GetAllMailFolders(ctx, *params, gs)
if err != nil {
return nil, err
}
mfs := containerFilter(prefix, allFolders)
purgables := make([]purgable, len(mfs)) purgables := make([]purgable, len(mfs))
for i, v := range mfs { for i, v := range mfs {
@ -261,11 +271,17 @@ func purgeCalendarFolders(
uid string, uid string,
) error { ) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllCalendars(ctx, gs, uid, prefix) params, err := exchangeQueryParamFactory(uid, path.EventsCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
allCalendars, err := exchange.GetAllCalendars(ctx, *params, gs)
if err != nil {
return nil, err
}
cfs := containerFilter(prefix, allCalendars)
purgables := make([]purgable, len(cfs)) purgables := make([]purgable, len(cfs))
for i, v := range cfs { for i, v := range cfs {
@ -291,11 +307,17 @@ func purgeContactFolders(
uid string, uid string,
) error { ) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) { getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllContactFolders(ctx, gs, uid, prefix) params, err := exchangeQueryParamFactory(uid, path.ContactsCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
allContainers, err := exchange.GetAllContactFolders(ctx, *params, gs)
if err != nil {
return nil, err
}
cfs := containerFilter(prefix, allContainers)
purgables := make([]purgable, len(cfs)) purgables := make([]purgable, len(cfs))
for i, v := range cfs { for i, v := range cfs {
@ -475,3 +497,45 @@ func userOrUsers(u string, us map[string]string) map[string]string {
return map[string]string{u: u} return map[string]string{u: u}
} }
// containerFilter filters container list based on prefix
// @returns cachedContainers that meet the requirements for purging.
func containerFilter(nameContains string, containers []graph.CachedContainer) []graph.CachedContainer {
f := filters.In(nameContains)
result := make([]graph.CachedContainer, 0)
for _, folder := range containers {
if f.Compare(*folder.GetDisplayName()) {
result = append(result, folder)
}
}
return result
}
func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.QueryParams, error) {
var scope selectors.ExchangeScope
switch category {
case path.ContactsCategory:
scope = selectors.NewExchangeBackup().ContactFolders([]string{user}, selectors.Any())[0]
case path.EmailCategory:
scope = selectors.NewExchangeBackup().MailFolders([]string{user}, selectors.Any())[0]
case path.EventsCategory:
scope = selectors.NewExchangeBackup().EventCalendars([]string{user}, selectors.Any())[0]
default:
return nil, fmt.Errorf("category %s not supported", category)
}
params := &graph.QueryParams{
User: user,
Scope: scope,
FailFast: false,
Credentials: account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)),
},
}
return params, nil
}

View File

@ -3,7 +3,6 @@ package exchange
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"github.com/hashicorp/go-multierror" "github.com/hashicorp/go-multierror"
absser "github.com/microsoft/kiota-abstractions-go/serialization" absser "github.com/microsoft/kiota-abstractions-go/serialization"
@ -140,88 +139,51 @@ func DeleteContactFolder(ctx context.Context, gs graph.Service, user, folderID s
// Returns a slice of {ID, DisplayName} tuples. // Returns a slice of {ID, DisplayName} tuples.
func GetAllMailFolders( func GetAllMailFolders(
ctx context.Context, ctx context.Context,
qp graph.QueryParams,
gs graph.Service, gs graph.Service,
user, nameContains string, ) ([]graph.CachedContainer, error) {
) ([]models.MailFolderable, error) { containers := make([]graph.CachedContainer, 0)
var (
mfs = []models.MailFolderable{}
err error
)
resp, err := GetAllFolderNamesForUser(ctx, gs, user) resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EmailCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
iter, err := msgraphgocore.NewPageIterator( for _, c := range resolver.Items() {
resp, gs.Adapter(), models.CreateMailFolderCollectionResponseFromDiscriminatorValue) directories := c.Path().Elements()
if err != nil { if len(directories) == 0 {
return nil, err continue
} }
cb := func(item any) bool { if qp.Scope.Matches(selectors.ExchangeMailFolder, directories[len(directories)-1]) {
folder, ok := item.(models.MailFolderable) containers = append(containers, c)
if !ok { }
err = errors.New("casting item to models.MailFolderable")
return false
} }
include := len(nameContains) == 0 || return containers, nil
(len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains))
if include {
mfs = append(mfs, folder)
}
return true
}
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
}
return mfs, err
} }
// GetAllCalendars retrieves all event calendars for the specified user. // GetAllCalendars retrieves all event calendars for the specified user.
// If nameContains is populated, only returns calendars matching that property. // If nameContains is populated, only returns calendars matching that property.
// Returns a slice of {ID, DisplayName} tuples. // Returns a slice of {ID, DisplayName} tuples.
func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains string) ([]graph.Container, error) { func GetAllCalendars(
var ( ctx context.Context,
cs = make(map[string]graph.Container) qp graph.QueryParams,
containers = make([]graph.Container, 0) gs graph.Service,
err, errs error ) ([]graph.CachedContainer, error) {
errUpdater = func(s string, e error) { containers := make([]graph.CachedContainer, 0)
errs = support.WrapAndAppend(s, e, errs)
}
)
resp, err := GetAllCalendarNamesForUser(ctx, gs, user) resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EventsCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
iter, err := msgraphgocore.NewPageIterator( for _, c := range resolver.Items() {
resp, gs.Adapter(), models.CreateCalendarCollectionResponseFromDiscriminatorValue) directories := c.Path().Elements()
if err != nil {
return nil, err if qp.Scope.Matches(selectors.ExchangeEventCalendar, directories[len(directories)-1]) {
containers = append(containers, c)
} }
cb := IterativeCollectCalendarContainers(
cs,
nameContains,
errUpdater,
)
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
}
if errs != nil {
return nil, errs
}
for _, calendar := range cs {
containers = append(containers, calendar)
} }
return containers, err return containers, err
@ -233,39 +195,31 @@ func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains s
// https://github.com/alcionai/corso/issues/1122 // https://github.com/alcionai/corso/issues/1122
func GetAllContactFolders( func GetAllContactFolders(
ctx context.Context, ctx context.Context,
qp graph.QueryParams,
gs graph.Service, gs graph.Service,
user, nameContains string, ) ([]graph.CachedContainer, error) {
) ([]graph.Container, error) {
var ( var (
cs = make(map[string]graph.Container) query string
containers = make([]graph.Container, 0) containers = make([]graph.CachedContainer, 0)
err, errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
}
) )
resp, err := GetAllContactFolderNamesForUser(ctx, gs, user) resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.ContactsCategory)
if err != nil { if err != nil {
return nil, err return nil, err
} }
iter, err := msgraphgocore.NewPageIterator( for _, c := range resolver.Items() {
resp, gs.Adapter(), models.CreateContactFolderCollectionResponseFromDiscriminatorValue) directories := c.Path().Elements()
if err != nil {
return nil, err if len(directories) == 0 {
query = DefaultContactFolder
} else {
query = directories[len(directories)-1]
} }
cb := IterativeCollectContactContainers( if qp.Scope.Matches(selectors.ExchangeContactFolder, query) {
cs, nameContains, errUpdater, containers = append(containers, c)
)
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
} }
for _, entry := range cs {
containers = append(containers, entry)
} }
return containers, err return containers, err

View File

@ -9,14 +9,21 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
) )
type ServiceFunctionsIntegrationSuite struct { type ServiceFunctionsIntegrationSuite struct {
suite.Suite suite.Suite
m365UserID string m365UserID string
creds account.M365Config
} }
const (
invalidUser = "fnords_mc_snarfens"
nonExistantLookup = "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√"
)
func TestServiceFunctionsIntegrationSuite(t *testing.T) { func TestServiceFunctionsIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny( if err := tester.RunOnAny(
tester.CorsoCITests, tester.CorsoCITests,
@ -30,7 +37,13 @@ func TestServiceFunctionsIntegrationSuite(t *testing.T) {
} }
func (suite *ServiceFunctionsIntegrationSuite) SetupSuite() { func (suite *ServiceFunctionsIntegrationSuite) SetupSuite() {
suite.m365UserID = tester.M365UserID(suite.T()) t := suite.T()
suite.m365UserID = tester.M365UserID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.creds = m365
} }
func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
@ -38,42 +51,73 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
defer flush() defer flush()
gs := loadService(suite.T()) gs := loadService(suite.T())
userID := tester.M365UserID(suite.T())
table := []struct { table := []struct {
name, contains, user string name, user string
expectCount assert.ComparisonAssertionFunc expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc expectErr assert.ErrorAssertionFunc
}{ }{
{ {
name: "plain lookup", name: "plain lookup",
user: suite.m365UserID, user: userID,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, selectors.Any())[0]
},
}, },
{ {
name: "root calendar", name: "Get Default Calendar",
contains: DefaultCalendar, user: userID,
user: suite.m365UserID,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{DefaultCalendar})[0]
},
},
{
name: "non-root calendar",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{"Birthdays"})[0]
},
}, },
{ {
name: "nonsense user", name: "nonsense user",
user: "fnords_mc_snarfens", user: invalidUser,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.Error, expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{invalidUser}, []string{DefaultContactFolder})[0]
},
}, },
{ {
name: "nonsense matcher", name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√", user: userID,
user: suite.m365UserID,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{userID}, []string{nonExistantLookup})[0]
},
}, },
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllCalendars(ctx, gs, test.user, test.contains) params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllCalendars(ctx, params, gs)
test.expectErr(t, err) test.expectErr(t, err)
test.expectCount(t, len(cals), 0) test.expectCount(t, len(cals), 0)
}) })
@ -88,8 +132,9 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
user := tester.M365UserID(suite.T()) user := tester.M365UserID(suite.T())
table := []struct { table := []struct {
name, contains, user string name, user string
expectCount assert.ComparisonAssertionFunc expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc expectErr assert.ErrorAssertionFunc
}{ }{
{ {
@ -97,31 +142,66 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
user: user, user: user,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, selectors.Any())[0]
},
}, },
{ {
name: "root folder", name: "default contact folder",
contains: "Contact", // DefaultContactFolder doesn't work here?
user: user, user: user,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{DefaultContactFolder})[0]
},
},
{
name: "Trial folder lookup",
user: user,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{"TrialFolder"})[0]
},
}, },
{ {
name: "nonsense user", name: "nonsense user",
user: "fnords_mc_snarfens", user: invalidUser,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.Error, expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{invalidUser}, []string{DefaultContactFolder})[0]
},
}, },
{ {
name: "nonsense matcher", name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: user, user: user,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{nonExistantLookup})[0]
},
}, },
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllContactFolders(ctx, gs, test.user, test.contains) params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllContactFolders(ctx, params, gs)
test.expectErr(t, err) test.expectErr(t, err)
test.expectCount(t, len(cals), 0) test.expectCount(t, len(cals), 0)
}) })
@ -133,42 +213,79 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
defer flush() defer flush()
gs := loadService(suite.T()) gs := loadService(suite.T())
userID := tester.M365UserID(suite.T())
table := []struct { table := []struct {
name, contains, user string name, user string
expectCount assert.ComparisonAssertionFunc expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc expectErr assert.ErrorAssertionFunc
}{ }{
{ {
name: "plain lookup", name: "plain lookup",
user: suite.m365UserID, user: userID,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, selectors.Any())[0]
},
}, },
{ {
name: "Root folder", name: "root folder",
contains: DefaultMailFolder, user: userID,
user: suite.m365UserID,
expectCount: assert.Greater, expectCount: assert.Greater,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{DefaultMailFolder})[0]
},
},
{
name: "Trial folder lookup",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{"Drafts"})[0]
},
}, },
{ {
name: "nonsense user", name: "nonsense user",
user: "fnords_mc_snarfens", user: invalidUser,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.Error, expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{invalidUser}, []string{DefaultMailFolder})[0]
},
}, },
{ {
name: "nonsense matcher", name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√", user: userID,
user: suite.m365UserID,
expectCount: assert.Equal, expectCount: assert.Equal,
expectErr: assert.NoError, expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{nonExistantLookup})[0]
},
}, },
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllMailFolders(ctx, gs, test.user, test.contains) params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllMailFolders(ctx, params, gs)
test.expectErr(t, err) test.expectErr(t, err)
test.expectCount(t, len(cals), 0) test.expectCount(t, len(cals), 0)
}) })

View File

@ -14,6 +14,8 @@ import (
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
) )
const ( const (
@ -65,3 +67,20 @@ func (handler *LoggingMiddleware) Intercept(
return pipeline.Next(req, middlewareIndex) return pipeline.Next(req, middlewareIndex)
} }
// ScopeToPathCategory helper function that maps selectors.ExchangeScope to path.CategoryType
func ScopeToPathCategory(scope selectors.ExchangeScope) path.CategoryType {
if scope.IncludesCategory(selectors.ExchangeMail) {
return path.EmailCategory
}
if scope.IncludesCategory(selectors.ExchangeContact) {
return path.ContactsCategory
}
if scope.IncludesCategory(selectors.ExchangeEvent) {
return path.EventsCategory
}
return path.UnknownCategory
}