GC: Purge: Updates to use graph.ContainerResolver (#1190)

## Description
purge.go to use the ContainerResolvers to collect containers from the Graph Connector.

NOTE: CollectFolders in service_query.go unchanged. `CollectFolderrs()` is stubbed for another PR. This will reduce the amount of line changes to be used in the future. 
<!-- Insert PR description-->

## Type of change

- [x] 🐛 Bugfix

## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
*closes  #1189<issue>

## Test Plan

- [x]  Unit test
This commit is contained in:
Danny 2022-10-19 10:08:03 -04:00 committed by GitHub
parent 5ff890b760
commit 5cd2583466
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 283 additions and 129 deletions

View File

@ -2,6 +2,7 @@ package main
import (
"context"
"fmt"
"os"
"time"
@ -18,7 +19,10 @@ import (
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
var purgeCmd = &cobra.Command{
@ -231,11 +235,17 @@ func purgeMailFolders(
uid string,
) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
mfs, err := exchange.GetAllMailFolders(ctx, gs, uid, prefix)
params, err := exchangeQueryParamFactory(uid, path.EmailCategory)
if err != nil {
return nil, err
}
allFolders, err := exchange.GetAllMailFolders(ctx, *params, gs)
if err != nil {
return nil, err
}
mfs := containerFilter(prefix, allFolders)
purgables := make([]purgable, len(mfs))
for i, v := range mfs {
@ -261,11 +271,17 @@ func purgeCalendarFolders(
uid string,
) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllCalendars(ctx, gs, uid, prefix)
params, err := exchangeQueryParamFactory(uid, path.EventsCategory)
if err != nil {
return nil, err
}
allCalendars, err := exchange.GetAllCalendars(ctx, *params, gs)
if err != nil {
return nil, err
}
cfs := containerFilter(prefix, allCalendars)
purgables := make([]purgable, len(cfs))
for i, v := range cfs {
@ -291,11 +307,17 @@ func purgeContactFolders(
uid string,
) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllContactFolders(ctx, gs, uid, prefix)
params, err := exchangeQueryParamFactory(uid, path.ContactsCategory)
if err != nil {
return nil, err
}
allContainers, err := exchange.GetAllContactFolders(ctx, *params, gs)
if err != nil {
return nil, err
}
cfs := containerFilter(prefix, allContainers)
purgables := make([]purgable, len(cfs))
for i, v := range cfs {
@ -475,3 +497,45 @@ func userOrUsers(u string, us map[string]string) map[string]string {
return map[string]string{u: u}
}
// containerFilter filters container list based on prefix
// @returns cachedContainers that meet the requirements for purging.
func containerFilter(nameContains string, containers []graph.CachedContainer) []graph.CachedContainer {
f := filters.In(nameContains)
result := make([]graph.CachedContainer, 0)
for _, folder := range containers {
if f.Compare(*folder.GetDisplayName()) {
result = append(result, folder)
}
}
return result
}
func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.QueryParams, error) {
var scope selectors.ExchangeScope
switch category {
case path.ContactsCategory:
scope = selectors.NewExchangeBackup().ContactFolders([]string{user}, selectors.Any())[0]
case path.EmailCategory:
scope = selectors.NewExchangeBackup().MailFolders([]string{user}, selectors.Any())[0]
case path.EventsCategory:
scope = selectors.NewExchangeBackup().EventCalendars([]string{user}, selectors.Any())[0]
default:
return nil, fmt.Errorf("category %s not supported", category)
}
params := &graph.QueryParams{
User: user,
Scope: scope,
FailFast: false,
Credentials: account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: common.First(tenant, os.Getenv(account.AzureTenantID)),
},
}
return params, nil
}

View File

@ -3,7 +3,6 @@ package exchange
import (
"context"
"fmt"
"strings"
"github.com/hashicorp/go-multierror"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
@ -140,88 +139,51 @@ func DeleteContactFolder(ctx context.Context, gs graph.Service, user, folderID s
// Returns a slice of {ID, DisplayName} tuples.
func GetAllMailFolders(
ctx context.Context,
qp graph.QueryParams,
gs graph.Service,
user, nameContains string,
) ([]models.MailFolderable, error) {
var (
mfs = []models.MailFolderable{}
err error
)
) ([]graph.CachedContainer, error) {
containers := make([]graph.CachedContainer, 0)
resp, err := GetAllFolderNamesForUser(ctx, gs, user)
resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EmailCategory)
if err != nil {
return nil, err
}
iter, err := msgraphgocore.NewPageIterator(
resp, gs.Adapter(), models.CreateMailFolderCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, err
}
cb := func(item any) bool {
folder, ok := item.(models.MailFolderable)
if !ok {
err = errors.New("casting item to models.MailFolderable")
return false
for _, c := range resolver.Items() {
directories := c.Path().Elements()
if len(directories) == 0 {
continue
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains))
if include {
mfs = append(mfs, folder)
if qp.Scope.Matches(selectors.ExchangeMailFolder, directories[len(directories)-1]) {
containers = append(containers, c)
}
return true
}
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
}
return mfs, err
return containers, nil
}
// GetAllCalendars retrieves all event calendars for the specified user.
// If nameContains is populated, only returns calendars matching that property.
// Returns a slice of {ID, DisplayName} tuples.
func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains string) ([]graph.Container, error) {
var (
cs = make(map[string]graph.Container)
containers = make([]graph.Container, 0)
err, errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
func GetAllCalendars(
ctx context.Context,
qp graph.QueryParams,
gs graph.Service,
) ([]graph.CachedContainer, error) {
containers := make([]graph.CachedContainer, 0)
resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.EventsCategory)
if err != nil {
return nil, err
}
for _, c := range resolver.Items() {
directories := c.Path().Elements()
if qp.Scope.Matches(selectors.ExchangeEventCalendar, directories[len(directories)-1]) {
containers = append(containers, c)
}
)
resp, err := GetAllCalendarNamesForUser(ctx, gs, user)
if err != nil {
return nil, err
}
iter, err := msgraphgocore.NewPageIterator(
resp, gs.Adapter(), models.CreateCalendarCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, err
}
cb := IterativeCollectCalendarContainers(
cs,
nameContains,
errUpdater,
)
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
}
if errs != nil {
return nil, errs
}
for _, calendar := range cs {
containers = append(containers, calendar)
}
return containers, err
@ -233,39 +195,31 @@ func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains s
// https://github.com/alcionai/corso/issues/1122
func GetAllContactFolders(
ctx context.Context,
qp graph.QueryParams,
gs graph.Service,
user, nameContains string,
) ([]graph.Container, error) {
) ([]graph.CachedContainer, error) {
var (
cs = make(map[string]graph.Container)
containers = make([]graph.Container, 0)
err, errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
query string
containers = make([]graph.CachedContainer, 0)
)
resolver, err := MaybeGetAndPopulateFolderResolver(ctx, qp, path.ContactsCategory)
if err != nil {
return nil, err
}
for _, c := range resolver.Items() {
directories := c.Path().Elements()
if len(directories) == 0 {
query = DefaultContactFolder
} else {
query = directories[len(directories)-1]
}
)
resp, err := GetAllContactFolderNamesForUser(ctx, gs, user)
if err != nil {
return nil, err
}
iter, err := msgraphgocore.NewPageIterator(
resp, gs.Adapter(), models.CreateContactFolderCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, err
}
cb := IterativeCollectContactContainers(
cs, nameContains, errUpdater,
)
if err := iter.Iterate(ctx, cb); err != nil {
return nil, err
}
for _, entry := range cs {
containers = append(containers, entry)
if qp.Scope.Matches(selectors.ExchangeContactFolder, query) {
containers = append(containers, c)
}
}
return containers, err

View File

@ -9,14 +9,21 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/selectors"
)
type ServiceFunctionsIntegrationSuite struct {
suite.Suite
m365UserID string
creds account.M365Config
}
const (
invalidUser = "fnords_mc_snarfens"
nonExistantLookup = "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√"
)
func TestServiceFunctionsIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
@ -30,7 +37,13 @@ func TestServiceFunctionsIntegrationSuite(t *testing.T) {
}
func (suite *ServiceFunctionsIntegrationSuite) SetupSuite() {
suite.m365UserID = tester.M365UserID(suite.T())
t := suite.T()
suite.m365UserID = tester.M365UserID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.creds = m365
}
func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
@ -38,42 +51,73 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
defer flush()
gs := loadService(suite.T())
userID := tester.M365UserID(suite.T())
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
name, user string
expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: suite.m365UserID,
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, selectors.Any())[0]
},
},
{
name: "root calendar",
contains: DefaultCalendar,
user: suite.m365UserID,
name: "Get Default Calendar",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{DefaultCalendar})[0]
},
},
{
name: "non-root calendar",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{"Birthdays"})[0]
},
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
user: invalidUser,
expectCount: assert.Equal,
expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{invalidUser}, []string{DefaultContactFolder})[0]
},
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: suite.m365UserID,
user: userID,
expectCount: assert.Equal,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{userID}, []string{nonExistantLookup})[0]
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllCalendars(ctx, gs, test.user, test.contains)
params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllCalendars(ctx, params, gs)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})
@ -88,40 +132,76 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
user := tester.M365UserID(suite.T())
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
name, user string
expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: user,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, selectors.Any())[0]
},
},
{
name: "root folder",
contains: "Contact", // DefaultContactFolder doesn't work here?
name: "default contact folder",
user: user,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{DefaultContactFolder})[0]
},
},
{
name: "Trial folder lookup",
user: user,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{"TrialFolder"})[0]
},
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
user: invalidUser,
expectCount: assert.Equal,
expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{invalidUser}, []string{DefaultContactFolder})[0]
},
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: user,
expectCount: assert.Equal,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{nonExistantLookup})[0]
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllContactFolders(ctx, gs, test.user, test.contains)
params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllContactFolders(ctx, params, gs)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})
@ -133,42 +213,79 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
defer flush()
gs := loadService(suite.T())
userID := tester.M365UserID(suite.T())
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
name, user string
expectCount assert.ComparisonAssertionFunc
getScope func(t *testing.T) selectors.ExchangeScope
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: suite.m365UserID,
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, selectors.Any())[0]
},
},
{
name: "Root folder",
contains: DefaultMailFolder,
user: suite.m365UserID,
name: "root folder",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{DefaultMailFolder})[0]
},
},
{
name: "Trial folder lookup",
user: userID,
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{"Drafts"})[0]
},
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
user: invalidUser,
expectCount: assert.Equal,
expectErr: assert.Error,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{invalidUser}, []string{DefaultMailFolder})[0]
},
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: suite.m365UserID,
user: userID,
expectCount: assert.Equal,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{nonExistantLookup})[0]
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := GetAllMailFolders(ctx, gs, test.user, test.contains)
params := graph.QueryParams{
User: test.user,
Scope: test.getScope(t),
FailFast: false,
Credentials: suite.creds,
}
cals, err := GetAllMailFolders(ctx, params, gs)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})

View File

@ -14,6 +14,8 @@ import (
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
const (
@ -65,3 +67,20 @@ func (handler *LoggingMiddleware) Intercept(
return pipeline.Next(req, middlewareIndex)
}
// ScopeToPathCategory helper function that maps selectors.ExchangeScope to path.CategoryType
func ScopeToPathCategory(scope selectors.ExchangeScope) path.CategoryType {
if scope.IncludesCategory(selectors.ExchangeMail) {
return path.EmailCategory
}
if scope.IncludesCategory(selectors.ExchangeContact) {
return path.ContactsCategory
}
if scope.IncludesCategory(selectors.ExchangeEvent) {
return path.EventsCategory
}
return path.UnknownCategory
}