larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

purge both mail and calendars (#813)

Browse Source 
expands the `purge` command to accept args for
purging mail folders, calendars, or both.  This
allows the test cleanup to ensure we aren't over-
populating either mail folders or calendars, thus
blocking CI actions
This commit is contained in:
Keepers 2022-09-12 17:35:16 -06:00 committed by GitHub
parent 0a7954b300
commit fa489782a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 555 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

346
src/cmd/purge/purge.go
View File

@ -13,14 +13,33 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
)
var purgeCmd = &cobra.Command{
Use: "purge",
Short: "Purge m365 data",
RunE: doFolderPurge,
Short: "Purge all types of m365 folders",
RunE: handleAllFolderPurge,
}
var mailCmd = &cobra.Command{
Use: "mail",
Short: "Purges mail folders",
RunE: handleMailFolderPurge,
}
var eventsCmd = &cobra.Command{
Use: "events",
Short: "Purges calendar event folders",
RunE: handleCalendarFolderPurge,
}
var contactsCmd = &cobra.Command{
Use: "contacts",
Short: "Purges contacts folders",
RunE: handleContactsFolderPurge,
}
var (
@ -30,13 +49,260 @@ var (
prefix string
)
func doFolderPurge(cmd *cobra.Command, args []string) error {
// ------------------------------------------------------------------------------------------
// CLI command handlers
// ------------------------------------------------------------------------------------------
func main() {
ctx := SetRootCmd(context.Background(), purgeCmd)
fs := purgeCmd.PersistentFlags()
fs.StringVar(&before, "before", "", "folders older than this date are deleted. (default: now in UTC)")
fs.StringVar(&user, "user", "", "m365 user id whose folders will be deleted")
cobra.CheckErr(purgeCmd.MarkPersistentFlagRequired("user"))
fs.StringVar(&tenant, "tenant", "", "m365 tenant containing the user")
fs.StringVar(&prefix, "prefix", "", "filters mail folders by displayName prefix")
cobra.CheckErr(purgeCmd.MarkPersistentFlagRequired("prefix"))
purgeCmd.AddCommand(mailCmd)
purgeCmd.AddCommand(eventsCmd)
purgeCmd.AddCommand(contactsCmd)
if err := purgeCmd.ExecuteContext(ctx); err != nil {
Info(purgeCmd.Context(), "Error: ", err.Error())
os.Exit(1)
}
}
func handleAllFolderPurge(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
gc, err := getGC(ctx)
if err != nil {
return err
}
t, err := getBoundaryTime(ctx)
if err != nil {
return err
}
err = purgeMailFolders(ctx, gc, t)
if err != nil {
return errors.Wrap(err, "purging mail folders")
}
err = purgeCalendarFolders(ctx, gc, t)
if err != nil {
return errors.Wrap(err, "purging calendar folders")
}
err = purgeContactFolders(ctx, gc, t)
if err != nil {
return errors.Wrap(err, "purging contacts folders")
}
return nil
}
func handleMailFolderPurge(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
gc, err := getGC(ctx)
if err != nil {
return err
}
t, err := getBoundaryTime(ctx)
if err != nil {
return err
}
return purgeMailFolders(ctx, gc, t)
}
func handleCalendarFolderPurge(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
gc, err := getGC(ctx)
if err != nil {
return err
}
t, err := getBoundaryTime(ctx)
if err != nil {
return err
}
return purgeCalendarFolders(ctx, gc, t)
}
func handleContactsFolderPurge(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
gc, err := getGC(ctx)
if err != nil {
return err
}
t, err := getBoundaryTime(ctx)
if err != nil {
return err
}
return purgeContactFolders(ctx, gc, t)
}
// ------------------------------------------------------------------------------------------
// Purge Controllers
// ------------------------------------------------------------------------------------------
type purgable interface {
GetDisplayName() *string
GetId() *string
}
// ----- mail
func purgeMailFolders(ctx context.Context, gc *connector.GraphConnector, boundary time.Time) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
mfs, err := exchange.GetAllMailFolders(gs, uid, prefix)
if err != nil {
return nil, err
}
purgables := make([]purgable, len(mfs))
for i, v := range mfs {
purgables[i] = v
}
return purgables, nil
}
deleter := func(gs graph.Service, uid, fid string) error {
return exchange.DeleteMailFolder(gs, uid, fid)
}
return purgeFolders(ctx, gc, boundary, "mail", getter, deleter)
}
// ----- calendars
func purgeCalendarFolders(ctx context.Context, gc *connector.GraphConnector, boundary time.Time) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllCalendars(gs, uid, prefix)
if err != nil {
return nil, err
}
purgables := make([]purgable, len(cfs))
for i, v := range cfs {
purgables[i] = v
}
return purgables, nil
}
deleter := func(gs graph.Service, uid, fid string) error {
return exchange.DeleteCalendar(gs, uid, fid)
}
return purgeFolders(ctx, gc, boundary, "calendar", getter, deleter)
}
// ----- contacts
func purgeContactFolders(ctx context.Context, gc *connector.GraphConnector, boundary time.Time) error {
getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
cfs, err := exchange.GetAllContactFolders(gs, uid, prefix)
if err != nil {
return nil, err
}
purgables := make([]purgable, len(cfs))
for i, v := range cfs {
purgables[i] = v
}
return purgables, nil
}
deleter := func(gs graph.Service, uid, fid string) error {
return exchange.DeleteContactFolder(gs, uid, fid)
}
return purgeFolders(ctx, gc, boundary, "contact", getter, deleter)
}
// ----- controller
func purgeFolders(
ctx context.Context,
gc *connector.GraphConnector,
boundary time.Time,
data string,
getter func(graph.Service, string, string) ([]purgable, error),
deleter func(graph.Service, string, string) error,
) error {
// get them folders
fs, err := getter(gc.Service(), user, prefix)
if err != nil {
return Only(ctx, errors.Wrapf(err, "retrieving %s folders", data))
}
stLen := len(common.SimpleDateTimeFormat)
// delete any that don't meet the boundary
for _, fld := range fs {
// compare the folder time to the deletion boundary time first
var (
del bool
displayName = *fld.GetDisplayName()
dnLen = len(displayName)
)
if dnLen > stLen {
dnSuff := displayName[dnLen-stLen:]
dnTime, err := common.ParseTime(dnSuff)
if err != nil {
Info(ctx, errors.Wrapf(err, "Error: deleting %s folder [%s]", data, displayName))
continue
}
del = dnTime.Before(boundary)
}
if !del {
continue
}
Infof(ctx, "Deleting %s folder: %s", data, displayName)
err = deleter(gc.Service(), user, *fld.GetId())
if err != nil {
Info(ctx, errors.Wrapf(err, "Error: deleting %s folder [%s]", data, displayName))
}
}
return nil
}
// ------------------------------------------------------------------------------------------
// Helpers
// ------------------------------------------------------------------------------------------
func getGC(ctx context.Context) (*connector.GraphConnector, error) {
// get account info
m365Cfg := account.M365Config{
M365: credentials.GetM365(),
@ -45,79 +311,31 @@ func doFolderPurge(cmd *cobra.Command, args []string) error {
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil {
return Only(ctx, errors.Wrap(err, "finding m365 account details"))
return nil, Only(ctx, errors.Wrap(err, "finding m365 account details"))
}
// build a graph connector
gc, err := connector.NewGraphConnector(acct)
if err != nil {
return Only(ctx, errors.Wrap(err, "connecting to graph api"))
return nil, Only(ctx, errors.Wrap(err, "connecting to graph api"))
}
// get them folders
mfs, err := exchange.GetAllMailFolders(gc.Service(), user, prefix)
if err != nil {
return Only(ctx, errors.Wrap(err, "retrieving mail folders"))
}
return gc, nil
}
func getBoundaryTime(ctx context.Context) (time.Time, error) {
// format the time input
beforeTime := time.Now().UTC()
var (
err error
boundaryTime = time.Now().UTC()
)
if len(before) > 0 {
beforeTime, err = common.ParseTime(before)
boundaryTime, err = common.ParseTime(before)
if err != nil {
return Only(ctx, errors.Wrap(err, "parsing before flag to time"))
return time.Time{}, Only(ctx, errors.Wrap(err, "parsing before flag to time"))
}
}
stLen := len(common.SimpleDateTimeFormat)
// delete files
for _, mf := range mfs {
// compare the folder time to the deletion boundary time first
var (
del bool
dnLen = len(mf.DisplayName)
)
if dnLen > stLen {
dnSuff := mf.DisplayName[dnLen-stLen:]
dnTime, err := common.ParseTime(dnSuff)
if err != nil {
Info(ctx, errors.Wrapf(err, "Error: deleting folder [%s]", mf.DisplayName))
continue
}
del = dnTime.Before(beforeTime)
}
if !del {
continue
}
Info(ctx, "Deleting folder: ", mf.DisplayName)
err = exchange.DeleteMailFolder(gc.Service(), user, mf.ID)
if err != nil {
Info(ctx, errors.Wrapf(err, "Error: deleting folder [%s]", mf.DisplayName))
}
}
return nil
}
func main() {
ctx := SetRootCmd(context.Background(), purgeCmd)
fs := purgeCmd.Flags()
fs.StringVar(&before, "before", "", "folders older than this date are deleted. (default: now in UTC)")
fs.StringVar(&user, "user", "", "m365 user id whose folders will be deleted")
cobra.CheckErr(purgeCmd.MarkFlagRequired("user"))
fs.StringVar(&tenant, "tenant", "", "m365 tenant containing the user")
fs.StringVar(&prefix, "prefix", "", "filters mail folders by displayName prefix")
cobra.CheckErr(purgeCmd.MarkFlagRequired("prefix"))
if err := purgeCmd.ExecuteContext(ctx); err != nil {
Info(purgeCmd.Context(), "Error: ", err.Error())
os.Exit(1)
}
return boundaryTime, nil
}

10
src/internal/connector/exchange/calendar.go
View File

@ -4,25 +4,25 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
)
// calendarDisplayable is a transformative struct that aligns
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the displayable interface.
type calendarDisplayable struct {
type CalendarDisplayable struct {
models.Calendarable
}
// GetDisplayName returns the *string of the calendar name
func (c calendarDisplayable) GetDisplayName() *string {
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// CreateCalendarDisplayable helper function to create the
// calendarDisplayable during msgraph-sdk-go iterative process
// @param entry is the input supplied by pageIterator.Iterate()
func CreateCalendarDisplayable(entry any) *calendarDisplayable {
func CreateCalendarDisplayable(entry any) *CalendarDisplayable {
calendar, ok := entry.(models.Calendarable)
if !ok {
return nil
}
return &calendarDisplayable{calendar}
return &CalendarDisplayable{calendar}
}

106
src/internal/connector/exchange/service_functions.go
View File

@ -86,11 +86,6 @@ func DeleteMailFolder(gs graph.Service, user, folderID string) error {
return gs.Client().UsersById(user).MailFoldersById(folderID).Delete()
}
type MailFolder struct {
ID string
DisplayName string
}
// CreateCalendar makes an event Calendar with the name in the user's M365 exchange account
// Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go
func CreateCalendar(gs graph.Service, user, calendarName string) (models.Calendarable, error) {
@ -124,9 +119,9 @@ func DeleteContactFolder(gs graph.Service, user, folderID string) error {
// GetAllMailFolders retrieves all mail folders for the specified user.
// If nameContains is populated, only returns mail matching that property.
// Returns a slice of {ID, DisplayName} tuples.
func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolder, error) {
func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]models.MailFolderable, error) {
var (
mfs = []MailFolder{}
mfs = []models.MailFolderable{}
err error
)
@ -141,20 +136,17 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde
return nil, err
}
cb := func(folderItem any) bool {
folder, ok := folderItem.(models.MailFolderable)
cb := func(item any) bool {
folder, ok := item.(models.MailFolderable)
if !ok {
err = errors.New("HasFolder() iteration failure")
err = errors.New("casting item to models.MailFolderable")
return false
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains))
if include {
mfs = append(mfs, MailFolder{
ID: *folder.GetId(),
DisplayName: *folder.GetDisplayName(),
})
mfs = append(mfs, folder)
}
return true
@ -167,6 +159,92 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde
return mfs, err
}
// GetAllCalendars retrieves all event calendars for the specified user.
// If nameContains is populated, only returns calendars matching that property.
// Returns a slice of {ID, DisplayName} tuples.
func GetAllCalendars(gs graph.Service, user, nameContains string) ([]CalendarDisplayable, error) {
var (
cs = []CalendarDisplayable{}
err error
)
resp, err := GetAllCalendarNamesForUser(gs, user)
if err != nil {
return nil, err
}
iter, err := msgraphgocore.NewPageIterator(
resp, gs.Adapter(), models.CreateCalendarCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, err
}
cb := func(item any) bool {
cal, ok := item.(models.Calendarable)
if !ok {
err = errors.New("casting item to models.Calendarable")
return false
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*cal.GetName(), nameContains))
if include {
cs = append(cs, *CreateCalendarDisplayable(cal))
}
return true
}
if err := iter.Iterate(cb); err != nil {
return nil, err
}
return cs, err
}
// GetAllContactFolders retrieves all contacts folders for the specified user.
// If nameContains is populated, only returns folders matching that property.
// Returns a slice of {ID, DisplayName} tuples.
func GetAllContactFolders(gs graph.Service, user, nameContains string) ([]models.ContactFolderable, error) {
var (
cs = []models.ContactFolderable{}
err error
)
resp, err := GetAllContactFolderNamesForUser(gs, user)
if err != nil {
return nil, err
}
iter, err := msgraphgocore.NewPageIterator(
resp, gs.Adapter(), models.CreateContactFolderCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, err
}
cb := func(item any) bool {
folder, ok := item.(models.ContactFolderable)
if !ok {
err = errors.New("casting item to models.ContactFolderable")
return false
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains))
if include {
cs = append(cs, folder)
}
return true
}
if err := iter.Iterate(cb); err != nil {
return nil, err
}
return cs, err
}
// GetContainerID query function to retrieve a container's M365 ID.
// @param containerName is the target's name, user-readable and case sensitive
// @param category switches query and iteration to support multiple exchange applications

176
src/internal/connector/exchange/service_functions_test.go Normal file
View File

@ -0,0 +1,176 @@
package exchange_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/tester"
)
type ServiceFunctionsIntegrationSuite struct {
suite.Suite
gc *connector.GraphConnector
m365UserID string
}
func TestServiceFunctionsIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(ServiceFunctionsIntegrationSuite))
}
func (suite *ServiceFunctionsIntegrationSuite) SetupSuite() {
t := suite.T()
_, err := tester.GetRequiredEnvSls(tester.AWSStorageCredEnvs)
require.NoError(t, err)
acct := tester.NewM365Account(t)
gc, err := connector.NewGraphConnector(acct)
require.NoError(t, err)
suite.gc = gc
suite.m365UserID = tester.M365UserID(t)
}
func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
gs := suite.gc.Service()
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "root calendar",
contains: "Calendar",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
expectCount: assert.Equal,
expectErr: assert.Error,
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: suite.m365UserID,
expectCount: assert.Equal,
expectErr: assert.NoError,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := exchange.GetAllCalendars(gs, test.user, test.contains)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})
}
}
func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
gs := suite.gc.Service()
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "root folder",
contains: "Contact",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
expectCount: assert.Equal,
expectErr: assert.Error,
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: suite.m365UserID,
expectCount: assert.Equal,
expectErr: assert.NoError,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := exchange.GetAllContactFolders(gs, test.user, test.contains)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})
}
}
func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
gs := suite.gc.Service()
table := []struct {
name, contains, user string
expectCount assert.ComparisonAssertionFunc
expectErr assert.ErrorAssertionFunc
}{
{
name: "plain lookup",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "Root folder",
contains: "Inbox",
user: suite.m365UserID,
expectCount: assert.Greater,
expectErr: assert.NoError,
},
{
name: "nonsense user",
user: "fnords_mc_snarfens",
expectCount: assert.Equal,
expectErr: assert.Error,
},
{
name: "nonsense matcher",
contains: "∂ç∂ç∂√≈∂ƒß∂ç√ßç√≈ç√ß∂ƒçß√ß≈∂ƒßç√",
user: suite.m365UserID,
expectCount: assert.Equal,
expectErr: assert.NoError,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
cals, err := exchange.GetAllMailFolders(gs, test.user, test.contains)
test.expectErr(t, err)
test.expectCount(t, len(cals), 0)
})
}
}