From 532922f662aaf6680d29a1d8a507c8daf51d13fa Mon Sep 17 00:00:00 2001 From: Keepers <104464746+ryanfkeepers@users.noreply.github.com> Date: Thu, 21 Jul 2022 15:08:03 -0600 Subject: [PATCH] introduce exchange info selector support (#379) * introduce exchange info selector support Adds support in selectors/exchange for queries based on backup.ExchangeInfo entries. This allows the declaration of selectors based on non-identifier details such as sender, subject, or receivedAt time. Changes Exclude scope matching from being an Any- match comparator to an All-match. This keeps exclude and include behavior identical, hopefully making less confusion for users. --- src/cli/backup/exchange.go | 16 +- src/internal/common/time.go | 25 ++ src/internal/common/time_test.go | 42 +++ src/internal/connector/graph_connector.go | 2 +- src/pkg/selectors/exchange.go | 256 ++++++++++++++----- src/pkg/selectors/exchange_test.go | 246 +++++++----------- src/pkg/selectors/exchangecategory_string.go | 23 +- src/pkg/selectors/selectors.go | 31 ++- 8 files changed, 407 insertions(+), 234 deletions(-) create mode 100644 src/internal/common/time.go create mode 100644 src/internal/common/time_test.go diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 8efba7feb..47e39f677 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -156,22 +156,22 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { func exchangeBackupCreateSelectors(all bool, users, data []string) selectors.Selector { sel := selectors.NewExchangeBackup() if all { - sel.Include(sel.Users(selectors.All())) + sel.Include(sel.Users(selectors.Any())) return sel.Selector } if len(data) == 0 { - sel.Include(sel.ContactFolders(user, selectors.All())) - sel.Include(sel.MailFolders(user, selectors.All())) - sel.Include(sel.Events(user, selectors.All())) + sel.Include(sel.ContactFolders(user, selectors.Any())) + sel.Include(sel.MailFolders(user, selectors.Any())) + sel.Include(sel.Events(user, selectors.Any())) } for _, d := range data { switch d { case dataContacts: - sel.Include(sel.ContactFolders(users, selectors.All())) + sel.Include(sel.ContactFolders(users, selectors.Any())) case dataEmail: - sel.Include(sel.MailFolders(users, selectors.All())) + sel.Include(sel.MailFolders(users, selectors.Any())) case dataEvents: - sel.Include(sel.Events(users, selectors.All())) + sel.Include(sel.Events(users, selectors.Any())) } } return sel.Selector @@ -318,7 +318,7 @@ func exchangeBackupDetailSelectors( // if only the backupID is provided, treat that as an --all query if lc+lcf+le+lef+lev+lu == 0 { - sel.Include(sel.Users(selectors.All())) + sel.Include(sel.Users(selectors.Any())) return sel.Selector } diff --git a/src/internal/common/time.go b/src/internal/common/time.go new file mode 100644 index 000000000..5e4203ea6 --- /dev/null +++ b/src/internal/common/time.go @@ -0,0 +1,25 @@ +package common + +import ( + "errors" + "time" +) + +// FormatTime produces the standard format for corso time values. +// Always formats into the UTC timezone. +func FormatTime(t time.Time) string { + return t.UTC().Format(time.RFC3339Nano) +} + +// ParseTime makes a best attempt to produce a time value from +// the provided string. Always returns a UTC timezone value. +func ParseTime(s string) (time.Time, error) { + if len(s) == 0 { + return time.Time{}, errors.New("cannot interpret an empty string as time.Time") + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, err + } + return t.UTC(), nil +} diff --git a/src/internal/common/time_test.go b/src/internal/common/time_test.go new file mode 100644 index 000000000..d291d6e9c --- /dev/null +++ b/src/internal/common/time_test.go @@ -0,0 +1,42 @@ +package common_test + +import ( + "testing" + "time" + + "github.com/alcionai/corso/internal/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type CommonTimeUnitSuite struct { + suite.Suite +} + +func TestCommonTimeUnitSuite(t *testing.T) { + suite.Run(t, new(CommonTimeUnitSuite)) +} + +func (suite *CommonTimeUnitSuite) TestFormatTime() { + t := suite.T() + now := time.Now() + result := common.FormatTime(now) + assert.Equal(t, now.UTC().Format(time.RFC3339Nano), result) +} + +func (suite *CommonTimeUnitSuite) TestParseTime() { + t := suite.T() + now := time.Now() + + nowStr := now.Format(time.RFC3339Nano) + result, err := common.ParseTime(nowStr) + require.NoError(t, err) + assert.Equal(t, now.UTC(), result) + + _, err = common.ParseTime("") + require.Error(t, err) + + _, err = common.ParseTime("flablabls") + require.Error(t, err) +} diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index a9c93d3cd..7f02f7edd 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -208,7 +208,7 @@ func (gc *GraphConnector) ExchangeDataCollection(ctx context.Context, selector s // TODO: handle "get mail for all users" // this would probably no-op without this check, // but we want it made obvious that we're punting. - if user == selectors.AllTgt { + if user == selectors.AnyTgt { errs = support.WrapAndAppend( "all-users", errors.New("all users selector currently not handled"), diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index eae1fe28c..006cfb293 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -3,6 +3,7 @@ package selectors import ( "strings" + "github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/pkg/backup" ) @@ -76,13 +77,22 @@ func (s Selector) ToExchangeRestore() (*ExchangeRestore, error) { // Exclude/Includes // Include appends the provided scopes to the selector's inclusion set. +// +// All parts of the scope must match for data to be included. +// Ex: Mail(u1, f1, m1) => only includes mail if it is owned by user u1, +// located in folder f1, and ID'd as m1. Use selectors.Any() to wildcard +// a scope value. No value will match if selectors.None() is provided. +// +// Group-level scopes will automatically apply the Any() wildcard to child +// properties. +// ex: User(u1) is the same as Mail(u1, Any(), Any()). func (s *exchange) Include(scopes ...[]exchangeScope) { if s.Includes == nil { s.Includes = []map[string]string{} } concat := []exchangeScope{} for _, scopeSl := range scopes { - concat = append(concat, extendExchangeScopeValues(All(), scopeSl)...) + concat = append(concat, extendExchangeScopeValues(scopeSl)...) } for _, sc := range concat { s.Includes = append(s.Includes, map[string]string(sc)) @@ -91,13 +101,24 @@ func (s *exchange) Include(scopes ...[]exchangeScope) { // Exclude appends the provided scopes to the selector's exclusion set. // Every Exclusion scope applies globally, affecting all inclusion scopes. +// +// All parts of the scope must match for data to be excluded. +// Ex: Mail(u1, f1, m1) => only excludes mail that is owned by user u1, +// located in folder f1, and ID'd as m1. Use selectors.Any() to wildcard +// a scope value. No value will match if selectors.None() is provided. +// +// Group-level scopes will automatically apply the Any() wildcard to +// child properties. +// ex: User(u1) automatically includes all mail, events, and contacts, +// therefore it is the same as selecting all of the following: +// Mail(u1, Any(), Any()), Event(u1, Any()), Contacts(u1, Any(), Any()) func (s *exchange) Exclude(scopes ...[]exchangeScope) { if s.Excludes == nil { s.Excludes = []map[string]string{} } concat := []exchangeScope{} for _, scopeSl := range scopes { - concat = append(concat, extendExchangeScopeValues(None(), scopeSl)...) + concat = append(concat, extendExchangeScopeValues(scopeSl)...) } for _, sc := range concat { s.Excludes = append(s.Excludes, map[string]string(sc)) @@ -106,20 +127,20 @@ func (s *exchange) Exclude(scopes ...[]exchangeScope) { // completes population for certain scope properties, according to the // expecations of Include and Exclude behavior. -func extendExchangeScopeValues(v []string, es []exchangeScope) []exchangeScope { - vv := join(v...) +func extendExchangeScopeValues(es []exchangeScope) []exchangeScope { + v := join(Any()...) for i := range es { switch es[i].Category() { case ExchangeContactFolder: - es[i][ExchangeContact.String()] = vv + es[i][ExchangeContact.String()] = v case ExchangeMailFolder: - es[i][ExchangeMail.String()] = vv + es[i][ExchangeMail.String()] = v case ExchangeUser: - es[i][ExchangeContactFolder.String()] = vv - es[i][ExchangeContact.String()] = vv - es[i][ExchangeEvent.String()] = vv - es[i][ExchangeMailFolder.String()] = vv - es[i][ExchangeMail.String()] = vv + es[i][ExchangeContactFolder.String()] = v + es[i][ExchangeContact.String()] = v + es[i][ExchangeEvent.String()] = v + es[i][ExchangeMailFolder.String()] = v + es[i][ExchangeMail.String()] = v } } return es @@ -130,7 +151,7 @@ func extendExchangeScopeValues(v []string, es []exchangeScope) []exchangeScope { // Produces one or more exchange contact scopes. // One scope is created per combination of users,folders,contacts. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) Contacts(users, folders, contacts []string) []exchangeScope { @@ -154,7 +175,7 @@ func (s *exchange) Contacts(users, folders, contacts []string) []exchangeScope { // Produces one or more exchange contact folder scopes. // One scope is created per combination of users,folders. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) ContactFolders(users, folders []string) []exchangeScope { @@ -174,7 +195,7 @@ func (s *exchange) ContactFolders(users, folders []string) []exchangeScope { // Produces one or more exchange event scopes. // One scope is created per combination of users,events. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) Events(users, events []string) []exchangeScope { @@ -194,7 +215,7 @@ func (s *exchange) Events(users, events []string) []exchangeScope { // Produces one or more mail scopes. // One scope is created per combination of users,folders,mails. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) Mails(users, folders, mails []string) []exchangeScope { @@ -218,7 +239,7 @@ func (s *exchange) Mails(users, folders, mails []string) []exchangeScope { // Produces one or more exchange mail folder scopes. // One scope is created per combination of users,folders. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) MailFolders(users, folders []string) []exchangeScope { @@ -238,7 +259,7 @@ func (s *exchange) MailFolders(users, folders []string) []exchangeScope { // Produces one or more exchange contact user scopes. // One scope is created per user entry. -// If any slice contains selectors.All, that slice is reduced to [selectors.All] +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) Users(users []string) []exchangeScope { @@ -252,6 +273,73 @@ func (s *exchange) Users(users []string) []exchangeScope { return scopes } +// Produces one or more exchange mail sender filter scopes. +// Matches any mail whose sender is equal to one of the provided strings. +// One scope is created per senderID entry. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (sr *ExchangeRestore) MailSender(senderID []string) []exchangeScope { + scopes := []exchangeScope{} + scopes = append(scopes, exchangeScope{ + scopeKeyGranularity: Item, + scopeKeyCategory: ExchangeMail.String(), + scopeKeyInfoFilter: ExchangeInfoMailSender.String(), + ExchangeInfoMailSender.String(): join(senderID...), + }) + return scopes +} + +// Produces one or more exchange mail subject filter scopes. +// Matches any mail whose mail subject contains one of the provided strings. +// One scope is created per subject entry. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (sr *ExchangeRestore) MailSubject(subjectSubstring []string) []exchangeScope { + scopes := []exchangeScope{} + scopes = append(scopes, exchangeScope{ + scopeKeyGranularity: Item, + scopeKeyCategory: ExchangeMail.String(), + scopeKeyInfoFilter: ExchangeInfoMailSubject.String(), + ExchangeInfoMailSubject.String(): join(subjectSubstring...), + }) + return scopes +} + +// Produces one or more exchange mail received-after filter scopes. +// Matches any mail which was received after the timestring. +// One scope is created per timeString entry. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (sr *ExchangeRestore) MailReceivedAfter(timeString []string) []exchangeScope { + scopes := []exchangeScope{} + scopes = append(scopes, exchangeScope{ + scopeKeyGranularity: Item, + scopeKeyCategory: ExchangeMail.String(), + scopeKeyInfoFilter: ExchangeInfoMailReceivedAfter.String(), + ExchangeInfoMailReceivedAfter.String(): join(timeString...), + }) + return scopes +} + +// Produces one or more exchange mail received-before filter scopes. +// Matches any mail which was received before the timestring. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (sr *ExchangeRestore) MailReceivedBefore(timeString []string) []exchangeScope { + scopes := []exchangeScope{} + scopes = append(scopes, exchangeScope{ + scopeKeyGranularity: Item, + scopeKeyCategory: ExchangeMail.String(), + scopeKeyInfoFilter: ExchangeInfoMailReceivedBefore.String(), + ExchangeInfoMailReceivedBefore.String(): join(timeString...), + }) + return scopes +} + // --------------------------------------------------------------------------- // Destination // --------------------------------------------------------------------------- @@ -311,16 +399,23 @@ func (s *exchange) Scopes() []exchangeScope { //go:generate stringer -type=exchangeCategory const ( ExchangeCategoryUnknown exchangeCategory = iota + // types of data identified by exchange ExchangeContact ExchangeContactFolder ExchangeEvent ExchangeMail ExchangeMailFolder ExchangeUser + // filterable topics identified by exchange + ExchangeInfoMailSender exchangeCategory = iota + 100 // offset to pad out future data additions + ExchangeInfoMailSubject + ExchangeInfoMailReceivedAfter + ExchangeInfoMailReceivedBefore ) func exchangeCatAtoI(s string) exchangeCategory { switch s { + // data types case ExchangeContact.String(): return ExchangeContact case ExchangeContactFolder.String(): @@ -333,6 +428,15 @@ func exchangeCatAtoI(s string) exchangeCategory { return ExchangeMailFolder case ExchangeUser.String(): return ExchangeUser + // filters + case ExchangeInfoMailSender.String(): + return ExchangeInfoMailSender + case ExchangeInfoMailSubject.String(): + return ExchangeInfoMailSubject + case ExchangeInfoMailReceivedAfter.String(): + return ExchangeInfoMailReceivedAfter + case ExchangeInfoMailReceivedBefore.String(): + return ExchangeInfoMailReceivedBefore default: return ExchangeCategoryUnknown } @@ -349,6 +453,11 @@ func (s exchangeScope) Category() exchangeCategory { return exchangeCatAtoI(s[scopeKeyCategory]) } +// Filer describes the specific filter, and its target values. +func (s exchangeScope) Filter() exchangeCategory { + return exchangeCatAtoI(s[scopeKeyInfoFilter]) +} + // IncludeCategory checks whether the scope includes a // certain category of data. // Ex: to check if the scope includes mail data: @@ -387,64 +496,87 @@ var categoryPathSet = map[exchangeCategory][]exchangeCategory{ ExchangeContact: {ExchangeUser, ExchangeContactFolder, ExchangeContact}, ExchangeEvent: {ExchangeUser, ExchangeEvent}, ExchangeMail: {ExchangeUser, ExchangeMailFolder, ExchangeMail}, + ExchangeUser: {ExchangeUser}, } -// includesPath returns true if all filters in the scope match the path. -func (s exchangeScope) includesPath(cat exchangeCategory, path []string) bool { - ids := exchangeIDPath(cat, path) +// matches returns true if either the path or the info matches the scope details. +func (s exchangeScope) matches(cat exchangeCategory, path []string, info *backup.ExchangeInfo) bool { + return s.matchesPath(cat, path) || s.matchesInfo(cat, info) +} + +// matchesInfo handles the standard behavior when comparing a scope and an exchangeInfo +// returns true if the scope and info match for the provided category. +func (s exchangeScope) matchesInfo(cat exchangeCategory, info *backup.ExchangeInfo) bool { + // we need values to match against + if info == nil { + return false + } + // the scope must define targets to match on + filterCat := s.Filter() + targets := s.Get(filterCat) + if len(targets) == 0 { + return false + } + if targets[0] == AnyTgt { + return true + } + if targets[0] == NoneTgt { + return false + } + // any of the targets for a given info filter may succeed. + for _, target := range targets { + switch filterCat { + case ExchangeInfoMailSender: + if target == info.Sender { + return true + } + case ExchangeInfoMailSubject: + if strings.Contains(info.Subject, target) { + return true + } + case ExchangeInfoMailReceivedAfter: + if target < common.FormatTime(info.Received) { + return true + } + case ExchangeInfoMailReceivedBefore: + if target > common.FormatTime(info.Received) { + return true + } + } + } + return false +} + +// matchesPath handles the standard behavior when comparing a scope and a path +// returns true if the scope and path match for the provided category. +func (s exchangeScope) matchesPath(cat exchangeCategory, path []string) bool { + pathIDs := exchangeIDPath(cat, path) for _, c := range categoryPathSet[cat] { target := s.Get(c) + // the scope must define the targets to match on if len(target) == 0 { return false } - id, ok := ids[c] + // None() fails all matches + if target[0] == NoneTgt { + return false + } + // the path must contain a value to match against + id, ok := pathIDs[c] if !ok { return false } - if target[0] != AllTgt && !contains(target, id) { - return false + // all parts of the scope must match + isAny := target[0] == AnyTgt + if !isAny { + if !contains(target, id) { + return false + } } } return true } -// includesInfo returns true if all filters in the scope match the info. -func (s exchangeScope) includesInfo(cat exchangeCategory, info *backup.ExchangeInfo) bool { - // todo: implement once filters used in scopes - if info == nil { - return false - } - return false -} - -// excludesPath returns true if all filters in the scope match the path. -func (s exchangeScope) excludesPath(cat exchangeCategory, path []string) bool { - ids := exchangeIDPath(cat, path) - for _, c := range categoryPathSet[cat] { - target := s.Get(c) - if len(target) == 0 { - return true - } - id, ok := ids[c] - if !ok { - return true - } - if target[0] == AllTgt || contains(target, id) { - return true - } - } - return false -} - -// excludesInfo returns true if all filters in the scope matche the info. -func (s exchangeScope) excludesInfo(cat exchangeCategory, info *backup.ExchangeInfo) bool { - // todo: implement once filters used in scopes - if info == nil { - return false - } - return false -} - // temporary helper until filters replace string values for scopes. func contains(super []string, sub string) bool { for _, s := range super { @@ -577,7 +709,7 @@ func matchExchangeEntry( ) bool { var included bool for _, inc := range incs { - if inc.includesPath(cat, path) || inc.includesInfo(cat, info) { + if inc.matches(cat, path, info) { included = true break } @@ -588,7 +720,7 @@ func matchExchangeEntry( var excluded bool for _, exc := range excs { - if exc.excludesPath(cat, path) || exc.excludesInfo(cat, info) { + if exc.matches(cat, path, info) { excluded = true break } diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index d673552f9..59efb0620 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -2,7 +2,9 @@ package selectors import ( "testing" + "time" + "github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/pkg/backup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -57,7 +59,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Exclude_Contacts() { const ( user = "user" - folder = AllTgt + folder = AnyTgt c1 = "c1" c2 = "c2" ) @@ -78,7 +80,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_Contacts() { const ( user = "user" - folder = AllTgt + folder = AnyTgt c1 = "c1" c2 = "c2" ) @@ -112,7 +114,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Exclude_ContactFolders() scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], user) assert.Equal(t, scope[ExchangeContactFolder.String()], join(f1, f2)) - assert.Equal(t, scope[ExchangeContact.String()], NoneTgt) + assert.Equal(t, scope[ExchangeContact.String()], AnyTgt) } func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_ContactFolders() { @@ -132,7 +134,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_ContactFolders() scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], user) assert.Equal(t, scope[ExchangeContactFolder.String()], join(f1, f2)) - assert.Equal(t, scope[ExchangeContact.String()], AllTgt) + assert.Equal(t, scope[ExchangeContact.String()], AnyTgt) assert.Equal(t, sel.Scopes()[0].Category(), ExchangeContactFolder) } @@ -183,7 +185,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Exclude_Mails() { const ( user = "user" - folder = AllTgt + folder = AnyTgt m1 = "m1" m2 = "m2" ) @@ -204,7 +206,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_Mails() { const ( user = "user" - folder = AllTgt + folder = AnyTgt m1 = "m1" m2 = "m2" ) @@ -238,7 +240,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Exclude_MailFolders() { scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], user) assert.Equal(t, scope[ExchangeMailFolder.String()], join(f1, f2)) - assert.Equal(t, scope[ExchangeMail.String()], NoneTgt) + assert.Equal(t, scope[ExchangeMail.String()], AnyTgt) } func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_MailFolders() { @@ -258,7 +260,7 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_MailFolders() { scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], user) assert.Equal(t, scope[ExchangeMailFolder.String()], join(f1, f2)) - assert.Equal(t, scope[ExchangeMail.String()], AllTgt) + assert.Equal(t, scope[ExchangeMail.String()], AnyTgt) assert.Equal(t, sel.Scopes()[0].Category(), ExchangeMailFolder) } @@ -278,11 +280,11 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Exclude_Users() { scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], join(u1, u2)) - assert.Equal(t, scope[ExchangeContact.String()], NoneTgt) - assert.Equal(t, scope[ExchangeContactFolder.String()], NoneTgt) - assert.Equal(t, scope[ExchangeEvent.String()], NoneTgt) - assert.Equal(t, scope[ExchangeMail.String()], NoneTgt) - assert.Equal(t, scope[ExchangeMailFolder.String()], NoneTgt) + assert.Equal(t, scope[ExchangeContact.String()], AnyTgt) + assert.Equal(t, scope[ExchangeContactFolder.String()], AnyTgt) + assert.Equal(t, scope[ExchangeEvent.String()], AnyTgt) + assert.Equal(t, scope[ExchangeMail.String()], AnyTgt) + assert.Equal(t, scope[ExchangeMailFolder.String()], AnyTgt) } func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_Users() { @@ -300,11 +302,11 @@ func (suite *ExchangeSourceSuite) TestExchangeSelector_Include_Users() { scope := scopes[0] assert.Equal(t, scope[ExchangeUser.String()], join(u1, u2)) - assert.Equal(t, scope[ExchangeContact.String()], AllTgt) - assert.Equal(t, scope[ExchangeContactFolder.String()], AllTgt) - assert.Equal(t, scope[ExchangeEvent.String()], AllTgt) - assert.Equal(t, scope[ExchangeMail.String()], AllTgt) - assert.Equal(t, scope[ExchangeMailFolder.String()], AllTgt) + assert.Equal(t, scope[ExchangeContact.String()], AnyTgt) + assert.Equal(t, scope[ExchangeContactFolder.String()], AnyTgt) + assert.Equal(t, scope[ExchangeEvent.String()], AnyTgt) + assert.Equal(t, scope[ExchangeMail.String()], AnyTgt) + assert.Equal(t, scope[ExchangeMailFolder.String()], AnyTgt) assert.Equal(t, sel.Scopes()[0].Category(), ExchangeUser) } @@ -359,19 +361,19 @@ func (suite *ExchangeSourceSuite) TestExchangeDestination_GetOrDefault() { } var allScopesExceptUnknown = map[string]string{ - ExchangeContact.String(): AllTgt, - ExchangeContactFolder.String(): AllTgt, - ExchangeEvent.String(): AllTgt, - ExchangeMail.String(): AllTgt, - ExchangeMailFolder.String(): AllTgt, - ExchangeUser.String(): AllTgt, + ExchangeContact.String(): AnyTgt, + ExchangeContactFolder.String(): AnyTgt, + ExchangeEvent.String(): AnyTgt, + ExchangeMail.String(): AnyTgt, + ExchangeMailFolder.String(): AnyTgt, + ExchangeUser.String(): AnyTgt, } func (suite *ExchangeSourceSuite) TestExchangeBackup_Scopes() { eb := NewExchangeBackup() eb.Includes = []map[string]string{allScopesExceptUnknown} // todo: swap the above for this - // eb := NewExchangeBackup().IncludeUsers(AllTgt) + // eb := NewExchangeBackup().IncludeUsers(AnyTgt) scopes := eb.Scopes() assert.Len(suite.T(), scopes, 1) @@ -448,7 +450,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_Get() { eb := NewExchangeBackup() eb.Includes = []map[string]string{allScopesExceptUnknown} // todo: swap the above for this - // eb := NewExchangeBackup().IncludeUsers(AllTgt) + // eb := NewExchangeBackup().IncludeUsers(AnyTgt) scope := eb.Scopes()[0] @@ -466,7 +468,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_Get() { None(), scope.Get(ExchangeCategoryUnknown)) - expect := All() + expect := Any() for _, test := range table { suite.T().Run(test.String(), func(t *testing.T) { assert.Equal(t, expect, scope.Get(test)) @@ -474,33 +476,55 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_Get() { } } -func (suite *ExchangeSourceSuite) TestExchangeScope_IncludesInfo() { +func (suite *ExchangeSourceSuite) TestExchangeScope_Include_MatchesInfo() { + es := NewExchangeRestore() const ( - TODO = "this is a placeholder, awaiting implemenation of filters" + sender = "smarf@2many.cooks" + subject = "I have seen the fnords!" ) var ( - es = NewExchangeRestore() + epoch = time.Time{} + now = time.Now() + then = now.Add(1 * time.Minute) + info = &backup.ExchangeInfo{ + Sender: sender, + Subject: subject, + Received: now, + } ) table := []struct { name string scope []exchangeScope - info *backup.ExchangeInfo expect assert.BoolAssertionFunc }{ - {"all user's items", es.Users(All()), nil, assert.False}, // false while a todo + {"any mail with a sender", es.MailSender(Any()), assert.True}, + {"no mail, regardless of sender", es.MailSender(None()), assert.False}, + {"mail from a different sender", es.MailSender([]string{"magoo@ma.goo"}), assert.False}, + {"mail from the matching sender", es.MailSender([]string{sender}), assert.True}, + {"mail with any subject", es.MailSubject(Any()), assert.True}, + {"no mail, regardless of subject", es.MailSubject(None()), assert.False}, + {"mail with a different subject", es.MailSubject([]string{"fancy"}), assert.False}, + {"mail with the matching subject", es.MailSubject([]string{subject}), assert.True}, + {"mail with a substring subject match", es.MailSubject([]string{subject[5:9]}), assert.True}, + {"mail received after the epoch", es.MailReceivedAfter([]string{common.FormatTime(epoch)}), assert.True}, + {"mail received after now", es.MailReceivedAfter([]string{common.FormatTime(now)}), assert.False}, + {"mail received after sometime later", es.MailReceivedAfter([]string{common.FormatTime(then)}), assert.False}, + {"mail received before the epoch", es.MailReceivedBefore([]string{common.FormatTime(epoch)}), assert.False}, + {"mail received before now", es.MailReceivedBefore([]string{common.FormatTime(now)}), assert.False}, + {"mail received before sometime later", es.MailReceivedBefore([]string{common.FormatTime(then)}), assert.True}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - scopes := extendExchangeScopeValues(All(), test.scope) + scopes := extendExchangeScopeValues(test.scope) for _, scope := range scopes { - test.expect(t, scope.includesInfo(ExchangeMail, test.info)) + test.expect(t, scope.matchesInfo(scope.Category(), info)) } }) } } -func (suite *ExchangeSourceSuite) TestExchangeScope_IncludesPath() { +func (suite *ExchangeSourceSuite) TestExchangeScope_MatchesPath() { const ( usr = "userID" fld = "mailFolder" @@ -516,95 +540,27 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_IncludesPath() { scope []exchangeScope expect assert.BoolAssertionFunc }{ - {"all user's items", es.Users(All()), assert.True}, + {"all user's items", es.Users(Any()), assert.True}, {"no user's items", es.Users(None()), assert.False}, {"matching user", es.Users([]string{usr}), assert.True}, {"non-maching user", es.Users([]string{"smarf"}), assert.False}, {"one of multiple users", es.Users([]string{"smarf", usr}), assert.True}, - {"all folders", es.MailFolders(All(), All()), assert.True}, - {"no folders", es.MailFolders(All(), None()), assert.False}, - {"matching folder", es.MailFolders(All(), []string{fld}), assert.True}, - {"non-matching folder", es.MailFolders(All(), []string{"smarf"}), assert.False}, - {"one of multiple folders", es.MailFolders(All(), []string{"smarf", fld}), assert.True}, - {"all mail", es.Mails(All(), All(), All()), assert.True}, - {"no mail", es.Mails(All(), All(), None()), assert.False}, - {"matching mail", es.Mails(All(), All(), []string{mail}), assert.True}, - {"non-matching mail", es.Mails(All(), All(), []string{"smarf"}), assert.False}, - {"one of multiple mails", es.Mails(All(), All(), []string{"smarf", mail}), assert.True}, + {"all folders", es.MailFolders(Any(), Any()), assert.True}, + {"no folders", es.MailFolders(Any(), None()), assert.False}, + {"matching folder", es.MailFolders(Any(), []string{fld}), assert.True}, + {"non-matching folder", es.MailFolders(Any(), []string{"smarf"}), assert.False}, + {"one of multiple folders", es.MailFolders(Any(), []string{"smarf", fld}), assert.True}, + {"all mail", es.Mails(Any(), Any(), Any()), assert.True}, + {"no mail", es.Mails(Any(), Any(), None()), assert.False}, + {"matching mail", es.Mails(Any(), Any(), []string{mail}), assert.True}, + {"non-matching mail", es.Mails(Any(), Any(), []string{"smarf"}), assert.False}, + {"one of multiple mails", es.Mails(Any(), Any(), []string{"smarf", mail}), assert.True}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - scopes := extendExchangeScopeValues(All(), test.scope) + scopes := extendExchangeScopeValues(test.scope) for _, scope := range scopes { - test.expect(t, scope.includesPath(ExchangeMail, path)) - } - }) - } -} - -func (suite *ExchangeSourceSuite) TestExchangeScope_ExcludesInfo() { - const ( - TODO = "this is a placeholder, awaiting implemenation of filters" - ) - var ( - es = NewExchangeRestore() - ) - - table := []struct { - name string - scope []exchangeScope - info *backup.ExchangeInfo - expect assert.BoolAssertionFunc - }{ - {"all user's items", es.Users(All()), nil, assert.False}, // false while a todo - } - for _, test := range table { - suite.T().Run(test.name, func(t *testing.T) { - scopes := extendExchangeScopeValues(None(), test.scope) - for _, scope := range scopes { - test.expect(t, scope.excludesInfo(ExchangeMail, test.info)) - } - }) - } -} - -func (suite *ExchangeSourceSuite) TestExchangeScope_ExcludesPath() { - const ( - usr = "userID" - fld = "mailFolder" - mail = "mailID" - ) - var ( - path = []string{"tid", usr, "mail", fld, mail} - es = NewExchangeRestore() - ) - - table := []struct { - name string - scope []exchangeScope - expect assert.BoolAssertionFunc - }{ - {"all user's items", es.Users(All()), assert.True}, - {"no user's items", es.Users(None()), assert.False}, - {"matching user", es.Users([]string{usr}), assert.True}, - {"non-maching user", es.Users([]string{"smarf"}), assert.False}, - {"one of multiple users", es.Users([]string{"smarf", usr}), assert.True}, - {"all folders", es.MailFolders(None(), All()), assert.True}, - {"no folders", es.MailFolders(None(), None()), assert.False}, - {"matching folder", es.MailFolders(None(), []string{fld}), assert.True}, - {"non-matching folder", es.MailFolders(None(), []string{"smarf"}), assert.False}, - {"one of multiple folders", es.MailFolders(None(), []string{"smarf", fld}), assert.True}, - {"all mail", es.Mails(None(), None(), All()), assert.True}, - {"no mail", es.Mails(None(), None(), None()), assert.False}, - {"matching mail", es.Mails(None(), None(), []string{mail}), assert.True}, - {"non-matching mail", es.Mails(None(), None(), []string{"smarf"}), assert.False}, - {"one of multiple mails", es.Mails(None(), None(), []string{"smarf", mail}), assert.True}, - } - for _, test := range table { - suite.T().Run(test.name, func(t *testing.T) { - scopes := extendExchangeScopeValues(None(), test.scope) - for _, scope := range scopes { - test.expect(t, scope.excludesPath(ExchangeMail, path)) + test.expect(t, scope.matchesPath(ExchangeMail, path)) } }) } @@ -688,7 +644,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) return er }, []string{}, @@ -698,7 +654,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(contact), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) return er }, arr(contact), @@ -708,7 +664,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(event), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) return er }, arr(event), @@ -718,7 +674,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) return er }, arr(mail), @@ -728,7 +684,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(contact, event, mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) return er }, arr(contact, event, mail), @@ -768,7 +724,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(contact, event, mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) er.Exclude(er.Contacts([]string{"uid"}, []string{"cfld"}, []string{"cid"})) return er }, @@ -779,7 +735,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(contact, event, mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) er.Exclude(er.Events([]string{"uid"}, []string{"eid"})) return er }, @@ -790,7 +746,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { makeDeets(contact, event, mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Users(All())) + er.Include(er.Users(Any())) er.Exclude(er.Mails([]string{"uid"}, []string{"mfld"}, []string{"mid"})) return er }, @@ -810,10 +766,10 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { func (suite *ExchangeSourceSuite) TestExchangeScopesByCategory() { var ( es = NewExchangeRestore() - users = es.Users(All()) - contacts = es.ContactFolders(All(), All()) - events = es.Events(All(), All()) - mail = es.MailFolders(All(), All()) + users = es.Users(Any()) + contacts = es.ContactFolders(Any(), Any()) + events = es.Events(Any(), Any()) + mail = es.MailFolders(Any(), Any()) ) type expect struct { contact int @@ -857,22 +813,16 @@ func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() { mail = "mailID" cat = ExchangeMail ) - include := func(s []exchangeScope) []exchangeScope { - return extendExchangeScopeValues(All(), s) - } - exclude := func(s []exchangeScope) []exchangeScope { - return extendExchangeScopeValues(None(), s) - } var ( es = NewExchangeRestore() - inAll = include(es.Users(All())) - inNone = include(es.Users(None())) - inMail = include(es.Mails(All(), All(), []string{mail})) - inOtherMail = include(es.Mails(All(), All(), []string{"smarf"})) - exAll = exclude(es.Users(All())) - exNone = exclude(es.Users(None())) - exMail = exclude(es.Mails(None(), None(), []string{mail})) - exOtherMail = exclude(es.Mails(None(), None(), []string{"smarf"})) + inAny = extendExchangeScopeValues(es.Users(Any())) + inNone = extendExchangeScopeValues(es.Users(None())) + inMail = extendExchangeScopeValues(es.Mails(Any(), Any(), []string{mail})) + inOtherMail = extendExchangeScopeValues(es.Mails(Any(), Any(), []string{"smarf"})) + exAny = extendExchangeScopeValues(es.Users(Any())) + exNone = extendExchangeScopeValues(es.Users(None())) + exMail = extendExchangeScopeValues(es.Mails(Any(), Any(), []string{mail})) + exOtherMail = extendExchangeScopeValues(es.Mails(Any(), Any(), []string{"smarf"})) path = []string{"tid", "user", "mail", "folder", mail} ) @@ -883,15 +833,15 @@ func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() { expect assert.BoolAssertionFunc }{ {"empty", nil, nil, assert.False}, - {"in all", inAll, nil, assert.True}, + {"in all", inAny, nil, assert.True}, {"in None", inNone, nil, assert.False}, {"in Mail", inMail, nil, assert.True}, {"in Other", inOtherMail, nil, assert.False}, - {"ex all", inAll, exAll, assert.False}, - {"ex None", inAll, exNone, assert.True}, - {"in Mail", inAll, exMail, assert.False}, - {"in Other", inAll, exOtherMail, assert.True}, - {"in and ex mail", inMail, exMail, assert.False}, + {"ex all", inAny, exAny, assert.False}, + {"ex None", inAny, exNone, assert.True}, + {"in Mail", inAny, exMail, assert.False}, + {"in Other", inAny, exOtherMail, assert.True}, + {"in and ex Mail", inMail, exMail, assert.False}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { diff --git a/src/pkg/selectors/exchangecategory_string.go b/src/pkg/selectors/exchangecategory_string.go index 89e5b244b..b27bf8307 100644 --- a/src/pkg/selectors/exchangecategory_string.go +++ b/src/pkg/selectors/exchangecategory_string.go @@ -15,15 +15,30 @@ func _() { _ = x[ExchangeMail-4] _ = x[ExchangeMailFolder-5] _ = x[ExchangeUser-6] + _ = x[ExchangeInfoMailSender-107] + _ = x[ExchangeInfoMailSubject-108] + _ = x[ExchangeInfoMailReceivedAfter-109] + _ = x[ExchangeInfoMailReceivedBefore-110] } -const _exchangeCategory_name = "ExchangeCategoryUnknownExchangeContactExchangeContactFolderExchangeEventExchangeMailExchangeMailFolderExchangeUser" +const ( + _exchangeCategory_name_0 = "ExchangeCategoryUnknownExchangeContactExchangeContactFolderExchangeEventExchangeMailExchangeMailFolderExchangeUser" + _exchangeCategory_name_1 = "ExchangeInfoMailSenderExchangeInfoMailSubjectExchangeInfoMailReceivedAfterExchangeInfoMailReceivedBefore" +) -var _exchangeCategory_index = [...]uint8{0, 23, 38, 59, 72, 84, 102, 114} +var ( + _exchangeCategory_index_0 = [...]uint8{0, 23, 38, 59, 72, 84, 102, 114} + _exchangeCategory_index_1 = [...]uint8{0, 22, 45, 74, 104} +) func (i exchangeCategory) String() string { - if i < 0 || i >= exchangeCategory(len(_exchangeCategory_index)-1) { + switch { + case 0 <= i && i <= 6: + return _exchangeCategory_name_0[_exchangeCategory_index_0[i]:_exchangeCategory_index_0[i+1]] + case 107 <= i && i <= 110: + i -= 107 + return _exchangeCategory_name_1[_exchangeCategory_index_1[i]:_exchangeCategory_index_1[i+1]] + default: return "exchangeCategory(" + strconv.FormatInt(int64(i), 10) + ")" } - return _exchangeCategory_name[_exchangeCategory_index[i]:_exchangeCategory_index[i+1]] } diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index dd3cc2e44..2c75ebd29 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -19,21 +19,27 @@ var ErrorBadSelectorCast = errors.New("wrong selector service type") const ( scopeKeyCategory = "category" scopeKeyGranularity = "granularity" + scopeKeyInfoFilter = "info_filter" ) +// The granularity exprerssed by the scope. Groups imply non-item granularity, +// such as a directory. Items are individual files or objects. const ( Group = "group" Item = "item" ) const ( - // AllTgt is the target value used to select "all data of " - // Ex: {user: u1, events: AllTgt) => all events for user u1. + // AnyTgt is the target value used to select "any data of " + // Ex: {user: u1, events: AnyTgt) => all events for user u1. // In the event that "*" conflicts with a user value, such as a // folder named "*", calls to corso should escape the value with "\*" - AllTgt = "*" + AnyTgt = "*" // NoneTgt is the target value used to select "no data of " - // Ex: {user: u1, events: NoneTgt} => no events for user u1. + // This is primarily a fallback for empty values. Adding NoneTgt or + // None() to any selector will force all matches() checks on that + // selector to fail. + // Ex: {user: u1, events: NoneTgt} => matches nothing. NoneTgt = "" delimiter = "," @@ -60,12 +66,15 @@ func newSelector(s service) Selector { } } -// All returns the set matching All values. -func All() []string { - return []string{AllTgt} +// Any returns the set matching any value. +func Any() []string { + return []string{AnyTgt} } // None returns the set matching None of the values. +// This is primarily a fallback for empty values. Adding None() +// to any selector will force all matches() checks on that selector +// to fail. func None() []string { return []string{NoneTgt} } @@ -98,9 +107,9 @@ func split(s string) []string { return strings.Split(s, delimiter) } -// if the provided slice contains All, returns [All] +// if the provided slice contains Any, returns [Any] // if the slice contains None, returns [None] -// if the slice contains All and None, returns the first +// if the slice contains Any and None, returns the first // if the slice is empty, returns [None] // otherwise returns the input unchanged func normalize(s []string) []string { @@ -108,8 +117,8 @@ func normalize(s []string) []string { return None() } for _, e := range s { - if e == AllTgt { - return All() + if e == AnyTgt { + return Any() } if e == NoneTgt { return None()