From 80db9a56c73616f569f84c38d1978bd9c01fe738 Mon Sep 17 00:00:00 2001 From: Keepers <104464746+ryanfkeepers@users.noreply.github.com> Date: Mon, 11 Jul 2022 14:44:08 -0600 Subject: [PATCH] reduce restore point details to matching refs (#310) * reduce restore point details to matching refs A selector should be able to reduce a set of restore point details to only those that pass its inclusion and exclusion rules. --- src/pkg/selectors/exchange.go | 194 ++++++++++++++++ src/pkg/selectors/exchange_test.go | 355 +++++++++++++++++++++++++++++ 2 files changed, 549 insertions(+) diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index 471b42ae1..e6db1c761 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -1,5 +1,11 @@ package selectors +import ( + "strings" + + "github.com/alcionai/corso/pkg/backup" +) + // --------------------------------------------------------------------------- // Selectors // --------------------------------------------------------------------------- @@ -298,3 +304,191 @@ func (s exchangeScope) Get(cat exchangeCategory) []string { } return split(v) } + +var categoryPathSet = map[exchangeCategory][]exchangeCategory{ + ExchangeContact: {ExchangeUser, ExchangeContactFolder, ExchangeContact}, + ExchangeEvent: {ExchangeUser, ExchangeEvent}, + ExchangeMail: {ExchangeUser, ExchangeMailFolder, ExchangeMail}, +} + +// includesPath returns true if all filters in the scope match the path. +func (s exchangeScope) includesPath(cat exchangeCategory, path []string) bool { + ids := idPath(cat, path) + for _, c := range categoryPathSet[cat] { + target := s.Get(c) + if len(target) == 0 { + return false + } + id, ok := ids[c] + if !ok { + return false + } + if target[0] != All && !contains(target, id) { + return false + } + } + return true +} + +// excludesPath returns true if all filters in the scope match the path. +func (s exchangeScope) excludesPath(cat exchangeCategory, path []string) bool { + ids := idPath(cat, path) + for _, c := range categoryPathSet[cat] { + target := s.Get(c) + if len(target) == 0 { + return true + } + id, ok := ids[c] + if !ok { + return true + } + if target[0] == All || contains(target, id) { + return true + } + } + return false +} + +// temporary helper until filters replace string values for scopes. +func contains(super []string, sub string) bool { + for _, s := range super { + if s == sub { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// Restore Point Filtering +// --------------------------------------------------------------------------- + +// transforms a path to a map of identified properties. +// Malformed (ie, short len) paths will return incomplete results. +// Example: +// [tenantID, userID, "mail", mailFolder, mailID] +// => {exchUser: userID, exchMailFolder: mailFolder, exchMail: mailID} +func idPath(cat exchangeCategory, path []string) map[exchangeCategory]string { + m := map[exchangeCategory]string{} + if len(path) == 0 { + return m + } + m[ExchangeUser] = path[1] + /* + TODO/Notice: + Mail and Contacts contain folder structures, identified + in this code as being at index 3. This assumes a single + folder, while in reality users can express subfolder + hierarchies of arbirary depth. Subfolder handling is coming + at a later time. + */ + switch cat { + case ExchangeContact: + if len(path) < 5 { + return m + } + m[ExchangeContactFolder] = path[3] + m[ExchangeContact] = path[4] + case ExchangeEvent: + if len(path) < 4 { + return m + } + m[ExchangeEvent] = path[3] + case ExchangeMail: + if len(path) < 5 { + return m + } + m[ExchangeMailFolder] = path[3] + m[ExchangeMail] = path[4] + } + return m +} + +// FilterDetails reduces the entries in a backupDetails struct to only +// those that match the inclusions and exclusions in the selector. +func (s *ExchangeRestore) FilterDetails(deets *backup.Details) []string { + if deets == nil { + return []string{} + } + + entIncs := exchangeScopesByCategory(s.Includes) + entExcs := exchangeScopesByCategory(s.Excludes) + + refs := []string{} + + for _, ent := range deets.Entries { + path := strings.Split(ent.RepoRef, "/") + // not all paths will be len=3. Most should be longer. + // This just protects us from panicing four lines later. + if len(path) < 3 { + continue + } + var cat exchangeCategory + switch path[2] { + case "contact": + cat = ExchangeContact + case "event": + cat = ExchangeEvent + case "mail": + cat = ExchangeMail + } + matched := matchExchangeEntry( + cat, + path, + entIncs[cat.String()], + entExcs[cat.String()]) + if matched { + refs = append(refs, ent.RepoRef) + } + } + + return refs +} + +// groups each scope by its category of data (contact, event, or mail). +// user-level scopes will duplicate to all three categories. +func exchangeScopesByCategory(scopes []map[string]string) map[string][]exchangeScope { + m := map[string][]exchangeScope{ + ExchangeContact.String(): {}, + ExchangeEvent.String(): {}, + ExchangeMail.String(): {}, + } + for _, msc := range scopes { + sc := exchangeScope(msc) + if sc.IncludesCategory(ExchangeContact) { + m[ExchangeContact.String()] = append(m[ExchangeContact.String()], sc) + } + if sc.IncludesCategory(ExchangeEvent) { + m[ExchangeEvent.String()] = append(m[ExchangeEvent.String()], sc) + } + if sc.IncludesCategory(ExchangeMail) { + m[ExchangeMail.String()] = append(m[ExchangeMail.String()], sc) + } + } + return m +} + +// compare each path to the included and excluded exchange scopes. Returns true +// if the path is included, and not excluded. +func matchExchangeEntry(cat exchangeCategory, path []string, incs, excs []exchangeScope) bool { + var included bool + for _, inc := range incs { + if inc.includesPath(cat, path) { + included = true + break + } + } + if !included { + return false + } + + var excluded bool + for _, exc := range excs { + if exc.excludesPath(cat, path) { + excluded = true + break + } + } + + return !excluded +} diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index b9b4251a7..bc1dc3c73 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -3,6 +3,7 @@ package selectors import ( "testing" + "github.com/alcionai/corso/pkg/backup" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -476,3 +477,357 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_Get() { }) } } + +func (suite *ExchangeSourceSuite) TestExchangeScope_IncludesPath() { + const ( + usr = "userID" + fld = "mailFolder" + mail = "mailID" + ) + var ( + path = []string{"tid", usr, "mail", fld, mail} + es = NewExchangeRestore("rpid") + ) + + table := []struct { + name string + scope exchangeScope + expect assert.BoolAssertionFunc + }{ + {"all user's items", es.Users(All), assert.True}, + {"no user's items", es.Users(None), assert.False}, + {"matching user", es.Users(usr), assert.True}, + {"non-maching user", es.Users("smarf"), assert.False}, + {"one of multiple users", es.Users("smarf", usr), assert.True}, + {"all folders", es.MailFolders(All, All), assert.True}, + {"no folders", es.MailFolders(All, None), assert.False}, + {"matching folder", es.MailFolders(All, fld), assert.True}, + {"non-matching folder", es.MailFolders(All, "smarf"), assert.False}, + {"one of multiple folders", es.MailFolders(All, "smarf", fld), assert.True}, + {"all mail", es.Mails(All, All, All), assert.True}, + {"no mail", es.Mails(All, All, None), assert.False}, + {"matching mail", es.Mails(All, All, mail), assert.True}, + {"non-matching mail", es.Mails(All, All, "smarf"), assert.False}, + {"one of multiple mails", es.Mails(All, All, "smarf", mail), assert.True}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + scope := extendExchangeScopeValues(All, test.scope) + test.expect(t, scope.includesPath(ExchangeMail, path)) + }) + } +} + +func (suite *ExchangeSourceSuite) TestExchangeScope_ExcludesPath() { + const ( + usr = "userID" + fld = "mailFolder" + mail = "mailID" + ) + var ( + path = []string{"tid", usr, "mail", fld, mail} + es = NewExchangeRestore("rpid") + ) + + table := []struct { + name string + scope exchangeScope + expect assert.BoolAssertionFunc + }{ + {"all user's items", es.Users(All), assert.True}, + {"no user's items", es.Users(None), assert.False}, + {"matching user", es.Users(usr), assert.True}, + {"non-maching user", es.Users("smarf"), assert.False}, + {"one of multiple users", es.Users("smarf", usr), assert.True}, + {"all folders", es.MailFolders(None, All), assert.True}, + {"no folders", es.MailFolders(None, None), assert.False}, + {"matching folder", es.MailFolders(None, fld), assert.True}, + {"non-matching folder", es.MailFolders(None, "smarf"), assert.False}, + {"one of multiple folders", es.MailFolders(None, "smarf", fld), assert.True}, + {"all mail", es.Mails(None, None, All), assert.True}, + {"no mail", es.Mails(None, None, None), assert.False}, + {"matching mail", es.Mails(None, None, mail), assert.True}, + {"non-matching mail", es.Mails(None, None, "smarf"), assert.False}, + {"one of multiple mails", es.Mails(None, None, "smarf", mail), assert.True}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + scope := extendExchangeScopeValues(None, test.scope) + test.expect(t, scope.excludesPath(ExchangeMail, path)) + }) + } +} + +func (suite *ExchangeSourceSuite) TestIdPath() { + table := []struct { + cat exchangeCategory + path []string + expect map[exchangeCategory]string + }{ + { + ExchangeContact, + []string{"tid", "uid", "contact", "cFld", "cid"}, + map[exchangeCategory]string{ + ExchangeUser: "uid", + ExchangeContactFolder: "cFld", + ExchangeContact: "cid", + }, + }, + { + ExchangeEvent, + []string{"tid", "uid", "event", "eid"}, + map[exchangeCategory]string{ + ExchangeUser: "uid", + ExchangeEvent: "eid", + }, + }, + { + ExchangeMail, + []string{"tid", "uid", "mail", "mFld", "mid"}, + map[exchangeCategory]string{ + ExchangeUser: "uid", + ExchangeMailFolder: "mFld", + ExchangeMail: "mid", + }, + }, + { + ExchangeCategoryUnknown, + []string{"tid", "uid", "contact", "cFld", "cid"}, + map[exchangeCategory]string{ + ExchangeUser: "uid", + }, + }, + } + for _, test := range table { + suite.T().Run(test.cat.String(), func(t *testing.T) {}) + } +} + +func (suite *ExchangeSourceSuite) TestExchangeRestore_FilterDetails() { + makeDeets := func(refs ...string) *backup.Details { + deets := &backup.Details{ + Entries: []backup.DetailsEntry{}, + } + for _, r := range refs { + deets.Entries = append(deets.Entries, backup.DetailsEntry{ + RepoRef: r, + }) + } + return deets + } + const ( + contact = "tid/uid/contact/cfld/cid" + event = "tid/uid/event/eid" + mail = "tid/uid/mail/mfld/mid" + ) + table := []struct { + name string + deets *backup.Details + makeSelector func() *ExchangeRestore + expect []string + }{ + { + "no refs", + makeDeets(), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + return er + }, + []string{}, + }, + { + "contact only", + makeDeets(contact), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + return er + }, + []string{contact}, + }, + { + "event only", + makeDeets(event), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + return er + }, + []string{event}, + }, + { + "mail only", + makeDeets(mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + return er + }, + []string{mail}, + }, + { + "all", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + return er + }, + []string{contact, event, mail}, + }, + { + "only match contact", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Contacts("uid", "cfld", "cid")) + return er + }, + []string{contact}, + }, + { + "only match event", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Events("uid", "eid")) + return er + }, + []string{event}, + }, + { + "only match mail", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Mails("uid", "mfld", "mid")) + return er + }, + []string{mail}, + }, + { + "exclude contact", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + er.Exclude(er.Contacts("uid", "cfld", "cid")) + return er + }, + []string{event, mail}, + }, + { + "exclude event", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + er.Exclude(er.Events("uid", "eid")) + return er + }, + []string{contact, mail}, + }, + { + "exclude mail", + makeDeets(contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore("rpid") + er.Include(er.Users(All)) + er.Exclude(er.Mails("uid", "mfld", "mid")) + return er + }, + []string{contact, event}, + }, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + sel := test.makeSelector() + results := sel.FilterDetails(test.deets) + assert.Equal(t, test.expect, results) + }) + } +} + +func (suite *ExchangeSourceSuite) TestExchangeScopesByCategory() { + var ( + es = NewExchangeRestore("rpid") + users = es.Users(All) + contacts = es.ContactFolders(All, All) + events = es.Events(All, All) + mail = es.MailFolders(All, All) + ) + type expect struct { + contact int + event int + mail int + } + type input []map[string]string + table := []struct { + name string + scopes input + expect expect + }{ + {"users: one of each", input{users}, expect{1, 1, 1}}, + {"contacts only", input{contacts}, expect{1, 0, 0}}, + {"events only", input{events}, expect{0, 1, 0}}, + {"mail only", input{mail}, expect{0, 0, 1}}, + {"all", input{users, contacts, events, mail}, expect{2, 2, 2}}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + result := exchangeScopesByCategory(test.scopes) + assert.Equal(t, test.expect.contact, len(result[ExchangeContact.String()])) + assert.Equal(t, test.expect.event, len(result[ExchangeEvent.String()])) + assert.Equal(t, test.expect.mail, len(result[ExchangeMail.String()])) + }) + } +} + +func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() { + const ( + mail = "mailID" + cat = ExchangeMail + ) + include := func(s map[string]string) exchangeScope { + return extendExchangeScopeValues(All, exchangeScope(s)) + } + exclude := func(s map[string]string) exchangeScope { + return extendExchangeScopeValues(None, exchangeScope(s)) + } + var ( + es = NewExchangeRestore("rpid") + inAll = include(es.Users(All)) + inNone = include(es.Users(None)) + inMail = include(es.Mails(All, All, mail)) + inOtherMail = include(es.Mails(All, All, "smarf")) + exAll = exclude(es.Users(All)) + exNone = exclude(es.Users(None)) + exMail = exclude(es.Mails(None, None, mail)) + exOtherMail = exclude(es.Mails(None, None, "smarf")) + path = []string{"tid", "user", "mail", "folder", mail} + ) + + table := []struct { + name string + includes []exchangeScope + excludes []exchangeScope + expect assert.BoolAssertionFunc + }{ + {"empty", []exchangeScope{}, []exchangeScope{}, assert.False}, + {"in all", []exchangeScope{inAll}, []exchangeScope{}, assert.True}, + {"in None", []exchangeScope{inNone}, []exchangeScope{}, assert.False}, + {"in Mail", []exchangeScope{inMail}, []exchangeScope{}, assert.True}, + {"in Other", []exchangeScope{inOtherMail}, []exchangeScope{}, assert.False}, + {"ex all", []exchangeScope{inAll}, []exchangeScope{exAll}, assert.False}, + {"ex None", []exchangeScope{inAll}, []exchangeScope{exNone}, assert.True}, + {"in Mail", []exchangeScope{inAll}, []exchangeScope{exMail}, assert.False}, + {"in Other", []exchangeScope{inAll}, []exchangeScope{exOtherMail}, assert.True}, + {"in and ex mail", []exchangeScope{inMail}, []exchangeScope{exMail}, assert.False}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + test.expect(t, matchExchangeEntry(cat, path, test.includes, test.excludes)) + }) + } +}