From c29d93e655f219f8a66d2a6094251734143992b9 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 1 Sep 2022 09:39:14 -0600 Subject: [PATCH] adds calendars as a folder structure to events (#714) ## Description Exchange events were amended to use calendars as a folder structure. This updates the selector to treat events as having folders similar to mail and contacts. ## Type of change Please check the type of change your PR introduces: - [x] :sunflower: Feature ## Issue(s) #501 ## Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/backup/exchange.go | 10 +-- src/cli/restore/exchange.go | 6 +- .../connector/graph_connector_test.go | 2 +- src/internal/operations/backup_test.go | 2 +- src/pkg/selectors/exchange.go | 76 ++++++++--------- src/pkg/selectors/exchange_test.go | 83 +++++++++++++++---- src/pkg/selectors/helpers_test.go | 2 +- 7 files changed, 114 insertions(+), 67 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 3bf5f6891..3eb5b2ec3 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -1,8 +1,6 @@ package backup import ( - "fmt" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -222,7 +220,7 @@ func exchangeBackupCreateSelectors(all bool, users, data []string) selectors.Sel if len(data) == 0 { sel.Include(sel.ContactFolders(user, selectors.Any())) sel.Include(sel.MailFolders(user, selectors.Any())) - sel.Include(sel.Events(user, selectors.Any())) + sel.Include(sel.EventCalendars(user, selectors.Any())) } for _, d := range data { @@ -232,7 +230,7 @@ func exchangeBackupCreateSelectors(all bool, users, data []string) selectors.Sel case dataEmail: sel.Include(sel.MailFolders(users, selectors.Any())) case dataEvents: - sel.Include(sel.Events(users, selectors.Any())) + sel.Include(sel.EventCalendars(users, selectors.Any())) } } @@ -441,9 +439,7 @@ func includeExchangeEvents(sel *selectors.ExchangeRestore, users, eventCalendars events = selectors.Any() } - fmt.Println("logging eventCalendars so the linter sees it 'used'. Will remove asap", eventCalendars) - - sel.Include(sel.Events(users, events)) + sel.Include(sel.Events(users, eventCalendars, events)) } // builds the info-selector filters for `backup details exchange` diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index b8fae05c4..15c704b74 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -1,8 +1,6 @@ package restore import ( - "fmt" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -240,9 +238,7 @@ func includeExchangeEvents(sel *selectors.ExchangeRestore, users, eventCalendars events = selectors.Any() } - fmt.Println("logging eventCalendars so the linter sees it 'used'. Will remove asap", eventCalendars) - - sel.Include(sel.Events(users, events)) + sel.Include(sel.Events(users, eventCalendars, events)) } // builds the info-selector filters for `restore exchange` diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 9d3921237..45b0647c9 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -213,7 +213,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() t := suite.T() connector := loadConnector(t) sel := selectors.NewExchangeBackup() - sel.Include(sel.Events([]string{suite.user}, selectors.Any())) + sel.Include(sel.EventCalendars([]string{suite.user}, selectors.Any())) scopes := sel.Scopes() suite.Equal(len(scopes), 1) collections, err := connector.createCollections(context.Background(), scopes[0]) diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 16ad4459c..d0b45e35e 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -159,7 +159,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { name: "Integration Exchange.Events", selectFunc: func() *selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.Events([]string{m365UserID}, selectors.Any())) + sel.Include(sel.EventCalendars([]string{m365UserID}, selectors.Any())) return &sel.Selector }, diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index f2be5b4fe..a7466ec38 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -160,8 +160,7 @@ func (s *exchange) DiscreteScopes(userPNs []string) []ExchangeScope { // ------------------- // Scope Factories -// Produces one or more exchange contact scopes. -// One scope is created per combination of users,folders,contacts. +// Contacts produces one or more exchange contact scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -177,8 +176,7 @@ func (s *exchange) Contacts(users, folders, contacts []string) []ExchangeScope { return scopes } -// Produces one or more exchange contact folder scopes. -// One scope is created per combination of users,folders. +// Contactfolders produces one or more exchange contact folder scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -193,24 +191,39 @@ func (s *exchange) ContactFolders(users, folders []string) []ExchangeScope { return scopes } -// Produces one or more exchange event scopes. -// One scope is created per combination of users,events. +// Events produces one or more exchange event scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] -func (s *exchange) Events(users, events []string) []ExchangeScope { +func (s *exchange) Events(users, calendars, events []string) []ExchangeScope { scopes := []ExchangeScope{} scopes = append( scopes, - makeScope[ExchangeScope](Item, ExchangeEvent, users, events), + makeScope[ExchangeScope](Item, ExchangeEvent, users, events). + set(ExchangeEventCalendar, calendars), + ) + + return scopes +} + +// EventCalendars produces one or more exchange event calendar scopes. +// Calendars act as folders to contain Events +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (s *exchange) EventCalendars(users, events []string) []ExchangeScope { + scopes := []ExchangeScope{} + + scopes = append( + scopes, + makeScope[ExchangeScope](Item, ExchangeEventCalendar, users, events), ) return scopes } // Produces one or more mail scopes. -// One scope is created per combination of users,folders,mails. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -227,7 +240,6 @@ func (s *exchange) Mails(users, folders, mails []string) []ExchangeScope { } // Produces one or more exchange mail folder scopes. -// One scope is created per combination of users,folders. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -243,7 +255,7 @@ func (s *exchange) MailFolders(users, folders []string) []ExchangeScope { } // Produces one or more exchange contact user scopes. -// One scope is created per user entry. +// Each user id generates three scopes, one for each data type: contact, event, and mail. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -252,7 +264,7 @@ func (s *exchange) Users(users []string) []ExchangeScope { scopes = append(scopes, makeScope[ExchangeScope](Group, ExchangeContactFolder, users, Any()), - makeScope[ExchangeScope](Item, ExchangeEvent, users, Any()), + makeScope[ExchangeScope](Item, ExchangeEventCalendar, users, Any()), makeScope[ExchangeScope](Group, ExchangeMailFolder, users, Any()), ) @@ -448,6 +460,7 @@ const ( ExchangeContact exchangeCategory = "ExchangeContact" ExchangeContactFolder exchangeCategory = "ExchangeContactFolder" ExchangeEvent exchangeCategory = "ExchangeEvent" + ExchangeEventCalendar exchangeCategory = "ExchangeEventCalendar" ExchangeMail exchangeCategory = "ExchangeMail" ExchangeMailFolder exchangeCategory = "ExchangeFolder" ExchangeUser exchangeCategory = "ExchangeUser" @@ -471,7 +484,7 @@ const ( // these types appear in the canonical Path for each type. var exchangePathSet = map[categorizer][]categorizer{ ExchangeContact: {ExchangeUser, ExchangeContactFolder, ExchangeContact}, - ExchangeEvent: {ExchangeUser, ExchangeEvent}, + ExchangeEvent: {ExchangeUser, ExchangeEventCalendar, ExchangeEvent}, ExchangeMail: {ExchangeUser, ExchangeMailFolder, ExchangeMail}, ExchangeUser: {ExchangeUser}, // the root category must be represented } @@ -492,9 +505,11 @@ func (ec exchangeCategory) leafCat() categorizer { switch ec { case ExchangeContact, ExchangeContactFolder, ExchangeFilterContactName: return ExchangeContact - case ExchangeEvent, ExchangeFilterEventRecurs, ExchangeFilterEventStartsAfter, - ExchangeFilterEventStartsBefore, ExchangeFilterEventSubject: + + case ExchangeEvent, ExchangeEventCalendar, ExchangeFilterEventRecurs, + ExchangeFilterEventStartsAfter, ExchangeFilterEventStartsBefore, ExchangeFilterEventSubject: return ExchangeEvent + case ExchangeMail, ExchangeMailFolder, ExchangeFilterMailReceivedAfter, ExchangeFilterMailReceivedBefore, ExchangeFilterMailSender, ExchangeFilterMailSubject: return ExchangeMail @@ -524,40 +539,22 @@ func (ec exchangeCategory) unknownCat() categorizer { // => {exchUser: userPN, exchMailFolder: mailFolder, exchMail: mailID} func (ec exchangeCategory) pathValues(path []string) map[categorizer]string { m := map[categorizer]string{} - if len(path) < 2 { + if len(path) < 5 { return m } m[ExchangeUser] = path[1] - /* - TODO/Notice: - Mail and Contacts contain folder structures, identified - in this code as being at index 3. This assumes a single - folder, while in reality users can express subfolder - hierarchies of arbirary depth. Subfolder handling is coming - at a later time. - */ + switch ec { case ExchangeContact: - if len(path) < 5 { - return m - } - m[ExchangeContactFolder] = path[3] m[ExchangeContact] = path[4] case ExchangeEvent: - if len(path) < 4 { - return m - } - - m[ExchangeEvent] = path[3] + m[ExchangeEventCalendar] = path[3] + m[ExchangeEvent] = path[4] case ExchangeMail: - if len(path) < 5 { - return m - } - m[ExchangeMailFolder] = path[3] m[ExchangeMail] = path[4] } @@ -641,8 +638,13 @@ func (s ExchangeScope) setDefaults() { switch s.Category() { case ExchangeContactFolder: s[ExchangeContact.String()] = passAny + + case ExchangeEventCalendar: + s[ExchangeEvent.String()] = passAny + case ExchangeMailFolder: s[ExchangeMail.String()] = passAny + case ExchangeUser: s[ExchangeContactFolder.String()] = passAny s[ExchangeContact.String()] = passAny diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index 4b05284ee..4d2cac525 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -169,9 +169,10 @@ func (suite *ExchangeSelectorSuite) TestExchangeSelector_Exclude_Events() { user = "user" e1 = "e1" e2 = "e2" + c1 = "c1" ) - sel.Exclude(sel.Events([]string{user}, []string{e1, e2})) + sel.Exclude(sel.Events([]string{user}, []string{c1}, []string{e1, e2})) scopes := sel.Excludes require.Len(t, scopes, 1) @@ -179,8 +180,34 @@ func (suite *ExchangeSelectorSuite) TestExchangeSelector_Exclude_Events() { t, ExchangeScope(scopes[0]), map[categorizer]string{ - ExchangeUser: user, - ExchangeEvent: join(e1, e2), + ExchangeUser: user, + ExchangeEventCalendar: c1, + ExchangeEvent: join(e1, e2), + }, + ) +} + +func (suite *ExchangeSelectorSuite) TestExchangeSelector_Exclude_EventCalendars() { + t := suite.T() + sel := NewExchangeBackup() + + const ( + user = "user" + c1 = "c1" + c2 = "c2" + ) + + sel.Exclude(sel.EventCalendars([]string{user}, []string{c1, c2})) + scopes := sel.Excludes + require.Len(t, scopes, 1) + + scopeMustHave( + t, + ExchangeScope(scopes[0]), + map[categorizer]string{ + ExchangeUser: user, + ExchangeEventCalendar: join(c1, c2), + ExchangeEvent: AnyTgt, }, ) } @@ -193,9 +220,10 @@ func (suite *ExchangeSelectorSuite) TestExchangeSelector_Include_Events() { user = "user" e1 = "e1" e2 = "e2" + c1 = "c1" ) - sel.Include(sel.Events([]string{user}, []string{e1, e2})) + sel.Include(sel.Events([]string{user}, []string{c1}, []string{e1, e2})) scopes := sel.Includes require.Len(t, scopes, 1) @@ -203,12 +231,36 @@ func (suite *ExchangeSelectorSuite) TestExchangeSelector_Include_Events() { t, ExchangeScope(scopes[0]), map[categorizer]string{ - ExchangeUser: user, - ExchangeEvent: join(e1, e2), + ExchangeUser: user, + ExchangeEventCalendar: c1, + ExchangeEvent: join(e1, e2), }, ) +} - assert.Equal(t, sel.Scopes()[0].Category(), ExchangeEvent) +func (suite *ExchangeSelectorSuite) TestExchangeSelector_Include_EventCalendars() { + t := suite.T() + sel := NewExchangeBackup() + + const ( + user = "user" + c1 = "c1" + c2 = "c2" + ) + + sel.Include(sel.EventCalendars([]string{user}, []string{c1, c2})) + scopes := sel.Includes + require.Len(t, scopes, 1) + + scopeMustHave( + t, + ExchangeScope(scopes[0]), + map[categorizer]string{ + ExchangeUser: user, + ExchangeEventCalendar: join(c1, c2), + ExchangeEvent: AnyTgt, + }, + ) } func (suite *ExchangeSelectorSuite) TestExchangeSelector_Exclude_Mails() { @@ -824,7 +876,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() { const ( contact = "tid/uid/contact/cfld/cid" - event = "tid/uid/event/eid" + event = "tid/uid/event/ecld/eid" mail = "tid/uid/mail/mfld/mid" ) @@ -903,7 +955,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() { makeDeets(contact, event, mail), func() *ExchangeRestore { er := NewExchangeRestore() - er.Include(er.Events([]string{"uid"}, []string{"eid"})) + er.Include(er.Events([]string{"uid"}, []string{"ecld"}, []string{"eid"})) return er }, arr(event), @@ -935,7 +987,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() { func() *ExchangeRestore { er := NewExchangeRestore() er.Include(er.Users(Any())) - er.Exclude(er.Events([]string{"uid"}, []string{"eid"})) + er.Exclude(er.Events([]string{"uid"}, []string{"ecld"}, []string{"eid"})) return er }, arr(contact, mail), @@ -967,7 +1019,7 @@ func (suite *ExchangeSelectorSuite) TestScopesByCategory() { es = NewExchangeRestore() users = es.Users(Any()) contacts = es.ContactFolders(Any(), Any()) - events = es.Events(Any(), Any()) + events = es.Events(Any(), Any(), Any()) mail = es.MailFolders(Any(), Any()) ) @@ -1174,10 +1226,11 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() { ExchangeContactFolder: contactPath[3], ExchangeContact: contactPath[4], } - eventPath := []string{"ten", "user", "event", "eventitem"} + eventPath := []string{"ten", "user", "event", "ecalendar", "eventitem"} eventMap := map[categorizer]string{ - ExchangeUser: eventPath[1], - ExchangeEvent: eventPath[3], + ExchangeUser: eventPath[1], + ExchangeEventCalendar: eventPath[3], + ExchangeEvent: eventPath[4], } mailPath := []string{"ten", "user", "mail", "mfolder", "mailitem"} mailMap := map[categorizer]string{ @@ -1206,7 +1259,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() { func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathKeys() { contact := []categorizer{ExchangeUser, ExchangeContactFolder, ExchangeContact} - event := []categorizer{ExchangeUser, ExchangeEvent} + event := []categorizer{ExchangeUser, ExchangeEventCalendar, ExchangeEvent} mail := []categorizer{ExchangeUser, ExchangeMailFolder, ExchangeMail} user := []categorizer{ExchangeUser} diff --git a/src/pkg/selectors/helpers_test.go b/src/pkg/selectors/helpers_test.go index a4883ce06..b29102ba9 100644 --- a/src/pkg/selectors/helpers_test.go +++ b/src/pkg/selectors/helpers_test.go @@ -136,7 +136,7 @@ func setScopesToDefault[T scopeT](ts []T) []T { func scopeMustHave[T scopeT](t *testing.T, sc T, m map[categorizer]string) { for k, v := range m { t.Run(k.String(), func(t *testing.T) { - assert.Equal(t, getCatValue(sc, k), split(v)) + assert.Equal(t, getCatValue(sc, k), split(v), "Key: %s", k) }) } }