From 6f7cf00188399c577ef03d872508a3ebb4241a55 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 9 Sep 2022 08:15:36 -0600 Subject: [PATCH] add cli integration tests for event/contact (#760) ## Description Adds foundational cli integration tests for backup/restore of events and calendars ## Type of change - [x] :robot: Test ## Issue(s) #501 ## Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- src/cli/backup/exchange.go | 14 +- src/cli/backup/exchange_integration_test.go | 189 ++++++++++++------- src/cli/backup/exchange_test.go | 2 +- src/cli/restore/exchange.go | 14 +- src/cli/restore/exchange_integration_test.go | 77 ++++++-- src/cli/restore/exchange_test.go | 2 +- src/internal/connector/graph_connector.go | 21 ++- src/pkg/selectors/exchange.go | 15 +- src/pkg/selectors/exchange_test.go | 47 +++-- src/pkg/selectors/scopes.go | 10 + 10 files changed, 265 insertions(+), 126 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 1990dd7a5..93768dcb9 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -444,19 +444,19 @@ func includeExchangeBackupDetailDataSelectors( lc, lcf := len(contacts), len(contactFolders) le, lef := len(emails), len(emailFolders) lev, lec := len(events), len(eventCalendars) - lu := len(users) - if lc+lcf+le+lef+lev+lec+lu == 0 { - return - } + // either scope the request to a set of users + if lc+lcf+le+lef+lev+lec == 0 { + if len(users) == 0 { + users = selectors.Any() + } - // if only users are provided, we only get one selector - if lu > 0 && lc+lcf+le+lef+lev+lec == 0 { sel.Include(sel.Users(users)) + return } - // otherwise, add selectors for each type of data + // or add selectors for each type of data includeExchangeContacts(sel, users, contactFolders, contacts) includeExchangeEmails(sel, users, emailFolders, email) includeExchangeEvents(sel, users, eventCalendars, events) diff --git a/src/cli/backup/exchange_integration_test.go b/src/cli/backup/exchange_integration_test.go index fb4e0b5f7..468116c53 100644 --- a/src/cli/backup/exchange_integration_test.go +++ b/src/cli/backup/exchange_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/path" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -24,6 +25,14 @@ import ( "github.com/alcionai/corso/src/pkg/storage" ) +var ( + email = path.EmailCategory + contacts = path.ContactsCategory + events = path.EventsCategory +) + +var backupDataSets = []path.CategoryType{email, contacts, events} + // --------------------------------------------------------------------------- // tests with no prior backup // --------------------------------------------------------------------------- @@ -69,6 +78,7 @@ func (suite *BackupExchangeIntegrationSuite) SetupSuite() { tester.TestCfgStorageProvider: "S3", tester.TestCfgPrefix: cfg.Prefix, } + suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force) require.NoError(t, err) @@ -81,29 +91,37 @@ func (suite *BackupExchangeIntegrationSuite) SetupSuite() { } func (suite *BackupExchangeIntegrationSuite) TestExchangeBackupCmd() { - ctx := config.SetViper(tester.NewContext(), suite.vpr) - t := suite.T() - - cmd := tester.StubRootCmd( - "backup", "create", "exchange", - "--config-file", suite.cfgFP, - "--user", suite.m365UserID, - "--data", "email") - cli.BuildCommandTree(cmd) - recorder := strings.Builder{} - cmd.SetOut(&recorder) - ctx = print.SetRootCmd(ctx, cmd) + for _, set := range backupDataSets { + recorder.Reset() - // run the command - require.NoError(t, cmd.ExecuteContext(ctx)) + suite.T().Run(set.String(), func(t *testing.T) { + ctx := config.SetViper(tester.NewContext(), suite.vpr) - // as an offhand check: the result should contain a string with the current hour - result := recorder.String() - assert.Contains(t, result, time.Now().UTC().Format("2006-01-02T15")) - // and the m365 user id - assert.Contains(t, result, suite.m365UserID) + cmd := tester.StubRootCmd( + "backup", "create", "exchange", + "--config-file", suite.cfgFP, + "--user", suite.m365UserID, + "--data", set.String()) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + + result := recorder.String() + t.Log("backup results", result) + + // as an offhand check: the result should contain a string with the current hour + assert.Contains(t, result, time.Now().UTC().Format("2006-01-02T15")) + // and the m365 user id + assert.Contains(t, result, suite.m365UserID) + }) + } } // --------------------------------------------------------------------------- @@ -118,7 +136,7 @@ type PreparedBackupExchangeIntegrationSuite struct { cfgFP string repo *repository.Repository m365UserID string - backupOp operations.BackupOperation + backupOps map[path.CategoryType]operations.BackupOperation } func TestPreparedBackupExchangeIntegrationSuite(t *testing.T) { @@ -162,68 +180,107 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() { suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st) require.NoError(t, err) - // some tests require an existing backup - sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"})) + suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) - suite.backupOp, err = suite.repo.NewBackup( - ctx, - sel.Selector, - control.NewOptions(false)) - require.NoError(t, suite.backupOp.Run(ctx)) - require.NoError(t, err) + for _, set := range backupDataSets { + var ( + sel = selectors.NewExchangeBackup() + scopes []selectors.ExchangeScope + ) + + switch set { + case email: + scopes = sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"}) + + case contacts: + scopes = sel.ContactFolders([]string{suite.m365UserID}, selectors.Any()) + + case events: + scopes = sel.EventCalendars([]string{suite.m365UserID}, selectors.Any()) + } + + sel.Include(scopes) + + bop, err := suite.repo.NewBackup( + ctx, + sel.Selector, + control.NewOptions(false)) + require.NoError(t, bop.Run(ctx)) + require.NoError(t, err) + + suite.backupOps[set] = bop + + // sanity check, ensure we can find the backup and its details immediately + _, err = suite.repo.Backup(ctx, bop.Results.BackupID) + require.NoError(t, err, "retrieving recent backup by ID") + _, _, err = suite.repo.BackupDetails(ctx, string(bop.Results.BackupID)) + require.NoError(t, err, "retrieving recent backup details by ID") + } } func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeListCmd() { - ctx := config.SetViper(tester.NewContext(), suite.vpr) - t := suite.T() - - cmd := tester.StubRootCmd( - "backup", "list", "exchange", - "--config-file", suite.cfgFP) - cli.BuildCommandTree(cmd) - recorder := strings.Builder{} - cmd.SetOut(&recorder) - ctx = print.SetRootCmd(ctx, cmd) + for _, set := range backupDataSets { + recorder.Reset() - // run the command - require.NoError(t, cmd.ExecuteContext(ctx)) + suite.T().Run(set.String(), func(t *testing.T) { + ctx := config.SetViper(tester.NewContext(), suite.vpr) - // compare the output - result := recorder.String() - assert.Contains(t, result, suite.backupOp.Results.BackupID) + cmd := tester.StubRootCmd( + "backup", "list", "exchange", + "--config-file", suite.cfgFP) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + + // compare the output + result := recorder.String() + assert.Contains(t, result, suite.backupOps[set].Results.BackupID) + }) + } } func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeDetailsCmd() { - ctx := config.SetViper(tester.NewContext(), suite.vpr) - t := suite.T() - - // fetch the details from the repo first - deets, _, err := suite.repo.BackupDetails(ctx, string(suite.backupOp.Results.BackupID)) - require.NoError(t, err) - - cmd := tester.StubRootCmd( - "backup", "details", "exchange", - "--config-file", suite.cfgFP, - "--backup", string(suite.backupOp.Results.BackupID)) - cli.BuildCommandTree(cmd) - recorder := strings.Builder{} - cmd.SetOut(&recorder) - ctx = print.SetRootCmd(ctx, cmd) + for _, set := range backupDataSets { + recorder.Reset() - // run the command - require.NoError(t, cmd.ExecuteContext(ctx)) + suite.T().Run(set.String(), func(t *testing.T) { + ctx := config.SetViper(tester.NewContext(), suite.vpr) + bID := suite.backupOps[set].Results.BackupID - // compare the output - result := recorder.String() + // fetch the details from the repo first + deets, _, err := suite.repo.BackupDetails(ctx, string(bID)) + require.NoError(t, err) - for i, ent := range deets.Entries { - t.Run(fmt.Sprintf("detail %d", i), func(t *testing.T) { - assert.Contains(t, result, ent.RepoRef) + cmd := tester.StubRootCmd( + "backup", "details", "exchange", + "--config-file", suite.cfgFP, + "--backup", string(bID)) + cli.BuildCommandTree(cmd) + + cmd.SetOut(&recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + + // compare the output + result := recorder.String() + + for i, ent := range deets.Entries { + t.Run(fmt.Sprintf("detail %d", i), func(t *testing.T) { + assert.Contains(t, result, ent.RepoRef) + }) + } }) } } diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index d6be3ca69..4ce8403b1 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -328,7 +328,7 @@ func (suite *ExchangeSuite) TestIncludeExchangeBackupDetailDataSelectors() { }{ { name: "no selectors", - expectIncludeLen: 0, + expectIncludeLen: 3, }, { name: "any users", diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index 0f869b028..fe9a0b241 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -242,19 +242,19 @@ func includeExchangeRestoreDataSelectors( lc, lcf := len(contacts), len(contactFolders) le, lef := len(emails), len(emailFolders) lev, lec := len(events), len(eventCalendars) - lu := len(users) - if lc+lcf+le+lef+lev+lec+lu == 0 { - return - } + // either scope the request to a set of users + if lc+lcf+le+lef+lev+lec == 0 { + if len(users) == 0 { + users = selectors.Any() + } - // if only users are provided, we only get one selector - if lu > 0 && lc+lcf+le+lef+lev+lec == 0 { sel.Include(sel.Users(users)) + return } - // otherwise, add selectors for each type of data + // or add selectors for each type of data includeExchangeContacts(sel, users, contactFolders, contacts) includeExchangeEmails(sel, users, emailFolders, email) includeExchangeEvents(sel, users, eventCalendars, events) diff --git a/src/cli/restore/exchange_integration_test.go b/src/cli/restore/exchange_integration_test.go index 7e3ca8359..d2e3a17cc 100644 --- a/src/cli/restore/exchange_integration_test.go +++ b/src/cli/restore/exchange_integration_test.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/cli" "github.com/alcionai/corso/src/cli/config" "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/path" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -18,6 +19,14 @@ import ( "github.com/alcionai/corso/src/pkg/storage" ) +var ( + email = path.EmailCategory + contacts = path.ContactsCategory + events = path.EventsCategory +) + +var backupDataSets = []path.CategoryType{email, contacts, events} + type RestoreExchangeIntegrationSuite struct { suite.Suite acct account.Account @@ -26,7 +35,7 @@ type RestoreExchangeIntegrationSuite struct { cfgFP string repo *repository.Repository m365UserID string - backupOp operations.BackupOperation + backupOps map[path.CategoryType]operations.BackupOperation } func TestRestoreExchangeIntegrationSuite(t *testing.T) { @@ -71,27 +80,57 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() { suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st) require.NoError(t, err) - // restoration requires an existing backup - sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"})) - suite.backupOp, err = suite.repo.NewBackup( - ctx, - sel.Selector, - control.NewOptions(false)) - require.NoError(t, suite.backupOp.Run(ctx)) - require.NoError(t, err) + suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) + + for _, set := range backupDataSets { + var ( + sel = selectors.NewExchangeBackup() + scopes []selectors.ExchangeScope + ) + + switch set { + case email: + scopes = sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"}) + + case contacts: + scopes = sel.ContactFolders([]string{suite.m365UserID}, selectors.Any()) + + case events: + scopes = sel.EventCalendars([]string{suite.m365UserID}, selectors.Any()) + } + + sel.Include(scopes) + + bop, err := suite.repo.NewBackup( + ctx, + sel.Selector, + control.NewOptions(false)) + require.NoError(t, bop.Run(ctx)) + require.NoError(t, err) + + suite.backupOps[set] = bop + + // sanity check, ensure we can find the backup and its details immediately + _, err = suite.repo.Backup(ctx, bop.Results.BackupID) + require.NoError(t, err, "retrieving recent backup by ID") + _, _, err = suite.repo.BackupDetails(ctx, string(bop.Results.BackupID)) + require.NoError(t, err, "retrieving recent backup details by ID") + } } func (suite *RestoreExchangeIntegrationSuite) TestExchangeRestoreCmd() { - ctx := config.SetViper(tester.NewContext(), suite.vpr) - t := suite.T() + for _, set := range backupDataSets { + suite.T().Run(set.String(), func(t *testing.T) { + ctx := config.SetViper(tester.NewContext(), suite.vpr) - cmd := tester.StubRootCmd( - "restore", "exchange", - "--config-file", suite.cfgFP, - "--backup", string(suite.backupOp.Results.BackupID)) - cli.BuildCommandTree(cmd) + cmd := tester.StubRootCmd( + "restore", "exchange", + "--config-file", suite.cfgFP, + "--backup", string(suite.backupOps[set].Results.BackupID)) + cli.BuildCommandTree(cmd) - // run the command - require.NoError(t, cmd.ExecuteContext(ctx)) + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + }) + } } diff --git a/src/cli/restore/exchange_test.go b/src/cli/restore/exchange_test.go index ad737669d..f522fc314 100644 --- a/src/cli/restore/exchange_test.go +++ b/src/cli/restore/exchange_test.go @@ -163,7 +163,7 @@ func (suite *ExchangeSuite) TestIncludeExchangeRestoreDataSelectors() { }{ { name: "no selectors", - expectIncludeLen: 0, + expectIncludeLen: 3, }, { name: "any users", diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index d615a1be1..09fdf9dc6 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -260,14 +260,29 @@ func (gc *GraphConnector) RestoreExchangeDataCollection( for _, dc := range dcs { var ( + user string + category path.CategoryType directory = strings.Join(dc.FullPath(), "") - user = dc.FullPath()[2] items = dc.Items() // TODO(ashmrtn): Update this when we have path struct support in collections. - category = path.ToCategoryType(dc.FullPath()[3]) - exit bool + exit bool ) + // email uses the new path format + category = path.ToCategoryType(dc.FullPath()[3]) + if category == path.UnknownCategory { + // events and calendar use the old path format + category = path.ToCategoryType(dc.FullPath()[2]) + } + + // get the user from the path index based on the path pattern. + switch category { + case path.EmailCategory: + user = dc.FullPath()[2] + case path.ContactsCategory, path.EventsCategory: + user = dc.FullPath()[1] + } + if _, ok := pathCounter[directory]; !ok { pathCounter[directory] = true folderID, errs = exchange.GetRestoreContainer(&gc.graphService, user, category) diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index 7de512d11..119c05194 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -555,22 +555,23 @@ func (ec exchangeCategory) unknownCat() categorizer { // => {exchUser: userPN, exchMailFolder: mailFolder, exchMail: mailID} func (ec exchangeCategory) pathValues(path []string) map[categorizer]string { m := map[categorizer]string{} - if len(path) < 6 { + if len(path) < 5 { return m } - m[ExchangeUser] = path[2] - switch ec { case ExchangeContact: - m[ExchangeContactFolder] = path[4] - m[ExchangeContact] = path[5] + m[ExchangeUser] = path[1] + m[ExchangeContactFolder] = path[3] + m[ExchangeContact] = path[4] case ExchangeEvent: - m[ExchangeEventCalendar] = path[4] - m[ExchangeEvent] = path[5] + m[ExchangeUser] = path[1] + m[ExchangeEventCalendar] = path[3] + m[ExchangeEvent] = path[4] case ExchangeMail: + m[ExchangeUser] = path[2] m[ExchangeMailFolder] = path[4] m[ExchangeMail] = path[5] } diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index cea6d1f10..aa5528b4e 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -1,6 +1,7 @@ package selectors import ( + "strings" "testing" "time" @@ -883,8 +884,11 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() { } var ( - contact = stubRepoRef(path.ExchangeService, path.ContactsCategory, "uid", "cfld", "cid") - event = stubRepoRef(path.ExchangeService, path.EventsCategory, "uid", "ecld", "eid") + // TODO: contacts and events currently do not comply with the mail path pattern + // contact = stubRepoRef(path.ExchangeService, path.ContactsCategory, "uid", "cfld", "cid") + // event = stubRepoRef(path.ExchangeService, path.EventsCategory, "uid", "ecld", "eid") + contact = strings.Join([]string{"tid", "uid", path.ContactsCategory.String(), "cfld", "cid"}, "/") + event = strings.Join([]string{"tid", "uid", path.EventsCategory.String(), "ecld", "eid"}, "/") mail = stubRepoRef(path.ExchangeService, path.EmailCategory, "uid", "mfld", "mid") ) @@ -1228,24 +1232,37 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_leafCat() { } func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() { - contactPath := stubPath(path.ExchangeService, path.ContactsCategory, "user", "cfolder", "contactitem") - contactMap := map[categorizer]string{ - ExchangeUser: contactPath[2], - ExchangeContactFolder: contactPath[4], - ExchangeContact: contactPath[5], - } - eventPath := stubPath(path.ExchangeService, path.EventsCategory, "user", "ecalendar", "eventitem") - eventMap := map[categorizer]string{ - ExchangeUser: eventPath[2], - ExchangeEventCalendar: eventPath[4], - ExchangeEvent: eventPath[5], - } + // TODO: currently events and contacts are non-compliant with email path patterns + // contactPath := stubPath(path.ExchangeService, path.ContactsCategory, "user", "cfolder", "contactitem") + // contactMap := map[categorizer]string{ + // ExchangeUser: contactPath[2], + // ExchangeContactFolder: contactPath[4], + // ExchangeContact: contactPath[5], + // } + // eventPath := stubPath(path.ExchangeService, path.EventsCategory, "user", "ecalendar", "eventitem") + // eventMap := map[categorizer]string{ + // ExchangeUser: eventPath[2], + // ExchangeEventCalendar: eventPath[4], + // ExchangeEvent: eventPath[5], + // } mailPath := stubPath(path.ExchangeService, path.EmailCategory, "user", "mfolder", "mailitem") mailMap := map[categorizer]string{ - ExchangeUser: contactPath[2], + ExchangeUser: mailPath[2], ExchangeMailFolder: mailPath[4], ExchangeMail: mailPath[5], } + contactPath := []string{"tid", "user", path.ContactsCategory.String(), "cfolder", "contactitem"} + contactMap := map[categorizer]string{ + ExchangeUser: contactPath[1], + ExchangeContactFolder: contactPath[3], + ExchangeContact: contactPath[4], + } + eventPath := []string{"tid", "user", path.EventsCategory.String(), "ecalendar", "eventitem"} + eventMap := map[categorizer]string{ + ExchangeUser: eventPath[1], + ExchangeEventCalendar: eventPath[3], + ExchangeEvent: eventPath[4], + } table := []struct { cat exchangeCategory diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index aaf9f403d..2f0a72236 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -300,6 +300,16 @@ func pathTypeIn(p []string) pathType { return exchangeEventPath } + // fallback for unmigrated events and contacts paths + switch p[2] { + case path.EmailCategory.String(): + return exchangeMailPath + case path.ContactsCategory.String(): + return exchangeContactPath + case path.EventsCategory.String(): + return exchangeEventPath + } + return unknownPathType }