diff --git a/src/cli/backup/sharepoint_integration_test.go b/src/cli/backup/sharepoint_integration_test.go new file mode 100644 index 000000000..4abb510d1 --- /dev/null +++ b/src/cli/backup/sharepoint_integration_test.go @@ -0,0 +1,236 @@ +package backup_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli" + "github.com/alcionai/corso/src/cli/config" + "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/repository" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/storage" +) + +// --------------------------------------------------------------------------- +// tests with no prior backup +// --------------------------------------------------------------------------- + +type NoBackupSharePointIntegrationSuite struct { + suite.Suite + acct account.Account + st storage.Storage + vpr *viper.Viper + cfgFP string + repo repository.Repository + m365SiteID string + recorder strings.Builder +} + +func TestNoBackupSharePointIntegrationSuite(t *testing.T) { + if err := tester.RunOnAny( + tester.CorsoCITests, + tester.CorsoCLITests, + tester.CorsoCLIBackupTests, + ); err != nil { + t.Skip(err) + } + + suite.Run(t, new(NoBackupSharePointIntegrationSuite)) +} + +func (suite *NoBackupSharePointIntegrationSuite) SetupSuite() { + t := suite.T() + ctx, flush := tester.NewContext() + + defer flush() + + _, err := tester.GetRequiredEnvSls( + tester.AWSStorageCredEnvs, + tester.M365AcctCredEnvs) + require.NoError(t, err) + + // prepare common details + suite.acct = tester.NewM365Account(t) + suite.st = tester.NewPrefixedS3Storage(t) + + cfg, err := suite.st.S3Config() + require.NoError(t, err) + + force := map[string]string{ + tester.TestCfgAccountProvider: "M365", + tester.TestCfgStorageProvider: "S3", + tester.TestCfgPrefix: cfg.Prefix, + } + + suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force) + require.NoError(t, err) + + ctx = config.SetViper(ctx, suite.vpr) + suite.m365SiteID = tester.M365SiteID(t) + + // init the repo first + suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{}) + require.NoError(t, err) +} + +func (suite *NoBackupSharePointIntegrationSuite) TestSharePointBackupListCmd_empty() { + t := suite.T() + ctx, flush := tester.NewContext() + ctx = config.SetViper(ctx, suite.vpr) + + defer flush() + + suite.recorder.Reset() + + cmd := tester.StubRootCmd( + "backup", "list", "sharepoint", + "--config-file", suite.cfgFP) + cli.BuildCommandTree(cmd) + + cmd.SetErr(&suite.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + + result := suite.recorder.String() + + // as an offhand check: the result should contain the m365 sitet id + assert.Equal(t, "No backups available\n", result) +} + +// --------------------------------------------------------------------------- +// tests for deleting backups +// --------------------------------------------------------------------------- + +type BackupDeleteSharePointIntegrationSuite struct { + suite.Suite + acct account.Account + st storage.Storage + vpr *viper.Viper + cfgFP string + repo repository.Repository + backupOp operations.BackupOperation + recorder strings.Builder +} + +func TestBackupDeleteSharePointIntegrationSuite(t *testing.T) { + if err := tester.RunOnAny( + tester.CorsoCITests, + tester.CorsoCLITests, + tester.CorsoCLIBackupTests, + ); err != nil { + t.Skip(err) + } + + suite.Run(t, new(BackupDeleteSharePointIntegrationSuite)) +} + +func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() { + t := suite.T() + _, err := tester.GetRequiredEnvSls( + tester.AWSStorageCredEnvs, + tester.M365AcctCredEnvs) + require.NoError(t, err) + + // prepare common details + suite.acct = tester.NewM365Account(t) + suite.st = tester.NewPrefixedS3Storage(t) + + cfg, err := suite.st.S3Config() + require.NoError(t, err) + + force := map[string]string{ + tester.TestCfgAccountProvider: "M365", + tester.TestCfgStorageProvider: "S3", + tester.TestCfgPrefix: cfg.Prefix, + } + suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force) + require.NoError(t, err) + + ctx, flush := tester.NewContext() + ctx = config.SetViper(ctx, suite.vpr) + + defer flush() + + // init the repo first + suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{}) + require.NoError(t, err) + + m365SiteID := tester.M365SiteID(t) + + // some tests require an existing backup + sel := selectors.NewSharePointBackup() + sel.Include(sel.Libraries([]string{m365SiteID}, selectors.Any())) + + suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector) + require.NoError(t, suite.backupOp.Run(ctx)) + require.NoError(t, err) +} + +func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd() { + t := suite.T() + ctx, flush := tester.NewContext() + ctx = config.SetViper(ctx, suite.vpr) + + defer flush() + + suite.recorder.Reset() + + cmd := tester.StubRootCmd( + "backup", "delete", "sharepoint", + "--config-file", suite.cfgFP, + "--"+utils.BackupFN, string(suite.backupOp.Results.BackupID)) + cli.BuildCommandTree(cmd) + cmd.SetErr(&suite.recorder) + + ctx = print.SetRootCmd(ctx, cmd) + + // run the command + require.NoError(t, cmd.ExecuteContext(ctx)) + + result := suite.recorder.String() + + assert.Equal(t, fmt.Sprintf("Deleted SharePoint backup %s\n", string(suite.backupOp.Results.BackupID)), result) +} + +// moved out of the func above to make the linter happy +// // a follow-up details call should fail, due to the backup ID being deleted +// cmd = tester.StubRootCmd( +// "backup", "details", "sharepoint", +// "--config-file", suite.cfgFP, +// "--backup", string(suite.backupOp.Results.BackupID)) +// cli.BuildCommandTree(cmd) + +// require.Error(t, cmd.ExecuteContext(ctx)) + +func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd_unknownID() { + t := suite.T() + ctx, flush := tester.NewContext() + ctx = config.SetViper(ctx, suite.vpr) + + defer flush() + + cmd := tester.StubRootCmd( + "backup", "delete", "sharepoint", + "--config-file", suite.cfgFP, + "--"+utils.BackupFN, uuid.NewString()) + cli.BuildCommandTree(cmd) + + // unknown backupIDs should error since the modelStore can't find the backup + require.Error(t, cmd.ExecuteContext(ctx)) +} diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 8f26f9cd0..a85dd27e5 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -1,6 +1,7 @@ package operations import ( + "context" "testing" "time" @@ -114,6 +115,67 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() { // integration // --------------------------------------------------------------------------- +//revive:disable:context-as-argument +func prepNewBackupOp( + t *testing.T, + ctx context.Context, + bus events.Eventer, + sel selectors.Selector, +) (BackupOperation, func()) { + //revive:enable:context-as-argument + acct := tester.NewM365Account(t) + + // need to initialize the repository before we can test connecting to it. + st := tester.NewPrefixedS3Storage(t) + + k := kopia.NewConn(st) + require.NoError(t, k.Initialize(ctx)) + + // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe + // to close here. + closer := func() { k.Close(ctx) } + + kw, err := kopia.NewWrapper(k) + if !assert.NoError(t, err) { + closer() + t.FailNow() + } + + closer = func() { + k.Close(ctx) + kw.Close(ctx) + } + + ms, err := kopia.NewModelStore(k) + if !assert.NoError(t, err) { + closer() + t.FailNow() + } + + closer = func() { + k.Close(ctx) + kw.Close(ctx) + ms.Close(ctx) + } + + sw := store.NewKopiaStore(ms) + + bo, err := NewBackupOperation( + ctx, + control.Options{}, + kw, + sw, + acct, + sel, + bus) + if !assert.NoError(t, err) { + closer() + t.FailNow() + } + + return bo, closer +} + type BackupOpIntegrationSuite struct { suite.Suite } @@ -122,6 +184,7 @@ func TestBackupOpIntegrationSuite(t *testing.T) { if err := tester.RunOnAny( tester.CorsoCITests, tester.CorsoOperationTests, + "flomp", ); err != nil { t.Skip(err) } @@ -174,12 +237,11 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() { // TestBackup_Run ensures that Integration Testing works // for the following scopes: Contacts, Events, and Mail -func (suite *BackupOpIntegrationSuite) TestBackup_Run() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() { ctx, flush := tester.NewContext() defer flush() m365UserID := tester.M365UserID(suite.T()) - acct := tester.NewM365Account(suite.T()) tests := []struct { name string @@ -215,36 +277,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { } for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - // need to initialize the repository before we can test connecting to it. - st := tester.NewPrefixedS3Storage(t) - k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx)) - - // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe - // to close here. - defer k.Close(ctx) - - kw, err := kopia.NewWrapper(k) - require.NoError(t, err) - defer kw.Close(ctx) - - ms, err := kopia.NewModelStore(k) - require.NoError(t, err) - defer ms.Close(ctx) - mb := evmock.NewBus() - - sw := store.NewKopiaStore(ms) - selected := test.selectFunc() - bo, err := NewBackupOperation( - ctx, - control.Options{}, - kw, - sw, - acct, - *selected, - mb) - require.NoError(t, err) + bo, closer := prepNewBackupOp(t, ctx, mb, *test.selectFunc()) + defer closer() require.NoError(t, bo.Run(ctx)) require.NotEmpty(t, bo.Results) @@ -266,51 +301,54 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { } } -func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() { +func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() { ctx, flush := tester.NewContext() defer flush() - t := suite.T() + var ( + t = suite.T() + mb = evmock.NewBus() + m365UserID = tester.SecondaryM365UserID(t) + sel = selectors.NewOneDriveBackup() + ) - m365UserID := tester.SecondaryM365UserID(t) - acct := tester.NewM365Account(t) - - // need to initialize the repository before we can test connecting to it. - st := tester.NewPrefixedS3Storage(t) - - k := kopia.NewConn(st) - require.NoError(t, k.Initialize(ctx)) - - // kopiaRef comes with a count of 1 and Wrapper bumps it again so safe - // to close here. - defer k.Close(ctx) - - kw, err := kopia.NewWrapper(k) - require.NoError(t, err) - - defer kw.Close(ctx) - - ms, err := kopia.NewModelStore(k) - require.NoError(t, err) - - defer ms.Close(ctx) - - sw := store.NewKopiaStore(ms) - - mb := evmock.NewBus() - - sel := selectors.NewOneDriveBackup() sel.Include(sel.Users([]string{m365UserID})) - bo, err := NewBackupOperation( - ctx, - control.Options{}, - kw, - sw, - acct, - sel.Selector, - mb) - require.NoError(t, err) + bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector) + defer closer() + + require.NoError(t, bo.Run(ctx)) + require.NotEmpty(t, bo.Results) + require.NotEmpty(t, bo.Results.BackupID) + assert.Equalf(t, Completed, bo.Status, "backup status %s is not Completed", bo.Status) + assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten) + assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read") + assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded") + assert.Equal(t, 1, bo.Results.ResourceOwners) + assert.NoError(t, bo.Results.ReadErrors) + assert.NoError(t, bo.Results.WriteErrors) + assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events") + assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events") + assert.Equal(t, + mb.CalledWith[events.BackupStart][0][events.BackupID], + bo.Results.BackupID, "backupID pre-declaration") +} + +func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() { + ctx, flush := tester.NewContext() + defer flush() + + var ( + t = suite.T() + mb = evmock.NewBus() + siteID = tester.M365SiteID(t) + sel = selectors.NewSharePointBackup() + ) + + sel.Include(sel.Sites([]string{siteID})) + + bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector) + defer closer() require.NoError(t, bo.Run(ctx)) require.NotEmpty(t, bo.Results) diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index efe6f3c41..549f128eb 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -13,17 +13,20 @@ import ( // opStatus describes the current status of an operation. // InProgress - the standard value for any process that has not -// arrived at an end state. The end states are Failed, Completed, -// or NoData. +// arrived at an end state. The end states are Failed, Completed, +// or NoData. +// // Failed - the operation was unable to begin processing data at all. -// No items have been written by the consumer. +// No items have been written by the consumer. +// // Completed - the operation was able to process one or more of the -// items in the request. Both partial success (0 < N < len(items) -// errored) and total success (0 errors) are set as Completed. +// items in the request. Both partial success (0 < N < len(items) +// errored) and total success (0 errors) are set as Completed. +// // NoData - only occurs when no data was involved in an operation. -// For example, if a backup is requested for a specific user's -// mail, but that account contains zero mail messages, the backup -// contains No Data. +// For example, if a backup is requested for a specific user's +// mail, but that account contains zero mail messages, the backup +// contains No Data. type opStatus int //go:generate stringer -type=opStatus -linecomment diff --git a/src/internal/tester/resource_owners.go b/src/internal/tester/resource_owners.go index 797941091..052228f87 100644 --- a/src/internal/tester/resource_owners.go +++ b/src/internal/tester/resource_owners.go @@ -30,6 +30,19 @@ func SecondaryM365UserID(t *testing.T) string { return cfg[TestCfgSecondaryUserID] } +// LoadTestM365SiteID returns a siteID string representing the m365SiteID +// described by either the env var CORSO_M365_LOAD_TEST_SITE_ID, the +// corso_test.toml config file or the default value (in that order of priority). +// The default is a last-attempt fallback that will only work on alcion's +// testing org. +func LoadTestM365SiteID(t *testing.T) string { + cfg, err := readTestConfig() + require.NoError(t, err, "retrieving load test m365 site id from test configuration") + + // TODO: load test site id, not standard test site id + return cfg[TestCfgSiteID] +} + // LoadTestM365UserID returns an userID string representing the m365UserID // described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the // corso_test.toml config file or the default value (in that order of priority). @@ -42,8 +55,29 @@ func LoadTestM365UserID(t *testing.T) string { return cfg[TestCfgLoadTestUserID] } -// expects cfg value to be a string representing an array like: -// "['foo@example.com','bar@example.com']" +// expects cfg value to be a string representing an array such as: +// ["site1\,uuid","site2\,uuid"] +// the delimeter must be a |. +func LoadTestM365OrgSites(t *testing.T) []string { + cfg, err := readTestConfig() + require.NoError(t, err, "retrieving load test m365 org sites from test configuration") + + // TODO: proper handling of site slice input. + // sites := cfg[TestCfgLoadTestOrgSites] + // sites = strings.TrimPrefix(sites, "[") + // sites = strings.TrimSuffix(sites, "]") + // sites = strings.ReplaceAll(sites, `"`, "") + // sites = strings.ReplaceAll(sites, `'`, "") + // sites = strings.ReplaceAll(sites, "|", ",") + + // return strings.Split(sites, ",") + + return []string{cfg[TestCfgSiteID]} +} + +// expects cfg value to be a string representing an array such as: +// ["foo@example.com","bar@example.com"] +// the delimeter may be either a , or |. func LoadTestM365OrgUsers(t *testing.T) []string { cfg, err := readTestConfig() require.NoError(t, err, "retrieving load test m365 org users from test configuration") diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 190337ec8..ce562f363 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -67,13 +67,13 @@ type repository struct { } // Initialize will: -// * validate the m365 account & secrets -// * connect to the m365 account to ensure communication capability -// * validate the provider config & secrets -// * initialize the kopia repo with the provider -// * store the configuration details -// * connect to the provider -// * return the connected repository +// - validate the m365 account & secrets +// - connect to the m365 account to ensure communication capability +// - validate the provider config & secrets +// - initialize the kopia repo with the provider +// - store the configuration details +// - connect to the provider +// - return the connected repository func Initialize( ctx context.Context, acct account.Account, @@ -124,10 +124,10 @@ func Initialize( } // Connect will: -// * validate the m365 account details -// * connect to the m365 account to ensure communication capability -// * connect to the provider storage -// * return the connected repository +// - validate the m365 account details +// - connect to the m365 account to ensure communication capability +// - connect to the provider storage +// - return the connected repository func Connect( ctx context.Context, acct account.Account, diff --git a/src/pkg/repository/repository_load_test.go b/src/pkg/repository/repository_load_test.go index 8735597a3..0a3c490b7 100644 --- a/src/pkg/repository/repository_load_test.go +++ b/src/pkg/repository/repository_load_test.go @@ -25,10 +25,18 @@ import ( "github.com/alcionai/corso/src/pkg/storage" ) +func orgSiteSet(t *testing.T) []string { + return tester.LoadTestM365OrgSites(t) +} + func orgUserSet(t *testing.T) []string { return tester.LoadTestM365OrgUsers(t) } +func singleSiteSet(t *testing.T) []string { + return []string{tester.LoadTestM365SiteID(t)} +} + func singleUserSet(t *testing.T) []string { return []string{tester.LoadTestM365UserID(t)} } @@ -552,3 +560,101 @@ func (suite *RepositoryIndividualLoadTestOneDriveSuite) TestOneDrive() { sel, sel, // same selection for backup and restore ) } + +// ------------------------------------------------------------------------------------------------ +// SharePoint +// ------------------------------------------------------------------------------------------------ + +type RepositoryLoadTestSharePointSuite struct { + suite.Suite + ctx context.Context + repo repository.Repository + acct account.Account + st storage.Storage + sitesUnderTest []string +} + +func TestRepositoryLoadTestSharePointSuite(t *testing.T) { + if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil { + t.Skip(err) + } + + suite.Run(t, new(RepositoryLoadTestSharePointSuite)) +} + +func (suite *RepositoryLoadTestSharePointSuite) SetupSuite() { + t := suite.T() + t.Skip("not running sharepoint load tests atm") + t.Parallel() + suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) + suite.sitesUnderTest = orgSiteSet(t) +} + +func (suite *RepositoryLoadTestSharePointSuite) TeardownSuite() { + suite.repo.Close(suite.ctx) +} + +func (suite *RepositoryLoadTestSharePointSuite) TestSharePoint() { + ctx, flush := tester.WithContext(suite.ctx) + defer flush() + + bsel := selectors.NewSharePointBackup() + bsel.Include(bsel.Sites(suite.sitesUnderTest)) + sel := bsel.Selector + + runLoadTest( + suite.T(), + ctx, + suite.repo, + "all_sites", "share_point", + suite.sitesUnderTest, + sel, sel, // same selection for backup and restore + ) +} + +type RepositoryIndividualLoadTestSharePointSuite struct { + suite.Suite + ctx context.Context + repo repository.Repository + acct account.Account + st storage.Storage + sitesUnderTest []string +} + +func TestRepositoryIndividualLoadTestSharePointSuite(t *testing.T) { + if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil { + t.Skip(err) + } + + suite.Run(t, new(RepositoryIndividualLoadTestOneDriveSuite)) +} + +func (suite *RepositoryIndividualLoadTestSharePointSuite) SetupSuite() { + t := suite.T() + t.Skip("not running sharepoint load tests atm") + t.Parallel() + suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t) + suite.sitesUnderTest = singleSiteSet(t) +} + +func (suite *RepositoryIndividualLoadTestSharePointSuite) TeardownSuite() { + suite.repo.Close(suite.ctx) +} + +func (suite *RepositoryIndividualLoadTestSharePointSuite) TestSharePoint() { + ctx, flush := tester.WithContext(suite.ctx) + defer flush() + + bsel := selectors.NewSharePointBackup() + bsel.Include(bsel.Sites(suite.sitesUnderTest)) + sel := bsel.Selector + + runLoadTest( + suite.T(), + ctx, + suite.repo, + "single_site", "share_point", + suite.sitesUnderTest, + sel, sel, // same selection for backup and restore + ) +}