add boilerplate integration tests for sharepoint (#1600)

## Type of change

- [x] 🤖 Test

## Issue(s)

* #1506

## Test Plan

- [x] 💚 E2E
This commit is contained in:
Keepers 2022-11-29 13:53:42 -07:00 committed by GitHub
parent ff9a174d52
commit f1ea464ad6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 508 additions and 91 deletions

View File

@ -0,0 +1,236 @@
package backup_test
import (
"fmt"
"strings"
"testing"
"github.com/google/uuid"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/storage"
)
// ---------------------------------------------------------------------------
// tests with no prior backup
// ---------------------------------------------------------------------------
type NoBackupSharePointIntegrationSuite struct {
suite.Suite
acct account.Account
st storage.Storage
vpr *viper.Viper
cfgFP string
repo repository.Repository
m365SiteID string
recorder strings.Builder
}
func TestNoBackupSharePointIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoCLITests,
tester.CorsoCLIBackupTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(NoBackupSharePointIntegrationSuite))
}
func (suite *NoBackupSharePointIntegrationSuite) SetupSuite() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
_, err := tester.GetRequiredEnvSls(
tester.AWSStorageCredEnvs,
tester.M365AcctCredEnvs)
require.NoError(t, err)
// prepare common details
suite.acct = tester.NewM365Account(t)
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
tester.TestCfgStorageProvider: "S3",
tester.TestCfgPrefix: cfg.Prefix,
}
suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force)
require.NoError(t, err)
ctx = config.SetViper(ctx, suite.vpr)
suite.m365SiteID = tester.M365SiteID(t)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
}
func (suite *NoBackupSharePointIntegrationSuite) TestSharePointBackupListCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext()
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "list", "sharepoint",
"--config-file", suite.cfgFP)
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
// as an offhand check: the result should contain the m365 sitet id
assert.Equal(t, "No backups available\n", result)
}
// ---------------------------------------------------------------------------
// tests for deleting backups
// ---------------------------------------------------------------------------
type BackupDeleteSharePointIntegrationSuite struct {
suite.Suite
acct account.Account
st storage.Storage
vpr *viper.Viper
cfgFP string
repo repository.Repository
backupOp operations.BackupOperation
recorder strings.Builder
}
func TestBackupDeleteSharePointIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoCLITests,
tester.CorsoCLIBackupTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(BackupDeleteSharePointIntegrationSuite))
}
func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() {
t := suite.T()
_, err := tester.GetRequiredEnvSls(
tester.AWSStorageCredEnvs,
tester.M365AcctCredEnvs)
require.NoError(t, err)
// prepare common details
suite.acct = tester.NewM365Account(t)
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
tester.TestCfgStorageProvider: "S3",
tester.TestCfgPrefix: cfg.Prefix,
}
suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force)
require.NoError(t, err)
ctx, flush := tester.NewContext()
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
m365SiteID := tester.M365SiteID(t)
// some tests require an existing backup
sel := selectors.NewSharePointBackup()
sel.Include(sel.Libraries([]string{m365SiteID}, selectors.Any()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))
require.NoError(t, err)
}
func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd() {
t := suite.T()
ctx, flush := tester.NewContext()
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
suite.recorder.Reset()
cmd := tester.StubRootCmd(
"backup", "delete", "sharepoint",
"--config-file", suite.cfgFP,
"--"+utils.BackupFN, string(suite.backupOp.Results.BackupID))
cli.BuildCommandTree(cmd)
cmd.SetErr(&suite.recorder)
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
assert.Equal(t, fmt.Sprintf("Deleted SharePoint backup %s\n", string(suite.backupOp.Results.BackupID)), result)
}
// moved out of the func above to make the linter happy
// // a follow-up details call should fail, due to the backup ID being deleted
// cmd = tester.StubRootCmd(
// "backup", "details", "sharepoint",
// "--config-file", suite.cfgFP,
// "--backup", string(suite.backupOp.Results.BackupID))
// cli.BuildCommandTree(cmd)
// require.Error(t, cmd.ExecuteContext(ctx))
func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd_unknownID() {
t := suite.T()
ctx, flush := tester.NewContext()
ctx = config.SetViper(ctx, suite.vpr)
defer flush()
cmd := tester.StubRootCmd(
"backup", "delete", "sharepoint",
"--config-file", suite.cfgFP,
"--"+utils.BackupFN, uuid.NewString())
cli.BuildCommandTree(cmd)
// unknown backupIDs should error since the modelStore can't find the backup
require.Error(t, cmd.ExecuteContext(ctx))
}

View File

@ -1,6 +1,7 @@
package operations package operations
import ( import (
"context"
"testing" "testing"
"time" "time"
@ -114,6 +115,67 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
// integration // integration
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
//revive:disable:context-as-argument
func prepNewBackupOp(
t *testing.T,
ctx context.Context,
bus events.Eventer,
sel selectors.Selector,
) (BackupOperation, func()) {
//revive:enable:context-as-argument
acct := tester.NewM365Account(t)
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx))
// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
// to close here.
closer := func() { k.Close(ctx) }
kw, err := kopia.NewWrapper(k)
if !assert.NoError(t, err) {
closer()
t.FailNow()
}
closer = func() {
k.Close(ctx)
kw.Close(ctx)
}
ms, err := kopia.NewModelStore(k)
if !assert.NoError(t, err) {
closer()
t.FailNow()
}
closer = func() {
k.Close(ctx)
kw.Close(ctx)
ms.Close(ctx)
}
sw := store.NewKopiaStore(ms)
bo, err := NewBackupOperation(
ctx,
control.Options{},
kw,
sw,
acct,
sel,
bus)
if !assert.NoError(t, err) {
closer()
t.FailNow()
}
return bo, closer
}
type BackupOpIntegrationSuite struct { type BackupOpIntegrationSuite struct {
suite.Suite suite.Suite
} }
@ -122,6 +184,7 @@ func TestBackupOpIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny( if err := tester.RunOnAny(
tester.CorsoCITests, tester.CorsoCITests,
tester.CorsoOperationTests, tester.CorsoOperationTests,
"flomp",
); err != nil { ); err != nil {
t.Skip(err) t.Skip(err)
} }
@ -174,12 +237,11 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
// TestBackup_Run ensures that Integration Testing works // TestBackup_Run ensures that Integration Testing works
// for the following scopes: Contacts, Events, and Mail // for the following scopes: Contacts, Events, and Mail
func (suite *BackupOpIntegrationSuite) TestBackup_Run() { func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() {
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
m365UserID := tester.M365UserID(suite.T()) m365UserID := tester.M365UserID(suite.T())
acct := tester.NewM365Account(suite.T())
tests := []struct { tests := []struct {
name string name string
@ -215,36 +277,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
} }
for _, test := range tests { for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx))
// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
// to close here.
defer k.Close(ctx)
kw, err := kopia.NewWrapper(k)
require.NoError(t, err)
defer kw.Close(ctx)
ms, err := kopia.NewModelStore(k)
require.NoError(t, err)
defer ms.Close(ctx)
mb := evmock.NewBus() mb := evmock.NewBus()
bo, closer := prepNewBackupOp(t, ctx, mb, *test.selectFunc())
sw := store.NewKopiaStore(ms) defer closer()
selected := test.selectFunc()
bo, err := NewBackupOperation(
ctx,
control.Options{},
kw,
sw,
acct,
*selected,
mb)
require.NoError(t, err)
require.NoError(t, bo.Run(ctx)) require.NoError(t, bo.Run(ctx))
require.NotEmpty(t, bo.Results) require.NotEmpty(t, bo.Results)
@ -266,51 +301,54 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
} }
} }
func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() { func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() {
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
t := suite.T() var (
t = suite.T()
mb = evmock.NewBus()
m365UserID = tester.SecondaryM365UserID(t)
sel = selectors.NewOneDriveBackup()
)
m365UserID := tester.SecondaryM365UserID(t)
acct := tester.NewM365Account(t)
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx))
// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
// to close here.
defer k.Close(ctx)
kw, err := kopia.NewWrapper(k)
require.NoError(t, err)
defer kw.Close(ctx)
ms, err := kopia.NewModelStore(k)
require.NoError(t, err)
defer ms.Close(ctx)
sw := store.NewKopiaStore(ms)
mb := evmock.NewBus()
sel := selectors.NewOneDriveBackup()
sel.Include(sel.Users([]string{m365UserID})) sel.Include(sel.Users([]string{m365UserID}))
bo, err := NewBackupOperation( bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector)
ctx, defer closer()
control.Options{},
kw, require.NoError(t, bo.Run(ctx))
sw, require.NotEmpty(t, bo.Results)
acct, require.NotEmpty(t, bo.Results.BackupID)
sel.Selector, assert.Equalf(t, Completed, bo.Status, "backup status %s is not Completed", bo.Status)
mb) assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten)
require.NoError(t, err) assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read")
assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded")
assert.Equal(t, 1, bo.Results.ResourceOwners)
assert.NoError(t, bo.Results.ReadErrors)
assert.NoError(t, bo.Results.WriteErrors)
assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events")
assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events")
assert.Equal(t,
mb.CalledWith[events.BackupStart][0][events.BackupID],
bo.Results.BackupID, "backupID pre-declaration")
}
func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
mb = evmock.NewBus()
siteID = tester.M365SiteID(t)
sel = selectors.NewSharePointBackup()
)
sel.Include(sel.Sites([]string{siteID}))
bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector)
defer closer()
require.NoError(t, bo.Run(ctx)) require.NoError(t, bo.Run(ctx))
require.NotEmpty(t, bo.Results) require.NotEmpty(t, bo.Results)

View File

@ -13,17 +13,20 @@ import (
// opStatus describes the current status of an operation. // opStatus describes the current status of an operation.
// InProgress - the standard value for any process that has not // InProgress - the standard value for any process that has not
// arrived at an end state. The end states are Failed, Completed, // arrived at an end state. The end states are Failed, Completed,
// or NoData. // or NoData.
//
// Failed - the operation was unable to begin processing data at all. // Failed - the operation was unable to begin processing data at all.
// No items have been written by the consumer. // No items have been written by the consumer.
//
// Completed - the operation was able to process one or more of the // Completed - the operation was able to process one or more of the
// items in the request. Both partial success (0 < N < len(items) // items in the request. Both partial success (0 < N < len(items)
// errored) and total success (0 errors) are set as Completed. // errored) and total success (0 errors) are set as Completed.
//
// NoData - only occurs when no data was involved in an operation. // NoData - only occurs when no data was involved in an operation.
// For example, if a backup is requested for a specific user's // For example, if a backup is requested for a specific user's
// mail, but that account contains zero mail messages, the backup // mail, but that account contains zero mail messages, the backup
// contains No Data. // contains No Data.
type opStatus int type opStatus int
//go:generate stringer -type=opStatus -linecomment //go:generate stringer -type=opStatus -linecomment

View File

@ -30,6 +30,19 @@ func SecondaryM365UserID(t *testing.T) string {
return cfg[TestCfgSecondaryUserID] return cfg[TestCfgSecondaryUserID]
} }
// LoadTestM365SiteID returns a siteID string representing the m365SiteID
// described by either the env var CORSO_M365_LOAD_TEST_SITE_ID, the
// corso_test.toml config file or the default value (in that order of priority).
// The default is a last-attempt fallback that will only work on alcion's
// testing org.
func LoadTestM365SiteID(t *testing.T) string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving load test m365 site id from test configuration")
// TODO: load test site id, not standard test site id
return cfg[TestCfgSiteID]
}
// LoadTestM365UserID returns an userID string representing the m365UserID // LoadTestM365UserID returns an userID string representing the m365UserID
// described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the // described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the
// corso_test.toml config file or the default value (in that order of priority). // corso_test.toml config file or the default value (in that order of priority).
@ -42,8 +55,29 @@ func LoadTestM365UserID(t *testing.T) string {
return cfg[TestCfgLoadTestUserID] return cfg[TestCfgLoadTestUserID]
} }
// expects cfg value to be a string representing an array like: // expects cfg value to be a string representing an array such as:
// "['foo@example.com','bar@example.com']" // ["site1\,uuid","site2\,uuid"]
// the delimeter must be a |.
func LoadTestM365OrgSites(t *testing.T) []string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving load test m365 org sites from test configuration")
// TODO: proper handling of site slice input.
// sites := cfg[TestCfgLoadTestOrgSites]
// sites = strings.TrimPrefix(sites, "[")
// sites = strings.TrimSuffix(sites, "]")
// sites = strings.ReplaceAll(sites, `"`, "")
// sites = strings.ReplaceAll(sites, `'`, "")
// sites = strings.ReplaceAll(sites, "|", ",")
// return strings.Split(sites, ",")
return []string{cfg[TestCfgSiteID]}
}
// expects cfg value to be a string representing an array such as:
// ["foo@example.com","bar@example.com"]
// the delimeter may be either a , or |.
func LoadTestM365OrgUsers(t *testing.T) []string { func LoadTestM365OrgUsers(t *testing.T) []string {
cfg, err := readTestConfig() cfg, err := readTestConfig()
require.NoError(t, err, "retrieving load test m365 org users from test configuration") require.NoError(t, err, "retrieving load test m365 org users from test configuration")

View File

@ -67,13 +67,13 @@ type repository struct {
} }
// Initialize will: // Initialize will:
// * validate the m365 account & secrets // - validate the m365 account & secrets
// * connect to the m365 account to ensure communication capability // - connect to the m365 account to ensure communication capability
// * validate the provider config & secrets // - validate the provider config & secrets
// * initialize the kopia repo with the provider // - initialize the kopia repo with the provider
// * store the configuration details // - store the configuration details
// * connect to the provider // - connect to the provider
// * return the connected repository // - return the connected repository
func Initialize( func Initialize(
ctx context.Context, ctx context.Context,
acct account.Account, acct account.Account,
@ -124,10 +124,10 @@ func Initialize(
} }
// Connect will: // Connect will:
// * validate the m365 account details // - validate the m365 account details
// * connect to the m365 account to ensure communication capability // - connect to the m365 account to ensure communication capability
// * connect to the provider storage // - connect to the provider storage
// * return the connected repository // - return the connected repository
func Connect( func Connect(
ctx context.Context, ctx context.Context,
acct account.Account, acct account.Account,

View File

@ -25,10 +25,18 @@ import (
"github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/storage"
) )
func orgSiteSet(t *testing.T) []string {
return tester.LoadTestM365OrgSites(t)
}
func orgUserSet(t *testing.T) []string { func orgUserSet(t *testing.T) []string {
return tester.LoadTestM365OrgUsers(t) return tester.LoadTestM365OrgUsers(t)
} }
func singleSiteSet(t *testing.T) []string {
return []string{tester.LoadTestM365SiteID(t)}
}
func singleUserSet(t *testing.T) []string { func singleUserSet(t *testing.T) []string {
return []string{tester.LoadTestM365UserID(t)} return []string{tester.LoadTestM365UserID(t)}
} }
@ -552,3 +560,101 @@ func (suite *RepositoryIndividualLoadTestOneDriveSuite) TestOneDrive() {
sel, sel, // same selection for backup and restore sel, sel, // same selection for backup and restore
) )
} }
// ------------------------------------------------------------------------------------------------
// SharePoint
// ------------------------------------------------------------------------------------------------
type RepositoryLoadTestSharePointSuite struct {
suite.Suite
ctx context.Context
repo repository.Repository
acct account.Account
st storage.Storage
sitesUnderTest []string
}
func TestRepositoryLoadTestSharePointSuite(t *testing.T) {
if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil {
t.Skip(err)
}
suite.Run(t, new(RepositoryLoadTestSharePointSuite))
}
func (suite *RepositoryLoadTestSharePointSuite) SetupSuite() {
t := suite.T()
t.Skip("not running sharepoint load tests atm")
t.Parallel()
suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
suite.sitesUnderTest = orgSiteSet(t)
}
func (suite *RepositoryLoadTestSharePointSuite) TeardownSuite() {
suite.repo.Close(suite.ctx)
}
func (suite *RepositoryLoadTestSharePointSuite) TestSharePoint() {
ctx, flush := tester.WithContext(suite.ctx)
defer flush()
bsel := selectors.NewSharePointBackup()
bsel.Include(bsel.Sites(suite.sitesUnderTest))
sel := bsel.Selector
runLoadTest(
suite.T(),
ctx,
suite.repo,
"all_sites", "share_point",
suite.sitesUnderTest,
sel, sel, // same selection for backup and restore
)
}
type RepositoryIndividualLoadTestSharePointSuite struct {
suite.Suite
ctx context.Context
repo repository.Repository
acct account.Account
st storage.Storage
sitesUnderTest []string
}
func TestRepositoryIndividualLoadTestSharePointSuite(t *testing.T) {
if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil {
t.Skip(err)
}
suite.Run(t, new(RepositoryIndividualLoadTestOneDriveSuite))
}
func (suite *RepositoryIndividualLoadTestSharePointSuite) SetupSuite() {
t := suite.T()
t.Skip("not running sharepoint load tests atm")
t.Parallel()
suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
suite.sitesUnderTest = singleSiteSet(t)
}
func (suite *RepositoryIndividualLoadTestSharePointSuite) TeardownSuite() {
suite.repo.Close(suite.ctx)
}
func (suite *RepositoryIndividualLoadTestSharePointSuite) TestSharePoint() {
ctx, flush := tester.WithContext(suite.ctx)
defer flush()
bsel := selectors.NewSharePointBackup()
bsel.Include(bsel.Sites(suite.sitesUnderTest))
sel := bsel.Selector
runLoadTest(
suite.T(),
ctx,
suite.repo,
"single_site", "share_point",
suite.sitesUnderTest,
sel, sel, // same selection for backup and restore
)
}