add boilerplate integration tests for sharepoint (#1600)

## Type of change

- [x] 🤖 Test

## Issue(s)

* #1506

## Test Plan

- [x] 💚 E2E

src/cli/backup/sharepoint_integration_test.go

@@ -0,0 +1,236 @@
+package backup_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/cli"
+	"github.com/alcionai/corso/src/cli/config"
+	"github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/operations"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/repository"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+// ---------------------------------------------------------------------------
+// tests with no prior backup
+// ---------------------------------------------------------------------------
+
+type NoBackupSharePointIntegrationSuite struct {
+	suite.Suite
+	acct       account.Account
+	st         storage.Storage
+	vpr        *viper.Viper
+	cfgFP      string
+	repo       repository.Repository
+	m365SiteID string
+	recorder   strings.Builder
+}
+
+func TestNoBackupSharePointIntegrationSuite(t *testing.T) {
+	if err := tester.RunOnAny(
+		tester.CorsoCITests,
+		tester.CorsoCLITests,
+		tester.CorsoCLIBackupTests,
+	); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(NoBackupSharePointIntegrationSuite))
+}
+
+func (suite *NoBackupSharePointIntegrationSuite) SetupSuite() {
+	t := suite.T()
+	ctx, flush := tester.NewContext()
+
+	defer flush()
+
+	_, err := tester.GetRequiredEnvSls(
+		tester.AWSStorageCredEnvs,
+		tester.M365AcctCredEnvs)
+	require.NoError(t, err)
+
+	// prepare common details
+	suite.acct = tester.NewM365Account(t)
+	suite.st = tester.NewPrefixedS3Storage(t)
+
+	cfg, err := suite.st.S3Config()
+	require.NoError(t, err)
+
+	force := map[string]string{
+		tester.TestCfgAccountProvider: "M365",
+		tester.TestCfgStorageProvider: "S3",
+		tester.TestCfgPrefix:          cfg.Prefix,
+	}
+
+	suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force)
+	require.NoError(t, err)
+
+	ctx = config.SetViper(ctx, suite.vpr)
+	suite.m365SiteID = tester.M365SiteID(t)
+
+	// init the repo first
+	suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
+	require.NoError(t, err)
+}
+
+func (suite *NoBackupSharePointIntegrationSuite) TestSharePointBackupListCmd_empty() {
+	t := suite.T()
+	ctx, flush := tester.NewContext()
+	ctx = config.SetViper(ctx, suite.vpr)
+
+	defer flush()
+
+	suite.recorder.Reset()
+
+	cmd := tester.StubRootCmd(
+		"backup", "list", "sharepoint",
+		"--config-file", suite.cfgFP)
+	cli.BuildCommandTree(cmd)
+
+	cmd.SetErr(&suite.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	require.NoError(t, cmd.ExecuteContext(ctx))
+
+	result := suite.recorder.String()
+
+	// as an offhand check: the result should contain the m365 sitet id
+	assert.Equal(t, "No backups available\n", result)
+}
+
+// ---------------------------------------------------------------------------
+// tests for deleting backups
+// ---------------------------------------------------------------------------
+
+type BackupDeleteSharePointIntegrationSuite struct {
+	suite.Suite
+	acct     account.Account
+	st       storage.Storage
+	vpr      *viper.Viper
+	cfgFP    string
+	repo     repository.Repository
+	backupOp operations.BackupOperation
+	recorder strings.Builder
+}
+
+func TestBackupDeleteSharePointIntegrationSuite(t *testing.T) {
+	if err := tester.RunOnAny(
+		tester.CorsoCITests,
+		tester.CorsoCLITests,
+		tester.CorsoCLIBackupTests,
+	); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(BackupDeleteSharePointIntegrationSuite))
+}
+
+func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() {
+	t := suite.T()
+	_, err := tester.GetRequiredEnvSls(
+		tester.AWSStorageCredEnvs,
+		tester.M365AcctCredEnvs)
+	require.NoError(t, err)
+
+	// prepare common details
+	suite.acct = tester.NewM365Account(t)
+	suite.st = tester.NewPrefixedS3Storage(t)
+
+	cfg, err := suite.st.S3Config()
+	require.NoError(t, err)
+
+	force := map[string]string{
+		tester.TestCfgAccountProvider: "M365",
+		tester.TestCfgStorageProvider: "S3",
+		tester.TestCfgPrefix:          cfg.Prefix,
+	}
+	suite.vpr, suite.cfgFP, err = tester.MakeTempTestConfigClone(t, force)
+	require.NoError(t, err)
+
+	ctx, flush := tester.NewContext()
+	ctx = config.SetViper(ctx, suite.vpr)
+
+	defer flush()
+
+	// init the repo first
+	suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
+	require.NoError(t, err)
+
+	m365SiteID := tester.M365SiteID(t)
+
+	// some tests require an existing backup
+	sel := selectors.NewSharePointBackup()
+	sel.Include(sel.Libraries([]string{m365SiteID}, selectors.Any()))
+
+	suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
+	require.NoError(t, suite.backupOp.Run(ctx))
+	require.NoError(t, err)
+}
+
+func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd() {
+	t := suite.T()
+	ctx, flush := tester.NewContext()
+	ctx = config.SetViper(ctx, suite.vpr)
+
+	defer flush()
+
+	suite.recorder.Reset()
+
+	cmd := tester.StubRootCmd(
+		"backup", "delete", "sharepoint",
+		"--config-file", suite.cfgFP,
+		"--"+utils.BackupFN, string(suite.backupOp.Results.BackupID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetErr(&suite.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	require.NoError(t, cmd.ExecuteContext(ctx))
+
+	result := suite.recorder.String()
+
+	assert.Equal(t, fmt.Sprintf("Deleted SharePoint backup %s\n", string(suite.backupOp.Results.BackupID)), result)
+}
+
+// moved out of the func above to make the linter happy
+// // a follow-up details call should fail, due to the backup ID being deleted
+// cmd = tester.StubRootCmd(
+// 	"backup", "details", "sharepoint",
+// 	"--config-file", suite.cfgFP,
+// 	"--backup", string(suite.backupOp.Results.BackupID))
+// cli.BuildCommandTree(cmd)
+
+// require.Error(t, cmd.ExecuteContext(ctx))
+
+func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd_unknownID() {
+	t := suite.T()
+	ctx, flush := tester.NewContext()
+	ctx = config.SetViper(ctx, suite.vpr)
+
+	defer flush()
+
+	cmd := tester.StubRootCmd(
+		"backup", "delete", "sharepoint",
+		"--config-file", suite.cfgFP,
+		"--"+utils.BackupFN, uuid.NewString())
+	cli.BuildCommandTree(cmd)
+
+	// unknown backupIDs should error since the modelStore can't find the backup
+	require.Error(t, cmd.ExecuteContext(ctx))
+}

src/internal/operations/backup_test.go

@@ -1,6 +1,7 @@
 package operations
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -114,6 +115,67 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
 // integration
 // ---------------------------------------------------------------------------
 
+//revive:disable:context-as-argument
+func prepNewBackupOp(
+	t *testing.T,
+	ctx context.Context,
+	bus events.Eventer,
+	sel selectors.Selector,
+) (BackupOperation, func()) {
+	//revive:enable:context-as-argument
+	acct := tester.NewM365Account(t)
+
+	// need to initialize the repository before we can test connecting to it.
+	st := tester.NewPrefixedS3Storage(t)
+
+	k := kopia.NewConn(st)
+	require.NoError(t, k.Initialize(ctx))
+
+	// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
+	// to close here.
+	closer := func() { k.Close(ctx) }
+
+	kw, err := kopia.NewWrapper(k)
+	if !assert.NoError(t, err) {
+		closer()
+		t.FailNow()
+	}
+
+	closer = func() {
+		k.Close(ctx)
+		kw.Close(ctx)
+	}
+
+	ms, err := kopia.NewModelStore(k)
+	if !assert.NoError(t, err) {
+		closer()
+		t.FailNow()
+	}
+
+	closer = func() {
+		k.Close(ctx)
+		kw.Close(ctx)
+		ms.Close(ctx)
+	}
+
+	sw := store.NewKopiaStore(ms)
+
+	bo, err := NewBackupOperation(
+		ctx,
+		control.Options{},
+		kw,
+		sw,
+		acct,
+		sel,
+		bus)
+	if !assert.NoError(t, err) {
+		closer()
+		t.FailNow()
+	}
+
+	return bo, closer
+}
+
 type BackupOpIntegrationSuite struct {
 	suite.Suite
 }
@@ -122,6 +184,7 @@ func TestBackupOpIntegrationSuite(t *testing.T) {
 	if err := tester.RunOnAny(
 		tester.CorsoCITests,
 		tester.CorsoOperationTests,
+		"flomp",
 	); err != nil {
 		t.Skip(err)
 	}
@@ -174,12 +237,11 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
 
 // TestBackup_Run ensures that Integration Testing works
 // for the following scopes: Contacts, Events, and Mail
-func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
+func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchange() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
 	m365UserID := tester.M365UserID(suite.T())
-	acct := tester.NewM365Account(suite.T())
 
 	tests := []struct {
 		name       string
@@ -215,36 +277,9 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
 	}
 	for _, test := range tests {
 		suite.T().Run(test.name, func(t *testing.T) {
-			// need to initialize the repository before we can test connecting to it.
-			st := tester.NewPrefixedS3Storage(t)
-			k := kopia.NewConn(st)
-			require.NoError(t, k.Initialize(ctx))
-
-			// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
-			// to close here.
-			defer k.Close(ctx)
-
-			kw, err := kopia.NewWrapper(k)
-			require.NoError(t, err)
-			defer kw.Close(ctx)
-
-			ms, err := kopia.NewModelStore(k)
-			require.NoError(t, err)
-			defer ms.Close(ctx)
-
 			mb := evmock.NewBus()
-
-			sw := store.NewKopiaStore(ms)
-			selected := test.selectFunc()
-			bo, err := NewBackupOperation(
-				ctx,
-				control.Options{},
-				kw,
-				sw,
-				acct,
-				*selected,
-				mb)
-			require.NoError(t, err)
+			bo, closer := prepNewBackupOp(t, ctx, mb, *test.selectFunc())
+			defer closer()
 
 			require.NoError(t, bo.Run(ctx))
 			require.NotEmpty(t, bo.Results)
@@ -266,51 +301,54 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
 	}
 }
 
-func (suite *BackupOpIntegrationSuite) TestBackupOneDrive_Run() {
+func (suite *BackupOpIntegrationSuite) TestBackup_Run_oneDrive() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	t := suite.T()
+	var (
+		t          = suite.T()
+		mb         = evmock.NewBus()
+		m365UserID = tester.SecondaryM365UserID(t)
+		sel        = selectors.NewOneDriveBackup()
+	)
 
-	m365UserID := tester.SecondaryM365UserID(t)
-	acct := tester.NewM365Account(t)
-
-	// need to initialize the repository before we can test connecting to it.
-	st := tester.NewPrefixedS3Storage(t)
-
-	k := kopia.NewConn(st)
-	require.NoError(t, k.Initialize(ctx))
-
-	// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
-	// to close here.
-	defer k.Close(ctx)
-
-	kw, err := kopia.NewWrapper(k)
-	require.NoError(t, err)
-
-	defer kw.Close(ctx)
-
-	ms, err := kopia.NewModelStore(k)
-	require.NoError(t, err)
-
-	defer ms.Close(ctx)
-
-	sw := store.NewKopiaStore(ms)
-
-	mb := evmock.NewBus()
-
-	sel := selectors.NewOneDriveBackup()
 	sel.Include(sel.Users([]string{m365UserID}))
 
-	bo, err := NewBackupOperation(
-		ctx,
-		control.Options{},
-		kw,
-		sw,
-		acct,
-		sel.Selector,
-		mb)
-	require.NoError(t, err)
+	bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector)
+	defer closer()
+
+	require.NoError(t, bo.Run(ctx))
+	require.NotEmpty(t, bo.Results)
+	require.NotEmpty(t, bo.Results.BackupID)
+	assert.Equalf(t, Completed, bo.Status, "backup status %s is not Completed", bo.Status)
+	assert.Equal(t, bo.Results.ItemsRead, bo.Results.ItemsWritten)
+	assert.Less(t, int64(0), bo.Results.BytesRead, "bytes read")
+	assert.Less(t, int64(0), bo.Results.BytesUploaded, "bytes uploaded")
+	assert.Equal(t, 1, bo.Results.ResourceOwners)
+	assert.NoError(t, bo.Results.ReadErrors)
+	assert.NoError(t, bo.Results.WriteErrors)
+	assert.Equal(t, 1, mb.TimesCalled[events.BackupStart], "backup-start events")
+	assert.Equal(t, 1, mb.TimesCalled[events.BackupEnd], "backup-end events")
+	assert.Equal(t,
+		mb.CalledWith[events.BackupStart][0][events.BackupID],
+		bo.Results.BackupID, "backupID pre-declaration")
+}
+
+func (suite *BackupOpIntegrationSuite) TestBackup_Run_sharePoint() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	var (
+		t      = suite.T()
+		mb     = evmock.NewBus()
+		siteID = tester.M365SiteID(t)
+		sel    = selectors.NewSharePointBackup()
+	)
+
+	sel.Include(sel.Sites([]string{siteID}))
+
+	bo, closer := prepNewBackupOp(t, ctx, mb, sel.Selector)
+	defer closer()
 
 	require.NoError(t, bo.Run(ctx))
 	require.NotEmpty(t, bo.Results)

src/internal/operations/operation.go

@@ -15,11 +15,14 @@ import (
 // InProgress - the standard value for any process that has not
 // arrived at an end state.  The end states are Failed, Completed,
 // or NoData.
+//
 // Failed - the operation was unable to begin processing data at all.
 // No items have been written by the consumer.
+//
 // Completed - the operation was able to process one or more of the
 // items in the request. Both partial success (0 < N < len(items)
 // errored) and total success (0 errors) are set as Completed.
+//
 // NoData - only occurs when no data was involved in an operation.
 // For example, if a backup is requested for a specific user's
 // mail, but that account contains zero mail messages, the backup

src/internal/tester/resource_owners.go

@@ -30,6 +30,19 @@ func SecondaryM365UserID(t *testing.T) string {
 	return cfg[TestCfgSecondaryUserID]
 }
 
+// LoadTestM365SiteID returns a siteID string representing the m365SiteID
+// described by either the env var CORSO_M365_LOAD_TEST_SITE_ID, the
+// corso_test.toml config file or the default value (in that order of priority).
+// The default is a last-attempt fallback that will only work on alcion's
+// testing org.
+func LoadTestM365SiteID(t *testing.T) string {
+	cfg, err := readTestConfig()
+	require.NoError(t, err, "retrieving load test m365 site id from test configuration")
+
+	// TODO: load test site id, not standard test site id
+	return cfg[TestCfgSiteID]
+}
+
 // LoadTestM365UserID returns an userID string representing the m365UserID
 // described by either the env var CORSO_M365_LOAD_TEST_USER_ID, the
 // corso_test.toml config file or the default value (in that order of priority).
@@ -42,8 +55,29 @@ func LoadTestM365UserID(t *testing.T) string {
 	return cfg[TestCfgLoadTestUserID]
 }
 
-// expects cfg value to be a string representing an array like:
-// "['foo@example.com','bar@example.com']"
+// expects cfg value to be a string representing an array such as:
+// ["site1\,uuid","site2\,uuid"]
+// the delimeter must be a |.
+func LoadTestM365OrgSites(t *testing.T) []string {
+	cfg, err := readTestConfig()
+	require.NoError(t, err, "retrieving load test m365 org sites from test configuration")
+
+	// TODO: proper handling of site slice input.
+	// sites := cfg[TestCfgLoadTestOrgSites]
+	// sites = strings.TrimPrefix(sites, "[")
+	// sites = strings.TrimSuffix(sites, "]")
+	// sites = strings.ReplaceAll(sites, `"`, "")
+	// sites = strings.ReplaceAll(sites, `'`, "")
+	// sites = strings.ReplaceAll(sites, "|", ",")
+
+	// return strings.Split(sites, ",")
+
+	return []string{cfg[TestCfgSiteID]}
+}
+
+// expects cfg value to be a string representing an array such as:
+// ["foo@example.com","bar@example.com"]
+// the delimeter may be either a , or |.
 func LoadTestM365OrgUsers(t *testing.T) []string {
 	cfg, err := readTestConfig()
 	require.NoError(t, err, "retrieving load test m365 org users from test configuration")

src/pkg/repository/repository.go

@@ -67,13 +67,13 @@ type repository struct {
 }
 
 // Initialize will:
-//  * validate the m365 account & secrets
-//  * connect to the m365 account to ensure communication capability
-//  * validate the provider config & secrets
-//  * initialize the kopia repo with the provider
-//  * store the configuration details
-//  * connect to the provider
-//  * return the connected repository
+//   - validate the m365 account & secrets
+//   - connect to the m365 account to ensure communication capability
+//   - validate the provider config & secrets
+//   - initialize the kopia repo with the provider
+//   - store the configuration details
+//   - connect to the provider
+//   - return the connected repository
 func Initialize(
 	ctx context.Context,
 	acct account.Account,
@@ -124,10 +124,10 @@ func Initialize(
 }
 
 // Connect will:
-//  * validate the m365 account details
-//  * connect to the m365 account to ensure communication capability
-//  * connect to the provider storage
-//  * return the connected repository
+//   - validate the m365 account details
+//   - connect to the m365 account to ensure communication capability
+//   - connect to the provider storage
+//   - return the connected repository
 func Connect(
 	ctx context.Context,
 	acct account.Account,

src/pkg/repository/repository_load_test.go

@@ -25,10 +25,18 @@ import (
 	"github.com/alcionai/corso/src/pkg/storage"
 )
 
+func orgSiteSet(t *testing.T) []string {
+	return tester.LoadTestM365OrgSites(t)
+}
+
 func orgUserSet(t *testing.T) []string {
 	return tester.LoadTestM365OrgUsers(t)
 }
 
+func singleSiteSet(t *testing.T) []string {
+	return []string{tester.LoadTestM365SiteID(t)}
+}
+
 func singleUserSet(t *testing.T) []string {
 	return []string{tester.LoadTestM365UserID(t)}
 }
@@ -552,3 +560,101 @@ func (suite *RepositoryIndividualLoadTestOneDriveSuite) TestOneDrive() {
 		sel, sel, // same selection for backup and restore
 	)
 }
+
+// ------------------------------------------------------------------------------------------------
+// SharePoint
+// ------------------------------------------------------------------------------------------------
+
+type RepositoryLoadTestSharePointSuite struct {
+	suite.Suite
+	ctx            context.Context
+	repo           repository.Repository
+	acct           account.Account
+	st             storage.Storage
+	sitesUnderTest []string
+}
+
+func TestRepositoryLoadTestSharePointSuite(t *testing.T) {
+	if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(RepositoryLoadTestSharePointSuite))
+}
+
+func (suite *RepositoryLoadTestSharePointSuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not running sharepoint load tests atm")
+	t.Parallel()
+	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.sitesUnderTest = orgSiteSet(t)
+}
+
+func (suite *RepositoryLoadTestSharePointSuite) TeardownSuite() {
+	suite.repo.Close(suite.ctx)
+}
+
+func (suite *RepositoryLoadTestSharePointSuite) TestSharePoint() {
+	ctx, flush := tester.WithContext(suite.ctx)
+	defer flush()
+
+	bsel := selectors.NewSharePointBackup()
+	bsel.Include(bsel.Sites(suite.sitesUnderTest))
+	sel := bsel.Selector
+
+	runLoadTest(
+		suite.T(),
+		ctx,
+		suite.repo,
+		"all_sites", "share_point",
+		suite.sitesUnderTest,
+		sel, sel, // same selection for backup and restore
+	)
+}
+
+type RepositoryIndividualLoadTestSharePointSuite struct {
+	suite.Suite
+	ctx            context.Context
+	repo           repository.Repository
+	acct           account.Account
+	st             storage.Storage
+	sitesUnderTest []string
+}
+
+func TestRepositoryIndividualLoadTestSharePointSuite(t *testing.T) {
+	if err := tester.RunOnAny(tester.CorsoLoadTests); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(RepositoryIndividualLoadTestOneDriveSuite))
+}
+
+func (suite *RepositoryIndividualLoadTestSharePointSuite) SetupSuite() {
+	t := suite.T()
+	t.Skip("not running sharepoint load tests atm")
+	t.Parallel()
+	suite.ctx, suite.repo, suite.acct, suite.st = initM365Repo(t)
+	suite.sitesUnderTest = singleSiteSet(t)
+}
+
+func (suite *RepositoryIndividualLoadTestSharePointSuite) TeardownSuite() {
+	suite.repo.Close(suite.ctx)
+}
+
+func (suite *RepositoryIndividualLoadTestSharePointSuite) TestSharePoint() {
+	ctx, flush := tester.WithContext(suite.ctx)
+	defer flush()
+
+	bsel := selectors.NewSharePointBackup()
+	bsel.Include(bsel.Sites(suite.sitesUnderTest))
+	sel := bsel.Selector
+
+	runLoadTest(
+		suite.T(),
+		ctx,
+		suite.repo,
+		"single_site", "share_point",
+		suite.sitesUnderTest,
+		sel, sel, // same selection for backup and restore
+	)
+}