cli changes for sharepoint lists (#4924)

cli changes for sharepoint lists
Changes previously approved in: https://github.com/alcionai/corso/pull/4850

#### Does this PR need a docs update or release note?
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

#### Issue(s)
#4754 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/cli/backup/sharepoint.go

@@ -194,10 +194,12 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error {
 				flags.SiteFN + " *")
 	}
 
+	allowedCats := utils.SharePointAllowedCategories()
+
 	for _, d := range cats {
-		if d != flags.DataLibraries && d != flags.DataPages {
+		if _, ok := allowedCats[d]; !ok {
 			return clues.New(
-				d + " is an unrecognized data type; either  " + flags.DataLibraries + "or " + flags.DataPages)
+				d + " is an unrecognized data type; only  " + flags.DataLibraries + " supported")
 		}
 	}
 
@@ -239,10 +241,11 @@ func addCategories(sel *selectors.SharePointBackup, cats []string) *selectors.Sh
 
 	for _, d := range cats {
 		switch d {
+		// [TODO] uncomment when lists are enabled
+		// case flags.DataLists:
+		// 	sel.Include(sel.Lists(selectors.Any()))
 		case flags.DataLibraries:
 			sel.Include(sel.LibraryFolders(selectors.Any()))
-		case flags.DataPages:
-			sel.Include(sel.Pages(selectors.Any()))
 		}
 	}
 

src/cli/backup/sharepoint_e2e_test.go

@@ -1,12 +1,14 @@
 package backup_test
 
 import (
+	"context"
 	"fmt"
 	"strings"
 	"testing"
 
 	"github.com/alcionai/clues"
 	"github.com/google/uuid"
+	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -27,7 +29,7 @@ import (
 )
 
 // ---------------------------------------------------------------------------
-// tests with no prior backup
+// tests that require no existing backups
 // ---------------------------------------------------------------------------
 
 type NoBackupSharePointE2ESuite struct {
@@ -79,6 +81,278 @@ func (suite *NoBackupSharePointE2ESuite) TestSharePointBackupListCmd_empty() {
 	assert.True(t, strings.HasSuffix(result, "No backups available\n"))
 }
 
+// ---------------------------------------------------------------------------
+// tests with no prior backup
+// ---------------------------------------------------------------------------
+
+type BackupSharepointE2ESuite struct {
+	tester.Suite
+	dpnd dependencies
+	its  intgTesterSetup
+}
+
+func TestBackupSharepointE2ESuite(t *testing.T) {
+	suite.Run(t, &BackupSharepointE2ESuite{Suite: tester.NewE2ESuite(
+		t,
+		[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs})})
+}
+
+func (suite *BackupSharepointE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.its = newIntegrationTesterSetup(t)
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
+}
+
+func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_lists() {
+	runSharepointBackupCategoryTest(suite, flags.DataLists)
+}
+
+func runSharepointBackupCategoryTest(suite *BackupSharepointE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildSharepointBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		suite.its.site.ID,
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+func (suite *BackupSharepointE2ESuite) TestSharepointBackupCmd_siteNotFound_lists() {
+	runSharepointBackupSiteNotFoundTest(suite, flags.DataLists)
+}
+
+func runSharepointBackupSiteNotFoundTest(suite *BackupSharepointE2ESuite, category string) {
+	recorder := strings.Builder{}
+	recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd, ctx := buildSharepointBackupCmd(
+		ctx,
+		suite.dpnd.configFilePath,
+		uuid.NewString(),
+		category,
+		&recorder)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+	assert.Contains(
+		t,
+		err.Error(),
+		"Invalid hostname for this tenancy", "error missing site not found")
+	assert.NotContains(t, err.Error(), "runtime error", "panic happened")
+
+	t.Logf("backup error message: %s", err.Error())
+
+	result := recorder.String()
+	t.Log("backup results", result)
+}
+
+// ---------------------------------------------------------------------------
+// tests prepared with a previous backup
+// ---------------------------------------------------------------------------
+
+type PreparedBackupSharepointE2ESuite struct {
+	tester.Suite
+	dpnd      dependencies
+	backupOps map[path.CategoryType]string
+	its       intgTesterSetup
+}
+
+func TestPreparedBackupSharepointE2ESuite(t *testing.T) {
+	suite.Run(t, &PreparedBackupSharepointE2ESuite{
+		Suite: tester.NewE2ESuite(
+			t,
+			[][]string{storeTD.AWSStorageCredEnvs, tconfig.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) SetupSuite() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	suite.its = newIntegrationTesterSetup(t)
+	suite.dpnd = prepM365Test(t, ctx, path.SharePointService)
+	suite.backupOps = make(map[path.CategoryType]string)
+
+	var (
+		sites = []string{suite.its.site.ID}
+		ins   = idname.NewCache(map[string]string{suite.its.site.ID: suite.its.site.ID})
+		cats  = []path.CategoryType{
+			path.ListsCategory,
+		}
+	)
+
+	for _, set := range cats {
+		var (
+			sel    = selectors.NewSharePointBackup(sites)
+			scopes []selectors.SharePointScope
+		)
+
+		switch set {
+		case path.ListsCategory:
+			scopes = testdata.SharePointBackupListsScope(sel)
+		}
+
+		sel.Include(scopes)
+
+		bop, err := suite.dpnd.repo.NewBackupWithLookup(ctx, sel.Selector, ins)
+		require.NoError(t, err, clues.ToCore(err))
+
+		err = bop.Run(ctx)
+		require.NoError(t, err, clues.ToCore(err))
+
+		bIDs := string(bop.Results.BackupID)
+
+		// sanity check, ensure we can find the backup and its details immediately
+		b, err := suite.dpnd.repo.Backup(ctx, string(bop.Results.BackupID))
+		require.NoError(t, err, "retrieving recent backup by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
+
+		_, b, errs := suite.dpnd.repo.GetBackupDetails(ctx, bIDs)
+		require.NoError(t, errs.Failure(), "retrieving recent backup details by ID")
+		require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
+		require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
+
+		suite.backupOps[set] = string(b.ID)
+	}
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_lists() {
+	runSharepointListCmdTest(suite, path.ListsCategory)
+}
+
+func runSharepointListCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+	assert.Contains(t, result, suite.backupOps[category])
+
+	t.Log("backup results", result)
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointListCmd_badID() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "list", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath,
+		"--backup", uuid.NewString())
+	cli.BuildCommandTree(cmd)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.Error(t, err, clues.ToCore(err))
+}
+
+func (suite *PreparedBackupSharepointE2ESuite) TestSharepointDetailsCmd_lists() {
+	runSharepointDetailsCmdTest(suite, path.ListsCategory)
+}
+
+func runSharepointDetailsCmdTest(suite *PreparedBackupSharepointE2ESuite, category path.CategoryType) {
+	suite.dpnd.recorder.Reset()
+
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	ctx = config.SetViper(ctx, suite.dpnd.vpr)
+
+	defer flush()
+
+	bID := suite.backupOps[category]
+
+	// fetch the details from the repo first
+	deets, _, errs := suite.dpnd.repo.GetBackupDetails(ctx, string(bID))
+	require.NoError(t, errs.Failure(), clues.ToCore(errs.Failure()))
+	require.Empty(t, errs.Recovered())
+
+	cmd := cliTD.StubRootCmd(
+		"backup", "details", "sharepoint",
+		"--config-file", suite.dpnd.configFilePath,
+		"--"+flags.BackupFN, string(bID))
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(&suite.dpnd.recorder)
+
+	ctx = print.SetRootCmd(ctx, cmd)
+
+	// run the command
+	err := cmd.ExecuteContext(ctx)
+	require.NoError(t, err, clues.ToCore(err))
+
+	// compare the output
+	result := suite.dpnd.recorder.String()
+
+	i := 0
+	foundList := 0
+
+	for _, ent := range deets.Entries {
+		if ent.SharePoint != nil && ent.SharePoint.ItemName != "" {
+			suite.Run(fmt.Sprintf("detail %d", i), func() {
+				assert.Contains(suite.T(), result, ent.ShortRef)
+			})
+			foundList++
+			i++
+		}
+	}
+
+	assert.GreaterOrEqual(t, foundList, 1)
+}
+
 // ---------------------------------------------------------------------------
 // tests for deleting backups
 // ---------------------------------------------------------------------------
@@ -216,3 +490,23 @@ func (suite *BackupDeleteSharePointE2ESuite) TestSharePointBackupDeleteCmd_NoBac
 	err := cmd.ExecuteContext(ctx)
 	require.Error(t, err, clues.ToCore(err))
 }
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func buildSharepointBackupCmd(
+	ctx context.Context,
+	configFile, site, category string,
+	recorder *strings.Builder,
+) (*cobra.Command, context.Context) {
+	cmd := cliTD.StubRootCmd(
+		"backup", "create", "sharepoint",
+		"--config-file", configFile,
+		"--"+flags.SiteIDFN, site,
+		"--"+flags.CategoryDataFN, category)
+	cli.BuildCommandTree(cmd)
+	cmd.SetOut(recorder)
+
+	return cmd, print.SetRootCmd(ctx, cmd)
+}

src/cli/backup/sharepoint_test.go

@@ -218,6 +218,7 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
 		name   string
 		site   []string
 		weburl []string
+		cats   []string
 		expect assert.ErrorAssertionFunc
 	}{
 		{
@@ -225,25 +226,63 @@ func (suite *SharePointUnitSuite) TestValidateSharePointBackupCreateFlags() {
 			expect: assert.Error,
 		},
 		{
-			name:   "sites",
+			name:   "sites but no category",
 			site:   []string{"smarf"},
 			expect: assert.NoError,
 		},
 		{
-			name:   "urls",
+			name:   "web urls but no category",
 			weburl: []string{"fnord"},
 			expect: assert.NoError,
 		},
 		{
-			name:   "both",
+			name:   "both web urls and sites but no category",
 			site:   []string{"smarf"},
 			weburl: []string{"fnord"},
 			expect: assert.NoError,
 		},
+		{
+			name:   "site with libraries category",
+			site:   []string{"smarf"},
+			cats:   []string{flags.DataLibraries},
+			expect: assert.NoError,
+		},
+		{
+			name:   "site with invalid category",
+			site:   []string{"smarf"},
+			cats:   []string{"invalid category"},
+			expect: assert.Error,
+		},
+		// [TODO]: Uncomment when lists are enabled
+
+		// {
+		// 	name:   "site with lists category",
+		// 	site:   []string{"smarf"},
+		// 	cats:   []string{flags.DataLists},
+		// 	expect: assert.NoError,
+		// },
+
+		// [TODO]: Uncomment when pages are enabled
+
+		// {
+		// 	name:   "site with pages category",
+		// 	site:   []string{"smarf"},
+		// 	cats:   []string{flags.DataPages},
+		// 	expect: assert.NoError,
+		// },
+
+		// [TODO]: Uncomment when pages & lists are enabled
+
+		// {
+		// 	name:   "site with all categories",
+		// 	site:   []string{"smarf"},
+		// 	cats:   []string{flags.DataLists, flags.DataPages, flags.DataLibraries},
+		// 	expect: assert.NoError,
+		// },
 	}
 	for _, test := range table {
 		suite.Run(test.name, func() {
-			err := validateSharePointBackupCreateFlags(test.site, test.weburl, nil)
+			err := validateSharePointBackupCreateFlags(test.site, test.weburl, test.cats)
 			test.expect(suite.T(), err, clues.ToCore(err))
 		})
 	}
@@ -329,6 +368,12 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
 			data:   []string{flags.DataPages},
 			expect: bothIDs,
 		},
+		{
+			name:   "Lists",
+			site:   bothIDs,
+			data:   []string{flags.DataLists},
+			expect: bothIDs,
+		},
 	}
 	for _, test := range table {
 		suite.Run(test.name, func() {

src/cli/flags/sharepoint.go

@@ -7,6 +7,7 @@ import (
 const (
 	DataLibraries = "libraries"
 	DataPages     = "pages"
+	DataLists     = "lists"
 )
 
 const (

src/cli/utils/sharepoint.go

@@ -66,6 +66,30 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts {
 	}
 }
 
+func SharePointAllowedCategories() map[string]struct{} {
+	return map[string]struct{}{
+		flags.DataLibraries: {},
+		// flags.DataLists:     {},
+	}
+}
+
+func AddCategories(sel *selectors.SharePointBackup, cats []string) *selectors.SharePointBackup {
+	if len(cats) == 0 {
+		sel.Include(sel.LibraryFolders(selectors.Any()), sel.Lists(selectors.Any()))
+	}
+
+	for _, d := range cats {
+		switch d {
+		case flags.DataLibraries:
+			sel.Include(sel.LibraryFolders(selectors.Any()))
+		case flags.DataLists:
+			sel.Include(sel.Lists(selectors.Any()))
+		}
+	}
+
+	return sel
+}
+
 // ValidateSharePointRestoreFlags checks common flags for correctness and interdependencies
 func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error {
 	if len(backupID) == 0 {

src/cli/utils/sharepoint_test.go

@@ -377,3 +377,47 @@ func (suite *SharePointUtilsSuite) TestValidateSharePointRestoreFlags() {
 		})
 	}
 }
+
+func (suite *SharePointUtilsSuite) TestAddSharepointCategories() {
+	table := []struct {
+		name           string
+		cats           []string
+		expectScopeLen int
+	}{
+		{
+			name:           "none",
+			cats:           []string{},
+			expectScopeLen: 2,
+		},
+		{
+			name:           "libraries",
+			cats:           []string{flags.DataLibraries},
+			expectScopeLen: 1,
+		},
+		{
+			name:           "lists",
+			cats:           []string{flags.DataLists},
+			expectScopeLen: 1,
+		},
+		{
+			name: "all allowed",
+			cats: []string{
+				flags.DataLibraries,
+				flags.DataLists,
+			},
+			expectScopeLen: 2,
+		},
+		{
+			name:           "bad inputs",
+			cats:           []string{"foo"},
+			expectScopeLen: 0,
+		},
+	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			sel := utils.AddCategories(selectors.NewSharePointBackup(selectors.Any()), test.cats)
+			scopes := sel.Scopes()
+			assert.Len(suite.T(), scopes, test.expectScopeLen)
+		})
+	}
+}