GC: SharePoint: BackUp: Pages (#2178)

## Description
- Adds logic to retrieve  `SharePoint.Pages` from M365
- Anchor PR for `SharePoint.Pages` feature support. Restore Pipeline 
PR to remain in Draft to ensure PR Train is stable until the solution to #2174 is implemented. 
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

## Issue(s)
* related to #2173
* related to #2071
## Test Plan
- [x] ⚡ Unit test

NOTE: Tests will fail in CI due to complications with #2086.

src/internal/connector/discovery/api/beta_service.go

@@ -11,23 +11,23 @@ import (
 // Service wraps BetaClient's functionality.
 // Abstraction created to comply loosely with graph.Servicer
 // methods for ease of switching between v1.0 and beta connnectors
-type Service struct {
+type BetaService struct {
 	client *betasdk.BetaClient
 }
 
-func (s Service) Client() *betasdk.BetaClient {
+func (s BetaService) Client() *betasdk.BetaClient {
 	return s.client
 }
 
-func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *Service {
+func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *BetaService {
-	return &Service{
+	return &BetaService{
 		client: betasdk.NewBetaClient(adpt),
 	}
 }
 
 // Seraialize writes an M365 parsable object into a byte array using the built-in
 // application/json writer within the adapter.
-func (s Service) Serialize(object absser.Parsable) ([]byte, error) {
+func (s BetaService) Serialize(object absser.Parsable) ([]byte, error) {
 	writer, err := s.client.Adapter().
 		GetSerializationWriterFactory().
 		GetSerializationWriter("application/json")

src/internal/connector/sharepoint/api/api.go

@@ -0,0 +1,6 @@
+package api
+
+type Tuple struct {
+	Name string
+	ID   string
+}

src/internal/connector/sharepoint/api/helper_test.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/stretchr/testify/require"
+)
+
+func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
+	adapter, err := graph.CreateAdapter(
+		credentials.AzureTenantID,
+		credentials.AzureClientID,
+		credentials.AzureClientSecret,
+	)
+	require.NoError(t, err)
+
+	return api.NewBetaService(adapter)
+}

src/internal/connector/sharepoint/api/pages.go

@@ -0,0 +1,93 @@
+package api
+
+import (
+	"context"
+
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
+	"github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
+	"github.com/alcionai/corso/src/internal/connector/graph/betasdk/sites"
+	"github.com/alcionai/corso/src/internal/connector/support"
+)
+
+// GetSitePages retrieves a collection of Pages related to the give Site.
+// Returns error if error experienced during the call
+func GetSitePage(
+	ctx context.Context,
+	serv *api.BetaService,
+	siteID string,
+	pages []string,
+) ([]models.SitePageable, error) {
+	col := make([]models.SitePageable, 0)
+	opts := retrieveSitePageOptions()
+
+	for _, entry := range pages {
+		page, err := serv.Client().SitesById(siteID).PagesById(entry).Get(ctx, opts)
+		if err != nil {
+			return nil, support.ConnectorStackErrorTraceWrap(err, "fetching page: "+entry)
+		}
+
+		col = append(col, page)
+	}
+
+	return col, nil
+}
+
+// fetchPages utility function to return the tuple of item
+func FetchPages(ctx context.Context, bs *api.BetaService, siteID string) ([]Tuple, error) {
+	var (
+		builder    = bs.Client().SitesById(siteID).Pages()
+		opts       = fetchPageOptions()
+		pageTuples = make([]Tuple, 0)
+	)
+
+	for {
+		resp, err := builder.Get(ctx, opts)
+		if err != nil {
+			return nil, support.ConnectorStackErrorTraceWrap(err, "failed fetching site page")
+		}
+
+		for _, entry := range resp.GetValue() {
+			pid := *entry.GetId()
+			temp := Tuple{pid, pid}
+
+			if entry.GetName() != nil {
+				temp.Name = *entry.GetName()
+			}
+
+			pageTuples = append(pageTuples, temp)
+		}
+
+		if resp.GetOdataNextLink() == nil {
+			break
+		}
+
+		builder = sites.NewItemPagesRequestBuilder(*resp.GetOdataNextLink(), bs.Client().Adapter())
+	}
+
+	return pageTuples, nil
+}
+
+// fetchPageOptions is used to return minimal information reltating to Site Pages
+// Pages API: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
+func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration {
+	fields := []string{"id", "name"}
+	options := &sites.ItemPagesRequestBuilderGetRequestConfiguration{
+		QueryParameters: &sites.ItemPagesRequestBuilderGetQueryParameters{
+			Select: fields,
+		},
+	}
+
+	return options
+}
+
+// retrievePageOptions returns options to expand
+func retrieveSitePageOptions() *sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration {
+	fields := []string{"canvasLayout"}
+	options := &sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration{
+		QueryParameters: &sites.ItemPagesSitePageItemRequestBuilderGetQueryParameters{
+			Expand: fields,
+		},
+	}
+
+	return options
+}

src/internal/connector/sharepoint/api/pages_test.go

@@ -0,0 +1,71 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/account"
+)
+
+type SharePointPageSuite struct {
+	suite.Suite
+	siteID string
+	creds  account.M365Config
+}
+
+func (suite *SharePointPageSuite) SetupSuite() {
+	t := suite.T()
+	tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
+
+	suite.siteID = tester.M365SiteID(t)
+	a := tester.NewM365Account(t)
+	m365, err := a.M365Config()
+	require.NoError(t, err)
+
+	suite.creds = m365
+}
+
+func TestSharePointPageSuite(t *testing.T) {
+	tester.RunOnAny(
+		t,
+		tester.CorsoCITests,
+		tester.CorsoGraphConnectorSharePointTests)
+	suite.Run(t, new(SharePointPageSuite))
+}
+
+func (suite *SharePointPageSuite) TestFetchPages() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	t := suite.T()
+	service := createTestBetaService(t, suite.creds)
+
+	pgs, err := FetchPages(ctx, service, suite.siteID)
+	assert.NoError(t, err)
+	require.NotNil(t, pgs)
+	assert.NotZero(t, len(pgs))
+
+	for _, entry := range pgs {
+		t.Logf("id: %s\t name: %s\n", entry.ID, entry.Name)
+	}
+}
+
+func (suite *SharePointPageSuite) TestGetSitePage() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	t := suite.T()
+	service := createTestBetaService(t, suite.creds)
+	tuples, err := FetchPages(ctx, service, suite.siteID)
+	require.NoError(t, err)
+	require.NotNil(t, tuples)
+
+	jobs := []string{tuples[0].ID}
+	pages, err := GetSitePage(ctx, service, suite.siteID, jobs)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, pages)
+}

src/internal/connector/sharepoint/collection.go

@@ -9,6 +9,7 @@ import (
 	kw "github.com/microsoft/kiota-serialization-json-go"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
@@ -46,6 +47,7 @@ type Collection struct {
 	jobs []string
 	// M365 IDs of the items of this collection
 	service       graph.Servicer
+	betaService   *api.BetaService
 	statusUpdater support.StatusUpdater
 }
 

src/internal/connector/sharepoint/collection_test.go

@@ -17,11 +17,27 @@ import (
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
 type SharePointCollectionSuite struct {
 	suite.Suite
+	siteID string
+	creds  account.M365Config
+}
+
+func (suite *SharePointCollectionSuite) SetupSuite() {
+	t := suite.T()
+	tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
+
+	suite.siteID = tester.M365SiteID(t)
+	a := tester.NewM365Account(t)
+	m365, err := a.M365Config()
+	require.NoError(t, err)
+
+	suite.creds = m365
 }
 
 func TestSharePointCollectionSuite(t *testing.T) {
@@ -95,20 +111,33 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
 	assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName)
 }
 
+func (suite *SharePointCollectionSuite) TestCollectPages() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	t := suite.T()
+	col, err := collectPages(
+		ctx,
+		suite.creds,
+		nil,
+		account.AzureTenantID,
+		suite.siteID,
+		nil,
+		&MockGraphService{},
+		control.Defaults(),
+	)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, col)
+}
+
 // TestRestoreListCollection verifies Graph Restore API for the List Collection
 func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
 	t := suite.T()
-	siteID := tester.M365SiteID(t)
-	a := tester.NewM365Account(t)
-	account, err := a.M365Config()
-	require.NoError(t, err)
-
-	service, err := createTestService(account)
-	require.NoError(t, err)
 
+	service := createTestService(t, suite.creds)
 	listing := mockconnector.GetMockListDefault("Mock List")
 	testName := "MockListing"
 	listing.SetDisplayName(&testName)
@@ -123,13 +152,13 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
 
 	destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting)
 
-	deets, err := restoreListItem(ctx, service, listData, siteID, destName)
+	deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
 	assert.NoError(t, err)
 	t.Logf("List created: %s\n", deets.SharePoint.ItemName)
 
 	// Clean-Up
 	var (
-		builder  = service.Client().SitesById(siteID).Lists()
+		builder  = service.Client().SitesById(suite.siteID).Lists()
 		isFound  bool
 		deleteID string
 	)
@@ -156,7 +185,7 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
 	}
 
 	if isFound {
-		err := DeleteList(ctx, service, siteID, deleteID)
+		err := DeleteList(ctx, service, suite.siteID, deleteID)
 		assert.NoError(t, err)
 	}
 }
@@ -168,25 +197,18 @@ func (suite *SharePointCollectionSuite) TestRestoreLocation() {
 	defer flush()
 
 	t := suite.T()
-	a := tester.NewM365Account(t)
-	account, err := a.M365Config()
-	require.NoError(t, err)
-
-	service, err := createTestService(account)
-	require.NoError(t, err)
 
+	service := createTestService(t, suite.creds)
 	rootFolder := "General_" + common.FormatNow(common.SimpleTimeTesting)
-	siteID := tester.M365SiteID(t)
+	folderID, err := createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder})
-
-	folderID, err := createRestoreFolders(ctx, service, siteID, []string{rootFolder})
 	assert.NoError(t, err)
 	t.Log("FolderID: " + folderID)
 
-	_, err = createRestoreFolders(ctx, service, siteID, []string{rootFolder, "Tsao"})
+	_, err = createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder, "Tsao"})
 	assert.NoError(t, err)
 
 	// CleanUp
-	siteDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil)
+	siteDrive, err := service.Client().SitesById(suite.siteID).Drive().Get(ctx, nil)
 	require.NoError(t, err)
 
 	driveID := *siteDrive.GetId()

src/internal/connector/sharepoint/data_collections.go

@@ -6,11 +6,14 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
+	sapi "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/internal/observe"
+	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
@@ -162,6 +165,55 @@ func collectLibraries(
 	return append(collections, odcs...), excludes, errs
 }
 
+// collectPages constructs a sharepoint Collections struct and Get()s the associated
+// M365 IDs for the associated Pages
+func collectPages(
+	ctx context.Context,
+	creds account.M365Config,
+	serv graph.Servicer,
+	tenantID, siteID string,
+	scope selectors.SharePointScope,
+	updater statusUpdater,
+	ctrlOpts control.Options,
+) ([]data.Collection, error) {
+	logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Pages collections")
+
+	spcs := make([]data.Collection, 0)
+
+	// make the betaClient
+	adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "adapter for betaservice not created")
+	}
+
+	betaService := api.NewBetaService(adpt)
+
+	tuples, err := sapi.FetchPages(ctx, betaService, siteID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, tuple := range tuples {
+		dir, err := path.Builder{}.Append(tuple.Name).
+			ToDataLayerSharePointPath(
+				tenantID,
+				siteID,
+				path.PagesCategory,
+				false)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create collection path for site: %s", siteID)
+		}
+
+		collection := NewCollection(dir, serv, updater.UpdateStatus)
+		collection.betaService = betaService
+		collection.AddJob(tuple.ID)
+
+		spcs = append(spcs, collection)
+	}
+
+	return spcs, nil
+}
+
 type folderMatcher struct {
 	scope selectors.SharePointScope
 }

src/internal/connector/sharepoint/helper_test.go

@@ -4,11 +4,11 @@ import (
 	"testing"
 
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
-	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
+	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/pkg/account"
 )
 
@@ -29,21 +29,22 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
 	return nil
 }
 
+func (ms *MockGraphService) UpdateStatus(*support.ConnectorOperationStatus) {
+}
+
 // ---------------------------------------------------------------------------
 // Helper Functions
 // ---------------------------------------------------------------------------
 
-func createTestService(credentials account.M365Config) (*graph.Service, error) {
+func createTestService(t *testing.T, credentials account.M365Config) *graph.Service {
 	adapter, err := graph.CreateAdapter(
 		credentials.AzureTenantID,
 		credentials.AzureClientID,
 		credentials.AzureClientSecret,
 	)
-	if err != nil {
+	require.NoError(t, err, "creating microsoft graph service for exchange")
-		return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
-	}
 
-	return graph.NewService(adapter), nil
+	return graph.NewService(adapter)
 }
 
 func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {

src/internal/connector/sharepoint/list_test.go

@@ -49,9 +49,7 @@ func (suite *SharePointSuite) TestLoadList() {
 	defer flush()
 
 	t := suite.T()
-	service, err := createTestService(suite.creds)
+	service := createTestService(t, suite.creds)
-	require.NoError(t, err)
-
 	tuples, err := preFetchLists(ctx, service, "root")
 	require.NoError(t, err)