larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: SharePoint: BackUp: Pages (#2178)

Browse Source 
## Description
- Adds logic to retrieve  `SharePoint.Pages` from M365
- Anchor PR for `SharePoint.Pages` feature support. Restore Pipeline 
PR to remain in Draft to ensure PR Train is stable until the solution to #2174 is implemented. 
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

## Issue(s)
* related to #2173
* related to #2071
## Test Plan
- [x] ⚡ Unit test

NOTE: Tests will fail in CI due to complications with #2086.
This commit is contained in:
Danny 2023-02-03 13:37:45 -05:00 committed by GitHub
parent 272a8b30fe
commit 6e12885787
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 301 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/internal/connector/discovery/api/beta_service.go
View File

@ -11,23 +11,23 @@ import (
// Service wraps BetaClient's functionality.
// Abstraction created to comply loosely with graph.Servicer
// methods for ease of switching between v1.0 and beta connnectors
type Service struct {
type BetaService struct {
client *betasdk.BetaClient
}
func (s Service) Client() *betasdk.BetaClient {
func (s BetaService) Client() *betasdk.BetaClient {
return s.client
}
func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *Service {
return &Service{
func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *BetaService {
return &BetaService{
client: betasdk.NewBetaClient(adpt),
}
}
// Seraialize writes an M365 parsable object into a byte array using the built-in
// application/json writer within the adapter.
func (s Service) Serialize(object absser.Parsable) ([]byte, error) {
func (s BetaService) Serialize(object absser.Parsable) ([]byte, error) {
writer, err := s.client.Adapter().
GetSerializationWriterFactory().
GetSerializationWriter("application/json")

6
src/internal/connector/sharepoint/api/api.go Normal file
View File

@ -0,0 +1,6 @@
package api
type Tuple struct {
Name string
ID string
}

21
src/internal/connector/sharepoint/api/helper_test.go Normal file
View File

@ -0,0 +1,21 @@
package api
import (
"testing"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/stretchr/testify/require"
)
func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret,
)
require.NoError(t, err)
return api.NewBetaService(adapter)
}

93
src/internal/connector/sharepoint/api/pages.go Normal file
View File

@ -0,0 +1,93 @@
package api
import (
"context"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk/sites"
"github.com/alcionai/corso/src/internal/connector/support"
)
// GetSitePages retrieves a collection of Pages related to the give Site.
// Returns error if error experienced during the call
func GetSitePage(
ctx context.Context,
serv *api.BetaService,
siteID string,
pages []string,
) ([]models.SitePageable, error) {
col := make([]models.SitePageable, 0)
opts := retrieveSitePageOptions()
for _, entry := range pages {
page, err := serv.Client().SitesById(siteID).PagesById(entry).Get(ctx, opts)
if err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "fetching page: "+entry)
}
col = append(col, page)
}
return col, nil
}
// fetchPages utility function to return the tuple of item
func FetchPages(ctx context.Context, bs *api.BetaService, siteID string) ([]Tuple, error) {
var (
builder = bs.Client().SitesById(siteID).Pages()
opts = fetchPageOptions()
pageTuples = make([]Tuple, 0)
)
for {
resp, err := builder.Get(ctx, opts)
if err != nil {
return nil, support.ConnectorStackErrorTraceWrap(err, "failed fetching site page")
}
for _, entry := range resp.GetValue() {
pid := *entry.GetId()
temp := Tuple{pid, pid}
if entry.GetName() != nil {
temp.Name = *entry.GetName()
}
pageTuples = append(pageTuples, temp)
}
if resp.GetOdataNextLink() == nil {
break
}
builder = sites.NewItemPagesRequestBuilder(*resp.GetOdataNextLink(), bs.Client().Adapter())
}
return pageTuples, nil
}
// fetchPageOptions is used to return minimal information reltating to Site Pages
// Pages API: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration {
fields := []string{"id", "name"}
options := &sites.ItemPagesRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.ItemPagesRequestBuilderGetQueryParameters{
Select: fields,
},
}
return options
}
// retrievePageOptions returns options to expand
func retrieveSitePageOptions() *sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration {
fields := []string{"canvasLayout"}
options := &sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.ItemPagesSitePageItemRequestBuilderGetQueryParameters{
Expand: fields,
},
}
return options
}

71
src/internal/connector/sharepoint/api/pages_test.go Normal file
View File

@ -0,0 +1,71 @@
package api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
)
type SharePointPageSuite struct {
suite.Suite
siteID string
creds account.M365Config
}
func (suite *SharePointPageSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
suite.siteID = tester.M365SiteID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.creds = m365
}
func TestSharePointPageSuite(t *testing.T) {
tester.RunOnAny(
t,
tester.CorsoCITests,
tester.CorsoGraphConnectorSharePointTests)
suite.Run(t, new(SharePointPageSuite))
}
func (suite *SharePointPageSuite) TestFetchPages() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
service := createTestBetaService(t, suite.creds)
pgs, err := FetchPages(ctx, service, suite.siteID)
assert.NoError(t, err)
require.NotNil(t, pgs)
assert.NotZero(t, len(pgs))
for _, entry := range pgs {
t.Logf("id: %s\t name: %s\n", entry.ID, entry.Name)
}
}
func (suite *SharePointPageSuite) TestGetSitePage() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
service := createTestBetaService(t, suite.creds)
tuples, err := FetchPages(ctx, service, suite.siteID)
require.NoError(t, err)
require.NotNil(t, tuples)
jobs := []string{tuples[0].ID}
pages, err := GetSitePage(ctx, service, suite.siteID, jobs)
assert.NoError(t, err)
assert.NotEmpty(t, pages)
}

2
src/internal/connector/sharepoint/collection.go
View File

@ -9,6 +9,7 @@ import (
kw "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -46,6 +47,7 @@ type Collection struct {
jobs []string
// M365 IDs of the items of this collection
service graph.Servicer
betaService *api.BetaService
statusUpdater support.StatusUpdater
}

64
src/internal/connector/sharepoint/collection_test.go
View File

@ -17,11 +17,27 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
)
type SharePointCollectionSuite struct {
suite.Suite
siteID string
creds account.M365Config
}
func (suite *SharePointCollectionSuite) SetupSuite() {
t := suite.T()
tester.MustGetEnvSets(t, tester.M365AcctCredEnvs)
suite.siteID = tester.M365SiteID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
suite.creds = m365
}
func TestSharePointCollectionSuite(t *testing.T) {
@ -95,20 +111,33 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName)
}
func (suite *SharePointCollectionSuite) TestCollectPages() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
col, err := collectPages(
ctx,
suite.creds,
nil,
account.AzureTenantID,
suite.siteID,
nil,
&MockGraphService{},
control.Defaults(),
)
assert.NoError(t, err)
assert.NotEmpty(t, col)
}
// TestRestoreListCollection verifies Graph Restore API for the List Collection
func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
siteID := tester.M365SiteID(t)
a := tester.NewM365Account(t)
account, err := a.M365Config()
require.NoError(t, err)
service, err := createTestService(account)
require.NoError(t, err)
service := createTestService(t, suite.creds)
listing := mockconnector.GetMockListDefault("Mock List")
testName := "MockListing"
listing.SetDisplayName(&testName)
@ -123,13 +152,13 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting)
deets, err := restoreListItem(ctx, service, listData, siteID, destName)
deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
assert.NoError(t, err)
t.Logf("List created: %s\n", deets.SharePoint.ItemName)
// Clean-Up
var (
builder = service.Client().SitesById(siteID).Lists()
builder = service.Client().SitesById(suite.siteID).Lists()
isFound bool
deleteID string
)
@ -156,7 +185,7 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
}
if isFound {
err := DeleteList(ctx, service, siteID, deleteID)
err := DeleteList(ctx, service, suite.siteID, deleteID)
assert.NoError(t, err)
}
}
@ -168,25 +197,18 @@ func (suite *SharePointCollectionSuite) TestRestoreLocation() {
defer flush()
t := suite.T()
a := tester.NewM365Account(t)
account, err := a.M365Config()
require.NoError(t, err)
service, err := createTestService(account)
require.NoError(t, err)
service := createTestService(t, suite.creds)
rootFolder := "General_" + common.FormatNow(common.SimpleTimeTesting)
siteID := tester.M365SiteID(t)
folderID, err := createRestoreFolders(ctx, service, siteID, []string{rootFolder})
folderID, err := createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder})
assert.NoError(t, err)
t.Log("FolderID: " + folderID)
_, err = createRestoreFolders(ctx, service, siteID, []string{rootFolder, "Tsao"})
_, err = createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder, "Tsao"})
assert.NoError(t, err)
// CleanUp
siteDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil)
siteDrive, err := service.Client().SitesById(suite.siteID).Drive().Get(ctx, nil)
require.NoError(t, err)
driveID := *siteDrive.GetId()

52
src/internal/connector/sharepoint/data_collections.go
View File

@ -6,11 +6,14 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
sapi "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -162,6 +165,55 @@ func collectLibraries(
return append(collections, odcs...), excludes, errs
}
// collectPages constructs a sharepoint Collections struct and Get()s the associated
// M365 IDs for the associated Pages
func collectPages(
ctx context.Context,
creds account.M365Config,
serv graph.Servicer,
tenantID, siteID string,
scope selectors.SharePointScope,
updater statusUpdater,
ctrlOpts control.Options,
) ([]data.Collection, error) {
logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Pages collections")
spcs := make([]data.Collection, 0)
// make the betaClient
adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret)
if err != nil {
return nil, errors.Wrap(err, "adapter for betaservice not created")
}
betaService := api.NewBetaService(adpt)
tuples, err := sapi.FetchPages(ctx, betaService, siteID)
if err != nil {
return nil, err
}
for _, tuple := range tuples {
dir, err := path.Builder{}.Append(tuple.Name).
ToDataLayerSharePointPath(
tenantID,
siteID,
path.PagesCategory,
false)
if err != nil {
return nil, errors.Wrapf(err, "failed to create collection path for site: %s", siteID)
}
collection := NewCollection(dir, serv, updater.UpdateStatus)
collection.betaService = betaService
collection.AddJob(tuple.ID)
spcs = append(spcs, collection)
}
return spcs, nil
}
type folderMatcher struct {
scope selectors.SharePointScope
}

13
src/internal/connector/sharepoint/helper_test.go
View File

@ -4,11 +4,11 @@ import (
"testing"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/account"
)
@ -29,21 +29,22 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
return nil
}
func (ms *MockGraphService) UpdateStatus(*support.ConnectorOperationStatus) {
}
// ---------------------------------------------------------------------------
// Helper Functions
// ---------------------------------------------------------------------------
func createTestService(credentials account.M365Config) (*graph.Service, error) {
func createTestService(t *testing.T, credentials account.M365Config) *graph.Service {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret,
)
if err != nil {
return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
}
require.NoError(t, err, "creating microsoft graph service for exchange")
return graph.NewService(adapter), nil
return graph.NewService(adapter)
}
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {

4
src/internal/connector/sharepoint/list_test.go
View File

@ -49,9 +49,7 @@ func (suite *SharePointSuite) TestLoadList() {
defer flush()
t := suite.T()
service, err := createTestService(suite.creds)
require.NoError(t, err)
service := createTestService(t, suite.creds)
tuples, err := preFetchLists(ctx, service, "root")
require.NoError(t, err)