larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'restore-page-flow-3' into restore-page-flow-4

Browse Source
This commit is contained in:
Danny Adams 2023-01-28 15:39:39 -05:00
commit c80dce034e
6 changed files with 308 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
working-directory: src
steps:
- uses: actions/checkout@v3
# single setup and sum cache handling here.
# the results will cascade onto both testing and linting.
- name: Setup Golang with cache

47
src/internal/connector/sharepoint/collection_test.go
View File

@ -205,6 +205,53 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
}
}
func (suite *SharePointCollectionSuite) TestRestoreSinglePage() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
siteID := tester.M365SiteID(t)
a := tester.NewM365Account(t)
account, err := a.M365Config()
require.NoError(t, err)
service := createTestBetaService(t, account)
destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting)
testName := "MockPage"
// Create Test Page
//nolint:lll
byteArray := []byte("{\"name\":\"test.aspx\",\"title\":\"test\",\"pageLayout\":\"article\",\"showComments\":true," +
"\"showRecommendedPages\":false,\"titleArea\":{\"enableGradientEffect\":true,\"imageWebUrl\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATETITLEIMAGE.JPG\"," +
"\"layout\":\"colorBlock\",\"showAuthor\":true,\"showPublishedDate\":false,\"showTextBlockAboveTitle\":false,\"textAboveTitle\":\"TEXTABOVETITLE\"," +
"\"textAlignment\":\"left\",\"imageSourceType\":2,\"title\":\"sample1\"}," +
"\"canvasLayout\":{\"horizontalSections\":[{\"layout\":\"oneThirdRightColumn\",\"id\":\"1\",\"emphasis\":\"none\",\"columns\":[{\"id\":\"1\",\"width\":8," +
"\"webparts\":[{\"id\":\"6f9230af-2a98-4952-b205-9ede4f9ef548\",\"innerHtml\":\"<p><b>Hello!</b></p>\"}]},{\"id\":\"2\",\"width\":4," +
"\"webparts\":[{\"id\":\"73d07dde-3474-4545-badb-f28ba239e0e1\",\"webPartType\":\"d1d91016-032f-456d-98a4-721247c305e8\",\"data\":{\"dataVersion\":\"1.9\"," +
"\"description\":\"Showanimageonyourpage\",\"title\":\"Image\",\"properties\":{\"imageSourceType\":2,\"altText\":\"\",\"overlayText\":\"\"," +
"\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\",\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\"," +
"\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"imgWidth\":4288,\"imgHeight\":2848,\"fixAspectRatio\":false,\"captionText\":\"\",\"alignment\":\"Center\"}," +
"\"serverProcessedContent\":{\"imageSources\":[{\"key\":\"imageSource\",\"value\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATEIMAGE1.JPG\"}]," +
"\"customMetadata\":[{\"key\":\"imageSource\",\"value\":{\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\"," +
"\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\",\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"width\":\"4288\",\"height\":\"2848\"}}]}}}]}]}]}}")
pageData := &Item{
id: testName,
data: io.NopCloser(bytes.NewReader(byteArray)),
}
info, err := restoreSitePage(ctx, service, pageData, siteID, destName)
require.NoError(t, err)
require.NotNil(t, info)
// Clean Up
pageID := info.SharePoint.ParentPath
err = DeleteSitePage(ctx, service, siteID, pageID)
assert.NoError(t, err)
}
// TestRestoreLocation temporary test for greater restore operation
// TODO delete after full functionality tested in GraphConnector
func (suite *SharePointCollectionSuite) TestRestoreLocation() {

168
src/internal/connector/sharepoint/restore.go
View File

@ -10,6 +10,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -68,7 +69,7 @@ func RestoreCollections(
deets,
errUpdater)
case path.ListsCategory:
metrics, canceled = RestoreCollection(
metrics, canceled = RestoreListCollection(
ctx,
service,
dc,
@ -77,11 +78,14 @@ func RestoreCollections(
errUpdater,
)
case path.PagesCategory:
errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category())
logger.Ctx(ctx).Error(errorMessage)
return nil, errors.New(errorMessage)
metrics, canceled = RestorePageCollection(
ctx,
service,
dc,
dest.ContainerName,
deets,
errUpdater,
)
default:
return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
}
@ -203,7 +207,7 @@ func restoreListItem(
return dii, nil
}
func RestoreCollection(
func RestoreListCollection(
ctx context.Context,
service graph.Servicer,
dc data.Collection,
@ -211,7 +215,7 @@ func RestoreCollection(
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath()))
ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath()))
defer end()
var (
@ -219,7 +223,7 @@ func RestoreCollection(
directory = dc.FullPath()
)
trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String())
trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from the collection
@ -270,3 +274,149 @@ func RestoreCollection(
}
}
}
// RestorePageCollection handles restoration of an individual site page collection.
// returns:
// - the collection's item and byte count metrics
// - the context cancellation station. True iff context is canceled.
func RestorePageCollection(
ctx context.Context,
service graph.Servicer,
dc data.Collection,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
defer end()
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
)
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from collection
items := dc.Items()
for {
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, true
case itemData, ok := <-items:
if !ok {
return metrics, false
}
metrics.Objects++
itemInfo, err := restoreSitePage(
ctx,
service,
itemData,
siteID,
restoreContainerName,
)
if err != nil {
errUpdater(itemData.UUID(), err)
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).Errorw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
continue
}
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
true,
itemInfo,
)
metrics.Successes++
}
}
}
// restoreSitePage handles the restoration of single site page to SharePoint.
// The new M365ID is placed within the details.ItemInfo
func restoreSitePage(
ctx context.Context,
service *betasdk.Service,
itemData data.Stream,
siteID, destName string,
) (details.ItemInfo, error) {
ctx, end := D.Span(ctx, "gc:sharepoint:restorePage", D.Label("item_uuid", itemData.UUID()))
defer end()
var (
dii = details.ItemInfo{}
pageID = itemData.UUID()
pageName = pageID
)
byteArray, err := io.ReadAll(itemData.ToReader())
if err != nil {
return dii, errors.Wrap(err, "reading sharepoint page bytes from stream")
}
// Hydrate Page
page, err := support.CreatePageFromBytes(byteArray)
if err != nil {
return dii, errors.Wrapf(err, "creating Page object %s", pageID)
}
pageNamePtr := page.GetName()
if pageNamePtr != nil {
pageName = *pageNamePtr
}
newName := fmt.Sprintf("%s_%s", destName, pageName)
page.SetName(&newName)
// Restore is a 2-Step Process in Graph API
// 1. Create the Page on the site
// 2. Publish the site
// See: https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta
restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil)
if err != nil {
sendErr := support.ConnectorStackErrorTraceWrap(
err,
"creating page from ID: %s"+pageName+" API Error Details",
)
return dii, sendErr
}
// Publish page to make visible
// See https://learn.microsoft.com/en-us/graph/api/sitepage-publish?view=graph-rest-beta
if restoredPage.GetWebUrl() == nil {
return dii, fmt.Errorf("creating page %s incomplete. Field `webURL` not populated", *restoredPage.GetId())
}
err = service.Client().
SitesById(siteID).
PagesById(*restoredPage.GetId()).Publish().Post(ctx, nil)
if err != nil {
return dii, support.ConnectorStackErrorTraceWrap(
err,
"publishing page ID: "+*restoredPage.GetId()+" API Error Details",
)
}
dii.SharePoint = sharePointPageInfo(restoredPage, int64(len(byteArray)))
// Storing new pageID in unused field.
dii.SharePoint.ParentPath = pageID
return dii, nil
}

15
src/internal/connector/sharepoint/site_page.go
View File

@ -80,6 +80,21 @@ func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration {
return options
}
// DeleteSitePage removes the selected page from the SharePoint Site
// https://learn.microsoft.com/en-us/graph/api/sitepage-delete?view=graph-rest-beta
func DeleteSitePage(
ctx context.Context,
serv *betasdk.Service,
siteID, pageID string,
) error {
err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil)
if err != nil {
return support.ConnectorStackErrorTraceWrap(err, "deleting page: "+pageID)
}
return nil
}
// retrievePageOptions returns options to expand
func retrieveSitePageOptions() *sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration {
fields := []string{"canvasLayout"}

23
src/internal/connector/support/m365Support.go
View File

@ -3,6 +3,7 @@ package support
import (
"strings"
bmodels "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
js "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -14,7 +15,7 @@ import (
func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Parsable, error) {
parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", bytes)
if err != nil {
return nil, errors.Wrap(err, "parsing byte array into m365 object")
return nil, errors.Wrap(err, "deserializing bytes into base m365 object")
}
anObject, err := parseNode.GetObjectValue(createFunc)
@ -29,7 +30,7 @@ func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Pa
func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) {
aMessage, err := CreateFromBytes(bytes, models.CreateMessageFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, "creating m365 exchange.Mail object from provided bytes")
return nil, errors.Wrap(err, "deserializing bytes to exchange message")
}
message := aMessage.(models.Messageable)
@ -42,7 +43,7 @@ func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) {
func CreateContactFromBytes(bytes []byte) (models.Contactable, error) {
parsable, err := CreateFromBytes(bytes, models.CreateContactFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, "creating m365 exchange.Contact object from provided bytes")
return nil, errors.Wrap(err, "deserializing bytes to exchange contact")
}
contact := parsable.(models.Contactable)
@ -54,7 +55,7 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) {
func CreateEventFromBytes(bytes []byte) (models.Eventable, error) {
parsable, err := CreateFromBytes(bytes, models.CreateEventFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, "creating m365 exchange.Event object from provided bytes")
return nil, errors.Wrap(err, "deserializing bytes to exchange event")
}
event := parsable.(models.Eventable)
@ -66,7 +67,7 @@ func CreateEventFromBytes(bytes []byte) (models.Eventable, error) {
func CreateListFromBytes(bytes []byte) (models.Listable, error) {
parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, "creating m365 sharepoint.List object from provided bytes")
return nil, errors.Wrap(err, "deserializing bytes to sharepoint list")
}
list := parsable.(models.Listable)
@ -74,6 +75,18 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) {
return list, nil
}
// CreatePageFromBytes transforms given bytes in models.SitePageable object
func CreatePageFromBytes(bytes []byte) (bmodels.SitePageable, error) {
parsable, err := CreateFromBytes(bytes, bmodels.CreateSitePageFromDiscriminatorValue)
if err != nil {
return nil, errors.Wrap(err, "deserializing bytes to sharepoint page")
}
page := parsable.(bmodels.SitePageable)
return page, nil
}
func HasAttachments(body models.ItemBodyable) bool {
if body.GetContent() == nil || body.GetContentType() == nil ||
*body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 {

74
src/internal/connector/support/m365Support_test.go
View File

@ -3,11 +3,13 @@ package support
import (
"testing"
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
bmodels "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
)
@ -19,6 +21,11 @@ func TestDataSupportSuite(t *testing.T) {
suite.Run(t, new(DataSupportSuite))
}
var (
empty = "Empty Bytes"
invalid = "Invalid Bytes"
)
// TestCreateMessageFromBytes verifies approved mockdata bytes can
// be successfully transformed into M365 Message data.
func (suite *DataSupportSuite) TestCreateMessageFromBytes() {
@ -60,13 +67,13 @@ func (suite *DataSupportSuite) TestCreateContactFromBytes() {
isNil assert.ValueAssertionFunc
}{
{
name: "Empty Bytes",
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
isNil: assert.Nil,
},
{
name: "Invalid Bytes",
name: invalid,
byteArray: []byte("A random sentence doesn't make an object"),
checkError: assert.Error,
isNil: assert.Nil,
@ -95,13 +102,13 @@ func (suite *DataSupportSuite) TestCreateEventFromBytes() {
isNil assert.ValueAssertionFunc
}{
{
name: "Empty Byes",
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
isNil: assert.Nil,
},
{
name: "Invalid Bytes",
name: invalid,
byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"),
checkError: assert.Error,
isNil: assert.Nil,
@ -133,13 +140,13 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() {
isNil assert.ValueAssertionFunc
}{
{
name: "Empty Byes",
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
isNil: assert.Nil,
},
{
name: "Invalid Bytes",
name: invalid,
byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"),
checkError: assert.Error,
isNil: assert.Nil,
@ -161,6 +168,61 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() {
}
}
func (suite *DataSupportSuite) TestCreatePageFromBytes() {
tests := []struct {
name string
checkError assert.ErrorAssertionFunc
isNil assert.ValueAssertionFunc
getBytes func(t *testing.T) []byte
}{
{
empty,
assert.Error,
assert.Nil,
func(t *testing.T) []byte {
return make([]byte, 0)
},
},
{
invalid,
assert.Error,
assert.Nil,
func(t *testing.T) []byte {
return []byte("snarf")
},
},
{
"Valid Page",
assert.NoError,
assert.NotNil,
func(t *testing.T) []byte {
pg := bmodels.NewSitePage()
title := "Tested"
pg.SetTitle(&title)
pg.SetName(&title)
pg.SetWebUrl(&title)
writer := kioser.NewJsonSerializationWriter()
err := pg.Serialize(writer)
require.NoError(t, err)
byteArray, err := writer.GetSerializedContent()
require.NoError(t, err)
return byteArray
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
result, err := CreatePageFromBytes(test.getBytes(t))
test.checkError(t, err)
test.isNil(t, result)
})
}
}
func (suite *DataSupportSuite) TestHasAttachments() {
tests := []struct {
name string