diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32acffa53..c1e993f12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: working-directory: src steps: - uses: actions/checkout@v3 - + # single setup and sum cache handling here. # the results will cascade onto both testing and linting. - name: Setup Golang with cache diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index 1a5ea2218..a7e648541 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -205,6 +205,53 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() { } } +func (suite *SharePointCollectionSuite) TestRestoreSinglePage() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + siteID := tester.M365SiteID(t) + a := tester.NewM365Account(t) + account, err := a.M365Config() + require.NoError(t, err) + + service := createTestBetaService(t, account) + + destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting) + testName := "MockPage" + + // Create Test Page + //nolint:lll + byteArray := []byte("{\"name\":\"test.aspx\",\"title\":\"test\",\"pageLayout\":\"article\",\"showComments\":true," + + "\"showRecommendedPages\":false,\"titleArea\":{\"enableGradientEffect\":true,\"imageWebUrl\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATETITLEIMAGE.JPG\"," + + "\"layout\":\"colorBlock\",\"showAuthor\":true,\"showPublishedDate\":false,\"showTextBlockAboveTitle\":false,\"textAboveTitle\":\"TEXTABOVETITLE\"," + + "\"textAlignment\":\"left\",\"imageSourceType\":2,\"title\":\"sample1\"}," + + "\"canvasLayout\":{\"horizontalSections\":[{\"layout\":\"oneThirdRightColumn\",\"id\":\"1\",\"emphasis\":\"none\",\"columns\":[{\"id\":\"1\",\"width\":8," + + "\"webparts\":[{\"id\":\"6f9230af-2a98-4952-b205-9ede4f9ef548\",\"innerHtml\":\"

Hello!

\"}]},{\"id\":\"2\",\"width\":4," + + "\"webparts\":[{\"id\":\"73d07dde-3474-4545-badb-f28ba239e0e1\",\"webPartType\":\"d1d91016-032f-456d-98a4-721247c305e8\",\"data\":{\"dataVersion\":\"1.9\"," + + "\"description\":\"Showanimageonyourpage\",\"title\":\"Image\",\"properties\":{\"imageSourceType\":2,\"altText\":\"\",\"overlayText\":\"\"," + + "\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\",\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\"," + + "\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"imgWidth\":4288,\"imgHeight\":2848,\"fixAspectRatio\":false,\"captionText\":\"\",\"alignment\":\"Center\"}," + + "\"serverProcessedContent\":{\"imageSources\":[{\"key\":\"imageSource\",\"value\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATEIMAGE1.JPG\"}]," + + "\"customMetadata\":[{\"key\":\"imageSource\",\"value\":{\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\"," + + "\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\",\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"width\":\"4288\",\"height\":\"2848\"}}]}}}]}]}]}}") + + pageData := &Item{ + id: testName, + data: io.NopCloser(bytes.NewReader(byteArray)), + } + + info, err := restoreSitePage(ctx, service, pageData, siteID, destName) + + require.NoError(t, err) + require.NotNil(t, info) + + // Clean Up + pageID := info.SharePoint.ParentPath + err = DeleteSitePage(ctx, service, siteID, pageID) + assert.NoError(t, err) +} + // TestRestoreLocation temporary test for greater restore operation // TODO delete after full functionality tested in GraphConnector func (suite *SharePointCollectionSuite) TestRestoreLocation() { diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index ef2b940bb..3ba8ce657 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/betasdk" "github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -68,7 +69,7 @@ func RestoreCollections( deets, errUpdater) case path.ListsCategory: - metrics, canceled = RestoreCollection( + metrics, canceled = RestoreListCollection( ctx, service, dc, @@ -77,11 +78,14 @@ func RestoreCollections( errUpdater, ) case path.PagesCategory: - errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category()) - logger.Ctx(ctx).Error(errorMessage) - - return nil, errors.New(errorMessage) - + metrics, canceled = RestorePageCollection( + ctx, + service, + dc, + dest.ContainerName, + deets, + errUpdater, + ) default: return nil, errors.Errorf("category %s not supported", dc.FullPath().Category()) } @@ -203,7 +207,7 @@ func restoreListItem( return dii, nil } -func RestoreCollection( +func RestoreListCollection( ctx context.Context, service graph.Servicer, dc data.Collection, @@ -211,7 +215,7 @@ func RestoreCollection( deets *details.Builder, errUpdater func(string, error), ) (support.CollectionMetrics, bool) { - ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath())) + ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath())) defer end() var ( @@ -219,7 +223,7 @@ func RestoreCollection( directory = dc.FullPath() ) - trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String()) + trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String()) siteID := directory.ResourceOwner() // Restore items from the collection @@ -270,3 +274,149 @@ func RestoreCollection( } } } + +// RestorePageCollection handles restoration of an individual site page collection. +// returns: +// - the collection's item and byte count metrics +// - the context cancellation station. True iff context is canceled. +func RestorePageCollection( + ctx context.Context, + service graph.Servicer, + dc data.Collection, + restoreContainerName string, + deets *details.Builder, + errUpdater func(string, error), +) (support.CollectionMetrics, bool) { + ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath())) + defer end() + + var ( + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + ) + + trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String()) + siteID := directory.ResourceOwner() + + // Restore items from collection + items := dc.Items() + + for { + select { + case <-ctx.Done(): + errUpdater("context canceled", ctx.Err()) + return metrics, true + + case itemData, ok := <-items: + if !ok { + return metrics, false + } + metrics.Objects++ + + itemInfo, err := restoreSitePage( + ctx, + service, + itemData, + siteID, + restoreContainerName, + ) + if err != nil { + errUpdater(itemData.UUID(), err) + continue + } + + metrics.TotalBytes += itemInfo.SharePoint.Size + + itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + if err != nil { + logger.Ctx(ctx).Errorw("transforming item to full path", "error", err) + errUpdater(itemData.UUID(), err) + + continue + } + + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + true, + itemInfo, + ) + + metrics.Successes++ + } + } +} + +// restoreSitePage handles the restoration of single site page to SharePoint. +// The new M365ID is placed within the details.ItemInfo +func restoreSitePage( + ctx context.Context, + service *betasdk.Service, + itemData data.Stream, + siteID, destName string, +) (details.ItemInfo, error) { + ctx, end := D.Span(ctx, "gc:sharepoint:restorePage", D.Label("item_uuid", itemData.UUID())) + defer end() + + var ( + dii = details.ItemInfo{} + pageID = itemData.UUID() + pageName = pageID + ) + + byteArray, err := io.ReadAll(itemData.ToReader()) + if err != nil { + return dii, errors.Wrap(err, "reading sharepoint page bytes from stream") + } + + // Hydrate Page + page, err := support.CreatePageFromBytes(byteArray) + if err != nil { + return dii, errors.Wrapf(err, "creating Page object %s", pageID) + } + + pageNamePtr := page.GetName() + if pageNamePtr != nil { + pageName = *pageNamePtr + } + + newName := fmt.Sprintf("%s_%s", destName, pageName) + page.SetName(&newName) + + // Restore is a 2-Step Process in Graph API + // 1. Create the Page on the site + // 2. Publish the site + // See: https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta + restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil) + if err != nil { + sendErr := support.ConnectorStackErrorTraceWrap( + err, + "creating page from ID: %s"+pageName+" API Error Details", + ) + + return dii, sendErr + } + + // Publish page to make visible + // See https://learn.microsoft.com/en-us/graph/api/sitepage-publish?view=graph-rest-beta + if restoredPage.GetWebUrl() == nil { + return dii, fmt.Errorf("creating page %s incomplete. Field `webURL` not populated", *restoredPage.GetId()) + } + + err = service.Client(). + SitesById(siteID). + PagesById(*restoredPage.GetId()).Publish().Post(ctx, nil) + if err != nil { + return dii, support.ConnectorStackErrorTraceWrap( + err, + "publishing page ID: "+*restoredPage.GetId()+" API Error Details", + ) + } + + dii.SharePoint = sharePointPageInfo(restoredPage, int64(len(byteArray))) + // Storing new pageID in unused field. + dii.SharePoint.ParentPath = pageID + + return dii, nil +} diff --git a/src/internal/connector/sharepoint/site_page.go b/src/internal/connector/sharepoint/site_page.go index e8ba0ab42..2cb4e41f6 100644 --- a/src/internal/connector/sharepoint/site_page.go +++ b/src/internal/connector/sharepoint/site_page.go @@ -80,6 +80,21 @@ func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration { return options } +// DeleteSitePage removes the selected page from the SharePoint Site +// https://learn.microsoft.com/en-us/graph/api/sitepage-delete?view=graph-rest-beta +func DeleteSitePage( + ctx context.Context, + serv *betasdk.Service, + siteID, pageID string, +) error { + err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil) + if err != nil { + return support.ConnectorStackErrorTraceWrap(err, "deleting page: "+pageID) + } + + return nil +} + // retrievePageOptions returns options to expand func retrieveSitePageOptions() *sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration { fields := []string{"canvasLayout"} diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index 99cb95577..0780a2b0e 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -3,6 +3,7 @@ package support import ( "strings" + bmodels "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models" absser "github.com/microsoft/kiota-abstractions-go/serialization" js "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -14,7 +15,7 @@ import ( func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Parsable, error) { parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", bytes) if err != nil { - return nil, errors.Wrap(err, "parsing byte array into m365 object") + return nil, errors.Wrap(err, "deserializing bytes into base m365 object") } anObject, err := parseNode.GetObjectValue(createFunc) @@ -29,7 +30,7 @@ func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Pa func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) { aMessage, err := CreateFromBytes(bytes, models.CreateMessageFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Mail object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange message") } message := aMessage.(models.Messageable) @@ -42,7 +43,7 @@ func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) { func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { parsable, err := CreateFromBytes(bytes, models.CreateContactFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Contact object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange contact") } contact := parsable.(models.Contactable) @@ -54,7 +55,7 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { parsable, err := CreateFromBytes(bytes, models.CreateEventFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Event object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange event") } event := parsable.(models.Eventable) @@ -66,7 +67,7 @@ func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { func CreateListFromBytes(bytes []byte) (models.Listable, error) { parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 sharepoint.List object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to sharepoint list") } list := parsable.(models.Listable) @@ -74,6 +75,18 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { return list, nil } +// CreatePageFromBytes transforms given bytes in models.SitePageable object +func CreatePageFromBytes(bytes []byte) (bmodels.SitePageable, error) { + parsable, err := CreateFromBytes(bytes, bmodels.CreateSitePageFromDiscriminatorValue) + if err != nil { + return nil, errors.Wrap(err, "deserializing bytes to sharepoint page") + } + + page := parsable.(bmodels.SitePageable) + + return page, nil +} + func HasAttachments(body models.ItemBodyable) bool { if body.GetContent() == nil || body.GetContentType() == nil || *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index dedde3536..946996431 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -3,11 +3,13 @@ package support import ( "testing" + kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + bmodels "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models" "github.com/alcionai/corso/src/internal/connector/mockconnector" ) @@ -19,6 +21,11 @@ func TestDataSupportSuite(t *testing.T) { suite.Run(t, new(DataSupportSuite)) } +var ( + empty = "Empty Bytes" + invalid = "Invalid Bytes" +) + // TestCreateMessageFromBytes verifies approved mockdata bytes can // be successfully transformed into M365 Message data. func (suite *DataSupportSuite) TestCreateMessageFromBytes() { @@ -60,13 +67,13 @@ func (suite *DataSupportSuite) TestCreateContactFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Bytes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("A random sentence doesn't make an object"), checkError: assert.Error, isNil: assert.Nil, @@ -95,13 +102,13 @@ func (suite *DataSupportSuite) TestCreateEventFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Byes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), checkError: assert.Error, isNil: assert.Nil, @@ -133,13 +140,13 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Byes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), checkError: assert.Error, isNil: assert.Nil, @@ -161,6 +168,61 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { } } +func (suite *DataSupportSuite) TestCreatePageFromBytes() { + tests := []struct { + name string + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + getBytes func(t *testing.T) []byte + }{ + { + empty, + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return make([]byte, 0) + }, + }, + { + invalid, + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return []byte("snarf") + }, + }, + { + "Valid Page", + assert.NoError, + assert.NotNil, + func(t *testing.T) []byte { + pg := bmodels.NewSitePage() + title := "Tested" + pg.SetTitle(&title) + pg.SetName(&title) + pg.SetWebUrl(&title) + + writer := kioser.NewJsonSerializationWriter() + err := pg.Serialize(writer) + require.NoError(t, err) + + byteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + return byteArray + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + result, err := CreatePageFromBytes(test.getBytes(t)) + test.checkError(t, err) + test.isNil(t, result) + }) + } +} + func (suite *DataSupportSuite) TestHasAttachments() { tests := []struct { name string