From da2d0d1779e39cc155444612bf79ce311559f776 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 19:54:37 -0500 Subject: [PATCH 01/16] Update /internal/connector/support/m365Support.go Issue #2169: Serialization logic for SitePageable data types. --- src/internal/connector/support/m365Support.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index c75787045..4abeda28c 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -74,6 +74,18 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { return list, nil } +// CreatePageFromBytes transforms given bytes in models.SitePageable object +func CreatePageFromBytes(bytes []byte) (models.SitePageable, error) { + parsable, err := CreateFromBytes(bytes, models.CreateSitePageFromDiscriminatorValue) + if err != nil { + return nil, errors.Wrap(err, "createing m365 sharepoint.Page object from provided bytes") + } + + page := parsable.(models.SitePageable) + + return page, nil +} + func HasAttachments(body models.ItemBodyable) bool { if body.GetContent() == nil || body.GetContentType() == nil || *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { From d38ccb1cfc9ebca247913dc77743ace262b32dd7 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 19:56:25 -0500 Subject: [PATCH 02/16] Update /internal/connector/support/m365Support_test.go Adds testing for SitePage creation. --- .../connector/support/m365Support_test.go | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index 387e79b7d..e388e6727 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -3,6 +3,7 @@ package support import ( "testing" + kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -133,7 +134,7 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Byes", + name: "Empty Bytes", byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, @@ -161,6 +162,61 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { } } +func (suite *DataSupportSuite) TestCreatePageFromBytes() { + tests := []struct { + name string + checkError assert.ErrorAssertionFunc + isNil assert.ValueAssertionFunc + getBytes func(t *testing.T) []byte + }{ + { + "Empty Bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return make([]byte, 0) + }, + }, + { + "Invalid Bytes", + assert.Error, + assert.Nil, + func(t *testing.T) []byte { + return []byte("snarf") + }, + }, + { + "Valid Page", + assert.NoError, + assert.NotNil, + func(t *testing.T) []byte { + pg := models.NewSitePage() + title := "Tested" + pg.SetTitle(&title) + pg.SetName(&title) + pg.SetWebUrl(&title) + + writer := kioser.NewJsonSerializationWriter() + err := pg.Serialize(writer) + require.NoError(t, err) + + byteArray, err := writer.GetSerializedContent() + require.NoError(t, err) + + return byteArray + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + result, err := CreatePageFromBytes(test.getBytes(t)) + test.checkError(t, err) + test.isNil(t, result) + }) + } +} + func (suite *DataSupportSuite) TestHasAttachments() { tests := []struct { name string From b3cb8fec46e986ca253816a78fe134204bb658ec Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 20:05:58 -0500 Subject: [PATCH 03/16] Updates to /conector/support/m365Support_test.go Use of constant for repeating test names. --- .../connector/support/m365Support_test.go | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index e388e6727..557a91750 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -20,6 +20,11 @@ func TestDataSupportSuite(t *testing.T) { suite.Run(t, new(DataSupportSuite)) } +var ( + empty = "Empty Bytes" + invalid = "Invalid Bytes" +) + // TestCreateMessageFromBytes verifies approved mockdata bytes can // be successfully transformed into M365 Message data. func (suite *DataSupportSuite) TestCreateMessageFromBytes() { @@ -61,13 +66,13 @@ func (suite *DataSupportSuite) TestCreateContactFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Bytes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("A random sentence doesn't make an object"), checkError: assert.Error, isNil: assert.Nil, @@ -96,13 +101,13 @@ func (suite *DataSupportSuite) TestCreateEventFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Byes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), checkError: assert.Error, isNil: assert.Nil, @@ -134,13 +139,13 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { isNil assert.ValueAssertionFunc }{ { - name: "Empty Bytes", + name: empty, byteArray: make([]byte, 0), checkError: assert.Error, isNil: assert.Nil, }, { - name: "Invalid Bytes", + name: invalid, byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"), checkError: assert.Error, isNil: assert.Nil, @@ -170,7 +175,7 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() { getBytes func(t *testing.T) []byte }{ { - "Empty Bytes", + empty, assert.Error, assert.Nil, func(t *testing.T) []byte { @@ -178,7 +183,7 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() { }, }, { - "Invalid Bytes", + invalid, assert.Error, assert.Nil, func(t *testing.T) []byte { From 607ca8317728c587a656f820c9f13e089c47b07c Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 20:52:35 -0500 Subject: [PATCH 04/16] Updates to /connector/sharepoint/restore.go Issue #2169: Restore logic for sharepoint page. --- src/internal/connector/sharepoint/restore.go | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index c0c16fd0f..b851924c2 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -270,3 +270,43 @@ func RestoreCollection( } } } + +func restoreSitePage( + ctx context.Context, + service graph.Servicer, + itemData data.Stream, + siteID, destName string, +) (details.ItemInfo, error) { + ctx, end := D.Span(ctx, "gc:sharepoint:restorePage", D.Label("item_uuid", itemData.UUID())) + defer end() + + var ( + dii = details.ItemInfo{} + pageName = itemData.UUID() + ) + + byteArray, err := io.ReadAll(itemData.ToReader()) + if err != nil { + return dii, errors.Wrap(err, "sharepoint restorePage failed to restore bytes from data.Stream") + } + + // Hydrate Page + page, err := support.CreatePageFromBytes(byteArray) + if err != nil { + return dii, errors.Wrapf(err, "failed to create Page object %s", pageName) + } + + restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil) + if err != nil { + sendErr := support.ConnectorStackErrorTraceWrap( + err, + "failure to create page from ID: %s"+pageName+" API Error Details", + ) + + return dii, sendErr + } + + dii.SharePoint = sharePointPageInfo(restoredPage, int64(len(byteArray))) + + return dii, nil +} From ca641e0116933f1b8daa45d445c0cc780af409ce Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 21:25:34 -0500 Subject: [PATCH 05/16] Updates to /connector/sharepoint/restore.go Issue #2169: Publish API added to restore of Page Item. --- src/internal/connector/sharepoint/restore.go | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index b851924c2..ad3691f3b 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -7,6 +7,7 @@ import ( "runtime/trace" "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -283,6 +284,7 @@ func restoreSitePage( var ( dii = details.ItemInfo{} pageName = itemData.UUID() + pageURL string ) byteArray, err := io.ReadAll(itemData.ToReader()) @@ -296,6 +298,8 @@ func restoreSitePage( return dii, errors.Wrapf(err, "failed to create Page object %s", pageName) } + // POST the Publish + // https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil) if err != nil { sendErr := support.ConnectorStackErrorTraceWrap( @@ -306,6 +310,23 @@ func restoreSitePage( return dii, sendErr } + // Publish page to make visible + // See https://learn.microsoft.com/en-us/graph/api/sitepage-publish?view=graph-rest-beta + if restoredPage.GetWebUrl() == nil { + return dii, fmt.Errorf("created page %s did not return webURL from API", *restoredPage.GetId()) + } + + pageURL = *restoredPage.GetWebUrl() + publishRequest := sites.NewItemPagesItemPublishRequestBuilder(pageURL, service.Adapter()) + + err = publishRequest.Post(ctx, nil) + if err != nil { + return dii, support.ConnectorStackErrorTraceWrap( + err, + "unable to publish page ID: "+*restoredPage.GetId()+" API Error Details", + ) + } + dii.SharePoint = sharePointPageInfo(restoredPage, int64(len(byteArray))) return dii, nil From d16b13bd64fe81b5b97735b4b2e522b55a473ee9 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 23:34:48 -0500 Subject: [PATCH 06/16] Update to /connector/sharepoint/site_page.go Issue #2169: Delete API added for SharePoint Page. --- src/internal/connector/sharepoint/site_page.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/internal/connector/sharepoint/site_page.go b/src/internal/connector/sharepoint/site_page.go index 237090b7c..0b2378053 100644 --- a/src/internal/connector/sharepoint/site_page.go +++ b/src/internal/connector/sharepoint/site_page.go @@ -79,3 +79,18 @@ func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration { return options } + +// DeleteSitePage removes the selected page from the SharePoint Site +// https://learn.microsoft.com/en-us/graph/api/sitepage-delete?view=graph-rest-beta +func DeleteSitePage( + ctx context.Context, + serv graph.Servicer, + siteID, pageID string, +) error { + err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil) + if err != nil { + return support.ConnectorStackErrorTraceWrap(err, "deleting page: "+pageID) + } + + return nil +} From 9ea6e2921b6ca7f0a0d57ed89ca10d01c2a7023b Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 23:36:08 -0500 Subject: [PATCH 07/16] Update to /connector/sharepoint/restore.go Issue #2169: Adds logic to append destination name to file name. Title is not altered. --- src/internal/connector/sharepoint/restore.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index ad3691f3b..84cdc26be 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -7,7 +7,6 @@ import ( "runtime/trace" "github.com/microsoftgraph/msgraph-beta-sdk-go/models" - "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -284,7 +283,6 @@ func restoreSitePage( var ( dii = details.ItemInfo{} pageName = itemData.UUID() - pageURL string ) byteArray, err := io.ReadAll(itemData.ToReader()) @@ -298,8 +296,13 @@ func restoreSitePage( return dii, errors.Wrapf(err, "failed to create Page object %s", pageName) } - // POST the Publish - // https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta + newName := fmt.Sprintf("%s_%s", destName, *page.GetName()) + page.SetName(&newName) + fmt.Printf("Page Name: %s\n", *page.GetName()) + // Restore is a 2-Step Process in Graph API + // 1. Create the Page on the site + // 2. Publish the site + // See: https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil) if err != nil { sendErr := support.ConnectorStackErrorTraceWrap( @@ -316,18 +319,19 @@ func restoreSitePage( return dii, fmt.Errorf("created page %s did not return webURL from API", *restoredPage.GetId()) } - pageURL = *restoredPage.GetWebUrl() - publishRequest := sites.NewItemPagesItemPublishRequestBuilder(pageURL, service.Adapter()) + pageID := *restoredPage.GetId() - err = publishRequest.Post(ctx, nil) + err = service.Client().SitesById(siteID).PagesById(pageID).Publish().Post(ctx, nil) if err != nil { return dii, support.ConnectorStackErrorTraceWrap( err, - "unable to publish page ID: "+*restoredPage.GetId()+" API Error Details", + "unable to publish page ID: "+pageID+" API Error Details", ) } dii.SharePoint = sharePointPageInfo(restoredPage, int64(len(byteArray))) + // Storing new pageID in unused field. + dii.SharePoint.ParentPath = pageID return dii, nil } From 09f92fabc6282ded3a2a7f635a98114bac825dac Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 21 Jan 2023 23:37:33 -0500 Subject: [PATCH 08/16] Updates to /connector/sharepoint/collection_test.go Issue #2169: Checks restore logic for site page. --- .../connector/sharepoint/collection_test.go | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index 3b91b1241..a0770da2f 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -161,6 +161,54 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() { } } +func (suite *SharePointCollectionSuite) TestRestoreSinglePage() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + siteID := tester.M365SiteID(t) + a := tester.NewM365Account(t) + account, err := a.M365Config() + require.NoError(t, err) + + service, err := createTestService(account) + require.NoError(t, err) + + destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting) + testName := "MockPage" + + // Create Test Page + //nolint:lll + byteArray := []byte("{\"name\":\"test.aspx\",\"title\":\"test\",\"pageLayout\":\"article\",\"showComments\":true," + + "\"showRecommendedPages\":false,\"titleArea\":{\"enableGradientEffect\":true,\"imageWebUrl\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATETITLEIMAGE.JPG\"," + + "\"layout\":\"colorBlock\",\"showAuthor\":true,\"showPublishedDate\":false,\"showTextBlockAboveTitle\":false,\"textAboveTitle\":\"TEXTABOVETITLE\"," + + "\"textAlignment\":\"left\",\"imageSourceType\":2,\"title\":\"sample1\"}," + + "\"canvasLayout\":{\"horizontalSections\":[{\"layout\":\"oneThirdRightColumn\",\"id\":\"1\",\"emphasis\":\"none\",\"columns\":[{\"id\":\"1\",\"width\":8," + + "\"webparts\":[{\"id\":\"6f9230af-2a98-4952-b205-9ede4f9ef548\",\"innerHtml\":\"

Hello!

\"}]},{\"id\":\"2\",\"width\":4," + + "\"webparts\":[{\"id\":\"73d07dde-3474-4545-badb-f28ba239e0e1\",\"webPartType\":\"d1d91016-032f-456d-98a4-721247c305e8\",\"data\":{\"dataVersion\":\"1.9\"," + + "\"description\":\"Showanimageonyourpage\",\"title\":\"Image\",\"properties\":{\"imageSourceType\":2,\"altText\":\"\",\"overlayText\":\"\"," + + "\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\",\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\"," + + "\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"imgWidth\":4288,\"imgHeight\":2848,\"fixAspectRatio\":false,\"captionText\":\"\",\"alignment\":\"Center\"}," + + "\"serverProcessedContent\":{\"imageSources\":[{\"key\":\"imageSource\",\"value\":\"/_LAYOUTS/IMAGES/VISUALTEMPLATEIMAGE1.JPG\"}]," + + "\"customMetadata\":[{\"key\":\"imageSource\",\"value\":{\"siteid\":\"0264cabe-6b92-450a-b162-b0c3d54fe5e8\",\"webid\":\"f3989670-cd37-4514-8ccb-0f7c2cbe5314\"," + + "\"listid\":\"bdb41041-eb06-474e-ac29-87093386bb14\",\"uniqueid\":\"d9f94b40-78ba-48d0-a39f-3cb23c2fe7eb\",\"width\":\"4288\",\"height\":\"2848\"}}]}}}]}]}]}}") + + pageData := &Item{ + id: testName, + data: io.NopCloser(bytes.NewReader(byteArray)), + } + + info, err := restoreSitePage(ctx, service, pageData, siteID, destName) + + require.NoError(t, err) + require.NotNil(t, info) + + // Clean Up + pageID := info.SharePoint.ParentPath + err = DeleteSitePage(ctx, service, siteID, pageID) + assert.NoError(t, err) +} + // TestRestoreLocation temporary test for greater restore operation // TODO delete after full functionality tested in GraphConnector func (suite *SharePointCollectionSuite) TestRestoreLocation() { From 22c070885b70e1b0398487de9dbb931f3cb8ad05 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sun, 22 Jan 2023 00:58:33 -0500 Subject: [PATCH 09/16] Updates to /connector/sharepoint/restore.go Issue #2169: Restore Collection logic added. --- src/internal/connector/sharepoint/restore.go | 78 +++++++++++++++++++- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 84cdc26be..030e91ca6 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -68,7 +68,7 @@ func RestoreCollections( deets, errUpdater) case path.ListsCategory: - metrics, canceled = RestoreCollection( + metrics, canceled = RestoreListCollection( ctx, service, dc, @@ -203,7 +203,7 @@ func restoreListItem( return dii, nil } -func RestoreCollection( +func RestoreListCollection( ctx context.Context, service graph.Servicer, dc data.Collection, @@ -211,7 +211,7 @@ func RestoreCollection( deets *details.Builder, errUpdater func(string, error), ) (support.CollectionMetrics, bool) { - ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath())) + ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath())) defer end() var ( @@ -219,7 +219,7 @@ func RestoreCollection( directory = dc.FullPath() ) - trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String()) + trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String()) siteID := directory.ResourceOwner() // Restore items from the collection @@ -271,6 +271,76 @@ func RestoreCollection( } } +func RestorePageCollection( + ctx context.Context, + service graph.Servicer, + dc data.Collection, + restoreContainerName string, + deets *details.Builder, + errUpdater func(string, error), +) (support.CollectionMetrics, bool) { + ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath())) + defer end() + + var ( + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + ) + + trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String()) + siteID := directory.ResourceOwner() + + // Restore items from collection + items := dc.Items() + + for { + select { + case <-ctx.Done(): + errUpdater("context canceled", ctx.Err()) + return metrics, true + + case itemData, ok := <-items: + if !ok { + return metrics, false + } + metrics.Objects++ + + itemInfo, err := restoreSitePage( + ctx, + service, + itemData, + siteID, + restoreContainerName, + ) + if err != nil { + errUpdater(itemData.UUID(), err) + continue + } + + metrics.TotalBytes += itemInfo.SharePoint.Size + + itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + if err != nil { + logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err) + errUpdater(itemData.UUID(), err) + + continue + } + + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + true, + itemInfo, + ) + + metrics.Successes++ + } + } + +} + func restoreSitePage( ctx context.Context, service graph.Servicer, From 018a57cb3066838d7bd6a26bb92e0a92ab3ac79f Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sun, 22 Jan 2023 01:14:02 -0500 Subject: [PATCH 10/16] GOFMT: white space removed --- src/internal/connector/sharepoint/restore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 030e91ca6..739dd89fd 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -338,7 +338,6 @@ func RestorePageCollection( metrics.Successes++ } } - } func restoreSitePage( From 51ad7458b4f3e28c8090199d8079ad3655ea81fa Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sun, 22 Jan 2023 01:31:59 -0500 Subject: [PATCH 11/16] Updates /connector/sharepoint/restore.go Issue #2169: Connects the restore pipeline for sharepoint pages. Comments also updated for SharePoint Pages. --- src/internal/connector/sharepoint/restore.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 739dd89fd..59d71d8a3 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -77,11 +77,14 @@ func RestoreCollections( errUpdater, ) case path.PagesCategory: - errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category()) - logger.Ctx(ctx).Error(errorMessage) - - return nil, errors.New(errorMessage) - + metrics, canceled = RestorePageCollection( + ctx, + service, + dc, + dest.ContainerName, + deets, + errUpdater, + ) default: return nil, errors.Errorf("category %s not supported", dc.FullPath().Category()) } @@ -271,6 +274,10 @@ func RestoreListCollection( } } +// RestorePageCollection handles restoration of an individual site page collection. +// returns: +// - the collection's item and byte count metrics +// - the context cancellation station. True iff context is canceled. func RestorePageCollection( ctx context.Context, service graph.Servicer, @@ -340,6 +347,8 @@ func RestorePageCollection( } } +// restoreSitePage handles the restoration of single site page to SharePoint. +// The new M365ID is placed within the details.ItemInfo func restoreSitePage( ctx context.Context, service graph.Servicer, From 3d807a13049c680d1fd036c311d7ab845e9c7fd5 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 28 Jan 2023 14:01:37 -0500 Subject: [PATCH 12/16] GOFMT: wsl --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32acffa53..c1e993f12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: working-directory: src steps: - uses: actions/checkout@v3 - + # single setup and sum cache handling here. # the results will cascade onto both testing and linting. - name: Setup Golang with cache From 0340ce67fa7169e4d9ab2c9c9de07041f0899d3d Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 28 Jan 2023 14:13:01 -0500 Subject: [PATCH 13/16] Updates to /support/m365Support.go PR Suggestion: Error messages updated for clarity. --- src/internal/connector/support/m365Support.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index cc5fb89e9..0780a2b0e 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -15,7 +15,7 @@ import ( func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Parsable, error) { parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", bytes) if err != nil { - return nil, errors.Wrap(err, "parsing byte array into m365 object") + return nil, errors.Wrap(err, "deserializing bytes into base m365 object") } anObject, err := parseNode.GetObjectValue(createFunc) @@ -30,7 +30,7 @@ func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Pa func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) { aMessage, err := CreateFromBytes(bytes, models.CreateMessageFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Mail object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange message") } message := aMessage.(models.Messageable) @@ -43,7 +43,7 @@ func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) { func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { parsable, err := CreateFromBytes(bytes, models.CreateContactFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Contact object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange contact") } contact := parsable.(models.Contactable) @@ -55,7 +55,7 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { parsable, err := CreateFromBytes(bytes, models.CreateEventFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 exchange.Event object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to exchange event") } event := parsable.(models.Eventable) @@ -67,7 +67,7 @@ func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { func CreateListFromBytes(bytes []byte) (models.Listable, error) { parsable, err := CreateFromBytes(bytes, models.CreateListFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "creating m365 sharepoint.List object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to sharepoint list") } list := parsable.(models.Listable) @@ -79,7 +79,7 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { func CreatePageFromBytes(bytes []byte) (bmodels.SitePageable, error) { parsable, err := CreateFromBytes(bytes, bmodels.CreateSitePageFromDiscriminatorValue) if err != nil { - return nil, errors.Wrap(err, "createing m365 sharepoint.Page object from provided bytes") + return nil, errors.Wrap(err, "deserializing bytes to sharepoint page") } page := parsable.(bmodels.SitePageable) From e77fd9c0d81c877ff027714277ed1357e9ae4e27 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 28 Jan 2023 14:43:38 -0500 Subject: [PATCH 14/16] Updates to internal/connecto/sharepoint/restore.go Error messages updated for clarity. --- src/internal/connector/sharepoint/restore.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 24af3c88d..5df59ad56 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -283,7 +283,8 @@ func restoreSitePage( var ( dii = details.ItemInfo{} - pageName = itemData.UUID() + pageID = itemData.UUID() + pageName = pageID ) byteArray, err := io.ReadAll(itemData.ToReader()) @@ -294,12 +295,17 @@ func restoreSitePage( // Hydrate Page page, err := support.CreatePageFromBytes(byteArray) if err != nil { - return dii, errors.Wrapf(err, "failed to create Page object %s", pageName) + return dii, errors.Wrapf(err, "failed to create Page object %s", pageID) } - newName := fmt.Sprintf("%s_%s", destName, *page.GetName()) + pageNamePtr := page.GetName() + if pageNamePtr != nil { + pageName = *pageNamePtr + } + + newName := fmt.Sprintf("%s_%s", destName, pageName) page.SetName(&newName) - fmt.Printf("Page Name: %s\n", *page.GetName()) + // Restore is a 2-Step Process in Graph API // 1. Create the Page on the site // 2. Publish the site From b349d9ed2ef05e7e93181ddf0f00275e2632b3e7 Mon Sep 17 00:00:00 2001 From: Danny Adams Date: Sat, 28 Jan 2023 15:07:05 -0500 Subject: [PATCH 15/16] Updates made /sharepoint/restore.go Variable assignment removed to accomodate website-linting failure. --- src/internal/connector/sharepoint/restore.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 5df59ad56..977411889 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -289,13 +289,13 @@ func restoreSitePage( byteArray, err := io.ReadAll(itemData.ToReader()) if err != nil { - return dii, errors.Wrap(err, "sharepoint restorePage failed to restore bytes from data.Stream") + return dii, errors.Wrap(err, "reading sharepoint page bytes from stream") } // Hydrate Page page, err := support.CreatePageFromBytes(byteArray) if err != nil { - return dii, errors.Wrapf(err, "failed to create Page object %s", pageID) + return dii, errors.Wrapf(err, "creating Page object %s", pageID) } pageNamePtr := page.GetName() @@ -314,7 +314,7 @@ func restoreSitePage( if err != nil { sendErr := support.ConnectorStackErrorTraceWrap( err, - "failure to create page from ID: %s"+pageName+" API Error Details", + "creating page from ID: %s"+pageName+" API Error Details", ) return dii, sendErr @@ -323,16 +323,16 @@ func restoreSitePage( // Publish page to make visible // See https://learn.microsoft.com/en-us/graph/api/sitepage-publish?view=graph-rest-beta if restoredPage.GetWebUrl() == nil { - return dii, fmt.Errorf("created page %s did not return webURL from API", *restoredPage.GetId()) + return dii, fmt.Errorf("creating page %s incomplete. Field `webURL` not populated", *restoredPage.GetId()) } - pageID := *restoredPage.GetId() - - err = service.Client().SitesById(siteID).PagesById(pageID).Publish().Post(ctx, nil) + err = service.Client(). + SitesById(siteID). + PagesById(*restoredPage.GetId()).Publish().Post(ctx, nil) if err != nil { return dii, support.ConnectorStackErrorTraceWrap( err, - "unable to publish page ID: "+pageID+" API Error Details", + "publishing page ID: "+*restoredPage.GetId()+" API Error Details", ) } From c439ffd74c41adaf0efca2a278079f72f9e157d7 Mon Sep 17 00:00:00 2001 From: Danny Date: Sat, 28 Jan 2023 15:16:25 -0500 Subject: [PATCH 16/16] Update src/internal/connector/sharepoint/restore.go Update for logging error. Co-authored-by: Keepers --- src/internal/connector/sharepoint/restore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index c68032ae5..3ba8ce657 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -329,7 +329,7 @@ func RestorePageCollection( itemPath, err := dc.FullPath().Append(itemData.UUID(), true) if err != nil { - logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err) + logger.Ctx(ctx).Errorw("transforming item to full path", "error", err) errUpdater(itemData.UUID(), err) continue