From 936789b6b4a5d4036eff76b1b178f51a0982d147 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 18 Jan 2023 22:19:15 -0500 Subject: [PATCH 1/3] GC: Backup: Sharepoint libraries to omit site drive. (#2098) ## Description Branch treats Drive `Site Pages` as a restricted directory. All `pages` within the directory, will be backed up via the Site Pages API. This adds an additional call to per collection to obtain the parent `driveName`. TODO: Create a Pipeline that distinguishes between SitePages, Libraries, and List for SharePoint ## Does this PR need a docs update or release note? - [x] :clock1: Yes, but in a later PR The documentation should state the assumption is that only `.aspx` files are to be found in the s ## Type of change - [x] :sunflower: Feature ## Issue(s) * closes #2072 ## Test Plan - [x] :zap: Unit test --- src/internal/connector/onedrive/collections.go | 13 ++++++++++--- src/internal/connector/onedrive/collections_test.go | 2 +- src/internal/connector/onedrive/drive.go | 9 +++++---- src/internal/connector/onedrive/item_test.go | 4 ++-- .../connector/sharepoint/data_collections_test.go | 2 +- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 7bdbfc8f5..6a59104f1 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -24,6 +24,7 @@ const ( OneDriveSource SharePointSource ) +const restrictedDirectory = "Site Pages" func (ds driveSource) toPathServiceCat() (path.ServiceType, path.CategoryType) { switch ds { @@ -102,8 +103,9 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { // Update the collection map with items from each drive for _, d := range drives { driveID := *d.GetId() + driveName := *d.GetName() - delta, paths, err := collectItems(ctx, c.service, driveID, c.UpdateCollections) + delta, paths, err := collectItems(ctx, c.service, driveID, driveName, c.UpdateCollections) if err != nil { return nil, err } @@ -162,7 +164,7 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { // A new collection is created for every drive folder (or package) func (c *Collections) UpdateCollections( ctx context.Context, - driveID string, + driveID, driveName string, items []models.DriveItemable, paths map[string]string, ) error { @@ -188,7 +190,7 @@ func (c *Collections) UpdateCollections( } // Skip items that don't match the folder selectors we were given. - if !includePath(ctx, c.matcher, collectionPath) { + if shouldSkipDrive(ctx, collectionPath, c.matcher, driveName) { logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String()) continue } @@ -239,6 +241,11 @@ func (c *Collections) UpdateCollections( return nil } +func shouldSkipDrive(ctx context.Context, drivePath path.Path, m folderMatcher, driveName string) bool { + return !includePath(ctx, m, drivePath) || + (drivePath.Category() == path.LibrariesCategory && restrictedDirectory == driveName) +} + // GetCanonicalPath constructs the standard path for the given source. func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) { var ( diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index f31ae8bab..5a0775edc 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -326,7 +326,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", tt.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index f063eec53..e98e8d35b 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -163,7 +163,7 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod // itemCollector functions collect the items found in a drive type itemCollector func( ctx context.Context, - driveID string, + driveID, driveName string, driveItems []models.DriveItemable, paths map[string]string, ) error @@ -173,7 +173,7 @@ type itemCollector func( func collectItems( ctx context.Context, service graph.Servicer, - driveID string, + driveID, driveName string, collector itemCollector, ) (string, map[string]string, error) { var ( @@ -219,7 +219,7 @@ func collectItems( ) } - err = collector(ctx, driveID, r.GetValue(), paths) + err = collector(ctx, driveID, driveName, r.GetValue(), paths) if err != nil { return "", nil, err } @@ -349,9 +349,10 @@ func GetAllFolders( ctx, gs, *d.GetId(), + *d.GetName(), func( innerCtx context.Context, - driveID string, + driveID, driveName string, items []models.DriveItemable, paths map[string]string, ) error { diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index d87878fc4..5c2e8c335 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -97,7 +97,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { // This item collector tries to find "a" drive item that is a file to test the reader function itemCollector := func( ctx context.Context, - driveID string, + driveID, driveName string, items []models.DriveItemable, paths map[string]string, ) error { @@ -110,7 +110,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { return nil } - _, _, err := collectItems(ctx, suite, suite.userDriveID, itemCollector) + _, _, err := collectItems(ctx, suite, suite.userDriveID, "General", itemCollector) require.NoError(suite.T(), err) // Test Requirement 2: Need a file diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index f52a642ba..2934b2fa1 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -96,7 +96,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { &MockGraphService{}, nil, control.Options{}) - err := c.UpdateCollections(ctx, "driveID", test.items, paths) + err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths) test.expect(t, err) assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths") assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") From 1db101ca1744806d7a635207319780ff1d06af2c Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 18 Jan 2023 22:42:31 -0500 Subject: [PATCH 2/3] GC: Restore: SharePoint add case for Page Category in Workflow. (#2114) ## Description Fix to allow selectors to include `SharePoint.Pages` during backup without causing backup workflow to fail. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :bug: Bugfix ## Issue(s) *closes #2113 ## Test Plan - [x] :muscle: Manual --- src/internal/connector/sharepoint/restore.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 943ef7773..ef2b940bb 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -76,6 +76,12 @@ func RestoreCollections( deets, errUpdater, ) + case path.PagesCategory: + errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category()) + logger.Ctx(ctx).Error(errorMessage) + + return nil, errors.New(errorMessage) + default: return nil, errors.Errorf("category %s not supported", dc.FullPath().Category()) } From 70ebaa6b0db148d82b442d26e09348eaa4397a3e Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 18 Jan 2023 23:06:30 -0500 Subject: [PATCH 3/3] MetaData: Pages Backup Info Support (#2084) ## Description SharePointInfo extended to include SitePageable objects. **NOTE:** Merge includes the beta graph package and will affect the compile time. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :sunflower: Feature ## Issue(s) * closes #2069 * related #2071 ## Test Plan - [x] :zap: Unit test --- .../connector/sharepoint/listInfo_test.go | 12 +- src/internal/connector/sharepoint/pageInfo.go | 42 ++++ .../connector/sharepoint/pageInfo_test.go | 47 +++++ .../connector/sharepoint/site_page.go | 187 ++++++++++++++++++ .../connector/sharepoint/site_pageable.go | 24 +++ 5 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 src/internal/connector/sharepoint/pageInfo.go create mode 100644 src/internal/connector/sharepoint/pageInfo_test.go create mode 100644 src/internal/connector/sharepoint/site_page.go create mode 100644 src/internal/connector/sharepoint/site_pageable.go diff --git a/src/internal/connector/sharepoint/listInfo_test.go b/src/internal/connector/sharepoint/listInfo_test.go index 14e484817..f56b1a396 100644 --- a/src/internal/connector/sharepoint/listInfo_test.go +++ b/src/internal/connector/sharepoint/listInfo_test.go @@ -20,33 +20,33 @@ func TestSharePointInfoSuite(t *testing.T) { func (suite *SharePointInfoSuite) TestSharePointInfo() { tests := []struct { - name string - listAndRP func() (models.Listable, *details.SharePointInfo) + name string + listAndDeets func() (models.Listable, *details.SharePointInfo) }{ { name: "Empty List", - listAndRP: func() (models.Listable, *details.SharePointInfo) { + listAndDeets: func() (models.Listable, *details.SharePointInfo) { i := &details.SharePointInfo{ItemType: details.SharePointItem} return models.NewList(), i }, }, { name: "Only Name", - listAndRP: func() (models.Listable, *details.SharePointInfo) { + listAndDeets: func() (models.Listable, *details.SharePointInfo) { aTitle := "Whole List" listing := models.NewList() listing.SetDisplayName(&aTitle) i := &details.SharePointInfo{ ItemType: details.SharePointItem, ItemName: aTitle, - Size: 10, } + return listing, i }, }, } for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - list, expected := test.listAndRP() + list, expected := test.listAndDeets() info := sharePointListInfo(list, 10) assert.Equal(t, expected.ItemType, info.ItemType) assert.Equal(t, expected.ItemName, info.ItemName) diff --git a/src/internal/connector/sharepoint/pageInfo.go b/src/internal/connector/sharepoint/pageInfo.go new file mode 100644 index 000000000..2c11863cb --- /dev/null +++ b/src/internal/connector/sharepoint/pageInfo.go @@ -0,0 +1,42 @@ +package sharepoint + +import ( + "time" + + "github.com/alcionai/corso/src/pkg/backup/details" +) + +// sharePointPageInfo propagates metadata from the SharePoint Page data type +// into searchable content. +// Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta +func sharePointPageInfo(page SitePageable, size int64) *details.SharePointInfo { + var ( + name, webURL string + created, modified time.Time + ) + + if page.GetTitle() != nil { + name = *page.GetTitle() + } + + if page.GetWebUrl() != nil { + webURL = *page.GetWebUrl() + } + + if page.GetCreatedDateTime() != nil { + created = *page.GetCreatedDateTime() + } + + if page.GetLastModifiedDateTime() != nil { + modified = *page.GetLastModifiedDateTime() + } + + return &details.SharePointInfo{ + ItemType: details.SharePointItem, + ItemName: name, + Created: created, + Modified: modified, + WebURL: webURL, + Size: size, + } +} diff --git a/src/internal/connector/sharepoint/pageInfo_test.go b/src/internal/connector/sharepoint/pageInfo_test.go new file mode 100644 index 000000000..6ea070fbe --- /dev/null +++ b/src/internal/connector/sharepoint/pageInfo_test.go @@ -0,0 +1,47 @@ +package sharepoint + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/alcionai/corso/src/pkg/backup/details" +) + +func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { + tests := []struct { + name string + pageAndDeets func() (SitePageable, *details.SharePointInfo) + }{ + { + name: "Empty Page", + pageAndDeets: func() (SitePageable, *details.SharePointInfo) { + deets := &details.SharePointInfo{ItemType: details.SharePointItem} + return NewSitePage(), deets + }, + }, + { + name: "Only Name", + pageAndDeets: func() (SitePageable, *details.SharePointInfo) { + title := "Blank Page" + sPage := NewSitePage() + sPage.SetTitle(&title) + deets := &details.SharePointInfo{ + ItemType: details.SharePointItem, + ItemName: title, + } + + return sPage, deets + }, + }, + } + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + paged, expected := test.pageAndDeets() + info := sharePointPageInfo(paged, 0) + assert.Equal(t, expected.ItemType, info.ItemType) + assert.Equal(t, expected.ItemName, info.ItemName) + assert.Equal(t, expected.WebURL, info.WebURL) + }) + } +} diff --git a/src/internal/connector/sharepoint/site_page.go b/src/internal/connector/sharepoint/site_page.go new file mode 100644 index 000000000..56bf7ad27 --- /dev/null +++ b/src/internal/connector/sharepoint/site_page.go @@ -0,0 +1,187 @@ +package sharepoint + +import ( + kioser "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-sdk-go/models" +) + +// SitePage provides operations to manage the minimal creation of a Site Page. +// Altered from original: github.com/microsoftgraph/msgraph-beta-sdk-go/models +// TODO: remove when Issue #2086 resolved +type SitePage struct { + models.BaseItem + // Indicates the layout of the content in a given SharePoint page, including horizontal sections and vertical section + // canvasLayout models.CanvasLayoutable + // Inherited from baseItem. + contentType models.ContentTypeInfoable + // The name of the page layout of the page. + // The possible values are: microsoftReserved, article, home, unknownFutureValue. + // pageLayout *models.PageLayoutType + // Indicates the promotion kind of the sitePage. The possible values are: + // microsoftReserved, page, newsPost, unknownFutureValue. + // promotionKind *models.PagePromotionType + // The publishing status and the MM.mm version of the page. + publishingState models.PublicationFacetable + // Reactions information for the page. + // reactions models.ReactionsFacetable + // Determines whether or not to show comments at the bottom of the page. + showComments *bool + // Determines whether or not to show recommended pages at the bottom of the page. + showRecommendedPages *bool + // Url of the sitePage's thumbnail image + //revive:disable:var-naming + thumbnailWebUrl *string + //revive:enable:var-naming + // Title of the sitePage. + title *string +} + +// Title area on the SharePoint page. +// titleArea models.TitleAreaable +// Collection of webparts on the SharePoint page +// webParts []models.WebPartable + +var _ SitePageable = &SitePage{} + +// NewSitePage instantiates a new sitePage and sets the default values. +func NewSitePage() *SitePage { + m := &SitePage{ + BaseItem: *models.NewBaseItem(), + } + odataTypeValue := "#microsoft.graph.sitePage" + m.SetOdataType(&odataTypeValue) + + return m +} + +// CreateSitePageFromDiscriminatorValue creates a new instance of the appropriate class based on discriminator value +func CreateSitePageFromDiscriminatorValue(parseNode kioser.ParseNode) (kioser.Parsable, error) { + return NewSitePage(), nil +} + +// GetContentType gets the contentType property value. Inherited from baseItem. +func (m *SitePage) GetContentType() models.ContentTypeInfoable { + return m.contentType +} + +// GetFieldDeserializers the deserialization information for the current model +// Altered from original. +func (m *SitePage) GetFieldDeserializers() map[string]func(kioser.ParseNode) error { + res := m.BaseItem.GetFieldDeserializers() + + return res +} + +// GetPublishingState gets the publishingState property value. The publishing status and the MM.mm version of the page. +func (m *SitePage) GetPublishingState() models.PublicationFacetable { + return m.publishingState +} + +// GetShowComments gets the showComments property value. +// Determines whether or not to show comments at the bottom of the page. +func (m *SitePage) GetShowComments() *bool { + return m.showComments +} + +// GetShowRecommendedPages gets the showRecommendedPages property value. +// Determines whether or not to show recommended pages at the bottom of the page. +func (m *SitePage) GetShowRecommendedPages() *bool { + return m.showRecommendedPages +} + +// GetThumbnailWebUrl gets the thumbnailWebUrl property value. Url of the sitePage's thumbnail image +// +//revive:disable:var-naming +func (m *SitePage) GetThumbnailWebUrl() *string { + return m.thumbnailWebUrl +} + +// GetTitle gets the title property value. Title of the sitePage. +func (m *SitePage) GetTitle() *string { + return m.title +} + +// Serialize serializes information the current object +func (m *SitePage) Serialize(writer kioser.SerializationWriter) error { + err := m.BaseItem.Serialize(writer) + if err != nil { + return err + } + + if m.GetContentType() != nil { + err = writer.WriteObjectValue("contentType", m.GetContentType()) + if err != nil { + return err + } + } + + if m.GetPublishingState() != nil { + err = writer.WriteObjectValue("publishingState", m.GetPublishingState()) + if err != nil { + return err + } + } + { + err = writer.WriteBoolValue("showComments", m.GetShowComments()) + if err != nil { + return err + } + } + { + err = writer.WriteBoolValue("showRecommendedPages", m.GetShowRecommendedPages()) + if err != nil { + return err + } + } + { + err = writer.WriteStringValue("thumbnailWebUrl", m.GetThumbnailWebUrl()) + if err != nil { + return err + } + } + { + err = writer.WriteStringValue("title", m.GetTitle()) + if err != nil { + return err + } + } + + return nil +} + +// SetContentType sets the contentType property value. Inherited from baseItem. +func (m *SitePage) SetContentType(value models.ContentTypeInfoable) { + m.contentType = value +} + +// SetPublishingState sets the publishingState property value. The publishing status and the MM.mm version of the page. +func (m *SitePage) SetPublishingState(value models.PublicationFacetable) { + m.publishingState = value +} + +// SetShowComments sets the showComments property value. +// Determines whether or not to show comments at the bottom of the page. +func (m *SitePage) SetShowComments(value *bool) { + m.showComments = value +} + +// SetShowRecommendedPages sets the showRecommendedPages property value. +// Determines whether or not to show recommended pages at the bottom of the page. +func (m *SitePage) SetShowRecommendedPages(value *bool) { + m.showRecommendedPages = value +} + +// SetThumbnailWebUrl sets the thumbnailWebUrl property value. +// Url of the sitePage's thumbnail image +// +//revive:disable:var-naming +func (m *SitePage) SetThumbnailWebUrl(value *string) { + m.thumbnailWebUrl = value +} + +//revive:enable:var-naming + +// SetTitle sets the title property value. Title of the sitePage. +func (m *SitePage) SetTitle(value *string) { + m.title = value +} diff --git a/src/internal/connector/sharepoint/site_pageable.go b/src/internal/connector/sharepoint/site_pageable.go new file mode 100644 index 000000000..d0626321b --- /dev/null +++ b/src/internal/connector/sharepoint/site_pageable.go @@ -0,0 +1,24 @@ +package sharepoint + +import ( + "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-sdk-go/models" +) + +// SitePageable adjusted from msgraph-beta-sdk-go for temporary testing +type SitePageable interface { + models.BaseItemable + serialization.Parsable + GetContentType() models.ContentTypeInfoable + GetPublishingState() models.PublicationFacetable + GetShowComments() *bool + GetShowRecommendedPages() *bool + GetThumbnailWebUrl() *string + GetTitle() *string + SetContentType(value models.ContentTypeInfoable) + SetPublishingState(value models.PublicationFacetable) + SetShowComments(value *bool) + SetShowRecommendedPages(value *bool) + SetThumbnailWebUrl(value *string) + SetTitle(value *string) +}