From 4bcfc9dadffa01780060efad1559838db732e026 Mon Sep 17 00:00:00 2001 From: Danny Date: Thu, 5 Jan 2023 09:59:48 -0500 Subject: [PATCH] GC: Restore: SharePoint List Workflow (#1964) ## Description Connects restore pipeline for List Category. CLI commands are not included, Documentation for List restore needs to be placed within the CLI PR ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :sunflower: Feature ## Issue(s) * closes #1962 * closes #1978 * closes #1935 ## Test Plan - [x] :zap: Unit test --- .../connector/mockconnector/mock_data_list.go | 4 + .../connector/sharepoint/collection_test.go | 23 +-- .../connector/sharepoint/helper_test.go | 42 ++--- src/internal/connector/sharepoint/list.go | 11 +- src/internal/connector/sharepoint/restore.go | 161 +++++++++++++++--- .../connector/support/m365Transform.go | 18 +- 6 files changed, 178 insertions(+), 81 deletions(-) diff --git a/src/internal/connector/mockconnector/mock_data_list.go b/src/internal/connector/mockconnector/mock_data_list.go index 18337f984..2994dd275 100644 --- a/src/internal/connector/mockconnector/mock_data_list.go +++ b/src/internal/connector/mockconnector/mock_data_list.go @@ -24,6 +24,10 @@ type MockListCollection struct { Names []string } +func (mlc *MockListCollection) SetPath(p path.Path) { + mlc.fullPath = p +} + func (mlc *MockListCollection) State() data.CollectionState { return data.NewState } diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index 086caa505..f69366f67 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -5,7 +5,7 @@ import ( "io" "testing" - kw "github.com/microsoft/kiota-serialization-json-go" + kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/sites" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -48,11 +48,8 @@ func (suite *SharePointCollectionSuite) TestSharePointDataReader_Valid() { func (suite *SharePointCollectionSuite) TestSharePointListCollection() { t := suite.T() - ow := kw.NewJsonSerializationWriter() - artistAndAlbum := map[string]string{ - "Our Love to Admire": "Interpol", - } - listing := mockconnector.GetMockList("Mock List", "Artist", artistAndAlbum) + ow := kioser.NewJsonSerializationWriter() + listing := mockconnector.GetMockListDefault("Mock List") testName := "MockListing" listing.SetDisplayName(&testName) @@ -92,30 +89,24 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() { assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName) } -func (suite *SharePointCollectionSuite) TestRestoreList() { +// TestRestoreListCollection verifies Graph Restore API for the List Collection +func (suite *SharePointCollectionSuite) TestRestoreListCollection() { ctx, flush := tester.NewContext() defer flush() t := suite.T() + siteID := tester.M365SiteID(t) a := tester.NewM365Account(t) account, err := a.M365Config() require.NoError(t, err) service, err := createTestService(account) require.NoError(t, err) - siteID := tester.M365SiteID(t) - ow := kw.NewJsonSerializationWriter() listing := mockconnector.GetMockListDefault("Mock List") testName := "MockListing" listing.SetDisplayName(&testName) - - err = ow.WriteObjectValue("", listing) - require.NoError(t, err) - - byteArray, err := ow.GetSerializedContent() - require.NoError(t, err) - + byteArray, err := service.Serialize(listing) require.NoError(t, err) listData := &Item{ diff --git a/src/internal/connector/sharepoint/helper_test.go b/src/internal/connector/sharepoint/helper_test.go index e0dcc5ddf..e716a5bae 100644 --- a/src/internal/connector/sharepoint/helper_test.go +++ b/src/internal/connector/sharepoint/helper_test.go @@ -17,12 +17,6 @@ import ( // --------------------------------------------------------------------------- type MockGraphService struct{} -type testService struct { - client msgraphsdk.GraphServiceClient - adapter msgraphsdk.GraphRequestAdapter - credentials account.M365Config -} - //------------------------------------------------------------ // Interface Functions: @See graph.Service //------------------------------------------------------------ @@ -35,37 +29,21 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter { return nil } -func (ts *testService) Client() *msgraphsdk.GraphServiceClient { - return &ts.client -} - -func (ts *testService) Adapter() *msgraphsdk.GraphRequestAdapter { - return &ts.adapter -} - // --------------------------------------------------------------------------- // Helper Functions // --------------------------------------------------------------------------- -func createTestService(credentials account.M365Config) (*testService, error) { - { - adapter, err := graph.CreateAdapter( - credentials.AzureTenantID, - credentials.AzureClientID, - credentials.AzureClientSecret, - ) - if err != nil { - return nil, errors.Wrap(err, "creating microsoft graph service for exchange") - } - - service := testService{ - adapter: *adapter, - client: *msgraphsdk.NewGraphServiceClient(adapter), - credentials: credentials, - } - - return &service, nil +func createTestService(credentials account.M365Config) (*graph.Service, error) { + adapter, err := graph.CreateAdapter( + credentials.AzureTenantID, + credentials.AzureClientID, + credentials.AzureClientSecret, + ) + if err != nil { + return nil, errors.Wrap(err, "creating microsoft graph service for exchange") } + + return graph.NewService(adapter), nil } func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string { diff --git a/src/internal/connector/sharepoint/list.go b/src/internal/connector/sharepoint/list.go index 391902ea1..101de9722 100644 --- a/src/internal/connector/sharepoint/list.go +++ b/src/internal/connector/sharepoint/list.go @@ -265,13 +265,8 @@ func fetchColumns( // for the following: // - ColumnLinks // - Columns -// The following two are not included: -// - ColumnPositions -// - BaseTypes -// These relationships are not included as they following error from the API: -// itemNotFound Item not found: error status code received from the API -// Current as of github.com/microsoftgraph/msgraph-sdk-go v0.40.0 -// TODO: Verify functionality after version upgrade or remove (dadams39) Check Stubs +// Expand queries not used to retrieve the above. Possibly more than 20. +// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters func fetchContentTypes( ctx context.Context, gs graph.Servicer, @@ -299,7 +294,6 @@ func fetchContentTypes( } cont.SetColumnLinks(links) - // TODO: stub for columPositions cs, err := fetchColumns(ctx, gs, siteID, listID, id) if err != nil { @@ -307,7 +301,6 @@ func fetchContentTypes( } cont.SetColumns(cs) - // TODO: stub for BaseTypes cTypes = append(cTypes, cont) } diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index 5e98829b8..943ef7773 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "io" + "runtime/trace" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -14,9 +16,23 @@ import ( D "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) +//---------------------------------------------------------------------------- +// SharePoint Restore WorkFlow: +// - RestoreCollections called by GC component +// -- Collections are iterated within, Control Flow Switch +// -- Switch: +// ---- Libraries restored via the same workflow as oneDrive +// ---- Lists call RestoreCollection() +// ----> for each data.Stream within Collection.Items() +// ----> restoreListItems() is called +// Restored List can be found in the Site's `Site content` page +// Restored Libraries can be found within the Site's `Pages` page +//------------------------------------------ + // RestoreCollections will restore the specified data collections into OneDrive func RestoreCollections( ctx context.Context, @@ -51,6 +67,15 @@ func RestoreCollections( dest.ContainerName, deets, errUpdater) + case path.ListsCategory: + metrics, canceled = RestoreCollection( + ctx, + service, + dc, + dest.ContainerName, + deets, + errUpdater, + ) default: return nil, errors.Errorf("category %s not supported", dc.FullPath().Category()) } @@ -92,7 +117,7 @@ func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID st // restoreListItem utility function restores a List to the siteID. // The name is changed to to Corso_Restore_{timeStame}_name // API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http -// Restored List can be verified within the Site contents +// Restored List can be verified within the Site contents. func restoreListItem( ctx context.Context, service graph.Servicer, @@ -103,9 +128,8 @@ func restoreListItem( defer end() var ( - dii = details.ItemInfo{} - itemName = itemData.UUID() - displayName = itemName + dii = details.ItemInfo{} + listName = itemData.UUID() ) byteArray, err := io.ReadAll(itemData.ToReader()) @@ -113,35 +137,130 @@ func restoreListItem( return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream") } // Create Item - newItem, err := support.CreateListFromBytes(byteArray) + oldList, err := support.CreateListFromBytes(byteArray) if err != nil { - return dii, errors.Wrapf(err, "failed to construct list item %s", itemName) + return dii, errors.Wrapf(err, "failed to build list item %s", listName) } - // If field "name" is set, this will trigger the following error: - // invalidRequest Cannot define a 'name' for a list as it is assigned by the server. Instead, provide 'displayName' - if newItem.GetName() != nil { - adtlData := newItem.GetAdditionalData() - adtlData["list_name"] = *newItem.GetName() - newItem.SetName(nil) + if oldList.GetDisplayName() != nil { + listName = *oldList.GetDisplayName() } - if newItem.GetDisplayName() != nil { - displayName = *newItem.GetDisplayName() + newName := fmt.Sprintf("%s_%s", destName, listName) + newList := support.ToListable(oldList, newName) + + contents := make([]models.ListItemable, 0) + + for _, itm := range oldList.GetItems() { + temp := support.CloneListItem(itm) + contents = append(contents, temp) } - newName := fmt.Sprintf("%s_%s", destName, displayName) - newItem.SetDisplayName(&newName) + newList.SetItems(contents) - // Restore to M365 store - restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newItem, nil) + // Restore to List base to M365 back store + restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil) if err != nil { - return dii, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + errorMsg := fmt.Sprintf( + "failure to create list foundation ID: %s API Error Details: %s", + itemData.UUID(), + support.ConnectorStackErrorTrace(err), + ) + + return dii, errors.Wrap(err, errorMsg) } - written := int64(len(byteArray)) + // Uploading of ListItems is conducted after the List is restored + // Reference: https://learn.microsoft.com/en-us/graph/api/listitem-create?view=graph-rest-1.0&tabs=http + if len(contents) > 0 { + for _, lItem := range contents { + _, err := service.Client(). + SitesById(siteID). + ListsById(*restoredList.GetId()). + Items(). + Post(ctx, lItem, nil) + if err != nil { + errorMsg := fmt.Sprintf( + "listItem failed for listID %s. Details: %s. Content: %v", + *restoredList.GetId(), + support.ConnectorStackErrorTrace(err), + lItem.GetAdditionalData(), + ) - dii.SharePoint = sharePointListInfo(restoredList, written) + return dii, errors.Wrap(err, errorMsg) + } + } + } + + dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray))) return dii, nil } + +func RestoreCollection( + ctx context.Context, + service graph.Servicer, + dc data.Collection, + restoreContainerName string, + deets *details.Builder, + errUpdater func(string, error), +) (support.CollectionMetrics, bool) { + ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath())) + defer end() + + var ( + metrics = support.CollectionMetrics{} + directory = dc.FullPath() + ) + + trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String()) + siteID := directory.ResourceOwner() + + // Restore items from the collection + items := dc.Items() + + for { + select { + case <-ctx.Done(): + errUpdater("context canceled", ctx.Err()) + return metrics, true + + case itemData, ok := <-items: + if !ok { + return metrics, false + } + metrics.Objects++ + + itemInfo, err := restoreListItem( + ctx, + service, + itemData, + siteID, + restoreContainerName, + ) + if err != nil { + errUpdater(itemData.UUID(), err) + continue + } + + metrics.TotalBytes += itemInfo.SharePoint.Size + + itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + if err != nil { + logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err) + errUpdater(itemData.UUID(), err) + + continue + } + + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + true, + itemInfo) + + metrics.Successes++ + } + } +} diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 14fa0cb5d..651689430 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -207,12 +207,24 @@ func ToListable(orig models.Listable, displayName string) models.Listable { } for _, cd := range orig.GetColumns() { - tag := *cd.GetDisplayName() - _, isLegacy := leg[tag] + var ( + displayName string + readOnly bool + ) + + if cd.GetDisplayName() != nil { + displayName = *cd.GetDisplayName() + } + + if cd.GetReadOnly() != nil { + readOnly = *cd.GetReadOnly() + } + + _, isLegacy := leg[displayName] // Skips columns that cannot be uploaded for models.ColumnDefinitionable: // - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type - if *cd.GetReadOnly() || tag == "Title" || isLegacy { + if readOnly || displayName == "Title" || isLegacy { continue }