GC: Restore: SharePoint List Workflow (#1964)

## Description
Connects restore pipeline for List Category. CLI commands are not included, Documentation for List restore needs to be placed within the CLI PR
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

## Issue(s)

* closes #1962<issue>
* closes #1978
* closes #1935

## Test Plan

- [x] ⚡ Unit test

src/internal/connector/mockconnector/mock_data_list.go

@@ -24,6 +24,10 @@ type MockListCollection struct {
 	Names    []string
 }
 
+func (mlc *MockListCollection) SetPath(p path.Path) {
+	mlc.fullPath = p
+}
+
 func (mlc *MockListCollection) State() data.CollectionState {
 	return data.NewState
 }

src/internal/connector/sharepoint/collection_test.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"testing"
 
-	kw "github.com/microsoft/kiota-serialization-json-go"
+	kioser "github.com/microsoft/kiota-serialization-json-go"
 	"github.com/microsoftgraph/msgraph-sdk-go/sites"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -48,11 +48,8 @@ func (suite *SharePointCollectionSuite) TestSharePointDataReader_Valid() {
 func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
 	t := suite.T()
 
-	ow := kw.NewJsonSerializationWriter()
+	ow := kioser.NewJsonSerializationWriter()
-	artistAndAlbum := map[string]string{
+	listing := mockconnector.GetMockListDefault("Mock List")
-		"Our Love to  Admire": "Interpol",
-	}
-	listing := mockconnector.GetMockList("Mock List", "Artist", artistAndAlbum)
 	testName := "MockListing"
 	listing.SetDisplayName(&testName)
 
@@ -92,30 +89,24 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
 	assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName)
 }
 
-func (suite *SharePointCollectionSuite) TestRestoreList() {
+// TestRestoreListCollection verifies Graph Restore API for the List Collection
+func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
 	t := suite.T()
+	siteID := tester.M365SiteID(t)
 	a := tester.NewM365Account(t)
 	account, err := a.M365Config()
 	require.NoError(t, err)
 
 	service, err := createTestService(account)
 	require.NoError(t, err)
-	siteID := tester.M365SiteID(t)
 
-	ow := kw.NewJsonSerializationWriter()
 	listing := mockconnector.GetMockListDefault("Mock List")
 	testName := "MockListing"
 	listing.SetDisplayName(&testName)
-
+	byteArray, err := service.Serialize(listing)
-	err = ow.WriteObjectValue("", listing)
-	require.NoError(t, err)
-
-	byteArray, err := ow.GetSerializedContent()
-	require.NoError(t, err)
-
 	require.NoError(t, err)
 
 	listData := &Item{

src/internal/connector/sharepoint/helper_test.go

@@ -17,12 +17,6 @@ import (
 // ---------------------------------------------------------------------------
 type MockGraphService struct{}
 
-type testService struct {
-	client      msgraphsdk.GraphServiceClient
-	adapter     msgraphsdk.GraphRequestAdapter
-	credentials account.M365Config
-}
-
 //------------------------------------------------------------
 // Interface Functions: @See graph.Service
 //------------------------------------------------------------
@@ -35,37 +29,21 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
 	return nil
 }
 
-func (ts *testService) Client() *msgraphsdk.GraphServiceClient {
-	return &ts.client
-}
-
-func (ts *testService) Adapter() *msgraphsdk.GraphRequestAdapter {
-	return &ts.adapter
-}
-
 // ---------------------------------------------------------------------------
 // Helper Functions
 // ---------------------------------------------------------------------------
 
-func createTestService(credentials account.M365Config) (*testService, error) {
+func createTestService(credentials account.M365Config) (*graph.Service, error) {
-	{
+	adapter, err := graph.CreateAdapter(
-		adapter, err := graph.CreateAdapter(
+		credentials.AzureTenantID,
-			credentials.AzureTenantID,
+		credentials.AzureClientID,
-			credentials.AzureClientID,
+		credentials.AzureClientSecret,
-			credentials.AzureClientSecret,
+	)
-		)
+	if err != nil {
-		if err != nil {
+		return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
-			return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
-		}
-
-		service := testService{
-			adapter:     *adapter,
-			client:      *msgraphsdk.NewGraphServiceClient(adapter),
-			credentials: credentials,
-		}
-
-		return &service, nil
 	}
+
+	return graph.NewService(adapter), nil
 }
 
 func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {

src/internal/connector/sharepoint/list.go

@@ -265,13 +265,8 @@ func fetchColumns(
 // for the following:
 // - ColumnLinks
 // - Columns
-// The following two are not included:
+// Expand queries not used to retrieve the above. Possibly more than 20.
-// - ColumnPositions
+// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
-// - BaseTypes
-// These relationships are not included as they following error from the API:
-// itemNotFound Item not found: error status code received from the API
-// Current as of github.com/microsoftgraph/msgraph-sdk-go v0.40.0
-// TODO: Verify functionality after version upgrade or remove (dadams39) Check Stubs
 func fetchContentTypes(
 	ctx context.Context,
 	gs graph.Servicer,
@@ -299,7 +294,6 @@ func fetchContentTypes(
 			}
 
 			cont.SetColumnLinks(links)
-			// TODO: stub for columPositions
 
 			cs, err := fetchColumns(ctx, gs, siteID, listID, id)
 			if err != nil {
@@ -307,7 +301,6 @@ func fetchContentTypes(
 			}
 
 			cont.SetColumns(cs)
-			// TODO: stub for BaseTypes
 
 			cTypes = append(cTypes, cont)
 		}

src/internal/connector/sharepoint/restore.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"runtime/trace"
 
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/src/internal/connector/graph"
@@ -14,9 +16,23 @@ import (
 	D "github.com/alcionai/corso/src/internal/diagnostics"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
+//----------------------------------------------------------------------------
+// SharePoint Restore WorkFlow:
+// - RestoreCollections called by GC component
+// -- Collections are iterated within, Control Flow Switch
+// -- Switch:
+// ---- Libraries restored via the same workflow as oneDrive
+// ---- Lists call RestoreCollection()
+// ----> for each data.Stream within  Collection.Items()
+// ----> restoreListItems() is called
+// Restored List can be found in the Site's `Site content` page
+// Restored Libraries can be found within the Site's `Pages` page
+//------------------------------------------
+
 // RestoreCollections will restore the specified data collections into OneDrive
 func RestoreCollections(
 	ctx context.Context,
@@ -51,6 +67,15 @@ func RestoreCollections(
 				dest.ContainerName,
 				deets,
 				errUpdater)
+		case path.ListsCategory:
+			metrics, canceled = RestoreCollection(
+				ctx,
+				service,
+				dc,
+				dest.ContainerName,
+				deets,
+				errUpdater,
+			)
 		default:
 			return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
 		}
@@ -92,7 +117,7 @@ func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID st
 // restoreListItem utility function restores a List to the siteID.
 // The name is changed to to Corso_Restore_{timeStame}_name
 // API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http
-// Restored List can be verified within the Site contents
+// Restored List can be verified within the Site contents.
 func restoreListItem(
 	ctx context.Context,
 	service graph.Servicer,
@@ -103,9 +128,8 @@ func restoreListItem(
 	defer end()
 
 	var (
-		dii         = details.ItemInfo{}
+		dii      = details.ItemInfo{}
-		itemName    = itemData.UUID()
+		listName = itemData.UUID()
-		displayName = itemName
 	)
 
 	byteArray, err := io.ReadAll(itemData.ToReader())
@@ -113,35 +137,130 @@ func restoreListItem(
 		return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream")
 	}
 	// Create Item
-	newItem, err := support.CreateListFromBytes(byteArray)
+	oldList, err := support.CreateListFromBytes(byteArray)
 	if err != nil {
-		return dii, errors.Wrapf(err, "failed to construct list item %s", itemName)
+		return dii, errors.Wrapf(err, "failed to build list item %s", listName)
 	}
 
-	// If field "name" is set, this will trigger the following error:
+	if oldList.GetDisplayName() != nil {
-	// invalidRequest Cannot define a 'name' for a list as it is assigned by the server. Instead, provide 'displayName'
+		listName = *oldList.GetDisplayName()
-	if newItem.GetName() != nil {
-		adtlData := newItem.GetAdditionalData()
-		adtlData["list_name"] = *newItem.GetName()
-		newItem.SetName(nil)
 	}
 
-	if newItem.GetDisplayName() != nil {
+	newName := fmt.Sprintf("%s_%s", destName, listName)
-		displayName = *newItem.GetDisplayName()
+	newList := support.ToListable(oldList, newName)
+
+	contents := make([]models.ListItemable, 0)
+
+	for _, itm := range oldList.GetItems() {
+		temp := support.CloneListItem(itm)
+		contents = append(contents, temp)
 	}
 
-	newName := fmt.Sprintf("%s_%s", destName, displayName)
+	newList.SetItems(contents)
-	newItem.SetDisplayName(&newName)
 
-	// Restore to M365 store
+	// Restore to List base to M365 back store
-	restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newItem, nil)
+	restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
 	if err != nil {
-		return dii, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
+		errorMsg := fmt.Sprintf(
+			"failure to create list foundation ID: %s API Error Details: %s",
+			itemData.UUID(),
+			support.ConnectorStackErrorTrace(err),
+		)
+
+		return dii, errors.Wrap(err, errorMsg)
 	}
 
-	written := int64(len(byteArray))
+	// Uploading of ListItems is conducted after the List is restored
+	// Reference: https://learn.microsoft.com/en-us/graph/api/listitem-create?view=graph-rest-1.0&tabs=http
+	if len(contents) > 0 {
+		for _, lItem := range contents {
+			_, err := service.Client().
+				SitesById(siteID).
+				ListsById(*restoredList.GetId()).
+				Items().
+				Post(ctx, lItem, nil)
+			if err != nil {
+				errorMsg := fmt.Sprintf(
+					"listItem  failed for listID %s. Details: %s. Content: %v",
+					*restoredList.GetId(),
+					support.ConnectorStackErrorTrace(err),
+					lItem.GetAdditionalData(),
+				)
 
-	dii.SharePoint = sharePointListInfo(restoredList, written)
+				return dii, errors.Wrap(err, errorMsg)
+			}
+		}
+	}
+
+	dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray)))
 
 	return dii, nil
 }
+
+func RestoreCollection(
+	ctx context.Context,
+	service graph.Servicer,
+	dc data.Collection,
+	restoreContainerName string,
+	deets *details.Builder,
+	errUpdater func(string, error),
+) (support.CollectionMetrics, bool) {
+	ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath()))
+	defer end()
+
+	var (
+		metrics   = support.CollectionMetrics{}
+		directory = dc.FullPath()
+	)
+
+	trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String())
+	siteID := directory.ResourceOwner()
+
+	// Restore items from the collection
+	items := dc.Items()
+
+	for {
+		select {
+		case <-ctx.Done():
+			errUpdater("context canceled", ctx.Err())
+			return metrics, true
+
+		case itemData, ok := <-items:
+			if !ok {
+				return metrics, false
+			}
+			metrics.Objects++
+
+			itemInfo, err := restoreListItem(
+				ctx,
+				service,
+				itemData,
+				siteID,
+				restoreContainerName,
+			)
+			if err != nil {
+				errUpdater(itemData.UUID(), err)
+				continue
+			}
+
+			metrics.TotalBytes += itemInfo.SharePoint.Size
+
+			itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
+			if err != nil {
+				logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
+				errUpdater(itemData.UUID(), err)
+
+				continue
+			}
+
+			deets.Add(
+				itemPath.String(),
+				itemPath.ShortRef(),
+				"",
+				true,
+				itemInfo)
+
+			metrics.Successes++
+		}
+	}
+}

src/internal/connector/support/m365Transform.go

@@ -207,12 +207,24 @@ func ToListable(orig models.Listable, displayName string) models.Listable {
 	}
 
 	for _, cd := range orig.GetColumns() {
-		tag := *cd.GetDisplayName()
+		var (
-		_, isLegacy := leg[tag]
+			displayName string
+			readOnly    bool
+		)
+
+		if cd.GetDisplayName() != nil {
+			displayName = *cd.GetDisplayName()
+		}
+
+		if cd.GetReadOnly() != nil {
+			readOnly = *cd.GetReadOnly()
+		}
+
+		_, isLegacy := leg[displayName]
 
 		// Skips columns that cannot be uploaded for models.ColumnDefinitionable:
 		// - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type
-		if *cd.GetReadOnly() || tag == "Title" || isLegacy {
+		if readOnly || displayName == "Title" || isLegacy {
 			continue
 		}