larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: Restore: SharePoint List Workflow (#1964)

Browse Source 
## Description
Connects restore pipeline for List Category. CLI commands are not included, Documentation for List restore needs to be placed within the CLI PR
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

## Issue(s)

* closes #1962<issue>
* closes #1978
* closes #1935

## Test Plan

- [x] ⚡ Unit test
This commit is contained in:
Danny 2023-01-05 09:59:48 -05:00 committed by GitHub
parent 1f26339813
commit 4bcfc9dadf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 178 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/internal/connector/mockconnector/mock_data_list.go
View File

@ -24,6 +24,10 @@ type MockListCollection struct {
Names []string
}
func (mlc *MockListCollection) SetPath(p path.Path) {
mlc.fullPath = p
}
func (mlc *MockListCollection) State() data.CollectionState {
return data.NewState
}

23
src/internal/connector/sharepoint/collection_test.go
View File

@ -5,7 +5,7 @@ import (
"io"
"testing"
kw "github.com/microsoft/kiota-serialization-json-go"
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -48,11 +48,8 @@ func (suite *SharePointCollectionSuite) TestSharePointDataReader_Valid() {
func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
t := suite.T()
ow := kw.NewJsonSerializationWriter()
artistAndAlbum := map[string]string{
"Our Love to Admire": "Interpol",
}
listing := mockconnector.GetMockList("Mock List", "Artist", artistAndAlbum)
ow := kioser.NewJsonSerializationWriter()
listing := mockconnector.GetMockListDefault("Mock List")
testName := "MockListing"
listing.SetDisplayName(&testName)
@ -92,30 +89,24 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName)
}
func (suite *SharePointCollectionSuite) TestRestoreList() {
// TestRestoreListCollection verifies Graph Restore API for the List Collection
func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
siteID := tester.M365SiteID(t)
a := tester.NewM365Account(t)
account, err := a.M365Config()
require.NoError(t, err)
service, err := createTestService(account)
require.NoError(t, err)
siteID := tester.M365SiteID(t)
ow := kw.NewJsonSerializationWriter()
listing := mockconnector.GetMockListDefault("Mock List")
testName := "MockListing"
listing.SetDisplayName(&testName)
err = ow.WriteObjectValue("", listing)
require.NoError(t, err)
byteArray, err := ow.GetSerializedContent()
require.NoError(t, err)
byteArray, err := service.Serialize(listing)
require.NoError(t, err)
listData := &Item{

26
src/internal/connector/sharepoint/helper_test.go
View File

@ -17,12 +17,6 @@ import (
// ---------------------------------------------------------------------------
type MockGraphService struct{}
type testService struct {
client msgraphsdk.GraphServiceClient
adapter msgraphsdk.GraphRequestAdapter
credentials account.M365Config
}
//------------------------------------------------------------
// Interface Functions: @See graph.Service
//------------------------------------------------------------
@ -35,20 +29,11 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
return nil
}
func (ts *testService) Client() *msgraphsdk.GraphServiceClient {
return &ts.client
}
func (ts *testService) Adapter() *msgraphsdk.GraphRequestAdapter {
return &ts.adapter
}
// ---------------------------------------------------------------------------
// Helper Functions
// ---------------------------------------------------------------------------
func createTestService(credentials account.M365Config) (*testService, error) {
{
func createTestService(credentials account.M365Config) (*graph.Service, error) {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
@ -58,14 +43,7 @@ func createTestService(credentials account.M365Config) (*testService, error) {
return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
}
service := testService{
adapter: *adapter,
client: *msgraphsdk.NewGraphServiceClient(adapter),
credentials: credentials,
}
return &service, nil
}
return graph.NewService(adapter), nil
}
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {

11
src/internal/connector/sharepoint/list.go
View File

@ -265,13 +265,8 @@ func fetchColumns(
// for the following:
// - ColumnLinks
// - Columns
// The following two are not included:
// - ColumnPositions
// - BaseTypes
// These relationships are not included as they following error from the API:
// itemNotFound Item not found: error status code received from the API
// Current as of github.com/microsoftgraph/msgraph-sdk-go v0.40.0
// TODO: Verify functionality after version upgrade or remove (dadams39) Check Stubs
// Expand queries not used to retrieve the above. Possibly more than 20.
// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
func fetchContentTypes(
ctx context.Context,
gs graph.Servicer,
@ -299,7 +294,6 @@ func fetchContentTypes(
}
cont.SetColumnLinks(links)
// TODO: stub for columPositions
cs, err := fetchColumns(ctx, gs, siteID, listID, id)
if err != nil {
@ -307,7 +301,6 @@ func fetchContentTypes(
}
cont.SetColumns(cs)
// TODO: stub for BaseTypes
cTypes = append(cTypes, cont)
}

159
src/internal/connector/sharepoint/restore.go
View File

@ -4,7 +4,9 @@ import (
"context"
"fmt"
"io"
"runtime/trace"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
@ -14,9 +16,23 @@ import (
D "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
//----------------------------------------------------------------------------
// SharePoint Restore WorkFlow:
// - RestoreCollections called by GC component
// -- Collections are iterated within, Control Flow Switch
// -- Switch:
// ---- Libraries restored via the same workflow as oneDrive
// ---- Lists call RestoreCollection()
// ----> for each data.Stream within Collection.Items()
// ----> restoreListItems() is called
// Restored List can be found in the Site's `Site content` page
// Restored Libraries can be found within the Site's `Pages` page
//------------------------------------------
// RestoreCollections will restore the specified data collections into OneDrive
func RestoreCollections(
ctx context.Context,
@ -51,6 +67,15 @@ func RestoreCollections(
dest.ContainerName,
deets,
errUpdater)
case path.ListsCategory:
metrics, canceled = RestoreCollection(
ctx,
service,
dc,
dest.ContainerName,
deets,
errUpdater,
)
default:
return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
}
@ -92,7 +117,7 @@ func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID st
// restoreListItem utility function restores a List to the siteID.
// The name is changed to to Corso_Restore_{timeStame}_name
// API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http
// Restored List can be verified within the Site contents
// Restored List can be verified within the Site contents.
func restoreListItem(
ctx context.Context,
service graph.Servicer,
@ -104,8 +129,7 @@ func restoreListItem(
var (
dii = details.ItemInfo{}
itemName = itemData.UUID()
displayName = itemName
listName = itemData.UUID()
)
byteArray, err := io.ReadAll(itemData.ToReader())
@ -113,35 +137,130 @@ func restoreListItem(
return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream")
}
// Create Item
newItem, err := support.CreateListFromBytes(byteArray)
oldList, err := support.CreateListFromBytes(byteArray)
if err != nil {
return dii, errors.Wrapf(err, "failed to construct list item %s", itemName)
return dii, errors.Wrapf(err, "failed to build list item %s", listName)
}
// If field "name" is set, this will trigger the following error:
// invalidRequest Cannot define a 'name' for a list as it is assigned by the server. Instead, provide 'displayName'
if newItem.GetName() != nil {
adtlData := newItem.GetAdditionalData()
adtlData["list_name"] = *newItem.GetName()
newItem.SetName(nil)
if oldList.GetDisplayName() != nil {
listName = *oldList.GetDisplayName()
}
if newItem.GetDisplayName() != nil {
displayName = *newItem.GetDisplayName()
newName := fmt.Sprintf("%s_%s", destName, listName)
newList := support.ToListable(oldList, newName)
contents := make([]models.ListItemable, 0)
for _, itm := range oldList.GetItems() {
temp := support.CloneListItem(itm)
contents = append(contents, temp)
}
newName := fmt.Sprintf("%s_%s", destName, displayName)
newItem.SetDisplayName(&newName)
newList.SetItems(contents)
// Restore to M365 store
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newItem, nil)
// Restore to List base to M365 back store
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
if err != nil {
return dii, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
errorMsg := fmt.Sprintf(
"failure to create list foundation ID: %s API Error Details: %s",
itemData.UUID(),
support.ConnectorStackErrorTrace(err),
)
return dii, errors.Wrap(err, errorMsg)
}
written := int64(len(byteArray))
// Uploading of ListItems is conducted after the List is restored
// Reference: https://learn.microsoft.com/en-us/graph/api/listitem-create?view=graph-rest-1.0&tabs=http
if len(contents) > 0 {
for _, lItem := range contents {
_, err := service.Client().
SitesById(siteID).
ListsById(*restoredList.GetId()).
Items().
Post(ctx, lItem, nil)
if err != nil {
errorMsg := fmt.Sprintf(
"listItem failed for listID %s. Details: %s. Content: %v",
*restoredList.GetId(),
support.ConnectorStackErrorTrace(err),
lItem.GetAdditionalData(),
)
dii.SharePoint = sharePointListInfo(restoredList, written)
return dii, errors.Wrap(err, errorMsg)
}
}
}
dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray)))
return dii, nil
}
func RestoreCollection(
ctx context.Context,
service graph.Servicer,
dc data.Collection,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath()))
defer end()
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
)
trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from the collection
items := dc.Items()
for {
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, true
case itemData, ok := <-items:
if !ok {
return metrics, false
}
metrics.Objects++
itemInfo, err := restoreListItem(
ctx,
service,
itemData,
siteID,
restoreContainerName,
)
if err != nil {
errUpdater(itemData.UUID(), err)
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
continue
}
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
true,
itemInfo)
metrics.Successes++
}
}
}

18
src/internal/connector/support/m365Transform.go
View File

@ -207,12 +207,24 @@ func ToListable(orig models.Listable, displayName string) models.Listable {
}
for _, cd := range orig.GetColumns() {
tag := *cd.GetDisplayName()
_, isLegacy := leg[tag]
var (
displayName string
readOnly bool
)
if cd.GetDisplayName() != nil {
displayName = *cd.GetDisplayName()
}
if cd.GetReadOnly() != nil {
readOnly = *cd.GetReadOnly()
}
_, isLegacy := leg[displayName]
// Skips columns that cannot be uploaded for models.ColumnDefinitionable:
// - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type
if *cd.GetReadOnly() || tag == "Title" || isLegacy {
if readOnly || displayName == "Title" || isLegacy {
continue
}