GC: Restore: SharePoint List Workflow (#1964)
## Description Connects restore pipeline for List Category. CLI commands are not included, Documentation for List restore needs to be placed within the CLI PR <!-- Insert PR description--> ## Does this PR need a docs update or release note? - [x] ⛔ No ## Type of change <!--- Please check the type of change your PR introduces: ---> - [x] 🌻 Feature ## Issue(s) * closes #1962<issue> * closes #1978 * closes #1935 ## Test Plan - [x] ⚡ Unit test
This commit is contained in:
Danny
GitHub
parent
1f26339813
commit
4bcfc9dadf
@ -24,6 +24,10 @@ type MockListCollection struct {
|
||||
Names []string
|
||||
}
|
||||
|
||||
func (mlc *MockListCollection) SetPath(p path.Path) {
|
||||
mlc.fullPath = p
|
||||
}
|
||||
|
||||
func (mlc *MockListCollection) State() data.CollectionState {
|
||||
return data.NewState
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
kw "github.com/microsoft/kiota-serialization-json-go"
|
||||
kioser "github.com/microsoft/kiota-serialization-json-go"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@ -48,11 +48,8 @@ func (suite *SharePointCollectionSuite) TestSharePointDataReader_Valid() {
|
||||
func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
|
||||
t := suite.T()
|
||||
|
||||
ow := kw.NewJsonSerializationWriter()
|
||||
artistAndAlbum := map[string]string{
|
||||
"Our Love to Admire": "Interpol",
|
||||
}
|
||||
listing := mockconnector.GetMockList("Mock List", "Artist", artistAndAlbum)
|
||||
ow := kioser.NewJsonSerializationWriter()
|
||||
listing := mockconnector.GetMockListDefault("Mock List")
|
||||
testName := "MockListing"
|
||||
listing.SetDisplayName(&testName)
|
||||
|
||||
@ -92,30 +89,24 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
|
||||
assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName)
|
||||
}
|
||||
|
||||
func (suite *SharePointCollectionSuite) TestRestoreList() {
|
||||
// TestRestoreListCollection verifies Graph Restore API for the List Collection
|
||||
func (suite *SharePointCollectionSuite) TestRestoreListCollection() {
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
t := suite.T()
|
||||
siteID := tester.M365SiteID(t)
|
||||
a := tester.NewM365Account(t)
|
||||
account, err := a.M365Config()
|
||||
require.NoError(t, err)
|
||||
|
||||
service, err := createTestService(account)
|
||||
require.NoError(t, err)
|
||||
siteID := tester.M365SiteID(t)
|
||||
|
||||
ow := kw.NewJsonSerializationWriter()
|
||||
listing := mockconnector.GetMockListDefault("Mock List")
|
||||
testName := "MockListing"
|
||||
listing.SetDisplayName(&testName)
|
||||
|
||||
err = ow.WriteObjectValue("", listing)
|
||||
require.NoError(t, err)
|
||||
|
||||
byteArray, err := ow.GetSerializedContent()
|
||||
require.NoError(t, err)
|
||||
|
||||
byteArray, err := service.Serialize(listing)
|
||||
require.NoError(t, err)
|
||||
|
||||
listData := &Item{
|
||||
|
||||
@ -17,12 +17,6 @@ import (
|
||||
// ---------------------------------------------------------------------------
|
||||
type MockGraphService struct{}
|
||||
|
||||
type testService struct {
|
||||
client msgraphsdk.GraphServiceClient
|
||||
adapter msgraphsdk.GraphRequestAdapter
|
||||
credentials account.M365Config
|
||||
}
|
||||
|
||||
//------------------------------------------------------------
|
||||
// Interface Functions: @See graph.Service
|
||||
//------------------------------------------------------------
|
||||
@ -35,37 +29,21 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ts *testService) Client() *msgraphsdk.GraphServiceClient {
|
||||
return &ts.client
|
||||
}
|
||||
|
||||
func (ts *testService) Adapter() *msgraphsdk.GraphRequestAdapter {
|
||||
return &ts.adapter
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper Functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func createTestService(credentials account.M365Config) (*testService, error) {
|
||||
{
|
||||
adapter, err := graph.CreateAdapter(
|
||||
credentials.AzureTenantID,
|
||||
credentials.AzureClientID,
|
||||
credentials.AzureClientSecret,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
|
||||
}
|
||||
|
||||
service := testService{
|
||||
adapter: *adapter,
|
||||
client: *msgraphsdk.NewGraphServiceClient(adapter),
|
||||
credentials: credentials,
|
||||
}
|
||||
|
||||
return &service, nil
|
||||
func createTestService(credentials account.M365Config) (*graph.Service, error) {
|
||||
adapter, err := graph.CreateAdapter(
|
||||
credentials.AzureTenantID,
|
||||
credentials.AzureClientID,
|
||||
credentials.AzureClientSecret,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
|
||||
}
|
||||
|
||||
return graph.NewService(adapter), nil
|
||||
}
|
||||
|
||||
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {
|
||||
|
||||
@ -265,13 +265,8 @@ func fetchColumns(
|
||||
// for the following:
|
||||
// - ColumnLinks
|
||||
// - Columns
|
||||
// The following two are not included:
|
||||
// - ColumnPositions
|
||||
// - BaseTypes
|
||||
// These relationships are not included as they following error from the API:
|
||||
// itemNotFound Item not found: error status code received from the API
|
||||
// Current as of github.com/microsoftgraph/msgraph-sdk-go v0.40.0
|
||||
// TODO: Verify functionality after version upgrade or remove (dadams39) Check Stubs
|
||||
// Expand queries not used to retrieve the above. Possibly more than 20.
|
||||
// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
|
||||
func fetchContentTypes(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
@ -299,7 +294,6 @@ func fetchContentTypes(
|
||||
}
|
||||
|
||||
cont.SetColumnLinks(links)
|
||||
// TODO: stub for columPositions
|
||||
|
||||
cs, err := fetchColumns(ctx, gs, siteID, listID, id)
|
||||
if err != nil {
|
||||
@ -307,7 +301,6 @@ func fetchContentTypes(
|
||||
}
|
||||
|
||||
cont.SetColumns(cs)
|
||||
// TODO: stub for BaseTypes
|
||||
|
||||
cTypes = append(cTypes, cont)
|
||||
}
|
||||
|
||||
@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"runtime/trace"
|
||||
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
@ -14,9 +16,23 @@ import (
|
||||
D "github.com/alcionai/corso/src/internal/diagnostics"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
//----------------------------------------------------------------------------
|
||||
// SharePoint Restore WorkFlow:
|
||||
// - RestoreCollections called by GC component
|
||||
// -- Collections are iterated within, Control Flow Switch
|
||||
// -- Switch:
|
||||
// ---- Libraries restored via the same workflow as oneDrive
|
||||
// ---- Lists call RestoreCollection()
|
||||
// ----> for each data.Stream within Collection.Items()
|
||||
// ----> restoreListItems() is called
|
||||
// Restored List can be found in the Site's `Site content` page
|
||||
// Restored Libraries can be found within the Site's `Pages` page
|
||||
//------------------------------------------
|
||||
|
||||
// RestoreCollections will restore the specified data collections into OneDrive
|
||||
func RestoreCollections(
|
||||
ctx context.Context,
|
||||
@ -51,6 +67,15 @@ func RestoreCollections(
|
||||
dest.ContainerName,
|
||||
deets,
|
||||
errUpdater)
|
||||
case path.ListsCategory:
|
||||
metrics, canceled = RestoreCollection(
|
||||
ctx,
|
||||
service,
|
||||
dc,
|
||||
dest.ContainerName,
|
||||
deets,
|
||||
errUpdater,
|
||||
)
|
||||
default:
|
||||
return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
|
||||
}
|
||||
@ -92,7 +117,7 @@ func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID st
|
||||
// restoreListItem utility function restores a List to the siteID.
|
||||
// The name is changed to to Corso_Restore_{timeStame}_name
|
||||
// API Reference: https://learn.microsoft.com/en-us/graph/api/list-create?view=graph-rest-1.0&tabs=http
|
||||
// Restored List can be verified within the Site contents
|
||||
// Restored List can be verified within the Site contents.
|
||||
func restoreListItem(
|
||||
ctx context.Context,
|
||||
service graph.Servicer,
|
||||
@ -103,9 +128,8 @@ func restoreListItem(
|
||||
defer end()
|
||||
|
||||
var (
|
||||
dii = details.ItemInfo{}
|
||||
itemName = itemData.UUID()
|
||||
displayName = itemName
|
||||
dii = details.ItemInfo{}
|
||||
listName = itemData.UUID()
|
||||
)
|
||||
|
||||
byteArray, err := io.ReadAll(itemData.ToReader())
|
||||
@ -113,35 +137,130 @@ func restoreListItem(
|
||||
return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream")
|
||||
}
|
||||
// Create Item
|
||||
newItem, err := support.CreateListFromBytes(byteArray)
|
||||
oldList, err := support.CreateListFromBytes(byteArray)
|
||||
if err != nil {
|
||||
return dii, errors.Wrapf(err, "failed to construct list item %s", itemName)
|
||||
return dii, errors.Wrapf(err, "failed to build list item %s", listName)
|
||||
}
|
||||
|
||||
// If field "name" is set, this will trigger the following error:
|
||||
// invalidRequest Cannot define a 'name' for a list as it is assigned by the server. Instead, provide 'displayName'
|
||||
if newItem.GetName() != nil {
|
||||
adtlData := newItem.GetAdditionalData()
|
||||
adtlData["list_name"] = *newItem.GetName()
|
||||
newItem.SetName(nil)
|
||||
if oldList.GetDisplayName() != nil {
|
||||
listName = *oldList.GetDisplayName()
|
||||
}
|
||||
|
||||
if newItem.GetDisplayName() != nil {
|
||||
displayName = *newItem.GetDisplayName()
|
||||
newName := fmt.Sprintf("%s_%s", destName, listName)
|
||||
newList := support.ToListable(oldList, newName)
|
||||
|
||||
contents := make([]models.ListItemable, 0)
|
||||
|
||||
for _, itm := range oldList.GetItems() {
|
||||
temp := support.CloneListItem(itm)
|
||||
contents = append(contents, temp)
|
||||
}
|
||||
|
||||
newName := fmt.Sprintf("%s_%s", destName, displayName)
|
||||
newItem.SetDisplayName(&newName)
|
||||
newList.SetItems(contents)
|
||||
|
||||
// Restore to M365 store
|
||||
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newItem, nil)
|
||||
// Restore to List base to M365 back store
|
||||
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
|
||||
if err != nil {
|
||||
return dii, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
|
||||
errorMsg := fmt.Sprintf(
|
||||
"failure to create list foundation ID: %s API Error Details: %s",
|
||||
itemData.UUID(),
|
||||
support.ConnectorStackErrorTrace(err),
|
||||
)
|
||||
|
||||
return dii, errors.Wrap(err, errorMsg)
|
||||
}
|
||||
|
||||
written := int64(len(byteArray))
|
||||
// Uploading of ListItems is conducted after the List is restored
|
||||
// Reference: https://learn.microsoft.com/en-us/graph/api/listitem-create?view=graph-rest-1.0&tabs=http
|
||||
if len(contents) > 0 {
|
||||
for _, lItem := range contents {
|
||||
_, err := service.Client().
|
||||
SitesById(siteID).
|
||||
ListsById(*restoredList.GetId()).
|
||||
Items().
|
||||
Post(ctx, lItem, nil)
|
||||
if err != nil {
|
||||
errorMsg := fmt.Sprintf(
|
||||
"listItem failed for listID %s. Details: %s. Content: %v",
|
||||
*restoredList.GetId(),
|
||||
support.ConnectorStackErrorTrace(err),
|
||||
lItem.GetAdditionalData(),
|
||||
)
|
||||
|
||||
dii.SharePoint = sharePointListInfo(restoredList, written)
|
||||
return dii, errors.Wrap(err, errorMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dii.SharePoint = sharePointListInfo(restoredList, int64(len(byteArray)))
|
||||
|
||||
return dii, nil
|
||||
}
|
||||
|
||||
func RestoreCollection(
|
||||
ctx context.Context,
|
||||
service graph.Servicer,
|
||||
dc data.Collection,
|
||||
restoreContainerName string,
|
||||
deets *details.Builder,
|
||||
errUpdater func(string, error),
|
||||
) (support.CollectionMetrics, bool) {
|
||||
ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath()))
|
||||
defer end()
|
||||
|
||||
var (
|
||||
metrics = support.CollectionMetrics{}
|
||||
directory = dc.FullPath()
|
||||
)
|
||||
|
||||
trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String())
|
||||
siteID := directory.ResourceOwner()
|
||||
|
||||
// Restore items from the collection
|
||||
items := dc.Items()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errUpdater("context canceled", ctx.Err())
|
||||
return metrics, true
|
||||
|
||||
case itemData, ok := <-items:
|
||||
if !ok {
|
||||
return metrics, false
|
||||
}
|
||||
metrics.Objects++
|
||||
|
||||
itemInfo, err := restoreListItem(
|
||||
ctx,
|
||||
service,
|
||||
itemData,
|
||||
siteID,
|
||||
restoreContainerName,
|
||||
)
|
||||
if err != nil {
|
||||
errUpdater(itemData.UUID(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
metrics.TotalBytes += itemInfo.SharePoint.Size
|
||||
|
||||
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
|
||||
if err != nil {
|
||||
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
|
||||
errUpdater(itemData.UUID(), err)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
deets.Add(
|
||||
itemPath.String(),
|
||||
itemPath.ShortRef(),
|
||||
"",
|
||||
true,
|
||||
itemInfo)
|
||||
|
||||
metrics.Successes++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -207,12 +207,24 @@ func ToListable(orig models.Listable, displayName string) models.Listable {
|
||||
}
|
||||
|
||||
for _, cd := range orig.GetColumns() {
|
||||
tag := *cd.GetDisplayName()
|
||||
_, isLegacy := leg[tag]
|
||||
var (
|
||||
displayName string
|
||||
readOnly bool
|
||||
)
|
||||
|
||||
if cd.GetDisplayName() != nil {
|
||||
displayName = *cd.GetDisplayName()
|
||||
}
|
||||
|
||||
if cd.GetReadOnly() != nil {
|
||||
readOnly = *cd.GetReadOnly()
|
||||
}
|
||||
|
||||
_, isLegacy := leg[displayName]
|
||||
|
||||
// Skips columns that cannot be uploaded for models.ColumnDefinitionable:
|
||||
// - ReadOnly, Title, or Legacy columns: Attachments, Edit, or Content Type
|
||||
if *cd.GetReadOnly() || tag == "Title" || isLegacy {
|
||||
if readOnly || displayName == "Title" || isLegacy {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user