larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add clues/fault to sharepoint restores (#2505)

Browse Source 
## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-02-17 15:05:32 -07:00 committed by GitHub
parent daad056d7e
commit 207232e8d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 102 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/internal/connector/data_collections.go
View File

@ -269,7 +269,7 @@ func (gc *GraphConnector) RestoreDataCollections(
case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets)
case selectors.ServiceSharePoint:
status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets)
status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets, errs)
default:
err = clues.Wrap(clues.New(selector.Service.String()), "service not supported")
}

25
src/internal/connector/sharepoint/listInfo.go
View File

@ -1,10 +1,9 @@
package sharepoint
import (
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/backup/details"
)
@ -12,26 +11,12 @@ import (
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo {
var (
name, webURL string
created, modified time.Time
name = ptr.Val(lst.GetDisplayName())
webURL = ptr.Val(lst.GetWebUrl())
created = ptr.Val(lst.GetCreatedDateTime())
modified = ptr.Val(lst.GetLastModifiedDateTime())
)
if lst.GetDisplayName() != nil {
name = *lst.GetDisplayName()
}
if lst.GetWebUrl() != nil {
webURL = *lst.GetWebUrl()
}
if lst.GetCreatedDateTime() != nil {
created = *lst.GetCreatedDateTime()
}
if lst.GetLastModifiedDateTime() != nil {
modified = *lst.GetLastModifiedDateTime()
}
return &details.SharePointInfo{
ItemType: details.SharePointItem,
ItemName: name,

25
src/internal/connector/sharepoint/pageInfo.go
View File

@ -1,8 +1,7 @@
package sharepoint
import (
"time"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
"github.com/alcionai/corso/src/pkg/backup/details"
)
@ -12,26 +11,12 @@ import (
// Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
func sharePointPageInfo(page models.SitePageable, size int64) *details.SharePointInfo {
var (
name, webURL string
created, modified time.Time
name = ptr.Val(page.GetTitle())
webURL = ptr.Val(page.GetWebUrl())
created = ptr.Val(page.GetCreatedDateTime())
modified = ptr.Val(page.GetLastModifiedDateTime())
)
if page.GetTitle() != nil {
name = *page.GetTitle()
}
if page.GetWebUrl() != nil {
webURL = *page.GetWebUrl()
}
if page.GetCreatedDateTime() != nil {
created = *page.GetCreatedDateTime()
}
if page.GetLastModifiedDateTime() != nil {
modified = *page.GetLastModifiedDateTime()
}
return &details.SharePointInfo{
ItemType: details.SharePointItem,
ItemName: name,

8
src/internal/connector/sharepoint/queries.go
View File

@ -6,6 +6,7 @@ import (
absser "github.com/microsoft/kiota-abstractions-go/serialization"
mssite "github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector/graph"
)
@ -19,5 +20,10 @@ func GetAllSitesForTenant(ctx context.Context, gs graph.Servicer) (absser.Parsab
},
}
return gs.Client().Sites().Get(ctx, options)
sites, err := gs.Client().Sites().Get(ctx, options)
if err != nil {
return nil, clues.Wrap(err, "getting sites").WithClues(ctx).With(graph.ErrData(err)...)
}
return sites, nil
}

177
src/internal/connector/sharepoint/restore.go
View File

@ -6,9 +6,10 @@ import (
"io"
"runtime/trace"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
discover "github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
@ -19,7 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -45,27 +46,29 @@ func RestoreCollections(
dest control.RestoreDestination,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Errors,
) (*support.ConnectorOperationStatus, error) {
var (
err error
restoreMetrics support.CollectionMetrics
restoreErrors error
)
errUpdater := func(id string, err error) {
restoreErrors = support.WrapAndAppend(id, err, restoreErrors)
}
// Iterate through the data collections and restore the contents of each
for _, dc := range dcs {
var (
metrics support.CollectionMetrics
canceled bool
category = dc.FullPath().Category()
metrics support.CollectionMetrics
ictx = clues.Add(ctx,
"category", category,
"destination", dest.ContainerName, // TODO: pii
"resource_owner", dc.FullPath().ResourceOwner()) // TODO: pii
)
switch dc.FullPath().Category() {
case path.LibrariesCategory:
metrics, _, _, canceled = onedrive.RestoreCollection(
ctx,
ictx,
backupVersion,
service,
dc,
@ -73,61 +76,59 @@ func RestoreCollections(
onedrive.SharePointSource,
dest.ContainerName,
deets,
errUpdater,
func(s string, err error) { errs.Add(err) },
map[string]string{},
false,
)
false)
case path.ListsCategory:
metrics, canceled = RestoreListCollection(
ctx,
metrics, err = RestoreListCollection(
ictx,
service,
dc,
dest.ContainerName,
deets,
errUpdater,
)
errs)
case path.PagesCategory:
metrics, canceled = RestorePageCollection(
ctx,
metrics, err = RestorePageCollection(
ictx,
creds,
dc,
dest.ContainerName,
deets,
errUpdater,
)
errs)
default:
return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
return nil, clues.Wrap(clues.New(category.String()), "category not supported")
}
restoreMetrics.Combine(metrics)
if canceled {
if canceled || err != nil {
break
}
}
return support.CreateStatus(
ctx,
support.Restore,
len(dcs),
restoreMetrics,
restoreErrors,
dest.ContainerName),
nil
status := support.CreateStatus(
ctx,
support.Restore,
len(dcs),
restoreMetrics,
err,
dest.ContainerName)
return status, err
}
// createRestoreFolders creates the restore folder hieararchy in the specified drive and returns the folder ID
// of the last folder entry given in the hiearchy
func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID string, restoreFolders []string,
func createRestoreFolders(
ctx context.Context,
service graph.Servicer,
siteID string,
restoreFolders []string,
) (string, error) {
// Get Main Drive for Site, Documents
mainDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil)
if err != nil {
return "", errors.Wrapf(
err,
"failed to get site drive root. details: %s",
support.ConnectorStackErrorTrace(err),
)
return "", clues.Wrap(err, "getting site drive root").WithClues(ctx).With(graph.ErrData(err)...)
}
return onedrive.CreateRestoreFolders(ctx, service, *mainDrive.GetId(), restoreFolders)
@ -146,6 +147,8 @@ func restoreListItem(
ctx, end := D.Span(ctx, "gc:sharepoint:restoreList", D.Label("item_uuid", itemData.UUID()))
defer end()
ctx = clues.Add(ctx, "list_item_id", itemData.UUID())
var (
dii = details.ItemInfo{}
listName = itemData.UUID()
@ -153,22 +156,23 @@ func restoreListItem(
byteArray, err := io.ReadAll(itemData.ToReader())
if err != nil {
return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream")
return dii, clues.Wrap(err, "reading backup data").WithClues(ctx)
}
// Create Item
oldList, err := support.CreateListFromBytes(byteArray)
if err != nil {
return dii, errors.Wrapf(err, "failed to build list item %s", listName)
return dii, clues.Wrap(err, "creating item").WithClues(ctx)
}
if oldList.GetDisplayName() != nil {
listName = *oldList.GetDisplayName()
}
newName := fmt.Sprintf("%s_%s", destName, listName)
newList := support.ToListable(oldList, newName)
contents := make([]models.ListItemable, 0)
var (
newName = fmt.Sprintf("%s_%s", destName, listName)
newList = support.ToListable(oldList, newName)
contents = make([]models.ListItemable, 0)
)
for _, itm := range oldList.GetItems() {
temp := support.CloneListItem(itm)
@ -180,13 +184,7 @@ func restoreListItem(
// Restore to List base to M365 back store
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
if err != nil {
errorMsg := fmt.Sprintf(
"failure to create list foundation ID: %s API Error Details: %s",
itemData.UUID(),
support.ConnectorStackErrorTrace(err),
)
return dii, errors.Wrap(err, errorMsg)
return dii, clues.Wrap(err, "restoring list").WithClues(ctx).With(graph.ErrData(err)...)
}
// Uploading of ListItems is conducted after the List is restored
@ -199,14 +197,10 @@ func restoreListItem(
Items().
Post(ctx, lItem, nil)
if err != nil {
errorMsg := fmt.Sprintf(
"listItem failed for listID %s. Details: %s. Content: %v",
*restoredList.GetId(),
support.ConnectorStackErrorTrace(err),
lItem.GetAdditionalData(),
)
return dii, errors.Wrap(err, errorMsg)
return dii, clues.Wrap(err, "restoring list items").
With("restored_list_id", ptr.Val(restoredList.GetId())).
WithClues(ctx).
With(graph.ErrData(err)...)
}
}
}
@ -222,31 +216,32 @@ func RestoreListCollection(
dc data.RestoreCollection,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
errs *fault.Errors,
) (support.CollectionMetrics, error) {
ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath()))
defer end()
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
siteID = directory.ResourceOwner()
items = dc.Items(ctx, errs)
)
trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from the collection
items := dc.Items(ctx, nil) // TODO: fault.Errors instead of nil
for {
if errs.Err() != nil {
return metrics, errs.Err()
}
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, true
return metrics, clues.Stack(ctx.Err()).WithClues(ctx)
case itemData, ok := <-items:
if !ok {
return metrics, false
return metrics, nil
}
metrics.Objects++
@ -255,10 +250,9 @@ func RestoreListCollection(
service,
itemData,
siteID,
restoreContainerName,
)
restoreContainerName)
if err != nil {
errUpdater(itemData.UUID(), err)
errs.Add(err)
continue
}
@ -266,9 +260,7 @@ func RestoreListCollection(
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
continue
}
@ -295,38 +287,41 @@ func RestorePageCollection(
dc data.RestoreCollection,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
defer end()
errs *fault.Errors,
) (support.CollectionMetrics, error) {
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
siteID = directory.ResourceOwner()
)
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
defer end()
adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret)
if err != nil {
return metrics, false
return metrics, clues.Wrap(err, "constructing graph client")
}
service := discover.NewBetaService(adpt)
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from collection
items := dc.Items(ctx, nil) // TODO: fault.Errors instead of nil
for {
if errs.Err() != nil {
return metrics, errs.Err()
}
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, true
return metrics, clues.Stack(ctx.Err()).WithClues(ctx)
case itemData, ok := <-items:
if !ok {
return metrics, false
return metrics, nil
}
metrics.Objects++
@ -335,10 +330,9 @@ func RestorePageCollection(
service,
itemData,
siteID,
restoreContainerName,
)
restoreContainerName)
if err != nil {
errUpdater(itemData.UUID(), err)
errs.Add(err)
continue
}
@ -346,9 +340,7 @@ func RestorePageCollection(
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).Errorw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
continue
}
@ -358,8 +350,7 @@ func RestorePageCollection(
"",
"", // TODO: implement locationRef
true,
itemInfo,
)
itemInfo)
metrics.Successes++
}