add clues/fault to sharepoint restores (#2505)

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/connector/data_collections.go

@@ -269,7 +269,7 @@ func (gc *GraphConnector) RestoreDataCollections(
 	case selectors.ServiceOneDrive:
 		status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets)
 	case selectors.ServiceSharePoint:
-		status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets)
+		status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets, errs)
 	default:
 		err = clues.Wrap(clues.New(selector.Service.String()), "service not supported")
 	}

src/internal/connector/sharepoint/listInfo.go

@@ -1,10 +1,9 @@
 package sharepoint
 
 import (
-	"time"
-
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 )
 
@@ -12,26 +11,12 @@ import (
 // List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
 func sharePointListInfo(lst models.Listable, size int64) *details.SharePointInfo {
 	var (
-		name, webURL      string
+		name     = ptr.Val(lst.GetDisplayName())
-		created, modified time.Time
+		webURL   = ptr.Val(lst.GetWebUrl())
+		created  = ptr.Val(lst.GetCreatedDateTime())
+		modified = ptr.Val(lst.GetLastModifiedDateTime())
 	)
 
-	if lst.GetDisplayName() != nil {
-		name = *lst.GetDisplayName()
-	}
-
-	if lst.GetWebUrl() != nil {
-		webURL = *lst.GetWebUrl()
-	}
-
-	if lst.GetCreatedDateTime() != nil {
-		created = *lst.GetCreatedDateTime()
-	}
-
-	if lst.GetLastModifiedDateTime() != nil {
-		modified = *lst.GetLastModifiedDateTime()
-	}
-
 	return &details.SharePointInfo{
 		ItemType: details.SharePointItem,
 		ItemName: name,

src/internal/connector/sharepoint/pageInfo.go

@@ -1,8 +1,7 @@
 package sharepoint
 
 import (
-	"time"
+	"github.com/alcionai/corso/src/internal/common/ptr"
-
 	"github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 )
@@ -12,26 +11,12 @@ import (
 // Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
 func sharePointPageInfo(page models.SitePageable, size int64) *details.SharePointInfo {
 	var (
-		name, webURL      string
+		name     = ptr.Val(page.GetTitle())
-		created, modified time.Time
+		webURL   = ptr.Val(page.GetWebUrl())
+		created  = ptr.Val(page.GetCreatedDateTime())
+		modified = ptr.Val(page.GetLastModifiedDateTime())
 	)
 
-	if page.GetTitle() != nil {
-		name = *page.GetTitle()
-	}
-
-	if page.GetWebUrl() != nil {
-		webURL = *page.GetWebUrl()
-	}
-
-	if page.GetCreatedDateTime() != nil {
-		created = *page.GetCreatedDateTime()
-	}
-
-	if page.GetLastModifiedDateTime() != nil {
-		modified = *page.GetLastModifiedDateTime()
-	}
-
 	return &details.SharePointInfo{
 		ItemType: details.SharePointItem,
 		ItemName: name,

src/internal/connector/sharepoint/queries.go

@@ -6,6 +6,7 @@ import (
 	absser "github.com/microsoft/kiota-abstractions-go/serialization"
 	mssite "github.com/microsoftgraph/msgraph-sdk-go/sites"
 
+	"github.com/alcionai/clues"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 )
 
@@ -19,5 +20,10 @@ func GetAllSitesForTenant(ctx context.Context, gs graph.Servicer) (absser.Parsab
 		},
 	}
 
-	return gs.Client().Sites().Get(ctx, options)
+	sites, err := gs.Client().Sites().Get(ctx, options)
+	if err != nil {
+		return nil, clues.Wrap(err, "getting sites").WithClues(ctx).With(graph.ErrData(err)...)
+	}
+
+	return sites, nil
 }

src/internal/connector/sharepoint/restore.go

@@ -6,9 +6,10 @@ import (
 	"io"
 	"runtime/trace"
 
+	"github.com/alcionai/clues"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/pkg/errors"
 
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	discover "github.com/alcionai/corso/src/internal/connector/discovery/api"
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
@@ -19,7 +20,7 @@ import (
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
-	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
@@ -45,27 +46,29 @@ func RestoreCollections(
 	dest control.RestoreDestination,
 	dcs []data.RestoreCollection,
 	deets *details.Builder,
+	errs *fault.Errors,
 ) (*support.ConnectorOperationStatus, error) {
 	var (
+		err            error
 		restoreMetrics support.CollectionMetrics
-		restoreErrors  error
 	)
 
-	errUpdater := func(id string, err error) {
-		restoreErrors = support.WrapAndAppend(id, err, restoreErrors)
-	}
-
 	// Iterate through the data collections and restore the contents of each
 	for _, dc := range dcs {
 		var (
-			metrics  support.CollectionMetrics
 			canceled bool
+			category = dc.FullPath().Category()
+			metrics  support.CollectionMetrics
+			ictx     = clues.Add(ctx,
+				"category", category,
+				"destination", dest.ContainerName, // TODO: pii
+				"resource_owner", dc.FullPath().ResourceOwner()) // TODO: pii
 		)
 
 		switch dc.FullPath().Category() {
 		case path.LibrariesCategory:
 			metrics, _, _, canceled = onedrive.RestoreCollection(
-				ctx,
+				ictx,
 				backupVersion,
 				service,
 				dc,
@@ -73,61 +76,59 @@ func RestoreCollections(
 				onedrive.SharePointSource,
 				dest.ContainerName,
 				deets,
-				errUpdater,
+				func(s string, err error) { errs.Add(err) },
 				map[string]string{},
-				false,
+				false)
-			)
 		case path.ListsCategory:
-			metrics, canceled = RestoreListCollection(
+			metrics, err = RestoreListCollection(
-				ctx,
+				ictx,
 				service,
 				dc,
 				dest.ContainerName,
 				deets,
-				errUpdater,
+				errs)
-			)
 		case path.PagesCategory:
-			metrics, canceled = RestorePageCollection(
+			metrics, err = RestorePageCollection(
-				ctx,
+				ictx,
 				creds,
 				dc,
 				dest.ContainerName,
 				deets,
-				errUpdater,
+				errs)
-			)
 		default:
-			return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
+			return nil, clues.Wrap(clues.New(category.String()), "category not supported")
 		}
 
 		restoreMetrics.Combine(metrics)
 
-		if canceled {
+		if canceled || err != nil {
 			break
 		}
 	}
 
-	return support.CreateStatus(
+	status := support.CreateStatus(
-			ctx,
+		ctx,
-			support.Restore,
+		support.Restore,
-			len(dcs),
+		len(dcs),
-			restoreMetrics,
+		restoreMetrics,
-			restoreErrors,
+		err,
-			dest.ContainerName),
+		dest.ContainerName)
-		nil
+
+	return status, err
 }
 
 // createRestoreFolders creates the restore folder hieararchy in the specified drive and returns the folder ID
 // of the last folder entry given in the hiearchy
-func createRestoreFolders(ctx context.Context, service graph.Servicer, siteID string, restoreFolders []string,
+func createRestoreFolders(
+	ctx context.Context,
+	service graph.Servicer,
+	siteID string,
+	restoreFolders []string,
 ) (string, error) {
 	// Get Main Drive for Site, Documents
 	mainDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil)
 	if err != nil {
-		return "", errors.Wrapf(
+		return "", clues.Wrap(err, "getting site drive root").WithClues(ctx).With(graph.ErrData(err)...)
-			err,
-			"failed to get site drive root. details: %s",
-			support.ConnectorStackErrorTrace(err),
-		)
 	}
 
 	return onedrive.CreateRestoreFolders(ctx, service, *mainDrive.GetId(), restoreFolders)
@@ -146,6 +147,8 @@ func restoreListItem(
 	ctx, end := D.Span(ctx, "gc:sharepoint:restoreList", D.Label("item_uuid", itemData.UUID()))
 	defer end()
 
+	ctx = clues.Add(ctx, "list_item_id", itemData.UUID())
+
 	var (
 		dii      = details.ItemInfo{}
 		listName = itemData.UUID()
@@ -153,22 +156,23 @@ func restoreListItem(
 
 	byteArray, err := io.ReadAll(itemData.ToReader())
 	if err != nil {
-		return dii, errors.Wrap(err, "sharepoint restoreItem failed to retrieve bytes from data.Stream")
+		return dii, clues.Wrap(err, "reading backup data").WithClues(ctx)
 	}
-	// Create Item
+
 	oldList, err := support.CreateListFromBytes(byteArray)
 	if err != nil {
-		return dii, errors.Wrapf(err, "failed to build list item %s", listName)
+		return dii, clues.Wrap(err, "creating item").WithClues(ctx)
 	}
 
 	if oldList.GetDisplayName() != nil {
 		listName = *oldList.GetDisplayName()
 	}
 
-	newName := fmt.Sprintf("%s_%s", destName, listName)
+	var (
-	newList := support.ToListable(oldList, newName)
+		newName  = fmt.Sprintf("%s_%s", destName, listName)
-
+		newList  = support.ToListable(oldList, newName)
-	contents := make([]models.ListItemable, 0)
+		contents = make([]models.ListItemable, 0)
+	)
 
 	for _, itm := range oldList.GetItems() {
 		temp := support.CloneListItem(itm)
@@ -180,13 +184,7 @@ func restoreListItem(
 	// Restore to List base to M365 back store
 	restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
 	if err != nil {
-		errorMsg := fmt.Sprintf(
+		return dii, clues.Wrap(err, "restoring list").WithClues(ctx).With(graph.ErrData(err)...)
-			"failure to create list foundation ID: %s API Error Details: %s",
-			itemData.UUID(),
-			support.ConnectorStackErrorTrace(err),
-		)
-
-		return dii, errors.Wrap(err, errorMsg)
 	}
 
 	// Uploading of ListItems is conducted after the List is restored
@@ -199,14 +197,10 @@ func restoreListItem(
 				Items().
 				Post(ctx, lItem, nil)
 			if err != nil {
-				errorMsg := fmt.Sprintf(
+				return dii, clues.Wrap(err, "restoring list items").
-					"listItem  failed for listID %s. Details: %s. Content: %v",
+					With("restored_list_id", ptr.Val(restoredList.GetId())).
-					*restoredList.GetId(),
+					WithClues(ctx).
-					support.ConnectorStackErrorTrace(err),
+					With(graph.ErrData(err)...)
-					lItem.GetAdditionalData(),
-				)
-
-				return dii, errors.Wrap(err, errorMsg)
 			}
 		}
 	}
@@ -222,31 +216,32 @@ func RestoreListCollection(
 	dc data.RestoreCollection,
 	restoreContainerName string,
 	deets *details.Builder,
-	errUpdater func(string, error),
+	errs *fault.Errors,
-) (support.CollectionMetrics, bool) {
+) (support.CollectionMetrics, error) {
 	ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath()))
 	defer end()
 
 	var (
 		metrics   = support.CollectionMetrics{}
 		directory = dc.FullPath()
+		siteID    = directory.ResourceOwner()
+		items     = dc.Items(ctx, errs)
 	)
 
 	trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
-	siteID := directory.ResourceOwner()
-
-	// Restore items from the collection
-	items := dc.Items(ctx, nil) // TODO: fault.Errors instead of nil
 
 	for {
+		if errs.Err() != nil {
+			return metrics, errs.Err()
+		}
+
 		select {
 		case <-ctx.Done():
-			errUpdater("context canceled", ctx.Err())
+			return metrics, clues.Stack(ctx.Err()).WithClues(ctx)
-			return metrics, true
 
 		case itemData, ok := <-items:
 			if !ok {
-				return metrics, false
+				return metrics, nil
 			}
 			metrics.Objects++
 
@@ -255,10 +250,9 @@ func RestoreListCollection(
 				service,
 				itemData,
 				siteID,
-				restoreContainerName,
+				restoreContainerName)
-			)
 			if err != nil {
-				errUpdater(itemData.UUID(), err)
+				errs.Add(err)
 				continue
 			}
 
@@ -266,9 +260,7 @@ func RestoreListCollection(
 
 			itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
 			if err != nil {
-				logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
+				errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
-				errUpdater(itemData.UUID(), err)
-
 				continue
 			}
 
@@ -295,38 +287,41 @@ func RestorePageCollection(
 	dc data.RestoreCollection,
 	restoreContainerName string,
 	deets *details.Builder,
-	errUpdater func(string, error),
+	errs *fault.Errors,
-) (support.CollectionMetrics, bool) {
+) (support.CollectionMetrics, error) {
-	ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
-	defer end()
-
 	var (
 		metrics   = support.CollectionMetrics{}
 		directory = dc.FullPath()
+		siteID    = directory.ResourceOwner()
 	)
 
+	trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
+	ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
+
+	defer end()
+
 	adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret)
 	if err != nil {
-		return metrics, false
+		return metrics, clues.Wrap(err, "constructing graph client")
 	}
 
 	service := discover.NewBetaService(adpt)
 
-	trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
-	siteID := directory.ResourceOwner()
-
 	// Restore items from collection
 	items := dc.Items(ctx, nil) // TODO: fault.Errors instead of nil
 
 	for {
+		if errs.Err() != nil {
+			return metrics, errs.Err()
+		}
+
 		select {
 		case <-ctx.Done():
-			errUpdater("context canceled", ctx.Err())
+			return metrics, clues.Stack(ctx.Err()).WithClues(ctx)
-			return metrics, true
 
 		case itemData, ok := <-items:
 			if !ok {
-				return metrics, false
+				return metrics, nil
 			}
 			metrics.Objects++
 
@@ -335,10 +330,9 @@ func RestorePageCollection(
 				service,
 				itemData,
 				siteID,
-				restoreContainerName,
+				restoreContainerName)
-			)
 			if err != nil {
-				errUpdater(itemData.UUID(), err)
+				errs.Add(err)
 				continue
 			}
 
@@ -346,9 +340,7 @@ func RestorePageCollection(
 
 			itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
 			if err != nil {
-				logger.Ctx(ctx).Errorw("transforming item to full path", "error", err)
+				errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
-				errUpdater(itemData.UUID(), err)
-
 				continue
 			}
 
@@ -358,8 +350,7 @@ func RestorePageCollection(
 				"",
 				"", // TODO: implement locationRef
 				true,
-				itemInfo,
+				itemInfo)
-			)
 
 			metrics.Successes++
 		}