GC: Restore: SharePoint: Collection logic (#2227)

## Description
Updates SharePoint Restore Collection Logic.
Test Suite included.
Restore Pipeline is not connected in this PR for ease of parsing. 

It is noted that there is a large amount of code duplication between Lists and Pages. Code Clean-Up will address these issues once issue #2174 has been handled.  As this will require the use of an HTTP client that is not necessary for other services. 
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x]  No 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature


## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* related to  #2169<issue>

## Test Plan

- [x]  Unit test
This commit is contained in:
Danny 2023-02-06 10:27:39 -05:00 committed by GitHub
parent d82b5cacdf
commit b3b5189e19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 97 additions and 10 deletions

View File

@ -293,7 +293,7 @@ func (gc *GraphConnector) RestoreDataCollections(
case selectors.ServiceOneDrive: case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets) status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets)
case selectors.ServiceSharePoint: case selectors.ServiceSharePoint:
status, err = sharepoint.RestoreCollections(ctx, backupVersion, gc.Service, dest, dcs, deets) status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets)
default: default:
err = errors.Errorf("restore data from service %s not supported", selector.Service.String()) err = errors.Errorf("restore data from service %s not supported", selector.Service.String())
} }

View File

@ -9,11 +9,14 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors" "github.com/pkg/errors"
discover "github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
D "github.com/alcionai/corso/src/internal/diagnostics" D "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -37,6 +40,7 @@ import (
func RestoreCollections( func RestoreCollections(
ctx context.Context, ctx context.Context,
backupVersion int, backupVersion int,
creds account.M365Config,
service graph.Servicer, service graph.Servicer,
dest control.RestoreDestination, dest control.RestoreDestination,
dcs []data.Collection, dcs []data.Collection,
@ -74,7 +78,7 @@ func RestoreCollections(
false, false,
) )
case path.ListsCategory: case path.ListsCategory:
metrics, canceled = RestoreCollection( metrics, canceled = RestoreListCollection(
ctx, ctx,
service, service,
dc, dc,
@ -83,11 +87,14 @@ func RestoreCollections(
errUpdater, errUpdater,
) )
case path.PagesCategory: case path.PagesCategory:
errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category()) metrics, canceled = RestorePageCollection(
logger.Ctx(ctx).Error(errorMessage) ctx,
creds,
return nil, errors.New(errorMessage) dc,
dest.ContainerName,
deets,
errUpdater,
)
default: default:
return nil, errors.Errorf("category %s not supported", dc.FullPath().Category()) return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
} }
@ -209,7 +216,7 @@ func restoreListItem(
return dii, nil return dii, nil
} }
func RestoreCollection( func RestoreListCollection(
ctx context.Context, ctx context.Context,
service graph.Servicer, service graph.Servicer,
dc data.Collection, dc data.Collection,
@ -217,7 +224,7 @@ func RestoreCollection(
deets *details.Builder, deets *details.Builder,
errUpdater func(string, error), errUpdater func(string, error),
) (support.CollectionMetrics, bool) { ) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restoreCollection", D.Label("path", dc.FullPath())) ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath()))
defer end() defer end()
var ( var (
@ -225,7 +232,7 @@ func RestoreCollection(
directory = dc.FullPath() directory = dc.FullPath()
) )
trace.Log(ctx, "gc:sharepoint:restoreCollection", directory.String()) trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
siteID := directory.ResourceOwner() siteID := directory.ResourceOwner()
// Restore items from the collection // Restore items from the collection
@ -276,3 +283,83 @@ func RestoreCollection(
} }
} }
} }
// RestorePageCollection handles restoration of an individual site page collection.
// returns:
// - the collection's item and byte count metrics
// - the context cancellation station. True iff context is canceled.
func RestorePageCollection(
ctx context.Context,
creds account.M365Config,
dc data.Collection,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:sharepoint:restorePageCollection", D.Label("path", dc.FullPath()))
defer end()
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
)
adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret)
if err != nil {
return metrics, false
}
service := discover.NewBetaService(adpt)
trace.Log(ctx, "gc:sharepoint:restorePageCollection", directory.String())
siteID := directory.ResourceOwner()
// Restore items from collection
items := dc.Items()
for {
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, true
case itemData, ok := <-items:
if !ok {
return metrics, false
}
metrics.Objects++
itemInfo, err := api.RestoreSitePage(
ctx,
service,
itemData,
siteID,
restoreContainerName,
)
if err != nil {
errUpdater(itemData.UUID(), err)
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).Errorw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
continue
}
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
true,
itemInfo,
)
metrics.Successes++
}
}
}