larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add deets to gc restore ops (#1029)

Browse Source 
## Description

In order for a restore operation to accurately print
out the list of items that were restored, it needs to
track item details within the GC restoration like is
done within kopia in backup.   These details will
not be stored in a modelStore.  But they will get
printed out on the CLI for end users.

Part 1 of multiple.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #977

## Test Plan

- [x] 💪 Manual
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-10-04 11:56:34 -06:00 committed by GitHub
parent dd34ecd5f7
commit d0560500d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 69 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/internal/connector/exchange/service_restore.go
View File

@ -13,6 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -288,6 +289,7 @@ func RestoreExchangeDataCollections(
gs graph.Service,
dest control.RestoreDestination,
dcs []data.Collection,
deets *details.Details,
) (*support.ConnectorOperationStatus, error) {
var (
pathCounter = map[string]bool{}
@ -303,7 +305,7 @@ func RestoreExchangeDataCollections(
}
for _, dc := range dcs {
temp, root, canceled := restoreCollection(ctx, gs, dc, rootFolder, pathCounter, dest, policy, errUpdater)
temp, root, canceled := restoreCollection(ctx, gs, dc, rootFolder, pathCounter, dest, policy, deets, errUpdater)
metrics.Combine(temp)
@ -333,6 +335,7 @@ func restoreCollection(
pathCounter map[string]bool,
dest control.RestoreDestination,
policy control.CollisionPolicy,
deets *details.Details,
errUpdater func(string, error),
) (support.CollectionMetrics, string, bool) {
defer trace.StartRegion(ctx, "gc:exchange:restoreCollection").End()
@ -392,6 +395,18 @@ func restoreCollection(
metrics.TotalBytes += int64(len(byteArray))
metrics.Successes++
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
continue
}
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
details.ItemInfo{})
}
}
}

10
src/internal/connector/graph_connector.go
View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/selectors"
@ -252,19 +253,20 @@ func (gc *GraphConnector) RestoreDataCollections(
selector selectors.Selector,
dest control.RestoreDestination,
dcs []data.Collection,
) error {
) (*details.Details, error) {
gc.region = trace.StartRegion(ctx, "connector:restore")
var (
status *support.ConnectorOperationStatus
err error
deets = &details.Details{}
)
switch selector.Service {
case selectors.ServiceExchange:
status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs)
status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs, deets)
case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs)
status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs, deets)
default:
err = errors.Errorf("restore data from service %s not supported", selector.Service.String())
}
@ -272,7 +274,7 @@ func (gc *GraphConnector) RestoreDataCollections(
gc.incrementAwaitingMessages()
gc.UpdateStatus(status)
return err
return deets, err
}
// createCollection - utility function that retrieves M365

4
src/internal/connector/graph_connector_disconnected_test.go
View File

@ -190,7 +190,9 @@ func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() {
}
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive)
assert.Error(t, gc.RestoreDataCollections(ctx, sel, dest, nil))
deets, err := gc.RestoreDataCollections(ctx, sel, dest, nil)
assert.Error(t, err)
assert.NotNil(t, deets)
status := gc.AwaitStatus()
assert.Equal(t, 0, status.ObjectCount)

15
src/internal/connector/graph_connector_test.go
View File

@ -408,8 +408,9 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
suite.T().Run(test.name, func(t *testing.T) {
ctx := context.Background()
err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col)
deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col)
require.NoError(t, err)
assert.NotNil(t, deets)
stats := suite.connector.AwaitStatus()
assert.Zero(t, stats.ObjectCount)
@ -687,13 +688,17 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
restoreGC := loadConnector(ctx, t)
restoreSel := getSelectorWith(test.service)
err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
require.NoError(t, err)
assert.NotNil(t, deets)
status := restoreGC.AwaitStatus()
assert.Equal(t, test.expectedRestoreFolders, status.FolderCount, "status.FolderCount")
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal(
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
t.Logf("Restore complete\n")
@ -822,14 +827,18 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
)
restoreGC := loadConnector(ctx, t)
err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections)
require.NoError(t, err)
require.NotNil(t, deets)
status := restoreGC.AwaitStatus()
// Always just 1 because it's just 1 collection.
assert.Equal(t, 1, status.FolderCount, "status.FolderCount")
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal(
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
t.Logf("Restore complete\n")
}

32
src/internal/connector/onedrive/restore.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
@ -50,6 +51,7 @@ func RestoreCollections(
service graph.Service,
dest control.RestoreDestination,
dcs []data.Collection,
deets *details.Details,
) (*support.ConnectorOperationStatus, error) {
var (
restoreMetrics support.CollectionMetrics
@ -62,7 +64,7 @@ func RestoreCollections(
// Iterate through the data collections and restore the contents of each
for _, dc := range dcs {
temp, canceled := restoreCollection(ctx, service, dc, dest.ContainerName, errUpdater)
temp, canceled := restoreCollection(ctx, service, dc, dest.ContainerName, deets, errUpdater)
restoreMetrics.Combine(temp)
@ -89,6 +91,7 @@ func restoreCollection(
service graph.Service,
dc data.Collection,
restoreContainerName string,
deets *details.Details,
errUpdater func(string, error),
) (support.CollectionMetrics, bool) {
defer trace.StartRegion(ctx, "gc:oneDrive:restoreCollection").End()
@ -139,12 +142,31 @@ func restoreCollection(
metrics.TotalBytes += int64(len(copyBuffer))
err := restoreItem(ctx, service, itemData, drivePath.driveID, restoreFolderID, copyBuffer)
err := restoreItem(ctx,
service,
itemData,
drivePath.driveID,
restoreFolderID,
copyBuffer)
if err != nil {
errUpdater(itemData.UUID(), err)
continue
}
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
continue
}
deets.Add(
itemPath.String(),
itemPath.ShortRef(),
"",
details.ItemInfo{})
metrics.Successes++
}
}
@ -196,7 +218,11 @@ func createRestoreFolders(ctx context.Context, service graph.Service, driveID st
}
// restoreItem will create a new item in the specified `parentFolderID` and upload the data.Stream
func restoreItem(ctx context.Context, service graph.Service, itemData data.Stream, driveID, parentFolderID string,
func restoreItem(
ctx context.Context,
service graph.Service,
itemData data.Stream,
driveID, parentFolderID string,
copyBuffer []byte,
) error {
defer trace.StartRegion(ctx, "gc:oneDrive:restoreItem").End()

5
src/internal/operations/restore.go
View File

@ -209,7 +209,8 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
return err
}
err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
// TODO: return details and print in CLI
_, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs)
if err != nil {
err = errors.Wrap(err, "restoring service data")
opStats.writeErr = err
@ -225,7 +226,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
return nil
}
// writes the restoreOperation outcome to the modelStore.
// persists details and statistics about the restore operation.
func (op *RestoreOperation) persistResults(
ctx context.Context,
started time.Time,