From d0560500d2640ba5c95bb250a7aea78895e03669 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 4 Oct 2022 11:56:34 -0600 Subject: [PATCH] add deets to gc restore ops (#1029) ## Description In order for a restore operation to accurately print out the list of items that were restored, it needs to track item details within the GC restoration like is done within kopia in backup. These details will not be stored in a modelStore. But they will get printed out on the CLI for end users. Part 1 of multiple. ## Type of change - [x] :sunflower: Feature ## Issue(s) * #977 ## Test Plan - [x] :muscle: Manual - [x] :green_heart: E2E --- .../connector/exchange/service_restore.go | 17 +++++++++- src/internal/connector/graph_connector.go | 10 +++--- .../graph_connector_disconnected_test.go | 4 ++- .../connector/graph_connector_test.go | 15 +++++++-- src/internal/connector/onedrive/restore.go | 32 +++++++++++++++++-- src/internal/operations/restore.go | 5 +-- 6 files changed, 69 insertions(+), 14 deletions(-) diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 8c259b46c..33833b9df 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -288,6 +289,7 @@ func RestoreExchangeDataCollections( gs graph.Service, dest control.RestoreDestination, dcs []data.Collection, + deets *details.Details, ) (*support.ConnectorOperationStatus, error) { var ( pathCounter = map[string]bool{} @@ -303,7 +305,7 @@ func RestoreExchangeDataCollections( } for _, dc := range dcs { - temp, root, canceled := restoreCollection(ctx, gs, dc, rootFolder, pathCounter, dest, policy, errUpdater) + temp, root, canceled := restoreCollection(ctx, gs, dc, rootFolder, pathCounter, dest, policy, deets, errUpdater) metrics.Combine(temp) @@ -333,6 +335,7 @@ func restoreCollection( pathCounter map[string]bool, dest control.RestoreDestination, policy control.CollisionPolicy, + deets *details.Details, errUpdater func(string, error), ) (support.CollectionMetrics, string, bool) { defer trace.StartRegion(ctx, "gc:exchange:restoreCollection").End() @@ -392,6 +395,18 @@ func restoreCollection( metrics.TotalBytes += int64(len(byteArray)) metrics.Successes++ + + itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + if err != nil { + logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err) + continue + } + + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + details.ItemInfo{}) } } } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index b3f405e29..3bff2de6a 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/selectors" @@ -252,19 +253,20 @@ func (gc *GraphConnector) RestoreDataCollections( selector selectors.Selector, dest control.RestoreDestination, dcs []data.Collection, -) error { +) (*details.Details, error) { gc.region = trace.StartRegion(ctx, "connector:restore") var ( status *support.ConnectorOperationStatus err error + deets = &details.Details{} ) switch selector.Service { case selectors.ServiceExchange: - status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs) + status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs, deets) case selectors.ServiceOneDrive: - status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs) + status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs, deets) default: err = errors.Errorf("restore data from service %s not supported", selector.Service.String()) } @@ -272,7 +274,7 @@ func (gc *GraphConnector) RestoreDataCollections( gc.incrementAwaitingMessages() gc.UpdateStatus(status) - return err + return deets, err } // createCollection - utility function that retrieves M365 diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index cfaca3698..dbfc343d2 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -190,7 +190,9 @@ func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() { } dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormatOneDrive) - assert.Error(t, gc.RestoreDataCollections(ctx, sel, dest, nil)) + deets, err := gc.RestoreDataCollections(ctx, sel, dest, nil) + assert.Error(t, err) + assert.NotNil(t, deets) status := gc.AwaitStatus() assert.Equal(t, 0, status.ObjectCount) diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 2845be9af..8de1fb994 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -408,8 +408,9 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { suite.T().Run(test.name, func(t *testing.T) { ctx := context.Background() - err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col) + deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col) require.NoError(t, err) + assert.NotNil(t, deets) stats := suite.connector.AwaitStatus() assert.Zero(t, stats.ObjectCount) @@ -687,13 +688,17 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { restoreGC := loadConnector(ctx, t) restoreSel := getSelectorWith(test.service) - err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) require.NoError(t, err) + assert.NotNil(t, deets) status := restoreGC.AwaitStatus() assert.Equal(t, test.expectedRestoreFolders, status.FolderCount, "status.FolderCount") assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, totalItems, status.Successful, "status.Successful") + assert.Equal( + t, totalItems, len(deets.Entries), + "details entries contains same item count as total successful items restored") t.Logf("Restore complete\n") @@ -822,14 +827,18 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames ) restoreGC := loadConnector(ctx, t) - err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) require.NoError(t, err) + require.NotNil(t, deets) status := restoreGC.AwaitStatus() // Always just 1 because it's just 1 collection. assert.Equal(t, 1, status.FolderCount, "status.FolderCount") assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, totalItems, status.Successful, "status.Successful") + assert.Equal( + t, totalItems, len(deets.Entries), + "details entries contains same item count as total successful items restored") t.Logf("Restore complete\n") } diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 5c22b39b1..41033244b 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -50,6 +51,7 @@ func RestoreCollections( service graph.Service, dest control.RestoreDestination, dcs []data.Collection, + deets *details.Details, ) (*support.ConnectorOperationStatus, error) { var ( restoreMetrics support.CollectionMetrics @@ -62,7 +64,7 @@ func RestoreCollections( // Iterate through the data collections and restore the contents of each for _, dc := range dcs { - temp, canceled := restoreCollection(ctx, service, dc, dest.ContainerName, errUpdater) + temp, canceled := restoreCollection(ctx, service, dc, dest.ContainerName, deets, errUpdater) restoreMetrics.Combine(temp) @@ -89,6 +91,7 @@ func restoreCollection( service graph.Service, dc data.Collection, restoreContainerName string, + deets *details.Details, errUpdater func(string, error), ) (support.CollectionMetrics, bool) { defer trace.StartRegion(ctx, "gc:oneDrive:restoreCollection").End() @@ -139,12 +142,31 @@ func restoreCollection( metrics.TotalBytes += int64(len(copyBuffer)) - err := restoreItem(ctx, service, itemData, drivePath.driveID, restoreFolderID, copyBuffer) + err := restoreItem(ctx, + service, + itemData, + drivePath.driveID, + restoreFolderID, + copyBuffer) if err != nil { errUpdater(itemData.UUID(), err) continue } + itemPath, err := dc.FullPath().Append(itemData.UUID(), true) + if err != nil { + logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err) + errUpdater(itemData.UUID(), err) + + continue + } + + deets.Add( + itemPath.String(), + itemPath.ShortRef(), + "", + details.ItemInfo{}) + metrics.Successes++ } } @@ -196,7 +218,11 @@ func createRestoreFolders(ctx context.Context, service graph.Service, driveID st } // restoreItem will create a new item in the specified `parentFolderID` and upload the data.Stream -func restoreItem(ctx context.Context, service graph.Service, itemData data.Stream, driveID, parentFolderID string, +func restoreItem( + ctx context.Context, + service graph.Service, + itemData data.Stream, + driveID, parentFolderID string, copyBuffer []byte, ) error { defer trace.StartRegion(ctx, "gc:oneDrive:restoreItem").End() diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 480f06ba2..661b1ce50 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -209,7 +209,8 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { return err } - err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs) + // TODO: return details and print in CLI + _, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs) if err != nil { err = errors.Wrap(err, "restoring service data") opStats.writeErr = err @@ -225,7 +226,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) { return nil } -// writes the restoreOperation outcome to the modelStore. +// persists details and statistics about the restore operation. func (op *RestoreOperation) persistResults( ctx context.Context, started time.Time,