larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cetnralize restoreConfig (#3563)

Browse Source 
centralizes all restore configuration management within a restoreConfig struct.  This struct is owned by the control package, which allows it to be utilized by both CLI and SDK consumers.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #3562

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-06-12 17:33:22 -06:00 committed by GitHub
parent de589a4571
commit 960e8b79a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 186 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/cli/restore/exchange.go
View File

@ -96,13 +96,13 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
dest := control.DefaultRestoreDestination(dttm.HumanReadable)
Infof(ctx, "Restoring to folder %s", dest.ContainerName)
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadable)
Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
sel := utils.IncludeExchangeRestoreDataSelectors(opts)
utils.FilterExchangeRestoreInfoSelectors(sel, opts)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, dest)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, restoreCfg)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to initialize Exchange restore"))
}

6
src/cli/restore/onedrive.go
View File

@ -97,13 +97,13 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
dest := control.DefaultRestoreDestination(dttm.HumanReadableDriveItem)
Infof(ctx, "Restoring to folder %s", dest.ContainerName)
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, dest)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, restoreCfg)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive restore"))
}

6
src/cli/restore/sharepoint.go
View File

@ -102,13 +102,13 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
dest := control.DefaultRestoreDestination(dttm.HumanReadableDriveItem)
Infof(ctx, "Restoring to folder %s", dest.ContainerName)
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
utils.FilterSharePointRestoreInfoSelectors(sel, opts)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, dest)
ro, err := r.NewRestore(ctx, utils.BackupIDFV, sel.Selector, restoreCfg)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to initialize SharePoint restore"))
}

16
src/cmd/factory/impl/common.go
View File

@ -83,9 +83,9 @@ func generateAndRestoreItems(
items: items,
}}
dest := control.DefaultRestoreDestination(dttm.SafeForTesting)
dest.ContainerName = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
dest := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location)
dataColls, err := buildCollections(
service,
@ -163,7 +163,7 @@ type collection struct {
func buildCollections(
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
dest control.RestoreConfig,
colls []collection,
) ([]data.RestoreCollection, error) {
collections := make([]data.RestoreCollection, 0, len(colls))
@ -224,9 +224,9 @@ func generateAndRestoreDriveItems(
ctx, flush := tester.NewContext(nil)
defer flush()
dest := control.DefaultRestoreDestination(dttm.SafeForTesting)
dest.ContainerName = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.ContainerName)
dest := control.DefaultRestoreConfig(dttm.SafeForTesting)
dest.Location = destFldr
print.Infof(ctx, "Restoring to folder %s", dest.Location)
var driveID string
@ -394,7 +394,7 @@ func generateAndRestoreDriveItems(
Service: service,
Tenant: tenantID,
ResourceOwners: []string{resourceOwner},
Dest: tester.DefaultTestRestoreDestination(""),
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
_, _, collections, _, err := connector.GetCollectionsAndExpected(

8
src/internal/connector/data_collections.go
View File

@ -233,7 +233,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections(
ctx context.Context,
backupVersion int,
sels selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Bus,
@ -251,13 +251,13 @@ func (gc *GraphConnector) ConsumeRestoreCollections(
switch sels.Service {
case selectors.ServiceExchange:
status, err = exchange.RestoreCollections(ctx, gc.AC, dest, dcs, deets, errs)
status, err = exchange.RestoreCollections(ctx, gc.AC, restoreCfg, dcs, deets, errs)
case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(
ctx,
onedrive.NewRestoreHandler(gc.AC),
backupVersion,
dest,
restoreCfg,
opts,
dcs,
deets,
@ -267,7 +267,7 @@ func (gc *GraphConnector) ConsumeRestoreCollections(
ctx,
backupVersion,
gc.AC,
dest,
restoreCfg,
opts,
dcs,
deets,

2
src/internal/connector/exchange/contacts_restore_test.go
View File

@ -51,7 +51,7 @@ func (suite *ContactsRestoreIntgSuite) TestCreateContainerDestination() {
path.EmailCategory,
suite.creds.AzureTenantID,
suite.userID,
tester.DefaultTestRestoreDestination("").ContainerName,
tester.DefaultTestRestoreConfig("").Location,
[]string{"Hufflepuff"},
[]string{"Ravenclaw"})
}

2
src/internal/connector/exchange/events_restore_test.go
View File

@ -51,7 +51,7 @@ func (suite *EventsRestoreIntgSuite) TestCreateContainerDestination() {
path.EmailCategory,
suite.creds.AzureTenantID,
suite.userID,
tester.DefaultTestRestoreDestination("").ContainerName,
tester.DefaultTestRestoreConfig("").Location,
[]string{"Durmstrang"},
[]string{"Beauxbatons"})
}

2
src/internal/connector/exchange/mail_restore_test.go
View File

@ -51,7 +51,7 @@ func (suite *MailRestoreIntgSuite) TestCreateContainerDestination() {
path.EmailCategory,
suite.creds.AzureTenantID,
suite.userID,
tester.DefaultTestRestoreDestination("").ContainerName,
tester.DefaultTestRestoreConfig("").Location,
[]string{"Griffindor", "Croix"},
[]string{"Griffindor", "Felicius"})
}

32
src/internal/connector/exchange/restore_test.go
View File

@ -54,7 +54,7 @@ func (suite *RestoreIntgSuite) TestRestoreContact() {
var (
userID = tester.M365UserID(t)
folderName = tester.DefaultTestRestoreDestination("contact").ContainerName
folderName = tester.DefaultTestRestoreConfig("contact").Location
handler = newContactRestoreHandler(suite.ac)
)
@ -88,7 +88,7 @@ func (suite *RestoreIntgSuite) TestRestoreEvent() {
var (
userID = tester.M365UserID(t)
subject = tester.DefaultTestRestoreDestination("event").ContainerName
subject = tester.DefaultTestRestoreConfig("event").Location
handler = newEventRestoreHandler(suite.ac)
)
@ -154,7 +154,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageBytes("Restore Exchange Object"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailobj").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailobj").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -167,7 +167,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithDirectAttachment("Restore 1 Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailwattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailwattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -180,7 +180,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithItemAttachmentEvent("Event Item Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("eventwattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("eventwattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -193,7 +193,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithItemAttachmentMail("Mail Item Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailitemattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailitemattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -209,7 +209,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailbasicattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailbasicattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -225,7 +225,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailnestattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailnestattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -241,7 +241,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailcontactattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailcontactattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -254,7 +254,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithNestedItemAttachmentEvent("Nested Item Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("nestedattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("nestedattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -267,7 +267,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithLargeAttachment("Restore Large Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("maillargeattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("maillargeattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -280,7 +280,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithTwoAttachments("Restore 2 Attachments"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailtwoattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailtwoattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -293,7 +293,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.MessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"),
category: path.EmailCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("mailrefattch").ContainerName
folderName := tester.DefaultTestRestoreConfig("mailrefattch").Location
folder, err := handlers[path.EmailCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -306,7 +306,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.ContactBytes("Test_Omega"),
category: path.ContactsCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("contact").ContainerName
folderName := tester.DefaultTestRestoreConfig("contact").Location
folder, err := handlers[path.ContactsCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -319,7 +319,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.EventBytes("Restored Event Object"),
category: path.EventsCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("event").ContainerName
folderName := tester.DefaultTestRestoreConfig("event").Location
calendar, err := handlers[path.EventsCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))
@ -332,7 +332,7 @@ func (suite *RestoreIntgSuite) TestRestoreExchangeObject() {
bytes: exchMock.EventWithAttachment("Restored Event Attachment"),
category: path.EventsCategory,
destination: func(t *testing.T, ctx context.Context) string {
folderName := tester.DefaultTestRestoreDestination("eventobj").ContainerName
folderName := tester.DefaultTestRestoreConfig("eventobj").Location
calendar, err := handlers[path.EventsCategory].
CreateContainer(ctx, userID, folderName, "")
require.NoError(t, err, clues.ToCore(err))

2
src/internal/connector/exchange/service_iterators_test.go
View File

@ -190,7 +190,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
getter mockGetter
resolver graph.ContainerResolver
scope selectors.ExchangeScope
failFast control.FailureBehavior
failFast control.FailurePolicy
expectErr assert.ErrorAssertionFunc
expectNewColls int
expectMetadataColls int

6
src/internal/connector/exchange/service_restore.go
View File

@ -27,7 +27,7 @@ import (
func RestoreCollections(
ctx context.Context,
ac api.Client,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Bus,
@ -76,7 +76,7 @@ func RestoreCollections(
containerID, gcr, err := createDestination(
ictx,
handler,
handler.formatRestoreDestination(dest.ContainerName, dc.FullPath()),
handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()),
userID,
directoryCache[category],
isNewCache,
@ -116,7 +116,7 @@ func RestoreCollections(
support.Restore,
len(dcs),
metrics,
dest.ContainerName)
restoreCfg.Location)
return status, el.Failure()
}

2
src/internal/connector/graph_connector_helper_test.go
View File

@ -936,7 +936,7 @@ func checkCollections(
category = returned.FullPath().Category()
expectedColData = expected[returned.FullPath().String()]
folders = returned.FullPath().Elements()
rootDir = folders[len(folders)-1] == config.Dest.ContainerName
rootDir = folders[len(folders)-1] == config.RestoreCfg.Location
)
// Need to iterate through all items even if we don't expect to find a match

2
src/internal/connector/graph_connector_onedrive_test_helper.go
View File

@ -339,7 +339,7 @@ func GetCollectionsAndExpected(
config.Service,
config.Tenant,
owner,
config.Dest,
config.RestoreCfg,
testCollections,
backupVersion,
)

36
src/internal/connector/graph_connector_test.go
View File

@ -293,8 +293,8 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
defer flush()
var (
dest = tester.DefaultTestRestoreDestination("")
sel = selectors.Selector{
restoreCfg = tester.DefaultTestRestoreConfig("")
sel = selectors.Selector{
Service: selectors.ServiceUnknown,
}
)
@ -303,7 +303,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
ctx,
version.Backup,
sel,
dest,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
@ -320,7 +320,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
}
func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
dest := tester.DefaultTestRestoreDestination("")
restoreCfg := tester.DefaultTestRestoreConfig("")
table := []struct {
name string
col []data.RestoreCollection
@ -381,7 +381,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
ctx,
version.Backup,
test.sel,
dest,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
@ -413,7 +413,7 @@ func runRestore(
) {
t.Logf(
"Restoring collections to %s for resourceOwners(s) %v\n",
config.Dest.ContainerName,
config.RestoreCfg.Location,
config.ResourceOwners)
start := time.Now()
@ -424,7 +424,7 @@ func runRestore(
ctx,
backupVersion,
restoreSel,
config.Dest,
config.RestoreCfg,
config.Opts,
collections,
fault.New(true))
@ -472,7 +472,7 @@ func runBackupAndCompare(
for _, ro := range config.ResourceOwners {
expectedDests = append(expectedDests, destAndCats{
resourceOwner: ro,
dest: config.Dest.ContainerName,
dest: config.RestoreCfg.Location,
cats: cats,
})
@ -536,7 +536,7 @@ func runRestoreBackupTest(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
Dest: tester.DefaultTestRestoreDestination(""),
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, totalKopiaItems, collections, expectedData, err := GetCollectionsAndExpected(
@ -581,7 +581,7 @@ func runRestoreTestWithVersion(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
Dest: tester.DefaultTestRestoreDestination(""),
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, _, collections, _, err := GetCollectionsAndExpected(
@ -618,7 +618,7 @@ func runRestoreBackupTestVersions(
Service: test.service,
Tenant: tenant,
ResourceOwners: resourceOwners,
Dest: tester.DefaultTestRestoreDestination(""),
RestoreCfg: tester.DefaultTestRestoreConfig(""),
}
totalItems, _, collections, _, err := GetCollectionsAndExpected(
@ -993,11 +993,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
allExpectedData := map[string]map[string][]byte{}
for i, collection := range test.collections {
// Get a dest per collection so they're independent.
dest := tester.DefaultTestRestoreDestination("")
// Get a restoreCfg per collection so they're independent.
restoreCfg := tester.DefaultTestRestoreConfig("")
expectedDests = append(expectedDests, destAndCats{
resourceOwner: suite.user,
dest: dest.ContainerName,
dest: restoreCfg.Location,
cats: map[path.CategoryType]struct{}{
collection.Category: {},
},
@ -1007,7 +1007,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
test.service,
suite.connector.tenant,
suite.user,
dest,
restoreCfg,
[]ColInfo{collection},
version.Backup,
)
@ -1023,7 +1023,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
"Restoring %v/%v collections to %s\n",
i+1,
len(test.collections),
dest.ContainerName,
restoreCfg.Location,
)
restoreGC := loadConnector(ctx, t, test.resource)
@ -1031,7 +1031,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
ctx,
version.Backup,
restoreSel,
dest,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
@ -1081,7 +1081,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
ci := ConfigInfo{
Opts: control.Options{RestorePermissions: true},
// Alright to be empty, needed for OneDrive.
Dest: control.RestoreDestination{},
RestoreCfg: control.RestoreConfig{},
}
// Pull the data prior to waiting for the status as otherwise it will

12
src/internal/connector/graph_connector_test_helper.go
View File

@ -43,7 +43,7 @@ type ConfigInfo struct {
Service path.ServiceType
Tenant string
ResourceOwners []string
Dest control.RestoreDestination
RestoreCfg control.RestoreConfig
}
func mustToDataLayerPath(
@ -66,15 +66,15 @@ func mustToDataLayerPath(
// combination of the location the data was recently restored to and where the
// data was originally in the hierarchy.
func backupOutputPathFromRestore(
restoreDest control.RestoreDestination,
restoreCfg control.RestoreConfig,
inputPath path.Path,
) (path.Path, error) {
base := []string{restoreDest.ContainerName}
base := []string{restoreCfg.Location}
// OneDrive has leading information like the drive ID.
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
folders := inputPath.Folders()
base = append(append([]string{}, folders[:3]...), restoreDest.ContainerName)
base = append(append([]string{}, folders[:3]...), restoreCfg.Location)
if len(folders) > 3 {
base = append(base, folders[3:]...)
@ -117,7 +117,7 @@ func (rc mockRestoreCollection) FetchItemByName(
func collectionsForInfo(
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
) (int, int, []data.RestoreCollection, map[string]map[string][]byte, error) {
@ -142,7 +142,7 @@ func collectionsForInfo(
mc := exchMock.NewCollection(pth, pth, len(info.Items))
baseDestPath, err := backupOutputPathFromRestore(dest, pth)
baseDestPath, err := backupOutputPathFromRestore(restoreCfg, pth)
if err != nil {
return totalItems, kopiaEntries, collections, expectedData, err
}

2
src/internal/connector/mock/connector.go
View File

@ -60,7 +60,7 @@ func (gc GraphConnector) ConsumeRestoreCollections(
_ context.Context,
_ int,
_ selectors.Selector,
_ control.RestoreDestination,
_ control.RestoreConfig,
_ control.Options,
_ []data.RestoreCollection,
_ *fault.Bus,

2
src/internal/connector/onedrive/item_test.go
View File

@ -155,7 +155,7 @@ func (suite *ItemIntegrationSuite) TestItemWriter() {
root, err := suite.service.ac.Drives().GetRootFolder(ctx, test.driveID)
require.NoError(t, err, clues.ToCore(err))
newFolderName := tester.DefaultTestRestoreDestination("folder").ContainerName
newFolderName := tester.DefaultTestRestoreConfig("folder").Location
t.Logf("creating folder %s", newFolderName)
newFolder, err := rh.PostItemInContainer(

10
src/internal/connector/onedrive/restore.go
View File

@ -64,7 +64,7 @@ func RestoreCollections(
ctx context.Context,
rh RestoreHandler,
backupVersion int,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection,
deets *details.Builder,
@ -79,7 +79,7 @@ func RestoreCollections(
ctx = clues.Add(
ctx,
"backup_version", backupVersion,
"destination", dest.ContainerName)
"restore_location", restoreCfg.Location)
// Reorder collections so that the parents directories are created
// before the child directories; a requirement for permissions.
@ -97,7 +97,7 @@ func RestoreCollections(
ictx = clues.Add(
ctx,
"category", dc.FullPath().Category(),
"destination", clues.Hide(dest.ContainerName),
"destination", clues.Hide(restoreCfg.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"full_path", dc.FullPath())
)
@ -108,7 +108,7 @@ func RestoreCollections(
backupVersion,
dc,
caches,
dest.ContainerName,
restoreCfg.Location,
deets,
opts.RestorePermissions,
errs)
@ -128,7 +128,7 @@ func RestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
dest.ContainerName)
restoreCfg.Location)
return status, el.Failure()
}

2
src/internal/connector/onedrive/url_cache_test.go
View File

@ -63,7 +63,7 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
t = suite.T()
ac = suite.ac.Drives()
driveID = suite.driveID
newFolderName = tester.DefaultTestRestoreDestination("folder").ContainerName
newFolderName = tester.DefaultTestRestoreConfig("folder").Location
driveItemPager = suite.ac.Drives().NewItemPager(driveID, "", api.DriveItemSelectDefault())
)

2
src/internal/connector/sharepoint/api/pages_test.go
View File

@ -92,7 +92,7 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() {
ctx, flush := tester.NewContext(t)
defer flush()
destName := tester.DefaultTestRestoreDestination("").ContainerName
destName := tester.DefaultTestRestoreConfig("").Location
testName := "MockPage"
// Create Test Page

2
src/internal/connector/sharepoint/collection_test.go
View File

@ -208,7 +208,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
info: sharePointListInfo(listing, int64(len(byteArray))),
}
destName := tester.DefaultTestRestoreDestination("").ContainerName
destName := tester.DefaultTestRestoreConfig("").Location
deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
assert.NoError(t, err, clues.ToCore(err))

12
src/internal/connector/sharepoint/restore.go
View File

@ -43,7 +43,7 @@ func RestoreCollections(
ctx context.Context,
backupVersion int,
ac api.Client,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection,
deets *details.Builder,
@ -71,7 +71,7 @@ func RestoreCollections(
metrics support.CollectionMetrics
ictx = clues.Add(ctx,
"category", category,
"destination", clues.Hide(dest.ContainerName),
"restore_location", restoreCfg.Location,
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"full_path", dc.FullPath())
)
@ -84,7 +84,7 @@ func RestoreCollections(
backupVersion,
dc,
caches,
dest.ContainerName,
restoreCfg.Location,
deets,
opts.RestorePermissions,
errs)
@ -94,7 +94,7 @@ func RestoreCollections(
ictx,
ac.Stable,
dc,
dest.ContainerName,
restoreCfg.Location,
deets,
errs)
@ -103,7 +103,7 @@ func RestoreCollections(
ictx,
ac.Stable,
dc,
dest.ContainerName,
restoreCfg.Location,
deets,
errs)
@ -127,7 +127,7 @@ func RestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
dest.ContainerName)
restoreCfg.Location)
return status, el.Failure()
}

10
src/internal/operations/backup_integration_test.go
View File

@ -381,14 +381,14 @@ func generateContainerOfItems(
items: items,
}}
dest := control.DefaultRestoreDestination(dttm.SafeForTesting)
dest.ContainerName = destFldr
restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
restoreCfg.Location = destFldr
dataColls := buildCollections(
t,
service,
tenantID, resourceOwner,
dest,
restoreCfg,
collections)
opts := control.Defaults()
@ -398,7 +398,7 @@ func generateContainerOfItems(
ctx,
backupVersion,
sel,
dest,
restoreCfg,
opts,
dataColls,
fault.New(true))
@ -443,7 +443,7 @@ func buildCollections(
t *testing.T,
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
colls []incrementalCollection,
) []data.RestoreCollection {
t.Helper()

2
src/internal/operations/inject/inject.go
View File

@ -37,7 +37,7 @@ type (
ctx context.Context,
backupVersion int,
selector selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Bus,

34
src/internal/operations/restore.go
View File

@ -35,11 +35,11 @@ import (
type RestoreOperation struct {
operation
BackupID model.StableID `json:"backupID"`
Destination control.RestoreDestination `json:"destination"`
Results RestoreResults `json:"results"`
Selectors selectors.Selector `json:"selectors"`
Version string `json:"version"`
BackupID model.StableID
Results RestoreResults
Selectors selectors.Selector
RestoreCfg control.RestoreConfig
Version string
acct account.Account
rc inject.RestoreConsumer
@ -61,17 +61,17 @@ func NewRestoreOperation(
acct account.Account,
backupID model.StableID,
sel selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
bus events.Eventer,
) (RestoreOperation, error) {
op := RestoreOperation{
operation: newOperation(opts, bus, kw, sw),
acct: acct,
BackupID: backupID,
Destination: dest,
Selectors: sel,
Version: "v0",
rc: rc,
operation: newOperation(opts, bus, kw, sw),
acct: acct,
BackupID: backupID,
RestoreCfg: restoreCfg,
Selectors: sel,
Version: "v0",
rc: rc,
}
if err := op.validate(); err != nil {
return RestoreOperation{}, err
@ -138,7 +138,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De
"tenant_id", clues.Hide(op.acct.ID()),
"backup_id", op.BackupID,
"service", op.Selectors.Service,
"destination_container", clues.Hide(op.Destination.ContainerName))
"destination_container", clues.Hide(op.RestoreCfg.Location))
defer func() {
op.bus.Event(
@ -257,7 +257,7 @@ func (op *RestoreOperation) do(
op.rc,
bup.Version,
op.Selectors,
op.Destination,
op.RestoreCfg,
op.Options,
dcs,
op.Errors)
@ -314,7 +314,7 @@ func consumeRestoreCollections(
rc inject.RestoreConsumer,
backupVersion int,
sel selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Bus,
@ -329,7 +329,7 @@ func consumeRestoreCollections(
ctx,
backupVersion,
sel,
dest,
restoreCfg,
opts,
dcs,
errs)

46
src/internal/operations/restore_test.go
View File

@ -46,11 +46,11 @@ func TestRestoreOpSuite(t *testing.T) {
func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
var (
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &mock.GraphConnector{}
now = time.Now()
dest = tester.DefaultTestRestoreDestination("")
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &mock.GraphConnector{}
now = time.Now()
restoreCfg = tester.DefaultTestRestoreConfig("")
)
table := []struct {
@ -113,7 +113,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
account.Account{},
"foo",
selectors.Selector{DiscreteOwner: "test"},
dest,
restoreCfg,
evmock.NewBus())
require.NoError(t, err, clues.ToCore(err))
@ -215,11 +215,11 @@ func (suite *RestoreOpIntegrationSuite) TearDownSuite() {
func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
var (
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &mock.GraphConnector{}
dest = tester.DefaultTestRestoreDestination("")
opts = control.Defaults()
kw = &kopia.Wrapper{}
sw = &store.Wrapper{}
gc = &mock.GraphConnector{}
restoreCfg = tester.DefaultTestRestoreConfig("")
opts = control.Defaults()
)
table := []struct {
@ -251,7 +251,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
tester.NewM365Account(t),
"backup-id",
selectors.Selector{DiscreteOwner: "test"},
dest,
restoreCfg,
evmock.NewBus())
test.errCheck(t, err, clues.ToCore(err))
})
@ -370,14 +370,14 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
tables := []struct {
name string
owner string
dest control.RestoreDestination
restoreCfg control.RestoreConfig
getSelector func(t *testing.T, owners []string) selectors.Selector
setup func(t *testing.T, kw *kopia.Wrapper, sw *store.Wrapper, acct account.Account, owner string) bupResults
}{
{
name: "Exchange_Restore",
owner: tester.M365UserID(suite.T()),
dest: tester.DefaultTestRestoreDestination(""),
name: "Exchange_Restore",
owner: tester.M365UserID(suite.T()),
restoreCfg: tester.DefaultTestRestoreConfig(""),
getSelector: func(t *testing.T, owners []string) selectors.Selector {
rsel := selectors.NewExchangeRestore(owners)
rsel.Include(rsel.AllData())
@ -387,9 +387,9 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
setup: setupExchangeBackup,
},
{
name: "SharePoint_Restore",
owner: tester.M365SiteID(suite.T()),
dest: control.DefaultRestoreDestination(dttm.SafeForTesting),
name: "SharePoint_Restore",
owner: tester.M365SiteID(suite.T()),
restoreCfg: control.DefaultRestoreConfig(dttm.SafeForTesting),
getSelector: func(t *testing.T, owners []string) selectors.Selector {
rsel := selectors.NewSharePointRestore(owners)
rsel.Include(rsel.AllData())
@ -423,7 +423,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() {
tester.NewM365Account(t),
bup.backupID,
test.getSelector(t, bup.selectorResourceOwners),
test.dest,
test.restoreCfg,
mb)
require.NoError(t, err, clues.ToCore(err))
@ -453,8 +453,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
defer flush()
var (
dest = tester.DefaultTestRestoreDestination("")
mb = evmock.NewBus()
restoreCfg = tester.DefaultTestRestoreConfig("")
mb = evmock.NewBus()
)
rsel := selectors.NewExchangeRestore(selectors.None())
@ -475,7 +475,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
tester.NewM365Account(t),
"backupID",
rsel.Selector,
dest,
restoreCfg,
mb)
require.NoError(t, err, clues.ToCore(err))

10
src/internal/tester/restore_destination.go
View File

@ -9,10 +9,10 @@ import (
const RestoreFolderPrefix = "Corso_Test"
func DefaultTestRestoreDestination(namespace string) control.RestoreDestination {
func DefaultTestRestoreConfig(namespace string) control.RestoreConfig {
var (
dest = control.DefaultRestoreDestination(dttm.SafeForTesting)
sft = dttm.FormatNow(dttm.SafeForTesting)
restoreCfg = control.DefaultRestoreConfig(dttm.SafeForTesting)
sft = dttm.FormatNow(dttm.SafeForTesting)
)
parts := []string{RestoreFolderPrefix, namespace, sft}
@ -20,7 +20,7 @@ func DefaultTestRestoreDestination(namespace string) control.RestoreDestination
parts = []string{RestoreFolderPrefix, sft}
}
dest.ContainerName = strings.Join(parts, "_")
restoreCfg.Location = strings.Join(parts, "_")
return dest
return restoreCfg
}

26
src/pkg/control/collisionpolicy_string.go
View File

@ -1,26 +0,0 @@
// Code generated by "stringer -type=CollisionPolicy"; DO NOT EDIT.
package control
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[Unknown-0]
_ = x[Copy-1]
_ = x[Skip-2]
_ = x[Replace-3]
}
const _CollisionPolicy_name = "UnknownCopySkipReplace"
var _CollisionPolicy_index = [...]uint8{0, 7, 11, 15, 22}
func (i CollisionPolicy) String() string {
if i < 0 || i >= CollisionPolicy(len(_CollisionPolicy_index)-1) {
return "CollisionPolicy(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _CollisionPolicy_name[_CollisionPolicy_index[i]:_CollisionPolicy_index[i+1]]
}

79
src/pkg/control/options.go
View File

@ -7,9 +7,8 @@ import (
// Options holds the optional configurations for a process
type Options struct {
Collision CollisionPolicy `json:"-"`
DisableMetrics bool `json:"disableMetrics"`
FailureHandling FailureBehavior `json:"failureHandling"`
FailureHandling FailurePolicy `json:"failureHandling"`
RestorePermissions bool `json:"restorePermissions"`
SkipReduce bool `json:"skipReduce"`
ToggleFeatures Toggles `json:"toggleFeatures"`
@ -17,8 +16,6 @@ type Options struct {
Repo repository.Options `json:"repo"`
}
type FailureBehavior string
type Parallelism struct {
// sets the collection buffer size before blocking.
CollectionBuffer int
@ -26,13 +23,15 @@ type Parallelism struct {
ItemFetch int
}
type FailurePolicy string
const (
// fails and exits the run immediately
FailFast FailureBehavior = "fail-fast"
FailFast FailurePolicy = "fail-fast"
// recovers whenever possible, reports non-zero recoveries as a failure
FailAfterRecovery FailureBehavior = "fail-after-recovery"
FailAfterRecovery FailurePolicy = "fail-after-recovery"
// recovers whenever possible, does not report recovery as failure
BestEffort FailureBehavior = "best-effort"
BestEffort FailurePolicy = "best-effort"
)
// Defaults provides an Options with the default values set.
@ -48,44 +47,50 @@ func Defaults() Options {
}
// ---------------------------------------------------------------------------
// Restore Item Collision Policy
// ---------------------------------------------------------------------------
// CollisionPolicy describes how the datalayer behaves in case of a collision.
type CollisionPolicy int
//go:generate stringer -type=CollisionPolicy
const (
Unknown CollisionPolicy = iota
Copy
Skip
Replace
)
// ---------------------------------------------------------------------------
// Restore Destination
// Restore Configuration
// ---------------------------------------------------------------------------
const (
defaultRestoreLocation = "Corso_Restore_"
)
// RestoreDestination is a POD that contains an override of the resource owner
// to restore data under and the name of the root of the restored container
// hierarchy.
type RestoreDestination struct {
// ResourceOwnerOverride overrides the default resource owner to restore to.
// If it is not populated items should be restored under the previous resource
// owner of the item.
ResourceOwnerOverride string
// ContainerName is the name of the root of the restored container hierarchy.
// This field must be populated for a restore.
ContainerName string
// CollisionPolicy describes how the datalayer behaves in case of a collision.
type CollisionPolicy string
const (
Unknown CollisionPolicy = ""
Skip CollisionPolicy = "skip"
Copy CollisionPolicy = "copy"
Replace CollisionPolicy = "replace"
)
// RestoreConfig contains
type RestoreConfig struct {
// Defines the per-item collision handling policy.
// Defaults to Skip.
OnCollision CollisionPolicy
// ProtectedResource specifies which resource the data will be restored to.
// If empty, restores to the same resource that was backed up.
// Defaults to empty.
ProtectedResource string
// Location specifies the container into which the data will be restored.
// Only accepts container names, does not accept IDs.
// If empty or "/", data will get restored in place, beginning at the root.
// Defaults to "Corso_Restore_<current_dttm>"
Location string
// Drive specifies the drive into which the data will be restored.
// If empty, data is restored to the same drive that was backed up.
// Defaults to empty.
Drive string
}
func DefaultRestoreDestination(timeFormat dttm.TimeFormat) RestoreDestination {
return RestoreDestination{
ContainerName: defaultRestoreLocation + dttm.FormatNow(timeFormat),
func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig {
return RestoreConfig{
OnCollision: Skip,
Location: defaultRestoreLocation + dttm.FormatNow(timeFormat),
}
}

4
src/pkg/repository/loadtest/repository_load_test.go
View File

@ -151,9 +151,9 @@ func runRestoreLoadTest(
t.Skip("restore load test is toggled off")
}
dest := tester.DefaultTestRestoreDestination("")
restoreCfg := tester.DefaultTestRestoreConfig("")
rst, err := r.NewRestore(ctx, backupID, restSel, dest)
rst, err := r.NewRestore(ctx, backupID, restSel, restoreCfg)
require.NoError(t, err, clues.ToCore(err))
doRestoreLoadTest(t, ctx, rst, service, bup.Results.ItemsWritten, usersUnderTest)

6
src/pkg/repository/repository.go
View File

@ -69,7 +69,7 @@ type Repository interface {
ctx context.Context,
backupID string,
sel selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error)
NewMaintenance(
ctx context.Context,
@ -336,7 +336,7 @@ func (r repository) NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
dest control.RestoreDestination,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error) {
gc, err := connectToM365(ctx, sel, r.Account)
if err != nil {
@ -352,7 +352,7 @@ func (r repository) NewRestore(
r.Account,
model.StableID(backupID),
sel,
dest,
restoreCfg,
r.Bus)
}

4
src/pkg/repository/repository_test.go
View File

@ -242,7 +242,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
defer flush()
acct := tester.NewM365Account(t)
dest := tester.DefaultTestRestoreDestination("")
restoreCfg := tester.DefaultTestRestoreConfig("")
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
@ -250,7 +250,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
r, err := repository.Initialize(ctx, acct, st, control.Defaults())
require.NoError(t, err, clues.ToCore(err))
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, dest)
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg)
require.NoError(t, err, clues.ToCore(err))
require.NotNil(t, ro)
}