look up restore resource if specified (#3853)

If the restore configuration specifies a protected resource as a restore target, use that as the destination for the restore.  First step is to ensure the provided target can be retrieved and identified.

---

#### Does this PR need a docs update or release note?

- [x] 🕐 Yes, but in a later PR

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3562

#### Test Plan

- [x]  Unit test
This commit is contained in:
Keepers 2023-07-26 11:24:53 -06:00 committed by GitHub
parent 7653af5143
commit 3613cb2aa0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
65 changed files with 1365 additions and 591 deletions

View File

@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] (beta) ## [Unreleased] (beta)
### Added
- Restore commands now accept an optional resource override with the `--to-resource` flag. This allows restores to recreate backup data within different mailboxes, sites, and users.
### Fixed ### Fixed
- SharePoint document libraries deleted after the last backup can now be restored. - SharePoint document libraries deleted after the last backup can now be restored.
- Restore requires the protected resource to have access to the service being restored.
## [v0.11.1] (beta) - 2023-07-20 ## [v0.11.1] (beta) - 2023-07-20

View File

@ -47,7 +47,7 @@ func prepM365Test(
vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force) vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force)
ctx = config.SetViper(ctx, vpr) ctx = config.SetViper(ctx, vpr)
repo, err := repository.Initialize(ctx, acct, st, control.Defaults()) repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
return acct, st, repo, vpr, recorder, cfgFP return acct, st, repo, vpr, recorder, cfgFP

View File

@ -9,11 +9,13 @@ import (
const ( const (
CollisionsFN = "collisions" CollisionsFN = "collisions"
DestinationFN = "destination" DestinationFN = "destination"
ToResourceFN = "to-resource"
) )
var ( var (
CollisionsFV string CollisionsFV string
DestinationFV string DestinationFV string
ToResourceFV string
) )
// AddRestoreConfigFlags adds the restore config flag set. // AddRestoreConfigFlags adds the restore config flag set.
@ -25,5 +27,8 @@ func AddRestoreConfigFlags(cmd *cobra.Command) {
"Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace)) "Sets the behavior for existing item collisions: "+string(control.Skip)+", "+string(control.Copy)+", or "+string(control.Replace))
fs.StringVar( fs.StringVar(
&DestinationFV, DestinationFN, "", &DestinationFV, DestinationFN, "",
"Overrides the destination where items get restored; '/' places items into their original location") "Overrides the folder where items get restored; '/' places items into their original location")
fs.StringVar(
&ToResourceFV, ToResourceFN, "",
"Overrides the protected resource (mailbox, site, user, etc) where data gets restored")
} }

View File

@ -200,7 +200,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
ctx = config.SetViper(ctx, vpr) ctx = config.SetViper(ctx, vpr)
// init the repo first // init the repo first
_, err = repository.Initialize(ctx, account.Account{}, st, control.Defaults()) _, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// then test it // then test it

View File

@ -84,6 +84,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
"--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.CollisionsFN, testdata.Collisions,
"--" + flags.DestinationFN, testdata.Destination, "--" + flags.DestinationFN, testdata.Destination,
"--" + flags.ToResourceFN, testdata.ToResource,
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
@ -125,6 +126,7 @@ func (suite *ExchangeUnitSuite) TestAddExchangeCommands() {
assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions)
assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination)
assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)

View File

@ -70,6 +70,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
"--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.CollisionsFN, testdata.Collisions,
"--" + flags.DestinationFN, testdata.Destination, "--" + flags.DestinationFN, testdata.Destination,
"--" + flags.ToResourceFN, testdata.ToResource,
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
@ -80,6 +81,9 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
"--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret,
"--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase,
// bool flags
"--" + flags.RestorePermissionsFN,
}) })
cmd.SetOut(new(bytes.Buffer)) // drop output cmd.SetOut(new(bytes.Buffer)) // drop output
@ -99,6 +103,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions)
assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination)
assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
@ -109,6 +114,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() {
assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV)
assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV)
assert.True(t, flags.RestorePermissionsFV)
}) })
} }
} }

View File

@ -75,6 +75,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.CollisionsFN, testdata.Collisions, "--" + flags.CollisionsFN, testdata.Collisions,
"--" + flags.DestinationFN, testdata.Destination, "--" + flags.DestinationFN, testdata.Destination,
"--" + flags.ToResourceFN, testdata.ToResource,
"--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID,
"--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey,
@ -85,6 +86,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
"--" + flags.AzureClientSecretFN, testdata.AzureClientSecret, "--" + flags.AzureClientSecretFN, testdata.AzureClientSecret,
"--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase,
// bool flags
"--" + flags.RestorePermissionsFN,
}) })
cmd.SetOut(new(bytes.Buffer)) // drop output cmd.SetOut(new(bytes.Buffer)) // drop output
@ -111,6 +115,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions) assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions)
assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination) assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination)
assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource)
assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV)
assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV)
@ -121,6 +126,9 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() {
assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV) assert.Equal(t, testdata.AzureClientSecret, flags.AzureClientSecretFV)
assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV)
// bool flags
assert.True(t, flags.RestorePermissionsFV)
}) })
} }
} }

View File

@ -8,7 +8,7 @@ import (
// Control produces the control options based on the user's flags. // Control produces the control options based on the user's flags.
func Control() control.Options { func Control() control.Options {
opt := control.Defaults() opt := control.DefaultOptions()
if flags.FailFastFV { if flags.FailFastFV {
opt.FailureHandling = control.FailFast opt.FailureHandling = control.FailFast
@ -21,7 +21,6 @@ func Control() control.Options {
opt.DeltaPageSize = dps opt.DeltaPageSize = dps
opt.DisableMetrics = flags.NoStatsFV opt.DisableMetrics = flags.NoStatsFV
opt.RestorePermissions = flags.RestorePermissionsFV
opt.SkipReduce = flags.SkipReduceFV opt.SkipReduce = flags.SkipReduceFV
opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV
opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV

View File

@ -18,16 +18,20 @@ type RestoreCfgOpts struct {
// DTTMFormat is the timestamp format appended // DTTMFormat is the timestamp format appended
// to the default folder name. Defaults to // to the default folder name. Defaults to
// dttm.HumanReadable. // dttm.HumanReadable.
DTTMFormat dttm.TimeFormat DTTMFormat dttm.TimeFormat
ProtectedResource string
RestorePermissions bool
Populated flags.PopulatedFlags Populated flags.PopulatedFlags
} }
func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts { func makeRestoreCfgOpts(cmd *cobra.Command) RestoreCfgOpts {
return RestoreCfgOpts{ return RestoreCfgOpts{
Collisions: flags.CollisionsFV, Collisions: flags.CollisionsFV,
Destination: flags.DestinationFV, Destination: flags.DestinationFV,
DTTMFormat: dttm.HumanReadable, DTTMFormat: dttm.HumanReadable,
ProtectedResource: flags.ToResourceFV,
RestorePermissions: flags.RestorePermissionsFV,
// populated contains the list of flags that appear in the // populated contains the list of flags that appear in the
// command, according to pflags. Use this to differentiate // command, according to pflags. Use this to differentiate
@ -67,6 +71,9 @@ func MakeRestoreConfig(
restoreCfg.Location = opts.Destination restoreCfg.Location = opts.Destination
} }
restoreCfg.ProtectedResource = opts.ProtectedResource
restoreCfg.IncludePermissions = opts.RestorePermissions
Infof(ctx, "Restoring to folder %s", restoreCfg.Location) Infof(ctx, "Restoring to folder %s", restoreCfg.Location)
return restoreCfg return restoreCfg

View File

@ -68,18 +68,18 @@ func (suite *RestoreCfgUnitSuite) TestValidateRestoreConfigFlags() {
} }
func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() { func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
rco := &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
}
table := []struct { table := []struct {
name string name string
rco *RestoreCfgOpts
populated flags.PopulatedFlags populated flags.PopulatedFlags
expect control.RestoreConfig expect control.RestoreConfig
}{ }{
{ {
name: "not populated", name: "not populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{}, populated: flags.PopulatedFlags{},
expect: control.RestoreConfig{ expect: control.RestoreConfig{
OnCollision: control.Skip, OnCollision: control.Skip,
@ -88,6 +88,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
}, },
{ {
name: "collision populated", name: "collision populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{ populated: flags.PopulatedFlags{
flags.CollisionsFN: {}, flags.CollisionsFN: {},
}, },
@ -98,6 +102,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
}, },
{ {
name: "destination populated", name: "destination populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{ populated: flags.PopulatedFlags{
flags.DestinationFN: {}, flags.DestinationFN: {},
}, },
@ -108,6 +116,10 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
}, },
{ {
name: "both populated", name: "both populated",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
},
populated: flags.PopulatedFlags{ populated: flags.PopulatedFlags{
flags.CollisionsFN: {}, flags.CollisionsFN: {},
flags.DestinationFN: {}, flags.DestinationFN: {},
@ -117,6 +129,23 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
Location: "destination", Location: "destination",
}, },
}, },
{
name: "with restore permissions",
rco: &RestoreCfgOpts{
Collisions: "collisions",
Destination: "destination",
RestorePermissions: true,
},
populated: flags.PopulatedFlags{
flags.CollisionsFN: {},
flags.DestinationFN: {},
},
expect: control.RestoreConfig{
OnCollision: control.CollisionPolicy("collisions"),
Location: "destination",
IncludePermissions: true,
},
},
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
@ -125,12 +154,13 @@ func (suite *RestoreCfgUnitSuite) TestMakeRestoreConfig() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
opts := *rco opts := *test.rco
opts.Populated = test.populated opts.Populated = test.populated
result := MakeRestoreConfig(ctx, opts) result := MakeRestoreConfig(ctx, opts)
assert.Equal(t, test.expect.OnCollision, result.OnCollision) assert.Equal(t, test.expect.OnCollision, result.OnCollision)
assert.Contains(t, result.Location, test.expect.Location) assert.Contains(t, result.Location, test.expect.Location)
assert.Equal(t, test.expect.IncludePermissions, result.IncludePermissions)
}) })
} }
} }

View File

@ -46,6 +46,7 @@ var (
Collisions = "collisions" Collisions = "collisions"
Destination = "destination" Destination = "destination"
ToResource = "toResource"
RestorePermissions = true RestorePermissions = true
DeltaPageSize = "deltaPageSize" DeltaPageSize = "deltaPageSize"

View File

@ -21,12 +21,12 @@ import (
odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub" odStub "github.com/alcionai/corso/src/internal/m365/onedrive/stub"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
m365Stub "github.com/alcionai/corso/src/internal/m365/stub" m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
@ -104,7 +104,15 @@ func generateAndRestoreItems(
print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination) print.Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, Destination)
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, dataColls, errs, ctr) rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
return ctrl.ConsumeRestoreCollections(ctx, rcc, dataColls, errs, ctr)
} }
// ------------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------------
@ -144,7 +152,7 @@ func getControllerAndVerifyResourceOwner(
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api") return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
} }
id, _, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, resourceOwner, nil) id, _, err := ctrl.PopulateProtectedResourceIDAndName(ctx, resourceOwner, nil)
if err != nil { if err != nil {
return nil, account.Account{}, nil, clues.Wrap(err, "verifying user") return nil, account.Account{}, nil, clues.Wrap(err, "verifying user")
} }
@ -407,10 +415,8 @@ func generateAndRestoreDriveItems(
// input, // input,
// version.Backup) // version.Backup)
opts := control.Options{ opts := control.DefaultOptions()
RestorePermissions: true, restoreCfg.IncludePermissions = true
ToggleFeatures: control.Toggles{},
}
config := m365Stub.ConfigInfo{ config := m365Stub.ConfigInfo{
Opts: opts, Opts: opts,
@ -418,7 +424,7 @@ func generateAndRestoreDriveItems(
Service: service, Service: service,
Tenant: tenantID, Tenant: tenantID,
ResourceOwners: []string{resourceOwner}, ResourceOwners: []string{resourceOwner},
RestoreCfg: testdata.DefaultRestoreConfig(""), RestoreCfg: restoreCfg,
} }
_, _, collections, _, err := m365Stub.GetCollectionsAndExpected( _, _, collections, _, err := m365Stub.GetCollectionsAndExpected(
@ -429,5 +435,13 @@ func generateAndRestoreDriveItems(
return nil, err return nil, err
} }
return ctrl.ConsumeRestoreCollections(ctx, version.Backup, sel, restoreCfg, opts, collections, errs, ctr) rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
return ctrl.ConsumeRestoreCollections(ctx, rcc, collections, errs, ctr)
} }

View File

@ -72,7 +72,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
subject, body, body, subject, body, body,
now, now, now, now) now, now, now, now)
}, },
control.Defaults(), control.DefaultOptions(),
errs, errs,
count.New()) count.New())
if err != nil { if err != nil {
@ -121,7 +121,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
exchMock.NoAttachments, exchMock.NoCancelledOccurrences, exchMock.NoAttachments, exchMock.NoCancelledOccurrences,
exchMock.NoExceptionOccurrences) exchMock.NoExceptionOccurrences)
}, },
control.Defaults(), control.DefaultOptions(),
errs, errs,
count.New()) count.New())
if err != nil { if err != nil {
@ -172,7 +172,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
"123-456-7890", "123-456-7890",
) )
}, },
control.Defaults(), control.DefaultOptions(),
errs, errs,
count.New()) count.New())
if err != nil { if err != nil {

View File

@ -28,6 +28,10 @@ type is struct {
name string name string
} }
func NewProvider(id, name string) *is {
return &is{id, name}
}
func (is is) ID() string { return is.id } func (is is) ID() string { return is.id }
func (is is) Name() string { return is.name } func (is is) Name() string { return is.name }

View File

@ -0,0 +1,60 @@
package idname
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type IDNameUnitSuite struct {
tester.Suite
}
func TestIDNameUnitSuite(t *testing.T) {
suite.Run(t, &IDNameUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *IDNameUnitSuite) TestAdd() {
table := []struct {
name string
inID string
inName string
searchID string
searchName string
}{
{
name: "basic",
inID: "foo",
inName: "bar",
searchID: "foo",
searchName: "bar",
},
{
name: "change casing",
inID: "FNORDS",
inName: "SMARF",
searchID: "fnords",
searchName: "smarf",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
cache := NewCache(nil)
cache.Add(test.inID, test.inName)
id, found := cache.IDOf(test.searchName)
assert.True(t, found)
assert.Equal(t, test.inID, id)
name, found := cache.NameOf(test.searchID)
assert.True(t, found)
assert.Equal(t, test.inName, name)
})
}
}

View File

@ -52,7 +52,7 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
) )
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
b, err := events.NewBus(ctx, s, a.ID(), control.Defaults()) b, err := events.NewBus(ctx, s, a.ID(), control.DefaultOptions())
require.NotEmpty(t, b) require.NotEmpty(t, b)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))

View File

@ -2,7 +2,6 @@ package m365
import ( import (
"context" "context"
"strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -44,7 +43,7 @@ func (ctrl *Controller) ProduceBackupCollections(
ctx, end := diagnostics.Span( ctx, end := diagnostics.Span(
ctx, ctx,
"m365:produceBackupCollections", "m365:produceBackupCollections",
diagnostics.Index("service", sels.Service.String())) diagnostics.Index("service", sels.PathService().String()))
defer end() defer end()
ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()})
@ -61,8 +60,8 @@ func (ctrl *Controller) ProduceBackupCollections(
serviceEnabled, canMakeDeltaQueries, err := checkServiceEnabled( serviceEnabled, canMakeDeltaQueries, err := checkServiceEnabled(
ctx, ctx,
ctrl.AC.Users(), ctrl.AC.Users(),
path.ServiceType(sels.Service), sels.PathService(),
sels.DiscreteOwner) owner.ID())
if err != nil { if err != nil {
return nil, nil, false, err return nil, nil, false, err
} }
@ -194,10 +193,8 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error {
ids = siteIDs ids = siteIDs
} }
resourceOwner := strings.ToLower(sels.DiscreteOwner) if !filters.Contains(ids).Compare(sels.ID()) {
return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner)
if !filters.Equal(ids).Compare(resourceOwner) {
return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_resource_owner", sels.DiscreteOwner)
} }
return nil return nil

View File

@ -57,7 +57,7 @@ func (suite *DataCollectionIntgSuite) SetupSuite() {
suite.tenantID = creds.AzureTenantID suite.tenantID = creds.AzureTenantID
suite.ac, err = api.NewClient(creds, control.Defaults()) suite.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
} }
@ -120,7 +120,7 @@ func (suite *DataCollectionIntgSuite) TestExchangeDataCollection() {
sel := test.getSelector(t) sel := test.getSelector(t)
uidn := inMock.NewProvider(sel.ID(), sel.Name()) uidn := inMock.NewProvider(sel.ID(), sel.Name())
ctrlOpts := control.Defaults() ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries ctrlOpts.ToggleFeatures.DisableDelta = !canMakeDeltaQueries
collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections( collections, excludes, canUsePreviousBackup, err := exchange.ProduceBackupCollections(
@ -239,7 +239,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner()
test.getSelector(t), test.getSelector(t),
nil, nil,
version.NoBackup, version.NoBackup,
control.Defaults(), control.DefaultOptions(),
fault.New(true)) fault.New(true))
assert.Error(t, err, clues.ToCore(err)) assert.Error(t, err, clues.ToCore(err))
assert.False(t, canUsePreviousBackup, "can use previous backup") assert.False(t, canUsePreviousBackup, "can use previous backup")
@ -296,7 +296,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
nil, nil,
ctrl.credentials, ctrl.credentials,
ctrl, ctrl,
control.Defaults(), control.DefaultOptions(),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup") assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -367,7 +367,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
siteIDs = []string{siteID} siteIDs = []string{siteID}
) )
id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
sel := selectors.NewSharePointBackup(siteIDs) sel := selectors.NewSharePointBackup(siteIDs)
@ -381,7 +381,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
sel.Selector, sel.Selector,
nil, nil,
version.NoBackup, version.NoBackup,
control.Defaults(), control.DefaultOptions(),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup") assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -414,7 +414,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
siteIDs = []string{siteID} siteIDs = []string{siteID}
) )
id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, siteID, nil) id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, siteID, nil)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
sel := selectors.NewSharePointBackup(siteIDs) sel := selectors.NewSharePointBackup(siteIDs)
@ -428,7 +428,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
sel.Selector, sel.Selector,
nil, nil,
version.NoBackup, version.NoBackup,
control.Defaults(), control.DefaultOptions(),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup") assert.True(t, canUsePreviousBackup, "can use previous backup")

View File

@ -84,10 +84,11 @@ func NewController(
AC: ac, AC: ac,
IDNameLookup: idname.NewCache(nil), IDNameLookup: idname.NewCache(nil),
credentials: creds, credentials: creds,
ownerLookup: rCli, ownerLookup: rCli,
tenant: acct.ID(), tenant: acct.ID(),
wg: &sync.WaitGroup{}, wg: &sync.WaitGroup{},
backupDriveIDNames: idname.NewCache(nil),
} }
return &ctrl, nil return &ctrl, nil
@ -150,10 +151,6 @@ func (ctrl *Controller) incrementAwaitingMessages() {
} }
func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) {
if ctrl.backupDriveIDNames == nil {
ctrl.backupDriveIDNames = idname.NewCache(map[string]string{})
}
if dii.SharePoint != nil { if dii.SharePoint != nil {
ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName)
} }
@ -249,15 +246,15 @@ func (r resourceClient) getOwnerIDAndNameFrom(
return id, name, nil return id, name, nil
} }
// PopulateOwnerIDAndNamesFrom takes the provided owner identifier and produces // PopulateProtectedResourceIDAndName takes the provided owner identifier and produces
// the owner's name and ID from that value. Returns an error if the owner is // the owner's name and ID from that value. Returns an error if the owner is
// not recognized by the current tenant. // not recognized by the current tenant.
// //
// The id-name swapper is optional. Some processes will look up all owners in // The id-name cacher is optional. Some processes will look up all owners in
// the tenant before reaching this step. In that case, the data gets handed // the tenant before reaching this step. In that case, the data gets handed
// down for this func to consume instead of performing further queries. The // down for this func to consume instead of performing further queries. The
// data gets stored inside the controller instance for later re-use. // data gets stored inside the controller instance for later re-use.
func (ctrl *Controller) PopulateOwnerIDAndNamesFrom( func (ctrl *Controller) PopulateProtectedResourceIDAndName(
ctx context.Context, ctx context.Context,
owner string, // input value, can be either id or name owner string, // input value, can be either id or name
ins idname.Cacher, ins idname.Cacher,

View File

@ -12,15 +12,18 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
inMock "github.com/alcionai/corso/src/internal/common/idname/mock" inMock "github.com/alcionai/corso/src/internal/common/idname/mock"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock" dataMock "github.com/alcionai/corso/src/internal/data/mock"
exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/m365/stub" "github.com/alcionai/corso/src/internal/m365/stub"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
@ -223,7 +226,7 @@ func (suite *ControllerUnitSuite) TestPopulateOwnerIDAndNamesFrom() {
ctrl := &Controller{ownerLookup: test.rc} ctrl := &Controller{ownerLookup: test.rc}
rID, rName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, test.owner, test.ins) rID, rName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, test.owner, test.ins)
test.expectErr(t, err, clues.ToCore(err)) test.expectErr(t, err, clues.ToCore(err))
assert.Equal(t, test.expectID, rID, "id") assert.Equal(t, test.expectID, rID, "id")
assert.Equal(t, test.expectName, rName, "name") assert.Equal(t, test.expectName, rName, "name")
@ -385,20 +388,24 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() {
} }
) )
restoreCfg.IncludePermissions = true
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
deets, err := suite.ctrl.ConsumeRestoreCollections( deets, err := suite.ctrl.ConsumeRestoreCollections(
ctx, ctx,
version.Backup, rcc,
sel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
[]data.RestoreCollection{&dataMock.Collection{}}, []data.RestoreCollection{&dataMock.Collection{}},
fault.New(true), fault.New(true),
count.New()) count.New())
assert.Error(t, err, clues.ToCore(err)) assert.Error(t, err, graph.ErrServiceNotEnabled, clues.ToCore(err))
assert.NotNil(t, deets) assert.Nil(t, deets)
status := suite.ctrl.Wait() status := suite.ctrl.Wait()
assert.Equal(t, 0, status.Objects) assert.Equal(t, 0, status.Objects)
@ -408,6 +415,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreFailsBadService() {
func (suite *ControllerIntegrationSuite) TestEmptyCollections() { func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
restoreCfg := testdata.DefaultRestoreConfig("") restoreCfg := testdata.DefaultRestoreConfig("")
restoreCfg.IncludePermissions = true
table := []struct { table := []struct {
name string name string
col []data.RestoreCollection col []data.RestoreCollection
@ -464,15 +473,17 @@ func (suite *ControllerIntegrationSuite) TestEmptyCollections() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: test.sel,
RestoreConfig: restoreCfg,
Selector: test.sel,
}
deets, err := suite.ctrl.ConsumeRestoreCollections( deets, err := suite.ctrl.ConsumeRestoreCollections(
ctx, ctx,
version.Backup, rcc,
test.sel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
test.col, test.col,
fault.New(true), fault.New(true),
count.New()) count.New())
@ -503,12 +514,18 @@ func runRestore(
restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService) restoreCtrl := newController(ctx, t, sci.Resource, path.ExchangeService)
restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true) restoreSel := getSelectorWith(t, sci.Service, sci.ResourceOwners, true)
rcc := inject.RestoreConsumerConfig{
BackupVersion: backupVersion,
Options: control.DefaultOptions(),
ProtectedResource: restoreSel,
RestoreConfig: sci.RestoreCfg,
Selector: restoreSel,
}
deets, err := restoreCtrl.ConsumeRestoreCollections( deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx, ctx,
backupVersion, rcc,
restoreSel,
sci.RestoreCfg,
sci.Opts,
collections, collections,
fault.New(true), fault.New(true),
count.New()) count.New())
@ -610,6 +627,7 @@ func runRestoreBackupTest(
tenant string, tenant string,
resourceOwners []string, resourceOwners []string,
opts control.Options, opts control.Options,
restoreCfg control.RestoreConfig,
) { ) {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -620,7 +638,7 @@ func runRestoreBackupTest(
Service: test.service, Service: test.service,
Tenant: tenant, Tenant: tenant,
ResourceOwners: resourceOwners, ResourceOwners: resourceOwners,
RestoreCfg: testdata.DefaultRestoreConfig(""), RestoreCfg: restoreCfg,
} }
totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected( totalItems, totalKopiaItems, collections, expectedData, err := stub.GetCollectionsAndExpected(
@ -655,6 +673,7 @@ func runRestoreTestWithVersion(
tenant string, tenant string,
resourceOwners []string, resourceOwners []string,
opts control.Options, opts control.Options,
restoreCfg control.RestoreConfig,
) { ) {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -665,7 +684,7 @@ func runRestoreTestWithVersion(
Service: test.service, Service: test.service,
Tenant: tenant, Tenant: tenant,
ResourceOwners: resourceOwners, ResourceOwners: resourceOwners,
RestoreCfg: testdata.DefaultRestoreConfig(""), RestoreCfg: restoreCfg,
} }
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
@ -692,7 +711,7 @@ func runRestoreBackupTestVersions(
tenant string, tenant string,
resourceOwners []string, resourceOwners []string,
opts control.Options, opts control.Options,
crc control.RestoreConfig, restoreCfg control.RestoreConfig,
) { ) {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
@ -703,7 +722,7 @@ func runRestoreBackupTestVersions(
Service: test.service, Service: test.service,
Tenant: tenant, Tenant: tenant,
ResourceOwners: resourceOwners, ResourceOwners: resourceOwners,
RestoreCfg: crc, RestoreCfg: restoreCfg,
} }
totalItems, _, collections, _, err := stub.GetCollectionsAndExpected( totalItems, _, collections, _, err := stub.GetCollectionsAndExpected(
@ -737,7 +756,7 @@ func runRestoreBackupTestVersions(
test.collectionsLatest) test.collectionsLatest)
} }
func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() { func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_core() {
bodyText := "This email has some text. However, all the text is on the same line." bodyText := "This email has some text. However, all the text is on the same line."
subjectText := "Test message for restore" subjectText := "Test message for restore"
@ -996,10 +1015,8 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup() {
test, test,
suite.ctrl.tenant, suite.ctrl.tenant,
[]string{suite.user}, []string{suite.user},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, control.DefaultRestoreConfig(dttm.HumanReadableDriveItem))
ToggleFeatures: control.Toggles{},
})
}) })
} }
} }
@ -1080,6 +1097,8 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
for i, collection := range test.collections { for i, collection := range test.collections {
// Get a restoreCfg per collection so they're independent. // Get a restoreCfg per collection so they're independent.
restoreCfg := testdata.DefaultRestoreConfig("") restoreCfg := testdata.DefaultRestoreConfig("")
restoreCfg.IncludePermissions = true
expectedDests = append(expectedDests, destAndCats{ expectedDests = append(expectedDests, destAndCats{
resourceOwner: suite.user, resourceOwner: suite.user,
dest: restoreCfg.Location, dest: restoreCfg.Location,
@ -1112,15 +1131,18 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
) )
restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService) restoreCtrl := newController(ctx, t, test.resourceCat, path.ExchangeService)
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
ProtectedResource: restoreSel,
RestoreConfig: restoreCfg,
Selector: restoreSel,
}
deets, err := restoreCtrl.ConsumeRestoreCollections( deets, err := restoreCtrl.ConsumeRestoreCollections(
ctx, ctx,
version.Backup, rcc,
restoreSel,
restoreCfg,
control.Options{
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
collections, collections,
fault.New(true), fault.New(true),
count.New()) count.New())
@ -1152,10 +1174,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
backupSel, backupSel,
nil, nil,
version.NoBackup, version.NoBackup,
control.Options{ control.DefaultOptions(),
RestorePermissions: true,
ToggleFeatures: control.Toggles{},
},
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup") assert.True(t, canUsePreviousBackup, "can use previous backup")
@ -1164,10 +1183,13 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
t.Log("Backup enumeration complete") t.Log("Backup enumeration complete")
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
ci := stub.ConfigInfo{ ci := stub.ConfigInfo{
Opts: control.Options{RestorePermissions: true}, Opts: control.DefaultOptions(),
// Alright to be empty, needed for OneDrive. // Alright to be empty, needed for OneDrive.
RestoreCfg: control.RestoreConfig{}, RestoreCfg: restoreCfg,
} }
// Pull the data prior to waiting for the status as otherwise it will // Pull the data prior to waiting for the status as otherwise it will
@ -1205,16 +1227,16 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
}, },
} }
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreBackupTest( runRestoreBackupTest(
suite.T(), suite.T(),
test, test,
suite.ctrl.tenant, suite.ctrl.tenant,
[]string{suite.user}, []string{suite.user},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
)
} }
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
@ -1233,8 +1255,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
sel.Include( sel.Include(
sel.ContactFolders([]string{selectors.NoneTgt}), sel.ContactFolders([]string{selectors.NoneTgt}),
sel.EventCalendars([]string{selectors.NoneTgt}), sel.EventCalendars([]string{selectors.NoneTgt}),
sel.MailFolders([]string{selectors.NoneTgt}), sel.MailFolders([]string{selectors.NoneTgt}))
)
return sel.Selector return sel.Selector
}, },
@ -1297,23 +1318,20 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
start = time.Now() start = time.Now()
) )
id, name, err := backupCtrl.PopulateOwnerIDAndNamesFrom(ctx, backupSel.DiscreteOwner, nil) id, name, err := backupCtrl.PopulateProtectedResourceIDAndName(ctx, backupSel.DiscreteOwner, nil)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
backupSel.SetDiscreteOwnerIDName(id, name) backupSel.SetDiscreteOwnerIDName(id, name)
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections( dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
ctx, ctx,
inMock.NewProvider(id, name), idname.NewProvider(id, name),
backupSel, backupSel,
nil, nil,
version.NoBackup, version.NoBackup,
control.Options{ control.DefaultOptions(),
RestorePermissions: false,
ToggleFeatures: control.Toggles{},
},
fault.New(true)) fault.New(true))
require.NoError(t, err) require.NoError(t, err, clues.ToCore(err))
assert.True(t, canUsePreviousBackup, "can use previous backup") assert.True(t, canUsePreviousBackup, "can use previous backup")
// No excludes yet because this isn't an incremental backup. // No excludes yet because this isn't an incremental backup.
assert.True(t, excludes.Empty()) assert.True(t, excludes.Empty())

View File

@ -414,7 +414,7 @@ func (suite *BackupIntgSuite) SetupSuite() {
creds, err := acct.M365Config() creds, err := acct.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.ac, err = api.NewClient(creds, control.Defaults()) suite.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.tenantID = creds.AzureTenantID suite.tenantID = creds.AzureTenantID
@ -466,7 +466,7 @@ func (suite *BackupIntgSuite) TestMailFetch() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ctrlOpts := control.Defaults() ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries ctrlOpts.ToggleFeatures.DisableDelta = !test.canMakeDeltaQueries
collections, err := createCollections( collections, err := createCollections(
@ -554,7 +554,7 @@ func (suite *BackupIntgSuite) TestDelta() {
inMock.NewProvider(userID, userID), inMock.NewProvider(userID, userID),
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Defaults(), control.DefaultOptions(),
func(status *support.ControllerOperationStatus) {}, func(status *support.ControllerOperationStatus) {},
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -587,7 +587,7 @@ func (suite *BackupIntgSuite) TestDelta() {
inMock.NewProvider(userID, userID), inMock.NewProvider(userID, userID),
test.scope, test.scope,
dps, dps,
control.Defaults(), control.DefaultOptions(),
func(status *support.ControllerOperationStatus) {}, func(status *support.ControllerOperationStatus) {},
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -633,7 +633,7 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() {
inMock.NewProvider(suite.user, suite.user), inMock.NewProvider(suite.user, suite.user),
sel.Scopes()[0], sel.Scopes()[0],
DeltaPaths{}, DeltaPaths{},
control.Defaults(), control.DefaultOptions(),
newStatusUpdater(t, &wg), newStatusUpdater(t, &wg),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -709,7 +709,7 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() {
inMock.NewProvider(suite.user, suite.user), inMock.NewProvider(suite.user, suite.user),
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Defaults(), control.DefaultOptions(),
newStatusUpdater(t, &wg), newStatusUpdater(t, &wg),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -834,7 +834,7 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() {
inMock.NewProvider(suite.user, suite.user), inMock.NewProvider(suite.user, suite.user),
test.scope, test.scope,
DeltaPaths{}, DeltaPaths{},
control.Defaults(), control.DefaultOptions(),
newStatusUpdater(t, &wg), newStatusUpdater(t, &wg),
fault.New(true)) fault.New(true))
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -1995,7 +1995,7 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ctrlOpts := control.Defaults() ctrlOpts := control.DefaultOptions()
ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter ctrlOpts.ToggleFeatures.DisableDelta = !deltaAfter
getter := test.getter getter := test.getter

View File

@ -178,7 +178,7 @@ func (suite *CollectionSuite) TestNewCollection_state() {
test.curr, test.prev, test.loc, test.curr, test.prev, test.loc,
0, 0,
&mockItemer{}, nil, &mockItemer{}, nil,
control.Defaults(), control.DefaultOptions(),
false) false)
assert.Equal(t, test.expect, c.State(), "collection state") assert.Equal(t, test.expect, c.State(), "collection state")
assert.Equal(t, test.curr, c.fullPath, "full path") assert.Equal(t, test.curr, c.fullPath, "full path")

View File

@ -699,7 +699,7 @@ func (suite *ContainerResolverSuite) SetupSuite() {
} }
func (suite *ContainerResolverSuite) TestPopulate() { func (suite *ContainerResolverSuite) TestPopulate() {
ac, err := api.NewClient(suite.credentials, control.Defaults()) ac, err := api.NewClient(suite.credentials, control.DefaultOptions())
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
eventFunc := func(t *testing.T) graph.ContainerResolver { eventFunc := func(t *testing.T) graph.ContainerResolver {

View File

@ -31,7 +31,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
its.creds = creds its.creds = creds
its.ac, err = api.NewClient(creds, control.Defaults()) its.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
its.userID = tconfig.GetM365UserID(ctx) its.userID = tconfig.GetM365UserID(ctx)

View File

@ -84,7 +84,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
ac, err := api.NewClient(suite.credentials, control.Defaults()) ac, err := api.NewClient(suite.credentials, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
acm := ac.Mail() acm := ac.Mail()

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/count"
@ -28,7 +29,7 @@ import (
func ConsumeRestoreCollections( func ConsumeRestoreCollections(
ctx context.Context, ctx context.Context,
ac api.Client, ac api.Client,
restoreCfg control.RestoreConfig, rcc inject.RestoreConsumerConfig,
dcs []data.RestoreCollection, dcs []data.RestoreCollection,
deets *details.Builder, deets *details.Builder,
errs *fault.Bus, errs *fault.Bus,
@ -39,16 +40,13 @@ func ConsumeRestoreCollections(
} }
var ( var (
userID = dcs[0].FullPath().ResourceOwner() resourceID = rcc.ProtectedResource.ID()
directoryCache = make(map[path.CategoryType]graph.ContainerResolver) directoryCache = make(map[path.CategoryType]graph.ContainerResolver)
handlers = restoreHandlers(ac) handlers = restoreHandlers(ac)
metrics support.CollectionMetrics metrics support.CollectionMetrics
el = errs.Local() el = errs.Local()
) )
// FIXME: should be user name
ctx = clues.Add(ctx, "resource_owner", clues.Hide(userID))
for _, dc := range dcs { for _, dc := range dcs {
if el.Failure() != nil { if el.Failure() != nil {
break break
@ -69,7 +67,7 @@ func ConsumeRestoreCollections(
} }
if directoryCache[category] == nil { if directoryCache[category] == nil {
gcr := handler.newContainerCache(userID) gcr := handler.newContainerCache(resourceID)
if err := gcr.Populate(ctx, errs, handler.defaultRootContainer()); err != nil { if err := gcr.Populate(ctx, errs, handler.defaultRootContainer()); err != nil {
return nil, clues.Wrap(err, "populating container cache") return nil, clues.Wrap(err, "populating container cache")
} }
@ -80,8 +78,8 @@ func ConsumeRestoreCollections(
containerID, gcc, err := createDestination( containerID, gcc, err := createDestination(
ictx, ictx,
handler, handler,
handler.formatRestoreDestination(restoreCfg.Location, dc.FullPath()), handler.formatRestoreDestination(rcc.RestoreConfig.Location, dc.FullPath()),
userID, resourceID,
directoryCache[category], directoryCache[category],
errs) errs)
if err != nil { if err != nil {
@ -92,7 +90,7 @@ func ConsumeRestoreCollections(
directoryCache[category] = gcc directoryCache[category] = gcc
ictx = clues.Add(ictx, "restore_destination_id", containerID) ictx = clues.Add(ictx, "restore_destination_id", containerID)
collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, userID, containerID) collisionKeyToItemID, err := handler.getItemsInContainerByCollisionKey(ctx, resourceID, containerID)
if err != nil { if err != nil {
el.AddRecoverable(ctx, clues.Wrap(err, "building item collision cache")) el.AddRecoverable(ctx, clues.Wrap(err, "building item collision cache"))
continue continue
@ -102,10 +100,10 @@ func ConsumeRestoreCollections(
ictx, ictx,
handler, handler,
dc, dc,
userID, resourceID,
containerID, containerID,
collisionKeyToItemID, collisionKeyToItemID,
restoreCfg.OnCollision, rcc.RestoreConfig.OnCollision,
deets, deets,
errs, errs,
ctr) ctr)
@ -126,7 +124,7 @@ func ConsumeRestoreCollections(
support.Restore, support.Restore,
len(dcs), len(dcs),
metrics, metrics,
restoreCfg.Location) rcc.RestoreConfig.Location)
return status, el.Failure() return status, el.Failure()
} }
@ -136,7 +134,7 @@ func restoreCollection(
ctx context.Context, ctx context.Context,
ir itemRestorer, ir itemRestorer,
dc data.RestoreCollection, dc data.RestoreCollection,
userID, destinationID string, resourceID, destinationID string,
collisionKeyToItemID map[string]string, collisionKeyToItemID map[string]string,
collisionPolicy control.CollisionPolicy, collisionPolicy control.CollisionPolicy,
deets *details.Builder, deets *details.Builder,
@ -187,7 +185,7 @@ func restoreCollection(
info, err := ir.restore( info, err := ir.restore(
ictx, ictx,
body, body,
userID, resourceID,
destinationID, destinationID,
collisionKeyToItemID, collisionKeyToItemID,
collisionPolicy, collisionPolicy,
@ -240,7 +238,7 @@ func createDestination(
ctx context.Context, ctx context.Context,
ca containerAPI, ca containerAPI,
destination *path.Builder, destination *path.Builder,
userID string, resourceID string,
gcr graph.ContainerResolver, gcr graph.ContainerResolver,
errs *fault.Bus, errs *fault.Bus,
) (string, graph.ContainerResolver, error) { ) (string, graph.ContainerResolver, error) {
@ -264,7 +262,7 @@ func createDestination(
ca, ca,
cache, cache,
restoreLoc, restoreLoc,
userID, resourceID,
containerParentID, containerParentID,
container, container,
errs) errs)
@ -285,7 +283,7 @@ func getOrPopulateContainer(
ca containerAPI, ca containerAPI,
gcr graph.ContainerResolver, gcr graph.ContainerResolver,
restoreLoc *path.Builder, restoreLoc *path.Builder,
userID, containerParentID, containerName string, resourceID, containerParentID, containerName string,
errs *fault.Bus, errs *fault.Bus,
) (string, error) { ) (string, error) {
cached, ok := gcr.LocationInCache(restoreLoc.String()) cached, ok := gcr.LocationInCache(restoreLoc.String())
@ -293,7 +291,7 @@ func getOrPopulateContainer(
return cached, nil return cached, nil
} }
c, err := ca.CreateContainer(ctx, userID, containerParentID, containerName) c, err := ca.CreateContainer(ctx, resourceID, containerParentID, containerName)
// 409 handling case: // 409 handling case:
// attempt to fetch the container by name and add that result to the cache. // attempt to fetch the container by name and add that result to the cache.
@ -301,7 +299,7 @@ func getOrPopulateContainer(
// sometimes the backend will create the folder despite the 5xx response, // sometimes the backend will create the folder despite the 5xx response,
// leaving our local containerResolver with inconsistent state. // leaving our local containerResolver with inconsistent state.
if graph.IsErrFolderExists(err) { if graph.IsErrFolderExists(err) {
cc, e := ca.GetContainerByName(ctx, userID, containerParentID, containerName) cc, e := ca.GetContainerByName(ctx, resourceID, containerParentID, containerName)
if e != nil { if e != nil {
err = clues.Stack(err, e) err = clues.Stack(err, e)
} else { } else {
@ -327,7 +325,7 @@ func uploadAttachments(
ctx context.Context, ctx context.Context,
ap attachmentPoster, ap attachmentPoster,
as []models.Attachmentable, as []models.Attachmentable,
userID, destinationID, itemID string, resourceID, destinationID, itemID string,
errs *fault.Bus, errs *fault.Bus,
) error { ) error {
el := errs.Local() el := errs.Local()
@ -340,7 +338,7 @@ func uploadAttachments(
err := uploadAttachment( err := uploadAttachment(
ctx, ctx,
ap, ap,
userID, resourceID,
destinationID, destinationID,
itemID, itemID,
a) a)

View File

@ -44,7 +44,7 @@ func (suite *RestoreIntgSuite) SetupSuite() {
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.credentials = m365 suite.credentials = m365
suite.ac, err = api.NewClient(m365, control.Defaults()) suite.ac, err = api.NewClient(m365, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
} }

View File

@ -796,8 +796,8 @@ func compareDriveItem(
assert.Equal(t, expectedMeta.FileName, itemMeta.FileName) assert.Equal(t, expectedMeta.FileName, itemMeta.FileName)
} }
if !mci.Opts.RestorePermissions { if !mci.RestoreCfg.IncludePermissions {
assert.Equal(t, 0, len(itemMeta.Permissions)) assert.Empty(t, itemMeta.Permissions, "no permissions should be included in restore")
return true return true
} }

View File

@ -27,6 +27,10 @@ type Controller struct {
Err error Err error
Stats data.CollectionStats Stats data.CollectionStats
ProtectedResourceID string
ProtectedResourceName string
ProtectedResourceErr error
} }
func (ctrl Controller) ProduceBackupCollections( func (ctrl Controller) ProduceBackupCollections(
@ -60,10 +64,7 @@ func (ctrl Controller) Wait() *data.CollectionStats {
func (ctrl Controller) ConsumeRestoreCollections( func (ctrl Controller) ConsumeRestoreCollections(
_ context.Context, _ context.Context,
_ int, _ inject.RestoreConsumerConfig,
_ selectors.Selector,
_ control.RestoreConfig,
_ control.Options,
_ []data.RestoreCollection, _ []data.RestoreCollection,
_ *fault.Bus, _ *fault.Bus,
_ *count.Bus, _ *count.Bus,
@ -84,3 +85,13 @@ func (ctrl Controller) ExportRestoreCollections(
) ([]export.Collection, error) { ) ([]export.Collection, error) {
return nil, ctrl.Err return nil, ctrl.Err
} }
func (ctrl Controller) PopulateProtectedResourceIDAndName(
ctx context.Context,
protectedResource string, // input value, can be either id or name
ins idname.Cacher,
) (string, string, error) {
return ctrl.ProtectedResourceID,
ctrl.ProtectedResourceName,
ctrl.ProtectedResourceErr
}

View File

@ -945,7 +945,7 @@ func (suite *CollectionUnitTestSuite) TestItemExtensions() {
nil, nil,
} }
opts := control.Defaults() opts := control.DefaultOptions()
opts.ItemExtensionFactory = append( opts.ItemExtensionFactory = append(
opts.ItemExtensionFactory, opts.ItemExtensionFactory,
test.factories...) test.factories...)

View File

@ -313,7 +313,7 @@ func (suite *OneDriveIntgSuite) SetupSuite() {
suite.creds = creds suite.creds = creds
suite.ac, err = api.NewClient(creds, control.Defaults()) suite.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
} }

View File

@ -23,6 +23,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata" "github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
@ -38,114 +39,11 @@ const (
maxUploadRetries = 3 maxUploadRetries = 3
) )
type driveInfo struct {
id string
name string
rootFolderID string
}
type restoreCaches struct {
BackupDriveIDName idname.Cacher
collisionKeyToItemID map[string]api.DriveItemIDType
DriveIDToDriveInfo map[string]driveInfo
DriveNameToDriveInfo map[string]driveInfo
Folders *folderCache
OldLinkShareIDToNewID map[string]string
OldPermIDToNewID map[string]string
ParentDirToMeta map[string]metadata.Metadata
pool sync.Pool
}
func (rc *restoreCaches) AddDrive(
ctx context.Context,
md models.Driveable,
grf GetRootFolderer,
) error {
di := driveInfo{
id: ptr.Val(md.GetId()),
name: ptr.Val(md.GetName()),
}
ctx = clues.Add(ctx, "drive_info", di)
root, err := grf.GetRootFolder(ctx, di.id)
if err != nil {
return clues.Wrap(err, "getting drive root id")
}
di.rootFolderID = ptr.Val(root.GetId())
rc.DriveIDToDriveInfo[di.id] = di
rc.DriveNameToDriveInfo[di.name] = di
return nil
}
// Populate looks up drive items available to the protectedResource
// and adds their info to the caches.
func (rc *restoreCaches) Populate(
ctx context.Context,
gdparf GetDrivePagerAndRootFolderer,
protectedResourceID string,
) error {
drives, err := api.GetAllDrives(
ctx,
gdparf.NewDrivePager(protectedResourceID, nil),
true,
maxDrivesRetries)
if err != nil {
return clues.Wrap(err, "getting drives")
}
for _, md := range drives {
if err := rc.AddDrive(ctx, md, gdparf); err != nil {
return clues.Wrap(err, "caching drive")
}
}
return nil
}
type GetDrivePagerAndRootFolderer interface {
GetRootFolderer
NewDrivePagerer
}
func NewRestoreCaches(
backupDriveIDNames idname.Cacher,
) *restoreCaches {
// avoid nil panics
if backupDriveIDNames == nil {
backupDriveIDNames = idname.NewCache(nil)
}
return &restoreCaches{
BackupDriveIDName: backupDriveIDNames,
collisionKeyToItemID: map[string]api.DriveItemIDType{},
DriveIDToDriveInfo: map[string]driveInfo{},
DriveNameToDriveInfo: map[string]driveInfo{},
Folders: NewFolderCache(),
OldLinkShareIDToNewID: map[string]string{},
OldPermIDToNewID: map[string]string{},
ParentDirToMeta: map[string]metadata.Metadata{},
// Buffer pool for uploads
pool: sync.Pool{
New: func() any {
b := make([]byte, graph.CopyBufferSize)
return &b
},
},
}
}
// ConsumeRestoreCollections will restore the specified data collections into OneDrive // ConsumeRestoreCollections will restore the specified data collections into OneDrive
func ConsumeRestoreCollections( func ConsumeRestoreCollections(
ctx context.Context, ctx context.Context,
rh RestoreHandler, rh RestoreHandler,
backupVersion int, rcc inject.RestoreConsumerConfig,
restoreCfg control.RestoreConfig,
opts control.Options,
backupDriveIDNames idname.Cacher, backupDriveIDNames idname.Cacher,
dcs []data.RestoreCollection, dcs []data.RestoreCollection,
deets *details.Builder, deets *details.Builder,
@ -153,16 +51,15 @@ func ConsumeRestoreCollections(
ctr *count.Bus, ctr *count.Bus,
) (*support.ControllerOperationStatus, error) { ) (*support.ControllerOperationStatus, error) {
var ( var (
restoreMetrics support.CollectionMetrics restoreMetrics support.CollectionMetrics
el = errs.Local() el = errs.Local()
caches = NewRestoreCaches(backupDriveIDNames) caches = NewRestoreCaches(backupDriveIDNames)
protectedResourceID = dcs[0].FullPath().ResourceOwner() fallbackDriveName = rcc.RestoreConfig.Location
fallbackDriveName = restoreCfg.Location
) )
ctx = clues.Add(ctx, "backup_version", backupVersion) ctx = clues.Add(ctx, "backup_version", rcc.BackupVersion)
err := caches.Populate(ctx, rh, protectedResourceID) err := caches.Populate(ctx, rh, rcc.ProtectedResource.ID())
if err != nil { if err != nil {
return nil, clues.Wrap(err, "initializing restore caches") return nil, clues.Wrap(err, "initializing restore caches")
} }
@ -183,19 +80,16 @@ func ConsumeRestoreCollections(
ictx = clues.Add( ictx = clues.Add(
ctx, ctx,
"category", dc.FullPath().Category(), "category", dc.FullPath().Category(),
"resource_owner", clues.Hide(protectedResourceID),
"full_path", dc.FullPath()) "full_path", dc.FullPath())
) )
metrics, err = RestoreCollection( metrics, err = RestoreCollection(
ictx, ictx,
rh, rh,
restoreCfg, rcc,
backupVersion,
dc, dc,
caches, caches,
deets, deets,
opts.RestorePermissions,
fallbackDriveName, fallbackDriveName,
errs, errs,
ctr.Local()) ctr.Local())
@ -215,7 +109,7 @@ func ConsumeRestoreCollections(
support.Restore, support.Restore,
len(dcs), len(dcs),
restoreMetrics, restoreMetrics,
restoreCfg.Location) rcc.RestoreConfig.Location)
return status, el.Failure() return status, el.Failure()
} }
@ -228,26 +122,23 @@ func ConsumeRestoreCollections(
func RestoreCollection( func RestoreCollection(
ctx context.Context, ctx context.Context,
rh RestoreHandler, rh RestoreHandler,
restoreCfg control.RestoreConfig, rcc inject.RestoreConsumerConfig,
backupVersion int,
dc data.RestoreCollection, dc data.RestoreCollection,
caches *restoreCaches, caches *restoreCaches,
deets *details.Builder, deets *details.Builder,
restorePerms bool, // TODD: move into restoreConfig
fallbackDriveName string, fallbackDriveName string,
errs *fault.Bus, errs *fault.Bus,
ctr *count.Bus, ctr *count.Bus,
) (support.CollectionMetrics, error) { ) (support.CollectionMetrics, error) {
var ( var (
metrics = support.CollectionMetrics{} metrics = support.CollectionMetrics{}
directory = dc.FullPath() directory = dc.FullPath()
protectedResourceID = directory.ResourceOwner() el = errs.Local()
el = errs.Local() metricsObjects int64
metricsObjects int64 metricsBytes int64
metricsBytes int64 metricsSuccess int64
metricsSuccess int64 wg sync.WaitGroup
wg sync.WaitGroup complete bool
complete bool
) )
ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory)) ctx, end := diagnostics.Span(ctx, "gc:drive:restoreCollection", diagnostics.Label("path", directory))
@ -263,7 +154,7 @@ func RestoreCollection(
rh, rh,
caches, caches,
drivePath, drivePath,
protectedResourceID, rcc.ProtectedResource.ID(),
fallbackDriveName) fallbackDriveName)
if err != nil { if err != nil {
return metrics, clues.Wrap(err, "ensuring drive exists") return metrics, clues.Wrap(err, "ensuring drive exists")
@ -281,8 +172,8 @@ func RestoreCollection(
// the drive into which this folder gets restored is tracked separately in drivePath. // the drive into which this folder gets restored is tracked separately in drivePath.
restoreDir := &path.Builder{} restoreDir := &path.Builder{}
if len(restoreCfg.Location) > 0 { if len(rcc.RestoreConfig.Location) > 0 {
restoreDir = restoreDir.Append(restoreCfg.Location) restoreDir = restoreDir.Append(rcc.RestoreConfig.Location)
} }
restoreDir = restoreDir.Append(drivePath.Folders...) restoreDir = restoreDir.Append(drivePath.Folders...)
@ -301,8 +192,8 @@ func RestoreCollection(
drivePath, drivePath,
dc, dc,
caches, caches,
backupVersion, rcc.BackupVersion,
restorePerms) rcc.RestoreConfig.IncludePermissions)
if err != nil { if err != nil {
return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx) return metrics, clues.Wrap(err, "getting permissions").WithClues(ctx)
} }
@ -316,7 +207,7 @@ func RestoreCollection(
dc.FullPath(), dc.FullPath(),
colMeta, colMeta,
caches, caches,
restorePerms) rcc.RestoreConfig.IncludePermissions)
if err != nil { if err != nil {
return metrics, clues.Wrap(err, "creating folders for restore") return metrics, clues.Wrap(err, "creating folders for restore")
} }
@ -390,14 +281,12 @@ func RestoreCollection(
itemInfo, skipped, err := restoreItem( itemInfo, skipped, err := restoreItem(
ictx, ictx,
rh, rh,
restoreCfg, rcc,
dc, dc,
backupVersion,
drivePath, drivePath,
restoreFolderID, restoreFolderID,
copyBuffer, copyBuffer,
caches, caches,
restorePerms,
itemData, itemData,
itemPath, itemPath,
ctr) ctr)
@ -440,14 +329,12 @@ func RestoreCollection(
func restoreItem( func restoreItem(
ctx context.Context, ctx context.Context,
rh RestoreHandler, rh RestoreHandler,
restoreCfg control.RestoreConfig, rcc inject.RestoreConsumerConfig,
fibn data.FetchItemByNamer, fibn data.FetchItemByNamer,
backupVersion int,
drivePath *path.DrivePath, drivePath *path.DrivePath,
restoreFolderID string, restoreFolderID string,
copyBuffer []byte, copyBuffer []byte,
caches *restoreCaches, caches *restoreCaches,
restorePerms bool,
itemData data.Stream, itemData data.Stream,
itemPath path.Path, itemPath path.Path,
ctr *count.Bus, ctr *count.Bus,
@ -455,11 +342,11 @@ func restoreItem(
itemUUID := itemData.UUID() itemUUID := itemData.UUID()
ctx = clues.Add(ctx, "item_id", itemUUID) ctx = clues.Add(ctx, "item_id", itemUUID)
if backupVersion < version.OneDrive1DataAndMetaFiles { if rcc.BackupVersion < version.OneDrive1DataAndMetaFiles {
itemInfo, err := restoreV0File( itemInfo, err := restoreV0File(
ctx, ctx,
rh, rh,
restoreCfg, rcc.RestoreConfig,
drivePath, drivePath,
fibn, fibn,
restoreFolderID, restoreFolderID,
@ -468,7 +355,7 @@ func restoreItem(
itemData, itemData,
ctr) ctr)
if err != nil { if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil return details.ItemInfo{}, true, nil
} }
@ -491,7 +378,7 @@ func restoreItem(
// Only the version.OneDrive1DataAndMetaFiles needed to deserialize the // Only the version.OneDrive1DataAndMetaFiles needed to deserialize the
// permission for child folders here. Later versions can request // permission for child folders here. Later versions can request
// permissions inline when processing the collection. // permissions inline when processing the collection.
if !restorePerms || backupVersion >= version.OneDrive4DirIncludesPermissions { if !rcc.RestoreConfig.IncludePermissions || rcc.BackupVersion >= version.OneDrive4DirIncludesPermissions {
return details.ItemInfo{}, true, nil return details.ItemInfo{}, true, nil
} }
@ -511,22 +398,21 @@ func restoreItem(
// only items with DataFileSuffix from this point on // only items with DataFileSuffix from this point on
if backupVersion < version.OneDrive6NameInMeta { if rcc.BackupVersion < version.OneDrive6NameInMeta {
itemInfo, err := restoreV1File( itemInfo, err := restoreV1File(
ctx, ctx,
rh, rh,
restoreCfg, rcc,
drivePath, drivePath,
fibn, fibn,
restoreFolderID, restoreFolderID,
copyBuffer, copyBuffer,
restorePerms,
caches, caches,
itemPath, itemPath,
itemData, itemData,
ctr) ctr)
if err != nil { if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil return details.ItemInfo{}, true, nil
} }
@ -541,18 +427,17 @@ func restoreItem(
itemInfo, err := restoreV6File( itemInfo, err := restoreV6File(
ctx, ctx,
rh, rh,
restoreCfg, rcc,
drivePath, drivePath,
fibn, fibn,
restoreFolderID, restoreFolderID,
copyBuffer, copyBuffer,
restorePerms,
caches, caches,
itemPath, itemPath,
itemData, itemData,
ctr) ctr)
if err != nil { if err != nil {
if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && restoreCfg.OnCollision == control.Skip { if errors.Is(err, graph.ErrItemAlreadyExistsConflict) && rcc.RestoreConfig.OnCollision == control.Skip {
return details.ItemInfo{}, true, nil return details.ItemInfo{}, true, nil
} }
@ -596,12 +481,11 @@ func restoreV0File(
func restoreV1File( func restoreV1File(
ctx context.Context, ctx context.Context,
rh RestoreHandler, rh RestoreHandler,
restoreCfg control.RestoreConfig, rcc inject.RestoreConsumerConfig,
drivePath *path.DrivePath, drivePath *path.DrivePath,
fibn data.FetchItemByNamer, fibn data.FetchItemByNamer,
restoreFolderID string, restoreFolderID string,
copyBuffer []byte, copyBuffer []byte,
restorePerms bool,
caches *restoreCaches, caches *restoreCaches,
itemPath path.Path, itemPath path.Path,
itemData data.Stream, itemData data.Stream,
@ -611,7 +495,7 @@ func restoreV1File(
itemID, itemInfo, err := restoreFile( itemID, itemInfo, err := restoreFile(
ctx, ctx,
restoreCfg, rcc.RestoreConfig,
rh, rh,
fibn, fibn,
trimmedName, trimmedName,
@ -627,7 +511,7 @@ func restoreV1File(
// Mark it as success without processing .meta // Mark it as success without processing .meta
// file if we are not restoring permissions // file if we are not restoring permissions
if !restorePerms { if !rcc.RestoreConfig.IncludePermissions {
return itemInfo, nil return itemInfo, nil
} }
@ -657,12 +541,11 @@ func restoreV1File(
func restoreV6File( func restoreV6File(
ctx context.Context, ctx context.Context,
rh RestoreHandler, rh RestoreHandler,
restoreCfg control.RestoreConfig, rcc inject.RestoreConsumerConfig,
drivePath *path.DrivePath, drivePath *path.DrivePath,
fibn data.FetchItemByNamer, fibn data.FetchItemByNamer,
restoreFolderID string, restoreFolderID string,
copyBuffer []byte, copyBuffer []byte,
restorePerms bool,
caches *restoreCaches, caches *restoreCaches,
itemPath path.Path, itemPath path.Path,
itemData data.Stream, itemData data.Stream,
@ -696,7 +579,7 @@ func restoreV6File(
itemID, itemInfo, err := restoreFile( itemID, itemInfo, err := restoreFile(
ctx, ctx,
restoreCfg, rcc.RestoreConfig,
rh, rh,
fibn, fibn,
meta.FileName, meta.FileName,
@ -712,10 +595,12 @@ func restoreV6File(
// Mark it as success without processing .meta // Mark it as success without processing .meta
// file if we are not restoring permissions // file if we are not restoring permissions
if !restorePerms { if !rcc.RestoreConfig.IncludePermissions {
return itemInfo, nil return itemInfo, nil
} }
fmt.Printf("\n-----\nrestorev6 %+v\n-----\n", rcc.RestoreConfig.IncludePermissions)
err = RestorePermissions( err = RestorePermissions(
ctx, ctx,
rh, rh,
@ -765,6 +650,8 @@ func CreateRestoreFolders(
return id, nil return id, nil
} }
fmt.Printf("\n-----\ncreatefolders %+v\n-----\n", restorePerms)
err = RestorePermissions( err = RestorePermissions(
ctx, ctx,
rh, rh,

View File

@ -0,0 +1,116 @@
package onedrive
import (
"context"
"sync"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/m365/onedrive/metadata"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type driveInfo struct {
id string
name string
rootFolderID string
}
type restoreCaches struct {
BackupDriveIDName idname.Cacher
collisionKeyToItemID map[string]api.DriveItemIDType
DriveIDToDriveInfo map[string]driveInfo
DriveNameToDriveInfo map[string]driveInfo
Folders *folderCache
OldLinkShareIDToNewID map[string]string
OldPermIDToNewID map[string]string
ParentDirToMeta map[string]metadata.Metadata
pool sync.Pool
}
func (rc *restoreCaches) AddDrive(
ctx context.Context,
md models.Driveable,
grf GetRootFolderer,
) error {
di := driveInfo{
id: ptr.Val(md.GetId()),
name: ptr.Val(md.GetName()),
}
ctx = clues.Add(ctx, "drive_info", di)
root, err := grf.GetRootFolder(ctx, di.id)
if err != nil {
return clues.Wrap(err, "getting drive root id")
}
di.rootFolderID = ptr.Val(root.GetId())
rc.DriveIDToDriveInfo[di.id] = di
rc.DriveNameToDriveInfo[di.name] = di
return nil
}
// Populate looks up drive items available to the protectedResource
// and adds their info to the caches.
func (rc *restoreCaches) Populate(
ctx context.Context,
gdparf GetDrivePagerAndRootFolderer,
protectedResourceID string,
) error {
drives, err := api.GetAllDrives(
ctx,
gdparf.NewDrivePager(protectedResourceID, nil),
true,
maxDrivesRetries)
if err != nil {
return clues.Wrap(err, "getting drives")
}
for _, md := range drives {
if err := rc.AddDrive(ctx, md, gdparf); err != nil {
return clues.Wrap(err, "caching drive")
}
}
return nil
}
type GetDrivePagerAndRootFolderer interface {
GetRootFolderer
NewDrivePagerer
}
func NewRestoreCaches(
backupDriveIDNames idname.Cacher,
) *restoreCaches {
// avoid nil panics
if backupDriveIDNames == nil {
backupDriveIDNames = idname.NewCache(nil)
}
return &restoreCaches{
BackupDriveIDName: backupDriveIDNames,
collisionKeyToItemID: map[string]api.DriveItemIDType{},
DriveIDToDriveInfo: map[string]driveInfo{},
DriveNameToDriveInfo: map[string]driveInfo{},
Folders: NewFolderCache(),
OldLinkShareIDToNewID: map[string]string{},
OldPermIDToNewID: map[string]string{},
ParentDirToMeta: map[string]metadata.Metadata{},
// Buffer pool for uploads
pool: sync.Pool{
New: func() any {
b := make([]byte, graph.CopyBufferSize)
return &b
},
},
}
}

View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
"github.com/alcionai/corso/src/internal/m365/onedrive/mock" "github.com/alcionai/corso/src/internal/m365/onedrive/mock"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
@ -512,21 +513,25 @@ func (suite *RestoreUnitSuite) TestRestoreItem_collisionHandling() {
ctr := count.New() ctr := count.New()
rcc := inject.RestoreConsumerConfig{
BackupVersion: version.Backup,
Options: control.DefaultOptions(),
RestoreConfig: restoreCfg,
}
_, skip, err := restoreItem( _, skip, err := restoreItem(
ctx, ctx,
rh, rh,
restoreCfg, rcc,
mock.FetchItemByName{ mock.FetchItemByName{
Item: &mock.Data{ Item: &mock.Data{
Reader: mock.FileRespReadCloser(mock.DriveFileMetaData), Reader: mock.FileRespReadCloser(mock.DriveFileMetaData),
}, },
}, },
version.Backup,
dp, dp,
"", "",
make([]byte, graph.CopyBufferSize), make([]byte, graph.CopyBufferSize),
caches, caches,
false,
&mock.Data{ &mock.Data{
ID: uuid.NewString(), ID: uuid.NewString(),
Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData), Reader: mock.FileRespReadCloser(mock.DriveFilePayloadData),

View File

@ -21,7 +21,7 @@ type oneDriveService struct {
} }
func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) { func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) {
ac, err := api.NewClient(credentials, control.Defaults()) ac, err := api.NewClient(credentials, control.DefaultOptions())
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -53,7 +53,7 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() {
creds, err := acct.M365Config() creds, err := acct.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.ac, err = api.NewClient(creds, control.Defaults()) suite.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user) drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user)

View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/graph"
odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/m365/onedrive/consts"
@ -223,9 +224,9 @@ func (suite *SharePointIntegrationSuite) TestPermissionsRestoreAndBackup() {
testPermissionsRestoreAndBackup(suite, version.Backup) testPermissionsRestoreAndBackup(suite, version.Backup)
} }
func (suite *SharePointIntegrationSuite) TestPermissionsBackupAndNoRestore() { func (suite *SharePointIntegrationSuite) TestRestoreNoPermissionsAndBackup() {
suite.T().Skip("Temporarily disabled due to CI issues") suite.T().Skip("Temporarily disabled due to CI issues")
testPermissionsBackupAndNoRestore(suite, version.Backup) testRestoreNoPermissionsAndBackup(suite, version.Backup)
} }
func (suite *SharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { func (suite *SharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() {
@ -290,8 +291,8 @@ func (suite *OneDriveIntegrationSuite) TestPermissionsRestoreAndBackup() {
testPermissionsRestoreAndBackup(suite, version.Backup) testPermissionsRestoreAndBackup(suite, version.Backup)
} }
func (suite *OneDriveIntegrationSuite) TestPermissionsBackupAndNoRestore() { func (suite *OneDriveIntegrationSuite) TestRestoreNoPermissionsAndBackup() {
testPermissionsBackupAndNoRestore(suite, version.Backup) testRestoreNoPermissionsAndBackup(suite, version.Backup)
} }
func (suite *OneDriveIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() { func (suite *OneDriveIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() {
@ -354,8 +355,8 @@ func (suite *OneDriveNightlySuite) TestPermissionsRestoreAndBackup() {
testPermissionsRestoreAndBackup(suite, version.OneDrive1DataAndMetaFiles) testPermissionsRestoreAndBackup(suite, version.OneDrive1DataAndMetaFiles)
} }
func (suite *OneDriveNightlySuite) TestPermissionsBackupAndNoRestore() { func (suite *OneDriveNightlySuite) TestRestoreNoPermissionsAndBackup() {
testPermissionsBackupAndNoRestore(suite, version.OneDrive1DataAndMetaFiles) testRestoreNoPermissionsAndBackup(suite, version.OneDrive1DataAndMetaFiles)
} }
func (suite *OneDriveNightlySuite) TestPermissionsInheritanceRestoreAndBackup() { func (suite *OneDriveNightlySuite) TestPermissionsInheritanceRestoreAndBackup() {
@ -517,19 +518,17 @@ func testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(
collectionsLatest: expected, collectionsLatest: expected,
} }
rc := testdata.DefaultRestoreConfig("od_restore_and_backup_multi") restoreCfg := testdata.DefaultRestoreConfig("od_restore_and_backup_multi")
rc.OnCollision = control.Replace restoreCfg.OnCollision = control.Replace
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions( runRestoreBackupTestVersions(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
rc)
}) })
} }
} }
@ -768,24 +767,22 @@ func testPermissionsRestoreAndBackup(suite oneDriveSuite, startVersion int) {
collectionsLatest: expected, collectionsLatest: expected,
} }
rc := testdata.DefaultRestoreConfig("perms_restore_and_backup") restoreCfg := testdata.DefaultRestoreConfig("perms_restore_and_backup")
rc.OnCollision = control.Replace restoreCfg.OnCollision = control.Replace
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions( runRestoreBackupTestVersions(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
rc)
}) })
} }
} }
func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) { func testRestoreNoPermissionsAndBackup(suite oneDriveSuite, startVersion int) {
t := suite.T() t := suite.T()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
@ -860,19 +857,19 @@ func testPermissionsBackupAndNoRestore(suite oneDriveSuite, startVersion int) {
collectionsLatest: expected, collectionsLatest: expected,
} }
rc := testdata.DefaultRestoreConfig("perms_backup_no_restore") restoreCfg := testdata.DefaultRestoreConfig("perms_backup_no_restore")
rc.OnCollision = control.Replace restoreCfg.OnCollision = control.Replace
restoreCfg.IncludePermissions = false
fmt.Printf("\n-----\nrcfg %+v\n-----\n", restoreCfg.IncludePermissions)
runRestoreBackupTestVersions( runRestoreBackupTestVersions(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: false, restoreCfg)
ToggleFeatures: control.Toggles{},
},
rc)
}) })
} }
} }
@ -1067,19 +1064,17 @@ func testPermissionsInheritanceRestoreAndBackup(suite oneDriveSuite, startVersio
collectionsLatest: expected, collectionsLatest: expected,
} }
rc := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup") restoreCfg := testdata.DefaultRestoreConfig("perms_inherit_restore_and_backup")
rc.OnCollision = control.Replace restoreCfg.OnCollision = control.Replace
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions( runRestoreBackupTestVersions(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
rc)
}) })
} }
} }
@ -1264,19 +1259,17 @@ func testLinkSharesInheritanceRestoreAndBackup(suite oneDriveSuite, startVersion
collectionsLatest: expected, collectionsLatest: expected,
} }
rc := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup") restoreCfg := testdata.DefaultRestoreConfig("linkshares_inherit_restore_and_backup")
rc.OnCollision = control.Replace restoreCfg.OnCollision = control.Replace
restoreCfg.IncludePermissions = true
runRestoreBackupTestVersions( runRestoreBackupTestVersions(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
rc)
}) })
} }
} }
@ -1383,16 +1376,16 @@ func testRestoreFolderNamedFolderRegression(
collectionsLatest: expected, collectionsLatest: expected,
} }
restoreCfg := control.DefaultRestoreConfig(dttm.HumanReadableDriveItem)
restoreCfg.IncludePermissions = true
runRestoreTestWithVersion( runRestoreTestWithVersion(
t, t,
testData, testData,
suite.Tenant(), suite.Tenant(),
[]string{suite.ResourceOwner()}, []string{suite.ResourceOwner()},
control.Options{ control.DefaultOptions(),
RestorePermissions: true, restoreCfg)
ToggleFeatures: control.Toggles{},
},
)
}) })
} }
} }

View File

@ -12,11 +12,11 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/onedrive"
"github.com/alcionai/corso/src/internal/m365/sharepoint" "github.com/alcionai/corso/src/internal/m365/sharepoint"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/path"
) )
// ConsumeRestoreCollections restores data from the specified collections // ConsumeRestoreCollections restores data from the specified collections
@ -24,10 +24,7 @@ import (
// SideEffect: status is updated at the completion of operation // SideEffect: status is updated at the completion of operation
func (ctrl *Controller) ConsumeRestoreCollections( func (ctrl *Controller) ConsumeRestoreCollections(
ctx context.Context, ctx context.Context,
backupVersion int, rcc inject.RestoreConsumerConfig,
sels selectors.Selector,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection, dcs []data.RestoreCollection,
errs *fault.Bus, errs *fault.Bus,
ctr *count.Bus, ctr *count.Bus,
@ -35,48 +32,64 @@ func (ctrl *Controller) ConsumeRestoreCollections(
ctx, end := diagnostics.Span(ctx, "m365:restore") ctx, end := diagnostics.Span(ctx, "m365:restore")
defer end() defer end()
ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: rcc.Selector.PathService()})
ctx = clues.Add(ctx, "restore_config", restoreCfg) ctx = clues.Add(ctx, "restore_config", rcc.RestoreConfig)
if len(dcs) == 0 { if len(dcs) == 0 {
return nil, clues.New("no data collections to restore") return nil, clues.New("no data collections to restore")
} }
serviceEnabled, _, err := checkServiceEnabled(
ctx,
ctrl.AC.Users(),
rcc.Selector.PathService(),
rcc.ProtectedResource.ID())
if err != nil {
return nil, err
}
if !serviceEnabled {
return nil, clues.Stack(graph.ErrServiceNotEnabled).WithClues(ctx)
}
var ( var (
status *support.ControllerOperationStatus service = rcc.Selector.PathService()
deets = &details.Builder{} status *support.ControllerOperationStatus
err error deets = &details.Builder{}
) )
switch sels.Service { switch service {
case selectors.ServiceExchange: case path.ExchangeService:
status, err = exchange.ConsumeRestoreCollections(ctx, ctrl.AC, restoreCfg, dcs, deets, errs, ctr) status, err = exchange.ConsumeRestoreCollections(
case selectors.ServiceOneDrive: ctx,
ctrl.AC,
rcc,
dcs,
deets,
errs,
ctr)
case path.OneDriveService:
status, err = onedrive.ConsumeRestoreCollections( status, err = onedrive.ConsumeRestoreCollections(
ctx, ctx,
onedrive.NewRestoreHandler(ctrl.AC), onedrive.NewRestoreHandler(ctrl.AC),
backupVersion, rcc,
restoreCfg,
opts,
ctrl.backupDriveIDNames, ctrl.backupDriveIDNames,
dcs, dcs,
deets, deets,
errs, errs,
ctr) ctr)
case selectors.ServiceSharePoint: case path.SharePointService:
status, err = sharepoint.ConsumeRestoreCollections( status, err = sharepoint.ConsumeRestoreCollections(
ctx, ctx,
backupVersion, rcc,
ctrl.AC, ctrl.AC,
restoreCfg,
opts,
ctrl.backupDriveIDNames, ctrl.backupDriveIDNames,
dcs, dcs,
deets, deets,
errs, errs,
ctr) ctr)
default: default:
err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") err = clues.Wrap(clues.New(service.String()), "service not supported")
} }
ctrl.incrementAwaitingMessages() ctrl.incrementAwaitingMessages()

View File

@ -107,7 +107,7 @@ func (suite *LibrariesBackupUnitSuite) TestUpdateCollections() {
tenantID, tenantID,
site, site,
nil, nil,
control.Defaults()) control.DefaultOptions())
c.CollectionMap = collMap c.CollectionMap = collMap
@ -201,7 +201,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
creds, err := a.M365Config() creds, err := a.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ac, err := api.NewClient(creds, control.Defaults()) ac, err := api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
col, err := collectPages( col, err := collectPages(
@ -210,7 +210,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
ac, ac,
mock.NewProvider(siteID, siteID), mock.NewProvider(siteID, siteID),
&MockGraphService{}, &MockGraphService{},
control.Defaults(), control.DefaultOptions(),
fault.New(true)) fault.New(true))
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
assert.NotEmpty(t, col) assert.NotEmpty(t, col)

View File

@ -43,7 +43,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() {
suite.creds = m365 suite.creds = m365
ac, err := api.NewClient(m365, control.Defaults()) ac, err := api.NewClient(m365, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.ac = ac suite.ac = ac
@ -168,7 +168,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
suite.ac, suite.ac,
test.category, test.category,
nil, nil,
control.Defaults()) control.DefaultOptions())
col.data <- test.getItem(t, test.itemName) col.data <- test.getItem(t, test.itemName)
readItems := []data.Stream{} readItems := []data.Stream{}

View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/onedrive" "github.com/alcionai/corso/src/internal/m365/onedrive"
betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api" betaAPI "github.com/alcionai/corso/src/internal/m365/sharepoint/api"
"github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/count"
@ -31,10 +32,8 @@ import (
// ConsumeRestoreCollections will restore the specified data collections into OneDrive // ConsumeRestoreCollections will restore the specified data collections into OneDrive
func ConsumeRestoreCollections( func ConsumeRestoreCollections(
ctx context.Context, ctx context.Context,
backupVersion int, rcc inject.RestoreConsumerConfig,
ac api.Client, ac api.Client,
restoreCfg control.RestoreConfig,
opts control.Options,
backupDriveIDNames idname.Cacher, backupDriveIDNames idname.Cacher,
dcs []data.RestoreCollection, dcs []data.RestoreCollection,
deets *details.Builder, deets *details.Builder,
@ -42,14 +41,13 @@ func ConsumeRestoreCollections(
ctr *count.Bus, ctr *count.Bus,
) (*support.ControllerOperationStatus, error) { ) (*support.ControllerOperationStatus, error) {
var ( var (
lrh = libraryRestoreHandler{ac} lrh = libraryRestoreHandler{ac}
protectedResourceID = dcs[0].FullPath().ResourceOwner() restoreMetrics support.CollectionMetrics
restoreMetrics support.CollectionMetrics caches = onedrive.NewRestoreCaches(backupDriveIDNames)
caches = onedrive.NewRestoreCaches(backupDriveIDNames) el = errs.Local()
el = errs.Local()
) )
err := caches.Populate(ctx, lrh, protectedResourceID) err := caches.Populate(ctx, lrh, rcc.ProtectedResource.ID())
if err != nil { if err != nil {
return nil, clues.Wrap(err, "initializing restore caches") return nil, clues.Wrap(err, "initializing restore caches")
} }
@ -70,7 +68,7 @@ func ConsumeRestoreCollections(
metrics support.CollectionMetrics metrics support.CollectionMetrics
ictx = clues.Add(ctx, ictx = clues.Add(ctx,
"category", category, "category", category,
"restore_location", restoreCfg.Location, "restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"full_path", dc.FullPath()) "full_path", dc.FullPath())
) )
@ -80,12 +78,10 @@ func ConsumeRestoreCollections(
metrics, err = onedrive.RestoreCollection( metrics, err = onedrive.RestoreCollection(
ictx, ictx,
lrh, lrh,
restoreCfg, rcc,
backupVersion,
dc, dc,
caches, caches,
deets, deets,
opts.RestorePermissions,
control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem), control.DefaultRestoreContainerName(dttm.HumanReadableDriveItem),
errs, errs,
ctr) ctr)
@ -95,7 +91,7 @@ func ConsumeRestoreCollections(
ictx, ictx,
ac.Stable, ac.Stable,
dc, dc,
restoreCfg.Location, rcc.RestoreConfig.Location,
deets, deets,
errs) errs)
@ -104,7 +100,7 @@ func ConsumeRestoreCollections(
ictx, ictx,
ac.Stable, ac.Stable,
dc, dc,
restoreCfg.Location, rcc.RestoreConfig.Location,
deets, deets,
errs) errs)
@ -128,7 +124,7 @@ func ConsumeRestoreCollections(
support.Restore, support.Restore,
len(dcs), len(dcs),
restoreMetrics, restoreMetrics,
restoreCfg.Location) rcc.RestoreConfig.Location)
return status, el.Failure() return status, el.Failure()
} }

View File

@ -68,8 +68,7 @@ func GetCollectionsAndExpected(
owner, owner,
config.RestoreCfg, config.RestoreCfg,
testCollections, testCollections,
backupVersion, backupVersion)
)
if err != nil { if err != nil {
return totalItems, totalKopiaItems, collections, expectedData, err return totalItems, totalKopiaItems, collections, expectedData, err
} }

View File

@ -362,7 +362,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
op, err := NewBackupOperation( op, err := NewBackupOperation(
ctx, ctx,
control.Defaults(), control.DefaultOptions(),
kw, kw,
sw, sw,
ctrl, ctrl,
@ -1137,7 +1137,7 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() {
creds, err := a.M365Config() creds, err := a.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
suite.ac, err = api.NewClient(creds, control.Defaults()) suite.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
} }
@ -1147,7 +1147,7 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
sw = &store.Wrapper{} sw = &store.Wrapper{}
ctrl = &mock.Controller{} ctrl = &mock.Controller{}
acct = tconfig.NewM365Account(suite.T()) acct = tconfig.NewM365Account(suite.T())
opts = control.Defaults() opts = control.DefaultOptions()
) )
table := []struct { table := []struct {

View File

@ -99,7 +99,7 @@ func (suite *ExportOpSuite) TestExportOperation_PersistResults() {
op, err := NewExportOperation( op, err := NewExportOperation(
ctx, ctx,
control.Defaults(), control.DefaultOptions(),
kw, kw,
sw, sw,
ctrl, ctrl,

View File

@ -27,7 +27,7 @@ func ControllerWithSelector(
ins idname.Cacher, ins idname.Cacher,
onFail func(), onFail func(),
) (*m365.Controller, selectors.Selector) { ) (*m365.Controller, selectors.Selector) {
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions())
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil { if onFail != nil {
onFail() onFail()
@ -36,7 +36,7 @@ func ControllerWithSelector(
t.FailNow() t.FailNow()
} }
id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil { if onFail != nil {
onFail() onFail()

View File

@ -0,0 +1,18 @@
package inject
import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/selectors"
)
// RestoreConsumerConfig is a container-of-things for holding options and
// configurations from various packages, which are widely used by all
// restore consumers independent of service or data category.
type RestoreConsumerConfig struct {
BackupVersion int
Options control.Options
ProtectedResource idname.Provider
RestoreConfig control.RestoreConfig
Selector selectors.Selector
}

View File

@ -37,10 +37,7 @@ type (
RestoreConsumer interface { RestoreConsumer interface {
ConsumeRestoreCollections( ConsumeRestoreCollections(
ctx context.Context, ctx context.Context,
backupVersion int, rcc RestoreConsumerConfig,
selector selectors.Selector,
restoreCfg control.RestoreConfig,
opts control.Options,
dcs []data.RestoreCollection, dcs []data.RestoreCollection,
errs *fault.Bus, errs *fault.Bus,
ctr *count.Bus, ctr *count.Bus,
@ -49,6 +46,7 @@ type (
Wait() *data.CollectionStats Wait() *data.CollectionStats
CacheItemInfoer CacheItemInfoer
PopulateProtectedResourceIDAndNamer
} }
CacheItemInfoer interface { CacheItemInfoer interface {
@ -76,6 +74,25 @@ type (
CacheItemInfoer CacheItemInfoer
} }
PopulateProtectedResourceIDAndNamer interface {
// PopulateProtectedResourceIDAndName takes the provided owner identifier and produces
// the owner's name and ID from that value. Returns an error if the owner is
// not recognized by the current tenant.
//
// The id-name cacher should be optional. Some processes will look up all owners in
// the tenant before reaching this step. In that case, the data gets handed
// down for this func to consume instead of performing further queries. The
// data gets stored inside the controller instance for later re-use.
PopulateProtectedResourceIDAndName(
ctx context.Context,
owner string, // input value, can be either id or name
ins idname.Cacher,
) (
id, name string,
err error,
)
}
RepoMaintenancer interface { RepoMaintenancer interface {
RepoMaintenance(ctx context.Context, opts repository.Maintenance) error RepoMaintenance(ctx context.Context, opts repository.Maintenance) error
} }

View File

@ -54,7 +54,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() {
mo, err := NewMaintenanceOperation( mo, err := NewMaintenanceOperation(
ctx, ctx,
control.Defaults(), control.DefaultOptions(),
kw, kw,
repository.Maintenance{ repository.Maintenance{
Type: repository.MetadataMaintenance, Type: repository.MetadataMaintenance,

View File

@ -26,7 +26,7 @@ func TestOperationSuite(t *testing.T) {
func (suite *OperationSuite) TestNewOperation() { func (suite *OperationSuite) TestNewOperation() {
t := suite.T() t := suite.T()
op := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, nil, nil) op := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, nil, nil)
assert.Greater(t, op.CreatedAt, time.Time{}) assert.Greater(t, op.CreatedAt, time.Time{})
} }
@ -46,7 +46,7 @@ func (suite *OperationSuite) TestOperation_Validate() {
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
err := newOperation(control.Defaults(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate() err := newOperation(control.DefaultOptions(), events.Bus{}, &count.Bus{}, test.kw, test.sw).validate()
test.errCheck(suite.T(), err, clues.ToCore(err)) test.errCheck(suite.T(), err, clues.ToCore(err))
}) })
} }

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/common/crash" "github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
@ -172,7 +173,7 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De
logger.CtxErr(ctx, err).Error("running restore") logger.CtxErr(ctx, err).Error("running restore")
if errors.Is(err, kopia.ErrNoRestorePath) { if errors.Is(err, kopia.ErrNoRestorePath) {
op.Errors.Fail(clues.New("empty backup or unknown path provided")) op.Errors.Fail(clues.Wrap(err, "empty backup or unknown path provided"))
} }
op.Errors.Fail(clues.Wrap(err, "running restore")) op.Errors.Fail(clues.Wrap(err, "running restore"))
@ -217,7 +218,19 @@ func (op *RestoreOperation) do(
return nil, clues.Wrap(err, "getting backup and details") return nil, clues.Wrap(err, "getting backup and details")
} }
observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(bup.Selector.DiscreteOwner)) restoreToProtectedResource, err := chooseRestoreResource(ctx, op.rc, op.RestoreCfg, bup.Selector)
if err != nil {
return nil, clues.Wrap(err, "getting destination protected resource")
}
ctx = clues.Add(
ctx,
"backup_protected_resource_id", bup.Selector.ID(),
"backup_protected_resource_name", clues.Hide(bup.Selector.Name()),
"restore_protected_resource_id", restoreToProtectedResource.ID(),
"restore_protected_resource_name", clues.Hide(restoreToProtectedResource.Name()))
observe.Message(ctx, "Restoring", observe.Bullet, clues.Hide(restoreToProtectedResource.Name()))
paths, err := formatDetailsForRestoration( paths, err := formatDetailsForRestoration(
ctx, ctx,
@ -232,8 +245,6 @@ func (op *RestoreOperation) do(
ctx = clues.Add( ctx = clues.Add(
ctx, ctx,
"resource_owner_id", bup.Selector.ID(),
"resource_owner_name", clues.Hide(bup.Selector.Name()),
"details_entries", len(deets.Entries), "details_entries", len(deets.Entries),
"details_paths", len(paths), "details_paths", len(paths),
"backup_snapshot_id", bup.SnapshotID, "backup_snapshot_id", bup.SnapshotID,
@ -254,7 +265,12 @@ func (op *RestoreOperation) do(
kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository") kopiaComplete := observe.MessageWithCompletion(ctx, "Enumerating items in repository")
defer close(kopiaComplete) defer close(kopiaComplete)
dcs, err := op.kopia.ProduceRestoreCollections(ctx, bup.SnapshotID, paths, opStats.bytesRead, op.Errors) dcs, err := op.kopia.ProduceRestoreCollections(
ctx,
bup.SnapshotID,
paths,
opStats.bytesRead,
op.Errors)
if err != nil { if err != nil {
return nil, clues.Wrap(err, "producing collections to restore") return nil, clues.Wrap(err, "producing collections to restore")
} }
@ -271,6 +287,7 @@ func (op *RestoreOperation) do(
ctx, ctx,
op.rc, op.rc,
bup.Version, bup.Version,
restoreToProtectedResource,
op.Selectors, op.Selectors,
op.RestoreCfg, op.RestoreCfg,
op.Options, op.Options,
@ -321,6 +338,24 @@ func (op *RestoreOperation) persistResults(
return op.Errors.Failure() return op.Errors.Failure()
} }
func chooseRestoreResource(
ctx context.Context,
pprian inject.PopulateProtectedResourceIDAndNamer,
restoreCfg control.RestoreConfig,
orig idname.Provider,
) (idname.Provider, error) {
if len(restoreCfg.ProtectedResource) == 0 {
return orig, nil
}
id, name, err := pprian.PopulateProtectedResourceIDAndName(
ctx,
restoreCfg.ProtectedResource,
nil)
return idname.NewProvider(id, name), clues.Stack(err).OrNil()
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Restorer funcs // Restorer funcs
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -329,6 +364,7 @@ func consumeRestoreCollections(
ctx context.Context, ctx context.Context,
rc inject.RestoreConsumer, rc inject.RestoreConsumer,
backupVersion int, backupVersion int,
toProtectedResource idname.Provider,
sel selectors.Selector, sel selectors.Selector,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
opts control.Options, opts control.Options,
@ -342,15 +378,15 @@ func consumeRestoreCollections(
close(complete) close(complete)
}() }()
deets, err := rc.ConsumeRestoreCollections( rcc := inject.RestoreConsumerConfig{
ctx, BackupVersion: backupVersion,
backupVersion, Options: opts,
sel, ProtectedResource: toProtectedResource,
restoreCfg, RestoreConfig: restoreCfg,
opts, Selector: sel,
dcs, }
errs,
ctr) deets, err := rc.ConsumeRestoreCollections(ctx, rcc, dcs, errs, ctr)
if err != nil { if err != nil {
return nil, clues.Wrap(err, "restoring collections") return nil, clues.Wrap(err, "restoring collections")
} }

View File

@ -10,6 +10,8 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/events"
evmock "github.com/alcionai/corso/src/internal/events/mock" evmock "github.com/alcionai/corso/src/internal/events/mock"
@ -37,15 +39,15 @@ import (
// unit // unit
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type RestoreOpSuite struct { type RestoreOpUnitSuite struct {
tester.Suite tester.Suite
} }
func TestRestoreOpSuite(t *testing.T) { func TestRestoreOpUnitSuite(t *testing.T) {
suite.Run(t, &RestoreOpSuite{Suite: tester.NewUnitSuite(t)}) suite.Run(t, &RestoreOpUnitSuite{Suite: tester.NewUnitSuite(t)})
} }
func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { func (suite *RestoreOpUnitSuite) TestRestoreOperation_PersistResults() {
var ( var (
kw = &kopia.Wrapper{} kw = &kopia.Wrapper{}
sw = &store.Wrapper{} sw = &store.Wrapper{}
@ -107,7 +109,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
op, err := NewRestoreOperation( op, err := NewRestoreOperation(
ctx, ctx,
control.Defaults(), control.DefaultOptions(),
kw, kw,
sw, sw,
ctrl, ctrl,
@ -135,6 +137,75 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
} }
} }
func (suite *RestoreOpUnitSuite) TestChooseRestoreResource() {
var (
id = "id"
name = "name"
cfgWithPR = control.DefaultRestoreConfig(dttm.HumanReadable)
)
cfgWithPR.ProtectedResource = "cfgid"
table := []struct {
name string
cfg control.RestoreConfig
ctrl *mock.Controller
orig idname.Provider
expectErr assert.ErrorAssertionFunc
expectProvider assert.ValueAssertionFunc
expectID string
expectName string
}{
{
name: "use original",
cfg: control.DefaultRestoreConfig(dttm.HumanReadable),
ctrl: &mock.Controller{
ProtectedResourceID: id,
ProtectedResourceName: name,
},
orig: idname.NewProvider("oid", "oname"),
expectErr: assert.NoError,
expectID: "oid",
expectName: "oname",
},
{
name: "look up resource with iface",
cfg: cfgWithPR,
ctrl: &mock.Controller{
ProtectedResourceID: id,
ProtectedResourceName: name,
},
orig: idname.NewProvider("oid", "oname"),
expectErr: assert.NoError,
expectID: id,
expectName: name,
},
{
name: "error looking up protected resource",
cfg: cfgWithPR,
ctrl: &mock.Controller{
ProtectedResourceErr: assert.AnError,
},
orig: idname.NewProvider("oid", "oname"),
expectErr: assert.Error,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
result, err := chooseRestoreResource(ctx, test.ctrl, test.cfg, test.orig)
test.expectErr(t, err, clues.ToCore(err))
require.NotNil(t, result)
assert.Equal(t, test.expectID, result.ID())
assert.Equal(t, test.expectName, result.Name())
})
}
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// integration // integration
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -216,7 +287,7 @@ func (suite *RestoreOpIntegrationSuite) TestNewRestoreOperation() {
sw = &store.Wrapper{} sw = &store.Wrapper{}
ctrl = &mock.Controller{} ctrl = &mock.Controller{}
restoreCfg = testdata.DefaultRestoreConfig("") restoreCfg = testdata.DefaultRestoreConfig("")
opts = control.Defaults() opts = control.DefaultOptions()
) )
table := []struct { table := []struct {
@ -275,12 +346,12 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
suite.acct, suite.acct,
resource.Users, resource.Users,
rsel.PathService(), rsel.PathService(),
control.Defaults()) control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ro, err := NewRestoreOperation( ro, err := NewRestoreOperation(
ctx, ctx,
control.Defaults(), control.DefaultOptions(),
suite.kw, suite.kw,
suite.sw, suite.sw,
ctrl, ctrl,

View File

@ -67,9 +67,9 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
{ {
name: "Mail", name: "Mail",
selector: func() *selectors.ExchangeBackup { selector: func() *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup([]string{suite.its.userID}) sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
sel.DiscreteOwner = suite.its.userID sel.DiscreteOwner = suite.its.user.ID
return sel return sel
}, },
@ -79,7 +79,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
{ {
name: "Contacts", name: "Contacts",
selector: func() *selectors.ExchangeBackup { selector: func() *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup([]string{suite.its.userID}) sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch())) sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()))
return sel return sel
}, },
@ -89,7 +89,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
{ {
name: "Calendar Events", name: "Calendar Events",
selector: func() *selectors.ExchangeBackup { selector: func() *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup([]string{suite.its.userID}) sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch())) sel.Include(sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()))
return sel return sel
}, },
@ -107,7 +107,7 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = test.selector().Selector sel = test.selector().Selector
opts = control.Defaults() opts = control.DefaultOptions()
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
) )
@ -258,9 +258,9 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
// later on during the tests. Putting their identifiers into the selector // later on during the tests. Putting their identifiers into the selector
// at this point is harmless. // at this point is harmless.
containers = []string{container1, container2, container3, containerRename} containers = []string{container1, container2, container3, containerRename}
sel = selectors.NewExchangeBackup([]string{suite.its.userID}) sel = selectors.NewExchangeBackup([]string{suite.its.user.ID})
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
opts = control.Defaults() opts = control.DefaultOptions()
) )
opts.ToggleFeatures = toggles opts.ToggleFeatures = toggles
@ -278,7 +278,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
creds, err := acct.M365Config() creds, err := acct.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ac, err := api.NewClient(creds, control.Defaults()) ac, err := api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// generate 3 new folders with two items each. // generate 3 new folders with two items each.
@ -295,7 +295,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
mailDBF := func(id, timeStamp, subject, body string) []byte { mailDBF := func(id, timeStamp, subject, body string) []byte {
return exchMock.MessageWith( return exchMock.MessageWith(
suite.its.userID, suite.its.userID, suite.its.userID, suite.its.user.ID, suite.its.user.ID, suite.its.user.ID,
subject, body, body, subject, body, body,
now, now, now, now) now, now, now, now)
} }
@ -312,7 +312,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
eventDBF := func(id, timeStamp, subject, body string) []byte { eventDBF := func(id, timeStamp, subject, body string) []byte {
return exchMock.EventWith( return exchMock.EventWith(
suite.its.userID, subject, body, body, suite.its.user.ID, subject, body, body,
exchMock.NoOriginalStartDate, now, now, exchMock.NoOriginalStartDate, now, now,
exchMock.NoRecurrence, exchMock.NoAttendees, exchMock.NoRecurrence, exchMock.NoAttendees,
exchMock.NoAttachments, exchMock.NoCancelledOccurrences, exchMock.NoAttachments, exchMock.NoCancelledOccurrences,
@ -578,7 +578,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
service, service,
category, category,
selectors.NewExchangeRestore([]string{uidn.ID()}).Selector, selectors.NewExchangeRestore([]string{uidn.ID()}).Selector,
creds.AzureTenantID, suite.its.userID, "", container3, creds.AzureTenantID, suite.its.user.ID, "", container3,
2, 2,
version.Backup, version.Backup,
gen.dbf) gen.dbf)
@ -897,7 +897,7 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio
// a backup is required to run restores // a backup is required to run restores
baseSel := selectors.NewExchangeBackup([]string{suite.its.userID}) baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
baseSel.Include( baseSel.Include(
// events cannot be run, for the same reason as incremental backups: the user needs // events cannot be run, for the same reason as incremental backups: the user needs
// to have their account recycled. // to have their account recycled.
@ -905,11 +905,11 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio
baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()), baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch())) baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
baseSel.DiscreteOwner = suite.its.userID baseSel.DiscreteOwner = suite.its.user.ID
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.Defaults() opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup)
@ -1272,3 +1272,216 @@ func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeWithAdvancedOptio
assert.Len(t, result, 0, "no items should have been added as copies") assert.Len(t, result, 0, "no items should have been added as copies")
}) })
} }
func (suite *ExchangeRestoreIntgSuite) TestRestore_Run_exchangeAlternateProtectedResource() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
// a backup is required to run restores
baseSel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
baseSel.Include(
// events cannot be run, for the same reason as incremental backups: the user needs
// to have their account recycled.
// base_sel.EventCalendars([]string{api.DefaultCalendar}, selectors.PrefixMatch()),
baseSel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()),
baseSel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
baseSel.DiscreteOwner = suite.its.user.ID
var (
mb = evmock.NewBus()
opts = control.DefaultOptions()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup)
defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false)
rsel, err := baseSel.ToExchangeRestore()
require.NoError(t, err, clues.ToCore(err))
var (
restoreCfg = ctrlTD.DefaultRestoreConfig("exchange_restore_to_user")
sel = rsel.Selector
userID = suite.its.user.ID
secondaryUserID = suite.its.secondaryUser.ID
uid = userID
acCont = suite.its.ac.Contacts()
acMail = suite.its.ac.Mail()
// acEvts = suite.its.ac.Events()
firstCtr = count.New()
)
restoreCfg.OnCollision = control.Copy
mb = evmock.NewBus()
// first restore to the current user
ro1, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
firstCtr,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro1, mb, false)
// get all files in folder, use these as the base
// set of files to compare against.
var (
userItemIDs = map[path.CategoryType]map[string]struct{}{}
userCollisionKeys = map[path.CategoryType]map[string]string{}
)
// --- contacts
cat := path.ContactsCategory
userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs(
t,
ctx,
acCont,
uid,
"",
restoreCfg.Location)
// --- events
// cat = path.EventsCategory
// userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs(
// t,
// ctx,
// acEvts,
// uid,
// "",
// restoreCfg.Location)
// --- mail
cat = path.EmailCategory
userItemIDs[cat], userCollisionKeys[cat] = getCollKeysAndItemIDs(
t,
ctx,
acMail,
uid,
"",
restoreCfg.Location,
api.MailInbox)
// then restore to the secondary user
uid = secondaryUserID
mb = evmock.NewBus()
secondCtr := count.New()
restoreCfg.ProtectedResource = uid
ro2, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
secondCtr,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro2, mb, false)
var (
secondaryItemIDs = map[path.CategoryType]map[string]struct{}{}
secondaryCollisionKeys = map[path.CategoryType]map[string]string{}
)
// --- contacts
cat = path.ContactsCategory
secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs(
t,
ctx,
acCont,
uid,
"",
restoreCfg.Location)
// --- events
// cat = path.EventsCategory
// secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs(
// t,
// ctx,
// acEvts,
// uid,
// "",
// restoreCfg.Location)
// --- mail
cat = path.EmailCategory
secondaryItemIDs[cat], secondaryCollisionKeys[cat] = getCollKeysAndItemIDs(
t,
ctx,
acMail,
uid,
"",
restoreCfg.Location,
api.MailInbox)
// compare restore results
for _, cat := range []path.CategoryType{path.ContactsCategory, path.EmailCategory, path.EventsCategory} {
assert.Equal(t, len(userItemIDs[cat]), len(secondaryItemIDs[cat]))
assert.ElementsMatch(t, maps.Keys(userCollisionKeys[cat]), maps.Keys(secondaryCollisionKeys[cat]))
}
}
type GetItemsKeysAndContainerByNameer interface {
GetItemIDsInContainer(
ctx context.Context,
userID, containerID string,
) (map[string]struct{}, error)
GetContainerByName(
ctx context.Context,
userID, parentContainerID, containerName string,
) (graph.Container, error)
GetItemsInContainerByCollisionKey(
ctx context.Context,
userID, containerID string,
) (map[string]string, error)
}
func getCollKeysAndItemIDs(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
gikacbn GetItemsKeysAndContainerByNameer,
userID, parentContainerID string,
containerNames ...string,
) (map[string]struct{}, map[string]string) {
var (
c graph.Container
err error
cID string
)
for _, cn := range containerNames {
pcid := parentContainerID
if len(cID) != 0 {
pcid = cID
}
c, err = gikacbn.GetContainerByName(ctx, userID, pcid, cn)
require.NoError(t, err, clues.ToCore(err))
cID = ptr.Val(c.GetId())
}
itemIDs, err := gikacbn.GetItemIDsInContainer(ctx, userID, cID)
require.NoError(t, err, clues.ToCore(err))
collisionKeys, err := gikacbn.GetItemsInContainerByCollisionKey(ctx, userID, cID)
require.NoError(t, err, clues.ToCore(err))
return itemIDs, collisionKeys
}

View File

@ -25,6 +25,7 @@ import (
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
"github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/internal/streamstore" "github.com/alcionai/corso/src/internal/streamstore"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/tester/tconfig"
@ -400,6 +401,7 @@ func generateContainerOfItems(
restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting) restoreCfg := control.DefaultRestoreConfig(dttm.SafeForTesting)
restoreCfg.Location = destFldr restoreCfg.Location = destFldr
restoreCfg.IncludePermissions = true
dataColls := buildCollections( dataColls := buildCollections(
t, t,
@ -408,15 +410,19 @@ func generateContainerOfItems(
restoreCfg, restoreCfg,
collections) collections)
opts := control.Defaults() opts := control.DefaultOptions()
opts.RestorePermissions = true
rcc := inject.RestoreConsumerConfig{
BackupVersion: backupVersion,
Options: opts,
ProtectedResource: sel,
RestoreConfig: restoreCfg,
Selector: sel,
}
deets, err := ctrl.ConsumeRestoreCollections( deets, err := ctrl.ConsumeRestoreCollections(
ctx, ctx,
backupVersion, rcc,
sel,
restoreCfg,
opts,
dataColls, dataColls,
fault.New(true), fault.New(true),
count.New()) count.New())
@ -535,7 +541,7 @@ func ControllerWithSelector(
ins idname.Cacher, ins idname.Cacher,
onFail func(*testing.T, context.Context), onFail func(*testing.T, context.Context),
) (*m365.Controller, selectors.Selector) { ) (*m365.Controller, selectors.Selector) {
ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.Defaults()) ctrl, err := m365.NewController(ctx, acct, cr, sel.PathService(), control.DefaultOptions())
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil { if onFail != nil {
onFail(t, ctx) onFail(t, ctx)
@ -544,7 +550,7 @@ func ControllerWithSelector(
t.FailNow() t.FailNow()
} }
id, name, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) id, name, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil { if onFail != nil {
onFail(t, ctx) onFail(t, ctx)
@ -562,15 +568,19 @@ func ControllerWithSelector(
// Suite Setup // Suite Setup
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type ids struct {
ID string
DriveID string
DriveRootFolderID string
}
type intgTesterSetup struct { type intgTesterSetup struct {
ac api.Client ac api.Client
gockAC api.Client gockAC api.Client
userID string user ids
userDriveID string secondaryUser ids
userDriveRootFolderID string site ids
siteID string secondarySite ids
siteDriveID string
siteDriveRootFolderID string
} }
func newIntegrationTesterSetup(t *testing.T) intgTesterSetup { func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
@ -585,43 +595,58 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
creds, err := a.M365Config() creds, err := a.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
its.ac, err = api.NewClient(creds, control.Defaults()) its.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
its.gockAC, err = mock.NewClient(creds) its.gockAC, err = mock.NewClient(creds)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// user drive its.user = userIDs(t, tconfig.M365UserID(t), its.ac)
its.secondaryUser = userIDs(t, tconfig.SecondaryM365UserID(t), its.ac)
its.userID = tconfig.M365UserID(t) its.site = siteIDs(t, tconfig.M365SiteID(t), its.ac)
its.secondarySite = siteIDs(t, tconfig.SecondaryM365SiteID(t), its.ac)
userDrive, err := its.ac.Users().GetDefaultDrive(ctx, its.userID)
require.NoError(t, err, clues.ToCore(err))
its.userDriveID = ptr.Val(userDrive.GetId())
userDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.userDriveID)
require.NoError(t, err, clues.ToCore(err))
its.userDriveRootFolderID = ptr.Val(userDriveRootFolder.GetId())
its.siteID = tconfig.M365SiteID(t)
// site
siteDrive, err := its.ac.Sites().GetDefaultDrive(ctx, its.siteID)
require.NoError(t, err, clues.ToCore(err))
its.siteDriveID = ptr.Val(siteDrive.GetId())
siteDriveRootFolder, err := its.ac.Drives().GetRootFolder(ctx, its.siteDriveID)
require.NoError(t, err, clues.ToCore(err))
its.siteDriveRootFolderID = ptr.Val(siteDriveRootFolder.GetId())
return its return its
} }
func userIDs(t *testing.T, id string, ac api.Client) ids {
ctx, flush := tester.NewContext(t)
defer flush()
r := ids{ID: id}
drive, err := ac.Users().GetDefaultDrive(ctx, id)
require.NoError(t, err, clues.ToCore(err))
r.DriveID = ptr.Val(drive.GetId())
driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID)
require.NoError(t, err, clues.ToCore(err))
r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId())
return r
}
func siteIDs(t *testing.T, id string, ac api.Client) ids {
ctx, flush := tester.NewContext(t)
defer flush()
r := ids{ID: id}
drive, err := ac.Sites().GetDefaultDrive(ctx, id)
require.NoError(t, err, clues.ToCore(err))
r.DriveID = ptr.Val(drive.GetId())
driveRootFolder, err := ac.Drives().GetRootFolder(ctx, r.DriveID)
require.NoError(t, err, clues.ToCore(err))
r.DriveRootFolderID = ptr.Val(driveRootFolder.GetId())
return r
}
func getTestExtensionFactories() []extensions.CreateItemExtensioner { func getTestExtensionFactories() []extensions.CreateItemExtensioner {
return []extensions.CreateItemExtensioner{ return []extensions.CreateItemExtensioner{
&extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{},

View File

@ -72,7 +72,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() {
osel = selectors.NewOneDriveBackup([]string{userID}) osel = selectors.NewOneDriveBackup([]string{userID})
ws = deeTD.DriveIDFromRepoRef ws = deeTD.DriveIDFromRepoRef
svc = path.OneDriveService svc = path.OneDriveService
opts = control.Defaults() opts = control.DefaultOptions()
) )
osel.Include(selTD.OneDriveBackupFolderScope(osel)) osel.Include(selTD.OneDriveBackupFolderScope(osel))
@ -106,7 +106,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() {
} }
func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() { func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() {
sel := selectors.NewOneDriveRestore([]string{suite.its.userID}) sel := selectors.NewOneDriveRestore([]string{suite.its.user.ID})
ic := func(cs []string) selectors.Selector { ic := func(cs []string) selectors.Selector {
sel.Include(sel.Folders(cs, selectors.PrefixMatch())) sel.Include(sel.Folders(cs, selectors.PrefixMatch()))
@ -117,10 +117,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() {
t *testing.T, t *testing.T,
ctx context.Context, ctx context.Context,
) string { ) string {
d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.userID) d, err := suite.its.ac.Users().GetDefaultDrive(ctx, suite.its.user.ID)
if err != nil { if err != nil {
err = graph.Wrap(ctx, err, "retrieving default user drive"). err = graph.Wrap(ctx, err, "retrieving default user drive").
With("user", suite.its.userID) With("user", suite.its.user.ID)
} }
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -137,8 +137,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_incrementalOneDrive() {
runDriveIncrementalTest( runDriveIncrementalTest(
suite, suite,
suite.its.userID, suite.its.user.ID,
suite.its.userID, suite.its.user.ID,
resource.Users, resource.Users,
path.OneDriveService, path.OneDriveService,
path.FilesCategory, path.FilesCategory,
@ -166,7 +166,7 @@ func runDriveIncrementalTest(
var ( var (
acct = tconfig.NewM365Account(t) acct = tconfig.NewM365Account(t)
opts = control.Defaults() opts = control.DefaultOptions()
mb = evmock.NewBus() mb = evmock.NewBus()
ws = deeTD.DriveIDFromRepoRef ws = deeTD.DriveIDFromRepoRef
@ -683,7 +683,7 @@ func runDriveIncrementalTest(
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.Defaults()) cleanCtrl, err := m365.NewController(ctx, acct, rc, sel.PathService(), control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
bod.ctrl = cleanCtrl bod.ctrl = cleanCtrl
@ -785,7 +785,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
var ( var (
acct = tconfig.NewM365Account(t) acct = tconfig.NewM365Account(t)
opts = control.Defaults() opts = control.DefaultOptions()
mb = evmock.NewBus() mb = evmock.NewBus()
categories = map[path.CategoryType][]string{ categories = map[path.CategoryType][]string{
@ -801,10 +801,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
acct, acct,
resource.Users, resource.Users,
path.OneDriveService, path.OneDriveService,
control.Defaults()) control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.userID) userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
uid := ptr.Val(userable.GetId()) uid := ptr.Val(userable.GetId())
@ -922,7 +922,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() {
osel = selectors.NewOneDriveBackup([]string{userID}) osel = selectors.NewOneDriveBackup([]string{userID})
ws = deeTD.DriveIDFromRepoRef ws = deeTD.DriveIDFromRepoRef
svc = path.OneDriveService svc = path.OneDriveService
opts = control.Defaults() opts = control.DefaultOptions()
) )
opts.ItemExtensionFactory = getTestExtensionFactories() opts.ItemExtensionFactory = getTestExtensionFactories()
@ -982,17 +982,17 @@ func (suite *OneDriveRestoreIntgSuite) SetupSuite() {
} }
func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() { func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveWithAdvancedOptions() {
sel := selectors.NewOneDriveBackup([]string{suite.its.userID}) sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID})
sel.Include(selTD.OneDriveBackupFolderScope(sel)) sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = suite.its.userID sel.DiscreteOwner = suite.its.user.ID
runDriveRestoreWithAdvancedOptions( runDriveRestoreWithAdvancedOptions(
suite.T(), suite.T(),
suite, suite,
suite.its.ac, suite.its.ac,
sel.Selector, sel.Selector,
suite.its.userDriveID, suite.its.user.DriveID,
suite.its.userDriveRootFolderID) suite.its.user.DriveRootFolderID)
} }
func runDriveRestoreWithAdvancedOptions( func runDriveRestoreWithAdvancedOptions(
@ -1009,7 +1009,7 @@ func runDriveRestoreWithAdvancedOptions(
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.Defaults() opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup)
@ -1250,3 +1250,173 @@ func runDriveRestoreWithAdvancedOptions(
assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy") assert.Subset(t, maps.Keys(currentFileIDs), maps.Keys(fileIDs), "original item should exist after copy")
}) })
} }
func (suite *OneDriveRestoreIntgSuite) TestRestore_Run_onedriveAlternateProtectedResource() {
sel := selectors.NewOneDriveBackup([]string{suite.its.user.ID})
sel.Include(selTD.OneDriveBackupFolderScope(sel))
sel.DiscreteOwner = suite.its.user.ID
runDriveRestoreToAlternateProtectedResource(
suite.T(),
suite,
suite.its.ac,
sel.Selector,
suite.its.user,
suite.its.secondaryUser)
}
func runDriveRestoreToAlternateProtectedResource(
t *testing.T,
suite tester.Suite,
ac api.Client,
sel selectors.Selector, // owner should match 'from', both Restore and Backup types work.
from, to ids,
) {
ctx, flush := tester.NewContext(t)
defer flush()
// a backup is required to run restores
var (
mb = evmock.NewBus()
opts = control.DefaultOptions()
)
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup)
defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false)
var (
restoreCfg = ctrlTD.DefaultRestoreConfig("drive_restore_to_resource")
fromCollisionKeys map[string]api.DriveItemIDType
fromItemIDs map[string]api.DriveItemIDType
acd = ac.Drives()
)
// first restore to the 'from' resource
suite.Run("restore original resource", func() {
mb = evmock.NewBus()
fromCtr := count.New()
driveID := from.DriveID
rootFolderID := from.DriveRootFolderID
restoreCfg.OnCollision = control.Copy
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
fromCtr,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro, mb, false)
// get all files in folder, use these as the base
// set of files to compare against.
fromItemIDs, fromCollisionKeys = getDriveCollKeysAndItemIDs(
t,
ctx,
acd,
driveID,
rootFolderID,
restoreCfg.Location,
selTD.TestFolderName)
})
// then restore to the 'to' resource
var (
toCollisionKeys map[string]api.DriveItemIDType
toItemIDs map[string]api.DriveItemIDType
)
suite.Run("restore to alternate resource", func() {
mb = evmock.NewBus()
toCtr := count.New()
driveID := to.DriveID
rootFolderID := to.DriveRootFolderID
restoreCfg.ProtectedResource = to.ID
ro, _ := prepNewTestRestoreOp(
t,
ctx,
bod.st,
bo.Results.BackupID,
mb,
toCtr,
sel,
opts,
restoreCfg)
runAndCheckRestore(t, ctx, &ro, mb, false)
// get all files in folder, use these as the base
// set of files to compare against.
toItemIDs, toCollisionKeys = getDriveCollKeysAndItemIDs(
t,
ctx,
acd,
driveID,
rootFolderID,
restoreCfg.Location,
selTD.TestFolderName)
})
// compare restore results
assert.Equal(t, len(fromItemIDs), len(toItemIDs))
assert.ElementsMatch(t, maps.Keys(fromCollisionKeys), maps.Keys(toCollisionKeys))
}
type GetItemsKeysAndFolderByNameer interface {
GetItemIDsInContainer(
ctx context.Context,
driveID, containerID string,
) (map[string]api.DriveItemIDType, error)
GetFolderByName(
ctx context.Context,
driveID, parentFolderID, folderName string,
) (models.DriveItemable, error)
GetItemsInContainerByCollisionKey(
ctx context.Context,
driveID, containerID string,
) (map[string]api.DriveItemIDType, error)
}
func getDriveCollKeysAndItemIDs(
t *testing.T,
ctx context.Context, //revive:disable-line:context-as-argument
gikafbn GetItemsKeysAndFolderByNameer,
driveID, parentContainerID string,
containerNames ...string,
) (map[string]api.DriveItemIDType, map[string]api.DriveItemIDType) {
var (
c models.DriveItemable
err error
cID string
)
for _, cn := range containerNames {
pcid := parentContainerID
if len(cID) != 0 {
pcid = cID
}
c, err = gikafbn.GetFolderByName(ctx, driveID, pcid, cn)
require.NoError(t, err, clues.ToCore(err))
cID = ptr.Val(c.GetId())
}
itemIDs, err := gikafbn.GetItemIDsInContainer(ctx, driveID, cID)
require.NoError(t, err, clues.ToCore(err))
collisionKeys, err := gikafbn.GetItemsInContainerByCollisionKey(ctx, driveID, cID)
require.NoError(t, err, clues.ToCore(err))
return itemIDs, collisionKeys
}

View File

@ -49,7 +49,7 @@ func (suite *SharePointBackupIntgSuite) SetupSuite() {
} }
func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
sel := selectors.NewSharePointRestore([]string{suite.its.siteID}) sel := selectors.NewSharePointRestore([]string{suite.its.site.ID})
ic := func(cs []string) selectors.Selector { ic := func(cs []string) selectors.Selector {
sel.Include(sel.LibraryFolders(cs, selectors.PrefixMatch())) sel.Include(sel.LibraryFolders(cs, selectors.PrefixMatch()))
@ -60,10 +60,10 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
t *testing.T, t *testing.T,
ctx context.Context, ctx context.Context,
) string { ) string {
d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.siteID) d, err := suite.its.ac.Sites().GetDefaultDrive(ctx, suite.its.site.ID)
if err != nil { if err != nil {
err = graph.Wrap(ctx, err, "retrieving default site drive"). err = graph.Wrap(ctx, err, "retrieving default site drive").
With("site", suite.its.siteID) With("site", suite.its.site.ID)
} }
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -80,8 +80,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
runDriveIncrementalTest( runDriveIncrementalTest(
suite, suite,
suite.its.siteID, suite.its.site.ID,
suite.its.userID, suite.its.user.ID,
resource.Sites, resource.Sites,
path.SharePointService, path.SharePointService,
path.LibrariesCategory, path.LibrariesCategory,
@ -91,7 +91,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
true) true)
} }
func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic() {
t := suite.T() t := suite.T()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
@ -99,8 +99,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() {
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) sel = selectors.NewSharePointBackup([]string{suite.its.site.ID})
opts = control.Defaults() opts = control.DefaultOptions()
) )
sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Include(selTD.SharePointBackupFolderScope(sel))
@ -116,7 +116,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() {
bod.sw, bod.sw,
&bo, &bo,
bod.sel, bod.sel,
suite.its.siteID, bod.sel.ID(),
path.LibrariesCategory) path.LibrariesCategory)
} }
@ -128,8 +128,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() {
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.siteID}) sel = selectors.NewSharePointBackup([]string{suite.its.site.ID})
opts = control.Defaults() opts = control.DefaultOptions()
tenID = tconfig.M365TenantID(t) tenID = tconfig.M365TenantID(t)
svc = path.SharePointService svc = path.SharePointService
ws = deeTD.DriveIDFromRepoRef ws = deeTD.DriveIDFromRepoRef
@ -150,7 +150,7 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() {
bod.sw, bod.sw,
&bo, &bo,
bod.sel, bod.sel,
suite.its.siteID, bod.sel.ID(),
path.LibrariesCategory) path.LibrariesCategory)
bID := bo.Results.BackupID bID := bo.Results.BackupID
@ -201,18 +201,33 @@ func (suite *SharePointRestoreIntgSuite) SetupSuite() {
} }
func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() { func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointWithAdvancedOptions() {
sel := selectors.NewSharePointBackup([]string{suite.its.siteID}) sel := selectors.NewSharePointBackup([]string{suite.its.site.ID})
sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Include(selTD.SharePointBackupFolderScope(sel))
sel.Filter(sel.Library("documents")) sel.Filter(sel.Library("documents"))
sel.DiscreteOwner = suite.its.siteID sel.DiscreteOwner = suite.its.site.ID
runDriveRestoreWithAdvancedOptions( runDriveRestoreWithAdvancedOptions(
suite.T(), suite.T(),
suite, suite,
suite.its.ac, suite.its.ac,
sel.Selector, sel.Selector,
suite.its.siteDriveID, suite.its.site.DriveID,
suite.its.siteDriveRootFolderID) suite.its.site.DriveRootFolderID)
}
func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointAlternateProtectedResource() {
sel := selectors.NewSharePointBackup([]string{suite.its.site.ID})
sel.Include(selTD.SharePointBackupFolderScope(sel))
sel.Filter(sel.Library("documents"))
sel.DiscreteOwner = suite.its.site.ID
runDriveRestoreToAlternateProtectedResource(
suite.T(),
suite,
suite.its.ac,
sel.Selector,
suite.its.site,
suite.its.secondarySite)
} }
func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() { func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives() {
@ -229,7 +244,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives
rc.OnCollision = control.Copy rc.OnCollision = control.Copy
// create a new drive // create a new drive
md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.siteID, rc.Location) md, err := suite.its.ac.Lists().PostDrive(ctx, suite.its.site.ID, rc.Location)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
driveID := ptr.Val(md.GetId()) driveID := ptr.Val(md.GetId())
@ -260,14 +275,14 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives
// run a backup // run a backup
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.Defaults() opts = control.DefaultOptions()
graphClient = suite.its.ac.Stable.Client() graphClient = suite.its.ac.Stable.Client()
) )
bsel := selectors.NewSharePointBackup([]string{suite.its.siteID}) bsel := selectors.NewSharePointBackup([]string{suite.its.site.ID})
bsel.Include(selTD.SharePointBackupFolderScope(bsel)) bsel.Include(selTD.SharePointBackupFolderScope(bsel))
bsel.Filter(bsel.Library(rc.Location)) bsel.Filter(bsel.Library(rc.Location))
bsel.DiscreteOwner = suite.its.siteID bsel.DiscreteOwner = suite.its.site.ID
bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup)
defer bod.close(t, ctx) defer bod.close(t, ctx)
@ -367,7 +382,7 @@ func (suite *SharePointRestoreIntgSuite) TestRestore_Run_sharepointDeletedDrives
pgr := suite.its.ac. pgr := suite.its.ac.
Drives(). Drives().
NewSiteDrivePager(suite.its.siteID, []string{"id", "name"}) NewSiteDrivePager(suite.its.site.ID, []string{"id", "name"})
drives, err := api.GetAllDrives(ctx, pgr, false, -1) drives, err := api.GetAllDrives(ctx, pgr, false, -1)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))

View File

@ -23,6 +23,7 @@ const (
// M365 config // M365 config
TestCfgAzureTenantID = "azure_tenantid" TestCfgAzureTenantID = "azure_tenantid"
TestCfgSecondarySiteID = "secondarym365siteid"
TestCfgSiteID = "m365siteid" TestCfgSiteID = "m365siteid"
TestCfgSiteURL = "m365siteurl" TestCfgSiteURL = "m365siteurl"
TestCfgUserID = "m365userid" TestCfgUserID = "m365userid"
@ -36,13 +37,14 @@ const (
// test specific env vars // test specific env vars
const ( const (
EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID"
EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS"
EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID"
EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL" EnvCorsoM365TestSiteURL = "CORSO_M365_TEST_SITE_URL"
EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID"
EnvCorsoSecondaryM365TestSiteID = "CORSO_SECONDARY_M365_TEST_SITE_ID"
EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID"
EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID" EnvCorsoTertiaryM365TestUserID = "CORSO_TERTIARY_M365_TEST_USER_ID"
EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID"
EnvCorsoM365LoadTestOrgUsers = "CORSO_M365_LOAD_TEST_ORG_USERS"
EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE" EnvCorsoTestConfigFilePath = "CORSO_TEST_CONFIG_FILE"
EnvCorsoUnlicensedM365TestUserID = "CORSO_M365_TEST_UNLICENSED_USER" EnvCorsoUnlicensedM365TestUserID = "CORSO_M365_TEST_UNLICENSED_USER"
) )
@ -147,13 +149,19 @@ func ReadTestConfig() (map[string]string, error) {
TestCfgSiteID, TestCfgSiteID,
os.Getenv(EnvCorsoM365TestSiteID), os.Getenv(EnvCorsoM365TestSiteID),
vpr.GetString(TestCfgSiteID), vpr.GetString(TestCfgSiteID),
"10rqc2.sharepoint.com,4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be") "4892edf5-2ebf-46be-a6e5-a40b2cbf1c1a,38ab6d06-fc82-4417-af93-22d8733c22be")
fallbackTo( fallbackTo(
testEnv, testEnv,
TestCfgSiteURL, TestCfgSiteURL,
os.Getenv(EnvCorsoM365TestSiteURL), os.Getenv(EnvCorsoM365TestSiteURL),
vpr.GetString(TestCfgSiteURL), vpr.GetString(TestCfgSiteURL),
"https://10rqc2.sharepoint.com/sites/CorsoCI") "https://10rqc2.sharepoint.com/sites/CorsoCI")
fallbackTo(
testEnv,
TestCfgSecondarySiteID,
os.Getenv(EnvCorsoSecondaryM365TestSiteID),
vpr.GetString(TestCfgSecondarySiteID),
"053684d8-ca6c-4376-a03e-2567816bb091,9b3e9abe-6a5e-4084-8b44-ea5a356fe02c")
fallbackTo( fallbackTo(
testEnv, testEnv,
TestCfgUnlicensedUserID, TestCfgUnlicensedUserID,

View File

@ -198,6 +198,17 @@ func GetM365SiteID(ctx context.Context) string {
return strings.ToLower(cfg[TestCfgSiteID]) return strings.ToLower(cfg[TestCfgSiteID])
} }
// SecondaryM365SiteID returns a siteID string representing the secondarym365SiteID described
// by either the env var CORSO_SECONDARY_M365_TEST_SITE_ID, the corso_test.toml config
// file or the default value (in that order of priority). The default is a
// last-attempt fallback that will only work on alcion's testing org.
func SecondaryM365SiteID(t *testing.T) string {
cfg, err := ReadTestConfig()
require.NoError(t, err, "retrieving secondary m365 site id from test configuration: %+v", clues.ToCore(err))
return strings.ToLower(cfg[TestCfgSecondarySiteID])
}
// UnlicensedM365UserID returns an userID string representing the m365UserID // UnlicensedM365UserID returns an userID string representing the m365UserID
// described by either the env var CORSO_M365_TEST_UNLICENSED_USER, the // described by either the env var CORSO_M365_TEST_UNLICENSED_USER, the
// corso_test.toml config file or the default value (in that order of priority). // corso_test.toml config file or the default value (in that order of priority).

View File

@ -15,7 +15,6 @@ type Options struct {
ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"`
Parallelism Parallelism `json:"parallelism"` Parallelism Parallelism `json:"parallelism"`
Repo repository.Options `json:"repo"` Repo repository.Options `json:"repo"`
RestorePermissions bool `json:"restorePermissions"`
SkipReduce bool `json:"skipReduce"` SkipReduce bool `json:"skipReduce"`
ToggleFeatures Toggles `json:"toggleFeatures"` ToggleFeatures Toggles `json:"toggleFeatures"`
} }
@ -38,8 +37,8 @@ const (
BestEffort FailurePolicy = "best-effort" BestEffort FailurePolicy = "best-effort"
) )
// Defaults provides an Options with the default values set. // DefaultOptions provides an Options with the default values set.
func Defaults() Options { func DefaultOptions() Options {
return Options{ return Options{
FailureHandling: FailAfterRecovery, FailureHandling: FailAfterRecovery,
DeltaPageSize: 500, DeltaPageSize: 500,

View File

@ -61,6 +61,10 @@ type RestoreConfig struct {
// up. // up.
// Defaults to empty. // Defaults to empty.
Drive string `json:"drive"` Drive string `json:"drive"`
// IncludePermissions toggles whether the restore will include the original
// folder- and item-level permissions.
IncludePermissions bool `json:"includePermissions"`
} }
func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig { func DefaultRestoreConfig(timeFormat dttm.TimeFormat) RestoreConfig {
@ -120,10 +124,11 @@ func (rc RestoreConfig) marshal() string {
func (rc RestoreConfig) concealed() RestoreConfig { func (rc RestoreConfig) concealed() RestoreConfig {
return RestoreConfig{ return RestoreConfig{
OnCollision: rc.OnCollision, OnCollision: rc.OnCollision,
ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(), ProtectedResource: clues.Hide(rc.ProtectedResource).Conceal(),
Location: path.LoggableDir(rc.Location), Location: path.LoggableDir(rc.Location),
Drive: clues.Hide(rc.Drive).Conceal(), Drive: clues.Hide(rc.Drive).Conceal(),
IncludePermissions: rc.IncludePermissions,
} }
} }

View File

@ -335,7 +335,7 @@ func (r repository) NewBackupWithLookup(
return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365")
} }
ownerID, ownerName, err := ctrl.PopulateOwnerIDAndNamesFrom(ctx, sel.DiscreteOwner, ins) ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins)
if err != nil { if err != nil {
return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details")
} }

View File

@ -60,7 +60,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() {
st, err := test.storage() st, err := test.storage()
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
_, err = Initialize(ctx, test.account, st, control.Defaults()) _, err = Initialize(ctx, test.account, st, control.DefaultOptions())
test.errCheck(t, err, clues.ToCore(err)) test.errCheck(t, err, clues.ToCore(err))
}) })
} }
@ -94,7 +94,7 @@ func (suite *RepositoryUnitSuite) TestConnect() {
st, err := test.storage() st, err := test.storage()
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
_, err = Connect(ctx, test.account, st, "not_found", control.Defaults()) _, err = Connect(ctx, test.account, st, "not_found", control.DefaultOptions())
test.errCheck(t, err, clues.ToCore(err)) test.errCheck(t, err, clues.ToCore(err))
}) })
} }
@ -137,7 +137,7 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
defer flush() defer flush()
st := test.storage(t) st := test.storage(t)
r, err := Initialize(ctx, test.account, st, control.Defaults()) r, err := Initialize(ctx, test.account, st, control.DefaultOptions())
if err == nil { if err == nil {
defer func() { defer func() {
err := r.Close(ctx) err := r.Close(ctx)
@ -186,11 +186,11 @@ func (suite *RepositoryIntegrationSuite) TestConnect() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// now re-connect // now re-connect
_, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.Defaults()) _, err = Connect(ctx, account.Account{}, st, repo.GetID(), control.DefaultOptions())
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
} }
@ -203,7 +203,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, account.Account{}, st, control.Defaults()) r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
oldID := r.GetID() oldID := r.GetID()
@ -212,7 +212,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// now re-connect // now re-connect
r, err = Connect(ctx, account.Account{}, st, oldID, control.Defaults()) r, err = Connect(ctx, account.Account{}, st, oldID, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, oldID, r.GetID()) assert.Equal(t, oldID, r.GetID())
} }
@ -228,7 +228,7 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults()) r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
userID := tconfig.M365UserID(t) userID := tconfig.M365UserID(t)
@ -250,7 +250,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults()) r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg) ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg)
@ -269,7 +269,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.Defaults()) r, err := Initialize(ctx, acct, st, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{})
@ -286,7 +286,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() {
// need to initialize the repository before we can test connecting to it. // need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t) st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.Defaults()) repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
require.NoError(t, err) require.NoError(t, err)
// now re-connect // now re-connect
@ -308,14 +308,14 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
{ {
name: "default options", name: "default options",
opts: func() control.Options { opts: func() control.Options {
return control.Defaults() return control.DefaultOptions()
}, },
expectedLen: 0, expectedLen: 0,
}, },
{ {
name: "options with an extension factory", name: "options with an extension factory",
opts: func() control.Options { opts: func() control.Options {
o := control.Defaults() o := control.DefaultOptions()
o.ItemExtensionFactory = append( o.ItemExtensionFactory = append(
o.ItemExtensionFactory, o.ItemExtensionFactory,
&extensions.MockItemExtensionFactory{}) &extensions.MockItemExtensionFactory{})
@ -327,7 +327,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
{ {
name: "options with multiple extension factories", name: "options with multiple extension factories",
opts: func() control.Options { opts: func() control.Options {
o := control.Defaults() o := control.DefaultOptions()
f := []extensions.CreateItemExtensioner{ f := []extensions.CreateItemExtensioner{
&extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{},
&extensions.MockItemExtensionFactory{}, &extensions.MockItemExtensionFactory{},

View File

@ -97,7 +97,7 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
creds, err := a.M365Config() creds, err := a.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
its.ac, err = api.NewClient(creds, control.Defaults()) its.ac, err = api.NewClient(creds, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
its.gockAC, err = mock.NewClient(creds) its.gockAC, err = mock.NewClient(creds)

View File

@ -143,12 +143,16 @@ func (suite *SitesIntgSuite) TestSites_GetByID() {
var ( var (
t = suite.T() t = suite.T()
siteID = tconfig.M365SiteID(t) siteID = tconfig.M365SiteID(t)
host = strings.Split(siteID, ",")[0] parts = strings.Split(siteID, ",")
shortID = strings.TrimPrefix(siteID, host+",") uuids = siteID
siteURL = tconfig.M365SiteURL(t) siteURL = tconfig.M365SiteURL(t)
modifiedSiteURL = siteURL + "foo" modifiedSiteURL = siteURL + "foo"
) )
if len(parts) == 3 {
uuids = strings.Join(parts[1:], ",")
}
sitesAPI := suite.its.ac.Sites() sitesAPI := suite.its.ac.Sites()
table := []struct { table := []struct {
@ -165,7 +169,7 @@ func (suite *SitesIntgSuite) TestSites_GetByID() {
}, },
{ {
name: "2 part id", name: "2 part id",
id: shortID, id: uuids,
expectErr: func(t *testing.T, err error) { expectErr: func(t *testing.T, err error) {
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
}, },
@ -191,13 +195,6 @@ func (suite *SitesIntgSuite) TestSites_GetByID() {
assert.NoError(t, err, clues.ToCore(err)) assert.NoError(t, err, clues.ToCore(err))
}, },
}, },
{
name: "host only",
id: host,
expectErr: func(t *testing.T, err error) {
assert.NoError(t, err, clues.ToCore(err))
},
},
{ {
name: "malformed url", name: "malformed url",
id: "barunihlda", id: "barunihlda",

View File

@ -329,7 +329,7 @@ func makeAC(
return api.Client{}, clues.Wrap(err, "getting m365 account creds") return api.Client{}, clues.Wrap(err, "getting m365 account creds")
} }
cli, err := api.NewClient(creds, control.Defaults()) cli, err := api.NewClient(creds, control.DefaultOptions())
if err != nil { if err != nil {
return api.Client{}, clues.Wrap(err, "constructing api client") return api.Client{}, clues.Wrap(err, "constructing api client")
} }

View File

@ -10,7 +10,7 @@ manner to a new folder. When you need more control over the results you can
use the advanced configuration options to change where and how your data use the advanced configuration options to change where and how your data
gets restored. gets restored.
## Destination ## Restore to target folder
The `--destination` flag lets you select the top-level folder where Corso will The `--destination` flag lets you select the top-level folder where Corso will
write all of the restored data. write all of the restored data.
@ -18,7 +18,7 @@ write all of the restored data.
### The default destination ### The default destination
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd` `corso restore onedrive --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
If the flag isn't provided, Corso will create a new folder with a standard name: If the flag isn't provided, Corso will create a new folder with a standard name:
@ -29,7 +29,7 @@ data integrity then this is always the safest option.
### An alternate destination ### An alternate destination
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd --destination /my-latest-restore` `corso restore onedrive --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
When a destination is manually specified, all restored will appear in that top-level When a destination is manually specified, all restored will appear in that top-level
@ -41,14 +41,14 @@ folder multiple times.
### The original location ### The original location
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd --destination /` `corso restore onedrive --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
You can restore items back to their original location by setting the destination You can restore items back to their original location by setting the destination
to `/`. This skips the creation of a top-level folder, and all restored items will to `/`. This skips the creation of a top-level folder, and all restored items will
appear back in their location at the time of backup. appear back in their location at the time of backup.
### Limitations ### Destination Limitations
* Destination won't create N-depth folder structures. `--destination a/b/c` * Destination won't create N-depth folder structures. `--destination a/b/c`
doesn't create three folders; it creates a single, top-level folder named `a/b/c`. doesn't create three folders; it creates a single, top-level folder named `a/b/c`.
@ -79,19 +79,19 @@ it still collides.
Collisions can be handled with three different configurations: `Skip`, `Copy`, Collisions can be handled with three different configurations: `Skip`, `Copy`,
and `Replace`. and `Replace`.
## Skip (default) ### Skip (default)
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd --collisions skip --destination /` `corso restore onedrive --collisions skip --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
When a collision is identified, the item is skipped and When a collision is identified, the item is skipped and
no restore is attempted. no restore is attempted.
## Copy ### Copy
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd --collisions copy --destination /my-latest-restore` `corso restore onedrive --collisions copy --destination /my-latest-restore --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
Item collisions create a copy of the item in the backup. The copy holds the backup Item collisions create a copy of the item in the backup. The copy holds the backup
@ -99,12 +99,31 @@ version of the item, leaving the current version unchanged. If necessary, change
item properties (such as filenames) to avoid additional collisions. Eg: item properties (such as filenames) to avoid additional collisions. Eg:
the copy of`reports.txt` is named `reports 1.txt`. the copy of`reports.txt` is named `reports 1.txt`.
## Replace ### Replace
<CodeBlock language="bash">{ <CodeBlock language="bash">{
`corso restore onedrive --backup abcd --collisions replace --destination /` `corso restore onedrive --collisions replace --destination / --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock> }</CodeBlock>
Collisions will entirely replace the current version of the item with the backup Collisions will entirely replace the current version of the item with the backup
version. If multiple existing items collide with the backup item, only one of the version. If multiple existing items collide with the backup item, only one of the
existing items is replaced. existing items is replaced.
## Restore to target resource
The `--to-resource` flag lets you select which resource will receive the restored data.
A resource can be a mailbox, user, sharepoint site, or other owner of data.
When restoring to a target resource, all other restore configuration behaves normally.
Data is restored into the default folder: `Corso_Restore_<current-date-time>` (unless a
`--destination` flag is added). When restoring in-place, collision policies are followed.
<CodeBlock language="bash">{
`corso restore onedrive --to-resource adelev@alcion.ai --backup a422895c-c20c-4b06-883d-b866db9f86ef`
}</CodeBlock>
### Resource Limitations
* The resource must exist. Corso won't create new mailboxes, users, or sites.
* The resource must have access to the service being restored. No restore will be
performed for an unlicensed resource.