larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow setting retention parameters on repo init (#3903)

Browse Source 
Take in information about retention so that
when the repo is initialized we can configure
retention

Not currently exposed by CLI layer

**Requires changing the interface for repo
init so SDK consumers will require changes**

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #3519

#### Test Plan

- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2023-07-28 08:39:17 -07:00 committed by GitHub
parent 8f30db4f6e
commit 1e126bd2af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 203 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/cli/backup/help_e2e_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
"github.com/alcionai/corso/src/pkg/storage/testdata"
@ -47,7 +48,12 @@ func prepM365Test(
vpr, cfgFP := tconfig.MakeTempTestConfigClone(t, force)
ctx = config.SetViper(ctx, vpr)
repo, err := repository.Initialize(ctx, acct, st, control.DefaultOptions())
repo, err := repository.Initialize(
ctx,
acct,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
return acct, st, repo, vpr, recorder, cfgFP

9
src/cli/repo/s3.go
View File

@ -15,6 +15,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/pkg/account"
rep "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -158,7 +159,13 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config"))
}
r, err := repository.Initialize(ctx, cfg.Account, cfg.Storage, opt)
// TODO(ashmrtn): Wire to flags for retention during repo init.
r, err := repository.Initialize(
ctx,
cfg.Account,
cfg.Storage,
opt,
rep.Retention{})
if err != nil {
if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) {
return nil

8
src/cli/repo/s3_e2e_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
)
@ -200,7 +201,12 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
ctx = config.SetViper(ctx, vpr)
// init the repo first
_, err = repository.Initialize(ctx, account.Account{}, st, control.DefaultOptions())
_, err = repository.Initialize(
ctx,
account.Account{},
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
// then test it

8
src/cli/restore/exchange_e2e_test.go
View File

@ -20,6 +20,7 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
@ -83,7 +84,12 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() {
)
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
suite.repo, err = repository.Initialize(
ctx,
suite.acct,
suite.st,
control.Options{},
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
suite.backupOps = make(map[path.CategoryType]operations.BackupOperation)

30
src/internal/kopia/conn.go
View File

@ -74,7 +74,11 @@ func NewConn(s storage.Storage) *conn {
}
}
func (w *conn) Initialize(ctx context.Context, opts repository.Options) error {
func (w *conn) Initialize(
ctx context.Context,
opts repository.Options,
retentionOpts repository.Retention,
) error {
bst, err := blobStoreByProvider(ctx, opts, w.storage)
if err != nil {
return clues.Wrap(err, "initializing storage")
@ -86,8 +90,23 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error {
return clues.Stack(err).WithClues(ctx)
}
// todo - issue #75: nil here should be a storage.NewRepoOptions()
if err = repo.Initialize(ctx, bst, nil, cfg.CorsoPassphrase); err != nil {
rOpts := retention.NewOpts()
if err := rOpts.Set(retentionOpts); err != nil {
return clues.Wrap(err, "setting retention configuration").WithClues(ctx)
}
blobCfg, _, err := rOpts.AsConfigs(ctx)
if err != nil {
return clues.Stack(err)
}
// Minimal config for retention if caller requested it.
kopiaOpts := repo.NewRepositoryOptions{
RetentionMode: blobCfg.RetentionMode,
RetentionPeriod: blobCfg.RetentionPeriod,
}
if err = repo.Initialize(ctx, bst, &kopiaOpts, cfg.CorsoPassphrase); err != nil {
if errors.Is(err, repo.ErrAlreadyInitialized) {
return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx)
}
@ -111,7 +130,10 @@ func (w *conn) Initialize(ctx context.Context, opts repository.Options) error {
return clues.Stack(err).WithClues(ctx)
}
return nil
// Calling with all parameters here will set extend object locks for
// maintenance. Parameters for actual retention should have been set during
// initialization and won't be updated again.
return clues.Stack(w.setRetentionParameters(ctx, retentionOpts)).OrNil()
}
func (w *conn) Connect(ctx context.Context, opts repository.Options) error {

82
src/internal/kopia/conn_test.go
View File

@ -7,12 +7,15 @@ import (
"time"
"github.com/alcionai/clues"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/storage"
@ -26,7 +29,7 @@ func openKopiaRepo(
st := storeTD.NewPrefixedS3Storage(t)
k := NewConn(st)
if err := k.Initialize(ctx, repository.Options{}); err != nil {
if err := k.Initialize(ctx, repository.Options{}, repository.Retention{}); err != nil {
return nil, err
}
@ -82,13 +85,13 @@ func (suite *WrapperIntegrationSuite) TestRepoExistsError() {
st := storeTD.NewPrefixedS3Storage(t)
k := NewConn(st)
err := k.Initialize(ctx, repository.Options{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
err = k.Close(ctx)
require.NoError(t, err, clues.ToCore(err))
err = k.Initialize(ctx, repository.Options{})
err = k.Initialize(ctx, repository.Options{}, repository.Retention{})
assert.Error(t, err, clues.ToCore(err))
assert.ErrorIs(t, err, ErrorRepoAlreadyExists)
}
@ -103,7 +106,7 @@ func (suite *WrapperIntegrationSuite) TestBadProviderErrors() {
st.Provider = storage.ProviderUnknown
k := NewConn(st)
err := k.Initialize(ctx, repository.Options{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
assert.Error(t, err, clues.ToCore(err))
}
@ -413,7 +416,7 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
st := storeTD.NewPrefixedS3Storage(t)
k := NewConn(st)
err := k.Initialize(ctx, opts)
err := k.Initialize(ctx, opts, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
kopiaOpts := k.ClientOptions()
@ -453,3 +456,72 @@ func (suite *WrapperIntegrationSuite) TestSetUserAndHost() {
err = k.Close(ctx)
assert.NoError(t, err, clues.ToCore(err))
}
// ---------------
// integration tests that require object locking to be enabled on the bucket.
// ---------------
type ConnRetentionIntegrationSuite struct {
tester.Suite
}
func TestConnRetentionIntegrationSuite(t *testing.T) {
suite.Run(t, &ConnRetentionIntegrationSuite{
Suite: tester.NewRetentionSuite(
t,
[][]string{storeTD.AWSStorageCredEnvs},
),
})
}
// Test that providing retention doesn't change anything but retention values
// from the default values that kopia uses.
func (suite *ConnRetentionIntegrationSuite) TestInitWithAndWithoutRetention() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
st1 := storeTD.NewPrefixedS3Storage(t)
k1 := NewConn(st1)
err := k1.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, "initializing repo 1: %v", clues.ToCore(err))
st2 := storeTD.NewPrefixedS3Storage(t)
k2 := NewConn(st2)
err = k2.Initialize(
ctx,
repository.Options{},
repository.Retention{
Mode: ptr.To(repository.GovernanceRetention),
Duration: ptr.To(time.Hour * 48),
Extend: ptr.To(true),
})
require.NoError(t, err, "initializing repo 2: %v", clues.ToCore(err))
dr1, ok := k1.Repository.(repo.DirectRepository)
require.True(t, ok, "getting direct repo 1")
dr2, ok := k2.Repository.(repo.DirectRepository)
require.True(t, ok, "getting direct repo 2")
format1 := dr1.FormatManager().ScrubbedContentFormat()
format2 := dr2.FormatManager().ScrubbedContentFormat()
assert.Equal(t, format1, format2)
blobCfg1, err := dr1.FormatManager().BlobCfgBlob()
require.NoError(t, err, "getting blob config 1: %v", clues.ToCore(err))
blobCfg2, err := dr2.FormatManager().BlobCfgBlob()
require.NoError(t, err, "getting retention config 2: %v", clues.ToCore(err))
assert.NotEqual(t, blobCfg1, blobCfg2)
// Check to make sure retention not enabled unexpectedly.
checkRetentionParams(t, ctx, k1, blob.RetentionMode(""), 0, assert.False)
// Some checks to make sure retention was fully initialized as expected.
checkRetentionParams(t, ctx, k2, blob.Governance, time.Hour*48, assert.True)
}

2
src/internal/kopia/model_store_test.go
View File

@ -808,7 +808,7 @@ func openConnAndModelStore(
st := storeTD.NewPrefixedS3Storage(t)
c := NewConn(st)
err := c.Initialize(ctx, repository.Options{})
err := c.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
defer func() {

2
src/internal/operations/maintenance_test.go
View File

@ -40,7 +40,7 @@ func (suite *MaintenanceOpIntegrationSuite) TestRepoMaintenance() {
ctx, flush := tester.NewContext(t)
defer flush()
err := k.Initialize(ctx, repository.Options{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
kw, err := kopia.NewWrapper(k)

2
src/internal/operations/restore_test.go
View File

@ -243,7 +243,7 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() {
suite.acct = tconfig.NewM365Account(t)
err := k.Initialize(ctx, repository.Options{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
suite.kopiaCloser = func(ctx context.Context) {

2
src/internal/operations/test/helper_test.go
View File

@ -102,7 +102,7 @@ func prepNewTestBackupOp(
k := kopia.NewConn(bod.st)
err := k.Initialize(ctx, repository.Options{})
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
defer func() {

3
src/internal/streamstore/collectables_test.go
View File

@ -44,7 +44,8 @@ func (suite *StreamStoreIntgSuite) SetupSubTest() {
st := storeTD.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx, repository.Options{}))
err := k.Initialize(ctx, repository.Options{}, repository.Retention{})
require.NoError(t, err, clues.ToCore(err))
suite.kcloser = func() { k.Close(ctx) }

3
src/pkg/repository/loadtest/repository_load_test.go
View File

@ -21,6 +21,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -102,7 +103,7 @@ func initM365Repo(t *testing.T) (
FailureHandling: control.FailFast,
}
repo, err := repository.Initialize(ctx, ac, st, opts)
repo, err := repository.Initialize(ctx, ac, st, opts, ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
return ctx, repo, ac, st

12
src/pkg/repository/repository.go
View File

@ -25,7 +25,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
rep "github.com/alcionai/corso/src/pkg/control/repository"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -82,7 +82,7 @@ type Repository interface {
) (operations.ExportOperation, error)
NewMaintenance(
ctx context.Context,
mOpts rep.Maintenance,
mOpts ctrlRepo.Maintenance,
) (operations.MaintenanceOperation, error)
DeleteBackup(ctx context.Context, id string) error
BackupGetter
@ -117,7 +117,8 @@ func (r repository) GetID() string {
// - validate the m365 account & secrets
// - connect to the m365 account to ensure communication capability
// - validate the provider config & secrets
// - initialize the kopia repo with the provider
// - initialize the kopia repo with the provider and retention parameters
// - update maintenance retention parameters as needed
// - store the configuration details
// - connect to the provider
// - return the connected repository
@ -126,6 +127,7 @@ func Initialize(
acct account.Account,
s storage.Storage,
opts control.Options,
retentionOpts ctrlRepo.Retention,
) (repo Repository, err error) {
ctx = clues.Add(
ctx,
@ -140,7 +142,7 @@ func Initialize(
}()
kopiaRef := kopia.NewConn(s)
if err := kopiaRef.Initialize(ctx, opts.Repo); err != nil {
if err := kopiaRef.Initialize(ctx, opts.Repo, retentionOpts); err != nil {
// replace common internal errors so that sdk users can check results with errors.Is()
if errors.Is(err, kopia.ErrorRepoAlreadyExists) {
return nil, clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx)
@ -408,7 +410,7 @@ func (r repository) NewRestore(
func (r repository) NewMaintenance(
ctx context.Context,
mOpts rep.Maintenance,
mOpts ctrlRepo.Maintenance,
) (operations.MaintenanceOperation, error) {
return operations.NewMaintenanceOperation(
ctx,

60
src/pkg/repository/repository_test.go
View File

@ -60,7 +60,12 @@ func (suite *RepositoryUnitSuite) TestInitialize() {
st, err := test.storage()
assert.NoError(t, err, clues.ToCore(err))
_, err = Initialize(ctx, test.account, st, control.DefaultOptions())
_, err = Initialize(
ctx,
test.account,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
test.errCheck(t, err, clues.ToCore(err))
})
}
@ -137,7 +142,12 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
defer flush()
st := test.storage(t)
r, err := Initialize(ctx, test.account, st, control.DefaultOptions())
r, err := Initialize(
ctx,
test.account,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
if err == nil {
defer func() {
err := r.Close(ctx)
@ -169,7 +179,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() {
st.SessionName = "corso-repository-test"
st.SessionDuration = roleDuration.String()
r, err := Initialize(ctx, account.Account{}, st, control.Options{})
r, err := Initialize(ctx, account.Account{}, st, control.Options{}, ctrlRepo.Retention{})
require.NoError(suite.T(), err)
defer func() {
@ -186,7 +196,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
repo, err := Initialize(
ctx,
account.Account{},
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
// now re-connect
@ -203,7 +218,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
r, err := Initialize(
ctx,
account.Account{},
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
oldID := r.GetID()
@ -228,7 +248,12 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
r, err := Initialize(
ctx,
acct,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
userID := tconfig.M365UserID(t)
@ -250,7 +275,12 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
r, err := Initialize(
ctx,
acct,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
ro, err := r.NewRestore(ctx, "backup-id", selectors.Selector{DiscreteOwner: "test"}, restoreCfg)
@ -269,7 +299,12 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := Initialize(ctx, acct, st, control.DefaultOptions())
r, err := Initialize(
ctx,
acct,
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err, clues.ToCore(err))
mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{})
@ -286,7 +321,12 @@ func (suite *RepositoryIntegrationSuite) TestConnect_DisableMetrics() {
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
repo, err := Initialize(ctx, account.Account{}, st, control.DefaultOptions())
repo, err := Initialize(
ctx,
account.Account{},
st,
control.DefaultOptions(),
ctrlRepo.Retention{})
require.NoError(t, err)
// now re-connect
@ -350,7 +390,7 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
ctx, flush := tester.NewContext(t)
defer flush()
repo, err := Initialize(ctx, acct, st, test.opts())
repo, err := Initialize(ctx, acct, st, test.opts(), ctrlRepo.Retention{})
require.NoError(t, err)
r := repo.(*repository)

9
src/pkg/repository/repository_unexported_test.go
View File

@ -240,7 +240,7 @@ func (suite *RepositoryModelIntgSuite) SetupSuite() {
require.NotNil(t, k)
err = k.Initialize(ctx, rep.Options{})
err = k.Initialize(ctx, rep.Options{}, rep.Retention{})
require.NoError(t, err, clues.ToCore(err))
err = k.Connect(ctx, rep.Options{})
@ -291,8 +291,11 @@ func (suite *RepositoryModelIntgSuite) TestGetRepositoryModel() {
k = kopia.NewConn(s)
)
require.NoError(t, k.Initialize(ctx, rep.Options{}))
require.NoError(t, k.Connect(ctx, rep.Options{}))
err := k.Initialize(ctx, rep.Options{}, rep.Retention{})
require.NoError(t, err, "initializing repo: %v", clues.ToCore(err))
err = k.Connect(ctx, rep.Options{})
require.NoError(t, err, "connecting to repo: %v", clues.ToCore(err))
defer k.Close(ctx)