From 179edfc08e2907520232248c2913540fb8a0f4b7 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 4 Aug 2023 12:08:04 -0700 Subject: [PATCH] Add hidden flags for setting retention during repo init (#3932) Add, parse, and configure info about retention when initializing a repo. These flags are currently marked as hidden Medium-term, they can be used in longevity tests when making the repo for the first time (we can configure this out of band if needed though) Long-term, they can be exposed to the user for immutable backup support --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * #3799 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/flags/retention.go | 50 ++++++ src/cli/repo/s3.go | 14 +- src/cli/utils/retention.go | 87 ++++++++++ src/cli/utils/retention_test.go | 156 ++++++++++++++++++ src/pkg/control/repository/repo.go | 16 +- .../repository/retentionmode_string.go | 4 +- 6 files changed, 316 insertions(+), 11 deletions(-) create mode 100644 src/cli/flags/retention.go create mode 100644 src/cli/utils/retention.go create mode 100644 src/cli/utils/retention_test.go diff --git a/src/cli/flags/retention.go b/src/cli/flags/retention.go new file mode 100644 index 000000000..38f717148 --- /dev/null +++ b/src/cli/flags/retention.go @@ -0,0 +1,50 @@ +package flags + +import ( + "time" + + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/pkg/control/repository" +) + +const ( + RetentionModeFN = "retention-mode" + RetentionDurationFN = "retention-duration" + ExtendRetentionFN = "extend-retention" +) + +var ( + RetentionModeFV string + RetentionDurationFV time.Duration + ExtendRetentionFV bool +) + +// AddRetentionConfigFlags adds the retention config flag set. +func AddRetentionConfigFlags(cmd *cobra.Command) { + fs := cmd.Flags() + fs.StringVar( + &RetentionModeFV, + RetentionModeFN, + repository.NoRetention.String(), + "Sets object locking mode (if any) to use in remote storage: "+ + repository.NoRetention.String()+", "+ + repository.GovernanceRetention.String()+", or "+ + repository.ComplianceRetention.String()) + cobra.CheckErr(fs.MarkHidden(RetentionModeFN)) + + fs.DurationVar( + &RetentionDurationFV, + RetentionDurationFN, + time.Duration(0), + "Set the amount of time to lock individual objects in remote storage") + cobra.CheckErr(fs.MarkHidden(RetentionDurationFN)) + + fs.BoolVar( + &ExtendRetentionFV, + ExtendRetentionFN, + false, + "Extends object locks during maintenance. "+ + "Extends locks by the most recently set value of "+RetentionDurationFN) + cobra.CheckErr(fs.MarkHidden(ExtendRetentionFN)) +} diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index b611e93e5..972701502 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -15,7 +15,6 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/pkg/account" - rep "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" @@ -50,7 +49,10 @@ func addS3Commands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case initCommand: - c, fs = utils.AddCommand(cmd, s3InitCmd()) + init := s3InitCmd() + flags.AddRetentionConfigFlags(init) + c, fs = utils.AddCommand(cmd, init) + case connectCommand: c, fs = utils.AddCommand(cmd, s3ConnectCmd()) } @@ -133,6 +135,11 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { opt := utils.ControlWithConfig(cfg) + retentionOpts, err := utils.MakeRetentionOpts(cmd) + if err != nil { + return Only(ctx, err) + } + // SendStartCorsoEvent uses distict ID as tenant ID because repoID is still not generated utils.SendStartCorsoEvent( ctx, @@ -159,13 +166,12 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to parse m365 account config")) } - // TODO(ashmrtn): Wire to flags for retention during repo init. r, err := repository.Initialize( ctx, cfg.Account, cfg.Storage, opt, - rep.Retention{}) + retentionOpts) if err != nil { if succeedIfExists && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil diff --git a/src/cli/utils/retention.go b/src/cli/utils/retention.go new file mode 100644 index 000000000..b943180c2 --- /dev/null +++ b/src/cli/utils/retention.go @@ -0,0 +1,87 @@ +package utils + +import ( + "time" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type retentionCfgOpts struct { + Mode string + Duration time.Duration + Extend bool + + Populated flags.PopulatedFlags +} + +func makeRetentionCfgOpts(cmd *cobra.Command) retentionCfgOpts { + return retentionCfgOpts{ + Mode: flags.RetentionModeFV, + Duration: flags.RetentionDurationFV, + Extend: flags.ExtendRetentionFV, + + // Populated contains the list of flags that appear in the command, + // according to pflags. Use this to differentiate between an "empty" and a + // "missing" value. + Populated: flags.GetPopulatedFlags(cmd), + } +} + +// validate checks common restore flags for correctness. +func (opts retentionCfgOpts) validate() error { + // Mode defaults to a valid value when pulled from flags. If coming from an + // empty struct will return invalid error. + if _, ok := repository.ValidRetentionModeNames()[opts.Mode]; !ok { + return clues.New("invalid retention mode " + opts.Mode) + } + + // TODO(ashmrtn): Add an upper bound check? + if opts.Duration < 0 { + return clues.New("negative retention duration") + } + + return nil +} + +// MakeRetentionConfig converts the current retentionCfgOpts into a +// repository.Retention struct for use in lower-layers of corso. +func (opts retentionCfgOpts) makeRetentionOpts() (repository.Retention, error) { + retention := repository.Retention{} + + if err := opts.validate(); err != nil { + return retention, clues.Stack(err) + } + + // Only populate the fields that the user passed so that we don't accidentally + // change retention values without meaning to. Even if the user passed the + // same value as the default for the flag it gets marked as populated. + if _, ok := opts.Populated[flags.RetentionModeFN]; ok { + mode, ok := repository.ValidRetentionModeNames()[opts.Mode] + if !ok { + // Not sure how we'd get here since we validate above, but just in case. + return retention, clues.New("invalid retention mode " + opts.Mode) + } + + retention.Mode = ptr.To(mode) + } + + if _, ok := opts.Populated[flags.RetentionDurationFN]; ok { + retention.Duration = ptr.To(opts.Duration) + } + + if _, ok := opts.Populated[flags.ExtendRetentionFN]; ok { + retention.Extend = ptr.To(opts.Extend) + } + + return retention, nil +} + +func MakeRetentionOpts(cmd *cobra.Command) (repository.Retention, error) { + opts, err := makeRetentionCfgOpts(cmd).makeRetentionOpts() + return opts, clues.Stack(err).OrNil() +} diff --git a/src/cli/utils/retention_test.go b/src/cli/utils/retention_test.go new file mode 100644 index 000000000..7063a3041 --- /dev/null +++ b/src/cli/utils/retention_test.go @@ -0,0 +1,156 @@ +package utils_test + +import ( + "testing" + "time" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control/repository" +) + +type RetentionCfgUnitSuite struct { + tester.Suite +} + +func TestRetentionCfgUnitSuite(t *testing.T) { + suite.Run(t, &RetentionCfgUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *RetentionCfgUnitSuite) TestMakeRetentionOpts() { + table := []struct { + name string + flags map[string]string + expectErr assert.ErrorAssertionFunc + expect repository.Retention + }{ + { + name: "Nothing Set", + expectErr: assert.NoError, + }, + { + name: "Invalid Mode", + flags: map[string]string{ + flags.RetentionModeFN: "foo", + }, + expectErr: assert.Error, + }, + { + name: "Negative Duration", + flags: map[string]string{ + flags.RetentionDurationFN: "-5h", + }, + expectErr: assert.Error, + }, + { + name: "Only Governance Mode", + flags: map[string]string{ + flags.RetentionModeFN: "governance", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + }, + }, + { + name: "Only Compliance Mode", + flags: map[string]string{ + flags.RetentionModeFN: "compliance", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.ComplianceRetention), + }, + }, + { + name: "Only No Retention Mode", + flags: map[string]string{ + flags.RetentionModeFN: "none", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.NoRetention), + }, + }, + { + name: "Mode And Duration", + flags: map[string]string{ + flags.RetentionModeFN: "governance", + flags.RetentionDurationFN: "48h", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + }, + }, + { + name: "Mode And Extend", + flags: map[string]string{ + flags.RetentionModeFN: "governance", + flags.ExtendRetentionFN: "false", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Extend: ptr.To(false), + }, + }, + { + name: "Duration And Extend", + flags: map[string]string{ + flags.RetentionDurationFN: "48h", + flags.ExtendRetentionFN: "true", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }, + }, + { + name: "All Set", + flags: map[string]string{ + flags.RetentionModeFN: "governance", + flags.RetentionDurationFN: "48h", + flags.ExtendRetentionFN: "true", + }, + expectErr: assert.NoError, + expect: repository.Retention{ + Mode: ptr.To(repository.GovernanceRetention), + Duration: ptr.To(time.Hour * 48), + Extend: ptr.To(true), + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{} + flags.AddRetentionConfigFlags(cmd) + fs := cmd.Flags() + + for fn, fv := range test.flags { + require.NoError(t, fs.Set(fn, fv), "setting flag values") + } + + result, err := utils.MakeRetentionOpts(cmd) + test.expectErr(t, err, "parsing flags into struct: %v", clues.ToCore(err)) + + if err != nil { + return + } + + assert.Equal(t, test.expect, result) + }) + } +} diff --git a/src/pkg/control/repository/repo.go b/src/pkg/control/repository/repo.go index 720e35331..0d80a1fda 100644 --- a/src/pkg/control/repository/repo.go +++ b/src/pkg/control/repository/repo.go @@ -54,16 +54,22 @@ const ( type RetentionMode int -// Can't be reordered as we rely on iota for numbering. -// //go:generate stringer -type=RetentionMode -linecomment const ( UnknownRetention RetentionMode = 0 - NoRetention RetentionMode = 1 - GovernanceRetention RetentionMode = 2 - ComplianceRetention RetentionMode = 3 + NoRetention RetentionMode = 1 // none + GovernanceRetention RetentionMode = 2 // governance + ComplianceRetention RetentionMode = 3 // compliance ) +func ValidRetentionModeNames() map[string]RetentionMode { + return map[string]RetentionMode{ + NoRetention.String(): NoRetention, + GovernanceRetention.String(): GovernanceRetention, + ComplianceRetention.String(): ComplianceRetention, + } +} + // Retention contains various options for configuring the retention mode. Takes // pointers instead of values so that we can tell the difference between an // unset value and a set but invalid value. This allows for partial diff --git a/src/pkg/control/repository/retentionmode_string.go b/src/pkg/control/repository/retentionmode_string.go index 470647ff2..b329b1303 100644 --- a/src/pkg/control/repository/retentionmode_string.go +++ b/src/pkg/control/repository/retentionmode_string.go @@ -14,9 +14,9 @@ func _() { _ = x[ComplianceRetention-3] } -const _RetentionMode_name = "UnknownRetentionNoRetentionGovernanceRetentionComplianceRetention" +const _RetentionMode_name = "UnknownRetentionnonegovernancecompliance" -var _RetentionMode_index = [...]uint8{0, 16, 27, 46, 65} +var _RetentionMode_index = [...]uint8{0, 16, 20, 30, 40} func (i RetentionMode) String() string { if i < 0 || i >= RetentionMode(len(_RetentionMode_index)-1) {