Add struct definitions and CLI helpers for backup options (#4860)

Define structs that will be used for backup options and add some CLI
helpers/tests to populate those structs with flag values

Rate limiter config is pulled out as a separate struct because it will
likely be used for backup and restore operations and it has values that
are [passed separately](505c06441a/src/internal/m365/backup.go (L232)) to the rate limiter config code

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Test Plan

- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

src/cli/backup/exchange_test.go

@@ -116,8 +116,20 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
 
 	opts := utils.MakeExchangeOpts(cmd)
 	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
+	assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(backupOpts.M365.DeltaPageSize)))
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+	assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
+	assert.True(t, backupOpts.M365.ExchangeImmutableIDs)
+	assert.True(t, backupOpts.ServiceRateLimiter.DisableSlidingWindowLimiter)
 
-	assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
 	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
 	assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(co.DeltaPageSize)))
 	assert.Equal(t, control.FailFast, co.FailureHandling)
@@ -126,6 +138,8 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() {
 	assert.True(t, co.ToggleFeatures.DisableDelta)
 	assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs)
 	assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter)
+
+	assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users)
 	flagsTD.AssertGenericBackupFlags(t, cmd)
 	flagsTD.AssertProviderFlags(t, cmd)
 	flagsTD.AssertStorageFlags(t, cmd)

src/cli/backup/groups_test.go

@@ -160,13 +160,24 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() {
 
 	opts := utils.MakeGroupsOpts(cmd)
 	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch))
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
+	assert.True(t, backupOpts.M365.DisableDeltaEndpoint)
 
-	assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
 	assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch))
 	assert.Equal(t, control.FailFast, co.FailureHandling)
 	assert.True(t, co.ToggleFeatures.DisableIncrementals)
 	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
 	assert.True(t, co.ToggleFeatures.DisableDelta)
+
+	assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups)
 	flagsTD.AssertGenericBackupFlags(t, cmd)
 	flagsTD.AssertProviderFlags(t, cmd)
 	flagsTD.AssertStorageFlags(t, cmd)

src/cli/backup/onedrive_test.go

@@ -108,11 +108,20 @@ func (suite *OneDriveUnitSuite) TestBackupCreateFlags() {
 
 	opts := utils.MakeOneDriveOpts(cmd)
 	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
 
-	assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
 	assert.Equal(t, control.FailFast, co.FailureHandling)
 	assert.True(t, co.ToggleFeatures.DisableIncrementals)
 	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+
+	assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users)
 	flagsTD.AssertGenericBackupFlags(t, cmd)
 	flagsTD.AssertProviderFlags(t, cmd)
 	flagsTD.AssertStorageFlags(t, cmd)

src/cli/backup/sharepoint_test.go

@@ -112,12 +112,21 @@ func (suite *SharePointUnitSuite) TestBackupCreateFlags() {
 
 	opts := utils.MakeSharePointOpts(cmd)
 	co := utils.Control()
+	backupOpts := utils.ParseBackupOptions()
+
+	// TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once
+	// restore flags are switched over too and we no longer parse flags beyond
+	// connection info into control.Options.
+	assert.Equal(t, control.FailFast, backupOpts.FailureHandling)
+	assert.True(t, backupOpts.Incrementals.ForceFullEnumeration)
+	assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh)
 
-	assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID)
-	assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL)
 	assert.Equal(t, control.FailFast, co.FailureHandling)
 	assert.True(t, co.ToggleFeatures.DisableIncrementals)
 	assert.True(t, co.ToggleFeatures.ForceItemDataDownload)
+
+	assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID)
+	assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL)
 	flagsTD.AssertGenericBackupFlags(t, cmd)
 	flagsTD.AssertProviderFlags(t, cmd)
 	flagsTD.AssertStorageFlags(t, cmd)

src/cli/utils/options.go

@@ -41,3 +41,27 @@ func ControlWithConfig(cfg config.RepoDetails) control.Options {
 
 	return opt
 }
+
+func ParseBackupOptions() control.BackupConfig {
+	opt := control.DefaultBackupConfig()
+
+	if flags.FailFastFV {
+		opt.FailureHandling = control.FailFast
+	}
+
+	dps := int32(flags.DeltaPageSizeFV)
+	if dps > 500 || dps < 1 {
+		dps = 500
+	}
+
+	opt.M365.DeltaPageSize = dps
+	opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV
+	opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV
+	opt.M365.UseDriveDeltaTree = flags.UseDeltaTreeFV
+	opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV
+	opt.Parallelism.ItemFetch = flags.FetchParallelismFV
+	opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV
+	opt.Incrementals.ForceItemDataRefresh = flags.ForceItemDataDownloadFV
+
+	return opt
+}

src/pkg/control/backup.go

@@ -0,0 +1,106 @@
+package control
+
+import (
+	"github.com/alcionai/corso/src/pkg/extensions"
+)
+
+// DefaultBackupOptions provides a Backup with the default values set.
+func DefaultBackupConfig() BackupConfig {
+	return BackupConfig{
+		FailureHandling: FailAfterRecovery,
+		Parallelism: Parallelism{
+			CollectionBuffer: 4,
+			ItemFetch:        4,
+		},
+		M365: BackupM365Config{
+			DeltaPageSize: 500,
+		},
+	}
+}
+
+// BackupConfig is the set of options used for backup operations. Each set of
+// options is only applied to the backup operation it's passed to. To use the
+// same set of options for multiple backup operations pass the struct to all
+// operations.
+type BackupConfig struct {
+	FailureHandling      FailurePolicy                      `json:"failureHandling"`
+	ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"`
+	Parallelism          Parallelism                        `json:"parallelism"`
+	ServiceRateLimiter   RateLimiter                        `json:"serviceRateLimiter"`
+	Incrementals         IncrementalsConfig                 `json:"incrementalsConfig"`
+	M365                 BackupM365Config                   `json:"m365Config"`
+
+	// PreviewLimits defines the number of items and/or amount of data to fetch on
+	// a best-effort basis for preview backups.
+	//
+	// Since this is not split out by service or data categories these limits
+	// apply independently to all data categories that appear in a single backup
+	// where they are set. For example, if doing a teams backup and there's both a
+	// SharePoint site and Messages available, both data categories would try to
+	// backup data until the set limits without paying attention to what the other
+	// had already backed up.
+	PreviewLimits PreviewItemLimits `json:"previewItemLimits"`
+}
+
+// BackupM365Config contains config options that are specific to backing up data
+// from M365 or Corso features that are only available during M365 backups.
+type BackupM365Config struct {
+	// DeltaPageSize controls the quantity of items fetched in each page during
+	// multi-page queries, such as graph api delta endpoints.
+	DeltaPageSize int32 `json:"deltaPageSize"`
+
+	// DisableDelta prevents backups from calling /delta endpoints and will force
+	// a full enumeration of all items. This is different from
+	// IncrementalsConfig.ForceFullEnumeration in that this does not even
+	// make use of delta endpoints if a delta token is available. This is
+	// necessary when the user has filled up their mailbox storage as Microsoft
+	// prevents the API from being able to make calls to /delta endpoints when a
+	// mailbox is over storage limits.
+	DisableDeltaEndpoint bool `json:"exchangeDeltaEndpoint,omitempty"`
+
+	// ExchangeImmutableIDs denotes whether Corso should store items with
+	// immutable Exchange IDs. This is only safe to set if the previous backup for
+	// incremental backups used immutable IDs or if a full backup is being done.
+	ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"`
+
+	// see: https://github.com/alcionai/corso/issues/4688
+	UseDriveDeltaTree bool `json:"useDriveDeltaTree"`
+}
+
+type Parallelism struct {
+	// CollectionBuffer sets the number of items in a collection to buffer before
+	// blocking.
+	CollectionBuffer int
+
+	// ItemFetch sets the number of items to fetch in parallel when populating
+	// items within a collection.
+	ItemFetch int
+}
+
+// PreviewItemLimits describes best-effort maximum values to attempt to reach in
+// this backup. Preview backups are used to demonstrate value by being quick to
+// create.
+type PreviewItemLimits struct {
+	MaxItems             int
+	MaxItemsPerContainer int
+	MaxContainers        int
+	MaxBytes             int64
+	MaxPages             int
+	Enabled              bool
+}
+
+// IncrementalsConfig contains options specific to incremental backups and
+// affects what data will be fetched from the external service being backed up.
+type IncrementalsConfig struct {
+	// ForceFullEnumeration prevents the use of a previous backup as the starting
+	// point for the current backup. All data in the external service will be
+	// enumerated whether or not it's changed. Per-item storage will only get
+	// updated if changes have occurred.
+	ForceFullEnumeration bool `json:"forceFullEnumeration,omitempty"`
+
+	// ForceItemDataRefresh causes the data for all enumerated items to replace
+	// stored data, even if no changes have been detected. Storage-side data
+	// deduplication still applies, but that's after item download, and items are
+	// always downloaded when this flag is set.
+	ForceItemDataRefresh bool `json:"forceItemDataRefresh,omitempty"`
+}

src/pkg/control/options.go

@@ -29,23 +29,10 @@ type Options struct {
 	PreviewLimits PreviewItemLimits `json:"previewItemLimits"`
 }
 
-type Parallelism struct {
+// RateLimiter is the set of options applied to any external service facing rate
-	// sets the collection buffer size before blocking.
+// limiters Corso may use during backups or restores.
-	CollectionBuffer int
+type RateLimiter struct {
-	// sets the parallelism of item population within a collection.
+	DisableSlidingWindowLimiter bool `json:"disableSlidingWindowLimiter"`
-	ItemFetch int
-}
-
-// PreviewItemLimits describes best-effort maximum values to attempt to reach in
-// this backup. Preview backups are used to demonstrate value by being quick to
-// create.
-type PreviewItemLimits struct {
-	MaxItems             int
-	MaxItemsPerContainer int
-	MaxContainers        int
-	MaxBytes             int64
-	MaxPages             int
-	Enabled              bool
 }
 
 type FailurePolicy string