From fd056119ddc0e15c10f6822fde993d92778fe469 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:11:52 -0800 Subject: [PATCH] Add struct definitions and CLI helpers for backup options (#4860) Define structs that will be used for backup options and add some CLI helpers/tests to populate those structs with flag values Rate limiter config is pulled out as a separate struct because it will likely be used for backup and restore operations and it has values that are [passed separately](https://github.com/alcionai/corso/blob/505c06441a223f84d6dd81b42a7a56c83ea30f7c/src/internal/m365/backup.go#L232) to the rate limiter config code --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/backup/exchange_test.go | 16 ++++- src/cli/backup/groups_test.go | 13 +++- src/cli/backup/onedrive_test.go | 11 +++- src/cli/backup/sharepoint_test.go | 13 +++- src/cli/utils/options.go | 24 +++++++ src/pkg/control/backup.go | 106 ++++++++++++++++++++++++++++++ src/pkg/control/options.go | 21 ++---- 7 files changed, 182 insertions(+), 22 deletions(-) create mode 100644 src/pkg/control/backup.go diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index de6a38449..1dda84484 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -116,8 +116,20 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() { opts := utils.MakeExchangeOpts(cmd) co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch)) + assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(backupOpts.M365.DeltaPageSize))) + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) + assert.True(t, backupOpts.M365.DisableDeltaEndpoint) + assert.True(t, backupOpts.M365.ExchangeImmutableIDs) + assert.True(t, backupOpts.ServiceRateLimiter.DisableSlidingWindowLimiter) - assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users) assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch)) assert.Equal(t, flagsTD.DeltaPageSize, strconv.Itoa(int(co.DeltaPageSize))) assert.Equal(t, control.FailFast, co.FailureHandling) @@ -126,6 +138,8 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() { assert.True(t, co.ToggleFeatures.DisableDelta) assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs) assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter) + + assert.ElementsMatch(t, flagsTD.MailboxInput, opts.Users) flagsTD.AssertGenericBackupFlags(t, cmd) flagsTD.AssertProviderFlags(t, cmd) flagsTD.AssertStorageFlags(t, cmd) diff --git a/src/cli/backup/groups_test.go b/src/cli/backup/groups_test.go index c3707b9d9..ab5623704 100644 --- a/src/cli/backup/groups_test.go +++ b/src/cli/backup/groups_test.go @@ -160,13 +160,24 @@ func (suite *GroupsUnitSuite) TestBackupCreateFlags() { opts := utils.MakeGroupsOpts(cmd) co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(backupOpts.Parallelism.ItemFetch)) + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) + assert.True(t, backupOpts.M365.DisableDeltaEndpoint) - assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups) assert.Equal(t, flagsTD.FetchParallelism, strconv.Itoa(co.Parallelism.ItemFetch)) assert.Equal(t, control.FailFast, co.FailureHandling) assert.True(t, co.ToggleFeatures.DisableIncrementals) assert.True(t, co.ToggleFeatures.ForceItemDataDownload) assert.True(t, co.ToggleFeatures.DisableDelta) + + assert.ElementsMatch(t, flagsTD.GroupsInput, opts.Groups) flagsTD.AssertGenericBackupFlags(t, cmd) flagsTD.AssertProviderFlags(t, cmd) flagsTD.AssertStorageFlags(t, cmd) diff --git a/src/cli/backup/onedrive_test.go b/src/cli/backup/onedrive_test.go index cfd12e57d..199ea3efb 100644 --- a/src/cli/backup/onedrive_test.go +++ b/src/cli/backup/onedrive_test.go @@ -108,11 +108,20 @@ func (suite *OneDriveUnitSuite) TestBackupCreateFlags() { opts := utils.MakeOneDriveOpts(cmd) co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) - assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users) assert.Equal(t, control.FailFast, co.FailureHandling) assert.True(t, co.ToggleFeatures.DisableIncrementals) assert.True(t, co.ToggleFeatures.ForceItemDataDownload) + + assert.ElementsMatch(t, flagsTD.UsersInput, opts.Users) flagsTD.AssertGenericBackupFlags(t, cmd) flagsTD.AssertProviderFlags(t, cmd) flagsTD.AssertStorageFlags(t, cmd) diff --git a/src/cli/backup/sharepoint_test.go b/src/cli/backup/sharepoint_test.go index 19cf76fb6..c1e548fe1 100644 --- a/src/cli/backup/sharepoint_test.go +++ b/src/cli/backup/sharepoint_test.go @@ -112,12 +112,21 @@ func (suite *SharePointUnitSuite) TestBackupCreateFlags() { opts := utils.MakeSharePointOpts(cmd) co := utils.Control() + backupOpts := utils.ParseBackupOptions() + + // TODO(ashmrtn): Remove flag checks on control.Options to control.Backup once + // restore flags are switched over too and we no longer parse flags beyond + // connection info into control.Options. + assert.Equal(t, control.FailFast, backupOpts.FailureHandling) + assert.True(t, backupOpts.Incrementals.ForceFullEnumeration) + assert.True(t, backupOpts.Incrementals.ForceItemDataRefresh) - assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID) - assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL) assert.Equal(t, control.FailFast, co.FailureHandling) assert.True(t, co.ToggleFeatures.DisableIncrementals) assert.True(t, co.ToggleFeatures.ForceItemDataDownload) + + assert.ElementsMatch(t, []string{strings.Join(flagsTD.SiteIDInput, ",")}, opts.SiteID) + assert.ElementsMatch(t, flagsTD.WebURLInput, opts.WebURL) flagsTD.AssertGenericBackupFlags(t, cmd) flagsTD.AssertProviderFlags(t, cmd) flagsTD.AssertStorageFlags(t, cmd) diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 0a6c4c811..9bbd94707 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -41,3 +41,27 @@ func ControlWithConfig(cfg config.RepoDetails) control.Options { return opt } + +func ParseBackupOptions() control.BackupConfig { + opt := control.DefaultBackupConfig() + + if flags.FailFastFV { + opt.FailureHandling = control.FailFast + } + + dps := int32(flags.DeltaPageSizeFV) + if dps > 500 || dps < 1 { + dps = 500 + } + + opt.M365.DeltaPageSize = dps + opt.M365.DisableDeltaEndpoint = flags.DisableDeltaFV + opt.M365.ExchangeImmutableIDs = flags.EnableImmutableIDFV + opt.M365.UseDriveDeltaTree = flags.UseDeltaTreeFV + opt.ServiceRateLimiter.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV + opt.Parallelism.ItemFetch = flags.FetchParallelismFV + opt.Incrementals.ForceFullEnumeration = flags.DisableIncrementalsFV + opt.Incrementals.ForceItemDataRefresh = flags.ForceItemDataDownloadFV + + return opt +} diff --git a/src/pkg/control/backup.go b/src/pkg/control/backup.go new file mode 100644 index 000000000..641a554de --- /dev/null +++ b/src/pkg/control/backup.go @@ -0,0 +1,106 @@ +package control + +import ( + "github.com/alcionai/corso/src/pkg/extensions" +) + +// DefaultBackupOptions provides a Backup with the default values set. +func DefaultBackupConfig() BackupConfig { + return BackupConfig{ + FailureHandling: FailAfterRecovery, + Parallelism: Parallelism{ + CollectionBuffer: 4, + ItemFetch: 4, + }, + M365: BackupM365Config{ + DeltaPageSize: 500, + }, + } +} + +// BackupConfig is the set of options used for backup operations. Each set of +// options is only applied to the backup operation it's passed to. To use the +// same set of options for multiple backup operations pass the struct to all +// operations. +type BackupConfig struct { + FailureHandling FailurePolicy `json:"failureHandling"` + ItemExtensionFactory []extensions.CreateItemExtensioner `json:"-"` + Parallelism Parallelism `json:"parallelism"` + ServiceRateLimiter RateLimiter `json:"serviceRateLimiter"` + Incrementals IncrementalsConfig `json:"incrementalsConfig"` + M365 BackupM365Config `json:"m365Config"` + + // PreviewLimits defines the number of items and/or amount of data to fetch on + // a best-effort basis for preview backups. + // + // Since this is not split out by service or data categories these limits + // apply independently to all data categories that appear in a single backup + // where they are set. For example, if doing a teams backup and there's both a + // SharePoint site and Messages available, both data categories would try to + // backup data until the set limits without paying attention to what the other + // had already backed up. + PreviewLimits PreviewItemLimits `json:"previewItemLimits"` +} + +// BackupM365Config contains config options that are specific to backing up data +// from M365 or Corso features that are only available during M365 backups. +type BackupM365Config struct { + // DeltaPageSize controls the quantity of items fetched in each page during + // multi-page queries, such as graph api delta endpoints. + DeltaPageSize int32 `json:"deltaPageSize"` + + // DisableDelta prevents backups from calling /delta endpoints and will force + // a full enumeration of all items. This is different from + // IncrementalsConfig.ForceFullEnumeration in that this does not even + // make use of delta endpoints if a delta token is available. This is + // necessary when the user has filled up their mailbox storage as Microsoft + // prevents the API from being able to make calls to /delta endpoints when a + // mailbox is over storage limits. + DisableDeltaEndpoint bool `json:"exchangeDeltaEndpoint,omitempty"` + + // ExchangeImmutableIDs denotes whether Corso should store items with + // immutable Exchange IDs. This is only safe to set if the previous backup for + // incremental backups used immutable IDs or if a full backup is being done. + ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"` + + // see: https://github.com/alcionai/corso/issues/4688 + UseDriveDeltaTree bool `json:"useDriveDeltaTree"` +} + +type Parallelism struct { + // CollectionBuffer sets the number of items in a collection to buffer before + // blocking. + CollectionBuffer int + + // ItemFetch sets the number of items to fetch in parallel when populating + // items within a collection. + ItemFetch int +} + +// PreviewItemLimits describes best-effort maximum values to attempt to reach in +// this backup. Preview backups are used to demonstrate value by being quick to +// create. +type PreviewItemLimits struct { + MaxItems int + MaxItemsPerContainer int + MaxContainers int + MaxBytes int64 + MaxPages int + Enabled bool +} + +// IncrementalsConfig contains options specific to incremental backups and +// affects what data will be fetched from the external service being backed up. +type IncrementalsConfig struct { + // ForceFullEnumeration prevents the use of a previous backup as the starting + // point for the current backup. All data in the external service will be + // enumerated whether or not it's changed. Per-item storage will only get + // updated if changes have occurred. + ForceFullEnumeration bool `json:"forceFullEnumeration,omitempty"` + + // ForceItemDataRefresh causes the data for all enumerated items to replace + // stored data, even if no changes have been detected. Storage-side data + // deduplication still applies, but that's after item download, and items are + // always downloaded when this flag is set. + ForceItemDataRefresh bool `json:"forceItemDataRefresh,omitempty"` +} diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index f086856f2..f1c946fcf 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -29,23 +29,10 @@ type Options struct { PreviewLimits PreviewItemLimits `json:"previewItemLimits"` } -type Parallelism struct { - // sets the collection buffer size before blocking. - CollectionBuffer int - // sets the parallelism of item population within a collection. - ItemFetch int -} - -// PreviewItemLimits describes best-effort maximum values to attempt to reach in -// this backup. Preview backups are used to demonstrate value by being quick to -// create. -type PreviewItemLimits struct { - MaxItems int - MaxItemsPerContainer int - MaxContainers int - MaxBytes int64 - MaxPages int - Enabled bool +// RateLimiter is the set of options applied to any external service facing rate +// limiters Corso may use during backups or restores. +type RateLimiter struct { + DisableSlidingWindowLimiter bool `json:"disableSlidingWindowLimiter"` } type FailurePolicy string