From 71b0831c7b3107fa20e06edee1da8a43dad82963 Mon Sep 17 00:00:00 2001 From: Abhishek Pandey Date: Wed, 15 Nov 2023 13:54:46 -0800 Subject: [PATCH] Add a control option to disable sliding window limiter (#4676) Add a killswitch for sliding window limiter. This is only relevant for exchange backups Setting this flag switches exchange backups to use the default token bucket rate limiter. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [x] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * # #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/cli/backup/exchange.go | 1 + src/cli/backup/exchange_test.go | 2 + src/cli/flags/options.go | 71 ++++++++++++++++++++------------- src/cli/utils/options.go | 1 + src/cli/utils/options_test.go | 3 ++ src/pkg/control/options.go | 6 +++ 6 files changed, 57 insertions(+), 27 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 83640a0bc..7753644b4 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -84,6 +84,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { flags.AddEnableImmutableIDFlag(c) flags.AddDeltaPageSizeFlag(c) flags.AddGenericBackupFlags(c) + flags.AddDisableSlidingWindowLimiterFlag(c) case listCommand: c, fs = utils.AddCommand(cmd, exchangeListCmd()) diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index f21757dd0..de6a38449 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -108,6 +108,7 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() { // bool flags "--" + flags.DisableDeltaFN, "--" + flags.EnableImmutableIDFN, + "--" + flags.DisableSlidingWindowLimiterFN, }, flagsTD.PreparedGenericBackupFlags(), flagsTD.PreparedProviderFlags(), @@ -124,6 +125,7 @@ func (suite *ExchangeUnitSuite) TestBackupCreateFlags() { assert.True(t, co.ToggleFeatures.ForceItemDataDownload) assert.True(t, co.ToggleFeatures.DisableDelta) assert.True(t, co.ToggleFeatures.ExchangeImmutableIDs) + assert.True(t, co.ToggleFeatures.DisableSlidingWindowLimiter) flagsTD.AssertGenericBackupFlags(t, cmd) flagsTD.AssertProviderFlags(t, cmd) flagsTD.AssertStorageFlags(t, cmd) diff --git a/src/cli/flags/options.go b/src/cli/flags/options.go index 5b0eaf215..2d043ae1f 100644 --- a/src/cli/flags/options.go +++ b/src/cli/flags/options.go @@ -5,36 +5,38 @@ import ( ) const ( - AlertsFN = "alerts" - DeltaPageSizeFN = "delta-page-size" - DisableDeltaFN = "disable-delta" - DisableIncrementalsFN = "disable-incrementals" - ForceItemDataDownloadFN = "force-item-data-download" - EnableImmutableIDFN = "enable-immutable-id" - FailFastFN = "fail-fast" - FailedItemsFN = "failed-items" - FetchParallelismFN = "fetch-parallelism" - NoStatsFN = "no-stats" - RecoveredErrorsFN = "recovered-errors" - NoPermissionsFN = "no-permissions" - RunModeFN = "run-mode" - SkippedItemsFN = "skipped-items" - SkipReduceFN = "skip-reduce" + AlertsFN = "alerts" + DeltaPageSizeFN = "delta-page-size" + DisableDeltaFN = "disable-delta" + DisableIncrementalsFN = "disable-incrementals" + DisableSlidingWindowLimiterFN = "disable-sliding-window-limiter" + ForceItemDataDownloadFN = "force-item-data-download" + EnableImmutableIDFN = "enable-immutable-id" + FailFastFN = "fail-fast" + FailedItemsFN = "failed-items" + FetchParallelismFN = "fetch-parallelism" + NoStatsFN = "no-stats" + RecoveredErrorsFN = "recovered-errors" + NoPermissionsFN = "no-permissions" + RunModeFN = "run-mode" + SkippedItemsFN = "skipped-items" + SkipReduceFN = "skip-reduce" ) var ( - DeltaPageSizeFV int - DisableDeltaFV bool - DisableIncrementalsFV bool - ForceItemDataDownloadFV bool - EnableImmutableIDFV bool - FailFastFV bool - FetchParallelismFV int - ListAlertsFV string - ListFailedItemsFV string - ListSkippedItemsFV string - ListRecoveredErrorsFV string - NoStatsFV bool + DeltaPageSizeFV int + DisableDeltaFV bool + DisableIncrementalsFV bool + DisableSlidingWindowLimiterFV bool + ForceItemDataDownloadFV bool + EnableImmutableIDFV bool + FailFastFV bool + FetchParallelismFV int + ListAlertsFV string + ListFailedItemsFV string + ListSkippedItemsFV string + ListRecoveredErrorsFV string + NoStatsFV bool // RunMode describes the type of run, such as: // flagtest, dry, run. Should default to 'run'. RunModeFV string @@ -159,3 +161,18 @@ func AddRunModeFlag(cmd *cobra.Command, persistent bool) { fs.StringVar(&RunModeFV, RunModeFN, "run", "What mode to run: dry, test, run. Defaults to run.") cobra.CheckErr(fs.MarkHidden(RunModeFN)) } + +// AddDisableSlidingWindowLimiterFN disables the experimental sliding window rate +// limiter for graph API requests. This is only relevant for exchange backups. +// Exchange restores continue to use the default token bucket rate limiter. +// Setting this flag switches exchange backups to use the default token bucket +// rate limiter. +func AddDisableSlidingWindowLimiterFlag(cmd *cobra.Command) { + fs := cmd.Flags() + fs.BoolVar( + &DisableSlidingWindowLimiterFV, + DisableSlidingWindowLimiterFN, + false, + "Disable sliding window rate limiter.") + cobra.CheckErr(fs.MarkHidden(DisableSlidingWindowLimiterFN)) +} diff --git a/src/cli/utils/options.go b/src/cli/utils/options.go index 670d45ca5..f5d62ee9e 100644 --- a/src/cli/utils/options.go +++ b/src/cli/utils/options.go @@ -25,6 +25,7 @@ func Control() control.Options { opt.ToggleFeatures.DisableIncrementals = flags.DisableIncrementalsFV opt.ToggleFeatures.ForceItemDataDownload = flags.ForceItemDataDownloadFV opt.ToggleFeatures.DisableDelta = flags.DisableDeltaFV + opt.ToggleFeatures.DisableSlidingWindowLimiter = flags.DisableSlidingWindowLimiterFV opt.ToggleFeatures.ExchangeImmutableIDs = flags.EnableImmutableIDFV opt.Parallelism.ItemFetch = flags.FetchParallelismFV diff --git a/src/cli/utils/options_test.go b/src/cli/utils/options_test.go index 91d5d8b98..dd17183d9 100644 --- a/src/cli/utils/options_test.go +++ b/src/cli/utils/options_test.go @@ -36,6 +36,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { assert.True(t, flags.SkipReduceFV, flags.SkipReduceFN) assert.Equal(t, 2, flags.FetchParallelismFV, flags.FetchParallelismFN) assert.Equal(t, 499, flags.DeltaPageSizeFV, flags.DeltaPageSizeFN) + assert.True(t, flags.DisableSlidingWindowLimiterFV, flags.DisableSlidingWindowLimiterFN) }, } @@ -50,6 +51,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { flags.AddSkipReduceFlag(cmd) flags.AddFetchParallelismFlag(cmd) flags.AddDeltaPageSizeFlag(cmd) + flags.AddDisableSlidingWindowLimiterFlag(cmd) // Test arg parsing for few args cmd.SetArgs([]string{ @@ -63,6 +65,7 @@ func (suite *OptionsUnitSuite) TestAddExchangeCommands() { "--" + flags.SkipReduceFN, "--" + flags.FetchParallelismFN, "2", "--" + flags.DeltaPageSizeFN, "499", + "--" + flags.DisableSlidingWindowLimiterFN, }) err := cmd.Execute() diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index 16f5436e7..7f158b5e1 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -87,4 +87,10 @@ type Toggles struct { // the protected resource. PreviewBackups are used to demonstrate value by // being quick to create. PreviewBackup bool `json:"previewBackup"` + + // DisableSlidingWindowLimiter disables the experimental sliding window rate + // limiter for graph API requests. This is only relevant for exchange backups. + // Setting this flag switches exchange backups to fallback to the default token + // bucket rate limiter. + DisableSlidingWindowLimiter bool `json:"disableSlidingWindowLimiter"` }