Add hidden flag to set fetch parallelism for Exchange from CLI (#2879)

#### Does this PR need a docs update or release note?

- [x]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [ ]  No

#### Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* closes #2877

#### Test Plan

- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2023-03-20 14:48:21 -07:00 committed by GitHub
parent 8b1c3a7073
commit a798932e4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 15 additions and 1 deletions

View File

@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- OneDrive item downloads that return 404 during backup (normally due to external deletion while Corso processes) are now skipped instead of quietly dropped. These items will appear in the skipped list alongside other skipped cases such as malware detection. - OneDrive item downloads that return 404 during backup (normally due to external deletion while Corso processes) are now skipped instead of quietly dropped. These items will appear in the skipped list alongside other skipped cases such as malware detection.
- Listing a single backup by id will also list the skipped and failed items that occurred during the backup. These can be filtered out with the flags `--failed-items hide`, `--skipped-items hide`, and `--recovered-errors hide`. - Listing a single backup by id will also list the skipped and failed items that occurred during the backup. These can be filtered out with the flags `--failed-items hide`, `--skipped-items hide`, and `--recovered-errors hide`.
- Enable incremental backups for OneDrive if permissions aren't being backed up. - Enable incremental backups for OneDrive if permissions aren't being backed up.
- Hidden flag to control parallelism for fetching Exchange items (`--fetch-parallelism`). May help reduce `ApplicationThrottled` errors but will slow down backup.
### Fixed ### Fixed
- Fix repo connect not working without a config file - Fix repo connect not working without a config file

View File

@ -117,6 +117,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
&exchangeData, &exchangeData,
utils.DataFN, nil, utils.DataFN, nil,
"Select one or more types of data to backup: "+dataEmail+", "+dataContacts+", or "+dataEvents) "Select one or more types of data to backup: "+dataEmail+", "+dataContacts+", or "+dataEvents)
options.AddFetchParallelismFlag(c)
options.AddOperationFlags(c) options.AddOperationFlags(c)
case listCommand: case listCommand:

View File

@ -17,6 +17,7 @@ func Control() control.Options {
opt.SkipReduce = skipReduce opt.SkipReduce = skipReduce
opt.ToggleFeatures.DisableIncrementals = disableIncrementals opt.ToggleFeatures.DisableIncrementals = disableIncrementals
opt.ToggleFeatures.EnablePermissionsBackup = enablePermissionsBackup opt.ToggleFeatures.EnablePermissionsBackup = enablePermissionsBackup
opt.ItemFetchParallelism = fetchParallelism
return opt return opt
} }
@ -30,6 +31,7 @@ var (
noStats bool noStats bool
restorePermissions bool restorePermissions bool
skipReduce bool skipReduce bool
fetchParallelism int
) )
// AddOperationFlags adds command-local operation flags // AddOperationFlags adds command-local operation flags
@ -62,6 +64,16 @@ func AddSkipReduceFlag(cmd *cobra.Command) {
cobra.CheckErr(fs.MarkHidden("skip-reduce")) cobra.CheckErr(fs.MarkHidden("skip-reduce"))
} }
func AddFetchParallelismFlag(cmd *cobra.Command) {
fs := cmd.Flags()
fs.IntVar(
&fetchParallelism,
"fetch-parallelism",
4,
"Control the number of concurrent data fetches for Exchange. Valid range is [1-4]. Default: 4")
cobra.CheckErr(fs.MarkHidden("fetch-parallelism"))
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Feature Flags // Feature Flags
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -82,7 +94,7 @@ func AddFeatureToggle(cmd *cobra.Command, effs ...exposeFeatureFlag) {
} }
} }
// Adds the hidden '--no-incrementals' cli flag which, when set, disables // Adds the hidden '--disable-incrementals' cli flag which, when set, disables
// incremental backups. // incremental backups.
func DisableIncrementals() func(*pflag.FlagSet) { func DisableIncrementals() func(*pflag.FlagSet) {
return func(fs *pflag.FlagSet) { return func(fs *pflag.FlagSet) {