diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 7d5d86d10..dbc5a7fd9 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -106,6 +106,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case createCommand: c, fs = utils.AddCommand(cmd, exchangeCreateCmd()) + options.AddFeatureFlags(cmd, options.ExchangeIncrementals()) c.Use = c.Use + " " + exchangeServiceCommandCreateUseSuffix c.Example = exchangeServiceCommandCreateExamples diff --git a/src/cli/options/options.go b/src/cli/options/options.go index 66b061267..2d72f8d68 100644 --- a/src/cli/options/options.go +++ b/src/cli/options/options.go @@ -2,10 +2,34 @@ package options import ( "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/alcionai/corso/src/pkg/control" ) +// Control produces the control options based on the user's flags. +func Control() control.Options { + opt := control.Defaults() + + if fastFail { + opt.FailFast = true + } + + if noStats { + opt.DisableMetrics = true + } + + if exchangeIncrementals { + opt.EnabledFeatures.ExchangeIncrementals = true + } + + return opt +} + +// --------------------------------------------------------------------------- +// Operations Flags +// --------------------------------------------------------------------------- + var ( fastFail bool noStats bool @@ -25,17 +49,32 @@ func AddGlobalOperationFlags(cmd *cobra.Command) { fs.BoolVar(&noStats, "no-stats", false, "disable anonymous usage statistics gathering") } -// Control produces the control options based on the user's flags. -func Control() control.Options { - opt := control.Defaults() +// --------------------------------------------------------------------------- +// Feature Flags +// --------------------------------------------------------------------------- - if fastFail { - opt.FailFast = true +var exchangeIncrementals bool + +type exposeFeatureFlag func(*pflag.FlagSet) + +// AddFeatureFlags adds CLI flags for each exposed feature flags to the +// persistent flag set within the command. +func AddFeatureFlags(cmd *cobra.Command, effs ...exposeFeatureFlag) { + fs := cmd.PersistentFlags() + for _, fflag := range effs { + fflag(fs) + } +} + +// Adds the '--exchange-incrementals' cli flag which, when set, enables +// incrementals data retrieval for exchange backups. +func ExchangeIncrementals() func(*pflag.FlagSet) { + return func(fs *pflag.FlagSet) { + fs.BoolVar( + &exchangeIncrementals, + "exchange-incrementals", + false, + "Enable incremental data retrieval in Exchange backups.") + cobra.CheckErr(fs.MarkHidden("exchange-incrementals")) } - - if noStats { - opt.DisableMetrics = true - } - - return opt } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index f97263054..e1b45396b 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -360,11 +360,9 @@ func FetchContactIDsFromDirectory( Contacts(). Delta() - // TODO(rkeepers): Awaiting full integration of incremental support, else this - // will cause unexpected behavior/errors. - // if len(oldDelta) > 0 { - // builder = msuser.NewUsersItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter()) - // } + if len(oldDelta) > 0 { + builder = msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter()) + } for { resp, err := builder.Get(ctx, options) @@ -434,11 +432,9 @@ func FetchMessageIDsFromDirectory( Messages(). Delta() - // TODO(rkeepers): Awaiting full integration of incremental support, else this - // will cause unexpected behavior/errors. - // if len(oldDelta) > 0 { - // builder = msuser.NewUsersItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()) - // } + if len(oldDelta) > 0 { + builder = msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()) + } for { resp, err := builder.Get(ctx, options) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 58eeb3297..6edb25f38 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -129,8 +129,9 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { }() oc := selectorToOwnersCats(op.Selectors) + srm := shouldRetrieveMetadata(op.Selectors, op.Options) - mans, mdColls, err := produceManifestsAndMetadata(ctx, op.kopia, op.store, oc, tenantID) + mans, mdColls, err := produceManifestsAndMetadata(ctx, op.kopia, op.store, oc, tenantID, srm) if err != nil { opStats.readErr = errors.Wrap(err, "connecting to M365") return opStats.readErr @@ -142,7 +143,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { return opStats.readErr } - cs, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, control.Options{}) + cs, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options) if err != nil { opStats.readErr = errors.Wrap(err, "retrieving data to backup") return opStats.readErr @@ -175,6 +176,12 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { return err } +// checker to see if conditions are correct for retrieving metadata like delta tokens +// and previous paths. +func shouldRetrieveMetadata(sel selectors.Selector, opts control.Options) bool { + return opts.EnabledFeatures.ExchangeIncrementals && sel.Service == selectors.ServiceExchange +} + // calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics. func produceManifestsAndMetadata( ctx context.Context, @@ -182,6 +189,7 @@ func produceManifestsAndMetadata( sw *store.Wrapper, oc *kopia.OwnersCats, tenantID string, + getMetadata bool, ) ([]*kopia.ManifestEntry, []data.Collection, error) { complete, closer := observe.MessageWithCompletion("Fetching backup heuristics:") defer func() { @@ -203,6 +211,10 @@ func produceManifestsAndMetadata( return nil, nil, err } + if !getMetadata { + return ms, nil, nil + } + for _, man := range ms { if len(man.IncompleteReason) > 0 { continue diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index fcfd3458b..47abe7111 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -4,9 +4,25 @@ import ( "github.com/alcionai/corso/src/internal/common" ) -const ( - defaultRestoreLocation = "Corso_Restore_" -) +// Options holds the optional configurations for a process +type Options struct { + Collision CollisionPolicy `json:"-"` + DisableMetrics bool `json:"disableMetrics"` + FailFast bool `json:"failFast"` + EnabledFeatures FeatureFlags `json:"enabledFeatures"` +} + +// Defaults provides an Options with the default values set. +func Defaults() Options { + return Options{ + FailFast: true, + EnabledFeatures: FeatureFlags{}, + } +} + +// --------------------------------------------------------------------------- +// Restore Item Collision Policy +// --------------------------------------------------------------------------- // CollisionPolicy describes how the datalayer behaves in case of a collision. type CollisionPolicy int @@ -19,19 +35,13 @@ const ( Replace ) -// Options holds the optional configurations for a process -type Options struct { - Collision CollisionPolicy `json:"-"` - DisableMetrics bool `json:"disableMetrics"` - FailFast bool `json:"failFast"` -} +// --------------------------------------------------------------------------- +// Restore Destination +// --------------------------------------------------------------------------- -// Defaults provides an Options with the default values set. -func Defaults() Options { - return Options{ - FailFast: true, - } -} +const ( + defaultRestoreLocation = "Corso_Restore_" +) // RestoreDestination is a POD that contains an override of the resource owner // to restore data under and the name of the root of the restored container @@ -51,3 +61,13 @@ func DefaultRestoreDestination(timeFormat common.TimeFormat) RestoreDestination ContainerName: defaultRestoreLocation + common.FormatNow(timeFormat), } } + +// --------------------------------------------------------------------------- +// Feature Flags +// --------------------------------------------------------------------------- + +type FeatureFlags struct { + // ExchangeIncrementals allow for re-use of delta links when backing up + // exchange data, reducing the amount of data pulled from graph. + ExchangeIncrementals bool `json:"incrementals,omitempty"` +}