enable incrementals with feature flags (#1903)

## Description

Allows the usage of incrementals via feature-
flag controls.  Feature flags can be enabled
on a per-flag basis in either the cli (the flag
is hidden from users, normally) or through the
sdk by directly toggling the flag property
in the control.Options.

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1901

## Test Plan

- [x] 💪 Manual

src/cli/backup/exchange.go

@@ -106,6 +106,7 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
 	switch cmd.Use {
 	case createCommand:
 		c, fs = utils.AddCommand(cmd, exchangeCreateCmd())
+		options.AddFeatureFlags(cmd, options.ExchangeIncrementals())
 
 		c.Use = c.Use + " " + exchangeServiceCommandCreateUseSuffix
 		c.Example = exchangeServiceCommandCreateExamples

src/cli/options/options.go

@@ -2,10 +2,34 @@ package options
 
 import (
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/alcionai/corso/src/pkg/control"
 )
 
+// Control produces the control options based on the user's flags.
+func Control() control.Options {
+	opt := control.Defaults()
+
+	if fastFail {
+		opt.FailFast = true
+	}
+
+	if noStats {
+		opt.DisableMetrics = true
+	}
+
+	if exchangeIncrementals {
+		opt.EnabledFeatures.ExchangeIncrementals = true
+	}
+
+	return opt
+}
+
+// ---------------------------------------------------------------------------
+// Operations Flags
+// ---------------------------------------------------------------------------
+
 var (
 	fastFail bool
 	noStats  bool
@@ -25,17 +49,32 @@ func AddGlobalOperationFlags(cmd *cobra.Command) {
 	fs.BoolVar(&noStats, "no-stats", false, "disable anonymous usage statistics gathering")
 }
 
-// Control produces the control options based on the user's flags.
+// ---------------------------------------------------------------------------
-func Control() control.Options {
+// Feature Flags
-	opt := control.Defaults()
+// ---------------------------------------------------------------------------
 
-	if fastFail {
+var exchangeIncrementals bool
-		opt.FailFast = true
+
+type exposeFeatureFlag func(*pflag.FlagSet)
+
+// AddFeatureFlags adds CLI flags for each exposed feature flags to the
+// persistent flag set within the command.
+func AddFeatureFlags(cmd *cobra.Command, effs ...exposeFeatureFlag) {
+	fs := cmd.PersistentFlags()
+	for _, fflag := range effs {
+		fflag(fs)
+	}
+}
+
+// Adds the '--exchange-incrementals' cli flag which, when set, enables
+// incrementals data retrieval for exchange backups.
+func ExchangeIncrementals() func(*pflag.FlagSet) {
+	return func(fs *pflag.FlagSet) {
+		fs.BoolVar(
+			&exchangeIncrementals,
+			"exchange-incrementals",
+			false,
+			"Enable incremental data retrieval in Exchange backups.")
+		cobra.CheckErr(fs.MarkHidden("exchange-incrementals"))
 	}
-
-	if noStats {
-		opt.DisableMetrics = true
-	}
-
-	return opt
 }

src/internal/connector/exchange/service_iterators.go

@@ -360,11 +360,9 @@ func FetchContactIDsFromDirectory(
 		Contacts().
 		Delta()
 
-	// TODO(rkeepers): Awaiting full integration of incremental support, else this
+	if len(oldDelta) > 0 {
-	// will cause unexpected behavior/errors.
+		builder = msuser.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter())
-	// if len(oldDelta) > 0 {
+	}
-	// 	builder = msuser.NewUsersItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter())
-	// }
 
 	for {
 		resp, err := builder.Get(ctx, options)
@@ -434,11 +432,9 @@ func FetchMessageIDsFromDirectory(
 		Messages().
 		Delta()
 
-	// TODO(rkeepers): Awaiting full integration of incremental support, else this
+	if len(oldDelta) > 0 {
-	// will cause unexpected behavior/errors.
+		builder = msuser.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())
-	// if len(oldDelta) > 0 {
+	}
-	// 	builder = msuser.NewUsersItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())
-	// }
 
 	for {
 		resp, err := builder.Get(ctx, options)

src/internal/operations/backup.go

@@ -129,8 +129,9 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 	}()
 
 	oc := selectorToOwnersCats(op.Selectors)
+	srm := shouldRetrieveMetadata(op.Selectors, op.Options)
 
-	mans, mdColls, err := produceManifestsAndMetadata(ctx, op.kopia, op.store, oc, tenantID)
+	mans, mdColls, err := produceManifestsAndMetadata(ctx, op.kopia, op.store, oc, tenantID, srm)
 	if err != nil {
 		opStats.readErr = errors.Wrap(err, "connecting to M365")
 		return opStats.readErr
@@ -142,7 +143,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		return opStats.readErr
 	}
 
-	cs, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, control.Options{})
+	cs, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options)
 	if err != nil {
 		opStats.readErr = errors.Wrap(err, "retrieving data to backup")
 		return opStats.readErr
@@ -175,6 +176,12 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 	return err
 }
 
+// checker to see if conditions are correct for retrieving metadata like delta tokens
+// and previous paths.
+func shouldRetrieveMetadata(sel selectors.Selector, opts control.Options) bool {
+	return opts.EnabledFeatures.ExchangeIncrementals && sel.Service == selectors.ServiceExchange
+}
+
 // calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics.
 func produceManifestsAndMetadata(
 	ctx context.Context,
@@ -182,6 +189,7 @@ func produceManifestsAndMetadata(
 	sw *store.Wrapper,
 	oc *kopia.OwnersCats,
 	tenantID string,
+	getMetadata bool,
 ) ([]*kopia.ManifestEntry, []data.Collection, error) {
 	complete, closer := observe.MessageWithCompletion("Fetching backup heuristics:")
 	defer func() {
@@ -203,6 +211,10 @@ func produceManifestsAndMetadata(
 		return nil, nil, err
 	}
 
+	if !getMetadata {
+		return ms, nil, nil
+	}
+
 	for _, man := range ms {
 		if len(man.IncompleteReason) > 0 {
 			continue

src/pkg/control/options.go

@@ -4,9 +4,25 @@ import (
 	"github.com/alcionai/corso/src/internal/common"
 )
 
-const (
+// Options holds the optional configurations for a process
-	defaultRestoreLocation = "Corso_Restore_"
+type Options struct {
-)
+	Collision       CollisionPolicy `json:"-"`
+	DisableMetrics  bool            `json:"disableMetrics"`
+	FailFast        bool            `json:"failFast"`
+	EnabledFeatures FeatureFlags    `json:"enabledFeatures"`
+}
+
+// Defaults provides an Options with the default values set.
+func Defaults() Options {
+	return Options{
+		FailFast:        true,
+		EnabledFeatures: FeatureFlags{},
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Restore Item Collision Policy
+// ---------------------------------------------------------------------------
 
 // CollisionPolicy describes how the datalayer behaves in case of a collision.
 type CollisionPolicy int
@@ -19,19 +35,13 @@ const (
 	Replace
 )
 
-// Options holds the optional configurations for a process
+// ---------------------------------------------------------------------------
-type Options struct {
+// Restore Destination
-	Collision      CollisionPolicy `json:"-"`
+// ---------------------------------------------------------------------------
-	DisableMetrics bool            `json:"disableMetrics"`
-	FailFast       bool            `json:"failFast"`
-}
 
-// Defaults provides an Options with the default values set.
+const (
-func Defaults() Options {
+	defaultRestoreLocation = "Corso_Restore_"
-	return Options{
+)
-		FailFast: true,
-	}
-}
 
 // RestoreDestination is a POD that contains an override of the resource owner
 // to restore data under and the name of the root of the restored container
@@ -51,3 +61,13 @@ func DefaultRestoreDestination(timeFormat common.TimeFormat) RestoreDestination
 		ContainerName: defaultRestoreLocation + common.FormatNow(timeFormat),
 	}
 }
+
+// ---------------------------------------------------------------------------
+// Feature Flags
+// ---------------------------------------------------------------------------
+
+type FeatureFlags struct {
+	// ExchangeIncrementals allow for re-use of delta links when backing up
+	// exchange data, reducing the amount of data pulled from graph.
+	ExchangeIncrementals bool `json:"incrementals,omitempty"`
+}