SDK option and logic to drop assist bases (#3983)

Add a way for SDK users to drop
kopia-assisted incremental bases
thus forcing item data redownload
if the item wasn't sourced from
a merge base

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #2360

#### Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

src/internal/operations/backup.go

@@ -57,6 +57,9 @@ type BackupOperation struct {
 
 	// when true, this allows for incremental backups instead of full data pulls
 	incremental bool
+	// When true, disables kopia-assisted incremental backups. This forces
+	// downloading and hashing all item data for items not in the merge base(s).
+	disableAssistBackup bool
 }
 
 // BackupResults aggregate the details of the result of the operation.
@@ -86,6 +89,7 @@ func NewBackupOperation(
 		BackupVersion:       version.Backup,
 		account:             acct,
 		incremental:         useIncrementalBackup(selector, opts),
+		disableAssistBackup: opts.ToggleFeatures.ForceItemDataDownload,
 		bp:                  bp,
 	}
 
@@ -180,7 +184,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
 		"resource_owner_name", clues.Hide(op.ResourceOwner.Name()),
 		"backup_id", op.Results.BackupID,
 		"service", op.Selectors.Service,
-		"incremental", op.incremental)
+		"incremental", op.incremental,
+		"disable_assist_backup", op.disableAssistBackup)
 
 	op.bus.Event(
 		ctx,
@@ -301,7 +306,8 @@ func (op *BackupOperation) do(
 		op.kopia,
 		reasons, fallbackReasons,
 		op.account.ID(),
-		op.incremental)
+		op.incremental,
+		op.disableAssistBackup)
 	if err != nil {
 		return nil, clues.Wrap(err, "producing manifests and metadata")
 	}
@@ -312,6 +318,10 @@ func (op *BackupOperation) do(
 		lastBackupVersion = mans.MinBackupVersion()
 	}
 
+	// TODO(ashmrtn): This should probably just return a collection that deletes
+	// the entire subtree instead of returning an additional bool. That way base
+	// selection is controlled completely by flags and merging is controlled
+	// completely by collections.
 	cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections(
 		ctx,
 		op.bp,

src/internal/operations/manifests.go

@@ -15,11 +15,44 @@ import (
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
-// calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics.
-// TODO(ashmrtn): Make this a helper function that always returns as much as
-// possible and call in another function that drops metadata and/or
-// kopia-assisted incremental bases based on flag values.
 func produceManifestsAndMetadata(
+	ctx context.Context,
+	bf inject.BaseFinder,
+	rp inject.RestoreProducer,
+	reasons, fallbackReasons []kopia.Reasoner,
+	tenantID string,
+	getMetadata, dropAssistBases bool,
+) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
+	bb, meta, useMergeBases, err := getManifestsAndMetadata(
+		ctx,
+		bf,
+		rp,
+		reasons,
+		fallbackReasons,
+		tenantID,
+		getMetadata)
+	if err != nil {
+		return nil, nil, false, clues.Stack(err)
+	}
+
+	if !useMergeBases || !getMetadata {
+		logger.Ctx(ctx).Debug("full backup requested, dropping merge bases")
+
+		bb.ClearMergeBases()
+	}
+
+	if dropAssistBases {
+		logger.Ctx(ctx).Debug("no caching requested, dropping assist bases")
+
+		bb.ClearAssistBases()
+	}
+
+	return bb, meta, useMergeBases, nil
+}
+
+// getManifestsAndMetadata calls kopia to retrieve prior backup manifests,
+// metadata collections to supply backup heuristics.
+func getManifestsAndMetadata(
 	ctx context.Context,
 	bf inject.BaseFinder,
 	rp inject.RestoreProducer,
@@ -52,12 +85,6 @@ func produceManifestsAndMetadata(
 		})
 
 	if !getMetadata {
-		logger.Ctx(ctx).Debug("full backup requested, dropping merge bases")
-
-		// TODO(ashmrtn): If this function is moved to be a helper function then
-		// move this change to the bases to the caller of this function.
-		bb.ClearMergeBases()
-
 		return bb, nil, false, nil
 	}
 

src/internal/operations/manifests_test.go

@@ -254,6 +254,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 		rp          mockRestoreProducer
 		reasons     []kopia.Reasoner
 		getMeta     bool
+		dropAssist  bool
 		assertErr   assert.ErrorAssertionFunc
 		assertB     assert.BoolAssertionFunc
 		expectDCS   []mockColl
@@ -390,6 +391,36 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 				makeMan("id2", "checkpoint", path.EmailCategory),
 			),
 		},
+		{
+			name: "one valid man, extra incomplete man, no assist bases",
+			bf: &mockBackupFinder{
+				data: map[string]kopia.BackupBases{
+					ro: kopia.NewMockBackupBases().WithMergeBases(
+						makeMan("id1", "", path.EmailCategory),
+					).WithAssistBases(
+						makeMan("id2", "checkpoint", path.EmailCategory),
+					),
+				},
+			},
+			rp: mockRestoreProducer{
+				collsByID: map[string][]data.RestoreCollection{
+					"id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id1"}}},
+					"id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}},
+				},
+			},
+			reasons: []kopia.Reasoner{
+				kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
+			},
+			getMeta:    true,
+			dropAssist: true,
+			assertErr:  assert.NoError,
+			assertB:    assert.True,
+			expectDCS:  []mockColl{{id: "id1"}},
+			expectMans: kopia.NewMockBackupBases().WithMergeBases(
+				makeMan("id1", "", path.EmailCategory),
+			).
+				ClearMockAssistBases(),
+		},
 		{
 			name: "multiple valid mans",
 			bf: &mockBackupFinder{
@@ -452,7 +483,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
 				&test.rp,
 				test.reasons, nil,
 				tid,
-				test.getMeta)
+				test.getMeta,
+				test.dropAssist)
 			test.assertErr(t, err, clues.ToCore(err))
 			test.assertB(t, b)
 
@@ -551,6 +583,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
 		reasons         []kopia.Reasoner
 		fallbackReasons []kopia.Reasoner
 		getMeta         bool
+		dropAssist      bool
 		assertErr       assert.ErrorAssertionFunc
 		assertB         assert.BoolAssertionFunc
 		expectDCS       []mockColl
@@ -604,6 +637,35 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
 				makeBackup(fbro, "fb_id1", path.EmailCategory),
 			),
 		},
+		{
+			name: "only fallbacks, no assist",
+			bf: &mockBackupFinder{
+				data: map[string]kopia.BackupBases{
+					fbro: kopia.NewMockBackupBases().WithMergeBases(
+						makeMan(fbro, "fb_id1", "", path.EmailCategory),
+					).WithBackups(
+						makeBackup(fbro, "fb_id1", path.EmailCategory),
+					),
+				},
+			},
+			rp: mockRestoreProducer{
+				collsByID: map[string][]data.RestoreCollection{
+					"fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}},
+				},
+			},
+			fallbackReasons: []kopia.Reasoner{fbEmailReason},
+			getMeta:         true,
+			dropAssist:      true,
+			assertErr:       assert.NoError,
+			assertB:         assert.True,
+			expectDCS:       []mockColl{{id: "fb_id1"}},
+			expectMans: kopia.NewMockBackupBases().WithMergeBases(
+				makeMan(fbro, "fb_id1", "", path.EmailCategory),
+			).WithBackups(
+				makeBackup(fbro, "fb_id1", path.EmailCategory),
+			).
+				ClearMockAssistBases(),
+		},
 		{
 			name: "complete mans and fallbacks",
 			bf: &mockBackupFinder{
@@ -734,6 +796,40 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
 				makeMan(ro, "id2", "checkpoint", path.EmailCategory),
 			),
 		},
+		{
+			name: "incomplete mans and complete fallbacks, no assist bases",
+			bf: &mockBackupFinder{
+				data: map[string]kopia.BackupBases{
+					ro: kopia.NewMockBackupBases().WithAssistBases(
+						makeMan(ro, "id2", "checkpoint", path.EmailCategory),
+					),
+					fbro: kopia.NewMockBackupBases().WithMergeBases(
+						makeMan(fbro, "fb_id1", "", path.EmailCategory),
+					).WithBackups(
+						makeBackup(fbro, "fb_id1", path.EmailCategory),
+					),
+				},
+			},
+			rp: mockRestoreProducer{
+				collsByID: map[string][]data.RestoreCollection{
+					"id2":    {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}},
+					"fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}},
+				},
+			},
+			reasons:         []kopia.Reasoner{emailReason},
+			fallbackReasons: []kopia.Reasoner{fbEmailReason},
+			getMeta:         true,
+			dropAssist:      true,
+			assertErr:       assert.NoError,
+			assertB:         assert.True,
+			expectDCS:       []mockColl{{id: "fb_id1"}},
+			expectMans: kopia.NewMockBackupBases().WithMergeBases(
+				makeMan(fbro, "fb_id1", "", path.EmailCategory),
+			).WithBackups(
+				makeBackup(fbro, "fb_id1", path.EmailCategory),
+			).
+				ClearMockAssistBases(),
+		},
 		{
 			name: "complete mans and incomplete fallbacks",
 			bf: &mockBackupFinder{
@@ -887,7 +983,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
 				&test.rp,
 				test.reasons, test.fallbackReasons,
 				tid,
-				test.getMeta)
+				test.getMeta,
+				test.dropAssist)
 			test.assertErr(t, err, clues.ToCore(err))
 			test.assertB(t, b)
 

src/pkg/control/options.go

@@ -62,6 +62,12 @@ type Toggles struct {
 	// DisableIncrementals prevents backups from using incremental lookups,
 	// forcing a new, complete backup of all data regardless of prior state.
 	DisableIncrementals bool `json:"exchangeIncrementals,omitempty"`
+	// ForceItemDataDownload disables finding cached items in previous failed
+	// backups (i.e. kopia-assisted incrementals). Data dedupe will still occur
+	// since that is based on content hashes. Items that have not changed since
+	// the previous backup (i.e. in the merge base) will not be redownloaded. Use
+	// DisableIncrementals to control that behavior.
+	ForceItemDataDownload bool `json:"forceItemDataDownload,omitempty"`
 	// DisableDelta prevents backups from using delta based lookups,
 	// forcing a backup by enumerating all items. This is different
 	// from DisableIncrementals in that this does not even makes use of