diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 4d1f43324..1853a824f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -57,6 +57,9 @@ type BackupOperation struct { // when true, this allows for incremental backups instead of full data pulls incremental bool + // When true, disables kopia-assisted incremental backups. This forces + // downloading and hashing all item data for items not in the merge base(s). + disableAssistBackup bool } // BackupResults aggregate the details of the result of the operation. @@ -79,14 +82,15 @@ func NewBackupOperation( bus events.Eventer, ) (BackupOperation, error) { op := BackupOperation{ - operation: newOperation(opts, bus, count.New(), kw, sw), - ResourceOwner: owner, - Selectors: selector, - Version: "v0", - BackupVersion: version.Backup, - account: acct, - incremental: useIncrementalBackup(selector, opts), - bp: bp, + operation: newOperation(opts, bus, count.New(), kw, sw), + ResourceOwner: owner, + Selectors: selector, + Version: "v0", + BackupVersion: version.Backup, + account: acct, + incremental: useIncrementalBackup(selector, opts), + disableAssistBackup: opts.ToggleFeatures.ForceItemDataDownload, + bp: bp, } if err := op.validate(); err != nil { @@ -180,7 +184,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { "resource_owner_name", clues.Hide(op.ResourceOwner.Name()), "backup_id", op.Results.BackupID, "service", op.Selectors.Service, - "incremental", op.incremental) + "incremental", op.incremental, + "disable_assist_backup", op.disableAssistBackup) op.bus.Event( ctx, @@ -301,7 +306,8 @@ func (op *BackupOperation) do( op.kopia, reasons, fallbackReasons, op.account.ID(), - op.incremental) + op.incremental, + op.disableAssistBackup) if err != nil { return nil, clues.Wrap(err, "producing manifests and metadata") } @@ -312,6 +318,10 @@ func (op *BackupOperation) do( lastBackupVersion = mans.MinBackupVersion() } + // TODO(ashmrtn): This should probably just return a collection that deletes + // the entire subtree instead of returning an additional bool. That way base + // selection is controlled completely by flags and merging is controlled + // completely by collections. cs, ssmb, canUsePreviousBackup, err := produceBackupDataCollections( ctx, op.bp, diff --git a/src/internal/operations/manifests.go b/src/internal/operations/manifests.go index 1c5d1716c..8ca339d26 100644 --- a/src/internal/operations/manifests.go +++ b/src/internal/operations/manifests.go @@ -15,11 +15,44 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -// calls kopia to retrieve prior backup manifests, metadata collections to supply backup heuristics. -// TODO(ashmrtn): Make this a helper function that always returns as much as -// possible and call in another function that drops metadata and/or -// kopia-assisted incremental bases based on flag values. func produceManifestsAndMetadata( + ctx context.Context, + bf inject.BaseFinder, + rp inject.RestoreProducer, + reasons, fallbackReasons []kopia.Reasoner, + tenantID string, + getMetadata, dropAssistBases bool, +) (kopia.BackupBases, []data.RestoreCollection, bool, error) { + bb, meta, useMergeBases, err := getManifestsAndMetadata( + ctx, + bf, + rp, + reasons, + fallbackReasons, + tenantID, + getMetadata) + if err != nil { + return nil, nil, false, clues.Stack(err) + } + + if !useMergeBases || !getMetadata { + logger.Ctx(ctx).Debug("full backup requested, dropping merge bases") + + bb.ClearMergeBases() + } + + if dropAssistBases { + logger.Ctx(ctx).Debug("no caching requested, dropping assist bases") + + bb.ClearAssistBases() + } + + return bb, meta, useMergeBases, nil +} + +// getManifestsAndMetadata calls kopia to retrieve prior backup manifests, +// metadata collections to supply backup heuristics. +func getManifestsAndMetadata( ctx context.Context, bf inject.BaseFinder, rp inject.RestoreProducer, @@ -52,12 +85,6 @@ func produceManifestsAndMetadata( }) if !getMetadata { - logger.Ctx(ctx).Debug("full backup requested, dropping merge bases") - - // TODO(ashmrtn): If this function is moved to be a helper function then - // move this change to the bases to the caller of this function. - bb.ClearMergeBases() - return bb, nil, false, nil } diff --git a/src/internal/operations/manifests_test.go b/src/internal/operations/manifests_test.go index 5fdf22424..26bc00a1f 100644 --- a/src/internal/operations/manifests_test.go +++ b/src/internal/operations/manifests_test.go @@ -254,6 +254,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { rp mockRestoreProducer reasons []kopia.Reasoner getMeta bool + dropAssist bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc expectDCS []mockColl @@ -390,6 +391,36 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { makeMan("id2", "checkpoint", path.EmailCategory), ), }, + { + name: "one valid man, extra incomplete man, no assist bases", + bf: &mockBackupFinder{ + data: map[string]kopia.BackupBases{ + ro: kopia.NewMockBackupBases().WithMergeBases( + makeMan("id1", "", path.EmailCategory), + ).WithAssistBases( + makeMan("id2", "checkpoint", path.EmailCategory), + ), + }, + }, + rp: mockRestoreProducer{ + collsByID: map[string][]data.RestoreCollection{ + "id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id1"}}}, + "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, + }, + }, + reasons: []kopia.Reasoner{ + kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory), + }, + getMeta: true, + dropAssist: true, + assertErr: assert.NoError, + assertB: assert.True, + expectDCS: []mockColl{{id: "id1"}}, + expectMans: kopia.NewMockBackupBases().WithMergeBases( + makeMan("id1", "", path.EmailCategory), + ). + ClearMockAssistBases(), + }, { name: "multiple valid mans", bf: &mockBackupFinder{ @@ -452,7 +483,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() { &test.rp, test.reasons, nil, tid, - test.getMeta) + test.getMeta, + test.dropAssist) test.assertErr(t, err, clues.ToCore(err)) test.assertB(t, b) @@ -551,6 +583,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb reasons []kopia.Reasoner fallbackReasons []kopia.Reasoner getMeta bool + dropAssist bool assertErr assert.ErrorAssertionFunc assertB assert.BoolAssertionFunc expectDCS []mockColl @@ -604,6 +637,35 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb makeBackup(fbro, "fb_id1", path.EmailCategory), ), }, + { + name: "only fallbacks, no assist", + bf: &mockBackupFinder{ + data: map[string]kopia.BackupBases{ + fbro: kopia.NewMockBackupBases().WithMergeBases( + makeMan(fbro, "fb_id1", "", path.EmailCategory), + ).WithBackups( + makeBackup(fbro, "fb_id1", path.EmailCategory), + ), + }, + }, + rp: mockRestoreProducer{ + collsByID: map[string][]data.RestoreCollection{ + "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, + }, + }, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, + getMeta: true, + dropAssist: true, + assertErr: assert.NoError, + assertB: assert.True, + expectDCS: []mockColl{{id: "fb_id1"}}, + expectMans: kopia.NewMockBackupBases().WithMergeBases( + makeMan(fbro, "fb_id1", "", path.EmailCategory), + ).WithBackups( + makeBackup(fbro, "fb_id1", path.EmailCategory), + ). + ClearMockAssistBases(), + }, { name: "complete mans and fallbacks", bf: &mockBackupFinder{ @@ -734,6 +796,40 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb makeMan(ro, "id2", "checkpoint", path.EmailCategory), ), }, + { + name: "incomplete mans and complete fallbacks, no assist bases", + bf: &mockBackupFinder{ + data: map[string]kopia.BackupBases{ + ro: kopia.NewMockBackupBases().WithAssistBases( + makeMan(ro, "id2", "checkpoint", path.EmailCategory), + ), + fbro: kopia.NewMockBackupBases().WithMergeBases( + makeMan(fbro, "fb_id1", "", path.EmailCategory), + ).WithBackups( + makeBackup(fbro, "fb_id1", path.EmailCategory), + ), + }, + }, + rp: mockRestoreProducer{ + collsByID: map[string][]data.RestoreCollection{ + "id2": {data.NoFetchRestoreCollection{Collection: mockColl{id: "id2"}}}, + "fb_id1": {data.NoFetchRestoreCollection{Collection: mockColl{id: "fb_id1"}}}, + }, + }, + reasons: []kopia.Reasoner{emailReason}, + fallbackReasons: []kopia.Reasoner{fbEmailReason}, + getMeta: true, + dropAssist: true, + assertErr: assert.NoError, + assertB: assert.True, + expectDCS: []mockColl{{id: "fb_id1"}}, + expectMans: kopia.NewMockBackupBases().WithMergeBases( + makeMan(fbro, "fb_id1", "", path.EmailCategory), + ).WithBackups( + makeBackup(fbro, "fb_id1", path.EmailCategory), + ). + ClearMockAssistBases(), + }, { name: "complete mans and incomplete fallbacks", bf: &mockBackupFinder{ @@ -887,7 +983,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb &test.rp, test.reasons, test.fallbackReasons, tid, - test.getMeta) + test.getMeta, + test.dropAssist) test.assertErr(t, err, clues.ToCore(err)) test.assertB(t, b) diff --git a/src/pkg/control/options.go b/src/pkg/control/options.go index 01c88b5eb..0f7d559aa 100644 --- a/src/pkg/control/options.go +++ b/src/pkg/control/options.go @@ -62,6 +62,12 @@ type Toggles struct { // DisableIncrementals prevents backups from using incremental lookups, // forcing a new, complete backup of all data regardless of prior state. DisableIncrementals bool `json:"exchangeIncrementals,omitempty"` + // ForceItemDataDownload disables finding cached items in previous failed + // backups (i.e. kopia-assisted incrementals). Data dedupe will still occur + // since that is based on content hashes. Items that have not changed since + // the previous backup (i.e. in the merge base) will not be redownloaded. Use + // DisableIncrementals to control that behavior. + ForceItemDataDownload bool `json:"forceItemDataDownload,omitempty"` // DisableDelta prevents backups from using delta based lookups, // forcing a backup by enumerating all items. This is different // from DisableIncrementals in that this does not even makes use of