Share count bus between graph clients and backup/restore operations (#4605)

<!-- PR description-->

With #4497 changes, any counters that we log in graph middlewares ( e.g. `APICallTokensConsumed` or `ThrottledAPICalls`) will no longer show up in `BackupOperation.Results.Counts`. 

This is because graph adapters and backup operation will be initialized with different count busses. Since the backup result counts are captured from the operation bus, we will lose out on any graph metrics captured in middleware layers. This PR unifies the busses. 

Ill also add a regression check for graph metrics shortly

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2023-11-06 22:41:11 -08:00 committed by GitHub
parent ea308055eb
commit ea6a57b2a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 174 additions and 116 deletions

View File

@ -204,7 +204,9 @@ func internalMiddleware(
&LoggingMiddleware{}, &LoggingMiddleware{},
throttler, throttler,
&RateLimiterMiddleware{}, &RateLimiterMiddleware{},
&MetricsMiddleware{}, &MetricsMiddleware{
counter: counter,
},
} }
if len(cc.appendMiddleware) > 0 { if len(cc.appendMiddleware) > 0 {

View File

@ -306,7 +306,9 @@ func kiotaMiddlewares(
mw, mw,
throttler, throttler,
&RateLimiterMiddleware{}, &RateLimiterMiddleware{},
&MetricsMiddleware{}) &MetricsMiddleware{
counter: counter,
})
if len(cc.appendMiddleware) > 0 { if len(cc.appendMiddleware) > 0 {
mw = append(mw, cc.appendMiddleware...) mw = append(mw, cc.appendMiddleware...)

View File

@ -83,9 +83,10 @@ func NewBackupOperation(
selector selectors.Selector, selector selectors.Selector,
owner idname.Provider, owner idname.Provider,
bus events.Eventer, bus events.Eventer,
counter *count.Bus,
) (BackupOperation, error) { ) (BackupOperation, error) {
op := BackupOperation{ op := BackupOperation{
operation: newOperation(opts, bus, count.New(), kw, sw), operation: newOperation(opts, bus, counter, kw, sw),
ResourceOwner: owner, ResourceOwner: owner,
Selectors: selector, Selectors: selector,
Version: "v0", Version: "v0",

View File

@ -429,7 +429,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_PersistResults() {
acct, acct,
sel, sel,
sel, sel,
evmock.NewBus()) evmock.NewBus(),
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
op.Errors.Fail(test.fail) op.Errors.Fail(test.fail)
@ -1487,7 +1488,8 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
test.acct, test.acct,
sel, sel,
sel, sel,
evmock.NewBus()) evmock.NewBus(),
count.New())
test.errCheck(t, err, clues.ToCore(err)) test.errCheck(t, err, clues.ToCore(err))
}) })
} }
@ -1930,7 +1932,8 @@ func (suite *AssistBackupIntegrationSuite) TestBackupTypesForFailureModes() {
acct, acct,
osel.Selector, osel.Selector,
selectors.Selector{DiscreteOwner: userID}, selectors.Selector{DiscreteOwner: userID},
evmock.NewBus()) evmock.NewBus(),
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = bo.Run(ctx) err = bo.Run(ctx)
@ -2245,7 +2248,8 @@ func (suite *AssistBackupIntegrationSuite) TestExtensionsIncrementals() {
acct, acct,
osel.Selector, osel.Selector,
selectors.Selector{DiscreteOwner: userID}, selectors.Selector{DiscreteOwner: userID},
evmock.NewBus()) evmock.NewBus(),
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = bo.Run(ctx) err = bo.Run(ctx)

View File

@ -26,6 +26,7 @@ import (
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata"
@ -181,7 +182,8 @@ func (suite *MaintenanceOpNightlySuite) TestRepoMaintenance_GarbageCollection()
acct, acct,
osel.Selector, osel.Selector,
selectors.Selector{DiscreteOwner: userID}, selectors.Selector{DiscreteOwner: userID},
evmock.NewBus()) evmock.NewBus(),
count.New())
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
err = bo.Run(ctx) err = bo.Run(ctx)

View File

@ -118,12 +118,13 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
sel = test.selector().Selector sel = test.selector().Selector
opts = control.DefaultOptions() opts = control.DefaultOptions()
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
sel = bod.sel sel = bod.sel
@ -185,7 +186,8 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
ctx, ctx,
bod, bod,
incMB, incMB,
opts) opts,
counter)
) )
runAndCheckBackup(t, ctx, &incBO, incMB, true) runAndCheckBackup(t, ctx, &incBO, incMB, true)
@ -238,10 +240,11 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchangeBasic_groups9Versio
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewExchangeBackup([]string{suite.its.user.ID}) counter = count.New()
opts = control.DefaultOptions() sel = selectors.NewExchangeBackup([]string{suite.its.user.ID})
ws = deeTD.DriveIDFromRepoRef opts = control.DefaultOptions()
ws = deeTD.DriveIDFromRepoRef
) )
sel.Include( sel.Include(
@ -255,7 +258,8 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchangeBasic_groups9Versio
mb, mb,
sel.Selector, sel.Selector,
opts, opts,
version.All8MigrateUserPNToID) version.All8MigrateUserPNToID,
counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -290,12 +294,14 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchangeBasic_groups9Versio
false) false)
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
notForcedFull := newTestBackupOp( notForcedFull := newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
mb, mb,
opts) opts,
counter)
notForcedFull.BackupVersion = version.Groups9Update notForcedFull.BackupVersion = version.Groups9Update
runAndCheckBackup(t, ctx, &notForcedFull, mb, false) runAndCheckBackup(t, ctx, &notForcedFull, mb, false)
@ -357,6 +363,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
var ( var (
acct = tconfig.NewM365Account(t) acct = tconfig.NewM365Account(t)
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
now = dttm.Now() now = dttm.Now()
service = path.ExchangeService service = path.ExchangeService
categories = map[path.CategoryType][][]string{ categories = map[path.CategoryType][][]string{
@ -379,7 +386,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
) )
opts.ToggleFeatures = toggles opts.ToggleFeatures = toggles
ctrl, sels := ControllerWithSelector(t, ctx, acct, sel.Selector, nil, nil) ctrl, sels := ControllerWithSelector(t, ctx, acct, sel.Selector, nil, nil, counter)
sel.DiscreteOwner = sels.ID() sel.DiscreteOwner = sels.ID()
sel.DiscreteOwnerName = sels.Name() sel.DiscreteOwnerName = sels.Name()
@ -578,7 +585,7 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
} }
} }
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
// run the initial backup // run the initial backup
@ -911,15 +918,16 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
var ( var (
t = suite.T() t = suite.T()
incMB = evmock.NewBus() incMB = evmock.NewBus()
atid = creds.AzureTenantID counter = count.New()
atid = creds.AzureTenantID
) )
ctx, flush := tester.WithContext(t, ctx) ctx, flush := tester.WithContext(t, ctx)
defer flush() defer flush()
incBO := newTestBackupOp(t, ctx, bod, incMB, opts) incBO := newTestBackupOp(t, ctx, bod, incMB, opts, counter)
suite.Run("PreTestSetup", func() { suite.Run("PreTestSetup", func() {
t := suite.T() t := suite.T()
@ -1043,11 +1051,12 @@ func (suite *ExchangeRestoreNightlyIntgSuite) TestRestore_Run_exchangeWithAdvanc
baseSel.DiscreteOwner = suite.its.user.ID baseSel.DiscreteOwner = suite.its.user.ID
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.DefaultOptions() counter = count.New()
opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -1353,11 +1362,12 @@ func (suite *ExchangeRestoreNightlyIntgSuite) TestRestore_Run_exchangeAlternateP
baseSel.DiscreteOwner = suite.its.user.ID baseSel.DiscreteOwner = suite.its.user.ID
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.DefaultOptions() counter = count.New()
opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, baseSel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)

View File

@ -88,6 +88,7 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsBasic_groups9VersionBum
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
sel = selectors.NewGroupsBackup([]string{suite.its.group.ID}) sel = selectors.NewGroupsBackup([]string{suite.its.group.ID})
opts = control.DefaultOptions() opts = control.DefaultOptions()
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
@ -103,7 +104,8 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsBasic_groups9VersionBum
mb, mb,
sel.Selector, sel.Selector,
opts, opts,
version.All8MigrateUserPNToID) version.All8MigrateUserPNToID,
counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -138,12 +140,14 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsBasic_groups9VersionBum
false) false)
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
forcedFull := newTestBackupOp( forcedFull := newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
mb, mb,
opts) opts,
counter)
forcedFull.BackupVersion = version.Groups9Update forcedFull.BackupVersion = version.Groups9Update
runAndCheckBackup(t, ctx, &forcedFull, mb, false) runAndCheckBackup(t, ctx, &forcedFull, mb, false)
@ -203,6 +207,7 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsBasic() {
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
sel = selectors.NewGroupsBackup([]string{suite.its.group.ID}) sel = selectors.NewGroupsBackup([]string{suite.its.group.ID})
opts = control.DefaultOptions() opts = control.DefaultOptions()
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
@ -212,7 +217,7 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsBasic() {
selTD.GroupsBackupLibraryFolderScope(sel), selTD.GroupsBackupLibraryFolderScope(sel),
selTD.GroupsBackupChannelScope(sel)) selTD.GroupsBackupChannelScope(sel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -254,12 +259,13 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsExtensions() {
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewGroupsBackup([]string{suite.its.group.ID}) counter = count.New()
opts = control.DefaultOptions() sel = selectors.NewGroupsBackup([]string{suite.its.group.ID})
tenID = tconfig.M365TenantID(t) opts = control.DefaultOptions()
svc = path.GroupsService tenID = tconfig.M365TenantID(t)
ws = deeTD.DriveIDFromRepoRef svc = path.GroupsService
ws = deeTD.DriveIDFromRepoRef
) )
opts.ItemExtensionFactory = getTestExtensionFactories() opts.ItemExtensionFactory = getTestExtensionFactories()
@ -267,7 +273,7 @@ func (suite *GroupsBackupIntgSuite) TestBackup_Run_groupsExtensions() {
// does not apply to channel messages // does not apply to channel messages
sel.Include(selTD.GroupsBackupLibraryFolderScope(sel)) sel.Include(selTD.GroupsBackupLibraryFolderScope(sel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)

View File

@ -115,6 +115,7 @@ func prepNewTestBackupOp(
sel selectors.Selector, sel selectors.Selector,
opts control.Options, opts control.Options,
backupVersion int, backupVersion int,
counter *count.Bus,
) ( ) (
operations.BackupOperation, operations.BackupOperation,
*backupOpDependencies, *backupOpDependencies,
@ -161,14 +162,16 @@ func prepNewTestBackupOp(
bod.acct, bod.acct,
sel, sel,
nil, nil,
bod.close) bod.close,
counter)
bo := newTestBackupOp( bo := newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
bus, bus,
opts) opts,
counter)
bo.BackupVersion = backupVersion bo.BackupVersion = backupVersion
bod.sss = streamstore.NewStreamer( bod.sss = streamstore.NewStreamer(
@ -189,6 +192,7 @@ func newTestBackupOp(
bod *backupOpDependencies, bod *backupOpDependencies,
bus events.Eventer, bus events.Eventer,
opts control.Options, opts control.Options,
counter *count.Bus,
) operations.BackupOperation { ) operations.BackupOperation {
bod.ctrl.IDNameLookup = idname.NewCache(map[string]string{bod.sel.ID(): bod.sel.Name()}) bod.ctrl.IDNameLookup = idname.NewCache(map[string]string{bod.sel.ID(): bod.sel.Name()})
@ -201,7 +205,8 @@ func newTestBackupOp(
bod.acct, bod.acct,
bod.sel, bod.sel,
bod.sel, bod.sel,
bus) bus,
counter)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
bod.close(t, ctx) bod.close(t, ctx)
t.FailNow() t.FailNow()
@ -561,13 +566,14 @@ func ControllerWithSelector(
sel selectors.Selector, sel selectors.Selector,
ins idname.Cacher, ins idname.Cacher,
onFail func(*testing.T, context.Context), onFail func(*testing.T, context.Context),
counter *count.Bus,
) (*m365.Controller, selectors.Selector) { ) (*m365.Controller, selectors.Selector) {
ctrl, err := m365.NewController( ctrl, err := m365.NewController(
ctx, ctx,
acct, acct,
sel.PathService(), sel.PathService(),
control.DefaultOptions(), control.DefaultOptions(),
count.New()) counter)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
if onFail != nil { if onFail != nil {
onFail(t, ctx) onFail(t, ctx)

View File

@ -71,18 +71,19 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDrive() {
defer flush() defer flush()
var ( var (
tenID = tconfig.M365TenantID(t) tenID = tconfig.M365TenantID(t)
mb = evmock.NewBus() mb = evmock.NewBus()
userID = tconfig.SecondaryM365UserID(t) counter = count.New()
osel = selectors.NewOneDriveBackup([]string{userID}) userID = tconfig.SecondaryM365UserID(t)
ws = deeTD.DriveIDFromRepoRef osel = selectors.NewOneDriveBackup([]string{userID})
svc = path.OneDriveService ws = deeTD.DriveIDFromRepoRef
opts = control.DefaultOptions() svc = path.OneDriveService
opts = control.DefaultOptions()
) )
osel.Include(selTD.OneDriveBackupFolderScope(osel)) osel.Include(selTD.OneDriveBackupFolderScope(osel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -117,11 +118,12 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveBasic_groups9Versio
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
userID = tconfig.SecondaryM365UserID(t) counter = count.New()
osel = selectors.NewOneDriveBackup([]string{userID}) userID = tconfig.SecondaryM365UserID(t)
ws = deeTD.DriveIDFromRepoRef osel = selectors.NewOneDriveBackup([]string{userID})
opts = control.DefaultOptions() ws = deeTD.DriveIDFromRepoRef
opts = control.DefaultOptions()
) )
osel.Include(selTD.OneDriveBackupFolderScope(osel)) osel.Include(selTD.OneDriveBackupFolderScope(osel))
@ -132,7 +134,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveBasic_groups9Versio
mb, mb,
osel.Selector, osel.Selector,
opts, opts,
version.All8MigrateUserPNToID) version.All8MigrateUserPNToID,
counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -167,12 +170,14 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveBasic_groups9Versio
false) false)
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
notForcedFull := newTestBackupOp( notForcedFull := newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
mb, mb,
opts) opts,
counter)
notForcedFull.BackupVersion = version.Groups9Update notForcedFull.BackupVersion = version.Groups9Update
runAndCheckBackup(t, ctx, &notForcedFull, mb, false) runAndCheckBackup(t, ctx, &notForcedFull, mb, false)
@ -282,10 +287,11 @@ func runDriveIncrementalTest(
defer flush() defer flush()
var ( var (
acct = tconfig.NewM365Account(t) acct = tconfig.NewM365Account(t)
opts = control.DefaultOptions() opts = control.DefaultOptions()
mb = evmock.NewBus() mb = evmock.NewBus()
ws = deeTD.DriveIDFromRepoRef counter = count.New()
ws = deeTD.DriveIDFromRepoRef
// `now` has to be formatted with SimpleDateTimeTesting as // `now` has to be formatted with SimpleDateTimeTesting as
// some drives cannot have `:` in file/folder names // some drives cannot have `:` in file/folder names
@ -315,7 +321,7 @@ func runDriveIncrementalTest(
creds, err := acct.M365Config() creds, err := acct.M365Config()
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
ctrl, sel := ControllerWithSelector(t, ctx, acct, sel, nil, nil) ctrl, sel := ControllerWithSelector(t, ctx, acct, sel, nil, nil, counter)
ac := ctrl.AC.Drives() ac := ctrl.AC.Drives()
rh := getRestoreHandler(ctrl.AC) rh := getRestoreHandler(ctrl.AC)
@ -436,7 +442,7 @@ func runDriveIncrementalTest(
locRef) locRef)
} }
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
sel = bod.sel sel = bod.sel
@ -825,14 +831,16 @@ func runDriveIncrementalTest(
bod.ctrl = cleanCtrl bod.ctrl = cleanCtrl
var ( var (
t = suite.T() t = suite.T()
incMB = evmock.NewBus() incMB = evmock.NewBus()
incBO = newTestBackupOp( counter = count.New()
incBO = newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
incMB, incMB,
opts) opts,
counter)
) )
ctx, flush := tester.WithContext(t, ctx) ctx, flush := tester.WithContext(t, ctx)
@ -986,6 +994,7 @@ func runDriveAssistBaseGroupsUpdate(
var ( var (
whatSet = deeTD.CategoryFromRepoRef whatSet = deeTD.CategoryFromRepoRef
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
opts = control.DefaultOptions() opts = control.DefaultOptions()
) )
@ -1001,7 +1010,8 @@ func runDriveAssistBaseGroupsUpdate(
mb, mb,
sel, sel,
opts, opts,
version.All8MigrateUserPNToID) version.All8MigrateUserPNToID,
counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
suite.Run("makeAssistBackup", func() { suite.Run("makeAssistBackup", func() {
@ -1035,8 +1045,9 @@ func runDriveAssistBaseGroupsUpdate(
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.DefaultOptions() counter = count.New()
opts = control.DefaultOptions()
) )
forcedFull := newTestBackupOp( forcedFull := newTestBackupOp(
@ -1044,7 +1055,8 @@ func runDriveAssistBaseGroupsUpdate(
ctx, ctx,
bod, bod,
mb, mb,
opts) opts,
counter)
forcedFull.BackupVersion = version.Groups9Update forcedFull.BackupVersion = version.Groups9Update
runAndCheckBackup(t, ctx, &forcedFull, mb, false) runAndCheckBackup(t, ctx, &forcedFull, mb, false)
@ -1106,9 +1118,10 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
defer flush() defer flush()
var ( var (
acct = tconfig.NewM365Account(t) acct = tconfig.NewM365Account(t)
opts = control.DefaultOptions() opts = control.DefaultOptions()
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
categories = map[path.CategoryType][][]string{ categories = map[path.CategoryType][][]string{
path.FilesCategory: {{bupMD.DeltaURLsFileName}, {bupMD.PreviousPathFileName}}, path.FilesCategory: {{bupMD.DeltaURLsFileName}, {bupMD.PreviousPathFileName}},
@ -1123,7 +1136,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
acct, acct,
path.OneDriveService, path.OneDriveService,
control.DefaultOptions(), control.DefaultOptions(),
count.New()) counter)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID) userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID)
@ -1135,7 +1148,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
oldsel := selectors.NewOneDriveBackup([]string{uname}) oldsel := selectors.NewOneDriveBackup([]string{uname})
oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel)) oldsel.Include(selTD.OneDriveBackupFolderScope(oldsel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, opts, 0) bo, bod := prepNewTestBackupOp(t, ctx, mb, oldsel.Selector, opts, 0, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
sel := bod.sel sel := bod.sel
@ -1163,7 +1176,7 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
var ( var (
incMB = evmock.NewBus() incMB = evmock.NewBus()
// the incremental backup op should have a proper user ID for the id. // the incremental backup op should have a proper user ID for the id.
incBO = newTestBackupOp(t, ctx, bod, incMB, opts) incBO = newTestBackupOp(t, ctx, bod, incMB, opts, counter)
) )
require.NotEqualf( require.NotEqualf(
@ -1234,20 +1247,21 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveExtensions() {
defer flush() defer flush()
var ( var (
tenID = tconfig.M365TenantID(t) tenID = tconfig.M365TenantID(t)
mb = evmock.NewBus() mb = evmock.NewBus()
userID = tconfig.SecondaryM365UserID(t) counter = count.New()
osel = selectors.NewOneDriveBackup([]string{userID}) userID = tconfig.SecondaryM365UserID(t)
ws = deeTD.DriveIDFromRepoRef osel = selectors.NewOneDriveBackup([]string{userID})
svc = path.OneDriveService ws = deeTD.DriveIDFromRepoRef
opts = control.DefaultOptions() svc = path.OneDriveService
opts = control.DefaultOptions()
) )
opts.ItemExtensionFactory = getTestExtensionFactories() opts.ItemExtensionFactory = getTestExtensionFactories()
osel.Include(selTD.OneDriveBackupFolderScope(osel)) osel.Include(selTD.OneDriveBackupFolderScope(osel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, osel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -1326,11 +1340,12 @@ func runDriveRestoreWithAdvancedOptions(
// a backup is required to run restores // a backup is required to run restores
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.DefaultOptions() counter = count.New()
opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -1598,11 +1613,12 @@ func runDriveRestoreToAlternateProtectedResource(
// a backup is required to run restores // a backup is required to run restores
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
opts = control.DefaultOptions() counter = count.New()
opts = control.DefaultOptions()
) )
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)

View File

@ -68,7 +68,7 @@ func prepNewTestRestoreOp(
backupStore storage.Storage, backupStore storage.Storage,
backupID model.StableID, backupID model.StableID,
bus events.Eventer, bus events.Eventer,
ctr *count.Bus, counter *count.Bus,
sel selectors.Selector, sel selectors.Selector,
opts control.Options, opts control.Options,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
@ -112,7 +112,8 @@ func prepNewTestRestoreOp(
rod.acct, rod.acct,
sel, sel,
nil, nil,
rod.close) rod.close,
counter)
ro := newTestRestoreOp( ro := newTestRestoreOp(
t, t,
@ -120,7 +121,7 @@ func prepNewTestRestoreOp(
rod, rod,
backupID, backupID,
bus, bus,
ctr, counter,
opts, opts,
restoreCfg) restoreCfg)
@ -142,7 +143,7 @@ func newTestRestoreOp(
rod *restoreOpDependencies, rod *restoreOpDependencies,
backupID model.StableID, backupID model.StableID,
bus events.Eventer, bus events.Eventer,
ctr *count.Bus, counter *count.Bus,
opts control.Options, opts control.Options,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
) operations.RestoreOperation { ) operations.RestoreOperation {
@ -159,7 +160,7 @@ func newTestRestoreOp(
rod.sel, rod.sel,
restoreCfg, restoreCfg,
bus, bus,
ctr) counter)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
rod.close(t, ctx) rod.close(t, ctx)
t.FailNow() t.FailNow()

View File

@ -53,10 +53,11 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic_groups9Ve
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) counter = count.New()
opts = control.DefaultOptions() sel = selectors.NewSharePointBackup([]string{suite.its.site.ID})
ws = deeTD.DriveIDFromRepoRef opts = control.DefaultOptions()
ws = deeTD.DriveIDFromRepoRef
) )
sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Include(selTD.SharePointBackupFolderScope(sel))
@ -67,7 +68,8 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic_groups9Ve
mb, mb,
sel.Selector, sel.Selector,
opts, opts,
version.All8MigrateUserPNToID) version.All8MigrateUserPNToID,
counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -102,12 +104,14 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic_groups9Ve
false) false)
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
notForcedFull := newTestBackupOp( notForcedFull := newTestBackupOp(
t, t,
ctx, ctx,
bod, bod,
mb, mb,
opts) opts,
counter)
notForcedFull.BackupVersion = version.Groups9Update notForcedFull.BackupVersion = version.Groups9Update
runAndCheckBackup(t, ctx, &notForcedFull, mb, false) runAndCheckBackup(t, ctx, &notForcedFull, mb, false)
@ -207,14 +211,15 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointBasic() {
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) counter = count.New()
opts = control.DefaultOptions() sel = selectors.NewSharePointBackup([]string{suite.its.site.ID})
opts = control.DefaultOptions()
) )
sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Include(selTD.SharePointBackupFolderScope(sel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -236,19 +241,20 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointExtensions() {
defer flush() defer flush()
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
sel = selectors.NewSharePointBackup([]string{suite.its.site.ID}) counter = count.New()
opts = control.DefaultOptions() sel = selectors.NewSharePointBackup([]string{suite.its.site.ID})
tenID = tconfig.M365TenantID(t) opts = control.DefaultOptions()
svc = path.SharePointService tenID = tconfig.M365TenantID(t)
ws = deeTD.DriveIDFromRepoRef svc = path.SharePointService
ws = deeTD.DriveIDFromRepoRef
) )
opts.ItemExtensionFactory = getTestExtensionFactories() opts.ItemExtensionFactory = getTestExtensionFactories()
sel.Include(selTD.SharePointBackupFolderScope(sel)) sel.Include(selTD.SharePointBackupFolderScope(sel))
bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, sel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)
@ -385,6 +391,7 @@ func (suite *SharePointRestoreNightlyIntgSuite) TestRestore_Run_sharepointDelete
// run a backup // run a backup
var ( var (
mb = evmock.NewBus() mb = evmock.NewBus()
counter = count.New()
opts = control.DefaultOptions() opts = control.DefaultOptions()
graphClient = suite.its.ac.Stable.Client() graphClient = suite.its.ac.Stable.Client()
) )
@ -394,7 +401,7 @@ func (suite *SharePointRestoreNightlyIntgSuite) TestRestore_Run_sharepointDelete
bsel.Filter(bsel.Library(rc.Location)) bsel.Filter(bsel.Library(rc.Location))
bsel.DiscreteOwner = suite.its.site.ID bsel.DiscreteOwner = suite.its.site.ID
bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup) bo, bod := prepNewTestBackupOp(t, ctx, mb, bsel.Selector, opts, version.Backup, counter)
defer bod.close(t, ctx) defer bod.close(t, ctx)
runAndCheckBackup(t, ctx, &bo, mb, false) runAndCheckBackup(t, ctx, &bo, mb, false)

View File

@ -93,7 +93,8 @@ func (r repository) NewBackupWithLookup(
r.Account, r.Account,
sel, sel,
sel, // the selector acts as an IDNamer for its discrete resource owner. sel, // the selector acts as an IDNamer for its discrete resource owner.
r.Bus) r.Bus,
r.counter)
} }
// Backup retrieves a backup by id. // Backup retrieves a backup by id.

View File

@ -85,7 +85,7 @@ func connectToM365(
r.Account, r.Account,
pst, pst,
r.Opts, r.Opts,
r.counter.Local()) r.counter)
if err != nil { if err != nil {
return nil, clues.Wrap(err, "creating m365 client controller") return nil, clues.Wrap(err, "creating m365 client controller")
} }