remove ctx embedding for counter (#4497)
#### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🧹 Tech Debt/Cleanup #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
54364ed6f0
commit
5e0014307c
@ -163,7 +163,12 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
sel := exchangeBackupCreateSelectors(flags.UserFV, flags.CategoryDataFV)
|
sel := exchangeBackupCreateSelectors(flags.UserFV, flags.CategoryDataFV)
|
||||||
|
|
||||||
ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true))
|
ins, err := utils.UsersMap(
|
||||||
|
ctx,
|
||||||
|
*acct,
|
||||||
|
utils.Control(),
|
||||||
|
r.Counter(),
|
||||||
|
fault.New(true))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
|
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
|
||||||
}
|
}
|
||||||
|
|||||||
@ -22,6 +22,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/repository"
|
"github.com/alcionai/corso/src/pkg/repository"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -35,13 +36,13 @@ import (
|
|||||||
|
|
||||||
// GockClient produces a new exchange api client that can be
|
// GockClient produces a new exchange api client that can be
|
||||||
// mocked using gock.
|
// mocked using gock.
|
||||||
func gockClient(creds account.M365Config) (api.Client, error) {
|
func gockClient(creds account.M365Config, counter *count.Bus) (api.Client, error) {
|
||||||
s, err := gmock.NewService(creds)
|
s, err := gmock.NewService(creds, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Client{}, err
|
return api.Client{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
li, err := gmock.NewService(creds, graph.NoTimeout())
|
li, err := gmock.NewService(creds, counter, graph.NoTimeout())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Client{}, err
|
return api.Client{}, err
|
||||||
}
|
}
|
||||||
@ -85,10 +86,13 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
|
|||||||
creds, err := its.acct.M365Config()
|
creds, err := its.acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.ac, err = api.NewClient(creds, control.DefaultOptions())
|
its.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.gockAC, err = gockClient(creds)
|
its.gockAC, err = gockClient(creds, count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
// user drive
|
// user drive
|
||||||
|
|||||||
@ -143,7 +143,12 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
|
|||||||
|
|
||||||
sel := oneDriveBackupCreateSelectors(flags.UserFV)
|
sel := oneDriveBackupCreateSelectors(flags.UserFV)
|
||||||
|
|
||||||
ins, err := utils.UsersMap(ctx, *acct, utils.Control(), fault.New(true))
|
ins, err := utils.UsersMap(
|
||||||
|
ctx,
|
||||||
|
*acct,
|
||||||
|
utils.Control(),
|
||||||
|
r.Counter(),
|
||||||
|
fault.New(true))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
|
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 users"))
|
||||||
}
|
}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/common/idname"
|
"github.com/alcionai/corso/src/internal/common/idname"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -17,9 +18,10 @@ func UsersMap(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
acct account.Account,
|
acct account.Account,
|
||||||
co control.Options,
|
co control.Options,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) (idname.Cacher, error) {
|
) (idname.Cacher, error) {
|
||||||
au, err := makeUserAPI(acct, co)
|
au, err := makeUserAPI(acct, co, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "constructing a graph client")
|
return nil, clues.Wrap(err, "constructing a graph client")
|
||||||
}
|
}
|
||||||
@ -27,13 +29,17 @@ func UsersMap(
|
|||||||
return au.GetAllIDsAndNames(ctx, errs)
|
return au.GetAllIDsAndNames(ctx, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeUserAPI(acct account.Account, co control.Options) (api.Users, error) {
|
func makeUserAPI(
|
||||||
|
acct account.Account,
|
||||||
|
co control.Options,
|
||||||
|
counter *count.Bus,
|
||||||
|
) (api.Users, error) {
|
||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Users{}, clues.Wrap(err, "getting m365 account creds")
|
return api.Users{}, clues.Wrap(err, "getting m365 account creds")
|
||||||
}
|
}
|
||||||
|
|
||||||
cli, err := api.NewClient(creds, co)
|
cli, err := api.NewClient(creds, co, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Users{}, clues.Wrap(err, "constructing api client")
|
return api.Users{}, clues.Wrap(err, "constructing api client")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -145,7 +145,12 @@ func getControllerAndVerifyResourceOwner(
|
|||||||
return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
|
return nil, account.Account{}, nil, clues.Wrap(err, "finding m365 account details")
|
||||||
}
|
}
|
||||||
|
|
||||||
ctrl, err := m365.NewController(ctx, acct, pst, control.Options{})
|
ctrl, err := m365.NewController(
|
||||||
|
ctx,
|
||||||
|
acct,
|
||||||
|
pst,
|
||||||
|
control.Options{},
|
||||||
|
count.New())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
|
return nil, account.Account{}, nil, clues.Wrap(err, "connecting to graph api")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/credentials"
|
"github.com/alcionai/corso/src/pkg/credentials"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -62,5 +63,5 @@ func GetAC() (api.Client, error) {
|
|||||||
AzureTenantID: os.Getenv(account.AzureTenantID),
|
AzureTenantID: os.Getenv(account.AzureTenantID),
|
||||||
}
|
}
|
||||||
|
|
||||||
return api.NewClient(creds, control.DefaultOptions())
|
return api.NewClient(creds, control.DefaultOptions(), count.New())
|
||||||
}
|
}
|
||||||
|
|||||||
@ -23,8 +23,8 @@ type (
|
|||||||
pmr prefixmatcher.StringSetReader,
|
pmr prefixmatcher.StringSetReader,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
buildTreeWithBase bool,
|
buildTreeWithBase bool,
|
||||||
errs *fault.Bus,
|
|
||||||
counter *count.Bus,
|
counter *count.Bus,
|
||||||
|
errs *fault.Bus,
|
||||||
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error)
|
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -152,8 +152,8 @@ func (w Wrapper) ConsumeBackupCollections(
|
|||||||
globalExcludeSet prefixmatcher.StringSetReader,
|
globalExcludeSet prefixmatcher.StringSetReader,
|
||||||
additionalTags map[string]string,
|
additionalTags map[string]string,
|
||||||
buildTreeWithBase bool,
|
buildTreeWithBase bool,
|
||||||
errs *fault.Bus,
|
|
||||||
counter *count.Bus,
|
counter *count.Bus,
|
||||||
|
errs *fault.Bus,
|
||||||
) (*BackupStats, *details.Builder, DetailsMergeInfoer, error) {
|
) (*BackupStats, *details.Builder, DetailsMergeInfoer, error) {
|
||||||
if w.c == nil {
|
if w.c == nil {
|
||||||
return nil, nil, nil, clues.Stack(errNotConnected).WithClues(ctx)
|
return nil, nil, nil, clues.Stack(errNotConnected).WithClues(ctx)
|
||||||
|
|||||||
@ -110,8 +110,8 @@ func BenchmarkHierarchyMerge(b *testing.B) {
|
|||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
counter,
|
||||||
counter)
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
assert.Zero(t, stats.IgnoredErrorCount)
|
assert.Zero(t, stats.IgnoredErrorCount)
|
||||||
|
|||||||
@ -895,8 +895,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
|||||||
nil,
|
nil,
|
||||||
tags,
|
tags,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
counter,
|
||||||
counter)
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files")
|
assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files")
|
||||||
@ -1215,8 +1215,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
|
|||||||
nil,
|
nil,
|
||||||
tags,
|
tags,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
counter,
|
||||||
counter)
|
fault.New(true))
|
||||||
assert.NoError(t, err, clues.ToCore(err))
|
assert.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files")
|
assert.Equal(t, test.expectedUploadedFiles, stats.TotalFileCount, "total files")
|
||||||
@ -1310,8 +1310,8 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
|
|||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
count.New(),
|
||||||
count.New())
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
err = k.Compression(ctx, "gzip")
|
err = k.Compression(ctx, "gzip")
|
||||||
@ -1406,8 +1406,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
|||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
true,
|
true,
|
||||||
errs,
|
counter,
|
||||||
counter)
|
errs)
|
||||||
require.Error(t, err, clues.ToCore(err))
|
require.Error(t, err, clues.ToCore(err))
|
||||||
assert.Zero(t, stats.ErrorCount, "error count")
|
assert.Zero(t, stats.ErrorCount, "error count")
|
||||||
assert.Zero(t, counter.Get(count.PersistenceErrors), "error count")
|
assert.Zero(t, counter.Get(count.PersistenceErrors), "error count")
|
||||||
@ -1494,8 +1494,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections()
|
|||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
count.New(),
|
||||||
count.New())
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
assert.Equal(t, BackupStats{}, *s)
|
assert.Equal(t, BackupStats{}, *s)
|
||||||
@ -1656,8 +1656,8 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
|
|||||||
nil,
|
nil,
|
||||||
nil,
|
nil,
|
||||||
false,
|
false,
|
||||||
fault.New(true),
|
counter,
|
||||||
counter)
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
require.Zero(t, stats.ErrorCount)
|
require.Zero(t, stats.ErrorCount)
|
||||||
require.Zero(t, counter.Get(count.PersistenceErrors))
|
require.Zero(t, counter.Get(count.PersistenceErrors))
|
||||||
@ -1786,8 +1786,8 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
|
|||||||
excluded,
|
excluded,
|
||||||
nil,
|
nil,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
counter,
|
||||||
counter)
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.Equal(t, test.expectedCachedItems, stats.CachedFileCount)
|
assert.Equal(t, test.expectedCachedItems, stats.CachedFileCount)
|
||||||
assert.Equal(t, test.expectedUncachedItems, stats.UncachedFileCount)
|
assert.Equal(t, test.expectedUncachedItems, stats.UncachedFileCount)
|
||||||
|
|||||||
@ -17,6 +17,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365/service/sharepoint"
|
"github.com/alcionai/corso/src/internal/m365/service/sharepoint"
|
||||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||||
bupMD "github.com/alcionai/corso/src/pkg/backup/metadata"
|
bupMD "github.com/alcionai/corso/src/pkg/backup/metadata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/filters"
|
"github.com/alcionai/corso/src/pkg/filters"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
@ -35,6 +36,7 @@ import (
|
|||||||
func (ctrl *Controller) ProduceBackupCollections(
|
func (ctrl *Controller) ProduceBackupCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
bpc inject.BackupProducerConfig,
|
bpc inject.BackupProducerConfig,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
||||||
service := bpc.Selector.PathService()
|
service := bpc.Selector.PathService()
|
||||||
@ -94,6 +96,7 @@ func (ctrl *Controller) ProduceBackupCollections(
|
|||||||
ctrl.AC,
|
ctrl.AC,
|
||||||
ctrl.credentials,
|
ctrl.credentials,
|
||||||
ctrl.UpdateStatus,
|
ctrl.UpdateStatus,
|
||||||
|
counter,
|
||||||
errs)
|
errs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, false, err
|
return nil, nil, false, err
|
||||||
|
|||||||
@ -22,6 +22,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/internal/version"
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -61,7 +62,10 @@ func (suite *DataCollectionIntgSuite) SetupSuite() {
|
|||||||
|
|
||||||
suite.tenantID = creds.AzureTenantID
|
suite.tenantID = creds.AzureTenantID
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -251,6 +255,7 @@ func (suite *DataCollectionIntgSuite) TestDataCollections_invalidResourceOwner()
|
|||||||
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
assert.Error(t, err, clues.ToCore(err))
|
assert.Error(t, err, clues.ToCore(err))
|
||||||
assert.False(t, canUsePreviousBackup, "can use previous backup")
|
assert.False(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -311,6 +316,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
|
|||||||
suite.ac,
|
suite.ac,
|
||||||
ctrl.credentials,
|
ctrl.credentials,
|
||||||
ctrl.UpdateStatus,
|
ctrl.UpdateStatus,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -398,6 +404,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
|
|||||||
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -448,6 +455,7 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
|
|||||||
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
cols, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -534,6 +542,7 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint()
|
|||||||
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -634,6 +643,7 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint_In
|
|||||||
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
collections, excludes, canUsePreviousBackup, err := ctrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -21,7 +22,10 @@ type oneDriveService struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) {
|
func NewOneDriveService(credentials account.M365Config) (*oneDriveService, error) {
|
||||||
ac, err := api.NewClient(credentials, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
credentials,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,6 +17,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/common/str"
|
"github.com/alcionai/corso/src/internal/common/str"
|
||||||
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
|
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
|
||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -191,6 +192,7 @@ func driveItemWriter(
|
|||||||
nicu NewItemContentUploader,
|
nicu NewItemContentUploader,
|
||||||
driveID, itemID string,
|
driveID, itemID string,
|
||||||
itemSize int64,
|
itemSize int64,
|
||||||
|
counter *count.Bus,
|
||||||
) (io.Writer, string, error) {
|
) (io.Writer, string, error) {
|
||||||
ctx = clues.Add(ctx, "upload_item_id", itemID)
|
ctx = clues.Add(ctx, "upload_item_id", itemID)
|
||||||
|
|
||||||
@ -199,7 +201,11 @@ func driveItemWriter(
|
|||||||
return nil, "", clues.Stack(err)
|
return nil, "", clues.Stack(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
iw := graph.NewLargeItemWriter(itemID, ptr.Val(icu.GetUploadUrl()), itemSize)
|
iw := graph.NewLargeItemWriter(
|
||||||
|
itemID,
|
||||||
|
ptr.Val(icu.GetUploadUrl()),
|
||||||
|
itemSize,
|
||||||
|
counter)
|
||||||
|
|
||||||
return iw, ptr.Val(icu.GetUploadUrl()), nil
|
return iw, ptr.Val(icu.GetUploadUrl()), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -232,7 +233,10 @@ func (suite *OneDriveIntgSuite) SetupSuite() {
|
|||||||
|
|
||||||
suite.creds = creds
|
suite.creds = creds
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -22,6 +22,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/control/testdata"
|
"github.com/alcionai/corso/src/pkg/control/testdata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -231,7 +232,8 @@ func (suite *ItemIntegrationSuite) TestItemWriter() {
|
|||||||
rh,
|
rh,
|
||||||
test.driveID,
|
test.driveID,
|
||||||
ptr.Val(newItem.GetId()),
|
ptr.Val(newItem.GetId()),
|
||||||
writeSize)
|
writeSize,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
// Using a 32 KB buffer for the copy allows us to validate the
|
// Using a 32 KB buffer for the copy allows us to validate the
|
||||||
|
|||||||
@ -788,7 +788,13 @@ func restoreFile(
|
|||||||
return "", details.ItemInfo{}, err
|
return "", details.ItemInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
w, uploadURL, err := driveItemWriter(ctx, ir, driveID, ptr.Val(newItem.GetId()), ss.Size())
|
w, uploadURL, err := driveItemWriter(
|
||||||
|
ctx,
|
||||||
|
ir,
|
||||||
|
driveID,
|
||||||
|
ptr.Val(newItem.GetId()),
|
||||||
|
ss.Size(),
|
||||||
|
ctr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", details.ItemInfo{}, clues.Wrap(err, "get item upload session")
|
return "", details.ItemInfo{}, clues.Wrap(err, "get item upload session")
|
||||||
}
|
}
|
||||||
@ -841,7 +847,11 @@ func restoreFile(
|
|||||||
|
|
||||||
// refresh the io.Writer to restart the upload
|
// refresh the io.Writer to restart the upload
|
||||||
// TODO: @vkamra verify if var session is the desired input
|
// TODO: @vkamra verify if var session is the desired input
|
||||||
w = graph.NewLargeItemWriter(ptr.Val(newItem.GetId()), uploadURL, ss.Size())
|
w = graph.NewLargeItemWriter(
|
||||||
|
ptr.Val(newItem.GetId()),
|
||||||
|
uploadURL,
|
||||||
|
ss.Size(),
|
||||||
|
ctr)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -24,6 +24,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/control/testdata"
|
"github.com/alcionai/corso/src/pkg/control/testdata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||||
@ -61,7 +62,10 @@ func (suite *URLCacheIntegrationSuite) SetupSuite() {
|
|||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user)
|
drive, err := suite.ac.Users().GetDefaultDrive(ctx, suite.user)
|
||||||
@ -159,7 +163,7 @@ func (suite *URLCacheIntegrationSuite) TestURLCacheBasic() {
|
|||||||
require.Equal(t, false, props.isDeleted)
|
require.Equal(t, false, props.isDeleted)
|
||||||
|
|
||||||
// Validate download URL
|
// Validate download URL
|
||||||
c := graph.NewNoTimeoutHTTPWrapper()
|
c := graph.NewNoTimeoutHTTPWrapper(count.New())
|
||||||
|
|
||||||
resp, err := c.Request(
|
resp, err := c.Request(
|
||||||
ctx,
|
ctx,
|
||||||
|
|||||||
@ -27,6 +27,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -432,7 +433,10 @@ func (suite *BackupIntgSuite) SetupSuite() {
|
|||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.tenantID = creds.AzureTenantID
|
suite.tenantID = creds.AzureTenantID
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -699,7 +700,10 @@ func (suite *ContainerResolverSuite) SetupSuite() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ContainerResolverSuite) TestPopulate() {
|
func (suite *ContainerResolverSuite) TestPopulate() {
|
||||||
ac, err := api.NewClient(suite.credentials, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
suite.credentials,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(suite.T(), err, clues.ToCore(err))
|
require.NoError(suite.T(), err, clues.ToCore(err))
|
||||||
|
|
||||||
eventFunc := func(t *testing.T) graph.ContainerResolver {
|
eventFunc := func(t *testing.T) graph.ContainerResolver {
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -31,7 +32,10 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
|
|||||||
|
|
||||||
its.creds = creds
|
its.creds = creds
|
||||||
|
|
||||||
its.ac, err = api.NewClient(creds, control.DefaultOptions())
|
its.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.userID = tconfig.GetM365UserID(ctx)
|
its.userID = tconfig.GetM365UserID(ctx)
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -83,7 +84,10 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
ac, err := api.NewClient(suite.credentials, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
suite.credentials,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
acm := ac.Mail()
|
acm := ac.Mail()
|
||||||
|
|||||||
@ -44,7 +44,10 @@ func (suite *RestoreIntgSuite) SetupSuite() {
|
|||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.credentials = m365
|
suite.credentials = m365
|
||||||
suite.ac, err = api.NewClient(m365, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
m365,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -24,6 +24,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
"github.com/alcionai/corso/src/pkg/backup/metadata"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -463,7 +464,10 @@ func (suite *BackupIntgSuite) SetupSuite() {
|
|||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.tenantID = creds.AzureTenantID
|
suite.tenantID = creds.AzureTenantID
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365/support"
|
"github.com/alcionai/corso/src/internal/m365/support"
|
||||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
@ -60,6 +61,7 @@ func CollectPages(
|
|||||||
ac api.Client,
|
ac api.Client,
|
||||||
scope selectors.SharePointScope,
|
scope selectors.SharePointScope,
|
||||||
su support.StatusUpdater,
|
su support.StatusUpdater,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, error) {
|
) ([]data.BackupCollection, error) {
|
||||||
logger.Ctx(ctx).Debug("creating SharePoint Pages collections")
|
logger.Ctx(ctx).Debug("creating SharePoint Pages collections")
|
||||||
@ -74,7 +76,8 @@ func CollectPages(
|
|||||||
adpt, err := graph.CreateAdapter(
|
adpt, err := graph.CreateAdapter(
|
||||||
creds.AzureTenantID,
|
creds.AzureTenantID,
|
||||||
creds.AzureClientID,
|
creds.AzureClientID,
|
||||||
creds.AzureClientSecret)
|
creds.AzureClientSecret,
|
||||||
|
counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "creating azure client adapter")
|
return nil, clues.Wrap(err, "creating azure client adapter")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -15,6 +15,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/internal/version"
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -48,12 +49,16 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
|
|||||||
var (
|
var (
|
||||||
siteID = tconfig.M365SiteID(t)
|
siteID = tconfig.M365SiteID(t)
|
||||||
a = tconfig.NewM365Account(t)
|
a = tconfig.NewM365Account(t)
|
||||||
|
counter = count.New()
|
||||||
)
|
)
|
||||||
|
|
||||||
creds, err := a.M365Config()
|
creds, err := a.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
ac, err := api.NewClient(creds, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
counter)
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
bpc := inject.BackupProducerConfig{
|
bpc := inject.BackupProducerConfig{
|
||||||
@ -71,6 +76,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
|
|||||||
ac,
|
ac,
|
||||||
sel.Lists(selectors.Any())[0],
|
sel.Lists(selectors.Any())[0],
|
||||||
(&MockGraphService{}).UpdateStatus,
|
(&MockGraphService{}).UpdateStatus,
|
||||||
|
counter,
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
assert.NoError(t, err, clues.ToCore(err))
|
assert.NoError(t, err, clues.ToCore(err))
|
||||||
assert.NotEmpty(t, col)
|
assert.NotEmpty(t, col)
|
||||||
|
|||||||
@ -22,6 +22,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
"github.com/alcionai/corso/src/pkg/control/testdata"
|
"github.com/alcionai/corso/src/pkg/control/testdata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -45,7 +46,10 @@ func (suite *SharePointCollectionSuite) SetupSuite() {
|
|||||||
|
|
||||||
suite.creds = m365
|
suite.creds = m365
|
||||||
|
|
||||||
ac, err := api.NewClient(m365, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
m365,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac = ac
|
suite.ac = ac
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
"github.com/alcionai/corso/src/internal/m365/support"
|
"github.com/alcionai/corso/src/internal/m365/support"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -50,7 +51,8 @@ func createTestService(t *testing.T, credentials account.M365Config) *graph.Serv
|
|||||||
adapter, err := graph.CreateAdapter(
|
adapter, err := graph.CreateAdapter(
|
||||||
credentials.AzureTenantID,
|
credentials.AzureTenantID,
|
||||||
credentials.AzureClientID,
|
credentials.AzureClientID,
|
||||||
credentials.AzureClientSecret)
|
credentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, "creating microsoft graph service for exchange", clues.ToCore(err))
|
require.NoError(t, err, "creating microsoft graph service for exchange", clues.ToCore(err))
|
||||||
|
|
||||||
return graph.NewService(adapter)
|
return graph.NewService(adapter)
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
)
|
)
|
||||||
@ -68,6 +69,7 @@ func NewController(
|
|||||||
acct account.Account,
|
acct account.Account,
|
||||||
pst path.ServiceType,
|
pst path.ServiceType,
|
||||||
co control.Options,
|
co control.Options,
|
||||||
|
counter *count.Bus,
|
||||||
) (*Controller, error) {
|
) (*Controller, error) {
|
||||||
graph.InitializeConcurrencyLimiter(ctx, pst == path.ExchangeService, co.Parallelism.ItemFetch)
|
graph.InitializeConcurrencyLimiter(ctx, pst == path.ExchangeService, co.Parallelism.ItemFetch)
|
||||||
|
|
||||||
@ -76,7 +78,7 @@ func NewController(
|
|||||||
return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx)
|
return nil, clues.Wrap(err, "retrieving m365 account configuration").WithClues(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
ac, err := api.NewClient(creds, co)
|
ac, err := api.NewClient(creds, co, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "creating api client").WithClues(ctx)
|
return nil, clues.Wrap(err, "creating api client").WithClues(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -637,6 +637,7 @@ func runBackupAndCompare(
|
|||||||
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -1232,6 +1233,7 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
|
|||||||
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
@ -1411,6 +1413,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
|
|||||||
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
dcs, excludes, canUsePreviousBackup, err := backupCtrl.ProduceBackupCollections(
|
||||||
ctx,
|
ctx,
|
||||||
bpc,
|
bpc,
|
||||||
|
count.New(),
|
||||||
fault.New(true))
|
fault.New(true))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
assert.True(t, canUsePreviousBackup, "can use previous backup")
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BetaClientSuite struct {
|
type BetaClientSuite struct {
|
||||||
@ -45,7 +46,8 @@ func (suite *BetaClientSuite) TestCreateBetaClient() {
|
|||||||
adpt, err := graph.CreateAdapter(
|
adpt, err := graph.CreateAdapter(
|
||||||
suite.credentials.AzureTenantID,
|
suite.credentials.AzureTenantID,
|
||||||
suite.credentials.AzureClientID,
|
suite.credentials.AzureClientID,
|
||||||
suite.credentials.AzureClientSecret)
|
suite.credentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
|
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
@ -66,7 +68,8 @@ func (suite *BetaClientSuite) TestBasicClientGetFunctionality() {
|
|||||||
adpt, err := graph.CreateAdapter(
|
adpt, err := graph.CreateAdapter(
|
||||||
suite.credentials.AzureTenantID,
|
suite.credentials.AzureTenantID,
|
||||||
suite.credentials.AzureClientID,
|
suite.credentials.AzureClientID,
|
||||||
suite.credentials.AzureClientSecret)
|
suite.credentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
client := NewBetaClient(adpt)
|
client := NewBetaClient(adpt)
|
||||||
|
|||||||
@ -292,6 +292,7 @@ func (tf *timedFence) RaiseFence(seconds time.Duration) {
|
|||||||
// throttlingMiddleware is used to ensure we don't overstep per-min request limits.
|
// throttlingMiddleware is used to ensure we don't overstep per-min request limits.
|
||||||
type throttlingMiddleware struct {
|
type throttlingMiddleware struct {
|
||||||
tf *timedFence
|
tf *timedFence
|
||||||
|
counter *count.Bus
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mw *throttlingMiddleware) Intercept(
|
func (mw *throttlingMiddleware) Intercept(
|
||||||
@ -314,8 +315,7 @@ func (mw *throttlingMiddleware) Intercept(
|
|||||||
return resp, nil
|
return resp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
countBus := count.Ctx(req.Context())
|
mw.counter.Inc(count.ThrottledAPICalls)
|
||||||
countBus.Inc(count.ThrottledAPICalls)
|
|
||||||
|
|
||||||
// if all prior conditions pass, we need to add a fence that blocks
|
// if all prior conditions pass, we need to add a fence that blocks
|
||||||
// calls, globally, from progressing until the timeout retry-after
|
// calls, globally, from progressing until the timeout retry-after
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import (
|
|||||||
"golang.org/x/net/context"
|
"golang.org/x/net/context"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ConcurrencyMWUnitTestSuite struct {
|
type ConcurrencyMWUnitTestSuite struct {
|
||||||
@ -307,7 +308,10 @@ func (suite *ConcurrencyMWUnitTestSuite) TestThrottlingMiddleware() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
tm := throttlingMiddleware{newTimedFence()}
|
tm := &throttlingMiddleware{
|
||||||
|
tf: newTimedFence(),
|
||||||
|
counter: count.New(),
|
||||||
|
}
|
||||||
|
|
||||||
req := &http.Request{}
|
req := &http.Request{}
|
||||||
req = req.WithContext(ctx)
|
req = req.WithContext(ctx)
|
||||||
|
|||||||
@ -13,6 +13,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/events"
|
"github.com/alcionai/corso/src/internal/events"
|
||||||
"github.com/alcionai/corso/src/internal/version"
|
"github.com/alcionai/corso/src/internal/version"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -36,12 +37,15 @@ type Requester interface {
|
|||||||
// and consume relatively unbound socket connections. It is important
|
// and consume relatively unbound socket connections. It is important
|
||||||
// to centralize this client to be passed downstream where api calls
|
// to centralize this client to be passed downstream where api calls
|
||||||
// can utilize it on a per-download basis.
|
// can utilize it on a per-download basis.
|
||||||
func NewHTTPWrapper(opts ...Option) *httpWrapper {
|
func NewHTTPWrapper(
|
||||||
|
counter *count.Bus,
|
||||||
|
opts ...Option,
|
||||||
|
) *httpWrapper {
|
||||||
var (
|
var (
|
||||||
cc = populateConfig(opts...)
|
cc = populateConfig(opts...)
|
||||||
rt = customTransport{
|
rt = customTransport{
|
||||||
n: pipeline{
|
n: pipeline{
|
||||||
middlewares: internalMiddleware(cc),
|
middlewares: internalMiddleware(cc, counter),
|
||||||
transport: defaultTransport(),
|
transport: defaultTransport(),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -66,9 +70,12 @@ func NewHTTPWrapper(opts ...Option) *httpWrapper {
|
|||||||
// and consume relatively unbound socket connections. It is important
|
// and consume relatively unbound socket connections. It is important
|
||||||
// to centralize this client to be passed downstream where api calls
|
// to centralize this client to be passed downstream where api calls
|
||||||
// can utilize it on a per-download basis.
|
// can utilize it on a per-download basis.
|
||||||
func NewNoTimeoutHTTPWrapper(opts ...Option) *httpWrapper {
|
func NewNoTimeoutHTTPWrapper(
|
||||||
|
counter *count.Bus,
|
||||||
|
opts ...Option,
|
||||||
|
) *httpWrapper {
|
||||||
opts = append(opts, NoTimeout())
|
opts = append(opts, NoTimeout())
|
||||||
return NewHTTPWrapper(opts...)
|
return NewHTTPWrapper(counter, opts...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -178,7 +185,15 @@ func defaultTransport() http.RoundTripper {
|
|||||||
return defaultTransport
|
return defaultTransport
|
||||||
}
|
}
|
||||||
|
|
||||||
func internalMiddleware(cc *clientConfig) []khttp.Middleware {
|
func internalMiddleware(
|
||||||
|
cc *clientConfig,
|
||||||
|
counter *count.Bus,
|
||||||
|
) []khttp.Middleware {
|
||||||
|
throttler := &throttlingMiddleware{
|
||||||
|
tf: newTimedFence(),
|
||||||
|
counter: counter,
|
||||||
|
}
|
||||||
|
|
||||||
mw := []khttp.Middleware{
|
mw := []khttp.Middleware{
|
||||||
&RetryMiddleware{
|
&RetryMiddleware{
|
||||||
MaxRetries: cc.maxRetries,
|
MaxRetries: cc.maxRetries,
|
||||||
@ -187,7 +202,7 @@ func internalMiddleware(cc *clientConfig) []khttp.Middleware {
|
|||||||
khttp.NewRetryHandler(),
|
khttp.NewRetryHandler(),
|
||||||
khttp.NewRedirectHandler(),
|
khttp.NewRedirectHandler(),
|
||||||
&LoggingMiddleware{},
|
&LoggingMiddleware{},
|
||||||
&throttlingMiddleware{newTimedFence()},
|
throttler,
|
||||||
&RateLimiterMiddleware{},
|
&RateLimiterMiddleware{},
|
||||||
&MetricsMiddleware{},
|
&MetricsMiddleware{},
|
||||||
}
|
}
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type HTTPWrapperIntgSuite struct {
|
type HTTPWrapperIntgSuite struct {
|
||||||
@ -34,7 +35,7 @@ func (suite *HTTPWrapperIntgSuite) TestNewHTTPWrapper() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
hw := NewHTTPWrapper()
|
hw := NewHTTPWrapper(count.New())
|
||||||
|
|
||||||
resp, err := hw.Request(
|
resp, err := hw.Request(
|
||||||
ctx,
|
ctx,
|
||||||
@ -107,7 +108,7 @@ func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_redirectMiddleware() {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
hw := NewHTTPWrapper(appendMiddleware(&mwResp))
|
hw := NewHTTPWrapper(count.New(), appendMiddleware(&mwResp))
|
||||||
|
|
||||||
resp, err := hw.Request(ctx, http.MethodGet, url, nil, nil)
|
resp, err := hw.Request(ctx, http.MethodGet, url, nil, nil)
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
@ -175,6 +176,7 @@ func (suite *HTTPWrapperUnitSuite) TestNewHTTPWrapper_http2StreamErrorRetries()
|
|||||||
}
|
}
|
||||||
|
|
||||||
hw := NewHTTPWrapper(
|
hw := NewHTTPWrapper(
|
||||||
|
count.New(),
|
||||||
appendMiddleware(&mwResp),
|
appendMiddleware(&mwResp),
|
||||||
MaxConnectionRetries(test.retries))
|
MaxConnectionRetries(test.retries))
|
||||||
|
|
||||||
|
|||||||
@ -323,7 +323,9 @@ func (mw RetryMiddleware) getRetryDelay(
|
|||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
// MetricsMiddleware aggregates per-request metrics on the events bus
|
// MetricsMiddleware aggregates per-request metrics on the events bus
|
||||||
type MetricsMiddleware struct{}
|
type MetricsMiddleware struct {
|
||||||
|
counter *count.Bus
|
||||||
|
}
|
||||||
|
|
||||||
const xmruHeader = "x-ms-resource-unit"
|
const xmruHeader = "x-ms-resource-unit"
|
||||||
|
|
||||||
@ -362,8 +364,7 @@ func (mw *MetricsMiddleware) Intercept(
|
|||||||
xmrui = 1
|
xmrui = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
countBus := count.Ctx(req.Context())
|
mw.counter.Add(count.APICallTokensConsumed, int64(xmrui))
|
||||||
countBus.Add(count.APICallTokensConsumed, int64(xmrui))
|
|
||||||
|
|
||||||
events.IncN(xmrui, events.APICall, xmruHeader)
|
events.IncN(xmrui, events.APICall, xmruHeader)
|
||||||
|
|
||||||
|
|||||||
@ -25,6 +25,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -109,7 +110,7 @@ func mockAdapter(
|
|||||||
var (
|
var (
|
||||||
clientOptions = msgraphsdkgo.GetDefaultClientOptions()
|
clientOptions = msgraphsdkgo.GetDefaultClientOptions()
|
||||||
cc = populateConfig(MinimumBackoff(10 * time.Millisecond))
|
cc = populateConfig(MinimumBackoff(10 * time.Millisecond))
|
||||||
middlewares = append(kiotaMiddlewares(&clientOptions, cc), mw)
|
middlewares = append(kiotaMiddlewares(&clientOptions, cc, count.New()), mw)
|
||||||
httpClient = msgraphgocore.GetDefaultClient(&clientOptions, middlewares...)
|
httpClient = msgraphgocore.GetDefaultClient(&clientOptions, middlewares...)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@ -7,13 +7,19 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) {
|
func NewService(
|
||||||
|
creds account.M365Config,
|
||||||
|
counter *count.Bus,
|
||||||
|
opts ...graph.Option,
|
||||||
|
) (*graph.Service, error) {
|
||||||
a, err := CreateAdapter(
|
a, err := CreateAdapter(
|
||||||
creds.AzureTenantID,
|
creds.AzureTenantID,
|
||||||
creds.AzureClientID,
|
creds.AzureClientID,
|
||||||
creds.AzureClientSecret,
|
creds.AzureClientSecret,
|
||||||
|
counter,
|
||||||
opts...)
|
opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "generating graph adapter")
|
return nil, clues.Wrap(err, "generating graph adapter")
|
||||||
@ -26,6 +32,7 @@ func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service,
|
|||||||
// enable interceptions via gock to make it mockable.
|
// enable interceptions via gock to make it mockable.
|
||||||
func CreateAdapter(
|
func CreateAdapter(
|
||||||
tenant, client, secret string,
|
tenant, client, secret string,
|
||||||
|
counter *count.Bus,
|
||||||
opts ...graph.Option,
|
opts ...graph.Option,
|
||||||
) (*msgraphsdkgo.GraphRequestAdapter, error) {
|
) (*msgraphsdkgo.GraphRequestAdapter, error) {
|
||||||
auth, err := graph.GetAuth(tenant, client, secret)
|
auth, err := graph.GetAuth(tenant, client, secret)
|
||||||
@ -33,7 +40,7 @@ func CreateAdapter(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient, _ := graph.KiotaHTTPClient(opts...)
|
httpClient, _ := graph.KiotaHTTPClient(counter, opts...)
|
||||||
|
|
||||||
// This makes sure that we are able to intercept any requests via
|
// This makes sure that we are able to intercept any requests via
|
||||||
// gock. Only necessary for testing.
|
// gock. Only necessary for testing.
|
||||||
|
|||||||
@ -17,6 +17,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/common/crash"
|
"github.com/alcionai/corso/src/internal/common/crash"
|
||||||
"github.com/alcionai/corso/src/internal/common/idname"
|
"github.com/alcionai/corso/src/internal/common/idname"
|
||||||
"github.com/alcionai/corso/src/internal/events"
|
"github.com/alcionai/corso/src/internal/events"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/filters"
|
"github.com/alcionai/corso/src/pkg/filters"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
@ -104,6 +105,7 @@ func (s Service) Serialize(object serialization.Parsable) ([]byte, error) {
|
|||||||
// to create a graph api client connection.
|
// to create a graph api client connection.
|
||||||
func CreateAdapter(
|
func CreateAdapter(
|
||||||
tenant, client, secret string,
|
tenant, client, secret string,
|
||||||
|
counter *count.Bus,
|
||||||
opts ...Option,
|
opts ...Option,
|
||||||
) (abstractions.RequestAdapter, error) {
|
) (abstractions.RequestAdapter, error) {
|
||||||
auth, err := GetAuth(tenant, client, secret)
|
auth, err := GetAuth(tenant, client, secret)
|
||||||
@ -111,7 +113,7 @@ func CreateAdapter(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
httpClient, cc := KiotaHTTPClient(opts...)
|
httpClient, cc := KiotaHTTPClient(counter, opts...)
|
||||||
|
|
||||||
adpt, err := msgraphsdkgo.NewGraphRequestAdapterWithParseNodeFactoryAndSerializationWriterFactoryAndHttpClient(
|
adpt, err := msgraphsdkgo.NewGraphRequestAdapterWithParseNodeFactoryAndSerializationWriterFactoryAndHttpClient(
|
||||||
auth,
|
auth,
|
||||||
@ -148,11 +150,14 @@ func GetAuth(tenant string, client string, secret string) (*kauth.AzureIdentityA
|
|||||||
// and consume relatively unbound socket connections. It is important
|
// and consume relatively unbound socket connections. It is important
|
||||||
// to centralize this client to be passed downstream where api calls
|
// to centralize this client to be passed downstream where api calls
|
||||||
// can utilize it on a per-download basis.
|
// can utilize it on a per-download basis.
|
||||||
func KiotaHTTPClient(opts ...Option) (*http.Client, *clientConfig) {
|
func KiotaHTTPClient(
|
||||||
|
counter *count.Bus,
|
||||||
|
opts ...Option,
|
||||||
|
) (*http.Client, *clientConfig) {
|
||||||
var (
|
var (
|
||||||
clientOptions = msgraphsdkgo.GetDefaultClientOptions()
|
clientOptions = msgraphsdkgo.GetDefaultClientOptions()
|
||||||
cc = populateConfig(opts...)
|
cc = populateConfig(opts...)
|
||||||
middlewares = kiotaMiddlewares(&clientOptions, cc)
|
middlewares = kiotaMiddlewares(&clientOptions, cc, counter)
|
||||||
httpClient = msgraphgocore.GetDefaultClient(&clientOptions, middlewares...)
|
httpClient = msgraphgocore.GetDefaultClient(&clientOptions, middlewares...)
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -271,6 +276,7 @@ func MaxConnectionRetries(max int) Option {
|
|||||||
func kiotaMiddlewares(
|
func kiotaMiddlewares(
|
||||||
options *msgraphgocore.GraphClientOptions,
|
options *msgraphgocore.GraphClientOptions,
|
||||||
cc *clientConfig,
|
cc *clientConfig,
|
||||||
|
counter *count.Bus,
|
||||||
) []khttp.Middleware {
|
) []khttp.Middleware {
|
||||||
mw := []khttp.Middleware{
|
mw := []khttp.Middleware{
|
||||||
msgraphgocore.NewGraphTelemetryHandler(options),
|
msgraphgocore.NewGraphTelemetryHandler(options),
|
||||||
@ -291,9 +297,14 @@ func kiotaMiddlewares(
|
|||||||
mw = append(mw, concurrencyLimitMiddlewareSingleton)
|
mw = append(mw, concurrencyLimitMiddlewareSingleton)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
throttler := &throttlingMiddleware{
|
||||||
|
tf: newTimedFence(),
|
||||||
|
counter: counter,
|
||||||
|
}
|
||||||
|
|
||||||
mw = append(
|
mw = append(
|
||||||
mw,
|
mw,
|
||||||
&throttlingMiddleware{newTimedFence()},
|
throttler,
|
||||||
&RateLimiterMiddleware{},
|
&RateLimiterMiddleware{},
|
||||||
&MetricsMiddleware{})
|
&MetricsMiddleware{})
|
||||||
|
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type GraphIntgSuite struct {
|
type GraphIntgSuite struct {
|
||||||
@ -59,7 +60,8 @@ func (suite *GraphIntgSuite) TestCreateAdapter() {
|
|||||||
adpt, err := CreateAdapter(
|
adpt, err := CreateAdapter(
|
||||||
suite.fakeCredentials.AzureTenantID,
|
suite.fakeCredentials.AzureTenantID,
|
||||||
suite.fakeCredentials.AzureClientID,
|
suite.fakeCredentials.AzureClientID,
|
||||||
suite.fakeCredentials.AzureClientSecret)
|
suite.fakeCredentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
|
|
||||||
assert.NoError(t, err, clues.ToCore(err))
|
assert.NoError(t, err, clues.ToCore(err))
|
||||||
assert.NotNil(t, adpt)
|
assert.NotNil(t, adpt)
|
||||||
@ -143,7 +145,7 @@ func (suite *GraphIntgSuite) TestHTTPClient() {
|
|||||||
suite.Run(test.name, func() {
|
suite.Run(test.name, func() {
|
||||||
t := suite.T()
|
t := suite.T()
|
||||||
|
|
||||||
cli, cc := KiotaHTTPClient(test.opts...)
|
cli, cc := KiotaHTTPClient(count.New(), test.opts...)
|
||||||
assert.NotNil(t, cli)
|
assert.NotNil(t, cli)
|
||||||
test.check(t, cli)
|
test.check(t, cli)
|
||||||
test.checkConfig(t, cc)
|
test.checkConfig(t, cc)
|
||||||
@ -156,7 +158,8 @@ func (suite *GraphIntgSuite) TestSerializationEndPoint() {
|
|||||||
adpt, err := CreateAdapter(
|
adpt, err := CreateAdapter(
|
||||||
suite.fakeCredentials.AzureTenantID,
|
suite.fakeCredentials.AzureTenantID,
|
||||||
suite.fakeCredentials.AzureClientID,
|
suite.fakeCredentials.AzureClientID,
|
||||||
suite.fakeCredentials.AzureClientSecret)
|
suite.fakeCredentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
serv := NewService(adpt)
|
serv := NewService(adpt)
|
||||||
@ -189,6 +192,7 @@ func (suite *GraphIntgSuite) TestAdapterWrap_catchesPanic() {
|
|||||||
suite.credentials.AzureTenantID,
|
suite.credentials.AzureTenantID,
|
||||||
suite.credentials.AzureClientID,
|
suite.credentials.AzureClientID,
|
||||||
suite.credentials.AzureClientSecret,
|
suite.credentials.AzureClientSecret,
|
||||||
|
count.New(),
|
||||||
appendMiddleware(&alwaysPanicMiddleware))
|
appendMiddleware(&alwaysPanicMiddleware))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
@ -210,13 +214,13 @@ func (suite *GraphIntgSuite) TestAdapterWrap_retriesConnectionClose() {
|
|||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
url := "https://graph.microsoft.com/fnords/beaux/regard"
|
url := "https://graph.microsoft.com/fnords/beaux/regard"
|
||||||
count := 0
|
retryInc := 0
|
||||||
|
|
||||||
// the panics should get caught and returned as errors
|
// the panics should get caught and returned as errors
|
||||||
alwaysECONNRESET := mwForceResp{
|
alwaysECONNRESET := mwForceResp{
|
||||||
err: syscall.ECONNRESET,
|
err: syscall.ECONNRESET,
|
||||||
alternate: func(req *http.Request) (bool, *http.Response, error) {
|
alternate: func(req *http.Request) (bool, *http.Response, error) {
|
||||||
count++
|
retryInc++
|
||||||
return false, nil, nil
|
return false, nil, nil
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -225,18 +229,19 @@ func (suite *GraphIntgSuite) TestAdapterWrap_retriesConnectionClose() {
|
|||||||
suite.credentials.AzureTenantID,
|
suite.credentials.AzureTenantID,
|
||||||
suite.credentials.AzureClientID,
|
suite.credentials.AzureClientID,
|
||||||
suite.credentials.AzureClientSecret,
|
suite.credentials.AzureClientSecret,
|
||||||
|
count.New(),
|
||||||
appendMiddleware(&alwaysECONNRESET))
|
appendMiddleware(&alwaysECONNRESET))
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
// the query doesn't matter
|
// the query doesn't matter
|
||||||
_, err = users.NewItemCalendarsItemEventsDeltaRequestBuilder(url, adpt).Get(ctx, nil)
|
_, err = users.NewItemCalendarsItemEventsDeltaRequestBuilder(url, adpt).Get(ctx, nil)
|
||||||
require.ErrorIs(t, err, syscall.ECONNRESET, clues.ToCore(err))
|
require.ErrorIs(t, err, syscall.ECONNRESET, clues.ToCore(err))
|
||||||
require.Equal(t, 16, count, "number of retries")
|
require.Equal(t, 16, retryInc, "number of retries")
|
||||||
|
|
||||||
count = 0
|
retryInc = 0
|
||||||
|
|
||||||
// the query doesn't matter
|
// the query doesn't matter
|
||||||
_, err = NewService(adpt).Client().Users().Get(ctx, nil)
|
_, err = NewService(adpt).Client().Users().Get(ctx, nil)
|
||||||
require.ErrorIs(t, err, syscall.ECONNRESET, clues.ToCore(err))
|
require.ErrorIs(t, err, syscall.ECONNRESET, clues.ToCore(err))
|
||||||
require.Equal(t, 16, count, "number of retries")
|
require.Equal(t, 16, retryInc, "number of retries")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -36,12 +37,16 @@ type largeItemWriter struct {
|
|||||||
client httpWrapper
|
client httpWrapper
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewLargeItemWriter(parentID, url string, size int64) *largeItemWriter {
|
func NewLargeItemWriter(
|
||||||
|
parentID, url string,
|
||||||
|
size int64,
|
||||||
|
counter *count.Bus,
|
||||||
|
) *largeItemWriter {
|
||||||
return &largeItemWriter{
|
return &largeItemWriter{
|
||||||
parentID: parentID,
|
parentID: parentID,
|
||||||
url: url,
|
url: url,
|
||||||
contentLength: size,
|
contentLength: size,
|
||||||
client: *NewNoTimeoutHTTPWrapper(),
|
client: *NewNoTimeoutHTTPWrapper(counter),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -16,6 +16,7 @@ import (
|
|||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type UploadSessionSuite struct {
|
type UploadSessionSuite struct {
|
||||||
@ -69,7 +70,7 @@ func (suite *UploadSessionSuite) TestWriter() {
|
|||||||
|
|
||||||
defer ts.Close()
|
defer ts.Close()
|
||||||
|
|
||||||
writer := NewLargeItemWriter("item", ts.URL, writeSize)
|
writer := NewLargeItemWriter("item", ts.URL, writeSize, count.New())
|
||||||
|
|
||||||
// Using a 32 KB buffer for the copy allows us to validate the
|
// Using a 32 KB buffer for the copy allows us to validate the
|
||||||
// multi-part upload. `io.CopyBuffer` will only write 32 KB at
|
// multi-part upload. `io.CopyBuffer` will only write 32 KB at
|
||||||
|
|||||||
@ -23,6 +23,7 @@ import (
|
|||||||
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
|
m365Stub "github.com/alcionai/corso/src/internal/m365/stub"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -1217,7 +1218,12 @@ func newController(
|
|||||||
) *Controller {
|
) *Controller {
|
||||||
a := tconfig.NewM365Account(t)
|
a := tconfig.NewM365Account(t)
|
||||||
|
|
||||||
controller, err := NewController(ctx, a, pst, control.Options{})
|
controller, err := NewController(
|
||||||
|
ctx,
|
||||||
|
a,
|
||||||
|
pst,
|
||||||
|
control.Options{},
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
return controller
|
return controller
|
||||||
|
|||||||
@ -40,6 +40,7 @@ type Controller struct {
|
|||||||
func (ctrl Controller) ProduceBackupCollections(
|
func (ctrl Controller) ProduceBackupCollections(
|
||||||
_ context.Context,
|
_ context.Context,
|
||||||
_ inject.BackupProducerConfig,
|
_ inject.BackupProducerConfig,
|
||||||
|
_ *count.Bus,
|
||||||
_ *fault.Bus,
|
_ *fault.Bus,
|
||||||
) (
|
) (
|
||||||
[]data.BackupCollection,
|
[]data.BackupCollection,
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -94,7 +95,10 @@ func (suite *groupsIntegrationSuite) SetupSuite() {
|
|||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.tenantID = creds.AzureTenantID
|
suite.tenantID = creds.AzureTenantID
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365/graph/betasdk/models"
|
"github.com/alcionai/corso/src/internal/m365/graph/betasdk/models"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type BetaUnitSuite struct {
|
type BetaUnitSuite struct {
|
||||||
@ -31,7 +32,8 @@ func (suite *BetaUnitSuite) TestBetaService_Adapter() {
|
|||||||
adpt, err := graph.CreateAdapter(
|
adpt, err := graph.CreateAdapter(
|
||||||
m365.AzureTenantID,
|
m365.AzureTenantID,
|
||||||
m365.AzureClientID,
|
m365.AzureClientID,
|
||||||
m365.AzureClientSecret)
|
m365.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
service := NewBetaService(adpt)
|
service := NewBetaService(adpt)
|
||||||
|
|||||||
@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control/testdata"
|
"github.com/alcionai/corso/src/pkg/control/testdata"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -26,7 +27,8 @@ func createTestBetaService(t *testing.T, credentials account.M365Config) *api.Be
|
|||||||
adapter, err := graph.CreateAdapter(
|
adapter, err := graph.CreateAdapter(
|
||||||
credentials.AzureTenantID,
|
credentials.AzureTenantID,
|
||||||
credentials.AzureClientID,
|
credentials.AzureClientID,
|
||||||
credentials.AzureClientSecret)
|
credentials.AzureClientSecret,
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
return api.NewBetaService(adapter)
|
return api.NewBetaService(adapter)
|
||||||
|
|||||||
@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -25,6 +26,7 @@ func ProduceBackupCollections(
|
|||||||
ac api.Client,
|
ac api.Client,
|
||||||
creds account.M365Config,
|
creds account.M365Config,
|
||||||
su support.StatusUpdater,
|
su support.StatusUpdater,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) {
|
) ([]data.BackupCollection, *prefixmatcher.StringSetMatcher, bool, error) {
|
||||||
b, err := bpc.Selector.ToSharePointBackup()
|
b, err := bpc.Selector.ToSharePointBackup()
|
||||||
@ -102,6 +104,7 @@ func ProduceBackupCollections(
|
|||||||
ac,
|
ac,
|
||||||
scope,
|
scope,
|
||||||
su,
|
su,
|
||||||
|
counter,
|
||||||
errs)
|
errs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
el.AddRecoverable(ctx, err)
|
el.AddRecoverable(ctx, err)
|
||||||
|
|||||||
@ -204,9 +204,6 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
|
|||||||
ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx})
|
ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx})
|
||||||
defer flushMetrics()
|
defer flushMetrics()
|
||||||
|
|
||||||
// for cases where we can't pass the counter down as part of a func call.
|
|
||||||
ctx = count.Embed(ctx, op.Counter)
|
|
||||||
|
|
||||||
// Check if the protected resource has the service enabled in order for us
|
// Check if the protected resource has the service enabled in order for us
|
||||||
// to run a backup.
|
// to run a backup.
|
||||||
enabled, err := op.bp.IsServiceEnabled(
|
enabled, err := op.bp.IsServiceEnabled(
|
||||||
@ -421,6 +418,7 @@ func (op *BackupOperation) do(
|
|||||||
mdColls,
|
mdColls,
|
||||||
lastBackupVersion,
|
lastBackupVersion,
|
||||||
op.Options,
|
op.Options,
|
||||||
|
op.Counter,
|
||||||
op.Errors)
|
op.Errors)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "producing backup data collections")
|
return nil, clues.Wrap(err, "producing backup data collections")
|
||||||
@ -441,8 +439,8 @@ func (op *BackupOperation) do(
|
|||||||
ssmb,
|
ssmb,
|
||||||
backupID,
|
backupID,
|
||||||
op.incremental && canUseMetadata && canUsePreviousBackup,
|
op.incremental && canUseMetadata && canUsePreviousBackup,
|
||||||
op.Errors,
|
op.Counter,
|
||||||
op.Counter)
|
op.Errors)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "persisting collection backups")
|
return nil, clues.Wrap(err, "persisting collection backups")
|
||||||
}
|
}
|
||||||
@ -500,6 +498,7 @@ func produceBackupDataCollections(
|
|||||||
metadata []data.RestoreCollection,
|
metadata []data.RestoreCollection,
|
||||||
lastBackupVersion int,
|
lastBackupVersion int,
|
||||||
ctrlOpts control.Options,
|
ctrlOpts control.Options,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
||||||
progressBar := observe.MessageWithCompletion(ctx, "Discovering items to backup")
|
progressBar := observe.MessageWithCompletion(ctx, "Discovering items to backup")
|
||||||
@ -513,7 +512,7 @@ func produceBackupDataCollections(
|
|||||||
Selector: sel,
|
Selector: sel,
|
||||||
}
|
}
|
||||||
|
|
||||||
return bp.ProduceBackupCollections(ctx, bpc, errs)
|
return bp.ProduceBackupCollections(ctx, bpc, counter, errs)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -531,8 +530,8 @@ func consumeBackupCollections(
|
|||||||
pmr prefixmatcher.StringSetReader,
|
pmr prefixmatcher.StringSetReader,
|
||||||
backupID model.StableID,
|
backupID model.StableID,
|
||||||
isIncremental bool,
|
isIncremental bool,
|
||||||
errs *fault.Bus,
|
|
||||||
counter *count.Bus,
|
counter *count.Bus,
|
||||||
|
errs *fault.Bus,
|
||||||
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) {
|
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) {
|
||||||
ctx = clues.Add(
|
ctx = clues.Add(
|
||||||
ctx,
|
ctx,
|
||||||
@ -555,8 +554,8 @@ func consumeBackupCollections(
|
|||||||
pmr,
|
pmr,
|
||||||
tags,
|
tags,
|
||||||
isIncremental,
|
isIncremental,
|
||||||
errs,
|
counter,
|
||||||
counter)
|
errs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if kopiaStats == nil {
|
if kopiaStats == nil {
|
||||||
return nil, nil, nil, clues.Stack(err)
|
return nil, nil, nil, clues.Stack(err)
|
||||||
|
|||||||
@ -136,8 +136,8 @@ func (mbu mockBackupConsumer) ConsumeBackupCollections(
|
|||||||
excluded prefixmatcher.StringSetReader,
|
excluded prefixmatcher.StringSetReader,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
buildTreeWithBase bool,
|
buildTreeWithBase bool,
|
||||||
errs *fault.Bus,
|
|
||||||
counter *count.Bus,
|
counter *count.Bus,
|
||||||
|
errs *fault.Bus,
|
||||||
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) {
|
) (*kopia.BackupStats, *details.Builder, kopia.DetailsMergeInfoer, error) {
|
||||||
if mbu.checkFunc != nil {
|
if mbu.checkFunc != nil {
|
||||||
mbu.checkFunc(backupReasons, bases, cs, tags, buildTreeWithBase)
|
mbu.checkFunc(backupReasons, bases, cs, tags, buildTreeWithBase)
|
||||||
@ -529,8 +529,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
|
|||||||
nil,
|
nil,
|
||||||
backupID,
|
backupID,
|
||||||
true,
|
true,
|
||||||
fault.New(true),
|
count.New(),
|
||||||
count.New())
|
fault.New(true))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() {
|
func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() {
|
||||||
@ -1439,7 +1439,10 @@ func (suite *BackupOpIntegrationSuite) SetupSuite() {
|
|||||||
creds, err := a.M365Config()
|
creds, err := a.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
suite.ac, err = api.NewClient(creds, control.DefaultOptions())
|
suite.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365"
|
"github.com/alcionai/corso/src/internal/m365"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -25,7 +26,12 @@ func ControllerWithSelector(
|
|||||||
ins idname.Cacher,
|
ins idname.Cacher,
|
||||||
onFail func(),
|
onFail func(),
|
||||||
) (*m365.Controller, selectors.Selector) {
|
) (*m365.Controller, selectors.Selector) {
|
||||||
ctrl, err := m365.NewController(ctx, acct, sel.PathService(), control.DefaultOptions())
|
ctrl, err := m365.NewController(
|
||||||
|
ctx,
|
||||||
|
acct,
|
||||||
|
sel.PathService(),
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
if !assert.NoError(t, err, clues.ToCore(err)) {
|
if !assert.NoError(t, err, clues.ToCore(err)) {
|
||||||
if onFail != nil {
|
if onFail != nil {
|
||||||
onFail()
|
onFail()
|
||||||
|
|||||||
@ -23,6 +23,7 @@ type (
|
|||||||
ProduceBackupCollections(
|
ProduceBackupCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
bpc BackupProducerConfig,
|
bpc BackupProducerConfig,
|
||||||
|
counter *count.Bus,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error)
|
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error)
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
||||||
"github.com/alcionai/corso/src/internal/m365"
|
"github.com/alcionai/corso/src/internal/m365"
|
||||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
@ -38,6 +39,7 @@ func NewMockBackupProducer(
|
|||||||
func (mbp *mockBackupProducer) ProduceBackupCollections(
|
func (mbp *mockBackupProducer) ProduceBackupCollections(
|
||||||
context.Context,
|
context.Context,
|
||||||
inject.BackupProducerConfig,
|
inject.BackupProducerConfig,
|
||||||
|
*count.Bus,
|
||||||
*fault.Bus,
|
*fault.Bus,
|
||||||
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
) ([]data.BackupCollection, prefixmatcher.StringSetReader, bool, error) {
|
||||||
if mbp.injectNonRecoverableErr {
|
if mbp.injectNonRecoverableErr {
|
||||||
|
|||||||
@ -136,8 +136,6 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De
|
|||||||
ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx})
|
ctx, flushMetrics := events.NewMetrics(ctx, logger.Writer{Ctx: ctx})
|
||||||
defer flushMetrics()
|
defer flushMetrics()
|
||||||
|
|
||||||
ctx = count.Embed(ctx, op.Counter)
|
|
||||||
|
|
||||||
cats, err := op.Selectors.AllHumanPathCategories()
|
cats, err := op.Selectors.AllHumanPathCategories()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// No need to exit over this, we'll just be missing a bit of info in the
|
// No need to exit over this, we'll just be missing a bit of info in the
|
||||||
|
|||||||
@ -344,7 +344,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_errorNoBackup() {
|
|||||||
ctx,
|
ctx,
|
||||||
suite.acct,
|
suite.acct,
|
||||||
rsel.PathService(),
|
rsel.PathService(),
|
||||||
control.DefaultOptions())
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
ro, err := NewRestoreOperation(
|
ro, err := NewRestoreOperation(
|
||||||
|
|||||||
@ -393,7 +393,10 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
|
|||||||
creds, err := acct.M365Config()
|
creds, err := acct.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
ac, err := api.NewClient(creds, control.DefaultOptions())
|
ac, err := api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
// generate 3 new folders with two items each.
|
// generate 3 new folders with two items each.
|
||||||
|
|||||||
@ -52,13 +52,13 @@ import (
|
|||||||
|
|
||||||
// GockClient produces a new exchange api client that can be
|
// GockClient produces a new exchange api client that can be
|
||||||
// mocked using gock.
|
// mocked using gock.
|
||||||
func gockClient(creds account.M365Config) (api.Client, error) {
|
func gockClient(creds account.M365Config, counter *count.Bus) (api.Client, error) {
|
||||||
s, err := gmock.NewService(creds)
|
s, err := gmock.NewService(creds, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Client{}, err
|
return api.Client{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
li, err := gmock.NewService(creds, graph.NoTimeout())
|
li, err := gmock.NewService(creds, counter, graph.NoTimeout())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Client{}, err
|
return api.Client{}, err
|
||||||
}
|
}
|
||||||
@ -562,7 +562,12 @@ func ControllerWithSelector(
|
|||||||
ins idname.Cacher,
|
ins idname.Cacher,
|
||||||
onFail func(*testing.T, context.Context),
|
onFail func(*testing.T, context.Context),
|
||||||
) (*m365.Controller, selectors.Selector) {
|
) (*m365.Controller, selectors.Selector) {
|
||||||
ctrl, err := m365.NewController(ctx, acct, sel.PathService(), control.DefaultOptions())
|
ctrl, err := m365.NewController(
|
||||||
|
ctx,
|
||||||
|
acct,
|
||||||
|
sel.PathService(),
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
if !assert.NoError(t, err, clues.ToCore(err)) {
|
if !assert.NoError(t, err, clues.ToCore(err)) {
|
||||||
if onFail != nil {
|
if onFail != nil {
|
||||||
onFail(t, ctx)
|
onFail(t, ctx)
|
||||||
@ -623,10 +628,15 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
|
|||||||
creds, err := a.M365Config()
|
creds, err := a.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.ac, err = api.NewClient(creds, control.DefaultOptions())
|
counter := count.New()
|
||||||
|
|
||||||
|
its.ac, err = api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
counter)
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.gockAC, err = gockClient(creds)
|
its.gockAC, err = gockClient(creds, counter)
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.user = userIDs(t, tconfig.M365UserID(t), its.ac)
|
its.user = userIDs(t, tconfig.M365UserID(t), its.ac)
|
||||||
|
|||||||
@ -814,7 +814,12 @@ func runDriveIncrementalTest(
|
|||||||
}
|
}
|
||||||
for _, test := range table {
|
for _, test := range table {
|
||||||
suite.Run(test.name, func() {
|
suite.Run(test.name, func() {
|
||||||
cleanCtrl, err := m365.NewController(ctx, acct, sel.PathService(), control.DefaultOptions())
|
cleanCtrl, err := m365.NewController(
|
||||||
|
ctx,
|
||||||
|
acct,
|
||||||
|
sel.PathService(),
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
bod.ctrl = cleanCtrl
|
bod.ctrl = cleanCtrl
|
||||||
@ -1117,7 +1122,8 @@ func (suite *OneDriveBackupIntgSuite) TestBackup_Run_oneDriveOwnerMigration() {
|
|||||||
ctx,
|
ctx,
|
||||||
acct,
|
acct,
|
||||||
path.OneDriveService,
|
path.OneDriveService,
|
||||||
control.DefaultOptions())
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID)
|
userable, err := ctrl.AC.Users().GetByID(ctx, suite.its.user.ID)
|
||||||
|
|||||||
@ -216,8 +216,8 @@ func write(
|
|||||||
prefixmatcher.NopReader[map[string]struct{}](),
|
prefixmatcher.NopReader[map[string]struct{}](),
|
||||||
nil,
|
nil,
|
||||||
false,
|
false,
|
||||||
errs,
|
count.New(),
|
||||||
count.New())
|
errs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", clues.Wrap(err, "storing marshalled bytes in repository")
|
return "", clues.Wrap(err, "storing marshalled bytes in repository")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,8 +1,6 @@
|
|||||||
package count
|
package count
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
|
|
||||||
"github.com/puzpuzpuz/xsync/v2"
|
"github.com/puzpuzpuz/xsync/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -116,33 +114,3 @@ func (b *Bus) TotalValues() map[string]int64 {
|
|||||||
|
|
||||||
return b.Values()
|
return b.Values()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// context embedding
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
type countKey string
|
|
||||||
|
|
||||||
const ctxKey countKey = "corsoCounter"
|
|
||||||
|
|
||||||
// Ctx retrieves the count.Bus embedded in the context.
|
|
||||||
func Ctx(ctx context.Context) *Bus {
|
|
||||||
l := ctx.Value(ctxKey)
|
|
||||||
if l == nil {
|
|
||||||
return New()
|
|
||||||
}
|
|
||||||
|
|
||||||
return l.(*Bus)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Embed allows users to embed their own count.Bus within the context.
|
|
||||||
func Embed(
|
|
||||||
ctx context.Context,
|
|
||||||
bus *Bus,
|
|
||||||
) context.Context {
|
|
||||||
if bus == nil {
|
|
||||||
bus = New()
|
|
||||||
}
|
|
||||||
|
|
||||||
return context.WithValue(ctx, ctxKey, bus)
|
|
||||||
}
|
|
||||||
|
|||||||
@ -79,7 +79,12 @@ func connectToM365(
|
|||||||
progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365")
|
progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365")
|
||||||
defer close(progressBar)
|
defer close(progressBar)
|
||||||
|
|
||||||
ctrl, err := m365.NewController(ctx, r.Account, pst, r.Opts)
|
ctrl, err := m365.NewController(
|
||||||
|
ctx,
|
||||||
|
r.Account,
|
||||||
|
pst,
|
||||||
|
r.Opts,
|
||||||
|
r.counter.Local())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "creating m365 client controller")
|
return nil, clues.Wrap(err, "creating m365 client controller")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
|
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/storage"
|
"github.com/alcionai/corso/src/pkg/storage"
|
||||||
@ -57,6 +58,8 @@ type Repositoryer interface {
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rcOpts ctrlRepo.Retention,
|
rcOpts ctrlRepo.Retention,
|
||||||
) (operations.RetentionConfigOperation, error)
|
) (operations.RetentionConfigOperation, error)
|
||||||
|
|
||||||
|
Counter() *count.Bus
|
||||||
}
|
}
|
||||||
|
|
||||||
// Repository contains storage provider information.
|
// Repository contains storage provider information.
|
||||||
@ -70,6 +73,7 @@ type repository struct {
|
|||||||
Opts control.Options
|
Opts control.Options
|
||||||
Provider DataProvider // the client controller used for external user data CRUD
|
Provider DataProvider // the client controller used for external user data CRUD
|
||||||
|
|
||||||
|
counter *count.Bus
|
||||||
Bus events.Eventer
|
Bus events.Eventer
|
||||||
dataLayer *kopia.Wrapper
|
dataLayer *kopia.Wrapper
|
||||||
modelStore *kopia.ModelStore
|
modelStore *kopia.ModelStore
|
||||||
@ -110,6 +114,7 @@ func New(
|
|||||||
Version: "v1",
|
Version: "v1",
|
||||||
Account: acct,
|
Account: acct,
|
||||||
Storage: st,
|
Storage: st,
|
||||||
|
counter: count.New(),
|
||||||
Bus: bus,
|
Bus: bus,
|
||||||
Opts: opts,
|
Opts: opts,
|
||||||
}
|
}
|
||||||
@ -336,6 +341,10 @@ func (r repository) NewRetentionConfig(
|
|||||||
r.Bus)
|
r.Bus)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r repository) Counter() *count.Bus {
|
||||||
|
return r.counter
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Repository ID Model
|
// Repository ID Model
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@ -11,6 +11,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
type AccessAPIIntgSuite struct {
|
type AccessAPIIntgSuite struct {
|
||||||
@ -109,7 +110,10 @@ func (suite *AccessAPIIntgSuite) TestGetToken() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
ac, err := NewClient(suite.its.ac.Credentials, control.DefaultOptions())
|
ac, err := NewClient(
|
||||||
|
suite.its.ac.Credentials,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
ac.Credentials = test.creds()
|
ac.Credentials = test.creds()
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -39,29 +40,44 @@ type Client struct {
|
|||||||
// graph api client.
|
// graph api client.
|
||||||
Requester graph.Requester
|
Requester graph.Requester
|
||||||
|
|
||||||
|
counter *count.Bus
|
||||||
|
|
||||||
options control.Options
|
options control.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient produces a new exchange api client. Must be used in
|
// NewClient produces a new exchange api client. Must be used in
|
||||||
// place of creating an ad-hoc client struct.
|
// place of creating an ad-hoc client struct.
|
||||||
func NewClient(creds account.M365Config, co control.Options) (Client, error) {
|
func NewClient(
|
||||||
s, err := NewService(creds)
|
creds account.M365Config,
|
||||||
|
co control.Options,
|
||||||
|
counter *count.Bus,
|
||||||
|
) (Client, error) {
|
||||||
|
s, err := NewService(creds, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Client{}, err
|
return Client{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
li, err := newLargeItemService(creds)
|
li, err := newLargeItemService(creds, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Client{}, err
|
return Client{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
rqr := graph.NewNoTimeoutHTTPWrapper()
|
rqr := graph.NewNoTimeoutHTTPWrapper(counter)
|
||||||
|
|
||||||
if co.DeltaPageSize < 1 || co.DeltaPageSize > maxDeltaPageSize {
|
if co.DeltaPageSize < 1 || co.DeltaPageSize > maxDeltaPageSize {
|
||||||
co.DeltaPageSize = maxDeltaPageSize
|
co.DeltaPageSize = maxDeltaPageSize
|
||||||
}
|
}
|
||||||
|
|
||||||
return Client{creds, s, li, rqr, co}, nil
|
cli := Client{
|
||||||
|
Credentials: creds,
|
||||||
|
Stable: s,
|
||||||
|
LargeItem: li,
|
||||||
|
Requester: rqr,
|
||||||
|
counter: counter,
|
||||||
|
options: co,
|
||||||
|
}
|
||||||
|
|
||||||
|
return cli, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// initConcurrencyLimit ensures that the graph concurrency limiter is
|
// initConcurrencyLimit ensures that the graph concurrency limiter is
|
||||||
@ -77,15 +93,20 @@ func InitConcurrencyLimit(ctx context.Context, pst path.ServiceType) {
|
|||||||
// so that in-flight state within the adapter doesn't get clobbered.
|
// so that in-flight state within the adapter doesn't get clobbered.
|
||||||
// Most calls should use the Client.Stable property instead of calling this
|
// Most calls should use the Client.Stable property instead of calling this
|
||||||
// func, unless it is explicitly necessary.
|
// func, unless it is explicitly necessary.
|
||||||
func (c Client) Service() (graph.Servicer, error) {
|
func (c Client) Service(counter *count.Bus) (graph.Servicer, error) {
|
||||||
return NewService(c.Credentials)
|
return NewService(c.Credentials, counter)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) {
|
func NewService(
|
||||||
|
creds account.M365Config,
|
||||||
|
counter *count.Bus,
|
||||||
|
opts ...graph.Option,
|
||||||
|
) (*graph.Service, error) {
|
||||||
a, err := graph.CreateAdapter(
|
a, err := graph.CreateAdapter(
|
||||||
creds.AzureTenantID,
|
creds.AzureTenantID,
|
||||||
creds.AzureClientID,
|
creds.AzureClientID,
|
||||||
creds.AzureClientSecret,
|
creds.AzureClientSecret,
|
||||||
|
counter,
|
||||||
opts...)
|
opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "generating graph api adapter")
|
return nil, clues.Wrap(err, "generating graph api adapter")
|
||||||
@ -94,8 +115,11 @@ func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service,
|
|||||||
return graph.NewService(a), nil
|
return graph.NewService(a), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newLargeItemService(creds account.M365Config) (*graph.Service, error) {
|
func newLargeItemService(
|
||||||
a, err := NewService(creds, graph.NoTimeout())
|
creds account.M365Config,
|
||||||
|
counter *count.Bus,
|
||||||
|
) (*graph.Service, error) {
|
||||||
|
a, err := NewService(creds, counter, graph.NoTimeout())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, clues.Wrap(err, "generating no-timeout graph adapter")
|
return nil, clues.Wrap(err, "generating no-timeout graph adapter")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -63,7 +63,7 @@ func (c Contacts) DeleteContainer(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := NewService(c.Credentials)
|
srv, err := NewService(c.Credentials, c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -230,7 +230,7 @@ func (c Contacts) DeleteItem(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := c.Service()
|
srv, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -239,7 +239,7 @@ func (c Drives) DeleteItem(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := c.Service()
|
srv, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Wrap(ctx, err, "creating adapter to delete item permission")
|
return graph.Wrap(ctx, err, "creating adapter to delete item permission")
|
||||||
}
|
}
|
||||||
@ -309,7 +309,7 @@ func (c Drives) DeleteItemPermission(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := c.Service()
|
srv, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Wrap(ctx, err, "creating adapter to delete item permission")
|
return graph.Wrap(ctx, err, "creating adapter to delete item permission")
|
||||||
}
|
}
|
||||||
|
|||||||
@ -73,7 +73,7 @@ func (c Events) DeleteContainer(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := NewService(c.Credentials)
|
srv, err := NewService(c.Credentials, c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -475,7 +475,7 @@ func (c Events) DeleteItem(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := c.Service()
|
srv, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -541,7 +541,7 @@ func (c Events) PostLargeAttachment(
|
|||||||
}
|
}
|
||||||
|
|
||||||
url := ptr.Val(us.GetUploadUrl())
|
url := ptr.Val(us.GetUploadUrl())
|
||||||
w := graph.NewLargeItemWriter(parentItemID, url, size)
|
w := graph.NewLargeItemWriter(parentItemID, url, size, c.counter)
|
||||||
copyBuffer := make([]byte, graph.AttachmentChunkSize)
|
copyBuffer := make([]byte, graph.AttachmentChunkSize)
|
||||||
|
|
||||||
_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
|
_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
|
||||||
|
|||||||
@ -42,7 +42,7 @@ func (c Groups) GetAll(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]models.Groupable, error) {
|
) ([]models.Groupable, error) {
|
||||||
service, err := c.Service()
|
service, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -110,7 +110,7 @@ func (c Groups) GetByID(
|
|||||||
identifier string,
|
identifier string,
|
||||||
_ CallConfig, // matching standards
|
_ CallConfig, // matching standards
|
||||||
) (models.Groupable, error) {
|
) (models.Groupable, error) {
|
||||||
service, err := c.Service()
|
service, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -229,7 +229,7 @@ func (c Groups) GetAllSites(
|
|||||||
return nil, clues.Wrap(err, "getting channels")
|
return nil, clues.Wrap(err, "getting channels")
|
||||||
}
|
}
|
||||||
|
|
||||||
service, err := c.Service()
|
service, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, graph.Stack(ctx, err)
|
return nil, graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -292,7 +292,7 @@ func (c Groups) GetRootSite(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
identifier string,
|
identifier string,
|
||||||
) (models.Siteable, error) {
|
) (models.Siteable, error) {
|
||||||
service, err := c.Service()
|
service, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,6 +19,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -27,13 +28,13 @@ import (
|
|||||||
|
|
||||||
// GockClient produces a new exchange api client that can be
|
// GockClient produces a new exchange api client that can be
|
||||||
// mocked using gock.
|
// mocked using gock.
|
||||||
func gockClient(creds account.M365Config) (Client, error) {
|
func gockClient(creds account.M365Config, counter *count.Bus) (Client, error) {
|
||||||
s, err := gmock.NewService(creds)
|
s, err := gmock.NewService(creds, counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Client{}, err
|
return Client{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
li, err := gmock.NewService(creds, graph.NoTimeout())
|
li, err := gmock.NewService(creds, counter, graph.NoTimeout())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Client{}, err
|
return Client{}, err
|
||||||
}
|
}
|
||||||
@ -128,10 +129,10 @@ func newIntegrationTesterSetup(t *testing.T) intgTesterSetup {
|
|||||||
creds, err := a.M365Config()
|
creds, err := a.M365Config()
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.ac, err = NewClient(creds, control.DefaultOptions())
|
its.ac, err = NewClient(creds, control.DefaultOptions(), count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
its.gockAC, err = gockClient(creds)
|
its.gockAC, err = gockClient(creds, count.New())
|
||||||
require.NoError(t, err, clues.ToCore(err))
|
require.NoError(t, err, clues.ToCore(err))
|
||||||
|
|
||||||
// user drive
|
// user drive
|
||||||
|
|||||||
@ -73,7 +73,7 @@ func (c Mail) DeleteContainer(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := NewService(c.Credentials)
|
srv, err := NewService(c.Credentials, c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -442,7 +442,7 @@ func (c Mail) DeleteItem(
|
|||||||
) error {
|
) error {
|
||||||
// deletes require unique http clients
|
// deletes require unique http clients
|
||||||
// https://github.com/alcionai/corso/issues/2707
|
// https://github.com/alcionai/corso/issues/2707
|
||||||
srv, err := NewService(c.Credentials)
|
srv, err := NewService(c.Credentials, c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return graph.Stack(ctx, err)
|
return graph.Stack(ctx, err)
|
||||||
}
|
}
|
||||||
@ -508,7 +508,7 @@ func (c Mail) PostLargeAttachment(
|
|||||||
}
|
}
|
||||||
|
|
||||||
url := ptr.Val(us.GetUploadUrl())
|
url := ptr.Val(us.GetUploadUrl())
|
||||||
w := graph.NewLargeItemWriter(parentItemID, url, size)
|
w := graph.NewLargeItemWriter(parentItemID, url, size, c.counter)
|
||||||
copyBuffer := make([]byte, graph.AttachmentChunkSize)
|
copyBuffer := make([]byte, graph.AttachmentChunkSize)
|
||||||
|
|
||||||
_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
|
_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
|
||||||
|
|||||||
@ -58,7 +58,7 @@ func (c Users) GetAll(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) ([]models.Userable, error) {
|
) ([]models.Userable, error) {
|
||||||
service, err := c.Service()
|
service, err := c.Service(c.counter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -109,7 +109,10 @@ func (suite *GroupsIntgSuite) TestGroups() {
|
|||||||
|
|
||||||
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
||||||
|
|
||||||
groups, err := m365.Groups(ctx, suite.acct, fault.New(true))
|
groups, err := m365.Groups(
|
||||||
|
ctx,
|
||||||
|
suite.acct,
|
||||||
|
fault.New(true))
|
||||||
assert.NoError(t, err, clues.ToCore(err))
|
assert.NoError(t, err, clues.ToCore(err))
|
||||||
assert.NotEmpty(t, groups)
|
assert.NotEmpty(t, groups)
|
||||||
|
|
||||||
@ -155,7 +158,10 @@ func (suite *GroupsIntgSuite) TestGroupsMap() {
|
|||||||
|
|
||||||
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
graph.InitializeConcurrencyLimiter(ctx, true, 4)
|
||||||
|
|
||||||
gm, err := m365.GroupsMap(ctx, suite.acct, fault.New(true))
|
gm, err := m365.GroupsMap(
|
||||||
|
ctx,
|
||||||
|
suite.acct,
|
||||||
|
fault.New(true))
|
||||||
assert.NoError(t, err, clues.ToCore(err))
|
assert.NoError(t, err, clues.ToCore(err))
|
||||||
assert.NotEmpty(t, gm)
|
assert.NotEmpty(t, gm)
|
||||||
|
|
||||||
@ -203,7 +209,10 @@ func (suite *GroupsIntgSuite) TestGroups_InvalidCredentials() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
groups, err := m365.Groups(ctx, test.acct(t), fault.New(true))
|
groups, err := m365.Groups(
|
||||||
|
ctx,
|
||||||
|
test.acct(t),
|
||||||
|
fault.New(true))
|
||||||
assert.Empty(t, groups, "returned no groups")
|
assert.Empty(t, groups, "returned no groups")
|
||||||
assert.NotNil(t, err)
|
assert.NotNil(t, err)
|
||||||
})
|
})
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/pkg/account"
|
"github.com/alcionai/corso/src/pkg/account"
|
||||||
"github.com/alcionai/corso/src/pkg/control"
|
"github.com/alcionai/corso/src/pkg/control"
|
||||||
|
"github.com/alcionai/corso/src/pkg/count"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||||
@ -36,7 +37,10 @@ func makeAC(
|
|||||||
return api.Client{}, clues.Wrap(err, "getting m365 account creds").WithClues(ctx)
|
return api.Client{}, clues.Wrap(err, "getting m365 account creds").WithClues(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
cli, err := api.NewClient(creds, control.DefaultOptions())
|
cli, err := api.NewClient(
|
||||||
|
creds,
|
||||||
|
control.DefaultOptions(),
|
||||||
|
count.New())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return api.Client{}, clues.Wrap(err, "constructing api client").WithClues(ctx)
|
return api.Client{}, clues.Wrap(err, "constructing api client").WithClues(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user