From 24911a358b88748b4ca4bddaacf5d3f07406e6e8 Mon Sep 17 00:00:00 2001 From: Keepers Date: Fri, 27 Jan 2023 12:34:00 -0700 Subject: [PATCH] quick graph client refactor (#2301) ## Description A couple minor refactors. First, combines the graph/service_helper.go and graph/service.go files into a single file (just lift'n'shift, no logic changes). Second, renames CreateHTTPClient to just HTTPClient. Third, replaces the new LargeItemClient constructor with extensible options for the http client, starting with a NoTimeout option. This will provide longer term maintenance over multiple constructors. This change sets up QoL for building multiple services and clients within api layer handlers, which will appear in an immediately following PR. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :broom: Tech Debt/Cleanup ## Issue(s) * #2299 ## Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cli/backup/sharepoint.go | 2 +- src/cmd/factory/impl/common.go | 2 +- src/cmd/getM365/getItem.go | 2 +- src/cmd/purge/purge.go | 2 +- .../connector/data_collections_test.go | 16 +- src/internal/connector/graph/service.go | 174 +++++++++++++++++- .../connector/graph/service_helper.go | 142 -------------- src/internal/connector/graph/service_test.go | 47 ++++- .../graph_connector_disconnected_test.go | 2 +- .../connector/graph_connector_test.go | 10 +- .../connector/onedrive/collection_test.go | 4 +- .../connector/onedrive/collections_test.go | 2 +- src/internal/connector/onedrive/drive_test.go | 2 +- src/internal/connector/onedrive/item_test.go | 2 +- .../sharepoint/data_collections_test.go | 2 +- .../operations/backup_integration_test.go | 2 +- src/internal/operations/operation.go | 2 +- src/pkg/services/m365/m365.go | 6 +- 18 files changed, 239 insertions(+), 182 deletions(-) delete mode 100644 src/internal/connector/graph/service_helper.go diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index fb4f7e766..9fa8afb6b 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -210,7 +210,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { defer utils.CloseRepo(ctx, r) - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, connector.Sites) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs")) } diff --git a/src/cmd/factory/impl/common.go b/src/cmd/factory/impl/common.go index 585118442..0ea6835dd 100644 --- a/src/cmd/factory/impl/common.go +++ b/src/cmd/factory/impl/common.go @@ -112,7 +112,7 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon } // build a graph connector - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, connector.Users) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users) if err != nil { return nil, account.Account{}, errors.Wrap(err, "connecting to graph api") } diff --git a/src/cmd/getM365/getItem.go b/src/cmd/getM365/getItem.go index 24ce81d9a..d24b27d38 100644 --- a/src/cmd/getM365/getItem.go +++ b/src/cmd/getM365/getItem.go @@ -178,7 +178,7 @@ func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config, return nil, m365Cfg, Only(ctx, errors.Wrap(err, "finding m365 account details")) } - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, connector.Users) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users) if err != nil { return nil, m365Cfg, Only(ctx, errors.Wrap(err, "connecting to graph API")) } diff --git a/src/cmd/purge/purge.go b/src/cmd/purge/purge.go index 32100772d..3d7074835 100644 --- a/src/cmd/purge/purge.go +++ b/src/cmd/purge/purge.go @@ -255,7 +255,7 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) { } // build a graph connector - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, connector.Users) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users) if err != nil { return nil, Only(ctx, errors.Wrap(err, "connecting to graph api")) } diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 57332ce1a..877975bb5 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -44,7 +44,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) SetupSuite() { tester.MustGetEnvVars(suite.T(), tester.M365AcctCredEnvs...) - suite.connector = loadConnector(ctx, suite.T(), graph.LargeItemClient(), AllResources) + suite.connector = loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), AllResources) suite.user = tester.M365UserID(suite.T()) suite.site = tester.M365SiteID(suite.T()) @@ -63,7 +63,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection selUsers := []string{suite.user} - connector := loadConnector(ctx, suite.T(), graph.LargeItemClient(), Users) + connector := loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -139,7 +139,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestDataCollections_invali owners := []string{"snuffleupagus"} - connector := loadConnector(ctx, suite.T(), graph.LargeItemClient(), Users) + connector := loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -215,7 +215,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti selSites := []string{suite.site} - connector := loadConnector(ctx, suite.T(), graph.LargeItemClient(), Sites) + connector := loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), Sites) tests := []struct { name string expected int @@ -244,7 +244,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti suite.T().Run(test.name, func(t *testing.T) { collections, err := sharepoint.DataCollections( ctx, - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), test.getSelector(), connector.credentials.AzureTenantID, connector.Service, @@ -300,7 +300,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) SetupSuite() { tester.MustGetEnvSets(suite.T(), tester.M365AcctCredEnvs) - suite.connector = loadConnector(ctx, suite.T(), graph.LargeItemClient(), Sites) + suite.connector = loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), Sites) suite.user = tester.M365UserID(suite.T()) tester.LogTimeOfTest(suite.T()) @@ -313,7 +313,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar var ( t = suite.T() siteID = tester.M365SiteID(t) - gc = loadConnector(ctx, t, graph.LargeItemClient(), Sites) + gc = loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), Sites) siteIDs = []string{siteID} ) @@ -337,7 +337,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar var ( t = suite.T() siteID = tester.M365SiteID(t) - gc = loadConnector(ctx, t, graph.LargeItemClient(), Sites) + gc = loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), Sites) siteIDs = []string{siteID} ) diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 7780e7941..fa4662014 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -2,15 +2,28 @@ package graph import ( "context" + "net/http" + "net/http/httputil" + "os" + "time" - absser "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/microsoft/kiota-abstractions-go/serialization" + ka "github.com/microsoft/kiota-authentication-azure-go" + khttp "github.com/microsoft/kiota-http-go" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/pkg/errors" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) +const ( + logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS" +) + // AllMetadataFileNames produces the standard set of filenames used to store graph // metadata such as delta tokens and folderID->path references. func AllMetadataFileNames() []string { @@ -23,6 +36,10 @@ type QueryParams struct { Credentials account.M365Config } +// --------------------------------------------------------------------------- +// Service Handler +// --------------------------------------------------------------------------- + var _ Servicer = &Service{} type Service struct { @@ -47,7 +64,7 @@ func (s Service) Client() *msgraphsdk.GraphServiceClient { // Seraialize writes an M365 parsable object into a byte array using the built-in // application/json writer within the adapter. -func (s Service) Serialize(object absser.Parsable) ([]byte, error) { +func (s Service) Serialize(object serialization.Parsable) ([]byte, error) { writer, err := s.adapter.GetSerializationWriterFactory().GetSerializationWriter("application/json") if err != nil || writer == nil { return nil, errors.Wrap(err, "creating json serialization writer") @@ -61,6 +78,90 @@ func (s Service) Serialize(object absser.Parsable) ([]byte, error) { return writer.GetSerializedContent() } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type clientConfig struct { + noTimeout bool +} + +type option func(*clientConfig) + +// populate constructs a clientConfig according to the provided options. +func (c *clientConfig) populate(opts ...option) *clientConfig { + for _, opt := range opts { + opt(c) + } + + return c +} + +// apply updates the http.Client with the expected options. +func (c *clientConfig) apply(hc *http.Client) { + if c.noTimeout { + hc.Timeout = 0 + } +} + +// NoTimeout sets the httpClient.Timeout to 0 (unlimited). +// The resulting client isn't suitable for most queries, due to the +// capacity for a call to persist forever. This configuration should +// only be used when downloading very large files. +func NoTimeout() option { + return func(c *clientConfig) { + c.noTimeout = true + } +} + +// CreateAdapter uses provided credentials to log into M365 using Kiota Azure Library +// with Azure identity package. An adapter object is a necessary to component +// to create *msgraphsdk.GraphServiceClient +func CreateAdapter(tenant, client, secret string, opts ...option) (*msgraphsdk.GraphRequestAdapter, error) { + // Client Provider: Uses Secret for access to tenant-level data + cred, err := azidentity.NewClientSecretCredential(tenant, client, secret, nil) + if err != nil { + return nil, errors.Wrap(err, "creating m365 client secret credentials") + } + + auth, err := ka.NewAzureIdentityAuthenticationProviderWithScopes( + cred, + []string{"https://graph.microsoft.com/.default"}, + ) + if err != nil { + return nil, errors.Wrap(err, "creating new AzureIdentityAuthentication") + } + + httpClient := HTTPClient(opts...) + + return msgraphsdk.NewGraphRequestAdapterWithParseNodeFactoryAndSerializationWriterFactoryAndHttpClient( + auth, nil, nil, httpClient) +} + +// HTTPClient creates the httpClient with middlewares and timeout configured +// +// Re-use of http clients is critical, or else we leak OS resources +// and consume relatively unbound socket connections. It is important +// to centralize this client to be passed downstream where api calls +// can utilize it on a per-download basis. +func HTTPClient(opts ...option) *http.Client { + clientOptions := msgraphsdk.GetDefaultClientOptions() + middlewares := msgraphgocore.GetDefaultMiddlewaresWithOptions(&clientOptions) + middlewares = append(middlewares, &LoggingMiddleware{}) + httpClient := msgraphgocore.GetDefaultClient(&clientOptions, middlewares...) + httpClient.Timeout = time.Second * 90 + + (&clientConfig{}). + populate(opts...). + apply(httpClient) + + return httpClient +} + +// --------------------------------------------------------------------------- +// Interfaces +// --------------------------------------------------------------------------- + type Servicer interface { // Client() returns msgraph Service client that can be used to process and execute // the majority of the queries to the M365 Backstore @@ -120,3 +221,72 @@ type ContainerResolver interface { // Items returns the containers in the cache. Items() []CachedContainer } + +// --------------------------------------------------------------------------- +// Client Middleware +// --------------------------------------------------------------------------- + +// LoggingMiddleware can be used to log the http request sent by the graph client +type LoggingMiddleware struct{} + +func (handler *LoggingMiddleware) Intercept( + pipeline khttp.Pipeline, + middlewareIndex int, + req *http.Request, +) (*http.Response, error) { + var ( + ctx = req.Context() + resp, err = pipeline.Next(req, middlewareIndex) + ) + + if resp == nil { + return resp, err + } + + if (resp.StatusCode / 100) == 2 { + if logger.DebugAPI || os.Getenv(logGraphRequestsEnvKey) != "" { + respDump, _ := httputil.DumpResponse(resp, false) + + metadata := []any{ + "idx", middlewareIndex, + "method", req.Method, + "status", resp.Status, + "statusCode", resp.StatusCode, + "requestLen", req.ContentLength, + "url", req.URL, + "response", respDump, + } + + logger.Ctx(ctx).Debugw("2xx graph api resp", metadata...) + } + + return resp, err + } + + if logger.DebugAPI || os.Getenv(logGraphRequestsEnvKey) != "" { + respDump, _ := httputil.DumpResponse(resp, true) + + metadata := []any{ + "idx", middlewareIndex, + "method", req.Method, + "status", resp.Status, + "statusCode", resp.StatusCode, + "requestLen", req.ContentLength, + "url", req.URL, + "response", string(respDump), + } + + logger.Ctx(ctx).Errorw("non-2xx graph api response", metadata...) + } else { + // special case for supportability: log all throttling cases. + if resp.StatusCode == http.StatusTooManyRequests { + logger.Ctx(ctx).Infow("graph api throttling", "method", req.Method, "url", req.URL) + } + + if resp.StatusCode != http.StatusTooManyRequests && (resp.StatusCode/100) != 2 { + logger.Ctx(ctx).Infow("graph api error", "status", resp.Status, "method", req.Method, "url", req.URL) + } + } + + return resp, err +} diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go deleted file mode 100644 index bf39fe194..000000000 --- a/src/internal/connector/graph/service_helper.go +++ /dev/null @@ -1,142 +0,0 @@ -package graph - -import ( - "net/http" - "net/http/httputil" - "os" - "time" - - az "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - ka "github.com/microsoft/kiota-authentication-azure-go" - khttp "github.com/microsoft/kiota-http-go" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/pkg/errors" - - "github.com/alcionai/corso/src/pkg/logger" -) - -const ( - logGraphRequestsEnvKey = "LOG_GRAPH_REQUESTS" -) - -// CreateAdapter uses provided credentials to log into M365 using Kiota Azure Library -// with Azure identity package. An adapter object is a necessary to component -// to create *msgraphsdk.GraphServiceClient -func CreateAdapter(tenant, client, secret string) (*msgraphsdk.GraphRequestAdapter, error) { - // Client Provider: Uses Secret for access to tenant-level data - cred, err := az.NewClientSecretCredential(tenant, client, secret, nil) - if err != nil { - return nil, errors.Wrap(err, "creating m365 client secret credentials") - } - - auth, err := ka.NewAzureIdentityAuthenticationProviderWithScopes( - cred, - []string{"https://graph.microsoft.com/.default"}, - ) - if err != nil { - return nil, errors.Wrap(err, "creating new AzureIdentityAuthentication") - } - - httpClient := CreateHTTPClient() - - return msgraphsdk.NewGraphRequestAdapterWithParseNodeFactoryAndSerializationWriterFactoryAndHttpClient( - auth, nil, nil, httpClient) -} - -// CreateHTTPClient creates the httpClient with middlewares and timeout configured -func CreateHTTPClient() *http.Client { - clientOptions := msgraphsdk.GetDefaultClientOptions() - middlewares := msgraphgocore.GetDefaultMiddlewaresWithOptions(&clientOptions) - middlewares = append(middlewares, &LoggingMiddleware{}) - httpClient := msgraphgocore.GetDefaultClient(&clientOptions, middlewares...) - httpClient.Timeout = time.Second * 90 - - return httpClient -} - -// LargeItemClient generates a client that's configured to handle -// large file downloads. This client isn't suitable for other queries -// due to loose restrictions on timeouts and such. -// -// Re-use of http clients is critical, or else we leak os resources -// and consume relatively unbound socket connections. It is important -// to centralize this client to be passed downstream where api calls -// can utilize it on a per-download basis. -// -// TODO: this should get owned by an API client layer, not the GC itself. -func LargeItemClient() *http.Client { - httpClient := CreateHTTPClient() - httpClient.Timeout = 0 // infinite timeout for pulling large files - - return httpClient -} - -// --------------------------------------------------------------------------- -// Logging Middleware -// --------------------------------------------------------------------------- - -// LoggingMiddleware can be used to log the http request sent by the graph client -type LoggingMiddleware struct{} - -func (handler *LoggingMiddleware) Intercept( - pipeline khttp.Pipeline, - middlewareIndex int, - req *http.Request, -) (*http.Response, error) { - var ( - ctx = req.Context() - resp, err = pipeline.Next(req, middlewareIndex) - ) - - if resp == nil { - return resp, err - } - - if (resp.StatusCode / 100) == 2 { - if logger.DebugAPI || os.Getenv(logGraphRequestsEnvKey) != "" { - respDump, _ := httputil.DumpResponse(resp, false) - - metadata := []any{ - "idx", middlewareIndex, - "method", req.Method, - "status", resp.Status, - "statusCode", resp.StatusCode, - "requestLen", req.ContentLength, - "url", req.URL, - "response", respDump, - } - - logger.Ctx(ctx).Debugw("2xx graph api resp", metadata...) - } - - return resp, err - } - - if logger.DebugAPI || os.Getenv(logGraphRequestsEnvKey) != "" { - respDump, _ := httputil.DumpResponse(resp, true) - - metadata := []any{ - "idx", middlewareIndex, - "method", req.Method, - "status", resp.Status, - "statusCode", resp.StatusCode, - "requestLen", req.ContentLength, - "url", req.URL, - "response", string(respDump), - } - - logger.Ctx(ctx).Errorw("non-2xx graph api response", metadata...) - } else { - // special case for supportability: log all throttling cases. - if resp.StatusCode == http.StatusTooManyRequests { - logger.Ctx(ctx).Infow("graph api throttling", "method", req.Method, "url", req.URL) - } - - if resp.StatusCode != http.StatusTooManyRequests && (resp.StatusCode/100) != 2 { - logger.Ctx(ctx).Infow("graph api error", "status", resp.Status, "method", req.Method, "url", req.URL) - } - } - - return resp, err -} diff --git a/src/internal/connector/graph/service_test.go b/src/internal/connector/graph/service_test.go index ee8c6bc29..14bdc9c36 100644 --- a/src/internal/connector/graph/service_test.go +++ b/src/internal/connector/graph/service_test.go @@ -1,14 +1,15 @@ -package graph_test +package graph import ( + "net/http" "testing" + "time" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" ) @@ -33,26 +34,54 @@ func (suite *GraphUnitSuite) SetupSuite() { func (suite *GraphUnitSuite) TestCreateAdapter() { t := suite.T() - adpt, err := graph.CreateAdapter( + adpt, err := CreateAdapter( suite.credentials.AzureTenantID, suite.credentials.AzureClientID, - suite.credentials.AzureClientSecret, - ) + suite.credentials.AzureClientSecret) assert.NoError(t, err) assert.NotNil(t, adpt) } +func (suite *GraphUnitSuite) TestHTTPClient() { + table := []struct { + name string + opts []option + check func(*testing.T, *http.Client) + }{ + { + name: "no options", + opts: []option{}, + check: func(t *testing.T, c *http.Client) { + assert.Equal(t, 90*time.Second, c.Timeout, "default timeout") + }, + }, + { + name: "no timeout", + opts: []option{NoTimeout()}, + check: func(t *testing.T, c *http.Client) { + assert.Equal(t, 0, int(c.Timeout), "unlimited timeout") + }, + }, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + cli := HTTPClient(test.opts...) + assert.NotNil(t, cli) + test.check(t, cli) + }) + } +} + func (suite *GraphUnitSuite) TestSerializationEndPoint() { t := suite.T() - adpt, err := graph.CreateAdapter( + adpt, err := CreateAdapter( suite.credentials.AzureTenantID, suite.credentials.AzureClientID, - suite.credentials.AzureClientSecret, - ) + suite.credentials.AzureClientSecret) require.NoError(t, err) - serv := graph.NewService(adpt) + serv := NewService(adpt) email := models.NewMessage() subject := "TestSerializationEndPoint" email.SetSubject(&subject) diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 711e55ff8..2f17ae026 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -66,7 +66,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() { for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - gc, err := NewGraphConnector(ctx, graph.LargeItemClient(), test.acct(t), Users) + gc, err := NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), test.acct(t), Users) assert.Nil(t, gc, test.name+" failed") assert.NotNil(t, err, test.name+"failed") }) diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 85ad3f45f..af4ae0824 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -156,7 +156,7 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() { tester.MustGetEnvSets(suite.T(), tester.M365AcctCredEnvs) - suite.connector = loadConnector(ctx, suite.T(), graph.LargeItemClient(), Users) + suite.connector = loadConnector(ctx, suite.T(), graph.HTTPClient(graph.NoTimeout()), Users) suite.user = tester.M365UserID(suite.T()) suite.acct = tester.NewM365Account(suite.T()) @@ -380,7 +380,7 @@ func runRestoreBackupTest( start := time.Now() - restoreGC := loadConnector(ctx, t, graph.LargeItemClient(), test.resource) + restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource) restoreSel := getSelectorWith(t, test.service, resourceOwners, true) deets, err := restoreGC.RestoreDataCollections( ctx, @@ -419,7 +419,7 @@ func runRestoreBackupTest( }) } - backupGC := loadConnector(ctx, t, graph.LargeItemClient(), test.resource) + backupGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Logf("Selective backup of %s\n", backupSel) @@ -870,7 +870,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames dest.ContainerName, ) - restoreGC := loadConnector(ctx, t, graph.LargeItemClient(), test.resource) + restoreGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource) deets, err := restoreGC.RestoreDataCollections(ctx, suite.acct, restoreSel, dest, collections) require.NoError(t, err) require.NotNil(t, deets) @@ -888,7 +888,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames // Run a backup and compare its output with what we put in. - backupGC := loadConnector(ctx, t, graph.LargeItemClient(), test.resource) + backupGC := loadConnector(ctx, t, graph.HTTPClient(graph.NoTimeout()), test.resource) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Log("Selective backup of", backupSel) diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index 6267ff017..b608e9068 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -151,7 +151,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { require.NoError(t, err) coll := NewCollection( - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), folderPath, "drive-id", suite, @@ -245,7 +245,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() { require.NoError(t, err) coll := NewCollection( - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), folderPath, "fakeDriveID", suite, diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 21ca061a9..b69253918 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -588,7 +588,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { outputFolderMap := map[string]string{} maps.Copy(outputFolderMap, tt.inputFolderMap) c := NewCollections( - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), tenant, user, OneDriveSource, diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index 755b7293b..631381240 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -147,7 +147,7 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() { NewOneDriveBackup([]string{test.user}). AllData()[0] odcs, err := NewCollections( - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), creds.AzureTenantID, test.user, OneDriveSource, diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 2b28f0910..5364d543c 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -126,7 +126,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { // Read data for the file - itemInfo, itemData, err := oneDriveItemReader(graph.LargeItemClient(), driveItem) + itemInfo, itemData, err := oneDriveItemReader(graph.HTTPClient(graph.NoTimeout()), driveItem) require.NoError(suite.T(), err) require.NotNil(suite.T(), itemInfo.OneDrive) require.NotEmpty(suite.T(), itemInfo.OneDrive.ItemName) diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 4daca0877..87aaa5c84 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -92,7 +92,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() { newPaths := map[string]string{} excluded := map[string]struct{}{} c := onedrive.NewCollections( - graph.LargeItemClient(), + graph.HTTPClient(graph.NoTimeout()), tenant, site, onedrive.SharePointSource, diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 83748baa2..f6e63b022 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -655,7 +655,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { m365, err := acct.M365Config() require.NoError(t, err) - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, connector.Users) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users) require.NoError(t, err) ac, err := api.NewClient(m365) diff --git a/src/internal/operations/operation.go b/src/internal/operations/operation.go index c910712fb..24391997e 100644 --- a/src/internal/operations/operation.go +++ b/src/internal/operations/operation.go @@ -108,7 +108,7 @@ func connectToM365( resource = connector.Sites } - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), acct, resource) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, resource) if err != nil { return nil, err } diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index e0dd75af9..984326b6d 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -21,7 +21,7 @@ type User struct { // Users returns a list of users in the specified M365 tenant // TODO: Implement paging support func Users(ctx context.Context, m365Account account.Account) ([]*User, error) { - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), m365Account, connector.Users) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), m365Account, connector.Users) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } @@ -77,7 +77,7 @@ func UserPNs(ctx context.Context, m365Account account.Account) ([]string, error) // SiteURLs returns a list of SharePoint site WebURLs in the specified M365 tenant func SiteURLs(ctx context.Context, m365Account account.Account) ([]string, error) { - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), m365Account, connector.Sites) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), m365Account, connector.Sites) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } @@ -87,7 +87,7 @@ func SiteURLs(ctx context.Context, m365Account account.Account) ([]string, error // SiteURLs returns a list of SharePoint sites IDs in the specified M365 tenant func SiteIDs(ctx context.Context, m365Account account.Account) ([]string, error) { - gc, err := connector.NewGraphConnector(ctx, graph.LargeItemClient(), m365Account, connector.Sites) + gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), m365Account, connector.Sites) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") }