Add test

src/internal/m365/collection/drive/collection.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/spatialcurrent/go-lazy/pkg/lazy"
 
 	"github.com/alcionai/corso/src/internal/common/idname"
@@ -386,9 +387,20 @@ func downloadContent(
 	logger.CtxErr(ctx, err).Debug("url cache miss: refetching from API")
 	counter.Inc(count.URLCacheMiss)
 
-	di, err := iaag.GetItem(ctx, driveID, ptr.Val(item.GetId()))
+	// di, err := iaag.GetItem(ctx, driveID, ptr.Val(item.GetId()))
-	if err != nil {
+	// if err != nil {
-		return nil, clues.Wrap(err, "retrieving expired item")
+	// 	return nil, clues.Wrap(err, "retrieving expired item")
+	// }
+
+	var di models.DriveItemable
+
+	for i := 0; i < 1000000; i++ {
+		di, err := iaag.GetItem(ctx, driveID, ptr.Val(item.GetId()))
+		if err != nil {
+			return nil, clues.Wrap(err, "retrieving expired item")
+		}
+
+		logger.Ctx(ctx).Debugf("iteration %d item id %s\n ", i, ptr.Val(di.GetId()))
 	}
 
 	cdi := custom.ToCustomDriveItem(di)

src/pkg/services/m365/api/graph/errors.go

@@ -52,7 +52,8 @@ const (
 	// This error occurs when an attempt is made to create a folder that has
 	// the same name as another folder in the same parent. Such duplicate folder
 	// names are not allowed by graph.
-	folderExists errorCode = "ErrorFolderExists"
+	folderExists   errorCode = "ErrorFolderExists"
+	invalidRequest errorCode = "invalidRequest"
 	// Some datacenters are returning this when we try to get the inbox of a user
 	// that doesn't exist.
 	invalidUser                 errorCode = "ErrorInvalidUser"
@@ -140,6 +141,14 @@ var (
 	ErrTokenExpired = clues.New("jwt token expired")
 )
 
+// IsErrInvalidRequest is to retry transient graph 400 errors with odata error code
+// set to invalidRequest.
+func IsErrInvalidRequest(err error, resp *http.Response) bool {
+	return resp != nil &&
+		resp.StatusCode == http.StatusBadRequest &&
+		parseODataErr(err).hasErrorCode(err, invalidRequest)
+}
+
 func IsErrApplicationThrottled(err error) bool {
 	return errors.Is(err, ErrApplicationThrottled) ||
 		parseODataErr(err).hasErrorCode(err, applicationThrottled)

src/pkg/services/m365/api/graph/middleware.go

@@ -206,6 +206,7 @@ func (mw RetryMiddleware) retryRequest(
 	// 1, there was a prior error OR the status code match retriable conditions.
 	// 3, the request method is retriable.
 	// 4, we haven't already hit maximum retries.
+
 	shouldRetry := (priorErr != nil || mw.isRetriableRespCode(ctx, resp)) &&
 		mw.isRetriableRequest(req) &&
 		executionCount < mw.MaxRetries
@@ -214,6 +215,10 @@ func (mw RetryMiddleware) retryRequest(
 		return resp, stackReq(ctx, req, resp, priorErr).OrNil()
 	}
 
+	if resp != nil && resp.StatusCode == http.StatusBadRequest {
+		logger.Ctx(ctx).Info("retrying 400_invalid_request")
+	}
+
 	executionCount++
 
 	delay := mw.getRetryDelay(req, resp, exponentialBackoff)
@@ -267,6 +272,10 @@ var retryableRespCodes = []int{
 	http.StatusBadGateway,
 }
 
+const (
+	invalidRequestMsg = "invalidRequest"
+)
+
 func (mw RetryMiddleware) isRetriableRespCode(ctx context.Context, resp *http.Response) bool {
 	if resp == nil {
 		return false
@@ -284,9 +293,16 @@ func (mw RetryMiddleware) isRetriableRespCode(ctx context.Context, resp *http.Re
 
 	// not a status code, but the message itself might indicate a connectivity issue that
 	// can be retried independent of the status code.
-	return strings.Contains(
+	first := strings.Contains(
 		strings.ToLower(getRespDump(ctx, resp, true)),
 		strings.ToLower(string(IOErrDuringRead)))
+
+	second := resp.StatusCode == http.StatusBadRequest &&
+		strings.Contains(
+			strings.ToLower(getRespDump(ctx, resp, true)),
+			strings.ToLower(invalidRequestMsg))
+
+	return first || second
 }
 
 func (mw RetryMiddleware) isRetriableRequest(req *http.Request) bool {

src/pkg/services/m365/api/graph/middleware_test.go

@@ -15,7 +15,6 @@ import (
 	msgraphsdkgo "github.com/microsoftgraph/msgraph-sdk-go"
 	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/microsoftgraph/msgraph-sdk-go/users"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -44,9 +43,14 @@ func newMWReturns(code int, body []byte, err error) mwReturns {
 
 	resp := &http.Response{
 		StatusCode: code,
-		Body:       brc,
+		//Status:     "",
+		Body: brc,
 	}
 
+	resp.Header = make(http.Header)
+
+	resp.Header.Add("Content-Type", "application/json")
+
 	if code == 0 {
 		resp = nil
 	}
@@ -154,9 +158,22 @@ func (suite *RetryMWIntgSuite) SetupSuite() {
 
 func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() {
 	var (
-		uri     = "https://graph.microsoft.com"
+		//uri     = "https://graph.microsoft.com"
-		urlPath = "/v1.0/users/user/messages/foo"
+		//urlPath = "/v1.0/users/"
-		url     = uri + urlPath
+		//url     = uri + urlPath
+		//1623c35a-b67a-473a-9b21-5e4891f22e70
+
+		jsonData = map[string]interface{}{
+			"error": map[string]interface{}{
+				"code":    "invalidRequest",
+				"message": "Invalid request",
+				"innerError": map[string]interface{}{
+					"date":              "mmddyy",
+					"request-id":        "rid",
+					"client-request-id": "cid",
+				},
+			},
+		}
 	)
 
 	tests := []struct {
@@ -167,13 +184,13 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() {
 		mw               testMW
 		expectErr        assert.ErrorAssertionFunc
 	}{
-		{
+		// {
-			name:             "200, no retries",
+		// 	name:             "200, no retries",
-			status:           http.StatusOK,
+		// 	status:           http.StatusOK,
-			providedErr:      nil,
+		// 	providedErr:      nil,
-			expectRetryCount: 0,
+		// 	expectRetryCount: 0,
-			expectErr:        assert.NoError,
+		// 	expectErr:        assert.NoError,
-		},
+		// },
 		{
 			name:             "400, no retries",
 			status:           http.StatusBadRequest,
@@ -252,10 +269,13 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() {
 			ctx, flush := tester.NewContext(t)
 			defer flush()
 
+			json_string, err := json.Marshal(jsonData)
+			require.NoError(t, err, clues.ToCore(err))
+
 			called := 0
 			mw := newTestMW(
 				func(*http.Request) { called++ },
-				newMWReturns(test.status, nil, test.providedErr))
+				newMWReturns(test.status, json_string, test.providedErr))
 			mw.repeatReturn0 = true
 
 			// Add a large timeout of 100 seconds to ensure that the ctx deadline
@@ -272,7 +292,11 @@ func (suite *RetryMWIntgSuite) TestRetryMiddleware_Intercept_byStatusCode() {
 			require.NoError(t, err, clues.ToCore(err))
 
 			// url doesn't fit the builder, but that shouldn't matter
-			_, err = users.NewCountRequestBuilder(url, adpt).Get(ctx, nil)
+			//_, err = users.NewCountRequestBuilder(url, adpt).Get(ctx, nil)
+
+			//Users().Get(context.Background(), nil)
+
+			_, err = NewService(adpt).Client().Users().Get(ctx, nil)
 			test.expectErr(t, err, clues.ToCore(err))
 
 			// -1 because the non-retried call always counts for one, then