Add dummy resp body

src/pkg/services/m365/api/graph/logging.go

@@ -42,6 +42,14 @@ func logResp(ctx context.Context, resp *http.Response) {
 		return
 	}
 
+	// Dump both request and response for graph 400 errors. This is a temporary
+	// measure to help us understand why graph is returning transient 400s.
+	if resp.StatusCode == http.StatusBadRequest {
+		log.With("request", getRequestDump(ctx, resp.Request, true)).
+			With("response", getRespDump(ctx, resp, logBody)).
+			Info("graph api bad request")
+	}
+
 	// Log api calls according to api debugging configurations.
 	switch respClass {
 	case 2:
@@ -61,6 +69,15 @@ func logResp(ctx context.Context, resp *http.Response) {
 	}
 }
 
+func logReq(ctx context.Context, req *http.Request) {
+	var (
+		log = logger.Ctx(ctx)
+	)
+
+	log.With("request", getRequestDump(ctx, req, true)).
+		Info("graph api req")
+}
+
 func getRespDump(ctx context.Context, resp *http.Response, getBody bool) string {
 	respDump, err := httputil.DumpResponse(resp, getBody)
 	if err != nil {
@@ -69,3 +86,12 @@ func getRespDump(ctx context.Context, resp *http.Response, getBody bool) string
 
 	return string(respDump)
 }
+
+func getRequestDump(ctx context.Context, req *http.Request, getBody bool) string {
+	reqDump, err := httputil.DumpRequest(req, getBody)
+	if err != nil {
+		logger.CtxErr(ctx, err).Error("dumping http request")
+	}
+
+	return string(reqDump)
+}

src/pkg/services/m365/api/graph/middleware.go

@@ -118,6 +118,8 @@ func (mw *LoggingMiddleware) Intercept(
 	middlewareIndex int,
 	req *http.Request,
 ) (*http.Response, error) {
+	logReq(req.Context(), req)
+
 	// call the next middleware
 	resp, err := pipeline.Next(req, middlewareIndex)
 	if resp == nil {

src/pkg/services/m365/api/graph/service.go

@@ -2,7 +2,9 @@ package graph
 
 import (
 	"context"
+	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
@@ -285,6 +287,27 @@ func kiotaMiddlewares(
 	cc *clientConfig,
 	counter *count.Bus,
 ) []khttp.Middleware {
+	// Add chaos handler to default middleware
+	chaosOpt := &khttp.ChaosHandlerOptions{
+		ChaosStrategy:   khttp.Random,
+		ChaosPercentage: 50,
+		StatusCode:      429,
+		ResponseBody: &http.Response{
+			StatusCode: 429,
+			// Retry-After header
+			Header: http.Header{
+				"Retry-After": []string{"1"},
+			},
+			// Dummy body
+			Body: io.NopCloser(strings.NewReader("hello")),
+		},
+	}
+
+	chaosHandler, err := khttp.NewChaosHandlerWithOptions(chaosOpt)
+	if err != nil {
+		panic(err)
+	}
+
 	retryOptions := khttp.RetryHandlerOptions{
 		ShouldRetry: func(
 			delay time.Duration,
@@ -311,6 +334,7 @@ func kiotaMiddlewares(
 		khttp.NewParametersNameDecodingHandler(),
 		khttp.NewUserAgentHandler(),
 		&LoggingMiddleware{},
+		chaosHandler,
 	}
 
 	// Optionally add concurrency limiter middleware if it has been initialized.