log non-2xx graph api calls (#1860)

## Description

Adds logging for non-2xx http communication
within the graph API client.  Logging req/resp
metadata is turned off by default, and requires
either a cli flag or os env var to enable.

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🤖 Test

## Issue(s)

* #1858

## Test Plan

- [x] 💚 E2E

.github/workflows/ci.yml

@@ -208,6 +208,7 @@ jobs:
           CORSO_M365_TEST_USER_ID: ${{ secrets.CORSO_M365_TEST_USER_ID }}
           CORSO_SECONDARY_M365_TEST_USER_ID: ${{ secrets.CORSO_SECONDARY_M365_TEST_USER_ID }}
           CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
+          LOG_GRAPH_REQUESTS: true
         run: |
           set -euo pipefail
           go test       \

.github/workflows/load_test.yml

@@ -52,11 +52,12 @@ jobs:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-          CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}
+          CORSO_LOAD_TESTS: true 
           CORSO_M365_LOAD_TEST_USER_ID: ${{ secrets.CORSO_M365_LOAD_TEST_USER_ID }}
           CORSO_M365_LOAD_TEST_ORG_USERS: ${{ secrets.CORSO_M365_LOAD_TEST_ORG_USERS }}
+          CORSO_PASSPHRASE: ${{ secrets.CORSO_PASSPHRASE }}
           IGNORE_LOAD_TEST_USER_ID: ${{ secrets.EXT_SDK_TEST_USER_ID }}
-          CORSO_LOAD_TESTS: true 
+          LOG_GRAPH_REQUESTS: true
         run: |
           set -euo pipefail
           go test       \

src/internal/connector/graph/service_helper.go

@@ -1,7 +1,6 @@
 package graph
 
 import (
-	"context"
 	nethttp "net/http"
 	"net/http/httputil"
 	"os"
@@ -51,12 +50,7 @@ func CreateAdapter(tenant, client, secret string) (*msgraphsdk.GraphRequestAdapt
 func CreateHTTPClient() *nethttp.Client {
 	clientOptions := msgraphsdk.GetDefaultClientOptions()
 	middlewares := msgraphgocore.GetDefaultMiddlewaresWithOptions(&clientOptions)
-
-	// When true, additional logging middleware support added for http request
-	if os.Getenv(logGraphRequestsEnvKey) != "" {
-		middlewares = append(middlewares, &LoggingMiddleware{})
-	}
-
+	middlewares = append(middlewares, &LoggingMiddleware{})
 	httpClient := msgraphgocore.GetDefaultClient(&clientOptions, middlewares...)
 	httpClient.Timeout = time.Second * 90
 
@@ -68,12 +62,39 @@ type LoggingMiddleware struct{}
 
 // Intercept implements the RequestInterceptor interface and decodes the parameters name
 func (handler *LoggingMiddleware) Intercept(
-	pipeline khttp.Pipeline, middlewareIndex int, req *nethttp.Request,
+	pipeline khttp.Pipeline,
+	middlewareIndex int,
+	req *nethttp.Request,
 ) (*nethttp.Response, error) {
-	requestDump, _ := httputil.DumpRequest(req, true)
-	logger.Ctx(context.TODO()).Infof("REQUEST: %s", string(requestDump))
+	var (
+		ctx       = req.Context()
+		resp, err = pipeline.Next(req, middlewareIndex)
+	)
 
-	return pipeline.Next(req, middlewareIndex)
+	if resp == nil {
+		return resp, err
+	}
+
+	if (resp.StatusCode / 100) == 2 {
+		return resp, err
+	}
+
+	if logger.DebugAPI || os.Getenv(logGraphRequestsEnvKey) != "" {
+		respDump, _ := httputil.DumpResponse(resp, true)
+
+		metadata := []any{
+			"method", req.Method,
+			"url", req.URL,
+			"requestLen", req.ContentLength,
+			"status", resp.Status,
+			"statusCode", resp.StatusCode,
+			"request", string(respDump),
+		}
+
+		logger.Ctx(ctx).Errorw("non-2xx graph api response", metadata...)
+	}
+
+	return resp, err
 }
 
 func StringToPathCategory(input string) path.CategoryType {

src/pkg/logger/logger.go

@@ -13,10 +13,11 @@ import (
 var (
 	logCore   *zapcore.Core
 	loggerton *zap.SugaredLogger
+
 	// logging level flag
-	// TODO: infer default based on environment.
 	llFlag = "info"
 
+	DebugAPI       bool
 	readableOutput bool
 )
 
@@ -31,6 +32,7 @@ const (
 )
 
 const (
+	debugAPIFN     = "debug-api-calls"
 	logLevelFN     = "log-level"
 	readableLogsFN = "readable-logs"
 )
@@ -43,6 +45,8 @@ func AddLogLevelFlag(cmd *cobra.Command) {
 	fs := cmd.PersistentFlags()
 	fs.StringVar(&llFlag, logLevelFN, "info", "set the log level to debug|info|warn|error")
 
+	fs.Bool(debugAPIFN, false, "add non-2xx request/response errors to logging")
+
 	fs.Bool(
 		readableLogsFN, false,
 		"minimizes log output for console readability: removes the file and date, colors the level")
@@ -57,6 +61,7 @@ func PreloadLogLevel() string {
 	fs := pflag.NewFlagSet("seed-logger", pflag.ContinueOnError)
 	fs.ParseErrorsWhitelist.UnknownFlags = true
 	fs.String(logLevelFN, "info", "set the log level to debug|info|warn|error")
+	fs.BoolVar(&DebugAPI, debugAPIFN, false, "add non-2xx request/response errors to logging")
 	fs.BoolVar(&readableOutput, readableLogsFN, false, "minimizes log output: removes the file and date, colors the level")
 	// prevents overriding the corso/cobra help processor
 	fs.BoolP("help", "h", false, "")