Bump up max page size preference for exchange delta requests (#2338)
## Description Reduces the number of roundtrips when requesting delta records for exchange TODO: Need to investigate OneDrive behavior - this header doesn't appear to work with OneDrive. ## Does this PR need a docs update or release note? - [x] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [ ] ⛔ No ## Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup ## Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #2332 ## Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
387f8e8cd7
commit
c510af3fda
@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||||||
|
|
||||||
- Document Corso's fault-tolerance and restartability features
|
- Document Corso's fault-tolerance and restartability features
|
||||||
- Add retries on timeouts and status code 500 for Exchange
|
- Add retries on timeouts and status code 500 for Exchange
|
||||||
|
- Increase page size preference for delta requests for Exchange to reduce number of roundtrips
|
||||||
|
|
||||||
## [v0.2.0] (alpha) - 2023-1-29
|
## [v0.2.0] (alpha) - 2023-1-29
|
||||||
|
|
||||||
|
|||||||
@ -3,6 +3,7 @@ package api
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
abstractions "github.com/microsoft/kiota-abstractions-go"
|
||||||
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
"github.com/microsoftgraph/msgraph-sdk-go/users"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -53,6 +54,16 @@ var (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// headerKeyPrefer is used to set query preferences
|
||||||
|
headerKeyPrefer = "Prefer"
|
||||||
|
// maxPageSizeHeaderFmt is used to indicate max page size
|
||||||
|
// preferences
|
||||||
|
maxPageSizeHeaderFmt = "odata.maxpagesize=%d"
|
||||||
|
// deltaMaxPageSize is the max page size to use for delta queries
|
||||||
|
deltaMaxPageSize = 200
|
||||||
|
)
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// exchange.Query Option Section
|
// exchange.Query Option Section
|
||||||
// These functions can be used to filter a response on M365
|
// These functions can be used to filter a response on M365
|
||||||
@ -71,8 +82,10 @@ func optionsForFolderMessagesDelta(
|
|||||||
requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
|
requestParameters := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
|
||||||
Select: selecting,
|
Select: selecting,
|
||||||
}
|
}
|
||||||
|
|
||||||
options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
|
options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
|
||||||
QueryParameters: requestParameters,
|
QueryParameters: requestParameters,
|
||||||
|
Headers: buildDeltaRequestHeaders(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return options, nil
|
return options, nil
|
||||||
@ -218,6 +231,7 @@ func optionsForContactFoldersItemDelta(
|
|||||||
|
|
||||||
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
|
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
|
||||||
QueryParameters: requestParameters,
|
QueryParameters: requestParameters,
|
||||||
|
Headers: buildDeltaRequestHeaders(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return options, nil
|
return options, nil
|
||||||
@ -275,3 +289,11 @@ func buildOptions(fields []string, allowed map[string]struct{}) ([]string, error
|
|||||||
|
|
||||||
return append(returnedOptions, fields...), nil
|
return append(returnedOptions, fields...), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildDeltaRequestHeaders returns the headers we add to delta page requests
|
||||||
|
func buildDeltaRequestHeaders() *abstractions.RequestHeaders {
|
||||||
|
headers := abstractions.NewRequestHeaders()
|
||||||
|
headers.Add(headerKeyPrefer, fmt.Sprintf(maxPageSizeHeaderFmt, deltaMaxPageSize))
|
||||||
|
|
||||||
|
return headers
|
||||||
|
}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
"github.com/alcionai/corso/src/internal/connector/graph/api"
|
"github.com/alcionai/corso/src/internal/connector/graph/api"
|
||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -82,6 +83,8 @@ func getItemsAddedAndRemovedFromContainer(
|
|||||||
return nil, nil, "", err
|
return nil, nil, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.Ctx(ctx).Infow("Got page", "items", len(items))
|
||||||
|
|
||||||
// iterate through the items in the page
|
// iterate through the items in the page
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
// if the additional data conains a `@removed` key, the value will either
|
// if the additional data conains a `@removed` key, the value will either
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user