follow-up refinements to m365/api (#3461)

some renaming, mostly refactoring the configuration (select and header declaration for each query) by combining the consts.go and query_param.go file into config.go, and making a more composable set of funcs and consts.

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #1996

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-05-22 12:37:01 -06:00 committed by GitHub
parent 11752d869c
commit 3d170c5b66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 533 additions and 498 deletions

View File

@ -20,15 +20,15 @@ func ParseBool(v string) bool {
return s
}
func FromMapToAny(k string, m map[string]any) (string, error) {
func AnyValueToString(k string, m map[string]any) (string, error) {
if len(m) == 0 {
return "", clues.New("missing entry").With("map_key", k)
}
return FromAny(m[k])
return AnyToString(m[k])
}
func FromAny(a any) (string, error) {
func AnyToString(a any) (string, error) {
if a == nil {
return "", clues.New("missing value")
}

View File

@ -6,7 +6,7 @@ import (
"github.com/alcionai/clues"
)
func FromMapToAny[T any](k string, m map[string]any) (T, error) {
func AnyValueToT[T any](k string, m map[string]any) (T, error) {
v, ok := m[k]
if !ok {
return *new(T), clues.New("entry not found")

View File

@ -1,51 +0,0 @@
package api
import (
"strings"
"github.com/alcionai/corso/src/internal/common/ptr"
)
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
// IsNextLinkValid separate check to investigate whether error is
func IsNextLinkValid(next string) bool {
return !strings.Contains(next, `users//`)
}
func NextLink(pl PageLinker) string {
return ptr.Val(pl.GetOdataNextLink())
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
return NextLink(pl), ptr.Val(pl.GetOdataDeltaLink())
}
type Valuer[T any] interface {
GetValue() []T
}
type PageLinkValuer[T any] interface {
PageLinker
Valuer[T]
}
// EmptyDeltaLinker is used to convert PageLinker to DeltaPageLinker
type EmptyDeltaLinker[T any] struct {
PageLinkValuer[T]
}
func (EmptyDeltaLinker[T]) GetOdataDeltaLink() *string {
return ptr.To("")
}
func (e EmptyDeltaLinker[T]) GetValue() []T {
return e.PageLinkValuer.GetValue()
}

View File

@ -1,147 +0,0 @@
package api_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester"
)
type mockNextLink struct {
nextLink *string
}
func (l mockNextLink) GetOdataNextLink() *string {
return l.nextLink
}
type mockDeltaNextLink struct {
mockNextLink
deltaLink *string
}
func (l mockDeltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
type APIUnitSuite struct {
tester.Suite
}
func TestAPIUnitSuite(t *testing.T) {
s := &APIUnitSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func (suite *APIUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.Run(name, func() {
t := suite.T()
l := mockDeltaNextLink{
mockNextLink: mockNextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := api.NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}
// TestIsLinkValid check to verify is nextLink guard check for logging
// Related to: https://github.com/alcionai/corso/issues/2520
//
//nolint:lll
func (suite *APIUnitSuite) TestIsLinkValid() {
invalidString := `https://graph.microsoft.com/v1.0/users//mailFolders//messages/microsoft.graph.delta()?$select=id%2CisRead`
tests := []struct {
name string
inputString string
isValid assert.BoolAssertionFunc
}{
{
name: "Empty",
inputString: emptyLink,
isValid: assert.True,
},
{
name: "Invalid",
inputString: invalidString,
isValid: assert.False,
},
{
name: "Valid",
inputString: `https://graph.microsoft.com/v1.0/users/aPerson/mailFolders/AMessage/messages/microsoft.graph.delta()?$select=id%2CisRead`,
isValid: assert.True,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
got := api.IsNextLinkValid(test.inputString)
test.isValid(suite.T(), got)
})
}
}

View File

@ -188,3 +188,19 @@ func CheckIDAndName(c Container) error {
return nil
}
// CheckIDNameAndParentFolderID is a validator that ensures the ID
// and name are populated and not zero valued.
func CheckIDNameAndParentFolderID(c Container) error {
err := CheckIDAndName(c)
if err != nil {
return err
}
pfid := ptr.Val(c.GetParentFolderId())
if len(pfid) == 0 {
return clues.New("container missing parent folder id").With("container_id", ptr.Val(c.GetId()))
}
return nil
}

View File

@ -18,7 +18,6 @@ import (
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock"
"github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -1205,7 +1204,7 @@ type mockItemPager struct {
getIdx int
}
func (p *mockItemPager) GetPage(context.Context) (gapi.DeltaPageLinker, error) {
func (p *mockItemPager) GetPage(context.Context) (api.DeltaPageLinker, error) {
if len(p.toReturn) <= p.getIdx {
return nil, assert.AnError
}
@ -1222,7 +1221,7 @@ func (p *mockItemPager) GetPage(context.Context) (gapi.DeltaPageLinker, error) {
func (p *mockItemPager) SetNext(string) {}
func (p *mockItemPager) Reset() {}
func (p *mockItemPager) ValuesIn(gapi.DeltaPageLinker) ([]models.DriveItemable, error) {
func (p *mockItemPager) ValuesIn(api.DeltaPageLinker) ([]models.DriveItemable, error) {
idx := p.getIdx
if idx > 0 {
// Return values lag by one since we increment in GetPage().

View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -90,39 +89,17 @@ type itemCollector func(
) error
type itemPager interface {
GetPage(context.Context) (gapi.DeltaPageLinker, error)
GetPage(context.Context) (api.DeltaPageLinker, error)
SetNext(nextLink string)
Reset()
ValuesIn(gapi.DeltaPageLinker) ([]models.DriveItemable, error)
ValuesIn(api.DeltaPageLinker) ([]models.DriveItemable, error)
}
func defaultItemPager(
servicer graph.Servicer,
driveID, link string,
) itemPager {
return api.NewItemPager(
servicer,
driveID,
link,
[]string{
"content.downloadUrl",
"createdBy",
"createdDateTime",
"file",
"folder",
"id",
"lastModifiedDateTime",
"name",
"package",
"parentReference",
"root",
"sharepointIds",
"size",
"deleted",
"malware",
"shared",
},
)
return api.NewItemPager(servicer, driveID, link, api.DriveItemSelectDefault())
}
// collectItems will enumerate all items in the specified drive and hand them to the
@ -196,7 +173,7 @@ func collectItems(
return DeltaUpdate{}, nil, nil, err
}
nextLink, deltaLink := gapi.NextAndDeltaLink(page)
nextLink, deltaLink := api.NextAndDeltaLink(page)
if len(deltaLink) > 0 {
newDeltaURL = deltaLink

View File

@ -1,8 +0,0 @@
package api
type NameID struct {
Name string
ID string
}
const fetchChannelSize = 5

View File

@ -1,22 +0,0 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret)
require.NoError(t, err, clues.ToCore(err))
return api.NewBetaService(adapter)
}

View File

@ -7,8 +7,6 @@ import (
"sync"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
@ -19,21 +17,25 @@ import (
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type NameID struct {
Name string
ID string
}
// GetSitePages retrieves a collection of Pages related to the give Site.
// Returns error if error experienced during the call
func GetSitePages(
ctx context.Context,
serv *api.BetaService,
serv *BetaService,
siteID string,
pages []string,
errs *fault.Bus,
) ([]betamodels.SitePageable, error) {
var (
col = make([]betamodels.SitePageable, 0)
semaphoreCh = make(chan struct{}, fetchChannelSize)
semaphoreCh = make(chan struct{}, 5)
opts = retrieveSitePageOptions()
el = errs.Local()
wg sync.WaitGroup
@ -82,25 +84,8 @@ func GetSitePages(
return col, el.Failure()
}
// GetSite returns a minimal Site with the SiteID and the WebURL
func GetSite(ctx context.Context, gs graph.Servicer, siteID string) (models.Siteable, error) {
// resp *sites.SiteItemRequestBuilderresp *sites.SiteItemRequestBuilde
options := &sites.SiteItemRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.SiteItemRequestBuilderGetQueryParameters{
Select: []string{"webUrl"},
},
}
resp, err := gs.Client().Sites().BySiteId(siteID).Get(ctx, options)
if err != nil {
return nil, err
}
return resp, nil
}
// fetchPages utility function to return the tuple of item
func FetchPages(ctx context.Context, bs *api.BetaService, siteID string) ([]NameID, error) {
func FetchPages(ctx context.Context, bs *BetaService, siteID string) ([]NameID, error) {
var (
builder = bs.Client().SitesById(siteID).Pages()
opts = fetchPageOptions()
@ -159,7 +144,7 @@ func fetchPageOptions() *betasites.ItemPagesRequestBuilderGetRequestConfiguratio
// https://github.com/alcionai/corso/issues/2707
func DeleteSitePage(
ctx context.Context,
serv *api.BetaService,
serv *BetaService,
siteID, pageID string,
) error {
err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil)
@ -184,7 +169,7 @@ func retrieveSitePageOptions() *betasites.ItemPagesSitePageItemRequestBuilderGet
func RestoreSitePage(
ctx context.Context,
service *api.BetaService,
service *BetaService,
itemData data.Stream,
siteID, destName string,
) (details.ItemInfo, error) {

View File

@ -10,20 +10,30 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
)
func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret)
require.NoError(t, err, clues.ToCore(err))
return api.NewBetaService(adapter)
}
type SharePointPageSuite struct {
tester.Suite
siteID string
creds account.M365Config
service *m365api.BetaService
service *api.BetaService
}
func (suite *SharePointPageSuite) SetupSuite() {

View File

@ -12,7 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
@ -21,7 +21,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type DataCategory int
@ -56,7 +56,7 @@ type Collection struct {
category DataCategory
service graph.Servicer
ctrl control.Options
betaService *m365api.BetaService
betaService *betaAPI.BetaService
statusUpdater support.StatusUpdater
}
@ -284,7 +284,7 @@ func (sc *Collection) retrievePages(
root := ptr.Val(parent.GetWebUrl())
pages, err := api.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs)
pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs)
if err != nil {
return metrics, err
}

View File

@ -9,7 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
@ -19,7 +19,6 @@ import (
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
)
type statusUpdater interface {
@ -256,9 +255,9 @@ func collectPages(
return nil, clues.Wrap(err, "creating azure client adapter")
}
betaService := m365api.NewBetaService(adpt)
betaService := betaAPI.NewBetaService(adpt)
tuples, err := api.FetchPages(ctx, betaService, site.ID())
tuples, err := betaAPI.FetchPages(ctx, betaService, site.ID())
if err != nil {
return nil, err
}

View File

@ -13,7 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics"
@ -23,7 +23,6 @@ import (
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
)
//----------------------------------------------------------------------------
@ -317,7 +316,7 @@ func RestorePageCollection(
var (
el = errs.Local()
service = m365api.NewBetaService(adpt)
service = betaAPI.NewBetaService(adpt)
items = dc.Items(ctx, errs)
)
@ -336,7 +335,7 @@ func RestorePageCollection(
}
metrics.Objects++
itemInfo, err := api.RestoreSitePage(
itemInfo, err := betaAPI.RestoreSitePage(
ctx,
service,
itemData,

View File

@ -0,0 +1,101 @@
package api
import (
"fmt"
"strings"
abstractions "github.com/microsoft/kiota-abstractions-go"
)
const (
maxNonDeltaPageSize = int32(999)
maxDeltaPageSize = int32(500)
)
// selectable values, case insensitive
// not comprehensive - just adding ones that might
// get easily misspelled.
// eg: we don't need a const for "id"
const (
parentFolderID = "parentFolderId"
displayName = "displayName"
userPrincipalName = "userPrincipalName"
)
// header keys
const (
headerKeyConsistencyLevel = "ConsistencyLevel"
headerKeyPrefer = "Prefer"
)
// header values
const (
idTypeImmutable = `IdType="ImmutableId"`
eventual = "eventual"
)
// ---------------------------------------------------------------------------
// not exported
// ---------------------------------------------------------------------------
func preferPageSize(size int32) string {
return fmt.Sprintf("odata.maxpagesize=%d", size)
}
func preferImmutableIDs(t bool) string {
if !t {
return ""
}
return idTypeImmutable
}
func newPreferHeaders(values ...string) *abstractions.RequestHeaders {
vs := []string{}
for _, v := range values {
if len(v) > 0 {
vs = append(vs, v)
}
}
headers := abstractions.NewRequestHeaders()
headers.Add(headerKeyPrefer, strings.Join(vs, ","))
return headers
}
func newEventualConsistencyHeaders() *abstractions.RequestHeaders {
headers := abstractions.NewRequestHeaders()
headers.Add(headerKeyConsistencyLevel, eventual)
return headers
}
// makes a slice with []string{"id", s...}
func idAnd(ss ...string) []string {
return append([]string{"id"}, ss...)
}
// ---------------------------------------------------------------------------
// exported
// ---------------------------------------------------------------------------
func DriveItemSelectDefault() []string {
return idAnd(
"content.downloadUrl",
"createdBy",
"createdDateTime",
"file",
"folder",
"lastModifiedDateTime",
"name",
"package",
"parentReference",
"root",
"sharepointIds",
"size",
"deleted",
"malware",
"shared")
}

View File

@ -1,3 +0,0 @@
package api
const maxPageSize = int32(999)

View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
@ -81,7 +80,7 @@ func (c Contacts) GetItem(
_ *fault.Bus, // no attachments to iterate over, so this goes unused
) (serialization.Parsable, *details.ExchangeInfo, error) {
options := &users.ItemContactsContactItemRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(false, immutableIDs),
Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
}
cont, err := c.Stable.Client().Users().ByUserId(user).Contacts().ByContactId(itemID).Get(ctx, options)
@ -96,13 +95,18 @@ func (c Contacts) GetContainerByID(
ctx context.Context,
userID, dirID string,
) (graph.Container, error) {
queryParams := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
config := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
Select: idAnd(displayName, parentFolderID),
},
}
resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, queryParams)
resp, err := c.Stable.Client().
Users().
ByUserId(userID).
ContactFolders().
ByContactFolderId(dirID).
Get(ctx, config)
if err != nil {
return nil, graph.Stack(ctx, err)
}
@ -126,9 +130,9 @@ func (c Contacts) EnumerateContainers(
return graph.Stack(ctx, err)
}
queryParams := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
config := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
Select: idAnd(displayName, parentFolderID),
},
}
@ -145,7 +149,7 @@ func (c Contacts) EnumerateContainers(
break
}
resp, err := builder.Get(ctx, queryParams)
resp, err := builder.Get(ctx, config)
if err != nil {
return graph.Stack(ctx, err)
}
@ -155,7 +159,7 @@ func (c Contacts) EnumerateContainers(
return el.Failure()
}
if err := graph.CheckIDAndName(fold); err != nil {
if err := graph.CheckIDNameAndParentFolderID(fold); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue
}
@ -201,25 +205,30 @@ func NewContactPager(
user, directoryID string,
immutableIDs bool,
) itemPager {
queryParams := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
config := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"},
Select: idAnd(parentFolderID),
},
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
builder := gs.Client().Users().ByUserId(user).ContactFolders().ByContactFolderId(directoryID).Contacts()
builder := gs.Client().
Users().
ByUserId(user).
ContactFolders().
ByContactFolderId(directoryID).
Contacts()
return &contactPager{gs, builder, queryParams}
return &contactPager{gs, builder, config}
}
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *contactPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return api.EmptyDeltaLinker[models.Contactable]{PageLinkValuer: resp}, nil
return EmptyDeltaLinker[models.Contactable]{PageLinkValuer: resp}, nil
}
func (p *contactPager) setNext(nextLink string) {
@ -229,7 +238,7 @@ func (p *contactPager) setNext(nextLink string) {
// non delta pagers don't need reset
func (p *contactPager) reset(context.Context) {}
func (p *contactPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *contactPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl)
}
@ -266,9 +275,9 @@ func NewContactDeltaPager(
) itemPager {
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"},
Select: idAnd(parentFolderID),
},
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder
@ -281,7 +290,7 @@ func NewContactDeltaPager(
return &contactDeltaPager{gs, user, directoryID, builder, options}
}
func (p *contactDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *contactDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
@ -298,7 +307,7 @@ func (p *contactDeltaPager) reset(ctx context.Context) {
p.builder = getContactDeltaBuilder(ctx, p.gs, p.user, p.directoryID, p.options)
}
func (p *contactDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *contactDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl)
}

View File

@ -3,11 +3,9 @@ package api
import (
"context"
"fmt"
"strings"
"time"
"github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
"github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
@ -15,7 +13,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/logger"
)
@ -34,22 +31,20 @@ type driveItemPager struct {
func NewItemPager(
gs graph.Servicer,
driveID, link string,
fields []string,
selectFields []string,
) *driveItemPager {
headers := abstractions.NewRequestHeaders()
preferHeaderItems := []string{
"deltashowremovedasdeleted",
"deltatraversepermissiongaps",
"deltashowsharingchanges",
"hierarchicalsharing",
}
headers.Add("Prefer", strings.Join(preferHeaderItems, ","))
requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{
Headers: headers,
Headers: newPreferHeaders(preferHeaderItems...),
QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{
Top: ptr.To(maxPageSize),
Select: fields,
Top: ptr.To(maxDeltaPageSize),
Select: selectFields,
},
}
@ -70,9 +65,9 @@ func NewItemPager(
return res
}
func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *driveItemPager) GetPage(ctx context.Context) (DeltaPageLinker, error) {
var (
resp api.DeltaPageLinker
resp DeltaPageLinker
err error
)
@ -97,7 +92,7 @@ func (p *driveItemPager) Reset() {
Delta()
}
func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) {
func (p *driveItemPager) ValuesIn(l DeltaPageLinker) ([]models.DriveItemable, error) {
return getValues[models.DriveItemable](l)
}
@ -139,9 +134,9 @@ type nopUserDrivePageLinker struct {
func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil }
func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
func (p *userDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
var (
resp api.PageLinker
resp PageLinker
err error
)
@ -167,7 +162,7 @@ func (p *userDrivePager) SetNext(link string) {
p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
func (p *userDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
nl, ok := l.(*nopUserDrivePageLinker)
if !ok || nl == nil {
return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l))
@ -216,9 +211,9 @@ func NewSiteDrivePager(
return res
}
func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
func (p *siteDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
var (
resp api.PageLinker
resp PageLinker
err error
)
@ -234,7 +229,7 @@ func (p *siteDrivePager) SetNext(link string) {
p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
func (p *siteDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
return getValues[models.Driveable](l)
}
@ -244,9 +239,9 @@ func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error)
// DrivePager pages through different types of drive owners
type DrivePager interface {
GetPage(context.Context) (api.PageLinker, error)
GetPage(context.Context) (PageLinker, error)
SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error)
ValuesIn(PageLinker) ([]models.Driveable, error)
}
// GetAllDrives fetches all drives for the given pager
@ -266,7 +261,7 @@ func GetAllDrives(
for {
var (
err error
page api.PageLinker
page PageLinker
)
// Retry Loop for Drive retrieval. Request can timeout
@ -315,7 +310,7 @@ func GetAllDrives(
// Helpers
// ---------------------------------------------------------------------------
func getValues[T any](l api.PageLinker) ([]T, error) {
func getValues[T any](l PageLinker) ([]T, error) {
page, ok := l.(interface{ GetValue() []T })
if !ok {
return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l))

View File

@ -14,7 +14,6 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -84,13 +83,18 @@ func (c Events) GetContainerByID(
return nil, graph.Stack(ctx, err)
}
queryParams := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
config := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{
Select: []string{"id", "name", "owner"},
Select: idAnd("name", "owner"),
},
}
cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, queryParams)
cal, err := service.Client().
Users().
ByUserId(userID).
Calendars().
ByCalendarId(containerID).
Get(ctx, config)
if err != nil {
return nil, graph.Stack(ctx, err).WithClues(ctx)
}
@ -147,23 +151,27 @@ func (c Events) GetItem(
var (
err error
event models.Eventable
header = buildPreferHeaders(false, immutableIDs)
itemOpts = &users.ItemEventsEventItemRequestBuilderGetRequestConfiguration{
Headers: header,
config = &users.ItemEventsEventItemRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
}
)
event, err = c.Stable.Client().Users().ByUserId(user).Events().ByEventId(itemID).Get(ctx, itemOpts)
event, err = c.Stable.Client().
Users().
ByUserId(user).
Events().
ByEventId(itemID).
Get(ctx, config)
if err != nil {
return nil, nil, graph.Stack(ctx, err)
}
if ptr.Val(event.GetHasAttachments()) || HasAttachments(event.GetBody()) {
options := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{
config := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"},
},
Headers: header,
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
attached, err := c.LargeItem.
@ -173,7 +181,7 @@ func (c Events) GetItem(
Events().
ByEventId(itemID).
Attachments().
Get(ctx, options)
Get(ctx, config)
if err != nil {
return nil, nil, graph.Wrap(ctx, err, "event attachment download")
}
@ -200,21 +208,25 @@ func (c Events) EnumerateContainers(
return graph.Stack(ctx, err)
}
queryParams := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
var (
el = errs.Local()
config = &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{
Select: []string{"id", "name"},
Select: idAnd("name"),
},
}
el := errs.Local()
builder := service.Client().Users().ByUserId(userID).Calendars()
builder = service.Client().
Users().
ByUserId(userID).
Calendars()
)
for {
if el.Failure() != nil {
break
}
resp, err := builder.Get(ctx, queryParams)
resp, err := builder.Get(ctx, config)
if err != nil {
return graph.Stack(ctx, err)
}
@ -279,7 +291,7 @@ func NewEventPager(
immutableIDs bool,
) (itemPager, error) {
options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
builder := gs.Client().Users().ByUserId(user).Calendars().ByCalendarId(calendarID).Events()
@ -287,13 +299,13 @@ func NewEventPager(
return &eventPager{gs, builder, options}, nil
}
func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *eventPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return api.EmptyDeltaLinker[models.Eventable]{PageLinkValuer: resp}, nil
return EmptyDeltaLinker[models.Eventable]{PageLinkValuer: resp}, nil
}
func (p *eventPager) setNext(nextLink string) {
@ -303,7 +315,7 @@ func (p *eventPager) setNext(nextLink string) {
// non delta pagers don't need reset
func (p *eventPager) reset(context.Context) {}
func (p *eventPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *eventPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl)
}
@ -328,7 +340,7 @@ func NewEventDeltaPager(
immutableIDs bool,
) (itemPager, error) {
options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
var builder *users.ItemCalendarsItemEventsDeltaRequestBuilder
@ -363,7 +375,7 @@ func getEventDeltaBuilder(
return builder
}
func (p *eventDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *eventDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
@ -380,7 +392,7 @@ func (p *eventDeltaPager) reset(ctx context.Context) {
p.builder = getEventDeltaBuilder(ctx, p.gs, p.user, p.calendarID, p.options)
}
func (p *eventDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *eventDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl)
}

View File

@ -3,22 +3,70 @@ package api
import (
"context"
"fmt"
"strings"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/logger"
)
// ---------------------------------------------------------------------------
// common interfaces and funcs
// ---------------------------------------------------------------------------
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
// IsNextLinkValid separate check to investigate whether error is
func IsNextLinkValid(next string) bool {
return !strings.Contains(next, `users//`)
}
func NextLink(pl PageLinker) string {
return ptr.Val(pl.GetOdataNextLink())
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
return NextLink(pl), ptr.Val(pl.GetOdataDeltaLink())
}
type Valuer[T any] interface {
GetValue() []T
}
type PageLinkValuer[T any] interface {
PageLinker
Valuer[T]
}
// EmptyDeltaLinker is used to convert PageLinker to DeltaPageLinker
type EmptyDeltaLinker[T any] struct {
PageLinkValuer[T]
}
func (EmptyDeltaLinker[T]) GetOdataDeltaLink() *string {
return ptr.To("")
}
func (e EmptyDeltaLinker[T]) GetValue() []T {
return e.PageLinkValuer.GetValue()
}
// ---------------------------------------------------------------------------
// generic handler for paging item ids in a container
// ---------------------------------------------------------------------------
type itemPager interface {
// getPage get a page with the specified options from graph
getPage(context.Context) (api.DeltaPageLinker, error)
getPage(context.Context) (DeltaPageLinker, error)
// setNext is used to pass in the next url got from graph
setNext(string)
// reset is used to clear delta url in delta pagers. When
@ -26,7 +74,7 @@ type itemPager interface {
// currently have and start a new delta query without the token.
reset(context.Context)
// valuesIn gets us the values in a page
valuesIn(api.PageLinker) ([]getIDAndAddtler, error)
valuesIn(PageLinker) ([]getIDAndAddtler, error)
}
type getIDAndAddtler interface {
@ -158,7 +206,7 @@ func getItemsAddedAndRemovedFromContainer(
}
}
nextLink, deltaLink := api.NextAndDeltaLink(resp)
nextLink, deltaLink := NextAndDeltaLink(resp)
// the deltaLink is kind of like a cursor for overall data state.
// once we run through pages of nextLinks, the last query will

View File

@ -2,6 +2,7 @@ package api
import (
"context"
"strings"
"testing"
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
@ -11,10 +12,30 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester"
)
// ---------------------------------------------------------------------------
// mock impls & stubs
// ---------------------------------------------------------------------------
type nextLink struct {
nextLink *string
}
func (l nextLink) GetOdataNextLink() *string {
return l.nextLink
}
type deltaNextLink struct {
nextLink
deltaLink *string
}
func (l deltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testPagerValue struct {
id string
removed bool
@ -51,7 +72,7 @@ type testPager struct {
needsReset bool
}
func (p *testPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *testPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
if p.errorCode != "" {
ierr := odataerrors.NewMainError()
ierr.SetCode(&p.errorCode)
@ -74,7 +95,7 @@ func (p *testPager) reset(context.Context) {
p.errorCode = ""
}
func (p *testPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *testPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
items := []getIDAndAddtler{}
for _, id := range p.added {
@ -88,15 +109,19 @@ func (p *testPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
return items, nil
}
type SharedAPIUnitSuite struct {
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
type ItemPagerUnitSuite struct {
tester.Suite
}
func TestSharedAPIUnitSuite(t *testing.T) {
suite.Run(t, &SharedAPIUnitSuite{Suite: tester.NewUnitSuite(t)})
func TestItemPagerUnitSuite(t *testing.T) {
suite.Run(t, &ItemPagerUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SharedAPIUnitSuite) TestGetAddedAndRemovedItemIDs() {
func (suite *ItemPagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
tests := []struct {
name string
pagerGetter func(context.Context, graph.Servicer, string, string, bool) (itemPager, error)
@ -260,3 +285,110 @@ func (suite *SharedAPIUnitSuite) TestGetAddedAndRemovedItemIDs() {
})
}
}
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
func (suite *ItemPagerUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.Run(name, func() {
t := suite.T()
l := deltaNextLink{
nextLink: nextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}
// TestIsLinkValid check to verify is nextLink guard check for logging
// Related to: https://github.com/alcionai/corso/issues/2520
//
//nolint:lll
func (suite *ItemPagerUnitSuite) TestIsLinkValid() {
invalidString := `https://graph.microsoft.com/v1.0/users//mailFolders//messages/microsoft.graph.delta()?$select=id%2CisRead`
tests := []struct {
name string
inputString string
isValid assert.BoolAssertionFunc
}{
{
name: "Empty",
inputString: emptyLink,
isValid: assert.True,
},
{
name: "Invalid",
inputString: invalidString,
isValid: assert.False,
},
{
name: "Valid",
inputString: `https://graph.microsoft.com/v1.0/users/aPerson/mailFolders/AMessage/messages/microsoft.graph.delta()?$select=id%2CisRead`,
isValid: assert.True,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
got := IsNextLinkValid(test.inputString)
test.isValid(suite.T(), got)
})
}
}

View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -123,9 +122,9 @@ func (c Mail) GetContainerByID(
return nil, graph.Stack(ctx, err)
}
queryParams := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
config := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"},
Select: idAnd(displayName, parentFolderID),
},
}
@ -134,7 +133,7 @@ func (c Mail) GetContainerByID(
ByUserId(userID).
MailFolders().
ByMailFolderId(dirID).
Get(ctx, queryParams)
Get(ctx, config)
if err != nil {
return nil, graph.Stack(ctx, err)
}
@ -153,14 +152,17 @@ func (c Mail) GetItem(
var (
size int64
mailBody models.ItemBodyable
)
// Will need adjusted if attachments start allowing paging.
headers := buildPreferHeaders(false, immutableIDs)
itemOpts := &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{
Headers: headers,
config = &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
}
)
mail, err := c.Stable.Client().Users().ByUserId(user).Messages().ByMessageId(itemID).Get(ctx, itemOpts)
mail, err := c.Stable.Client().
Users().
ByUserId(user).
Messages().
ByMessageId(itemID).
Get(ctx, config)
if err != nil {
return nil, nil, graph.Stack(ctx, err)
}
@ -177,11 +179,11 @@ func (c Mail) GetItem(
return mail, MailInfo(mail, size), nil
}
options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{
attachConfig := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"},
},
Headers: headers,
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
attached, err := c.LargeItem.
@ -191,7 +193,7 @@ func (c Mail) GetItem(
Messages().
ByMessageId(itemID).
Attachments().
Get(ctx, options)
Get(ctx, attachConfig)
if err == nil {
for _, a := range attached.GetValue() {
attachSize := ptr.Val(a.GetSize())
@ -215,7 +217,7 @@ func (c Mail) GetItem(
logger.CtxErr(ctx, err).Info("fetching all attachments by id")
// Getting size just to log in case of error
options.QueryParameters.Select = []string{"id", "size"}
attachConfig.QueryParameters.Select = []string{"id", "size"}
attachments, err := c.LargeItem.
Client().
@ -224,7 +226,7 @@ func (c Mail) GetItem(
Messages().
ByMessageId(itemID).
Attachments().
Get(ctx, options)
Get(ctx, attachConfig)
if err != nil {
return nil, nil, graph.Wrap(ctx, err, "getting mail attachment ids")
}
@ -232,11 +234,11 @@ func (c Mail) GetItem(
atts := []models.Attachmentable{}
for _, a := range attachments.GetValue() {
options := &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetRequestConfiguration{
attachConfig := &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"},
},
Headers: headers,
Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
}
att, err := c.Stable.
@ -247,10 +249,9 @@ func (c Mail) GetItem(
ByMessageId(itemID).
Attachments().
ByAttachmentId(ptr.Val(a.GetId())).
Get(ctx, options)
Get(ctx, attachConfig)
if err != nil {
return nil, nil,
graph.Wrap(ctx, err, "getting mail attachment").
return nil, nil, graph.Wrap(ctx, err, "getting mail attachment").
With("attachment_id", ptr.Val(a.GetId()), "attachment_size", ptr.Val(a.GetSize()))
}
@ -277,7 +278,7 @@ func NewMailFolderPager(service graph.Servicer, user string) mailFolderPager {
return mailFolderPager{service, builder}
}
func (p *mailFolderPager) getPage(ctx context.Context) (api.PageLinker, error) {
func (p *mailFolderPager) getPage(ctx context.Context) (PageLinker, error) {
page, err := p.builder.Get(ctx, nil)
if err != nil {
return nil, graph.Stack(ctx, err)
@ -290,7 +291,7 @@ func (p *mailFolderPager) setNext(nextLink string) {
p.builder = users.NewItemMailFoldersRequestBuilder(nextLink, p.service.Adapter())
}
func (p *mailFolderPager) valuesIn(pl api.PageLinker) ([]models.MailFolderable, error) {
func (p *mailFolderPager) valuesIn(pl PageLinker) ([]models.MailFolderable, error) {
// Ideally this should be `users.ItemMailFoldersResponseable`, but
// that is not a thing as stable returns different result
page, ok := pl.(models.MailFolderCollectionResponseable)
@ -336,17 +337,22 @@ func (c Mail) EnumerateContainers(
return graph.Stack(ctx, err)
}
for _, v := range resp {
for _, fold := range resp {
if el.Failure() != nil {
break
}
if err := graph.CheckIDNameAndParentFolderID(fold); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue
}
fctx := clues.Add(
ctx,
"container_id", ptr.Val(v.GetId()),
"container_name", ptr.Val(v.GetDisplayName()))
"container_id", ptr.Val(fold.GetId()),
"container_name", ptr.Val(fold.GetDisplayName()))
temp := graph.NewCacheFolder(v, nil, nil)
temp := graph.NewCacheFolder(fold, nil, nil)
if err := fn(&temp); err != nil {
errs.AddRecoverable(graph.Stack(fctx, err).Label(fault.LabelForceNoBackupCreation))
continue
@ -382,11 +388,11 @@ func NewMailPager(
user, directoryID string,
immutableIDs bool,
) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
config := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"},
Select: idAnd("isRead"),
},
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
builder := gs.Client().
@ -396,16 +402,16 @@ func NewMailPager(
ByMailFolderId(directoryID).
Messages()
return &mailPager{gs, builder, queryParams}
return &mailPager{gs, builder, config}
}
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *mailPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
page, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return api.EmptyDeltaLinker[models.Messageable]{PageLinkValuer: page}, nil
return EmptyDeltaLinker[models.Messageable]{PageLinkValuer: page}, nil
}
func (p *mailPager) setNext(nextLink string) {
@ -415,7 +421,7 @@ func (p *mailPager) setNext(nextLink string) {
// non delta pagers don't have reset
func (p *mailPager) reset(context.Context) {}
func (p *mailPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *mailPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl)
}
@ -457,11 +463,11 @@ func NewMailDeltaPager(
user, directoryID, oldDelta string,
immutableIDs bool,
) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
config := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"},
Select: idAnd("isRead"),
},
Headers: buildPreferHeaders(true, immutableIDs),
Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
}
var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder
@ -469,13 +475,13 @@ func NewMailDeltaPager(
if len(oldDelta) > 0 {
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())
} else {
builder = getMailDeltaBuilder(ctx, gs, user, directoryID, queryParams)
builder = getMailDeltaBuilder(ctx, gs, user, directoryID, config)
}
return &mailDeltaPager{gs, user, directoryID, builder, queryParams}
return &mailDeltaPager{gs, user, directoryID, builder, config}
}
func (p *mailDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
func (p *mailDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
page, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
@ -498,7 +504,7 @@ func (p *mailDeltaPager) reset(ctx context.Context) {
Delta()
}
func (p *mailDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
func (p *mailDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl)
}

View File

@ -6,14 +6,14 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type PageLinker struct {
type PageLink struct {
Link *string
}
func (pl *PageLinker) GetOdataNextLink() *string {
func (pl *PageLink) GetOdataNextLink() *string {
return pl.Link
}
@ -36,7 +36,7 @@ func (p *DrivePager) GetPage(context.Context) (api.PageLinker, error) {
idx := p.GetIdx
p.GetIdx++
return &PageLinker{p.ToReturn[idx].NextLink}, p.ToReturn[idx].Err
return &PageLink{p.ToReturn[idx].NextLink}, p.ToReturn[idx].Err
}
func (p *DrivePager) SetNext(string) {}

View File

@ -1,26 +0,0 @@
package api
import (
"fmt"
"strings"
abstractions "github.com/microsoft/kiota-abstractions-go"
)
// buildPreferHeaders returns the headers we add to item delta page requests.
func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders {
var allHeaders []string
if pageSize {
allHeaders = append(allHeaders, fmt.Sprintf("odata.maxpagesize=%d", maxPageSize))
}
if immutableID {
allHeaders = append(allHeaders, `IdType="ImmutableId"`)
}
headers := abstractions.NewRequestHeaders()
headers.Add("Prefer", strings.Join(allHeaders, ","))
return headers
}

View File

@ -35,6 +35,20 @@ type Sites struct {
// methods
// ---------------------------------------------------------------------------
// GetSite returns a minimal Site with the SiteID and the WebURL
// TODO: delete in favor of sites.GetByID()
func GetSite(ctx context.Context, gs graph.Servicer, siteID string) (models.Siteable, error) {
resp, err := gs.Client().
Sites().
BySiteId(siteID).
Get(ctx, nil)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
// GetAll retrieves all sites.
func (c Sites) GetAll(ctx context.Context, errs *fault.Bus) ([]models.Siteable, error) {
service, err := c.Service()

View File

@ -7,7 +7,6 @@ import (
"strings"
"github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
@ -122,12 +121,6 @@ func (ui *UserInfo) CanMakeDeltaQueries() bool {
// methods
// ---------------------------------------------------------------------------
const (
userSelectID = "id"
userSelectPrincipalName = "userPrincipalName"
userSelectDisplayName = "displayName"
)
// Filter out both guest users, and (for on-prem installations) non-synced users.
// The latter filter makes an assumption that no on-prem users are guests; this might
// require more fine-tuned controls in the future.
@ -145,13 +138,10 @@ var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType ne 'Guest'"
var t = true
func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration {
headers := abstractions.NewRequestHeaders()
headers.Add("ConsistencyLevel", "eventual")
return &users.UsersRequestBuilderGetRequestConfiguration{
Headers: headers,
Headers: newEventualConsistencyHeaders(),
QueryParameters: &users.UsersRequestBuilderGetQueryParameters{
Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName},
Select: idAnd(userPrincipalName, displayName),
Filter: fs,
Count: &t,
},
@ -261,7 +251,7 @@ func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
userInfo := newUserInfo()
requestParameters := users.ItemMailFoldersRequestBuilderGetQueryParameters{
Select: []string{"id"},
Select: idAnd(),
Top: ptr.To[int32](1), // if we get any folders, then we have access.
}
@ -396,90 +386,90 @@ func (c Users) getMailboxSettings(
additionalData := settings.GetAdditionalData()
mi.ArchiveFolder, err = str.FromMapToAny("archiveFolder", additionalData)
mi.ArchiveFolder, err = str.AnyValueToString("archiveFolder", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Timezone, err = str.FromMapToAny("timeZone", additionalData)
mi.Timezone, err = str.AnyValueToString("timeZone", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DateFormat, err = str.FromMapToAny("dateFormat", additionalData)
mi.DateFormat, err = str.AnyValueToString("dateFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.TimeFormat, err = str.FromMapToAny("timeFormat", additionalData)
mi.TimeFormat, err = str.AnyValueToString("timeFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Purpose, err = str.FromMapToAny("userPurpose", additionalData)
mi.Purpose, err = str.AnyValueToString("userPurpose", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DelegateMeetMsgDeliveryOpt, err = str.FromMapToAny("delegateMeetingMessageDeliveryOptions", additionalData)
mi.DelegateMeetMsgDeliveryOpt, err = str.AnyValueToString("delegateMeetingMessageDeliveryOptions", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode automatic replies settings
replySetting, err := tform.FromMapToAny[map[string]any]("automaticRepliesSetting", additionalData)
replySetting, err := tform.AnyValueToT[map[string]any]("automaticRepliesSetting", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.Status, err = str.FromMapToAny("status", replySetting)
mi.AutomaticRepliesSetting.Status, err = str.AnyValueToString("status", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalAudience, err = str.FromMapToAny("externalAudience", replySetting)
mi.AutomaticRepliesSetting.ExternalAudience, err = str.AnyValueToString("externalAudience", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.FromMapToAny("externalReplyMessage", replySetting)
mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.AnyValueToString("externalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.FromMapToAny("internalReplyMessage", replySetting)
mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.AnyValueToString("internalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode scheduledStartDateTime
startDateTime, err := tform.FromMapToAny[map[string]any]("scheduledStartDateTime", replySetting)
startDateTime, err := tform.AnyValueToT[map[string]any]("scheduledStartDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.FromMapToAny("dateTime", startDateTime)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.AnyValueToString("dateTime", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.FromMapToAny("timeZone", startDateTime)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.AnyValueToString("timeZone", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
endDateTime, err := tform.FromMapToAny[map[string]any]("scheduledEndDateTime", replySetting)
endDateTime, err := tform.AnyValueToT[map[string]any]("scheduledEndDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.FromMapToAny("dateTime", endDateTime)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.AnyValueToString("dateTime", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.FromMapToAny("timeZone", endDateTime)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.AnyValueToString("timeZone", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// Language decode
language, err := tform.FromMapToAny[map[string]any]("language", additionalData)
language, err := tform.AnyValueToT[map[string]any]("language", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.DisplayName, err = str.FromMapToAny("displayName", language)
mi.Language.DisplayName, err = str.AnyValueToString("displayName", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.Locale, err = str.FromMapToAny("locale", language)
mi.Language.Locale, err = str.AnyValueToString("locale", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// working hours
workingHours, err := tform.FromMapToAny[map[string]any]("workingHours", additionalData)
workingHours, err := tform.AnyValueToT[map[string]any]("workingHours", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.StartTime, err = str.FromMapToAny("startTime", workingHours)
mi.WorkingHours.StartTime, err = str.AnyValueToString("startTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.EndTime, err = str.FromMapToAny("endTime", workingHours)
mi.WorkingHours.EndTime, err = str.AnyValueToString("endTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
timeZone, err := tform.FromMapToAny[map[string]any]("timeZone", workingHours)
timeZone, err := tform.AnyValueToT[map[string]any]("timeZone", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.TimeZone.Name, err = str.FromMapToAny("name", timeZone)
mi.WorkingHours.TimeZone.Name, err = str.AnyValueToString("name", timeZone)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
days, err := tform.FromMapToAny[[]any]("daysOfWeek", workingHours)
days, err := tform.AnyValueToT[[]any]("daysOfWeek", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
for _, day := range days {
s, err := str.FromAny(day)
s, err := str.AnyToString(day)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.DaysOfWeek = append(mi.WorkingHours.DaysOfWeek, s)
}