larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

follow-up refinements to m365/api (#3461)

Browse Source 
some renaming, mostly refactoring the configuration (select and header declaration for each query) by combining the consts.go and query_param.go file into config.go, and making a more composable set of funcs and consts.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #1996

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-05-22 12:37:01 -06:00 committed by GitHub
parent 11752d869c
commit 3d170c5b66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 533 additions and 498 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/internal/common/str/str.go
View File

@ -20,15 +20,15 @@ func ParseBool(v string) bool {
return s return s
} }
func FromMapToAny(k string, m map[string]any) (string, error) { func AnyValueToString(k string, m map[string]any) (string, error) {
if len(m) == 0 { if len(m) == 0 {
return "", clues.New("missing entry").With("map_key", k) return "", clues.New("missing entry").With("map_key", k)
} }
return FromAny(m[k]) return AnyToString(m[k])
} }
func FromAny(a any) (string, error) { func AnyToString(a any) (string, error) {
if a == nil { if a == nil {
return "", clues.New("missing value") return "", clues.New("missing value")
} }

2
src/internal/common/tform/tform.go
View File

@ -6,7 +6,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
) )
func FromMapToAny[T any](k string, m map[string]any) (T, error) { func AnyValueToT[T any](k string, m map[string]any) (T, error) {
v, ok := m[k] v, ok := m[k]
if !ok { if !ok {
return *new(T), clues.New("entry not found") return *new(T), clues.New("entry not found")

51
src/internal/connector/graph/api/api.go
View File

@ -1,51 +0,0 @@
package api
import (
"strings"
"github.com/alcionai/corso/src/internal/common/ptr"
)
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
// IsNextLinkValid separate check to investigate whether error is
func IsNextLinkValid(next string) bool {
return !strings.Contains(next, `users//`)
}
func NextLink(pl PageLinker) string {
return ptr.Val(pl.GetOdataNextLink())
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
return NextLink(pl), ptr.Val(pl.GetOdataDeltaLink())
}
type Valuer[T any] interface {
GetValue() []T
}
type PageLinkValuer[T any] interface {
PageLinker
Valuer[T]
}
// EmptyDeltaLinker is used to convert PageLinker to DeltaPageLinker
type EmptyDeltaLinker[T any] struct {
PageLinkValuer[T]
}
func (EmptyDeltaLinker[T]) GetOdataDeltaLink() *string {
return ptr.To("")
}
func (e EmptyDeltaLinker[T]) GetValue() []T {
return e.PageLinkValuer.GetValue()
}

147
src/internal/connector/graph/api/api_test.go
View File

@ -1,147 +0,0 @@
package api_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester"
)
type mockNextLink struct {
nextLink *string
}
func (l mockNextLink) GetOdataNextLink() *string {
return l.nextLink
}
type mockDeltaNextLink struct {
mockNextLink
deltaLink *string
}
func (l mockDeltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
type APIUnitSuite struct {
tester.Suite
}
func TestAPIUnitSuite(t *testing.T) {
s := &APIUnitSuite{
Suite: tester.NewUnitSuite(t),
}
suite.Run(t, s)
}
func (suite *APIUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.Run(name, func() {
t := suite.T()
l := mockDeltaNextLink{
mockNextLink: mockNextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := api.NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}
// TestIsLinkValid check to verify is nextLink guard check for logging
// Related to: https://github.com/alcionai/corso/issues/2520
//
//nolint:lll
func (suite *APIUnitSuite) TestIsLinkValid() {
invalidString := `https://graph.microsoft.com/v1.0/users//mailFolders//messages/microsoft.graph.delta()?$select=id%2CisRead`
tests := []struct {
name string
inputString string
isValid assert.BoolAssertionFunc
}{
{
name: "Empty",
inputString: emptyLink,
isValid: assert.True,
},
{
name: "Invalid",
inputString: invalidString,
isValid: assert.False,
},
{
name: "Valid",
inputString: `https://graph.microsoft.com/v1.0/users/aPerson/mailFolders/AMessage/messages/microsoft.graph.delta()?$select=id%2CisRead`,
isValid: assert.True,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
got := api.IsNextLinkValid(test.inputString)
test.isValid(suite.T(), got)
})
}
}

16
src/internal/connector/graph/cache_container.go
View File

@ -188,3 +188,19 @@ func CheckIDAndName(c Container) error {
return nil return nil
} }
// CheckIDNameAndParentFolderID is a validator that ensures the ID
// and name are populated and not zero valued.
func CheckIDNameAndParentFolderID(c Container) error {
err := CheckIDAndName(c)
if err != nil {
return err
}
pfid := ptr.Val(c.GetParentFolderId())
if len(pfid) == 0 {
return clues.New("container missing parent folder id").With("container_id", ptr.Val(c.GetId()))
}
return nil
}

5
src/internal/connector/onedrive/collections_test.go
View File

@ -18,7 +18,6 @@ import (
"github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/prefixmatcher"
pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
@ -1205,7 +1204,7 @@ type mockItemPager struct {
getIdx int getIdx int
} }
func (p *mockItemPager) GetPage(context.Context) (gapi.DeltaPageLinker, error) { func (p *mockItemPager) GetPage(context.Context) (api.DeltaPageLinker, error) {
if len(p.toReturn) <= p.getIdx { if len(p.toReturn) <= p.getIdx {
return nil, assert.AnError return nil, assert.AnError
} }
@ -1222,7 +1221,7 @@ func (p *mockItemPager) GetPage(context.Context) (gapi.DeltaPageLinker, error) {
func (p *mockItemPager) SetNext(string) {} func (p *mockItemPager) SetNext(string) {}
func (p *mockItemPager) Reset() {} func (p *mockItemPager) Reset() {}
func (p *mockItemPager) ValuesIn(gapi.DeltaPageLinker) ([]models.DriveItemable, error) { func (p *mockItemPager) ValuesIn(api.DeltaPageLinker) ([]models.DriveItemable, error) {
idx := p.getIdx idx := p.getIdx
if idx > 0 { if idx > 0 {
// Return values lag by one since we increment in GetPage(). // Return values lag by one since we increment in GetPage().

31
src/internal/connector/onedrive/drive.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -90,39 +89,17 @@ type itemCollector func(
) error ) error
type itemPager interface { type itemPager interface {
GetPage(context.Context) (gapi.DeltaPageLinker, error) GetPage(context.Context) (api.DeltaPageLinker, error)
SetNext(nextLink string) SetNext(nextLink string)
Reset() Reset()
ValuesIn(gapi.DeltaPageLinker) ([]models.DriveItemable, error) ValuesIn(api.DeltaPageLinker) ([]models.DriveItemable, error)
} }
func defaultItemPager( func defaultItemPager(
servicer graph.Servicer, servicer graph.Servicer,
driveID, link string, driveID, link string,
) itemPager { ) itemPager {
return api.NewItemPager( return api.NewItemPager(servicer, driveID, link, api.DriveItemSelectDefault())
servicer,
driveID,
link,
[]string{
"content.downloadUrl",
"createdBy",
"createdDateTime",
"file",
"folder",
"id",
"lastModifiedDateTime",
"name",
"package",
"parentReference",
"root",
"sharepointIds",
"size",
"deleted",
"malware",
"shared",
},
)
} }
// collectItems will enumerate all items in the specified drive and hand them to the // collectItems will enumerate all items in the specified drive and hand them to the
@ -196,7 +173,7 @@ func collectItems(
return DeltaUpdate{}, nil, nil, err return DeltaUpdate{}, nil, nil, err
} }
nextLink, deltaLink := gapi.NextAndDeltaLink(page) nextLink, deltaLink := api.NextAndDeltaLink(page)
if len(deltaLink) > 0 { if len(deltaLink) > 0 {
newDeltaURL = deltaLink newDeltaURL = deltaLink

8
src/internal/connector/sharepoint/api/api.go
View File

@ -1,8 +0,0 @@
package api
type NameID struct {
Name string
ID string
}
const fetchChannelSize = 5

0
src/pkg/services/m365/api/beta_service.go → src/internal/connector/sharepoint/api/beta_service.go
View File

0
src/pkg/services/m365/api/beta_service_test.go → src/internal/connector/sharepoint/api/beta_service_test.go
View File

22
src/internal/connector/sharepoint/api/helper_test.go
View File

@ -1,22 +0,0 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret)
require.NoError(t, err, clues.ToCore(err))
return api.NewBetaService(adapter)
}

35
src/internal/connector/sharepoint/api/pages.go
View File

@ -7,8 +7,6 @@ import (
"sync" "sync"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
@ -19,21 +17,25 @@ import (
"github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/services/m365/api"
) )
type NameID struct {
Name string
ID string
}
// GetSitePages retrieves a collection of Pages related to the give Site. // GetSitePages retrieves a collection of Pages related to the give Site.
// Returns error if error experienced during the call // Returns error if error experienced during the call
func GetSitePages( func GetSitePages(
ctx context.Context, ctx context.Context,
serv *api.BetaService, serv *BetaService,
siteID string, siteID string,
pages []string, pages []string,
errs *fault.Bus, errs *fault.Bus,
) ([]betamodels.SitePageable, error) { ) ([]betamodels.SitePageable, error) {
var ( var (
col = make([]betamodels.SitePageable, 0) col = make([]betamodels.SitePageable, 0)
semaphoreCh = make(chan struct{}, fetchChannelSize) semaphoreCh = make(chan struct{}, 5)
opts = retrieveSitePageOptions() opts = retrieveSitePageOptions()
el = errs.Local() el = errs.Local()
wg sync.WaitGroup wg sync.WaitGroup
@ -82,25 +84,8 @@ func GetSitePages(
return col, el.Failure() return col, el.Failure()
} }
// GetSite returns a minimal Site with the SiteID and the WebURL
func GetSite(ctx context.Context, gs graph.Servicer, siteID string) (models.Siteable, error) {
// resp *sites.SiteItemRequestBuilderresp *sites.SiteItemRequestBuilde
options := &sites.SiteItemRequestBuilderGetRequestConfiguration{
QueryParameters: &sites.SiteItemRequestBuilderGetQueryParameters{
Select: []string{"webUrl"},
},
}
resp, err := gs.Client().Sites().BySiteId(siteID).Get(ctx, options)
if err != nil {
return nil, err
}
return resp, nil
}
// fetchPages utility function to return the tuple of item // fetchPages utility function to return the tuple of item
func FetchPages(ctx context.Context, bs *api.BetaService, siteID string) ([]NameID, error) { func FetchPages(ctx context.Context, bs *BetaService, siteID string) ([]NameID, error) {
var ( var (
builder = bs.Client().SitesById(siteID).Pages() builder = bs.Client().SitesById(siteID).Pages()
opts = fetchPageOptions() opts = fetchPageOptions()
@ -159,7 +144,7 @@ func fetchPageOptions() *betasites.ItemPagesRequestBuilderGetRequestConfiguratio
// https://github.com/alcionai/corso/issues/2707 // https://github.com/alcionai/corso/issues/2707
func DeleteSitePage( func DeleteSitePage(
ctx context.Context, ctx context.Context,
serv *api.BetaService, serv *BetaService,
siteID, pageID string, siteID, pageID string,
) error { ) error {
err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil) err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil)
@ -184,7 +169,7 @@ func retrieveSitePageOptions() *betasites.ItemPagesSitePageItemRequestBuilderGet
func RestoreSitePage( func RestoreSitePage(
ctx context.Context, ctx context.Context,
service *api.BetaService, service *BetaService,
itemData data.Stream, itemData data.Stream,
siteID, destName string, siteID, destName string,
) (details.ItemInfo, error) { ) (details.ItemInfo, error) {

14
src/internal/connector/sharepoint/api/pages_test.go
View File

@ -10,20 +10,30 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api" "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock" spMock "github.com/alcionai/corso/src/internal/connector/sharepoint/mock"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
) )
func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService {
adapter, err := graph.CreateAdapter(
credentials.AzureTenantID,
credentials.AzureClientID,
credentials.AzureClientSecret)
require.NoError(t, err, clues.ToCore(err))
return api.NewBetaService(adapter)
}
type SharePointPageSuite struct { type SharePointPageSuite struct {
tester.Suite tester.Suite
siteID string siteID string
creds account.M365Config creds account.M365Config
service *m365api.BetaService service *api.BetaService
} }
func (suite *SharePointPageSuite) SetupSuite() { func (suite *SharePointPageSuite) SetupSuite() {

8
src/internal/connector/sharepoint/collection.go
View File

@ -12,7 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api" betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
@ -21,7 +21,7 @@ import (
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
) )
type DataCategory int type DataCategory int
@ -56,7 +56,7 @@ type Collection struct {
category DataCategory category DataCategory
service graph.Servicer service graph.Servicer
ctrl control.Options ctrl control.Options
betaService *m365api.BetaService betaService *betaAPI.BetaService
statusUpdater support.StatusUpdater statusUpdater support.StatusUpdater
} }
@ -284,7 +284,7 @@ func (sc *Collection) retrievePages(
root := ptr.Val(parent.GetWebUrl()) root := ptr.Val(parent.GetWebUrl())
pages, err := api.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs) pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs)
if err != nil { if err != nil {
return metrics, err return metrics, err
} }

7
src/internal/connector/sharepoint/data_collections.go
View File

@ -9,7 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/prefixmatcher"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api" betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
@ -19,7 +19,6 @@ import (
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
) )
type statusUpdater interface { type statusUpdater interface {
@ -256,9 +255,9 @@ func collectPages(
return nil, clues.Wrap(err, "creating azure client adapter") return nil, clues.Wrap(err, "creating azure client adapter")
} }
betaService := m365api.NewBetaService(adpt) betaService := betaAPI.NewBetaService(adpt)
tuples, err := api.FetchPages(ctx, betaService, site.ID()) tuples, err := betaAPI.FetchPages(ctx, betaService, site.ID())
if err != nil { if err != nil {
return nil, err return nil, err
} }

7
src/internal/connector/sharepoint/restore.go
View File

@ -13,7 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive" "github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api" betaAPI "github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/diagnostics"
@ -23,7 +23,6 @@ import (
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
m365api "github.com/alcionai/corso/src/pkg/services/m365/api"
) )
//---------------------------------------------------------------------------- //----------------------------------------------------------------------------
@ -317,7 +316,7 @@ func RestorePageCollection(
var ( var (
el = errs.Local() el = errs.Local()
service = m365api.NewBetaService(adpt) service = betaAPI.NewBetaService(adpt)
items = dc.Items(ctx, errs) items = dc.Items(ctx, errs)
) )
@ -336,7 +335,7 @@ func RestorePageCollection(
} }
metrics.Objects++ metrics.Objects++
itemInfo, err := api.RestoreSitePage( itemInfo, err := betaAPI.RestoreSitePage(
ctx, ctx,
service, service,
itemData, itemData,

101
src/pkg/services/m365/api/config.go Normal file
View File

@ -0,0 +1,101 @@
package api
import (
"fmt"
"strings"
abstractions "github.com/microsoft/kiota-abstractions-go"
)
const (
maxNonDeltaPageSize = int32(999)
maxDeltaPageSize = int32(500)
)
// selectable values, case insensitive
// not comprehensive - just adding ones that might
// get easily misspelled.
// eg: we don't need a const for "id"
const (
parentFolderID = "parentFolderId"
displayName = "displayName"
userPrincipalName = "userPrincipalName"
)
// header keys
const (
headerKeyConsistencyLevel = "ConsistencyLevel"
headerKeyPrefer = "Prefer"
)
// header values
const (
idTypeImmutable = `IdType="ImmutableId"`
eventual = "eventual"
)
// ---------------------------------------------------------------------------
// not exported
// ---------------------------------------------------------------------------
func preferPageSize(size int32) string {
return fmt.Sprintf("odata.maxpagesize=%d", size)
}
func preferImmutableIDs(t bool) string {
if !t {
return ""
}
return idTypeImmutable
}
func newPreferHeaders(values ...string) *abstractions.RequestHeaders {
vs := []string{}
for _, v := range values {
if len(v) > 0 {
vs = append(vs, v)
}
}
headers := abstractions.NewRequestHeaders()
headers.Add(headerKeyPrefer, strings.Join(vs, ","))
return headers
}
func newEventualConsistencyHeaders() *abstractions.RequestHeaders {
headers := abstractions.NewRequestHeaders()
headers.Add(headerKeyConsistencyLevel, eventual)
return headers
}
// makes a slice with []string{"id", s...}
func idAnd(ss ...string) []string {
return append([]string{"id"}, ss...)
}
// ---------------------------------------------------------------------------
// exported
// ---------------------------------------------------------------------------
func DriveItemSelectDefault() []string {
return idAnd(
"content.downloadUrl",
"createdBy",
"createdDateTime",
"file",
"folder",
"lastModifiedDateTime",
"name",
"package",
"parentReference",
"root",
"sharepointIds",
"size",
"deleted",
"malware",
"shared")
}

3
src/pkg/services/m365/api/consts.go
View File

@ -1,3 +0,0 @@
package api
const maxPageSize = int32(999)

51
src/pkg/services/m365/api/contacts.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
@ -81,7 +80,7 @@ func (c Contacts) GetItem(
_ *fault.Bus, // no attachments to iterate over, so this goes unused _ *fault.Bus, // no attachments to iterate over, so this goes unused
) (serialization.Parsable, *details.ExchangeInfo, error) { ) (serialization.Parsable, *details.ExchangeInfo, error) {
options := &users.ItemContactsContactItemRequestBuilderGetRequestConfiguration{ options := &users.ItemContactsContactItemRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(false, immutableIDs), Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
} }
cont, err := c.Stable.Client().Users().ByUserId(user).Contacts().ByContactId(itemID).Get(ctx, options) cont, err := c.Stable.Client().Users().ByUserId(user).Contacts().ByContactId(itemID).Get(ctx, options)
@ -96,13 +95,18 @@ func (c Contacts) GetContainerByID(
ctx context.Context, ctx context.Context,
userID, dirID string, userID, dirID string,
) (graph.Container, error) { ) (graph.Container, error) {
queryParams := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{ config := &users.ItemContactFoldersContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemContactFoldersContactFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"}, Select: idAnd(displayName, parentFolderID),
}, },
} }
resp, err := c.Stable.Client().Users().ByUserId(userID).ContactFolders().ByContactFolderId(dirID).Get(ctx, queryParams) resp, err := c.Stable.Client().
Users().
ByUserId(userID).
ContactFolders().
ByContactFolderId(dirID).
Get(ctx, config)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
@ -126,9 +130,9 @@ func (c Contacts) EnumerateContainers(
return graph.Stack(ctx, err) return graph.Stack(ctx, err)
} }
queryParams := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{ config := &users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemContactFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"}, Select: idAnd(displayName, parentFolderID),
}, },
} }
@ -145,7 +149,7 @@ func (c Contacts) EnumerateContainers(
break break
} }
resp, err := builder.Get(ctx, queryParams) resp, err := builder.Get(ctx, config)
if err != nil { if err != nil {
return graph.Stack(ctx, err) return graph.Stack(ctx, err)
} }
@ -155,7 +159,7 @@ func (c Contacts) EnumerateContainers(
return el.Failure() return el.Failure()
} }
if err := graph.CheckIDAndName(fold); err != nil { if err := graph.CheckIDNameAndParentFolderID(fold); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation)) errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue continue
} }
@ -201,25 +205,30 @@ func NewContactPager(
user, directoryID string, user, directoryID string,
immutableIDs bool, immutableIDs bool,
) itemPager { ) itemPager {
queryParams := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{ config := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"}, Select: idAnd(parentFolderID),
}, },
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
builder := gs.Client().Users().ByUserId(user).ContactFolders().ByContactFolderId(directoryID).Contacts() builder := gs.Client().
Users().
ByUserId(user).
ContactFolders().
ByContactFolderId(directoryID).
Contacts()
return &contactPager{gs, builder, queryParams} return &contactPager{gs, builder, config}
} }
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *contactPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options) resp, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
return api.EmptyDeltaLinker[models.Contactable]{PageLinkValuer: resp}, nil return EmptyDeltaLinker[models.Contactable]{PageLinkValuer: resp}, nil
} }
func (p *contactPager) setNext(nextLink string) { func (p *contactPager) setNext(nextLink string) {
@ -229,7 +238,7 @@ func (p *contactPager) setNext(nextLink string) {
// non delta pagers don't need reset // non delta pagers don't need reset
func (p *contactPager) reset(context.Context) {} func (p *contactPager) reset(context.Context) {}
func (p *contactPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *contactPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl) return toValues[models.Contactable](pl)
} }
@ -266,9 +275,9 @@ func NewContactDeltaPager(
) itemPager { ) itemPager {
options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{ options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "parentFolderId"}, Select: idAnd(parentFolderID),
}, },
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder var builder *users.ItemContactFoldersItemContactsDeltaRequestBuilder
@ -281,7 +290,7 @@ func NewContactDeltaPager(
return &contactDeltaPager{gs, user, directoryID, builder, options} return &contactDeltaPager{gs, user, directoryID, builder, options}
} }
func (p *contactDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *contactDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options) resp, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
@ -298,7 +307,7 @@ func (p *contactDeltaPager) reset(ctx context.Context) {
p.builder = getContactDeltaBuilder(ctx, p.gs, p.user, p.directoryID, p.options) p.builder = getContactDeltaBuilder(ctx, p.gs, p.user, p.directoryID, p.options)
} }
func (p *contactDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *contactDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl) return toValues[models.Contactable](pl)
} }

39
src/pkg/services/m365/api/drive_pager.go
View File

@ -3,11 +3,9 @@ package api
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"time" "time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
"github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/sites" "github.com/microsoftgraph/msgraph-sdk-go/sites"
@ -15,7 +13,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts" onedrive "github.com/alcionai/corso/src/internal/connector/onedrive/consts"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
) )
@ -34,22 +31,20 @@ type driveItemPager struct {
func NewItemPager( func NewItemPager(
gs graph.Servicer, gs graph.Servicer,
driveID, link string, driveID, link string,
fields []string, selectFields []string,
) *driveItemPager { ) *driveItemPager {
headers := abstractions.NewRequestHeaders()
preferHeaderItems := []string{ preferHeaderItems := []string{
"deltashowremovedasdeleted", "deltashowremovedasdeleted",
"deltatraversepermissiongaps", "deltatraversepermissiongaps",
"deltashowsharingchanges", "deltashowsharingchanges",
"hierarchicalsharing", "hierarchicalsharing",
} }
headers.Add("Prefer", strings.Join(preferHeaderItems, ","))
requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{ requestConfig := &drives.ItemItemsItemDeltaRequestBuilderGetRequestConfiguration{
Headers: headers, Headers: newPreferHeaders(preferHeaderItems...),
QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{ QueryParameters: &drives.ItemItemsItemDeltaRequestBuilderGetQueryParameters{
Top: ptr.To(maxPageSize), Top: ptr.To(maxDeltaPageSize),
Select: fields, Select: selectFields,
}, },
} }
@ -70,9 +65,9 @@ func NewItemPager(
return res return res
} }
func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *driveItemPager) GetPage(ctx context.Context) (DeltaPageLinker, error) {
var ( var (
resp api.DeltaPageLinker resp DeltaPageLinker
err error err error
) )
@ -97,7 +92,7 @@ func (p *driveItemPager) Reset() {
Delta() Delta()
} }
func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) { func (p *driveItemPager) ValuesIn(l DeltaPageLinker) ([]models.DriveItemable, error) {
return getValues[models.DriveItemable](l) return getValues[models.DriveItemable](l)
} }
@ -139,9 +134,9 @@ type nopUserDrivePageLinker struct {
func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil } func (nl nopUserDrivePageLinker) GetOdataNextLink() *string { return nil }
func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { func (p *userDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
var ( var (
resp api.PageLinker resp PageLinker
err error err error
) )
@ -167,7 +162,7 @@ func (p *userDrivePager) SetNext(link string) {
p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) p.builder = users.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
} }
func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { func (p *userDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
nl, ok := l.(*nopUserDrivePageLinker) nl, ok := l.(*nopUserDrivePageLinker)
if !ok || nl == nil { if !ok || nl == nil {
return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l)) return nil, clues.New(fmt.Sprintf("improper page linker struct for user drives: %T", l))
@ -216,9 +211,9 @@ func NewSiteDrivePager(
return res return res
} }
func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { func (p *siteDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
var ( var (
resp api.PageLinker resp PageLinker
err error err error
) )
@ -234,7 +229,7 @@ func (p *siteDrivePager) SetNext(link string) {
p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) p.builder = sites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
} }
func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { func (p *siteDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
return getValues[models.Driveable](l) return getValues[models.Driveable](l)
} }
@ -244,9 +239,9 @@ func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error)
// DrivePager pages through different types of drive owners // DrivePager pages through different types of drive owners
type DrivePager interface { type DrivePager interface {
GetPage(context.Context) (api.PageLinker, error) GetPage(context.Context) (PageLinker, error)
SetNext(nextLink string) SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error) ValuesIn(PageLinker) ([]models.Driveable, error)
} }
// GetAllDrives fetches all drives for the given pager // GetAllDrives fetches all drives for the given pager
@ -266,7 +261,7 @@ func GetAllDrives(
for { for {
var ( var (
err error err error
page api.PageLinker page PageLinker
) )
// Retry Loop for Drive retrieval. Request can timeout // Retry Loop for Drive retrieval. Request can timeout
@ -315,7 +310,7 @@ func GetAllDrives(
// Helpers // Helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func getValues[T any](l api.PageLinker) ([]T, error) { func getValues[T any](l PageLinker) ([]T, error) {
page, ok := l.(interface{ GetValue() []T }) page, ok := l.(interface{ GetValue() []T })
if !ok { if !ok {
return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l)) return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l))

70
src/pkg/services/m365/api/events.go
View File

@ -14,7 +14,6 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -84,13 +83,18 @@ func (c Events) GetContainerByID(
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
queryParams := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ config := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{
Select: []string{"id", "name", "owner"}, Select: idAnd("name", "owner"),
}, },
} }
cal, err := service.Client().Users().ByUserId(userID).Calendars().ByCalendarId(containerID).Get(ctx, queryParams) cal, err := service.Client().
Users().
ByUserId(userID).
Calendars().
ByCalendarId(containerID).
Get(ctx, config)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err).WithClues(ctx) return nil, graph.Stack(ctx, err).WithClues(ctx)
} }
@ -145,25 +149,29 @@ func (c Events) GetItem(
errs *fault.Bus, errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error) { ) (serialization.Parsable, *details.ExchangeInfo, error) {
var ( var (
err error err error
event models.Eventable event models.Eventable
header = buildPreferHeaders(false, immutableIDs) config = &users.ItemEventsEventItemRequestBuilderGetRequestConfiguration{
itemOpts = &users.ItemEventsEventItemRequestBuilderGetRequestConfiguration{ Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
Headers: header,
} }
) )
event, err = c.Stable.Client().Users().ByUserId(user).Events().ByEventId(itemID).Get(ctx, itemOpts) event, err = c.Stable.Client().
Users().
ByUserId(user).
Events().
ByEventId(itemID).
Get(ctx, config)
if err != nil { if err != nil {
return nil, nil, graph.Stack(ctx, err) return nil, nil, graph.Stack(ctx, err)
} }
if ptr.Val(event.GetHasAttachments()) || HasAttachments(event.GetBody()) { if ptr.Val(event.GetHasAttachments()) || HasAttachments(event.GetBody()) {
options := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{ config := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"}, Expand: []string{"microsoft.graph.itemattachment/item"},
}, },
Headers: header, Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
attached, err := c.LargeItem. attached, err := c.LargeItem.
@ -173,7 +181,7 @@ func (c Events) GetItem(
Events(). Events().
ByEventId(itemID). ByEventId(itemID).
Attachments(). Attachments().
Get(ctx, options) Get(ctx, config)
if err != nil { if err != nil {
return nil, nil, graph.Wrap(ctx, err, "event attachment download") return nil, nil, graph.Wrap(ctx, err, "event attachment download")
} }
@ -200,21 +208,25 @@ func (c Events) EnumerateContainers(
return graph.Stack(ctx, err) return graph.Stack(ctx, err)
} }
queryParams := &users.ItemCalendarsRequestBuilderGetRequestConfiguration{ var (
QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{ el = errs.Local()
Select: []string{"id", "name"}, config = &users.ItemCalendarsRequestBuilderGetRequestConfiguration{
}, QueryParameters: &users.ItemCalendarsRequestBuilderGetQueryParameters{
} Select: idAnd("name"),
},
el := errs.Local() }
builder := service.Client().Users().ByUserId(userID).Calendars() builder = service.Client().
Users().
ByUserId(userID).
Calendars()
)
for { for {
if el.Failure() != nil { if el.Failure() != nil {
break break
} }
resp, err := builder.Get(ctx, queryParams) resp, err := builder.Get(ctx, config)
if err != nil { if err != nil {
return graph.Stack(ctx, err) return graph.Stack(ctx, err)
} }
@ -279,7 +291,7 @@ func NewEventPager(
immutableIDs bool, immutableIDs bool,
) (itemPager, error) { ) (itemPager, error) {
options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{ options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
builder := gs.Client().Users().ByUserId(user).Calendars().ByCalendarId(calendarID).Events() builder := gs.Client().Users().ByUserId(user).Calendars().ByCalendarId(calendarID).Events()
@ -287,13 +299,13 @@ func NewEventPager(
return &eventPager{gs, builder, options}, nil return &eventPager{gs, builder, options}, nil
} }
func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *eventPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options) resp, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
return api.EmptyDeltaLinker[models.Eventable]{PageLinkValuer: resp}, nil return EmptyDeltaLinker[models.Eventable]{PageLinkValuer: resp}, nil
} }
func (p *eventPager) setNext(nextLink string) { func (p *eventPager) setNext(nextLink string) {
@ -303,7 +315,7 @@ func (p *eventPager) setNext(nextLink string) {
// non delta pagers don't need reset // non delta pagers don't need reset
func (p *eventPager) reset(context.Context) {} func (p *eventPager) reset(context.Context) {}
func (p *eventPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *eventPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl) return toValues[models.Eventable](pl)
} }
@ -328,7 +340,7 @@ func NewEventDeltaPager(
immutableIDs bool, immutableIDs bool,
) (itemPager, error) { ) (itemPager, error) {
options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
var builder *users.ItemCalendarsItemEventsDeltaRequestBuilder var builder *users.ItemCalendarsItemEventsDeltaRequestBuilder
@ -363,7 +375,7 @@ func getEventDeltaBuilder(
return builder return builder
} }
func (p *eventDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *eventDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options) resp, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
@ -380,7 +392,7 @@ func (p *eventDeltaPager) reset(ctx context.Context) {
p.builder = getEventDeltaBuilder(ctx, p.gs, p.user, p.calendarID, p.options) p.builder = getEventDeltaBuilder(ctx, p.gs, p.user, p.calendarID, p.options)
} }
func (p *eventDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *eventDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl) return toValues[models.Eventable](pl)
} }

56
src/pkg/services/m365/api/item_pager.go
View File

@ -3,22 +3,70 @@ package api
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
) )
// ---------------------------------------------------------------------------
// common interfaces and funcs
// ---------------------------------------------------------------------------
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
// IsNextLinkValid separate check to investigate whether error is
func IsNextLinkValid(next string) bool {
return !strings.Contains(next, `users//`)
}
func NextLink(pl PageLinker) string {
return ptr.Val(pl.GetOdataNextLink())
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
return NextLink(pl), ptr.Val(pl.GetOdataDeltaLink())
}
type Valuer[T any] interface {
GetValue() []T
}
type PageLinkValuer[T any] interface {
PageLinker
Valuer[T]
}
// EmptyDeltaLinker is used to convert PageLinker to DeltaPageLinker
type EmptyDeltaLinker[T any] struct {
PageLinkValuer[T]
}
func (EmptyDeltaLinker[T]) GetOdataDeltaLink() *string {
return ptr.To("")
}
func (e EmptyDeltaLinker[T]) GetValue() []T {
return e.PageLinkValuer.GetValue()
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// generic handler for paging item ids in a container // generic handler for paging item ids in a container
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type itemPager interface { type itemPager interface {
// getPage get a page with the specified options from graph // getPage get a page with the specified options from graph
getPage(context.Context) (api.DeltaPageLinker, error) getPage(context.Context) (DeltaPageLinker, error)
// setNext is used to pass in the next url got from graph // setNext is used to pass in the next url got from graph
setNext(string) setNext(string)
// reset is used to clear delta url in delta pagers. When // reset is used to clear delta url in delta pagers. When
@ -26,7 +74,7 @@ type itemPager interface {
// currently have and start a new delta query without the token. // currently have and start a new delta query without the token.
reset(context.Context) reset(context.Context)
// valuesIn gets us the values in a page // valuesIn gets us the values in a page
valuesIn(api.PageLinker) ([]getIDAndAddtler, error) valuesIn(PageLinker) ([]getIDAndAddtler, error)
} }
type getIDAndAddtler interface { type getIDAndAddtler interface {
@ -158,7 +206,7 @@ func getItemsAddedAndRemovedFromContainer(
} }
} }
nextLink, deltaLink := api.NextAndDeltaLink(resp) nextLink, deltaLink := NextAndDeltaLink(resp)
// the deltaLink is kind of like a cursor for overall data state. // the deltaLink is kind of like a cursor for overall data state.
// once we run through pages of nextLinks, the last query will // once we run through pages of nextLinks, the last query will

146
src/pkg/services/m365/api/item_pager_test.go
View File

@ -2,6 +2,7 @@ package api
import ( import (
"context" "context"
"strings"
"testing" "testing"
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
@ -11,10 +12,30 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
) )
// ---------------------------------------------------------------------------
// mock impls & stubs
// ---------------------------------------------------------------------------
type nextLink struct {
nextLink *string
}
func (l nextLink) GetOdataNextLink() *string {
return l.nextLink
}
type deltaNextLink struct {
nextLink
deltaLink *string
}
func (l deltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testPagerValue struct { type testPagerValue struct {
id string id string
removed bool removed bool
@ -51,7 +72,7 @@ type testPager struct {
needsReset bool needsReset bool
} }
func (p *testPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *testPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
if p.errorCode != "" { if p.errorCode != "" {
ierr := odataerrors.NewMainError() ierr := odataerrors.NewMainError()
ierr.SetCode(&p.errorCode) ierr.SetCode(&p.errorCode)
@ -74,7 +95,7 @@ func (p *testPager) reset(context.Context) {
p.errorCode = "" p.errorCode = ""
} }
func (p *testPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *testPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
items := []getIDAndAddtler{} items := []getIDAndAddtler{}
for _, id := range p.added { for _, id := range p.added {
@ -88,15 +109,19 @@ func (p *testPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) {
return items, nil return items, nil
} }
type SharedAPIUnitSuite struct { // ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
type ItemPagerUnitSuite struct {
tester.Suite tester.Suite
} }
func TestSharedAPIUnitSuite(t *testing.T) { func TestItemPagerUnitSuite(t *testing.T) {
suite.Run(t, &SharedAPIUnitSuite{Suite: tester.NewUnitSuite(t)}) suite.Run(t, &ItemPagerUnitSuite{Suite: tester.NewUnitSuite(t)})
} }
func (suite *SharedAPIUnitSuite) TestGetAddedAndRemovedItemIDs() { func (suite *ItemPagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
tests := []struct { tests := []struct {
name string name string
pagerGetter func(context.Context, graph.Servicer, string, string, bool) (itemPager, error) pagerGetter func(context.Context, graph.Servicer, string, string, bool) (itemPager, error)
@ -260,3 +285,110 @@ func (suite *SharedAPIUnitSuite) TestGetAddedAndRemovedItemIDs() {
}) })
} }
} }
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
func (suite *ItemPagerUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.Run(name, func() {
t := suite.T()
l := deltaNextLink{
nextLink: nextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}
// TestIsLinkValid check to verify is nextLink guard check for logging
// Related to: https://github.com/alcionai/corso/issues/2520
//
//nolint:lll
func (suite *ItemPagerUnitSuite) TestIsLinkValid() {
invalidString := `https://graph.microsoft.com/v1.0/users//mailFolders//messages/microsoft.graph.delta()?$select=id%2CisRead`
tests := []struct {
name string
inputString string
isValid assert.BoolAssertionFunc
}{
{
name: "Empty",
inputString: emptyLink,
isValid: assert.True,
},
{
name: "Invalid",
inputString: invalidString,
isValid: assert.False,
},
{
name: "Valid",
inputString: `https://graph.microsoft.com/v1.0/users/aPerson/mailFolders/AMessage/messages/microsoft.graph.delta()?$select=id%2CisRead`,
isValid: assert.True,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
got := IsNextLinkValid(test.inputString)
test.isValid(suite.T(), got)
})
}
}

88
src/pkg/services/m365/api/mail.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -123,9 +122,9 @@ func (c Mail) GetContainerByID(
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
queryParams := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{ config := &users.ItemMailFoldersMailFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemMailFoldersMailFolderItemRequestBuilderGetQueryParameters{
Select: []string{"id", "displayName", "parentFolderId"}, Select: idAnd(displayName, parentFolderID),
}, },
} }
@ -134,7 +133,7 @@ func (c Mail) GetContainerByID(
ByUserId(userID). ByUserId(userID).
MailFolders(). MailFolders().
ByMailFolderId(dirID). ByMailFolderId(dirID).
Get(ctx, queryParams) Get(ctx, config)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
@ -153,14 +152,17 @@ func (c Mail) GetItem(
var ( var (
size int64 size int64
mailBody models.ItemBodyable mailBody models.ItemBodyable
config = &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
}
) )
// Will need adjusted if attachments start allowing paging.
headers := buildPreferHeaders(false, immutableIDs)
itemOpts := &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{
Headers: headers,
}
mail, err := c.Stable.Client().Users().ByUserId(user).Messages().ByMessageId(itemID).Get(ctx, itemOpts) mail, err := c.Stable.Client().
Users().
ByUserId(user).
Messages().
ByMessageId(itemID).
Get(ctx, config)
if err != nil { if err != nil {
return nil, nil, graph.Stack(ctx, err) return nil, nil, graph.Stack(ctx, err)
} }
@ -177,11 +179,11 @@ func (c Mail) GetItem(
return mail, MailInfo(mail, size), nil return mail, MailInfo(mail, size), nil
} }
options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{ attachConfig := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"}, Expand: []string{"microsoft.graph.itemattachment/item"},
}, },
Headers: headers, Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
attached, err := c.LargeItem. attached, err := c.LargeItem.
@ -191,7 +193,7 @@ func (c Mail) GetItem(
Messages(). Messages().
ByMessageId(itemID). ByMessageId(itemID).
Attachments(). Attachments().
Get(ctx, options) Get(ctx, attachConfig)
if err == nil { if err == nil {
for _, a := range attached.GetValue() { for _, a := range attached.GetValue() {
attachSize := ptr.Val(a.GetSize()) attachSize := ptr.Val(a.GetSize())
@ -215,7 +217,7 @@ func (c Mail) GetItem(
logger.CtxErr(ctx, err).Info("fetching all attachments by id") logger.CtxErr(ctx, err).Info("fetching all attachments by id")
// Getting size just to log in case of error // Getting size just to log in case of error
options.QueryParameters.Select = []string{"id", "size"} attachConfig.QueryParameters.Select = []string{"id", "size"}
attachments, err := c.LargeItem. attachments, err := c.LargeItem.
Client(). Client().
@ -224,7 +226,7 @@ func (c Mail) GetItem(
Messages(). Messages().
ByMessageId(itemID). ByMessageId(itemID).
Attachments(). Attachments().
Get(ctx, options) Get(ctx, attachConfig)
if err != nil { if err != nil {
return nil, nil, graph.Wrap(ctx, err, "getting mail attachment ids") return nil, nil, graph.Wrap(ctx, err, "getting mail attachment ids")
} }
@ -232,11 +234,11 @@ func (c Mail) GetItem(
atts := []models.Attachmentable{} atts := []models.Attachmentable{}
for _, a := range attachments.GetValue() { for _, a := range attachments.GetValue() {
options := &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetRequestConfiguration{ attachConfig := &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"}, Expand: []string{"microsoft.graph.itemattachment/item"},
}, },
Headers: headers, Headers: newPreferHeaders(preferImmutableIDs(immutableIDs)),
} }
att, err := c.Stable. att, err := c.Stable.
@ -247,11 +249,10 @@ func (c Mail) GetItem(
ByMessageId(itemID). ByMessageId(itemID).
Attachments(). Attachments().
ByAttachmentId(ptr.Val(a.GetId())). ByAttachmentId(ptr.Val(a.GetId())).
Get(ctx, options) Get(ctx, attachConfig)
if err != nil { if err != nil {
return nil, nil, return nil, nil, graph.Wrap(ctx, err, "getting mail attachment").
graph.Wrap(ctx, err, "getting mail attachment"). With("attachment_id", ptr.Val(a.GetId()), "attachment_size", ptr.Val(a.GetSize()))
With("attachment_id", ptr.Val(a.GetId()), "attachment_size", ptr.Val(a.GetSize()))
} }
atts = append(atts, att) atts = append(atts, att)
@ -277,7 +278,7 @@ func NewMailFolderPager(service graph.Servicer, user string) mailFolderPager {
return mailFolderPager{service, builder} return mailFolderPager{service, builder}
} }
func (p *mailFolderPager) getPage(ctx context.Context) (api.PageLinker, error) { func (p *mailFolderPager) getPage(ctx context.Context) (PageLinker, error) {
page, err := p.builder.Get(ctx, nil) page, err := p.builder.Get(ctx, nil)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
@ -290,7 +291,7 @@ func (p *mailFolderPager) setNext(nextLink string) {
p.builder = users.NewItemMailFoldersRequestBuilder(nextLink, p.service.Adapter()) p.builder = users.NewItemMailFoldersRequestBuilder(nextLink, p.service.Adapter())
} }
func (p *mailFolderPager) valuesIn(pl api.PageLinker) ([]models.MailFolderable, error) { func (p *mailFolderPager) valuesIn(pl PageLinker) ([]models.MailFolderable, error) {
// Ideally this should be `users.ItemMailFoldersResponseable`, but // Ideally this should be `users.ItemMailFoldersResponseable`, but
// that is not a thing as stable returns different result // that is not a thing as stable returns different result
page, ok := pl.(models.MailFolderCollectionResponseable) page, ok := pl.(models.MailFolderCollectionResponseable)
@ -336,17 +337,22 @@ func (c Mail) EnumerateContainers(
return graph.Stack(ctx, err) return graph.Stack(ctx, err)
} }
for _, v := range resp { for _, fold := range resp {
if el.Failure() != nil { if el.Failure() != nil {
break break
} }
if err := graph.CheckIDNameAndParentFolderID(fold); err != nil {
errs.AddRecoverable(graph.Stack(ctx, err).Label(fault.LabelForceNoBackupCreation))
continue
}
fctx := clues.Add( fctx := clues.Add(
ctx, ctx,
"container_id", ptr.Val(v.GetId()), "container_id", ptr.Val(fold.GetId()),
"container_name", ptr.Val(v.GetDisplayName())) "container_name", ptr.Val(fold.GetDisplayName()))
temp := graph.NewCacheFolder(v, nil, nil) temp := graph.NewCacheFolder(fold, nil, nil)
if err := fn(&temp); err != nil { if err := fn(&temp); err != nil {
errs.AddRecoverable(graph.Stack(fctx, err).Label(fault.LabelForceNoBackupCreation)) errs.AddRecoverable(graph.Stack(fctx, err).Label(fault.LabelForceNoBackupCreation))
continue continue
@ -382,11 +388,11 @@ func NewMailPager(
user, directoryID string, user, directoryID string,
immutableIDs bool, immutableIDs bool,
) itemPager { ) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{ config := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"}, Select: idAnd("isRead"),
}, },
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
builder := gs.Client(). builder := gs.Client().
@ -396,16 +402,16 @@ func NewMailPager(
ByMailFolderId(directoryID). ByMailFolderId(directoryID).
Messages() Messages()
return &mailPager{gs, builder, queryParams} return &mailPager{gs, builder, config}
} }
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *mailPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
page, err := p.builder.Get(ctx, p.options) page, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
} }
return api.EmptyDeltaLinker[models.Messageable]{PageLinkValuer: page}, nil return EmptyDeltaLinker[models.Messageable]{PageLinkValuer: page}, nil
} }
func (p *mailPager) setNext(nextLink string) { func (p *mailPager) setNext(nextLink string) {
@ -415,7 +421,7 @@ func (p *mailPager) setNext(nextLink string) {
// non delta pagers don't have reset // non delta pagers don't have reset
func (p *mailPager) reset(context.Context) {} func (p *mailPager) reset(context.Context) {}
func (p *mailPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *mailPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl) return toValues[models.Messageable](pl)
} }
@ -457,11 +463,11 @@ func NewMailDeltaPager(
user, directoryID, oldDelta string, user, directoryID, oldDelta string,
immutableIDs bool, immutableIDs bool,
) itemPager { ) itemPager {
queryParams := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{ config := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{ QueryParameters: &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetQueryParameters{
Select: []string{"id", "isRead"}, Select: idAnd("isRead"),
}, },
Headers: buildPreferHeaders(true, immutableIDs), Headers: newPreferHeaders(preferPageSize(maxDeltaPageSize), preferImmutableIDs(immutableIDs)),
} }
var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder var builder *users.ItemMailFoldersItemMessagesDeltaRequestBuilder
@ -469,13 +475,13 @@ func NewMailDeltaPager(
if len(oldDelta) > 0 { if len(oldDelta) > 0 {
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter()) builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())
} else { } else {
builder = getMailDeltaBuilder(ctx, gs, user, directoryID, queryParams) builder = getMailDeltaBuilder(ctx, gs, user, directoryID, config)
} }
return &mailDeltaPager{gs, user, directoryID, builder, queryParams} return &mailDeltaPager{gs, user, directoryID, builder, config}
} }
func (p *mailDeltaPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { func (p *mailDeltaPager) getPage(ctx context.Context) (DeltaPageLinker, error) {
page, err := p.builder.Get(ctx, p.options) page, err := p.builder.Get(ctx, p.options)
if err != nil { if err != nil {
return nil, graph.Stack(ctx, err) return nil, graph.Stack(ctx, err)
@ -498,7 +504,7 @@ func (p *mailDeltaPager) reset(ctx context.Context) {
Delta() Delta()
} }
func (p *mailDeltaPager) valuesIn(pl api.PageLinker) ([]getIDAndAddtler, error) { func (p *mailDeltaPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl) return toValues[models.Messageable](pl)
} }

8
src/pkg/services/m365/api/mock/drive.go → src/pkg/services/m365/api/mock/drive_pager.go
View File

@ -6,14 +6,14 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
) )
type PageLinker struct { type PageLink struct {
Link *string Link *string
} }
func (pl *PageLinker) GetOdataNextLink() *string { func (pl *PageLink) GetOdataNextLink() *string {
return pl.Link return pl.Link
} }
@ -36,7 +36,7 @@ func (p *DrivePager) GetPage(context.Context) (api.PageLinker, error) {
idx := p.GetIdx idx := p.GetIdx
p.GetIdx++ p.GetIdx++
return &PageLinker{p.ToReturn[idx].NextLink}, p.ToReturn[idx].Err return &PageLink{p.ToReturn[idx].NextLink}, p.ToReturn[idx].Err
} }
func (p *DrivePager) SetNext(string) {} func (p *DrivePager) SetNext(string) {}

26
src/pkg/services/m365/api/query_params.go
View File

@ -1,26 +0,0 @@
package api
import (
"fmt"
"strings"
abstractions "github.com/microsoft/kiota-abstractions-go"
)
// buildPreferHeaders returns the headers we add to item delta page requests.
func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders {
var allHeaders []string
if pageSize {
allHeaders = append(allHeaders, fmt.Sprintf("odata.maxpagesize=%d", maxPageSize))
}
if immutableID {
allHeaders = append(allHeaders, `IdType="ImmutableId"`)
}
headers := abstractions.NewRequestHeaders()
headers.Add("Prefer", strings.Join(allHeaders, ","))
return headers
}

14
src/pkg/services/m365/api/sites.go
View File

@ -35,6 +35,20 @@ type Sites struct {
// methods // methods
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// GetSite returns a minimal Site with the SiteID and the WebURL
// TODO: delete in favor of sites.GetByID()
func GetSite(ctx context.Context, gs graph.Servicer, siteID string) (models.Siteable, error) {
resp, err := gs.Client().
Sites().
BySiteId(siteID).
Get(ctx, nil)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
// GetAll retrieves all sites. // GetAll retrieves all sites.
func (c Sites) GetAll(ctx context.Context, errs *fault.Bus) ([]models.Siteable, error) { func (c Sites) GetAll(ctx context.Context, errs *fault.Bus) ([]models.Siteable, error) {
service, err := c.Service() service, err := c.Service()

70
src/pkg/services/m365/api/users.go
View File

@ -7,7 +7,6 @@ import (
"strings" "strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
abstractions "github.com/microsoft/kiota-abstractions-go"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users" "github.com/microsoftgraph/msgraph-sdk-go/users"
@ -122,12 +121,6 @@ func (ui *UserInfo) CanMakeDeltaQueries() bool {
// methods // methods
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const (
userSelectID = "id"
userSelectPrincipalName = "userPrincipalName"
userSelectDisplayName = "displayName"
)
// Filter out both guest users, and (for on-prem installations) non-synced users. // Filter out both guest users, and (for on-prem installations) non-synced users.
// The latter filter makes an assumption that no on-prem users are guests; this might // The latter filter makes an assumption that no on-prem users are guests; this might
// require more fine-tuned controls in the future. // require more fine-tuned controls in the future.
@ -145,13 +138,10 @@ var userFilterNoGuests = "onPremisesSyncEnabled eq true OR userType ne 'Guest'"
var t = true var t = true
func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration { func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration {
headers := abstractions.NewRequestHeaders()
headers.Add("ConsistencyLevel", "eventual")
return &users.UsersRequestBuilderGetRequestConfiguration{ return &users.UsersRequestBuilderGetRequestConfiguration{
Headers: headers, Headers: newEventualConsistencyHeaders(),
QueryParameters: &users.UsersRequestBuilderGetQueryParameters{ QueryParameters: &users.UsersRequestBuilderGetQueryParameters{
Select: []string{userSelectID, userSelectPrincipalName, userSelectDisplayName}, Select: idAnd(userPrincipalName, displayName),
Filter: fs, Filter: fs,
Count: &t, Count: &t,
}, },
@ -261,7 +251,7 @@ func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
userInfo := newUserInfo() userInfo := newUserInfo()
requestParameters := users.ItemMailFoldersRequestBuilderGetQueryParameters{ requestParameters := users.ItemMailFoldersRequestBuilderGetQueryParameters{
Select: []string{"id"}, Select: idAnd(),
Top: ptr.To[int32](1), // if we get any folders, then we have access. Top: ptr.To[int32](1), // if we get any folders, then we have access.
} }
@ -396,90 +386,90 @@ func (c Users) getMailboxSettings(
additionalData := settings.GetAdditionalData() additionalData := settings.GetAdditionalData()
mi.ArchiveFolder, err = str.FromMapToAny("archiveFolder", additionalData) mi.ArchiveFolder, err = str.AnyValueToString("archiveFolder", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Timezone, err = str.FromMapToAny("timeZone", additionalData) mi.Timezone, err = str.AnyValueToString("timeZone", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DateFormat, err = str.FromMapToAny("dateFormat", additionalData) mi.DateFormat, err = str.AnyValueToString("dateFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.TimeFormat, err = str.FromMapToAny("timeFormat", additionalData) mi.TimeFormat, err = str.AnyValueToString("timeFormat", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Purpose, err = str.FromMapToAny("userPurpose", additionalData) mi.Purpose, err = str.AnyValueToString("userPurpose", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.DelegateMeetMsgDeliveryOpt, err = str.FromMapToAny("delegateMeetingMessageDeliveryOptions", additionalData) mi.DelegateMeetMsgDeliveryOpt, err = str.AnyValueToString("delegateMeetingMessageDeliveryOptions", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode automatic replies settings // decode automatic replies settings
replySetting, err := tform.FromMapToAny[map[string]any]("automaticRepliesSetting", additionalData) replySetting, err := tform.AnyValueToT[map[string]any]("automaticRepliesSetting", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.Status, err = str.FromMapToAny("status", replySetting) mi.AutomaticRepliesSetting.Status, err = str.AnyValueToString("status", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalAudience, err = str.FromMapToAny("externalAudience", replySetting) mi.AutomaticRepliesSetting.ExternalAudience, err = str.AnyValueToString("externalAudience", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.FromMapToAny("externalReplyMessage", replySetting) mi.AutomaticRepliesSetting.ExternalReplyMessage, err = str.AnyValueToString("externalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.FromMapToAny("internalReplyMessage", replySetting) mi.AutomaticRepliesSetting.InternalReplyMessage, err = str.AnyValueToString("internalReplyMessage", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// decode scheduledStartDateTime // decode scheduledStartDateTime
startDateTime, err := tform.FromMapToAny[map[string]any]("scheduledStartDateTime", replySetting) startDateTime, err := tform.AnyValueToT[map[string]any]("scheduledStartDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.FromMapToAny("dateTime", startDateTime) mi.AutomaticRepliesSetting.ScheduledStartDateTime.DateTime, err = str.AnyValueToString("dateTime", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.FromMapToAny("timeZone", startDateTime) mi.AutomaticRepliesSetting.ScheduledStartDateTime.Timezone, err = str.AnyValueToString("timeZone", startDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
endDateTime, err := tform.FromMapToAny[map[string]any]("scheduledEndDateTime", replySetting) endDateTime, err := tform.AnyValueToT[map[string]any]("scheduledEndDateTime", replySetting)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.FromMapToAny("dateTime", endDateTime) mi.AutomaticRepliesSetting.ScheduledEndDateTime.DateTime, err = str.AnyValueToString("dateTime", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.FromMapToAny("timeZone", endDateTime) mi.AutomaticRepliesSetting.ScheduledEndDateTime.Timezone, err = str.AnyValueToString("timeZone", endDateTime)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// Language decode // Language decode
language, err := tform.FromMapToAny[map[string]any]("language", additionalData) language, err := tform.AnyValueToT[map[string]any]("language", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.DisplayName, err = str.FromMapToAny("displayName", language) mi.Language.DisplayName, err = str.AnyValueToString("displayName", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.Language.Locale, err = str.FromMapToAny("locale", language) mi.Language.Locale, err = str.AnyValueToString("locale", language)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
// working hours // working hours
workingHours, err := tform.FromMapToAny[map[string]any]("workingHours", additionalData) workingHours, err := tform.AnyValueToT[map[string]any]("workingHours", additionalData)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.StartTime, err = str.FromMapToAny("startTime", workingHours) mi.WorkingHours.StartTime, err = str.AnyValueToString("startTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.EndTime, err = str.FromMapToAny("endTime", workingHours) mi.WorkingHours.EndTime, err = str.AnyValueToString("endTime", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
timeZone, err := tform.FromMapToAny[map[string]any]("timeZone", workingHours) timeZone, err := tform.AnyValueToT[map[string]any]("timeZone", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.TimeZone.Name, err = str.FromMapToAny("name", timeZone) mi.WorkingHours.TimeZone.Name, err = str.AnyValueToString("name", timeZone)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
days, err := tform.FromMapToAny[[]any]("daysOfWeek", workingHours) days, err := tform.AnyValueToT[[]any]("daysOfWeek", workingHours)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
for _, day := range days { for _, day := range days {
s, err := str.FromAny(day) s, err := str.AnyToString(day)
mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err) mi.ErrGetMailBoxSetting = appendIfErr(mi.ErrGetMailBoxSetting, err)
mi.WorkingHours.DaysOfWeek = append(mi.WorkingHours.DaysOfWeek, s) mi.WorkingHours.DaysOfWeek = append(mi.WorkingHours.DaysOfWeek, s)
} }