larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create helper functions/interfaces to retrieve next and delta links (#2308)

Browse Source 
## Description

Create a shared set of helper functions and interfaces for getting next and delta links from Graph API responses. This helps standardize how they are handled with respect to nil/empty values and makes comparisons easier in other code

## Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No 

## Type of change

- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #2264 

## Test Plan

- [ ] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2023-01-27 15:13:16 -08:00 committed by GitHub
parent 57accfc9c4
commit 704a0f8878
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 175 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/internal/connector/exchange/api/contacts.go
View File

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
)
@ -173,7 +174,7 @@ type contactPager struct {
options *users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration
}
func (p *contactPager) getPage(ctx context.Context) (pageLinker, error) {
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
return p.builder.Get(ctx, p.options)
}
@ -181,7 +182,7 @@ func (p *contactPager) setNext(nextLink string) {
p.builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(nextLink, p.gs.Adapter())
}
func (p *contactPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) {
func (p *contactPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl)
}

5
src/internal/connector/exchange/api/events.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
@ -203,7 +204,7 @@ type eventPager struct {
options *users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration
}
func (p *eventPager) getPage(ctx context.Context) (pageLinker, error) {
func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
return resp, err
}
@ -212,7 +213,7 @@ func (p *eventPager) setNext(nextLink string) {
p.builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(nextLink, p.gs.Adapter())
}
func (p *eventPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) {
func (p *eventPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl)
}

5
src/internal/connector/exchange/api/mail.go
View File

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
@ -198,7 +199,7 @@ type mailPager struct {
options *users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration
}
func (p *mailPager) getPage(ctx context.Context) (pageLinker, error) {
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
return p.builder.Get(ctx, p.options)
}
@ -206,7 +207,7 @@ func (p *mailPager) setNext(nextLink string) {
p.builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(nextLink, p.gs.Adapter())
}
func (p *mailPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) {
func (p *mailPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl)
}

22
src/internal/connector/exchange/api/shared.go
View File

@ -6,6 +6,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
)
@ -14,14 +15,9 @@ import (
// ---------------------------------------------------------------------------
type itemPager interface {
getPage(context.Context) (pageLinker, error)
getPage(context.Context) (api.DeltaPageLinker, error)
setNext(string)
valuesIn(pageLinker) ([]getIDAndAddtler, error)
}
type pageLinker interface {
GetOdataDeltaLink() *string
GetOdataNextLink() *string
valuesIn(api.DeltaPageLinker) ([]getIDAndAddtler, error)
}
type getIDAndAddtler interface {
@ -98,24 +94,24 @@ func getItemsAddedAndRemovedFromContainer(
}
}
nextLink, delta := api.NextAndDeltaLink(resp)
// the deltaLink is kind of like a cursor for overall data state.
// once we run through pages of nextLinks, the last query will
// produce a deltaLink instead (if supported), which we'll use on
// the next backup to only get the changes since this run.
delta := resp.GetOdataDeltaLink()
if delta != nil && len(*delta) > 0 {
deltaURL = *delta
if len(delta) > 0 {
deltaURL = delta
}
// the nextLink is our page cursor within this query.
// if we have more data to retrieve, we'll have a
// nextLink instead of a deltaLink.
nextLink := resp.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
if len(nextLink) == 0 {
break
}
pager.setNext(*nextLink)
pager.setNext(nextLink)
}
return addedIDs, removedIDs, deltaURL, nil

30
src/internal/connector/graph/api/api.go Normal file
View File

@ -0,0 +1,30 @@
package api
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
func NextLink(pl PageLinker) string {
next := pl.GetOdataNextLink()
if next == nil || len(*next) == 0 {
return ""
}
return *next
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
next := NextLink(pl)
delta := pl.GetOdataDeltaLink()
if delta == nil || len(*delta) == 0 {
return next, ""
}
return next, *delta
}

114
src/internal/connector/graph/api/api_test.go Normal file
View File

@ -0,0 +1,114 @@
package api_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph/api"
)
type mockNextLink struct {
nextLink *string
}
func (l mockNextLink) GetOdataNextLink() *string {
return l.nextLink
}
type mockDeltaNextLink struct {
mockNextLink
deltaLink *string
}
func (l mockDeltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
type APIUnitSuite struct {
suite.Suite
}
func TestAPIUnitSuite(t *testing.T) {
suite.Run(t, new(APIUnitSuite))
}
func (suite *APIUnitSuite) TestNextLink() {
for _, test := range nextLinkInputs {
suite.T().Run(test.name, func(t *testing.T) {
l := mockNextLink{nextLink: test.inputLink}
assert.Equal(t, test.expectedLink, api.NextLink(l))
})
}
}
func (suite *APIUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.T().Run(name, func(t *testing.T) {
l := mockDeltaNextLink{
mockNextLink: mockNextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := api.NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}

13
src/internal/connector/onedrive/api/drive.go
View File

@ -9,12 +9,9 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
)
type PageLinker interface {
GetOdataNextLink() *string
}
type userDrivePager struct {
gs graph.Servicer
builder *msusers.ItemDrivesRequestBuilder
@ -41,7 +38,7 @@ func NewUserDrivePager(
return res
}
func (p *userDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
return p.builder.Get(ctx, p.options)
}
@ -49,7 +46,7 @@ func (p *userDrivePager) SetNext(link string) {
p.builder = msusers.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *userDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
page, ok := l.(interface{ GetValue() []models.Driveable })
if !ok {
return nil, errors.Errorf(
@ -87,7 +84,7 @@ func NewSiteDrivePager(
return res
}
func (p *siteDrivePager) GetPage(ctx context.Context) (PageLinker, error) {
func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
return p.builder.Get(ctx, p.options)
}
@ -95,7 +92,7 @@ func (p *siteDrivePager) SetNext(link string) {
p.builder = mssites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
}
func (p *siteDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) {
func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
page, ok := l.(interface{ GetValue() []models.Driveable })
if !ok {
return nil, errors.Errorf(

13
src/internal/connector/onedrive/drive.go
View File

@ -14,6 +14,7 @@ import (
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/logger"
@ -36,9 +37,9 @@ const (
)
type drivePager interface {
GetPage(context.Context) (api.PageLinker, error)
GetPage(context.Context) (gapi.PageLinker, error)
SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error)
ValuesIn(gapi.PageLinker) ([]models.Driveable, error)
}
func PagerForSource(
@ -64,7 +65,7 @@ func drives(
) ([]models.Driveable, error) {
var (
err error
page api.PageLinker
page gapi.PageLinker
numberOfRetries = getDrivesRetries
drives = []models.Driveable{}
)
@ -111,12 +112,12 @@ func drives(
drives = append(drives, tmp...)
nextLink := page.GetOdataNextLink()
if nextLink == nil || len(*nextLink) == 0 {
nextLink := gapi.NextLink(page)
if len(nextLink) == 0 {
break
}
pager.SetNext(*nextLink)
pager.SetNext(nextLink)
}
logger.Ctx(ctx).Debugf("Found %d drives", len(drives))

2
src/internal/connector/onedrive/drive_test.go
View File

@ -14,7 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"