Create helper functions/interfaces to retrieve next and delta links (#2308)

## Description

Create a shared set of helper functions and interfaces for getting next and delta links from Graph API responses. This helps standardize how they are handled with respect to nil/empty values and makes comparisons easier in other code

## Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No 

## Type of change

- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #2264 

## Test Plan

- [ ] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2023-01-27 15:13:16 -08:00 committed by GitHub
parent 57accfc9c4
commit 704a0f8878
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 175 additions and 34 deletions

View File

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
) )
@ -173,7 +174,7 @@ type contactPager struct {
options *users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration options *users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration
} }
func (p *contactPager) getPage(ctx context.Context) (pageLinker, error) { func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
return p.builder.Get(ctx, p.options) return p.builder.Get(ctx, p.options)
} }
@ -181,7 +182,7 @@ func (p *contactPager) setNext(nextLink string) {
p.builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(nextLink, p.gs.Adapter()) p.builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(nextLink, p.gs.Adapter())
} }
func (p *contactPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { func (p *contactPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Contactable](pl) return toValues[models.Contactable](pl)
} }

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -203,7 +204,7 @@ type eventPager struct {
options *users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration options *users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration
} }
func (p *eventPager) getPage(ctx context.Context) (pageLinker, error) { func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options) resp, err := p.builder.Get(ctx, p.options)
return resp, err return resp, err
} }
@ -212,7 +213,7 @@ func (p *eventPager) setNext(nextLink string) {
p.builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(nextLink, p.gs.Adapter()) p.builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(nextLink, p.gs.Adapter())
} }
func (p *eventPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { func (p *eventPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Eventable](pl) return toValues[models.Eventable](pl)
} }

View File

@ -13,6 +13,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -198,7 +199,7 @@ type mailPager struct {
options *users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration options *users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration
} }
func (p *mailPager) getPage(ctx context.Context) (pageLinker, error) { func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
return p.builder.Get(ctx, p.options) return p.builder.Get(ctx, p.options)
} }
@ -206,7 +207,7 @@ func (p *mailPager) setNext(nextLink string) {
p.builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(nextLink, p.gs.Adapter()) p.builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(nextLink, p.gs.Adapter())
} }
func (p *mailPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { func (p *mailPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl) return toValues[models.Messageable](pl)
} }

View File

@ -6,6 +6,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
) )
@ -14,14 +15,9 @@ import (
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type itemPager interface { type itemPager interface {
getPage(context.Context) (pageLinker, error) getPage(context.Context) (api.DeltaPageLinker, error)
setNext(string) setNext(string)
valuesIn(pageLinker) ([]getIDAndAddtler, error) valuesIn(api.DeltaPageLinker) ([]getIDAndAddtler, error)
}
type pageLinker interface {
GetOdataDeltaLink() *string
GetOdataNextLink() *string
} }
type getIDAndAddtler interface { type getIDAndAddtler interface {
@ -98,24 +94,24 @@ func getItemsAddedAndRemovedFromContainer(
} }
} }
nextLink, delta := api.NextAndDeltaLink(resp)
// the deltaLink is kind of like a cursor for overall data state. // the deltaLink is kind of like a cursor for overall data state.
// once we run through pages of nextLinks, the last query will // once we run through pages of nextLinks, the last query will
// produce a deltaLink instead (if supported), which we'll use on // produce a deltaLink instead (if supported), which we'll use on
// the next backup to only get the changes since this run. // the next backup to only get the changes since this run.
delta := resp.GetOdataDeltaLink() if len(delta) > 0 {
if delta != nil && len(*delta) > 0 { deltaURL = delta
deltaURL = *delta
} }
// the nextLink is our page cursor within this query. // the nextLink is our page cursor within this query.
// if we have more data to retrieve, we'll have a // if we have more data to retrieve, we'll have a
// nextLink instead of a deltaLink. // nextLink instead of a deltaLink.
nextLink := resp.GetOdataNextLink() if len(nextLink) == 0 {
if nextLink == nil || len(*nextLink) == 0 {
break break
} }
pager.setNext(*nextLink) pager.setNext(nextLink)
} }
return addedIDs, removedIDs, deltaURL, nil return addedIDs, removedIDs, deltaURL, nil

View File

@ -0,0 +1,30 @@
package api
type PageLinker interface {
GetOdataNextLink() *string
}
type DeltaPageLinker interface {
PageLinker
GetOdataDeltaLink() *string
}
func NextLink(pl PageLinker) string {
next := pl.GetOdataNextLink()
if next == nil || len(*next) == 0 {
return ""
}
return *next
}
func NextAndDeltaLink(pl DeltaPageLinker) (string, string) {
next := NextLink(pl)
delta := pl.GetOdataDeltaLink()
if delta == nil || len(*delta) == 0 {
return next, ""
}
return next, *delta
}

View File

@ -0,0 +1,114 @@
package api_test
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph/api"
)
type mockNextLink struct {
nextLink *string
}
func (l mockNextLink) GetOdataNextLink() *string {
return l.nextLink
}
type mockDeltaNextLink struct {
mockNextLink
deltaLink *string
}
func (l mockDeltaNextLink) GetOdataDeltaLink() *string {
return l.deltaLink
}
type testInput struct {
name string
inputLink *string
expectedLink string
}
// Needs to be var not const so we can take the address of it.
var (
emptyLink = ""
link = "foo"
link2 = "bar"
nextLinkInputs = []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
inputLink: &link,
expectedLink: link,
},
}
)
type APIUnitSuite struct {
suite.Suite
}
func TestAPIUnitSuite(t *testing.T) {
suite.Run(t, new(APIUnitSuite))
}
func (suite *APIUnitSuite) TestNextLink() {
for _, test := range nextLinkInputs {
suite.T().Run(test.name, func(t *testing.T) {
l := mockNextLink{nextLink: test.inputLink}
assert.Equal(t, test.expectedLink, api.NextLink(l))
})
}
}
func (suite *APIUnitSuite) TestNextAndDeltaLink() {
deltaTable := []testInput{
{
name: "empty",
inputLink: &emptyLink,
expectedLink: "",
},
{
name: "nil",
inputLink: nil,
expectedLink: "",
},
{
name: "non_empty",
// Use a different link so we can see if the results get swapped or something.
inputLink: &link2,
expectedLink: link2,
},
}
for _, next := range nextLinkInputs {
for _, delta := range deltaTable {
name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_")
suite.T().Run(name, func(t *testing.T) {
l := mockDeltaNextLink{
mockNextLink: mockNextLink{nextLink: next.inputLink},
deltaLink: delta.inputLink,
}
gotNext, gotDelta := api.NextAndDeltaLink(l)
assert.Equal(t, next.expectedLink, gotNext)
assert.Equal(t, delta.expectedLink, gotDelta)
})
}
}
}

View File

@ -9,12 +9,9 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
) )
type PageLinker interface {
GetOdataNextLink() *string
}
type userDrivePager struct { type userDrivePager struct {
gs graph.Servicer gs graph.Servicer
builder *msusers.ItemDrivesRequestBuilder builder *msusers.ItemDrivesRequestBuilder
@ -41,7 +38,7 @@ func NewUserDrivePager(
return res return res
} }
func (p *userDrivePager) GetPage(ctx context.Context) (PageLinker, error) { func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
return p.builder.Get(ctx, p.options) return p.builder.Get(ctx, p.options)
} }
@ -49,7 +46,7 @@ func (p *userDrivePager) SetNext(link string) {
p.builder = msusers.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) p.builder = msusers.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
} }
func (p *userDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) { func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
page, ok := l.(interface{ GetValue() []models.Driveable }) page, ok := l.(interface{ GetValue() []models.Driveable })
if !ok { if !ok {
return nil, errors.Errorf( return nil, errors.Errorf(
@ -87,7 +84,7 @@ func NewSiteDrivePager(
return res return res
} }
func (p *siteDrivePager) GetPage(ctx context.Context) (PageLinker, error) { func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
return p.builder.Get(ctx, p.options) return p.builder.Get(ctx, p.options)
} }
@ -95,7 +92,7 @@ func (p *siteDrivePager) SetNext(link string) {
p.builder = mssites.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) p.builder = mssites.NewItemDrivesRequestBuilder(link, p.gs.Adapter())
} }
func (p *siteDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) { func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) {
page, ok := l.(interface{ GetValue() []models.Driveable }) page, ok := l.(interface{ GetValue() []models.Driveable })
if !ok { if !ok {
return nil, errors.Errorf( return nil, errors.Errorf(

View File

@ -14,6 +14,7 @@ import (
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
@ -36,9 +37,9 @@ const (
) )
type drivePager interface { type drivePager interface {
GetPage(context.Context) (api.PageLinker, error) GetPage(context.Context) (gapi.PageLinker, error)
SetNext(nextLink string) SetNext(nextLink string)
ValuesIn(api.PageLinker) ([]models.Driveable, error) ValuesIn(gapi.PageLinker) ([]models.Driveable, error)
} }
func PagerForSource( func PagerForSource(
@ -64,7 +65,7 @@ func drives(
) ([]models.Driveable, error) { ) ([]models.Driveable, error) {
var ( var (
err error err error
page api.PageLinker page gapi.PageLinker
numberOfRetries = getDrivesRetries numberOfRetries = getDrivesRetries
drives = []models.Driveable{} drives = []models.Driveable{}
) )
@ -111,12 +112,12 @@ func drives(
drives = append(drives, tmp...) drives = append(drives, tmp...)
nextLink := page.GetOdataNextLink() nextLink := gapi.NextLink(page)
if nextLink == nil || len(*nextLink) == 0 { if len(nextLink) == 0 {
break break
} }
pager.SetNext(*nextLink) pager.SetNext(nextLink)
} }
logger.Ctx(ctx).Debugf("Found %d drives", len(drives)) logger.Ctx(ctx).Debugf("Found %d drives", len(drives))

View File

@ -14,7 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"