diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go index 85fea19fc..2e9014235 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/internal/connector/exchange/api/contacts.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" ) @@ -173,7 +174,7 @@ type contactPager struct { options *users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration } -func (p *contactPager) getPage(ctx context.Context) (pageLinker, error) { +func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { return p.builder.Get(ctx, p.options) } @@ -181,7 +182,7 @@ func (p *contactPager) setNext(nextLink string) { p.builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(nextLink, p.gs.Adapter()) } -func (p *contactPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { +func (p *contactPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) { return toValues[models.Contactable](pl) } diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index cd968056d..b3e6f9467 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" @@ -203,7 +204,7 @@ type eventPager struct { options *users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration } -func (p *eventPager) getPage(ctx context.Context) (pageLinker, error) { +func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { resp, err := p.builder.Get(ctx, p.options) return resp, err } @@ -212,7 +213,7 @@ func (p *eventPager) setNext(nextLink string) { p.builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(nextLink, p.gs.Adapter()) } -func (p *eventPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { +func (p *eventPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) { return toValues[models.Eventable](pl) } diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index f4d2d2e33..4c9d564f0 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" @@ -198,7 +199,7 @@ type mailPager struct { options *users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration } -func (p *mailPager) getPage(ctx context.Context) (pageLinker, error) { +func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { return p.builder.Get(ctx, p.options) } @@ -206,7 +207,7 @@ func (p *mailPager) setNext(nextLink string) { p.builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(nextLink, p.gs.Adapter()) } -func (p *mailPager) valuesIn(pl pageLinker) ([]getIDAndAddtler, error) { +func (p *mailPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error) { return toValues[models.Messageable](pl) } diff --git a/src/internal/connector/exchange/api/shared.go b/src/internal/connector/exchange/api/shared.go index dfe64d716..d89ce7411 100644 --- a/src/internal/connector/exchange/api/shared.go +++ b/src/internal/connector/exchange/api/shared.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/support" ) @@ -14,14 +15,9 @@ import ( // --------------------------------------------------------------------------- type itemPager interface { - getPage(context.Context) (pageLinker, error) + getPage(context.Context) (api.DeltaPageLinker, error) setNext(string) - valuesIn(pageLinker) ([]getIDAndAddtler, error) -} - -type pageLinker interface { - GetOdataDeltaLink() *string - GetOdataNextLink() *string + valuesIn(api.DeltaPageLinker) ([]getIDAndAddtler, error) } type getIDAndAddtler interface { @@ -98,24 +94,24 @@ func getItemsAddedAndRemovedFromContainer( } } + nextLink, delta := api.NextAndDeltaLink(resp) + // the deltaLink is kind of like a cursor for overall data state. // once we run through pages of nextLinks, the last query will // produce a deltaLink instead (if supported), which we'll use on // the next backup to only get the changes since this run. - delta := resp.GetOdataDeltaLink() - if delta != nil && len(*delta) > 0 { - deltaURL = *delta + if len(delta) > 0 { + deltaURL = delta } // the nextLink is our page cursor within this query. // if we have more data to retrieve, we'll have a // nextLink instead of a deltaLink. - nextLink := resp.GetOdataNextLink() - if nextLink == nil || len(*nextLink) == 0 { + if len(nextLink) == 0 { break } - pager.setNext(*nextLink) + pager.setNext(nextLink) } return addedIDs, removedIDs, deltaURL, nil diff --git a/src/internal/connector/graph/api/api.go b/src/internal/connector/graph/api/api.go new file mode 100644 index 000000000..abcf29a24 --- /dev/null +++ b/src/internal/connector/graph/api/api.go @@ -0,0 +1,30 @@ +package api + +type PageLinker interface { + GetOdataNextLink() *string +} + +type DeltaPageLinker interface { + PageLinker + GetOdataDeltaLink() *string +} + +func NextLink(pl PageLinker) string { + next := pl.GetOdataNextLink() + if next == nil || len(*next) == 0 { + return "" + } + + return *next +} + +func NextAndDeltaLink(pl DeltaPageLinker) (string, string) { + next := NextLink(pl) + + delta := pl.GetOdataDeltaLink() + if delta == nil || len(*delta) == 0 { + return next, "" + } + + return next, *delta +} diff --git a/src/internal/connector/graph/api/api_test.go b/src/internal/connector/graph/api/api_test.go new file mode 100644 index 000000000..37932396d --- /dev/null +++ b/src/internal/connector/graph/api/api_test.go @@ -0,0 +1,114 @@ +package api_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/connector/graph/api" +) + +type mockNextLink struct { + nextLink *string +} + +func (l mockNextLink) GetOdataNextLink() *string { + return l.nextLink +} + +type mockDeltaNextLink struct { + mockNextLink + deltaLink *string +} + +func (l mockDeltaNextLink) GetOdataDeltaLink() *string { + return l.deltaLink +} + +type testInput struct { + name string + inputLink *string + expectedLink string +} + +// Needs to be var not const so we can take the address of it. +var ( + emptyLink = "" + link = "foo" + link2 = "bar" + + nextLinkInputs = []testInput{ + { + name: "empty", + inputLink: &emptyLink, + expectedLink: "", + }, + { + name: "nil", + inputLink: nil, + expectedLink: "", + }, + { + name: "non_empty", + inputLink: &link, + expectedLink: link, + }, + } +) + +type APIUnitSuite struct { + suite.Suite +} + +func TestAPIUnitSuite(t *testing.T) { + suite.Run(t, new(APIUnitSuite)) +} + +func (suite *APIUnitSuite) TestNextLink() { + for _, test := range nextLinkInputs { + suite.T().Run(test.name, func(t *testing.T) { + l := mockNextLink{nextLink: test.inputLink} + assert.Equal(t, test.expectedLink, api.NextLink(l)) + }) + } +} + +func (suite *APIUnitSuite) TestNextAndDeltaLink() { + deltaTable := []testInput{ + { + name: "empty", + inputLink: &emptyLink, + expectedLink: "", + }, + { + name: "nil", + inputLink: nil, + expectedLink: "", + }, + { + name: "non_empty", + // Use a different link so we can see if the results get swapped or something. + inputLink: &link2, + expectedLink: link2, + }, + } + + for _, next := range nextLinkInputs { + for _, delta := range deltaTable { + name := strings.Join([]string{next.name, "next", delta.name, "delta"}, "_") + + suite.T().Run(name, func(t *testing.T) { + l := mockDeltaNextLink{ + mockNextLink: mockNextLink{nextLink: next.inputLink}, + deltaLink: delta.inputLink, + } + gotNext, gotDelta := api.NextAndDeltaLink(l) + + assert.Equal(t, next.expectedLink, gotNext) + assert.Equal(t, delta.expectedLink, gotDelta) + }) + } + } +} diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 7e5cfcaac..6dd7d46a1 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -9,12 +9,9 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/graph/api" ) -type PageLinker interface { - GetOdataNextLink() *string -} - type userDrivePager struct { gs graph.Servicer builder *msusers.ItemDrivesRequestBuilder @@ -41,7 +38,7 @@ func NewUserDrivePager( return res } -func (p *userDrivePager) GetPage(ctx context.Context) (PageLinker, error) { +func (p *userDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { return p.builder.Get(ctx, p.options) } @@ -49,7 +46,7 @@ func (p *userDrivePager) SetNext(link string) { p.builder = msusers.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) } -func (p *userDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) { +func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { page, ok := l.(interface{ GetValue() []models.Driveable }) if !ok { return nil, errors.Errorf( @@ -87,7 +84,7 @@ func NewSiteDrivePager( return res } -func (p *siteDrivePager) GetPage(ctx context.Context) (PageLinker, error) { +func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) { return p.builder.Get(ctx, p.options) } @@ -95,7 +92,7 @@ func (p *siteDrivePager) SetNext(link string) { p.builder = mssites.NewItemDrivesRequestBuilder(link, p.gs.Adapter()) } -func (p *siteDrivePager) ValuesIn(l PageLinker) ([]models.Driveable, error) { +func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { page, ok := l.(interface{ GetValue() []models.Driveable }) if !ok { return nil, errors.Errorf( diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 4bc56aec0..6270ec08a 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -14,6 +14,7 @@ import ( "golang.org/x/exp/maps" "github.com/alcionai/corso/src/internal/connector/graph" + gapi "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/logger" @@ -36,9 +37,9 @@ const ( ) type drivePager interface { - GetPage(context.Context) (api.PageLinker, error) + GetPage(context.Context) (gapi.PageLinker, error) SetNext(nextLink string) - ValuesIn(api.PageLinker) ([]models.Driveable, error) + ValuesIn(gapi.PageLinker) ([]models.Driveable, error) } func PagerForSource( @@ -64,7 +65,7 @@ func drives( ) ([]models.Driveable, error) { var ( err error - page api.PageLinker + page gapi.PageLinker numberOfRetries = getDrivesRetries drives = []models.Driveable{} ) @@ -111,12 +112,12 @@ func drives( drives = append(drives, tmp...) - nextLink := page.GetOdataNextLink() - if nextLink == nil || len(*nextLink) == 0 { + nextLink := gapi.NextLink(page) + if len(nextLink) == 0 { break } - pager.SetNext(*nextLink) + pager.SetNext(nextLink) } logger.Ctx(ctx).Debugf("Found %d drives", len(drives)) diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index 6ace0c3c2..66449a917 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -14,7 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" + "github.com/alcionai/corso/src/internal/connector/graph/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger"