diff --git a/src/internal/connector/onedrive/api/drive.go b/src/internal/connector/onedrive/api/drive.go index 6dd7d46a1..fea6e53a7 100644 --- a/src/internal/connector/onedrive/api/drive.go +++ b/src/internal/connector/onedrive/api/drive.go @@ -3,6 +3,7 @@ package api import ( "context" + msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" mssites "github.com/microsoftgraph/msgraph-sdk-go/sites" msusers "github.com/microsoftgraph/msgraph-sdk-go/users" @@ -12,6 +13,65 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph/api" ) +func getValues[T any](l api.PageLinker) ([]T, error) { + page, ok := l.(interface{ GetValue() []T }) + if !ok { + return nil, errors.Errorf( + "response of type [%T] does not comply with GetValue() interface", + l, + ) + } + + return page.GetValue(), nil +} + +// max we can do is 999 +const pageSize = int32(999) + +type driveItemPager struct { + gs graph.Servicer + builder *msdrives.ItemRootDeltaRequestBuilder + options *msdrives.ItemRootDeltaRequestBuilderGetRequestConfiguration +} + +func NewItemPager( + gs graph.Servicer, + driveID, link string, + fields []string, +) *driveItemPager { + pageCount := pageSize + requestConfig := &msdrives.ItemRootDeltaRequestBuilderGetRequestConfiguration{ + QueryParameters: &msdrives.ItemRootDeltaRequestBuilderGetQueryParameters{ + Top: &pageCount, + Select: fields, + }, + } + + res := &driveItemPager{ + gs: gs, + options: requestConfig, + builder: gs.Client().DrivesById(driveID).Root().Delta(), + } + + if len(link) > 0 { + res.builder = msdrives.NewItemRootDeltaRequestBuilder(link, gs.Adapter()) + } + + return res +} + +func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, error) { + return p.builder.Get(ctx, p.options) +} + +func (p *driveItemPager) SetNext(link string) { + p.builder = msdrives.NewItemRootDeltaRequestBuilder(link, p.gs.Adapter()) +} + +func (p *driveItemPager) ValuesIn(l api.DeltaPageLinker) ([]models.DriveItemable, error) { + return getValues[models.DriveItemable](l) +} + type userDrivePager struct { gs graph.Servicer builder *msusers.ItemDrivesRequestBuilder @@ -47,15 +107,7 @@ func (p *userDrivePager) SetNext(link string) { } func (p *userDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { - page, ok := l.(interface{ GetValue() []models.Driveable }) - if !ok { - return nil, errors.Errorf( - "response of type [%T] does not comply with GetValue() interface", - l, - ) - } - - return page.GetValue(), nil + return getValues[models.Driveable](l) } type siteDrivePager struct { @@ -93,13 +145,5 @@ func (p *siteDrivePager) SetNext(link string) { } func (p *siteDrivePager) ValuesIn(l api.PageLinker) ([]models.Driveable, error) { - page, ok := l.(interface{ GetValue() []models.Driveable }) - if !ok { - return nil, errors.Errorf( - "response of type [%T] does not comply with GetValue() interface", - l, - ) - } - - return page.GetValue(), nil + return getValues[models.Driveable](l) } diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 35b81f7f2..1e36b1ea3 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -65,6 +65,13 @@ type Collections struct { // for a OneDrive folder CollectionMap map[string]data.Collection + // Not the most ideal, but allows us to change the pager function for testing + // as needed. This will allow us to mock out some scenarios during testing. + itemPagerFunc func( + servicer graph.Servicer, + driveID, link string, + ) itemPager + // Track stats from drive enumeration. Represents the items backed up. NumItems int NumFiles int @@ -88,6 +95,7 @@ func NewCollections( source: source, matcher: matcher, CollectionMap: map[string]data.Collection{}, + itemPagerFunc: defaultItemPager, service: service, statusUpdater: statusUpdater, ctrl: ctrlOpts, @@ -266,7 +274,11 @@ func (c *Collections) Get( delta, paths, excluded, err := collectItems( ctx, - c.service, + c.itemPagerFunc( + c.service, + driveID, + "", + ), driveID, driveName, c.UpdateCollections, diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 6270ec08a..8e1578caf 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -7,7 +7,6 @@ import ( "time" msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive" - msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" "github.com/pkg/errors" @@ -135,11 +134,42 @@ type itemCollector func( excluded map[string]struct{}, ) error +type itemPager interface { + GetPage(context.Context) (gapi.DeltaPageLinker, error) + SetNext(nextLink string) + ValuesIn(gapi.DeltaPageLinker) ([]models.DriveItemable, error) +} + +func defaultItemPager( + servicer graph.Servicer, + driveID, link string, +) itemPager { + return api.NewItemPager( + servicer, + driveID, + link, + []string{ + "content.downloadUrl", + "createdBy", + "createdDateTime", + "file", + "folder", + "id", + "lastModifiedDateTime", + "name", + "package", + "parentReference", + "root", + "size", + }, + ) +} + // collectItems will enumerate all items in the specified drive and hand them to the // provided `collector` method func collectItems( ctx context.Context, - service graph.Servicer, + pager itemPager, driveID, driveName string, collector itemCollector, ) (string, map[string]string, map[string]struct{}, error) { @@ -154,34 +184,8 @@ func collectItems( maps.Copy(newPaths, oldPaths) - // TODO: Specify a timestamp in the delta query - // https://docs.microsoft.com/en-us/graph/api/driveitem-delta? - // view=graph-rest-1.0&tabs=http#example-4-retrieving-delta-results-using-a-timestamp - builder := service.Client().DrivesById(driveID).Root().Delta() - pageCount := int32(999) // max we can do is 999 - requestFields := []string{ - "content.downloadUrl", - "createdBy", - "createdDateTime", - "file", - "folder", - "id", - "lastModifiedDateTime", - "name", - "package", - "parentReference", - "root", - "size", - } - requestConfig := &msdrives.ItemRootDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: &msdrives.ItemRootDeltaRequestBuilderGetQueryParameters{ - Top: &pageCount, - Select: requestFields, - }, - } - for { - r, err := builder.Get(ctx, requestConfig) + page, err := pager.GetPage(ctx) if err != nil { return "", nil, nil, errors.Wrapf( err, @@ -190,23 +194,29 @@ func collectItems( ) } - err = collector(ctx, driveID, driveName, r.GetValue(), oldPaths, newPaths, excluded) + vals, err := pager.ValuesIn(page) + if err != nil { + return "", nil, nil, errors.Wrap(err, "extracting items from response") + } + + err = collector(ctx, driveID, driveName, vals, oldPaths, newPaths, excluded) if err != nil { return "", nil, nil, err } - if r.GetOdataDeltaLink() != nil && len(*r.GetOdataDeltaLink()) > 0 { - newDeltaURL = *r.GetOdataDeltaLink() + nextLink, deltaLink := gapi.NextAndDeltaLink(page) + + if len(deltaLink) > 0 { + newDeltaURL = deltaLink } // Check if there are more items - nextLink := r.GetOdataNextLink() - if nextLink == nil { + if len(nextLink) == 0 { break } - logger.Ctx(ctx).Debugf("Found %s nextLink", *nextLink) - builder = msdrives.NewItemRootDeltaRequestBuilder(*nextLink, service.Adapter()) + logger.Ctx(ctx).Debugw("Found nextLink", "link", nextLink) + pager.SetNext(nextLink) } return newDeltaURL, newPaths, excluded, nil @@ -318,7 +328,11 @@ func GetAllFolders( for _, d := range drives { _, _, _, err = collectItems( ctx, - gs, + defaultItemPager( + gs, + *d.GetId(), + "", + ), *d.GetId(), *d.GetName(), func( diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 938748ca2..6a8894ebf 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -115,7 +115,17 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() { return nil } - _, _, _, err := collectItems(ctx, suite, suite.userDriveID, "General", itemCollector) + _, _, _, err := collectItems( + ctx, + defaultItemPager( + suite, + suite.userDriveID, + "", + ), + suite.userDriveID, + "General", + itemCollector, + ) require.NoError(suite.T(), err) // Test Requirement 2: Need a file