Backup: SharePoint: List content retrieval (#1633)
## Description Logic for List retrieval from M365 back store. Keeping separate from Collection creation logic as there are a lot of calls to retrieve all supporting relationship data from the backstore. Adds collection to ## Type of change <!--- Please check the type of change your PR introduces: ---> - [x] 🌻 Feature ## Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> *closes #1475<issue> ## Test Plan - [x] ⚡ Unit test
This commit is contained in:
Danny
GitHub
parent
6dcbc8f2a1
commit
e767bb0b77
@ -1,12 +1,16 @@
|
||||
package sharepoint
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
kw "github.com/microsoft/kiota-serialization-json-go"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
@ -63,6 +67,7 @@ func (sc *Collection) FullPath() path.Path {
|
||||
}
|
||||
|
||||
func (sc *Collection) Items() <-chan data.Stream {
|
||||
go sc.populate(context.TODO())
|
||||
return sc.data
|
||||
}
|
||||
|
||||
@ -100,3 +105,62 @@ func (sc *Collection) finishPopulation(ctx context.Context, success int, totalBy
|
||||
sc.fullPath.Folder())
|
||||
logger.Ctx(ctx).Debug(status.String())
|
||||
}
|
||||
|
||||
// populate utility function to retrieve data from back store for a given collection
|
||||
func (sc *Collection) populate(ctx context.Context) {
|
||||
var (
|
||||
success int
|
||||
totalBytes, arrayLength int64
|
||||
errs error
|
||||
writer = kw.NewJsonSerializationWriter()
|
||||
)
|
||||
|
||||
// TODO: Insert correct ID for CollectionProgress
|
||||
colProgress, closer := observe.CollectionProgress("name", sc.fullPath.Category().String(), sc.fullPath.Folder())
|
||||
go closer()
|
||||
|
||||
defer func() {
|
||||
close(colProgress)
|
||||
sc.finishPopulation(ctx, success, totalBytes, errs)
|
||||
}()
|
||||
|
||||
// sc.jobs contains query = all of the site IDs.
|
||||
for _, id := range sc.jobs {
|
||||
// Retrieve list data from M365
|
||||
lists, err := loadLists(ctx, sc.service, id)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(id, err, errs)
|
||||
}
|
||||
// Write Data and Send
|
||||
for _, lst := range lists {
|
||||
err = writer.WriteObjectValue("", lst)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(*lst.GetId(), err, errs)
|
||||
continue
|
||||
}
|
||||
|
||||
byteArray, err := writer.GetSerializedContent()
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(*lst.GetId(), err, errs)
|
||||
continue
|
||||
}
|
||||
|
||||
writer.Close()
|
||||
|
||||
arrayLength = int64(len(byteArray))
|
||||
|
||||
if arrayLength > 0 {
|
||||
totalBytes += arrayLength
|
||||
|
||||
success++
|
||||
sc.data <- &Item{
|
||||
id: *lst.GetId(),
|
||||
data: io.NopCloser(bytes.NewReader(byteArray)),
|
||||
info: sharePointListInfo(lst, arrayLength),
|
||||
}
|
||||
|
||||
colProgress <- struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,7 +12,6 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/mockconnector"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
@ -43,9 +42,6 @@ func (suite *SharePointCollectionSuite) TestSharePointDataReader_Valid() {
|
||||
// SharePoint collection and to use the data stream channel.
|
||||
func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
|
||||
t := suite.T()
|
||||
ctx, flush := tester.NewContext()
|
||||
|
||||
defer flush()
|
||||
|
||||
ow := kw.NewJsonSerializationWriter()
|
||||
listing := mockconnector.GetMockList("Mock List")
|
||||
@ -73,7 +69,6 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() {
|
||||
data: io.NopCloser(bytes.NewReader(byteArray)),
|
||||
info: sharePointListInfo(listing, int64(len(byteArray))),
|
||||
}
|
||||
col.finishPopulation(ctx, 0, 0, nil)
|
||||
|
||||
readItems := []data.Stream{}
|
||||
for item := range col.Items() {
|
||||
|
||||
@ -57,6 +57,12 @@ func DataCollections(
|
||||
defer close(foldersComplete)
|
||||
|
||||
switch scope.Category().PathType() {
|
||||
// TODO path.ListCategory: PR
|
||||
// collect Lists
|
||||
// done?
|
||||
case path.ListsCategory:
|
||||
return nil, fmt.Errorf("sharePoint list collections not supported")
|
||||
|
||||
case path.LibrariesCategory:
|
||||
spcs, err := collectLibraries(
|
||||
ctx,
|
||||
|
||||
52
src/internal/connector/sharepoint/helper_test.go
Normal file
52
src/internal/connector/sharepoint/helper_test.go
Normal file
@ -0,0 +1,52 @@
|
||||
package sharepoint
|
||||
|
||||
import (
|
||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
)
|
||||
|
||||
type testService struct {
|
||||
client msgraphsdk.GraphServiceClient
|
||||
adapter msgraphsdk.GraphRequestAdapter
|
||||
credentials account.M365Config
|
||||
}
|
||||
|
||||
//------------------------------------------------------------
|
||||
// Functions to comply with graph.Service Interface
|
||||
//------------------------------------------------------------
|
||||
|
||||
func (ts *testService) Client() *msgraphsdk.GraphServiceClient {
|
||||
return &ts.client
|
||||
}
|
||||
|
||||
func (ts *testService) Adapter() *msgraphsdk.GraphRequestAdapter {
|
||||
return &ts.adapter
|
||||
}
|
||||
|
||||
func (ts *testService) ErrPolicy() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func createTestService(credentials account.M365Config) (*testService, error) {
|
||||
{
|
||||
adapter, err := graph.CreateAdapter(
|
||||
credentials.AzureTenantID,
|
||||
credentials.AzureClientID,
|
||||
credentials.AzureClientSecret,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "creating microsoft graph service for exchange")
|
||||
}
|
||||
|
||||
service := testService{
|
||||
adapter: *adapter,
|
||||
client: *msgraphsdk.NewGraphServiceClient(adapter),
|
||||
credentials: credentials,
|
||||
}
|
||||
|
||||
return &service, nil
|
||||
}
|
||||
}
|
||||
284
src/internal/connector/sharepoint/list.go
Normal file
284
src/internal/connector/sharepoint/list.go
Normal file
@ -0,0 +1,284 @@
|
||||
package sharepoint
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists/item/columns"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists/item/contenttypes"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists/item/contenttypes/item/columnlinks"
|
||||
tc "github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists/item/contenttypes/item/columns"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites/item/lists/item/items"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
)
|
||||
|
||||
// list.go contains additional functions to help retrieve SharePoint List data from M365
|
||||
// SharePoint lists represent lists on a site. Inherits additional properties from
|
||||
// baseItem: https://learn.microsoft.com/en-us/graph/api/resources/baseitem?view=graph-rest-1.0
|
||||
// The full details concerning SharePoint Lists can
|
||||
// be found at: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
|
||||
// Note additional calls are required for the relationships that exist outside of the object properties.
|
||||
|
||||
// loadLists is a utility function to populate the List object.
|
||||
// @param siteID the M365 ID that represents the SharePoint Site
|
||||
// Makes additional calls to retrieve the following relationships:
|
||||
// - Columns
|
||||
// - ContentTypes
|
||||
// - List Items
|
||||
func loadLists(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
siteID string,
|
||||
) ([]models.Listable, error) {
|
||||
var (
|
||||
prefix = gs.Client().SitesById(siteID)
|
||||
builder = prefix.Lists()
|
||||
results = make([]models.Listable, 0)
|
||||
errs error
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, support.WrapAndAppend(support.ConnectorStackErrorTrace(err), err, errs)
|
||||
}
|
||||
|
||||
for _, entry := range resp.GetValue() {
|
||||
id := *entry.GetId()
|
||||
|
||||
cols, err := fetchColumns(ctx, gs, siteID, id, "")
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(siteID, err, errs)
|
||||
continue
|
||||
}
|
||||
|
||||
entry.SetColumns(cols)
|
||||
|
||||
cTypes, err := fetchContentTypes(ctx, gs, siteID, id)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(siteID, err, errs)
|
||||
continue
|
||||
}
|
||||
|
||||
entry.SetContentTypes(cTypes)
|
||||
|
||||
lItems, err := fetchListItems(ctx, gs, siteID, id)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(siteID, err, errs)
|
||||
continue
|
||||
}
|
||||
|
||||
entry.SetItems(lItems)
|
||||
|
||||
results = append(results, entry)
|
||||
}
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = lists.NewListsRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
return nil, errs
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// fetchListItems utility for retrieving ListItem data and the associated relationship
|
||||
// data. Additional call append data to the tracked items, and do not create additional collections.
|
||||
// Additional Call:
|
||||
// * Fields
|
||||
func fetchListItems(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
siteID, listID string,
|
||||
) ([]models.ListItemable, error) {
|
||||
var (
|
||||
prefix = gs.Client().SitesById(siteID).ListsById(listID)
|
||||
builder = prefix.Items()
|
||||
itms = make([]models.ListItemable, 0)
|
||||
errs error
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
|
||||
}
|
||||
|
||||
for _, itm := range resp.GetValue() {
|
||||
newPrefix := prefix.ItemsById(*itm.GetId())
|
||||
|
||||
fields, err := newPrefix.Fields().Get(ctx, nil)
|
||||
if err != nil {
|
||||
errs = errors.Wrap(err, support.ConnectorStackErrorTrace(err))
|
||||
}
|
||||
|
||||
itm.SetFields(fields)
|
||||
|
||||
itms = append(itms, itm)
|
||||
}
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = items.NewItemsRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
return nil, errors.Wrap(errs, "fetchListItem unsuccessful")
|
||||
}
|
||||
|
||||
return itms, nil
|
||||
}
|
||||
|
||||
// fetchColumns utility function to return columns from a site.
|
||||
// An additional call required to check for details concerning the SourceColumn.
|
||||
// For additional details: https://learn.microsoft.com/en-us/graph/api/resources/columndefinition?view=graph-rest-1.0
|
||||
// TODO: Refactor on if/else (dadams39)
|
||||
func fetchColumns(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
siteID, listID, cTypeID string,
|
||||
) ([]models.ColumnDefinitionable, error) {
|
||||
cs := make([]models.ColumnDefinitionable, 0)
|
||||
|
||||
if len(cTypeID) == 0 {
|
||||
builder := gs.Client().SitesById(siteID).ListsById(listID).Columns()
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, support.WrapAndAppend(support.ConnectorStackErrorTrace(err), err, nil)
|
||||
}
|
||||
|
||||
cs = append(cs, resp.GetValue()...)
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = columns.NewColumnsRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
} else {
|
||||
builder := gs.Client().SitesById(siteID).ListsById(listID).ContentTypesById(cTypeID).Columns()
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
|
||||
}
|
||||
|
||||
cs = append(cs, resp.GetValue()...)
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = tc.NewColumnsRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
}
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
||||
// fetchContentTypes retrieves all data for content type. Additional queries required
|
||||
// for the following:
|
||||
// - ColumnLinks
|
||||
// - Columns
|
||||
// The following two are not included:
|
||||
// - ColumnPositions
|
||||
// - BaseTypes
|
||||
// These relationships are not included as they following error from the API:
|
||||
// itemNotFound Item not found: error status code received from the API
|
||||
// Current as of github.com/microsoftgraph/msgraph-sdk-go v0.40.0
|
||||
// TODO: Verify functionality after version upgrade or remove (dadams39) Check Stubs
|
||||
func fetchContentTypes(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
siteID, listID string,
|
||||
) ([]models.ContentTypeable, error) {
|
||||
var (
|
||||
cTypes = make([]models.ContentTypeable, 0)
|
||||
builder = gs.Client().SitesById(siteID).ListsById(listID).ContentTypes()
|
||||
errs error
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, support.WrapAndAppend(support.ConnectorStackErrorTrace(err), err, errs)
|
||||
}
|
||||
|
||||
for _, cont := range resp.GetValue() {
|
||||
id := *cont.GetId()
|
||||
|
||||
links, err := fetchColumnLinks(ctx, gs, siteID, listID, id)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend("unable to add column links to list", err, errs)
|
||||
break
|
||||
}
|
||||
|
||||
cont.SetColumnLinks(links)
|
||||
// TODO: stub for columPositions
|
||||
|
||||
cs, err := fetchColumns(ctx, gs, siteID, listID, id)
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend("unable to populate columns for contentType", err, errs)
|
||||
}
|
||||
|
||||
cont.SetColumns(cs)
|
||||
// TODO: stub for BaseTypes
|
||||
|
||||
cTypes = append(cTypes, cont)
|
||||
}
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = contenttypes.NewContentTypesRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
return nil, errs
|
||||
}
|
||||
|
||||
return cTypes, nil
|
||||
}
|
||||
|
||||
func fetchColumnLinks(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
siteID, listID, cTypeID string,
|
||||
) ([]models.ColumnLinkable, error) {
|
||||
var (
|
||||
builder = gs.Client().SitesById(siteID).ListsById(listID).ContentTypesById(cTypeID).ColumnLinks()
|
||||
links = make([]models.ColumnLinkable, 0)
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
|
||||
}
|
||||
|
||||
links = append(links, resp.GetValue()...)
|
||||
|
||||
if resp.GetOdataNextLink() == nil {
|
||||
break
|
||||
}
|
||||
|
||||
builder = columnlinks.NewColumnLinksRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
|
||||
}
|
||||
|
||||
return links, nil
|
||||
}
|
||||
61
src/internal/connector/sharepoint/list_test.go
Normal file
61
src/internal/connector/sharepoint/list_test.go
Normal file
@ -0,0 +1,61 @@
|
||||
package sharepoint
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
)
|
||||
|
||||
type SharePointSuite struct {
|
||||
suite.Suite
|
||||
creds account.M365Config
|
||||
}
|
||||
|
||||
func (suite *SharePointSuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
a := tester.NewM365Account(t)
|
||||
m365, err := a.M365Config()
|
||||
require.NoError(t, err)
|
||||
|
||||
suite.creds = m365
|
||||
}
|
||||
|
||||
func TestSharePointSuite(t *testing.T) {
|
||||
if err := tester.RunOnAny(
|
||||
tester.CorsoCITests,
|
||||
); err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
suite.Run(t, new(SharePointSuite))
|
||||
}
|
||||
|
||||
// Test LoadList --> Retrieves all data from backStore
|
||||
// Functions tested:
|
||||
// - fetchListItems()
|
||||
// - fetchColumns()
|
||||
// - fetchContentColumns()
|
||||
// - fetchContentTypes()
|
||||
// - fetchColumnLinks
|
||||
// TODO: upgrade passed github.com/microsoftgraph/msgraph-sdk-go v0.40.0
|
||||
// to verify if these 2 calls are valid
|
||||
// - fetchContentBaseTypes
|
||||
// - fetchColumnPositions
|
||||
func (suite *SharePointSuite) TestLoadList() {
|
||||
ctx, flush := tester.NewContext()
|
||||
defer flush()
|
||||
|
||||
t := suite.T()
|
||||
service, err := createTestService(suite.creds)
|
||||
require.NoError(t, err)
|
||||
|
||||
lists, err := loadLists(ctx, service, "root")
|
||||
assert.NoError(t, err)
|
||||
assert.Greater(t, len(lists), 0)
|
||||
t.Logf("Length: %d\n", len(lists))
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user