enables sharepoint to use lists backup handler for lists ops (#4920)
enables sharepoint to use lists backup handler for lists ops Changes previously approved in: - https://github.com/alcionai/corso/pull/4786 - https://github.com/alcionai/corso/pull/4787 - https://github.com/alcionai/corso/pull/4909 #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [x] 🌻 Feature #### Issue(s) #4754 #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
Hitesh Pattanayak
GitHub
parent
00662c4cd9
commit
dc47001cba
@ -274,7 +274,6 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
|
||||
ctrl := newController(ctx, suite.T(), path.SharePointService)
|
||||
tests := []struct {
|
||||
name string
|
||||
expected int
|
||||
getSelector func() selectors.Selector
|
||||
}{
|
||||
{
|
||||
@ -286,8 +285,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Lists",
|
||||
expected: 0,
|
||||
name: "Lists",
|
||||
getSelector: func() selectors.Selector {
|
||||
sel := selectors.NewSharePointBackup(selSites)
|
||||
sel.Include(sel.Lists(selectors.Any()))
|
||||
@ -329,8 +327,8 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
|
||||
}
|
||||
|
||||
// we don't know an exact count of drives this will produce,
|
||||
// but it should be more than one.
|
||||
assert.Less(t, test.expected, len(collections))
|
||||
// but it should be more than zero.
|
||||
assert.NotEmpty(t, collections)
|
||||
|
||||
for _, coll := range collections {
|
||||
for object := range coll.Items(ctx, fault.New(true)) {
|
||||
@ -465,7 +463,8 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
|
||||
assert.True(t, excludes.Empty())
|
||||
|
||||
for _, collection := range cols {
|
||||
t.Logf("Path: %s\n", collection.FullPath().String())
|
||||
assert.Equal(t, path.SharePointService, collection.FullPath().Service())
|
||||
assert.Equal(t, path.ListsCategory, collection.FullPath().Category())
|
||||
|
||||
for item := range collection.Items(ctx, fault.New(true)) {
|
||||
t.Log("File: " + item.ID())
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/m365/collection/drive"
|
||||
betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
|
||||
@ -123,13 +124,14 @@ func CollectPages(
|
||||
}
|
||||
|
||||
collection := NewCollection(
|
||||
nil,
|
||||
dir,
|
||||
ac,
|
||||
scope,
|
||||
su,
|
||||
bpc.Options)
|
||||
collection.SetBetaService(betaService)
|
||||
collection.AddJob(tuple.ID)
|
||||
collection.AddItem(tuple.ID)
|
||||
|
||||
spcs = append(spcs, collection)
|
||||
}
|
||||
@ -139,6 +141,7 @@ func CollectPages(
|
||||
|
||||
func CollectLists(
|
||||
ctx context.Context,
|
||||
bh backupHandler,
|
||||
bpc inject.BackupProducerConfig,
|
||||
ac api.Client,
|
||||
tenantID string,
|
||||
@ -151,14 +154,15 @@ func CollectLists(
|
||||
var (
|
||||
el = errs.Local()
|
||||
spcs = make([]data.BackupCollection, 0)
|
||||
acc = api.CallConfig{Select: idAnd("displayName")}
|
||||
)
|
||||
|
||||
lists, err := PreFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID())
|
||||
lists, err := bh.GetItems(ctx, acc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, tuple := range lists {
|
||||
for _, list := range lists {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
}
|
||||
@ -169,21 +173,32 @@ func CollectLists(
|
||||
path.SharePointService,
|
||||
path.ListsCategory,
|
||||
false,
|
||||
tuple.Name)
|
||||
ptr.Val(list.GetId()))
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating list collection path"))
|
||||
}
|
||||
|
||||
collection := NewCollection(
|
||||
bh,
|
||||
dir,
|
||||
ac,
|
||||
scope,
|
||||
su,
|
||||
bpc.Options)
|
||||
collection.AddJob(tuple.ID)
|
||||
collection.AddItem(ptr.Val(list.GetId()))
|
||||
|
||||
spcs = append(spcs, collection)
|
||||
}
|
||||
|
||||
return spcs, el.Failure()
|
||||
}
|
||||
|
||||
func idAnd(ss ...string) []string {
|
||||
id := []string{"id"}
|
||||
|
||||
if len(ss) == 0 {
|
||||
return id
|
||||
}
|
||||
|
||||
return append(id, ss...)
|
||||
}
|
||||
|
||||
@ -21,26 +21,26 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
)
|
||||
|
||||
type SharePointPagesSuite struct {
|
||||
type SharePointSuite struct {
|
||||
tester.Suite
|
||||
}
|
||||
|
||||
func TestSharePointPagesSuite(t *testing.T) {
|
||||
suite.Run(t, &SharePointPagesSuite{
|
||||
func TestSharePointSuite(t *testing.T) {
|
||||
suite.Run(t, &SharePointSuite{
|
||||
Suite: tester.NewIntegrationSuite(
|
||||
t,
|
||||
[][]string{tconfig.M365AcctCredEnvs}),
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *SharePointPagesSuite) SetupSuite() {
|
||||
func (suite *SharePointSuite) SetupSuite() {
|
||||
ctx, flush := tester.NewContext(suite.T())
|
||||
defer flush()
|
||||
|
||||
graph.InitializeConcurrencyLimiter(ctx, false, 4)
|
||||
}
|
||||
|
||||
func (suite *SharePointPagesSuite) TestCollectPages() {
|
||||
func (suite *SharePointSuite) TestCollectPages() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
@ -81,3 +81,47 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
|
||||
assert.NoError(t, err, clues.ToCore(err))
|
||||
assert.NotEmpty(t, col)
|
||||
}
|
||||
|
||||
func (suite *SharePointSuite) TestCollectLists() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
var (
|
||||
siteID = tconfig.M365SiteID(t)
|
||||
a = tconfig.NewM365Account(t)
|
||||
counter = count.New()
|
||||
)
|
||||
|
||||
creds, err := a.M365Config()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
ac, err := api.NewClient(
|
||||
creds,
|
||||
control.DefaultOptions(),
|
||||
counter)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
bpc := inject.BackupProducerConfig{
|
||||
LastBackupVersion: version.NoBackup,
|
||||
Options: control.DefaultOptions(),
|
||||
ProtectedResource: mock.NewProvider(siteID, siteID),
|
||||
}
|
||||
|
||||
sel := selectors.NewSharePointBackup([]string{siteID})
|
||||
|
||||
bh := NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists())
|
||||
|
||||
col, err := CollectLists(
|
||||
ctx,
|
||||
bh,
|
||||
bpc,
|
||||
ac,
|
||||
creds.AzureTenantID,
|
||||
sel.Lists(selectors.Any())[0],
|
||||
(&MockGraphService{}).UpdateStatus,
|
||||
fault.New(true))
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
assert.Less(t, 0, len(col))
|
||||
}
|
||||
|
||||
@ -4,10 +4,12 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoft/kiota-abstractions-go/serialization"
|
||||
kjson "github.com/microsoft/kiota-serialization-json-go"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
@ -41,25 +43,29 @@ const (
|
||||
|
||||
var _ data.BackupCollection = &Collection{}
|
||||
|
||||
// Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported
|
||||
// by the oneDrive.Collection as the calls are identical for populating the Collection
|
||||
// Collection is the SharePoint.List or SharePoint.Page implementation of data.Collection.
|
||||
|
||||
// SharePoint.Libraries collections are supported by the oneDrive.Collection
|
||||
// as the calls are identical for populating the Collection
|
||||
type Collection struct {
|
||||
// data is the container for each individual SharePoint.List
|
||||
data chan data.Item
|
||||
// stream is the container for each individual SharePoint item of (page/list)
|
||||
stream chan data.Item
|
||||
// fullPath indicates the hierarchy within the collection
|
||||
fullPath path.Path
|
||||
// jobs contain the SharePoint.Site.ListIDs for the associated list(s).
|
||||
jobs []string
|
||||
// jobs contain the SharePoint.List.IDs or SharePoint.Page.IDs
|
||||
items []string
|
||||
// M365 IDs of the items of this collection
|
||||
category path.CategoryType
|
||||
client api.Sites
|
||||
ctrl control.Options
|
||||
betaService *betaAPI.BetaService
|
||||
statusUpdater support.StatusUpdater
|
||||
getter getItemByIDer
|
||||
}
|
||||
|
||||
// NewCollection helper function for creating a Collection
|
||||
func NewCollection(
|
||||
getter getItemByIDer,
|
||||
folderPath path.Path,
|
||||
ac api.Client,
|
||||
scope selectors.SharePointScope,
|
||||
@ -68,8 +74,9 @@ func NewCollection(
|
||||
) *Collection {
|
||||
c := &Collection{
|
||||
fullPath: folderPath,
|
||||
jobs: make([]string, 0),
|
||||
data: make(chan data.Item, collectionChannelBufferSize),
|
||||
items: make([]string, 0),
|
||||
getter: getter,
|
||||
stream: make(chan data.Item, collectionChannelBufferSize),
|
||||
client: ac.Sites(),
|
||||
statusUpdater: statusUpdater,
|
||||
category: scope.Category().PathType(),
|
||||
@ -83,9 +90,9 @@ func (sc *Collection) SetBetaService(betaService *betaAPI.BetaService) {
|
||||
sc.betaService = betaService
|
||||
}
|
||||
|
||||
// AddJob appends additional objectID to job field
|
||||
func (sc *Collection) AddJob(objID string) {
|
||||
sc.jobs = append(sc.jobs, objID)
|
||||
// AddItem appends additional itemID to items field
|
||||
func (sc *Collection) AddItem(itemID string) {
|
||||
sc.items = append(sc.items, itemID)
|
||||
}
|
||||
|
||||
func (sc *Collection) FullPath() path.Path {
|
||||
@ -98,6 +105,10 @@ func (sc Collection) PreviousPath() path.Path {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sc Collection) LocationPath() *path.Builder {
|
||||
return path.Builder{}.Append(sc.fullPath.Folders()...)
|
||||
}
|
||||
|
||||
func (sc Collection) State() data.CollectionState {
|
||||
return data.NewState
|
||||
}
|
||||
@ -110,21 +121,21 @@ func (sc *Collection) Items(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
) <-chan data.Item {
|
||||
go sc.populate(ctx, errs)
|
||||
return sc.data
|
||||
go sc.streamItems(ctx, errs)
|
||||
return sc.stream
|
||||
}
|
||||
|
||||
func (sc *Collection) finishPopulation(
|
||||
ctx context.Context,
|
||||
metrics support.CollectionMetrics,
|
||||
metrics *support.CollectionMetrics,
|
||||
) {
|
||||
close(sc.data)
|
||||
close(sc.stream)
|
||||
|
||||
status := support.CreateStatus(
|
||||
ctx,
|
||||
support.Backup,
|
||||
1, // 1 folder
|
||||
metrics,
|
||||
*metrics,
|
||||
sc.fullPath.Folder(false))
|
||||
|
||||
logger.Ctx(ctx).Debug(status.String())
|
||||
@ -134,128 +145,98 @@ func (sc *Collection) finishPopulation(
|
||||
}
|
||||
}
|
||||
|
||||
// populate utility function to retrieve data from back store for a given collection
|
||||
func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) {
|
||||
metrics, _ := sc.runPopulate(ctx, errs)
|
||||
sc.finishPopulation(ctx, metrics)
|
||||
}
|
||||
|
||||
func (sc *Collection) runPopulate(
|
||||
// streamItems utility function to retrieve data from back store for a given collection
|
||||
func (sc *Collection) streamItems(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
) (support.CollectionMetrics, error) {
|
||||
var (
|
||||
err error
|
||||
metrics support.CollectionMetrics
|
||||
writer = kjson.NewJsonSerializationWriter()
|
||||
)
|
||||
|
||||
// TODO: Insert correct ID for CollectionProgress
|
||||
colProgress := observe.CollectionProgress(
|
||||
ctx,
|
||||
sc.fullPath.Category().HumanString(),
|
||||
sc.fullPath.Folders())
|
||||
defer close(colProgress)
|
||||
|
||||
) {
|
||||
// Switch retrieval function based on category
|
||||
switch sc.category {
|
||||
case path.ListsCategory:
|
||||
metrics, err = sc.retrieveLists(ctx, writer, colProgress, errs)
|
||||
sc.streamLists(ctx, errs)
|
||||
case path.PagesCategory:
|
||||
metrics, err = sc.retrievePages(ctx, sc.client, writer, colProgress, errs)
|
||||
sc.retrievePages(ctx, sc.client, errs)
|
||||
}
|
||||
|
||||
return metrics, err
|
||||
}
|
||||
|
||||
// retrieveLists utility function for collection that downloads and serializes
|
||||
// streamLists utility function for collection that downloads and serializes
|
||||
// models.Listable objects based on M365 IDs from the jobs field.
|
||||
func (sc *Collection) retrieveLists(
|
||||
func (sc *Collection) streamLists(
|
||||
ctx context.Context,
|
||||
wtr *kjson.JsonSerializationWriter,
|
||||
progress chan<- struct{},
|
||||
errs *fault.Bus,
|
||||
) (support.CollectionMetrics, error) {
|
||||
) {
|
||||
var (
|
||||
metrics support.CollectionMetrics
|
||||
el = errs.Local()
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
lists, err := loadSiteLists(
|
||||
ctx,
|
||||
sc.client.Stable,
|
||||
sc.fullPath.ProtectedResource(),
|
||||
sc.jobs,
|
||||
errs)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
}
|
||||
defer sc.finishPopulation(ctx, &metrics)
|
||||
|
||||
// TODO: Insert correct ID for CollectionProgress
|
||||
progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders())
|
||||
defer close(progress)
|
||||
|
||||
semaphoreCh := make(chan struct{}, fetchChannelSize)
|
||||
defer close(semaphoreCh)
|
||||
|
||||
metrics.Objects += len(lists)
|
||||
// For each models.Listable, object is serialized and the metrics are collected.
|
||||
// The progress is objected via the passed in channel.
|
||||
for _, lst := range lists {
|
||||
for _, listID := range sc.items {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
byteArray, err := serializeContent(ctx, wtr, lst)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation))
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
semaphoreCh <- struct{}{}
|
||||
|
||||
size := int64(len(byteArray))
|
||||
sc.handleListItems(ctx, semaphoreCh, progress, listID, el, &metrics)
|
||||
|
||||
if size > 0 {
|
||||
metrics.Bytes += size
|
||||
|
||||
metrics.Successes++
|
||||
|
||||
item, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
ptr.Val(lst.GetId()),
|
||||
details.ItemInfo{SharePoint: ListToSPInfo(lst, size)})
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
|
||||
continue
|
||||
}
|
||||
|
||||
sc.data <- item
|
||||
progress <- struct{}{}
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
return metrics, el.Failure()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (sc *Collection) retrievePages(
|
||||
ctx context.Context,
|
||||
as api.Sites,
|
||||
wtr *kjson.JsonSerializationWriter,
|
||||
progress chan<- struct{},
|
||||
errs *fault.Bus,
|
||||
) (support.CollectionMetrics, error) {
|
||||
) {
|
||||
var (
|
||||
metrics support.CollectionMetrics
|
||||
el = errs.Local()
|
||||
)
|
||||
|
||||
defer sc.finishPopulation(ctx, &metrics)
|
||||
|
||||
// TODO: Insert correct ID for CollectionProgress
|
||||
progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders())
|
||||
defer close(progress)
|
||||
|
||||
wtr := kjson.NewJsonSerializationWriter()
|
||||
defer wtr.Close()
|
||||
|
||||
betaService := sc.betaService
|
||||
if betaService == nil {
|
||||
return metrics, clues.NewWC(ctx, "beta service required")
|
||||
logger.Ctx(ctx).Error(clues.New("beta service required"))
|
||||
return
|
||||
}
|
||||
|
||||
parent, err := as.GetByID(ctx, sc.fullPath.ProtectedResource(), api.CallConfig{})
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
logger.Ctx(ctx).Error(err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
root := ptr.Val(parent.GetWebUrl())
|
||||
|
||||
pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.jobs, errs)
|
||||
pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.items, errs)
|
||||
if err != nil {
|
||||
return metrics, err
|
||||
logger.Ctx(ctx).Error(err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
metrics.Objects = len(pages)
|
||||
@ -275,25 +256,25 @@ func (sc *Collection) retrievePages(
|
||||
|
||||
size := int64(len(byteArray))
|
||||
|
||||
if size > 0 {
|
||||
metrics.Bytes += size
|
||||
metrics.Successes++
|
||||
|
||||
item, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
ptr.Val(pg.GetId()),
|
||||
details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)})
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
|
||||
continue
|
||||
}
|
||||
|
||||
sc.data <- item
|
||||
progress <- struct{}{}
|
||||
if size == 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return metrics, el.Failure()
|
||||
metrics.Bytes += size
|
||||
metrics.Successes++
|
||||
|
||||
item, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
ptr.Val(pg.GetId()),
|
||||
details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)})
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
|
||||
continue
|
||||
}
|
||||
|
||||
sc.stream <- item
|
||||
progress <- struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func serializeContent(
|
||||
@ -315,3 +296,79 @@ func serializeContent(
|
||||
|
||||
return byteArray, nil
|
||||
}
|
||||
|
||||
func (sc *Collection) handleListItems(
|
||||
ctx context.Context,
|
||||
semaphoreCh chan struct{},
|
||||
progress chan<- struct{},
|
||||
listID string,
|
||||
el *fault.Bus,
|
||||
metrics *support.CollectionMetrics,
|
||||
) {
|
||||
defer func() { <-semaphoreCh }()
|
||||
|
||||
writer := kjson.NewJsonSerializationWriter()
|
||||
defer writer.Close()
|
||||
|
||||
var (
|
||||
list models.Listable
|
||||
info *details.SharePointInfo
|
||||
err error
|
||||
)
|
||||
|
||||
list, info, err = sc.getter.GetItemByID(ctx, listID)
|
||||
if err != nil {
|
||||
err = clues.WrapWC(ctx, err, "getting list data").Label(fault.LabelForceNoBackupCreation)
|
||||
el.AddRecoverable(ctx, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
metrics.Objects++
|
||||
|
||||
if err := writer.WriteObjectValue("", list); err != nil {
|
||||
err = clues.WrapWC(ctx, err, "writing list to serializer").Label(fault.LabelForceNoBackupCreation)
|
||||
el.AddRecoverable(ctx, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
entryBytes, err := writer.GetSerializedContent()
|
||||
if err != nil {
|
||||
err = clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation)
|
||||
el.AddRecoverable(ctx, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
size := int64(len(entryBytes))
|
||||
|
||||
if size == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
metrics.Bytes += size
|
||||
metrics.Successes++
|
||||
|
||||
template := ""
|
||||
if list != nil && list.GetList() != nil {
|
||||
template = ptr.Val(list.GetList().GetTemplate())
|
||||
}
|
||||
|
||||
rc := io.NopCloser(bytes.NewReader(entryBytes))
|
||||
itemInfo := details.ItemInfo{
|
||||
SharePoint: info,
|
||||
NotRecoverable: template == api.WebTemplateExtensionsListTemplateName,
|
||||
}
|
||||
|
||||
item, err := data.NewPrefetchedItemWithInfo(rc, listID, itemInfo)
|
||||
if err != nil {
|
||||
err = clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation)
|
||||
el.AddRecoverable(ctx, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
sc.stream <- item
|
||||
progress <- struct{}{}
|
||||
}
|
||||
|
||||
@ -7,21 +7,22 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
kioser "github.com/microsoft/kiota-serialization-json-go"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/m365/collection/site/mock"
|
||||
betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
|
||||
spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
|
||||
"github.com/alcionai/corso/src/internal/m365/support"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/control/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
@ -76,7 +77,9 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
|
||||
tables := []struct {
|
||||
name, itemName string
|
||||
notRecoverable bool
|
||||
scope selectors.SharePointScope
|
||||
getter getItemByIDer
|
||||
getDir func(t *testing.T) path.Path
|
||||
getItem func(t *testing.T, itemName string) data.Item
|
||||
}{
|
||||
@ -84,6 +87,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
name: "List",
|
||||
itemName: "MockListing",
|
||||
scope: sel.Lists(selectors.Any())[0],
|
||||
getter: &mock.ListHandler{},
|
||||
getDir: func(t *testing.T) path.Path {
|
||||
dir, err := path.Build(
|
||||
tenant,
|
||||
@ -107,10 +111,61 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
byteArray, err := ow.GetSerializedContent()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
info := &details.SharePointInfo{
|
||||
ItemName: name,
|
||||
}
|
||||
|
||||
data, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
name,
|
||||
details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))})
|
||||
details.ItemInfo{SharePoint: info})
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
return data
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "List with wte template",
|
||||
itemName: "MockListing",
|
||||
notRecoverable: true,
|
||||
scope: sel.Lists(selectors.Any())[0],
|
||||
getter: &mock.ListHandler{},
|
||||
getDir: func(t *testing.T) path.Path {
|
||||
dir, err := path.Build(
|
||||
tenant,
|
||||
user,
|
||||
path.SharePointService,
|
||||
path.ListsCategory,
|
||||
false,
|
||||
dirRoot)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
return dir
|
||||
},
|
||||
getItem: func(t *testing.T, name string) data.Item {
|
||||
ow := kioser.NewJsonSerializationWriter()
|
||||
|
||||
listInfo := models.NewListInfo()
|
||||
listInfo.SetTemplate(ptr.To("webTemplateExtensionsList"))
|
||||
|
||||
listing := spMock.ListDefault(name)
|
||||
listing.SetDisplayName(&name)
|
||||
listing.SetList(listInfo)
|
||||
|
||||
err := ow.WriteObjectValue("", listing)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
byteArray, err := ow.GetSerializedContent()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
info := &details.SharePointInfo{
|
||||
ItemName: name,
|
||||
}
|
||||
|
||||
data, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
name,
|
||||
details.ItemInfo{SharePoint: info, NotRecoverable: true})
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
return data
|
||||
@ -120,6 +175,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
name: "Pages",
|
||||
itemName: "MockPages",
|
||||
scope: sel.Pages(selectors.Any())[0],
|
||||
getter: nil,
|
||||
getDir: func(t *testing.T) path.Path {
|
||||
dir, err := path.Build(
|
||||
tenant,
|
||||
@ -156,12 +212,13 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
defer flush()
|
||||
|
||||
col := NewCollection(
|
||||
test.getter,
|
||||
test.getDir(t),
|
||||
suite.ac,
|
||||
test.scope,
|
||||
nil,
|
||||
control.DefaultOptions())
|
||||
col.data <- test.getItem(t, test.itemName)
|
||||
col.stream <- test.getItem(t, test.itemName)
|
||||
|
||||
readItems := []data.Item{}
|
||||
|
||||
@ -180,68 +237,100 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
|
||||
assert.NotNil(t, info)
|
||||
assert.NotNil(t, info.SharePoint)
|
||||
assert.Equal(t, test.itemName, info.SharePoint.ItemName)
|
||||
assert.Equal(t, test.notRecoverable, info.NotRecoverable)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRestoreListCollection verifies Graph Restore API for the List Collection
|
||||
func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
|
||||
t := suite.T()
|
||||
// https://github.com/microsoftgraph/msgraph-sdk-go/issues/490
|
||||
t.Skip("disabled until upstream issue with list restore is fixed.")
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
service := createTestService(t, suite.creds)
|
||||
listing := spMock.ListDefault("Mock List")
|
||||
testName := "MockListing"
|
||||
listing.SetDisplayName(&testName)
|
||||
byteArray, err := service.Serialize(listing)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
listData, err := data.NewPrefetchedItemWithInfo(
|
||||
io.NopCloser(bytes.NewReader(byteArray)),
|
||||
testName,
|
||||
details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))})
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
destName := testdata.DefaultRestoreConfig("").Location
|
||||
|
||||
deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
|
||||
assert.NoError(t, err, clues.ToCore(err))
|
||||
t.Logf("List created: %s\n", deets.SharePoint.ItemName)
|
||||
|
||||
// Clean-Up
|
||||
func (suite *SharePointCollectionSuite) TestCollection_streamItems() {
|
||||
var (
|
||||
builder = service.Client().Sites().BySiteId(suite.siteID).Lists()
|
||||
isFound bool
|
||||
deleteID string
|
||||
t = suite.T()
|
||||
statusUpdater = func(*support.ControllerOperationStatus) {}
|
||||
tenant = "some"
|
||||
resource = "siteid"
|
||||
list = "list"
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
assert.NoError(t, err, "getting site lists", clues.ToCore(err))
|
||||
table := []struct {
|
||||
name string
|
||||
category path.CategoryType
|
||||
items []string
|
||||
getDir func(t *testing.T) path.Path
|
||||
}{
|
||||
{
|
||||
name: "no items",
|
||||
items: []string{},
|
||||
category: path.ListsCategory,
|
||||
getDir: func(t *testing.T) path.Path {
|
||||
dir, err := path.Build(
|
||||
tenant,
|
||||
resource,
|
||||
path.SharePointService,
|
||||
path.ListsCategory,
|
||||
false,
|
||||
list)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
for _, temp := range resp.GetValue() {
|
||||
if ptr.Val(temp.GetDisplayName()) == deets.SharePoint.ItemName {
|
||||
isFound = true
|
||||
deleteID = ptr.Val(temp.GetId())
|
||||
return dir
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with items",
|
||||
items: []string{"list1", "list2", "list3"},
|
||||
category: path.ListsCategory,
|
||||
getDir: func(t *testing.T) path.Path {
|
||||
dir, err := path.Build(
|
||||
tenant,
|
||||
resource,
|
||||
path.SharePointService,
|
||||
path.ListsCategory,
|
||||
false,
|
||||
list)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
// Get Next Link
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsRequestBuilder(link, service.Adapter())
|
||||
return dir
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t.Log("running test", test)
|
||||
|
||||
if isFound {
|
||||
err := DeleteList(ctx, service, suite.siteID, deleteID)
|
||||
assert.NoError(t, err, clues.ToCore(err))
|
||||
var (
|
||||
errs = fault.New(true)
|
||||
itemCount int
|
||||
)
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
col := &Collection{
|
||||
fullPath: test.getDir(t),
|
||||
category: test.category,
|
||||
items: test.items,
|
||||
getter: &mock.ListHandler{},
|
||||
stream: make(chan data.Item),
|
||||
statusUpdater: statusUpdater,
|
||||
}
|
||||
|
||||
itemMap := func(js []string) map[string]struct{} {
|
||||
m := make(map[string]struct{})
|
||||
for _, j := range js {
|
||||
m[j] = struct{}{}
|
||||
}
|
||||
return m
|
||||
}(test.items)
|
||||
|
||||
go col.streamItems(ctx, errs)
|
||||
|
||||
for item := range col.stream {
|
||||
itemCount++
|
||||
_, ok := itemMap[item.ID()]
|
||||
assert.True(t, ok, "should fetch item")
|
||||
}
|
||||
|
||||
assert.NoError(t, errs.Failure())
|
||||
assert.Equal(t, len(test.items), itemCount, "should see all expected items")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,16 +1,9 @@
|
||||
package site
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/m365/support"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@ -42,18 +35,3 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
|
||||
|
||||
func (ms *MockGraphService) UpdateStatus(*support.ControllerOperationStatus) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper functions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func createTestService(t *testing.T, credentials account.M365Config) *graph.Service {
|
||||
adapter, err := graph.CreateAdapter(
|
||||
credentials.AzureTenantID,
|
||||
credentials.AzureClientID,
|
||||
credentials.AzureClientSecret,
|
||||
count.New())
|
||||
require.NoError(t, err, "creating microsoft graph service for exchange", clues.ToCore(err))
|
||||
|
||||
return graph.NewService(adapter)
|
||||
}
|
||||
|
||||
@ -1,440 +0,0 @@
|
||||
package site
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/sites"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
)
|
||||
|
||||
// ListToSPInfo translates models.Listable metadata into searchable content
|
||||
// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
|
||||
func ListToSPInfo(lst models.Listable, size int64) *details.SharePointInfo {
|
||||
var (
|
||||
name = ptr.Val(lst.GetDisplayName())
|
||||
webURL = ptr.Val(lst.GetWebUrl())
|
||||
created = ptr.Val(lst.GetCreatedDateTime())
|
||||
modified = ptr.Val(lst.GetLastModifiedDateTime())
|
||||
)
|
||||
|
||||
return &details.SharePointInfo{
|
||||
ItemType: details.SharePointList,
|
||||
ItemName: name,
|
||||
Created: created,
|
||||
Modified: modified,
|
||||
WebURL: webURL,
|
||||
Size: size,
|
||||
}
|
||||
}
|
||||
|
||||
type ListTuple struct {
|
||||
ID string
|
||||
Name string
|
||||
}
|
||||
|
||||
func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration {
|
||||
selecting := []string{"id", "displayName"}
|
||||
queryOptions := sites.ItemListsRequestBuilderGetQueryParameters{
|
||||
Select: selecting,
|
||||
}
|
||||
options := &sites.ItemListsRequestBuilderGetRequestConfiguration{
|
||||
QueryParameters: &queryOptions,
|
||||
}
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
func PreFetchLists(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID string,
|
||||
) ([]ListTuple, error) {
|
||||
var (
|
||||
builder = gs.Client().Sites().BySiteId(siteID).Lists()
|
||||
options = preFetchListOptions()
|
||||
listTuples = make([]ListTuple, 0)
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, options)
|
||||
if err != nil {
|
||||
return nil, graph.Wrap(ctx, err, "getting lists")
|
||||
}
|
||||
|
||||
for _, entry := range resp.GetValue() {
|
||||
var (
|
||||
id = ptr.Val(entry.GetId())
|
||||
name = ptr.Val(entry.GetDisplayName())
|
||||
temp = ListTuple{ID: id, Name: name}
|
||||
)
|
||||
|
||||
if len(name) == 0 {
|
||||
temp.Name = id
|
||||
}
|
||||
|
||||
listTuples = append(listTuples, temp)
|
||||
}
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsRequestBuilder(link, gs.Adapter())
|
||||
}
|
||||
|
||||
return listTuples, nil
|
||||
}
|
||||
|
||||
// list.go contains additional functions to help retrieve SharePoint List data from M365
|
||||
// SharePoint lists represent lists on a site. Inherits additional properties from
|
||||
// baseItem: https://learn.microsoft.com/en-us/graph/api/resources/baseitem?view=graph-rest-1.0
|
||||
// The full details concerning SharePoint Lists can
|
||||
// be found at: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
|
||||
// Note additional calls are required for the relationships that exist outside of the object properties.
|
||||
|
||||
// loadSiteLists is a utility function to populate a collection of SharePoint.List
|
||||
// objects associated with a given siteID.
|
||||
// @param siteID the M365 ID that represents the SharePoint Site
|
||||
// Makes additional calls to retrieve the following relationships:
|
||||
// - Columns
|
||||
// - ContentTypes
|
||||
// - List Items
|
||||
func loadSiteLists(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID string,
|
||||
listIDs []string,
|
||||
errs *fault.Bus,
|
||||
) ([]models.Listable, error) {
|
||||
var (
|
||||
results = make([]models.Listable, 0)
|
||||
semaphoreCh = make(chan struct{}, fetchChannelSize)
|
||||
el = errs.Local()
|
||||
wg sync.WaitGroup
|
||||
m sync.Mutex
|
||||
)
|
||||
|
||||
defer close(semaphoreCh)
|
||||
|
||||
updateLists := func(list models.Listable) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
|
||||
results = append(results, list)
|
||||
}
|
||||
|
||||
for _, listID := range listIDs {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
semaphoreCh <- struct{}{}
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func(id string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-semaphoreCh }()
|
||||
|
||||
var (
|
||||
entry models.Listable
|
||||
err error
|
||||
)
|
||||
|
||||
entry, err = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(id).Get(ctx, nil)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting site list"))
|
||||
return
|
||||
}
|
||||
|
||||
cols, cTypes, lItems, err := fetchListContents(ctx, gs, siteID, id, errs)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.Wrap(err, "getting list contents"))
|
||||
return
|
||||
}
|
||||
|
||||
entry.SetColumns(cols)
|
||||
entry.SetContentTypes(cTypes)
|
||||
entry.SetItems(lItems)
|
||||
updateLists(entry)
|
||||
}(listID)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
return results, el.Failure()
|
||||
}
|
||||
|
||||
// fetchListContents utility function to retrieve associated M365 relationships
|
||||
// which are not included with the standard List query:
|
||||
// - Columns, ContentTypes, ListItems
|
||||
func fetchListContents(
|
||||
ctx context.Context,
|
||||
service graph.Servicer,
|
||||
siteID, listID string,
|
||||
errs *fault.Bus,
|
||||
) (
|
||||
[]models.ColumnDefinitionable,
|
||||
[]models.ContentTypeable,
|
||||
[]models.ListItemable,
|
||||
error,
|
||||
) {
|
||||
cols, err := fetchColumns(ctx, service, siteID, listID, "")
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
cTypes, err := fetchContentTypes(ctx, service, siteID, listID, errs)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
lItems, err := fetchListItems(ctx, service, siteID, listID, errs)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return cols, cTypes, lItems, nil
|
||||
}
|
||||
|
||||
// fetchListItems utility for retrieving ListItem data and the associated relationship
|
||||
// data. Additional call append data to the tracked items, and do not create additional collections.
|
||||
// Additional Call:
|
||||
// * Fields
|
||||
func fetchListItems(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID, listID string,
|
||||
errs *fault.Bus,
|
||||
) ([]models.ListItemable, error) {
|
||||
var (
|
||||
prefix = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID)
|
||||
builder = prefix.Items()
|
||||
itms = make([]models.ListItemable, 0)
|
||||
el = errs.Local()
|
||||
)
|
||||
|
||||
for {
|
||||
if errs.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, itm := range resp.GetValue() {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
newPrefix := prefix.Items().ByListItemId(ptr.Val(itm.GetId()))
|
||||
|
||||
fields, err := newPrefix.Fields().Get(ctx, nil)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting list fields"))
|
||||
continue
|
||||
}
|
||||
|
||||
itm.SetFields(fields)
|
||||
|
||||
itms = append(itms, itm)
|
||||
}
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsItemItemsRequestBuilder(link, gs.Adapter())
|
||||
}
|
||||
|
||||
return itms, el.Failure()
|
||||
}
|
||||
|
||||
// fetchColumns utility function to return columns from a site.
|
||||
// An additional call required to check for details concerning the SourceColumn.
|
||||
// For additional details: https://learn.microsoft.com/en-us/graph/api/resources/columndefinition?view=graph-rest-1.0
|
||||
// TODO: Refactor on if/else (dadams39)
|
||||
func fetchColumns(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID, listID, cTypeID string,
|
||||
) ([]models.ColumnDefinitionable, error) {
|
||||
cs := make([]models.ColumnDefinitionable, 0)
|
||||
|
||||
if len(cTypeID) == 0 {
|
||||
builder := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Columns()
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, graph.Wrap(ctx, err, "getting list columns")
|
||||
}
|
||||
|
||||
cs = append(cs, resp.GetValue()...)
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsItemColumnsRequestBuilder(link, gs.Adapter())
|
||||
}
|
||||
} else {
|
||||
builder := gs.Client().
|
||||
Sites().
|
||||
BySiteId(siteID).
|
||||
Lists().
|
||||
ByListId(listID).
|
||||
ContentTypes().
|
||||
ByContentTypeId(cTypeID).
|
||||
Columns()
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, graph.Wrap(ctx, err, "getting content columns")
|
||||
}
|
||||
|
||||
cs = append(cs, resp.GetValue()...)
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsItemContentTypesItemColumnsRequestBuilder(link, gs.Adapter())
|
||||
}
|
||||
}
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
||||
// fetchContentTypes retrieves all data for content type. Additional queries required
|
||||
// for the following:
|
||||
// - ColumnLinks
|
||||
// - Columns
|
||||
// Expand queries not used to retrieve the above. Possibly more than 20.
|
||||
// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
|
||||
func fetchContentTypes(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID, listID string,
|
||||
errs *fault.Bus,
|
||||
) ([]models.ContentTypeable, error) {
|
||||
var (
|
||||
el = errs.Local()
|
||||
cTypes = make([]models.ContentTypeable, 0)
|
||||
builder = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).ContentTypes()
|
||||
)
|
||||
|
||||
for {
|
||||
if errs.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, cont := range resp.GetValue() {
|
||||
if el.Failure() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
id := ptr.Val(cont.GetId())
|
||||
|
||||
links, err := fetchColumnLinks(ctx, gs, siteID, listID, id)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, err)
|
||||
continue
|
||||
}
|
||||
|
||||
cont.SetColumnLinks(links)
|
||||
|
||||
cs, err := fetchColumns(ctx, gs, siteID, listID, id)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, err)
|
||||
continue
|
||||
}
|
||||
|
||||
cont.SetColumns(cs)
|
||||
|
||||
cTypes = append(cTypes, cont)
|
||||
}
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsItemContentTypesRequestBuilder(link, gs.Adapter())
|
||||
}
|
||||
|
||||
return cTypes, el.Failure()
|
||||
}
|
||||
|
||||
func fetchColumnLinks(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID, listID, cTypeID string,
|
||||
) ([]models.ColumnLinkable, error) {
|
||||
var (
|
||||
builder = gs.Client().
|
||||
Sites().
|
||||
BySiteId(siteID).
|
||||
Lists().
|
||||
ByListId(listID).
|
||||
ContentTypes().
|
||||
ByContentTypeId(cTypeID).
|
||||
ColumnLinks()
|
||||
links = make([]models.ColumnLinkable, 0)
|
||||
)
|
||||
|
||||
for {
|
||||
resp, err := builder.Get(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, graph.Wrap(ctx, err, "getting column links")
|
||||
}
|
||||
|
||||
links = append(links, resp.GetValue()...)
|
||||
|
||||
link, ok := ptr.ValOK(resp.GetOdataNextLink())
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
builder = sites.NewItemListsItemContentTypesItemColumnLinksRequestBuilder(
|
||||
link,
|
||||
gs.Adapter())
|
||||
}
|
||||
|
||||
return links, nil
|
||||
}
|
||||
|
||||
// DeleteList removes a list object from a site.
|
||||
// deletes require unique http clients
|
||||
// https://github.com/alcionai/corso/issues/2707
|
||||
func DeleteList(
|
||||
ctx context.Context,
|
||||
gs graph.Servicer,
|
||||
siteID, listID string,
|
||||
) error {
|
||||
err := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Delete(ctx, nil)
|
||||
if err != nil {
|
||||
return graph.Wrap(ctx, err, "deleting list")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -1,112 +0,0 @@
|
||||
package site
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||
)
|
||||
|
||||
type ListsUnitSuite struct {
|
||||
tester.Suite
|
||||
creds account.M365Config
|
||||
}
|
||||
|
||||
func (suite *ListsUnitSuite) SetupSuite() {
|
||||
t := suite.T()
|
||||
a := tconfig.NewM365Account(t)
|
||||
m365, err := a.M365Config()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
suite.creds = m365
|
||||
|
||||
ctx, flush := tester.NewContext(suite.T())
|
||||
defer flush()
|
||||
|
||||
graph.InitializeConcurrencyLimiter(ctx, false, 4)
|
||||
}
|
||||
|
||||
func TestListsUnitSuite(t *testing.T) {
|
||||
suite.Run(t, &ListsUnitSuite{
|
||||
Suite: tester.NewIntegrationSuite(
|
||||
t,
|
||||
[][]string{tconfig.M365AcctCredEnvs}),
|
||||
})
|
||||
}
|
||||
|
||||
// Test LoadList --> Retrieves all data from backStore
|
||||
// Functions tested:
|
||||
// - fetchListItems()
|
||||
// - fetchColumns()
|
||||
// - fetchContentColumns()
|
||||
// - fetchContentTypes()
|
||||
// - fetchColumnLinks
|
||||
// TODO: upgrade passed github.com/microsoftgraph/msgraph-sdk-go v0.40.0
|
||||
// to verify if these 2 calls are valid
|
||||
// - fetchContentBaseTypes
|
||||
// - fetchColumnPositions
|
||||
func (suite *ListsUnitSuite) TestLoadList() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
service := createTestService(t, suite.creds)
|
||||
tuples, err := PreFetchLists(ctx, service, "root")
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
job := []string{tuples[0].ID}
|
||||
lists, err := loadSiteLists(ctx, service, "root", job, fault.New(true))
|
||||
assert.NoError(t, err, clues.ToCore(err))
|
||||
assert.Greater(t, len(lists), 0)
|
||||
t.Logf("Length: %d\n", len(lists))
|
||||
}
|
||||
|
||||
func (suite *ListsUnitSuite) TestSharePointInfo() {
|
||||
tests := []struct {
|
||||
name string
|
||||
listAndDeets func() (models.Listable, *details.SharePointInfo)
|
||||
}{
|
||||
{
|
||||
name: "Empty List",
|
||||
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
|
||||
i := &details.SharePointInfo{ItemType: details.SharePointList}
|
||||
return models.NewList(), i
|
||||
},
|
||||
}, {
|
||||
name: "Only Name",
|
||||
listAndDeets: func() (models.Listable, *details.SharePointInfo) {
|
||||
aTitle := "Whole List"
|
||||
listing := models.NewList()
|
||||
listing.SetDisplayName(&aTitle)
|
||||
i := &details.SharePointInfo{
|
||||
ItemType: details.SharePointList,
|
||||
ItemName: aTitle,
|
||||
}
|
||||
|
||||
return listing, i
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
list, expected := test.listAndDeets()
|
||||
info := ListToSPInfo(list, 10)
|
||||
assert.Equal(t, expected.ItemType, info.ItemType)
|
||||
assert.Equal(t, expected.ItemName, info.ItemName)
|
||||
assert.Equal(t, expected.WebURL, info.WebURL)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -200,7 +200,7 @@ func restoreListItem(
|
||||
}
|
||||
}
|
||||
|
||||
dii.SharePoint = ListToSPInfo(restoredList, int64(len(byteArray)))
|
||||
dii.SharePoint = api.ListToSPInfo(restoredList)
|
||||
|
||||
return dii, nil
|
||||
}
|
||||
|
||||
@ -55,8 +55,11 @@ func ProduceBackupCollections(
|
||||
|
||||
switch scope.Category().PathType() {
|
||||
case path.ListsCategory:
|
||||
bh := site.NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists())
|
||||
|
||||
spcs, err = site.CollectLists(
|
||||
ctx,
|
||||
bh,
|
||||
bpc,
|
||||
ac,
|
||||
creds.AzureTenantID,
|
||||
|
||||
@ -18,6 +18,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/tester/tconfig"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
@ -61,6 +62,67 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() {
|
||||
sel.Selector)
|
||||
}
|
||||
|
||||
func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointList() {
|
||||
t := suite.T()
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
var (
|
||||
resourceID = suite.its.Site.ID
|
||||
sel = selectors.NewSharePointBackup([]string{resourceID})
|
||||
tenID = tconfig.M365TenantID(t)
|
||||
mb = evmock.NewBus()
|
||||
counter = count.New()
|
||||
ws = deeTD.CategoryFromRepoRef
|
||||
)
|
||||
|
||||
sel.Include(selTD.SharePointBackupListsScope(sel))
|
||||
|
||||
bo, bod := PrepNewTestBackupOp(
|
||||
t,
|
||||
ctx,
|
||||
mb,
|
||||
sel.Selector,
|
||||
control.DefaultOptions(),
|
||||
version.Backup,
|
||||
counter)
|
||||
defer bod.Close(t, ctx)
|
||||
|
||||
RunAndCheckBackup(t, ctx, &bo, mb, false)
|
||||
CheckBackupIsInManifests(
|
||||
t,
|
||||
ctx,
|
||||
bod.KW,
|
||||
bod.SW,
|
||||
&bo,
|
||||
bod.Sel,
|
||||
bod.Sel.ID(),
|
||||
path.ListsCategory)
|
||||
|
||||
bID := bo.Results.BackupID
|
||||
|
||||
_, expectDeets := deeTD.GetDeetsInBackup(
|
||||
t,
|
||||
ctx,
|
||||
bID,
|
||||
tenID,
|
||||
bod.Sel.ID(),
|
||||
path.SharePointService,
|
||||
ws,
|
||||
bod.KMS,
|
||||
bod.SSS)
|
||||
deeTD.CheckBackupDetails(
|
||||
t,
|
||||
ctx,
|
||||
bID,
|
||||
ws,
|
||||
bod.KMS,
|
||||
bod.SSS,
|
||||
expectDeets,
|
||||
false)
|
||||
}
|
||||
|
||||
func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
|
||||
runSharePointIncrementalBackupTests(suite, suite.its, control.DefaultOptions())
|
||||
}
|
||||
|
||||
6
src/pkg/selectors/testdata/sharepoint.go
vendored
6
src/pkg/selectors/testdata/sharepoint.go
vendored
@ -4,8 +4,14 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
const TestListName = "test-list"
|
||||
|
||||
// SharePointBackupFolderScope is the standard folder scope that should be used
|
||||
// in integration backups with sharepoint.
|
||||
func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope {
|
||||
return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch())
|
||||
}
|
||||
|
||||
func SharePointBackupListsScope(sel *selectors.SharePointBackup) []selectors.SharePointScope {
|
||||
return sel.ListItems([]string{TestListName}, selectors.Any())
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user