enables sharepoint to use lists backup handler for lists ops (#4920)

enables sharepoint to use lists backup handler for lists ops
Changes previously approved in:
-  https://github.com/alcionai/corso/pull/4786
- https://github.com/alcionai/corso/pull/4787
- https://github.com/alcionai/corso/pull/4909

#### Does this PR need a docs update or release note?
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

#### Issue(s)
#4754 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/m365/backup_test.go

@@ -274,7 +274,6 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
 	ctrl := newController(ctx, suite.T(), path.SharePointService)
 	tests := []struct {
 		name        string
-		expected    int
 		getSelector func() selectors.Selector
 	}{
 		{
@@ -286,8 +285,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
 			},
 		},
 		{
-			name:     "Lists",
+			name: "Lists",
-			expected: 0,
 			getSelector: func() selectors.Selector {
 				sel := selectors.NewSharePointBackup(selSites)
 				sel.Include(sel.Lists(selectors.Any()))
@@ -329,8 +327,8 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() {
 			}
 
 			// we don't know an exact count of drives this will produce,
-			// but it should be more than one.
+			// but it should be more than zero.
-			assert.Less(t, test.expected, len(collections))
+			assert.NotEmpty(t, collections)
 
 			for _, coll := range collections {
 				for object := range coll.Items(ctx, fault.New(true)) {
@@ -465,7 +463,8 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {
 	assert.True(t, excludes.Empty())
 
 	for _, collection := range cols {
-		t.Logf("Path: %s\n", collection.FullPath().String())
+		assert.Equal(t, path.SharePointService, collection.FullPath().Service())
+		assert.Equal(t, path.ListsCategory, collection.FullPath().Category())
 
 		for item := range collection.Items(ctx, fault.New(true)) {
 			t.Log("File: " + item.ID())

src/internal/m365/collection/site/backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/alcionai/clues"
 
 	"github.com/alcionai/corso/src/internal/common/prefixmatcher"
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/internal/m365/collection/drive"
 	betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
@@ -123,13 +124,14 @@ func CollectPages(
 		}
 
 		collection := NewCollection(
+			nil,
 			dir,
 			ac,
 			scope,
 			su,
 			bpc.Options)
 		collection.SetBetaService(betaService)
-		collection.AddJob(tuple.ID)
+		collection.AddItem(tuple.ID)
 
 		spcs = append(spcs, collection)
 	}
@@ -139,6 +141,7 @@ func CollectPages(
 
 func CollectLists(
 	ctx context.Context,
+	bh backupHandler,
 	bpc inject.BackupProducerConfig,
 	ac api.Client,
 	tenantID string,
@@ -151,14 +154,15 @@ func CollectLists(
 	var (
 		el   = errs.Local()
 		spcs = make([]data.BackupCollection, 0)
+		acc  = api.CallConfig{Select: idAnd("displayName")}
 	)
 
-	lists, err := PreFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID())
+	lists, err := bh.GetItems(ctx, acc)
 	if err != nil {
 		return nil, err
 	}
 
-	for _, tuple := range lists {
+	for _, list := range lists {
 		if el.Failure() != nil {
 			break
 		}
@@ -169,21 +173,32 @@ func CollectLists(
 			path.SharePointService,
 			path.ListsCategory,
 			false,
-			tuple.Name)
+			ptr.Val(list.GetId()))
 		if err != nil {
 			el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating list collection path"))
 		}
 
 		collection := NewCollection(
+			bh,
 			dir,
 			ac,
 			scope,
 			su,
 			bpc.Options)
-		collection.AddJob(tuple.ID)
+		collection.AddItem(ptr.Val(list.GetId()))
 
 		spcs = append(spcs, collection)
 	}
 
 	return spcs, el.Failure()
 }
+
+func idAnd(ss ...string) []string {
+	id := []string{"id"}
+
+	if len(ss) == 0 {
+		return id
+	}
+
+	return append(id, ss...)
+}

src/internal/m365/collection/site/backup_test.go

@@ -21,26 +21,26 @@ import (
 	"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
 )
 
-type SharePointPagesSuite struct {
+type SharePointSuite struct {
 	tester.Suite
 }
 
-func TestSharePointPagesSuite(t *testing.T) {
+func TestSharePointSuite(t *testing.T) {
-	suite.Run(t, &SharePointPagesSuite{
+	suite.Run(t, &SharePointSuite{
 		Suite: tester.NewIntegrationSuite(
 			t,
 			[][]string{tconfig.M365AcctCredEnvs}),
 	})
 }
 
-func (suite *SharePointPagesSuite) SetupSuite() {
+func (suite *SharePointSuite) SetupSuite() {
 	ctx, flush := tester.NewContext(suite.T())
 	defer flush()
 
 	graph.InitializeConcurrencyLimiter(ctx, false, 4)
 }
 
-func (suite *SharePointPagesSuite) TestCollectPages() {
+func (suite *SharePointSuite) TestCollectPages() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
@@ -81,3 +81,47 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
 	assert.NoError(t, err, clues.ToCore(err))
 	assert.NotEmpty(t, col)
 }
+
+func (suite *SharePointSuite) TestCollectLists() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	var (
+		siteID  = tconfig.M365SiteID(t)
+		a       = tconfig.NewM365Account(t)
+		counter = count.New()
+	)
+
+	creds, err := a.M365Config()
+	require.NoError(t, err, clues.ToCore(err))
+
+	ac, err := api.NewClient(
+		creds,
+		control.DefaultOptions(),
+		counter)
+	require.NoError(t, err, clues.ToCore(err))
+
+	bpc := inject.BackupProducerConfig{
+		LastBackupVersion: version.NoBackup,
+		Options:           control.DefaultOptions(),
+		ProtectedResource: mock.NewProvider(siteID, siteID),
+	}
+
+	sel := selectors.NewSharePointBackup([]string{siteID})
+
+	bh := NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists())
+
+	col, err := CollectLists(
+		ctx,
+		bh,
+		bpc,
+		ac,
+		creds.AzureTenantID,
+		sel.Lists(selectors.Any())[0],
+		(&MockGraphService{}).UpdateStatus,
+		fault.New(true))
+	require.NoError(t, err, clues.ToCore(err))
+	assert.Less(t, 0, len(col))
+}

src/internal/m365/collection/site/collection.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"sync"
 
 	"github.com/alcionai/clues"
 	"github.com/microsoft/kiota-abstractions-go/serialization"
 	kjson "github.com/microsoft/kiota-serialization-json-go"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/internal/data"
@@ -41,25 +43,29 @@ const (
 
 var _ data.BackupCollection = &Collection{}
 
-// Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported
+// Collection is the SharePoint.List or SharePoint.Page implementation of data.Collection.
-// by the oneDrive.Collection as the calls are identical for populating the Collection
+
+// SharePoint.Libraries collections are supported by the oneDrive.Collection
+// as the calls are identical for populating the Collection
 type Collection struct {
-	// data is the container for each individual SharePoint.List
+	// stream is the container for each individual SharePoint item of (page/list)
-	data chan data.Item
+	stream chan data.Item
 	// fullPath indicates the hierarchy within the collection
 	fullPath path.Path
-	// jobs contain the SharePoint.Site.ListIDs for the associated list(s).
+	// jobs contain the SharePoint.List.IDs or SharePoint.Page.IDs
-	jobs []string
+	items []string
 	// M365 IDs of the items of this collection
 	category      path.CategoryType
 	client        api.Sites
 	ctrl          control.Options
 	betaService   *betaAPI.BetaService
 	statusUpdater support.StatusUpdater
+	getter        getItemByIDer
 }
 
 // NewCollection helper function for creating a Collection
 func NewCollection(
+	getter getItemByIDer,
 	folderPath path.Path,
 	ac api.Client,
 	scope selectors.SharePointScope,
@@ -68,8 +74,9 @@ func NewCollection(
 ) *Collection {
 	c := &Collection{
 		fullPath:      folderPath,
-		jobs:          make([]string, 0),
+		items:         make([]string, 0),
-		data:          make(chan data.Item, collectionChannelBufferSize),
+		getter:        getter,
+		stream:        make(chan data.Item, collectionChannelBufferSize),
 		client:        ac.Sites(),
 		statusUpdater: statusUpdater,
 		category:      scope.Category().PathType(),
@@ -83,9 +90,9 @@ func (sc *Collection) SetBetaService(betaService *betaAPI.BetaService) {
 	sc.betaService = betaService
 }
 
-// AddJob appends additional objectID to job field
+// AddItem appends additional itemID to items field
-func (sc *Collection) AddJob(objID string) {
+func (sc *Collection) AddItem(itemID string) {
-	sc.jobs = append(sc.jobs, objID)
+	sc.items = append(sc.items, itemID)
 }
 
 func (sc *Collection) FullPath() path.Path {
@@ -98,6 +105,10 @@ func (sc Collection) PreviousPath() path.Path {
 	return nil
 }
 
+func (sc Collection) LocationPath() *path.Builder {
+	return path.Builder{}.Append(sc.fullPath.Folders()...)
+}
+
 func (sc Collection) State() data.CollectionState {
 	return data.NewState
 }
@@ -110,21 +121,21 @@ func (sc *Collection) Items(
 	ctx context.Context,
 	errs *fault.Bus,
 ) <-chan data.Item {
-	go sc.populate(ctx, errs)
+	go sc.streamItems(ctx, errs)
-	return sc.data
+	return sc.stream
 }
 
 func (sc *Collection) finishPopulation(
 	ctx context.Context,
-	metrics support.CollectionMetrics,
+	metrics *support.CollectionMetrics,
 ) {
-	close(sc.data)
+	close(sc.stream)
 
 	status := support.CreateStatus(
 		ctx,
 		support.Backup,
 		1, // 1 folder
-		metrics,
+		*metrics,
 		sc.fullPath.Folder(false))
 
 	logger.Ctx(ctx).Debug(status.String())
@@ -134,128 +145,98 @@ func (sc *Collection) finishPopulation(
 	}
 }
 
-// populate utility function to retrieve data from back store for a given collection
+// streamItems utility function to retrieve data from back store for a given collection
-func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) {
+func (sc *Collection) streamItems(
-	metrics, _ := sc.runPopulate(ctx, errs)
-	sc.finishPopulation(ctx, metrics)
-}
-
-func (sc *Collection) runPopulate(
 	ctx context.Context,
 	errs *fault.Bus,
-) (support.CollectionMetrics, error) {
+) {
-	var (
-		err     error
-		metrics support.CollectionMetrics
-		writer  = kjson.NewJsonSerializationWriter()
-	)
-
-	// TODO: Insert correct ID for CollectionProgress
-	colProgress := observe.CollectionProgress(
-		ctx,
-		sc.fullPath.Category().HumanString(),
-		sc.fullPath.Folders())
-	defer close(colProgress)
-
 	// Switch retrieval function based on category
 	switch sc.category {
 	case path.ListsCategory:
-		metrics, err = sc.retrieveLists(ctx, writer, colProgress, errs)
+		sc.streamLists(ctx, errs)
 	case path.PagesCategory:
-		metrics, err = sc.retrievePages(ctx, sc.client, writer, colProgress, errs)
+		sc.retrievePages(ctx, sc.client, errs)
 	}
-
-	return metrics, err
 }
 
-// retrieveLists utility function for collection that downloads and serializes
+// streamLists utility function for collection that downloads and serializes
 // models.Listable objects based on M365 IDs from the jobs field.
-func (sc *Collection) retrieveLists(
+func (sc *Collection) streamLists(
 	ctx context.Context,
-	wtr *kjson.JsonSerializationWriter,
-	progress chan<- struct{},
 	errs *fault.Bus,
-) (support.CollectionMetrics, error) {
+) {
 	var (
 		metrics support.CollectionMetrics
 		el      = errs.Local()
+		wg      sync.WaitGroup
 	)
 
-	lists, err := loadSiteLists(
+	defer sc.finishPopulation(ctx, &metrics)
-		ctx,
+
-		sc.client.Stable,
+	// TODO: Insert correct ID for CollectionProgress
-		sc.fullPath.ProtectedResource(),
+	progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders())
-		sc.jobs,
+	defer close(progress)
-		errs)
+
-	if err != nil {
+	semaphoreCh := make(chan struct{}, fetchChannelSize)
-		return metrics, err
+	defer close(semaphoreCh)
-	}
 
-	metrics.Objects += len(lists)
 	// For each models.Listable, object is serialized and the metrics are collected.
 	// The progress is objected via the passed in channel.
-	for _, lst := range lists {
+	for _, listID := range sc.items {
 		if el.Failure() != nil {
 			break
 		}
 
-		byteArray, err := serializeContent(ctx, wtr, lst)
+		wg.Add(1)
-		if err != nil {
+		semaphoreCh <- struct{}{}
-			el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation))
-			continue
-		}
 
-		size := int64(len(byteArray))
+		sc.handleListItems(ctx, semaphoreCh, progress, listID, el, &metrics)
 
-		if size > 0 {
+		wg.Done()
-			metrics.Bytes += size
-
-			metrics.Successes++
-
-			item, err := data.NewPrefetchedItemWithInfo(
-				io.NopCloser(bytes.NewReader(byteArray)),
-				ptr.Val(lst.GetId()),
-				details.ItemInfo{SharePoint: ListToSPInfo(lst, size)})
-			if err != nil {
-				el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
-				continue
-			}
-
-			sc.data <- item
-			progress <- struct{}{}
-		}
 	}
 
-	return metrics, el.Failure()
+	wg.Wait()
 }
 
 func (sc *Collection) retrievePages(
 	ctx context.Context,
 	as api.Sites,
-	wtr *kjson.JsonSerializationWriter,
-	progress chan<- struct{},
 	errs *fault.Bus,
-) (support.CollectionMetrics, error) {
+) {
 	var (
 		metrics support.CollectionMetrics
 		el      = errs.Local()
 	)
 
+	defer sc.finishPopulation(ctx, &metrics)
+
+	// TODO: Insert correct ID for CollectionProgress
+	progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders())
+	defer close(progress)
+
+	wtr := kjson.NewJsonSerializationWriter()
+	defer wtr.Close()
+
 	betaService := sc.betaService
 	if betaService == nil {
-		return metrics, clues.NewWC(ctx, "beta service required")
+		logger.Ctx(ctx).Error(clues.New("beta service required"))
+		return
 	}
 
 	parent, err := as.GetByID(ctx, sc.fullPath.ProtectedResource(), api.CallConfig{})
 	if err != nil {
-		return metrics, err
+		logger.Ctx(ctx).Error(err)
+
+		return
 	}
 
 	root := ptr.Val(parent.GetWebUrl())
 
-	pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.jobs, errs)
+	pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.items, errs)
 	if err != nil {
-		return metrics, err
+		logger.Ctx(ctx).Error(err)
+
+		return
 	}
 
 	metrics.Objects = len(pages)
@@ -275,25 +256,25 @@ func (sc *Collection) retrievePages(
 
 		size := int64(len(byteArray))
 
-		if size > 0 {
+		if size == 0 {
-			metrics.Bytes += size
+			return
-			metrics.Successes++
-
-			item, err := data.NewPrefetchedItemWithInfo(
-				io.NopCloser(bytes.NewReader(byteArray)),
-				ptr.Val(pg.GetId()),
-				details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)})
-			if err != nil {
-				el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
-				continue
-			}
-
-			sc.data <- item
-			progress <- struct{}{}
 		}
-	}
 
-	return metrics, el.Failure()
+		metrics.Bytes += size
+		metrics.Successes++
+
+		item, err := data.NewPrefetchedItemWithInfo(
+			io.NopCloser(bytes.NewReader(byteArray)),
+			ptr.Val(pg.GetId()),
+			details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)})
+		if err != nil {
+			el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation))
+			continue
+		}
+
+		sc.stream <- item
+		progress <- struct{}{}
+	}
 }
 
 func serializeContent(
@@ -315,3 +296,79 @@ func serializeContent(
 
 	return byteArray, nil
 }
+
+func (sc *Collection) handleListItems(
+	ctx context.Context,
+	semaphoreCh chan struct{},
+	progress chan<- struct{},
+	listID string,
+	el *fault.Bus,
+	metrics *support.CollectionMetrics,
+) {
+	defer func() { <-semaphoreCh }()
+
+	writer := kjson.NewJsonSerializationWriter()
+	defer writer.Close()
+
+	var (
+		list models.Listable
+		info *details.SharePointInfo
+		err  error
+	)
+
+	list, info, err = sc.getter.GetItemByID(ctx, listID)
+	if err != nil {
+		err = clues.WrapWC(ctx, err, "getting list data").Label(fault.LabelForceNoBackupCreation)
+		el.AddRecoverable(ctx, err)
+
+		return
+	}
+
+	metrics.Objects++
+
+	if err := writer.WriteObjectValue("", list); err != nil {
+		err = clues.WrapWC(ctx, err, "writing list to serializer").Label(fault.LabelForceNoBackupCreation)
+		el.AddRecoverable(ctx, err)
+
+		return
+	}
+
+	entryBytes, err := writer.GetSerializedContent()
+	if err != nil {
+		err = clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation)
+		el.AddRecoverable(ctx, err)
+
+		return
+	}
+
+	size := int64(len(entryBytes))
+
+	if size == 0 {
+		return
+	}
+
+	metrics.Bytes += size
+	metrics.Successes++
+
+	template := ""
+	if list != nil && list.GetList() != nil {
+		template = ptr.Val(list.GetList().GetTemplate())
+	}
+
+	rc := io.NopCloser(bytes.NewReader(entryBytes))
+	itemInfo := details.ItemInfo{
+		SharePoint:     info,
+		NotRecoverable: template == api.WebTemplateExtensionsListTemplateName,
+	}
+
+	item, err := data.NewPrefetchedItemWithInfo(rc, listID, itemInfo)
+	if err != nil {
+		err = clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation)
+		el.AddRecoverable(ctx, err)
+
+		return
+	}
+
+	sc.stream <- item
+	progress <- struct{}{}
+}

src/internal/m365/collection/site/collection_test.go

@@ -7,21 +7,22 @@ import (
 
 	"github.com/alcionai/clues"
 	kioser "github.com/microsoft/kiota-serialization-json-go"
-	"github.com/microsoftgraph/msgraph-sdk-go/sites"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/internal/m365/collection/site/mock"
 	betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api"
 	spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock"
+	"github.com/alcionai/corso/src/internal/m365/support"
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/internal/tester/tconfig"
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
-	"github.com/alcionai/corso/src/pkg/control/testdata"
 	"github.com/alcionai/corso/src/pkg/count"
 	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/path"
@@ -76,7 +77,9 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 
 	tables := []struct {
 		name, itemName string
+		notRecoverable bool
 		scope          selectors.SharePointScope
+		getter         getItemByIDer
 		getDir         func(t *testing.T) path.Path
 		getItem        func(t *testing.T, itemName string) data.Item
 	}{
@@ -84,6 +87,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 			name:     "List",
 			itemName: "MockListing",
 			scope:    sel.Lists(selectors.Any())[0],
+			getter:   &mock.ListHandler{},
 			getDir: func(t *testing.T) path.Path {
 				dir, err := path.Build(
 					tenant,
@@ -107,10 +111,61 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 				byteArray, err := ow.GetSerializedContent()
 				require.NoError(t, err, clues.ToCore(err))
 
+				info := &details.SharePointInfo{
+					ItemName: name,
+				}
+
 				data, err := data.NewPrefetchedItemWithInfo(
 					io.NopCloser(bytes.NewReader(byteArray)),
 					name,
-					details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))})
+					details.ItemInfo{SharePoint: info})
+				require.NoError(t, err, clues.ToCore(err))
+
+				return data
+			},
+		},
+		{
+			name:           "List with wte template",
+			itemName:       "MockListing",
+			notRecoverable: true,
+			scope:          sel.Lists(selectors.Any())[0],
+			getter:         &mock.ListHandler{},
+			getDir: func(t *testing.T) path.Path {
+				dir, err := path.Build(
+					tenant,
+					user,
+					path.SharePointService,
+					path.ListsCategory,
+					false,
+					dirRoot)
+				require.NoError(t, err, clues.ToCore(err))
+
+				return dir
+			},
+			getItem: func(t *testing.T, name string) data.Item {
+				ow := kioser.NewJsonSerializationWriter()
+
+				listInfo := models.NewListInfo()
+				listInfo.SetTemplate(ptr.To("webTemplateExtensionsList"))
+
+				listing := spMock.ListDefault(name)
+				listing.SetDisplayName(&name)
+				listing.SetList(listInfo)
+
+				err := ow.WriteObjectValue("", listing)
+				require.NoError(t, err, clues.ToCore(err))
+
+				byteArray, err := ow.GetSerializedContent()
+				require.NoError(t, err, clues.ToCore(err))
+
+				info := &details.SharePointInfo{
+					ItemName: name,
+				}
+
+				data, err := data.NewPrefetchedItemWithInfo(
+					io.NopCloser(bytes.NewReader(byteArray)),
+					name,
+					details.ItemInfo{SharePoint: info, NotRecoverable: true})
 				require.NoError(t, err, clues.ToCore(err))
 
 				return data
@@ -120,6 +175,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 			name:     "Pages",
 			itemName: "MockPages",
 			scope:    sel.Pages(selectors.Any())[0],
+			getter:   nil,
 			getDir: func(t *testing.T) path.Path {
 				dir, err := path.Build(
 					tenant,
@@ -156,12 +212,13 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 			defer flush()
 
 			col := NewCollection(
+				test.getter,
 				test.getDir(t),
 				suite.ac,
 				test.scope,
 				nil,
 				control.DefaultOptions())
-			col.data <- test.getItem(t, test.itemName)
+			col.stream <- test.getItem(t, test.itemName)
 
 			readItems := []data.Item{}
 
@@ -180,68 +237,100 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
 			assert.NotNil(t, info)
 			assert.NotNil(t, info.SharePoint)
 			assert.Equal(t, test.itemName, info.SharePoint.ItemName)
+			assert.Equal(t, test.notRecoverable, info.NotRecoverable)
 		})
 	}
 }
 
-// TestRestoreListCollection verifies Graph Restore API for the List Collection
+func (suite *SharePointCollectionSuite) TestCollection_streamItems() {
-func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
-	t := suite.T()
-	// https://github.com/microsoftgraph/msgraph-sdk-go/issues/490
-	t.Skip("disabled until upstream issue with list restore is fixed.")
-
-	ctx, flush := tester.NewContext(t)
-	defer flush()
-
-	service := createTestService(t, suite.creds)
-	listing := spMock.ListDefault("Mock List")
-	testName := "MockListing"
-	listing.SetDisplayName(&testName)
-	byteArray, err := service.Serialize(listing)
-	require.NoError(t, err, clues.ToCore(err))
-
-	listData, err := data.NewPrefetchedItemWithInfo(
-		io.NopCloser(bytes.NewReader(byteArray)),
-		testName,
-		details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))})
-	require.NoError(t, err, clues.ToCore(err))
-
-	destName := testdata.DefaultRestoreConfig("").Location
-
-	deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
-	assert.NoError(t, err, clues.ToCore(err))
-	t.Logf("List created: %s\n", deets.SharePoint.ItemName)
-
-	// Clean-Up
 	var (
-		builder  = service.Client().Sites().BySiteId(suite.siteID).Lists()
+		t             = suite.T()
-		isFound  bool
+		statusUpdater = func(*support.ControllerOperationStatus) {}
-		deleteID string
+		tenant        = "some"
+		resource      = "siteid"
+		list          = "list"
 	)
 
-	for {
+	table := []struct {
-		resp, err := builder.Get(ctx, nil)
+		name     string
-		assert.NoError(t, err, "getting site lists", clues.ToCore(err))
+		category path.CategoryType
+		items    []string
+		getDir   func(t *testing.T) path.Path
+	}{
+		{
+			name:     "no items",
+			items:    []string{},
+			category: path.ListsCategory,
+			getDir: func(t *testing.T) path.Path {
+				dir, err := path.Build(
+					tenant,
+					resource,
+					path.SharePointService,
+					path.ListsCategory,
+					false,
+					list)
+				require.NoError(t, err, clues.ToCore(err))
 
-		for _, temp := range resp.GetValue() {
+				return dir
-			if ptr.Val(temp.GetDisplayName()) == deets.SharePoint.ItemName {
+			},
-				isFound = true
+		},
-				deleteID = ptr.Val(temp.GetId())
+		{
+			name:     "with items",
+			items:    []string{"list1", "list2", "list3"},
+			category: path.ListsCategory,
+			getDir: func(t *testing.T) path.Path {
+				dir, err := path.Build(
+					tenant,
+					resource,
+					path.SharePointService,
+					path.ListsCategory,
+					false,
+					list)
+				require.NoError(t, err, clues.ToCore(err))
 
-				break
+				return dir
-			}
+			},
-		}
+		},
-		// Get Next Link
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsRequestBuilder(link, service.Adapter())
 	}
+	for _, test := range table {
+		suite.Run(test.name, func() {
+			t.Log("running test", test)
 
-	if isFound {
+			var (
-		err := DeleteList(ctx, service, suite.siteID, deleteID)
+				errs      = fault.New(true)
-		assert.NoError(t, err, clues.ToCore(err))
+				itemCount int
+			)
+
+			ctx, flush := tester.NewContext(t)
+			defer flush()
+
+			col := &Collection{
+				fullPath:      test.getDir(t),
+				category:      test.category,
+				items:         test.items,
+				getter:        &mock.ListHandler{},
+				stream:        make(chan data.Item),
+				statusUpdater: statusUpdater,
+			}
+
+			itemMap := func(js []string) map[string]struct{} {
+				m := make(map[string]struct{})
+				for _, j := range js {
+					m[j] = struct{}{}
+				}
+				return m
+			}(test.items)
+
+			go col.streamItems(ctx, errs)
+
+			for item := range col.stream {
+				itemCount++
+				_, ok := itemMap[item.ID()]
+				assert.True(t, ok, "should fetch item")
+			}
+
+			assert.NoError(t, errs.Failure())
+			assert.Equal(t, len(test.items), itemCount, "should see all expected items")
+		})
 	}
 }

src/internal/m365/collection/site/helper_test.go

@@ -1,16 +1,9 @@
 package site
 
 import (
-	"testing"
-
-	"github.com/alcionai/clues"
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
-	"github.com/stretchr/testify/require"
 
 	"github.com/alcionai/corso/src/internal/m365/support"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/count"
-	"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
 )
 
 // ---------------------------------------------------------------------------
@@ -42,18 +35,3 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
 
 func (ms *MockGraphService) UpdateStatus(*support.ControllerOperationStatus) {
 }
-
-// ---------------------------------------------------------------------------
-// Helper functions
-// ---------------------------------------------------------------------------
-
-func createTestService(t *testing.T, credentials account.M365Config) *graph.Service {
-	adapter, err := graph.CreateAdapter(
-		credentials.AzureTenantID,
-		credentials.AzureClientID,
-		credentials.AzureClientSecret,
-		count.New())
-	require.NoError(t, err, "creating microsoft graph service for exchange", clues.ToCore(err))
-
-	return graph.NewService(adapter)
-}

src/internal/m365/collection/site/lists.go

@@ -1,440 +0,0 @@
-package site
-
-import (
-	"context"
-	"sync"
-
-	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/microsoftgraph/msgraph-sdk-go/sites"
-
-	"github.com/alcionai/corso/src/internal/common/ptr"
-	"github.com/alcionai/corso/src/pkg/backup/details"
-	"github.com/alcionai/corso/src/pkg/fault"
-	"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
-)
-
-// ListToSPInfo translates models.Listable metadata into searchable content
-// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
-func ListToSPInfo(lst models.Listable, size int64) *details.SharePointInfo {
-	var (
-		name     = ptr.Val(lst.GetDisplayName())
-		webURL   = ptr.Val(lst.GetWebUrl())
-		created  = ptr.Val(lst.GetCreatedDateTime())
-		modified = ptr.Val(lst.GetLastModifiedDateTime())
-	)
-
-	return &details.SharePointInfo{
-		ItemType: details.SharePointList,
-		ItemName: name,
-		Created:  created,
-		Modified: modified,
-		WebURL:   webURL,
-		Size:     size,
-	}
-}
-
-type ListTuple struct {
-	ID   string
-	Name string
-}
-
-func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration {
-	selecting := []string{"id", "displayName"}
-	queryOptions := sites.ItemListsRequestBuilderGetQueryParameters{
-		Select: selecting,
-	}
-	options := &sites.ItemListsRequestBuilderGetRequestConfiguration{
-		QueryParameters: &queryOptions,
-	}
-
-	return options
-}
-
-func PreFetchLists(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID string,
-) ([]ListTuple, error) {
-	var (
-		builder    = gs.Client().Sites().BySiteId(siteID).Lists()
-		options    = preFetchListOptions()
-		listTuples = make([]ListTuple, 0)
-	)
-
-	for {
-		resp, err := builder.Get(ctx, options)
-		if err != nil {
-			return nil, graph.Wrap(ctx, err, "getting lists")
-		}
-
-		for _, entry := range resp.GetValue() {
-			var (
-				id   = ptr.Val(entry.GetId())
-				name = ptr.Val(entry.GetDisplayName())
-				temp = ListTuple{ID: id, Name: name}
-			)
-
-			if len(name) == 0 {
-				temp.Name = id
-			}
-
-			listTuples = append(listTuples, temp)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsRequestBuilder(link, gs.Adapter())
-	}
-
-	return listTuples, nil
-}
-
-// list.go contains additional functions to help retrieve SharePoint List data from M365
-// SharePoint lists represent lists on a site. Inherits additional properties from
-// baseItem: https://learn.microsoft.com/en-us/graph/api/resources/baseitem?view=graph-rest-1.0
-// The full details concerning SharePoint Lists can
-// be found at: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0
-// Note additional calls are required for the relationships that exist outside of the object properties.
-
-// loadSiteLists is a utility function to populate a collection of SharePoint.List
-// objects associated with a given siteID.
-// @param siteID the M365 ID that represents the SharePoint Site
-// Makes additional calls to retrieve the following relationships:
-// - Columns
-// - ContentTypes
-// - List Items
-func loadSiteLists(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID string,
-	listIDs []string,
-	errs *fault.Bus,
-) ([]models.Listable, error) {
-	var (
-		results     = make([]models.Listable, 0)
-		semaphoreCh = make(chan struct{}, fetchChannelSize)
-		el          = errs.Local()
-		wg          sync.WaitGroup
-		m           sync.Mutex
-	)
-
-	defer close(semaphoreCh)
-
-	updateLists := func(list models.Listable) {
-		m.Lock()
-		defer m.Unlock()
-
-		results = append(results, list)
-	}
-
-	for _, listID := range listIDs {
-		if el.Failure() != nil {
-			break
-		}
-
-		semaphoreCh <- struct{}{}
-
-		wg.Add(1)
-
-		go func(id string) {
-			defer wg.Done()
-			defer func() { <-semaphoreCh }()
-
-			var (
-				entry models.Listable
-				err   error
-			)
-
-			entry, err = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(id).Get(ctx, nil)
-			if err != nil {
-				el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting site list"))
-				return
-			}
-
-			cols, cTypes, lItems, err := fetchListContents(ctx, gs, siteID, id, errs)
-			if err != nil {
-				el.AddRecoverable(ctx, clues.Wrap(err, "getting list contents"))
-				return
-			}
-
-			entry.SetColumns(cols)
-			entry.SetContentTypes(cTypes)
-			entry.SetItems(lItems)
-			updateLists(entry)
-		}(listID)
-	}
-
-	wg.Wait()
-
-	return results, el.Failure()
-}
-
-// fetchListContents utility function to retrieve associated M365 relationships
-// which are not included with the standard List query:
-// - Columns, ContentTypes, ListItems
-func fetchListContents(
-	ctx context.Context,
-	service graph.Servicer,
-	siteID, listID string,
-	errs *fault.Bus,
-) (
-	[]models.ColumnDefinitionable,
-	[]models.ContentTypeable,
-	[]models.ListItemable,
-	error,
-) {
-	cols, err := fetchColumns(ctx, service, siteID, listID, "")
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	cTypes, err := fetchContentTypes(ctx, service, siteID, listID, errs)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	lItems, err := fetchListItems(ctx, service, siteID, listID, errs)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	return cols, cTypes, lItems, nil
-}
-
-// fetchListItems utility for retrieving ListItem data and the associated relationship
-// data. Additional call append data to the tracked items, and do not create additional collections.
-// Additional Call:
-// * Fields
-func fetchListItems(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID, listID string,
-	errs *fault.Bus,
-) ([]models.ListItemable, error) {
-	var (
-		prefix  = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID)
-		builder = prefix.Items()
-		itms    = make([]models.ListItemable, 0)
-		el      = errs.Local()
-	)
-
-	for {
-		if errs.Failure() != nil {
-			break
-		}
-
-		resp, err := builder.Get(ctx, nil)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, itm := range resp.GetValue() {
-			if el.Failure() != nil {
-				break
-			}
-
-			newPrefix := prefix.Items().ByListItemId(ptr.Val(itm.GetId()))
-
-			fields, err := newPrefix.Fields().Get(ctx, nil)
-			if err != nil {
-				el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting list fields"))
-				continue
-			}
-
-			itm.SetFields(fields)
-
-			itms = append(itms, itm)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsItemItemsRequestBuilder(link, gs.Adapter())
-	}
-
-	return itms, el.Failure()
-}
-
-// fetchColumns utility function to return columns from a site.
-// An additional call required to check for details concerning the SourceColumn.
-// For additional details:  https://learn.microsoft.com/en-us/graph/api/resources/columndefinition?view=graph-rest-1.0
-// TODO: Refactor on if/else (dadams39)
-func fetchColumns(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID, listID, cTypeID string,
-) ([]models.ColumnDefinitionable, error) {
-	cs := make([]models.ColumnDefinitionable, 0)
-
-	if len(cTypeID) == 0 {
-		builder := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Columns()
-
-		for {
-			resp, err := builder.Get(ctx, nil)
-			if err != nil {
-				return nil, graph.Wrap(ctx, err, "getting list columns")
-			}
-
-			cs = append(cs, resp.GetValue()...)
-
-			link, ok := ptr.ValOK(resp.GetOdataNextLink())
-			if !ok {
-				break
-			}
-
-			builder = sites.NewItemListsItemColumnsRequestBuilder(link, gs.Adapter())
-		}
-	} else {
-		builder := gs.Client().
-			Sites().
-			BySiteId(siteID).
-			Lists().
-			ByListId(listID).
-			ContentTypes().
-			ByContentTypeId(cTypeID).
-			Columns()
-
-		for {
-			resp, err := builder.Get(ctx, nil)
-			if err != nil {
-				return nil, graph.Wrap(ctx, err, "getting content columns")
-			}
-
-			cs = append(cs, resp.GetValue()...)
-
-			link, ok := ptr.ValOK(resp.GetOdataNextLink())
-			if !ok {
-				break
-			}
-
-			builder = sites.NewItemListsItemContentTypesItemColumnsRequestBuilder(link, gs.Adapter())
-		}
-	}
-
-	return cs, nil
-}
-
-// fetchContentTypes retrieves all data for content type. Additional queries required
-// for the following:
-// - ColumnLinks
-// - Columns
-// Expand queries not used to retrieve the above. Possibly more than 20.
-// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters
-func fetchContentTypes(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID, listID string,
-	errs *fault.Bus,
-) ([]models.ContentTypeable, error) {
-	var (
-		el      = errs.Local()
-		cTypes  = make([]models.ContentTypeable, 0)
-		builder = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).ContentTypes()
-	)
-
-	for {
-		if errs.Failure() != nil {
-			break
-		}
-
-		resp, err := builder.Get(ctx, nil)
-		if err != nil {
-			return nil, err
-		}
-
-		for _, cont := range resp.GetValue() {
-			if el.Failure() != nil {
-				break
-			}
-
-			id := ptr.Val(cont.GetId())
-
-			links, err := fetchColumnLinks(ctx, gs, siteID, listID, id)
-			if err != nil {
-				el.AddRecoverable(ctx, err)
-				continue
-			}
-
-			cont.SetColumnLinks(links)
-
-			cs, err := fetchColumns(ctx, gs, siteID, listID, id)
-			if err != nil {
-				el.AddRecoverable(ctx, err)
-				continue
-			}
-
-			cont.SetColumns(cs)
-
-			cTypes = append(cTypes, cont)
-		}
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsItemContentTypesRequestBuilder(link, gs.Adapter())
-	}
-
-	return cTypes, el.Failure()
-}
-
-func fetchColumnLinks(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID, listID, cTypeID string,
-) ([]models.ColumnLinkable, error) {
-	var (
-		builder = gs.Client().
-			Sites().
-			BySiteId(siteID).
-			Lists().
-			ByListId(listID).
-			ContentTypes().
-			ByContentTypeId(cTypeID).
-			ColumnLinks()
-		links = make([]models.ColumnLinkable, 0)
-	)
-
-	for {
-		resp, err := builder.Get(ctx, nil)
-		if err != nil {
-			return nil, graph.Wrap(ctx, err, "getting column links")
-		}
-
-		links = append(links, resp.GetValue()...)
-
-		link, ok := ptr.ValOK(resp.GetOdataNextLink())
-		if !ok {
-			break
-		}
-
-		builder = sites.NewItemListsItemContentTypesItemColumnLinksRequestBuilder(
-			link,
-			gs.Adapter())
-	}
-
-	return links, nil
-}
-
-// DeleteList removes a list object from a site.
-// deletes require unique http clients
-// https://github.com/alcionai/corso/issues/2707
-func DeleteList(
-	ctx context.Context,
-	gs graph.Servicer,
-	siteID, listID string,
-) error {
-	err := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Delete(ctx, nil)
-	if err != nil {
-		return graph.Wrap(ctx, err, "deleting list")
-	}
-
-	return nil
-}

src/internal/m365/collection/site/lists_test.go

@@ -1,112 +0,0 @@
-package site
-
-import (
-	"testing"
-
-	"github.com/alcionai/clues"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
-
-	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/internal/tester/tconfig"
-	"github.com/alcionai/corso/src/pkg/account"
-	"github.com/alcionai/corso/src/pkg/backup/details"
-	"github.com/alcionai/corso/src/pkg/fault"
-	"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
-)
-
-type ListsUnitSuite struct {
-	tester.Suite
-	creds account.M365Config
-}
-
-func (suite *ListsUnitSuite) SetupSuite() {
-	t := suite.T()
-	a := tconfig.NewM365Account(t)
-	m365, err := a.M365Config()
-	require.NoError(t, err, clues.ToCore(err))
-
-	suite.creds = m365
-
-	ctx, flush := tester.NewContext(suite.T())
-	defer flush()
-
-	graph.InitializeConcurrencyLimiter(ctx, false, 4)
-}
-
-func TestListsUnitSuite(t *testing.T) {
-	suite.Run(t, &ListsUnitSuite{
-		Suite: tester.NewIntegrationSuite(
-			t,
-			[][]string{tconfig.M365AcctCredEnvs}),
-	})
-}
-
-// Test LoadList --> Retrieves all data from backStore
-// Functions tested:
-// - fetchListItems()
-// - fetchColumns()
-// - fetchContentColumns()
-// - fetchContentTypes()
-// - fetchColumnLinks
-// TODO: upgrade passed github.com/microsoftgraph/msgraph-sdk-go v0.40.0
-// to verify if these 2 calls are valid
-// - fetchContentBaseTypes
-// - fetchColumnPositions
-func (suite *ListsUnitSuite) TestLoadList() {
-	t := suite.T()
-
-	ctx, flush := tester.NewContext(t)
-	defer flush()
-
-	service := createTestService(t, suite.creds)
-	tuples, err := PreFetchLists(ctx, service, "root")
-	require.NoError(t, err, clues.ToCore(err))
-
-	job := []string{tuples[0].ID}
-	lists, err := loadSiteLists(ctx, service, "root", job, fault.New(true))
-	assert.NoError(t, err, clues.ToCore(err))
-	assert.Greater(t, len(lists), 0)
-	t.Logf("Length: %d\n", len(lists))
-}
-
-func (suite *ListsUnitSuite) TestSharePointInfo() {
-	tests := []struct {
-		name         string
-		listAndDeets func() (models.Listable, *details.SharePointInfo)
-	}{
-		{
-			name: "Empty List",
-			listAndDeets: func() (models.Listable, *details.SharePointInfo) {
-				i := &details.SharePointInfo{ItemType: details.SharePointList}
-				return models.NewList(), i
-			},
-		}, {
-			name: "Only Name",
-			listAndDeets: func() (models.Listable, *details.SharePointInfo) {
-				aTitle := "Whole List"
-				listing := models.NewList()
-				listing.SetDisplayName(&aTitle)
-				i := &details.SharePointInfo{
-					ItemType: details.SharePointList,
-					ItemName: aTitle,
-				}
-
-				return listing, i
-			},
-		},
-	}
-	for _, test := range tests {
-		suite.Run(test.name, func() {
-			t := suite.T()
-
-			list, expected := test.listAndDeets()
-			info := ListToSPInfo(list, 10)
-			assert.Equal(t, expected.ItemType, info.ItemType)
-			assert.Equal(t, expected.ItemName, info.ItemName)
-			assert.Equal(t, expected.WebURL, info.WebURL)
-		})
-	}
-}

src/internal/m365/collection/site/restore.go

@@ -200,7 +200,7 @@ func restoreListItem(
 		}
 	}
 
-	dii.SharePoint = ListToSPInfo(restoredList, int64(len(byteArray)))
+	dii.SharePoint = api.ListToSPInfo(restoredList)
 
 	return dii, nil
 }

src/internal/m365/service/sharepoint/backup.go

@@ -55,8 +55,11 @@ func ProduceBackupCollections(
 
 		switch scope.Category().PathType() {
 		case path.ListsCategory:
+			bh := site.NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists())
+
 			spcs, err = site.CollectLists(
 				ctx,
+				bh,
 				bpc,
 				ac,
 				creds.AzureTenantID,

src/internal/operations/test/m365/sharepoint/sharepoint_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/internal/tester/tconfig"
 	"github.com/alcionai/corso/src/internal/version"
+	deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata"
 	"github.com/alcionai/corso/src/pkg/control"
 	ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
 	"github.com/alcionai/corso/src/pkg/count"
@@ -61,6 +62,67 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() {
 		sel.Selector)
 }
 
+func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointList() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	var (
+		resourceID = suite.its.Site.ID
+		sel        = selectors.NewSharePointBackup([]string{resourceID})
+		tenID      = tconfig.M365TenantID(t)
+		mb         = evmock.NewBus()
+		counter    = count.New()
+		ws         = deeTD.CategoryFromRepoRef
+	)
+
+	sel.Include(selTD.SharePointBackupListsScope(sel))
+
+	bo, bod := PrepNewTestBackupOp(
+		t,
+		ctx,
+		mb,
+		sel.Selector,
+		control.DefaultOptions(),
+		version.Backup,
+		counter)
+	defer bod.Close(t, ctx)
+
+	RunAndCheckBackup(t, ctx, &bo, mb, false)
+	CheckBackupIsInManifests(
+		t,
+		ctx,
+		bod.KW,
+		bod.SW,
+		&bo,
+		bod.Sel,
+		bod.Sel.ID(),
+		path.ListsCategory)
+
+	bID := bo.Results.BackupID
+
+	_, expectDeets := deeTD.GetDeetsInBackup(
+		t,
+		ctx,
+		bID,
+		tenID,
+		bod.Sel.ID(),
+		path.SharePointService,
+		ws,
+		bod.KMS,
+		bod.SSS)
+	deeTD.CheckBackupDetails(
+		t,
+		ctx,
+		bID,
+		ws,
+		bod.KMS,
+		bod.SSS,
+		expectDeets,
+		false)
+}
+
 func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() {
 	runSharePointIncrementalBackupTests(suite, suite.its, control.DefaultOptions())
 }

src/pkg/selectors/testdata/sharepoint.go

@@ -4,8 +4,14 @@ import (
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
+const TestListName = "test-list"
+
 // SharePointBackupFolderScope is the standard folder scope that should be used
 // in integration backups with sharepoint.
 func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope {
 	return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch())
 }
+
+func SharePointBackupListsScope(sel *selectors.SharePointBackup) []selectors.SharePointScope {
+	return sel.ListItems([]string{TestListName}, selectors.Any())
+}