From dc47001cba049be98346b13768175ee893aa9025 Mon Sep 17 00:00:00 2001 From: Hitesh Pattanayak <48874082+HiteshRepo@users.noreply.github.com> Date: Fri, 22 Dec 2023 19:56:23 +0530 Subject: [PATCH] enables sharepoint to use lists backup handler for lists ops (#4920) enables sharepoint to use lists backup handler for lists ops Changes previously approved in: - https://github.com/alcionai/corso/pull/4786 - https://github.com/alcionai/corso/pull/4787 - https://github.com/alcionai/corso/pull/4909 #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) #4754 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/m365/backup_test.go | 11 +- src/internal/m365/collection/site/backup.go | 25 +- .../m365/collection/site/backup_test.go | 54 ++- .../m365/collection/site/collection.go | 267 ++++++----- .../m365/collection/site/collection_test.go | 201 +++++--- .../m365/collection/site/helper_test.go | 22 - src/internal/m365/collection/site/lists.go | 440 ------------------ .../m365/collection/site/lists_test.go | 112 ----- src/internal/m365/collection/site/restore.go | 2 +- .../m365/service/sharepoint/backup.go | 3 + .../test/m365/sharepoint/sharepoint_test.go | 62 +++ src/pkg/selectors/testdata/sharepoint.go | 6 + 12 files changed, 453 insertions(+), 752 deletions(-) delete mode 100644 src/internal/m365/collection/site/lists.go delete mode 100644 src/internal/m365/collection/site/lists_test.go diff --git a/src/internal/m365/backup_test.go b/src/internal/m365/backup_test.go index bc1293157..1c1c7b3d5 100644 --- a/src/internal/m365/backup_test.go +++ b/src/internal/m365/backup_test.go @@ -274,7 +274,6 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { ctrl := newController(ctx, suite.T(), path.SharePointService) tests := []struct { name string - expected int getSelector func() selectors.Selector }{ { @@ -286,8 +285,7 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { }, }, { - name: "Lists", - expected: 0, + name: "Lists", getSelector: func() selectors.Selector { sel := selectors.NewSharePointBackup(selSites) sel.Include(sel.Lists(selectors.Any())) @@ -329,8 +327,8 @@ func (suite *DataCollectionIntgSuite) TestSharePointDataCollection() { } // we don't know an exact count of drives this will produce, - // but it should be more than one. - assert.Less(t, test.expected, len(collections)) + // but it should be more than zero. + assert.NotEmpty(t, collections) for _, coll := range collections { for object := range coll.Items(ctx, fault.New(true)) { @@ -465,7 +463,8 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { assert.True(t, excludes.Empty()) for _, collection := range cols { - t.Logf("Path: %s\n", collection.FullPath().String()) + assert.Equal(t, path.SharePointService, collection.FullPath().Service()) + assert.Equal(t, path.ListsCategory, collection.FullPath().Category()) for item := range collection.Items(ctx, fault.New(true)) { t.Log("File: " + item.ID()) diff --git a/src/internal/m365/collection/site/backup.go b/src/internal/m365/collection/site/backup.go index c759462cd..22636f2f7 100644 --- a/src/internal/m365/collection/site/backup.go +++ b/src/internal/m365/collection/site/backup.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common/prefixmatcher" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/drive" betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" @@ -123,13 +124,14 @@ func CollectPages( } collection := NewCollection( + nil, dir, ac, scope, su, bpc.Options) collection.SetBetaService(betaService) - collection.AddJob(tuple.ID) + collection.AddItem(tuple.ID) spcs = append(spcs, collection) } @@ -139,6 +141,7 @@ func CollectPages( func CollectLists( ctx context.Context, + bh backupHandler, bpc inject.BackupProducerConfig, ac api.Client, tenantID string, @@ -151,14 +154,15 @@ func CollectLists( var ( el = errs.Local() spcs = make([]data.BackupCollection, 0) + acc = api.CallConfig{Select: idAnd("displayName")} ) - lists, err := PreFetchLists(ctx, ac.Stable, bpc.ProtectedResource.ID()) + lists, err := bh.GetItems(ctx, acc) if err != nil { return nil, err } - for _, tuple := range lists { + for _, list := range lists { if el.Failure() != nil { break } @@ -169,21 +173,32 @@ func CollectLists( path.SharePointService, path.ListsCategory, false, - tuple.Name) + ptr.Val(list.GetId())) if err != nil { el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "creating list collection path")) } collection := NewCollection( + bh, dir, ac, scope, su, bpc.Options) - collection.AddJob(tuple.ID) + collection.AddItem(ptr.Val(list.GetId())) spcs = append(spcs, collection) } return spcs, el.Failure() } + +func idAnd(ss ...string) []string { + id := []string{"id"} + + if len(ss) == 0 { + return id + } + + return append(id, ss...) +} diff --git a/src/internal/m365/collection/site/backup_test.go b/src/internal/m365/collection/site/backup_test.go index d80abe358..4f7251b79 100644 --- a/src/internal/m365/collection/site/backup_test.go +++ b/src/internal/m365/collection/site/backup_test.go @@ -21,26 +21,26 @@ import ( "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) -type SharePointPagesSuite struct { +type SharePointSuite struct { tester.Suite } -func TestSharePointPagesSuite(t *testing.T) { - suite.Run(t, &SharePointPagesSuite{ +func TestSharePointSuite(t *testing.T) { + suite.Run(t, &SharePointSuite{ Suite: tester.NewIntegrationSuite( t, [][]string{tconfig.M365AcctCredEnvs}), }) } -func (suite *SharePointPagesSuite) SetupSuite() { +func (suite *SharePointSuite) SetupSuite() { ctx, flush := tester.NewContext(suite.T()) defer flush() graph.InitializeConcurrencyLimiter(ctx, false, 4) } -func (suite *SharePointPagesSuite) TestCollectPages() { +func (suite *SharePointSuite) TestCollectPages() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -81,3 +81,47 @@ func (suite *SharePointPagesSuite) TestCollectPages() { assert.NoError(t, err, clues.ToCore(err)) assert.NotEmpty(t, col) } + +func (suite *SharePointSuite) TestCollectLists() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + siteID = tconfig.M365SiteID(t) + a = tconfig.NewM365Account(t) + counter = count.New() + ) + + creds, err := a.M365Config() + require.NoError(t, err, clues.ToCore(err)) + + ac, err := api.NewClient( + creds, + control.DefaultOptions(), + counter) + require.NoError(t, err, clues.ToCore(err)) + + bpc := inject.BackupProducerConfig{ + LastBackupVersion: version.NoBackup, + Options: control.DefaultOptions(), + ProtectedResource: mock.NewProvider(siteID, siteID), + } + + sel := selectors.NewSharePointBackup([]string{siteID}) + + bh := NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists()) + + col, err := CollectLists( + ctx, + bh, + bpc, + ac, + creds.AzureTenantID, + sel.Lists(selectors.Any())[0], + (&MockGraphService{}).UpdateStatus, + fault.New(true)) + require.NoError(t, err, clues.ToCore(err)) + assert.Less(t, 0, len(col)) +} diff --git a/src/internal/m365/collection/site/collection.go b/src/internal/m365/collection/site/collection.go index 00107e72d..491360060 100644 --- a/src/internal/m365/collection/site/collection.go +++ b/src/internal/m365/collection/site/collection.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "io" + "sync" "github.com/alcionai/clues" "github.com/microsoft/kiota-abstractions-go/serialization" kjson "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" @@ -41,25 +43,29 @@ const ( var _ data.BackupCollection = &Collection{} -// Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported -// by the oneDrive.Collection as the calls are identical for populating the Collection +// Collection is the SharePoint.List or SharePoint.Page implementation of data.Collection. + +// SharePoint.Libraries collections are supported by the oneDrive.Collection +// as the calls are identical for populating the Collection type Collection struct { - // data is the container for each individual SharePoint.List - data chan data.Item + // stream is the container for each individual SharePoint item of (page/list) + stream chan data.Item // fullPath indicates the hierarchy within the collection fullPath path.Path - // jobs contain the SharePoint.Site.ListIDs for the associated list(s). - jobs []string + // jobs contain the SharePoint.List.IDs or SharePoint.Page.IDs + items []string // M365 IDs of the items of this collection category path.CategoryType client api.Sites ctrl control.Options betaService *betaAPI.BetaService statusUpdater support.StatusUpdater + getter getItemByIDer } // NewCollection helper function for creating a Collection func NewCollection( + getter getItemByIDer, folderPath path.Path, ac api.Client, scope selectors.SharePointScope, @@ -68,8 +74,9 @@ func NewCollection( ) *Collection { c := &Collection{ fullPath: folderPath, - jobs: make([]string, 0), - data: make(chan data.Item, collectionChannelBufferSize), + items: make([]string, 0), + getter: getter, + stream: make(chan data.Item, collectionChannelBufferSize), client: ac.Sites(), statusUpdater: statusUpdater, category: scope.Category().PathType(), @@ -83,9 +90,9 @@ func (sc *Collection) SetBetaService(betaService *betaAPI.BetaService) { sc.betaService = betaService } -// AddJob appends additional objectID to job field -func (sc *Collection) AddJob(objID string) { - sc.jobs = append(sc.jobs, objID) +// AddItem appends additional itemID to items field +func (sc *Collection) AddItem(itemID string) { + sc.items = append(sc.items, itemID) } func (sc *Collection) FullPath() path.Path { @@ -98,6 +105,10 @@ func (sc Collection) PreviousPath() path.Path { return nil } +func (sc Collection) LocationPath() *path.Builder { + return path.Builder{}.Append(sc.fullPath.Folders()...) +} + func (sc Collection) State() data.CollectionState { return data.NewState } @@ -110,21 +121,21 @@ func (sc *Collection) Items( ctx context.Context, errs *fault.Bus, ) <-chan data.Item { - go sc.populate(ctx, errs) - return sc.data + go sc.streamItems(ctx, errs) + return sc.stream } func (sc *Collection) finishPopulation( ctx context.Context, - metrics support.CollectionMetrics, + metrics *support.CollectionMetrics, ) { - close(sc.data) + close(sc.stream) status := support.CreateStatus( ctx, support.Backup, 1, // 1 folder - metrics, + *metrics, sc.fullPath.Folder(false)) logger.Ctx(ctx).Debug(status.String()) @@ -134,128 +145,98 @@ func (sc *Collection) finishPopulation( } } -// populate utility function to retrieve data from back store for a given collection -func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) { - metrics, _ := sc.runPopulate(ctx, errs) - sc.finishPopulation(ctx, metrics) -} - -func (sc *Collection) runPopulate( +// streamItems utility function to retrieve data from back store for a given collection +func (sc *Collection) streamItems( ctx context.Context, errs *fault.Bus, -) (support.CollectionMetrics, error) { - var ( - err error - metrics support.CollectionMetrics - writer = kjson.NewJsonSerializationWriter() - ) - - // TODO: Insert correct ID for CollectionProgress - colProgress := observe.CollectionProgress( - ctx, - sc.fullPath.Category().HumanString(), - sc.fullPath.Folders()) - defer close(colProgress) - +) { // Switch retrieval function based on category switch sc.category { case path.ListsCategory: - metrics, err = sc.retrieveLists(ctx, writer, colProgress, errs) + sc.streamLists(ctx, errs) case path.PagesCategory: - metrics, err = sc.retrievePages(ctx, sc.client, writer, colProgress, errs) + sc.retrievePages(ctx, sc.client, errs) } - - return metrics, err } -// retrieveLists utility function for collection that downloads and serializes +// streamLists utility function for collection that downloads and serializes // models.Listable objects based on M365 IDs from the jobs field. -func (sc *Collection) retrieveLists( +func (sc *Collection) streamLists( ctx context.Context, - wtr *kjson.JsonSerializationWriter, - progress chan<- struct{}, errs *fault.Bus, -) (support.CollectionMetrics, error) { +) { var ( metrics support.CollectionMetrics el = errs.Local() + wg sync.WaitGroup ) - lists, err := loadSiteLists( - ctx, - sc.client.Stable, - sc.fullPath.ProtectedResource(), - sc.jobs, - errs) - if err != nil { - return metrics, err - } + defer sc.finishPopulation(ctx, &metrics) + + // TODO: Insert correct ID for CollectionProgress + progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders()) + defer close(progress) + + semaphoreCh := make(chan struct{}, fetchChannelSize) + defer close(semaphoreCh) - metrics.Objects += len(lists) // For each models.Listable, object is serialized and the metrics are collected. // The progress is objected via the passed in channel. - for _, lst := range lists { + for _, listID := range sc.items { if el.Failure() != nil { break } - byteArray, err := serializeContent(ctx, wtr, lst) - if err != nil { - el.AddRecoverable(ctx, clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation)) - continue - } + wg.Add(1) + semaphoreCh <- struct{}{} - size := int64(len(byteArray)) + sc.handleListItems(ctx, semaphoreCh, progress, listID, el, &metrics) - if size > 0 { - metrics.Bytes += size - - metrics.Successes++ - - item, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(byteArray)), - ptr.Val(lst.GetId()), - details.ItemInfo{SharePoint: ListToSPInfo(lst, size)}) - if err != nil { - el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation)) - continue - } - - sc.data <- item - progress <- struct{}{} - } + wg.Done() } - return metrics, el.Failure() + wg.Wait() } func (sc *Collection) retrievePages( ctx context.Context, as api.Sites, - wtr *kjson.JsonSerializationWriter, - progress chan<- struct{}, errs *fault.Bus, -) (support.CollectionMetrics, error) { +) { var ( metrics support.CollectionMetrics el = errs.Local() ) + defer sc.finishPopulation(ctx, &metrics) + + // TODO: Insert correct ID for CollectionProgress + progress := observe.CollectionProgress(ctx, sc.fullPath.Category().HumanString(), sc.fullPath.Folders()) + defer close(progress) + + wtr := kjson.NewJsonSerializationWriter() + defer wtr.Close() + betaService := sc.betaService if betaService == nil { - return metrics, clues.NewWC(ctx, "beta service required") + logger.Ctx(ctx).Error(clues.New("beta service required")) + return } parent, err := as.GetByID(ctx, sc.fullPath.ProtectedResource(), api.CallConfig{}) if err != nil { - return metrics, err + logger.Ctx(ctx).Error(err) + + return } root := ptr.Val(parent.GetWebUrl()) - pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.jobs, errs) + pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ProtectedResource(), sc.items, errs) if err != nil { - return metrics, err + logger.Ctx(ctx).Error(err) + + return } metrics.Objects = len(pages) @@ -275,25 +256,25 @@ func (sc *Collection) retrievePages( size := int64(len(byteArray)) - if size > 0 { - metrics.Bytes += size - metrics.Successes++ - - item, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(byteArray)), - ptr.Val(pg.GetId()), - details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)}) - if err != nil { - el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation)) - continue - } - - sc.data <- item - progress <- struct{}{} + if size == 0 { + return } - } - return metrics, el.Failure() + metrics.Bytes += size + metrics.Successes++ + + item, err := data.NewPrefetchedItemWithInfo( + io.NopCloser(bytes.NewReader(byteArray)), + ptr.Val(pg.GetId()), + details.ItemInfo{SharePoint: pageToSPInfo(pg, root, size)}) + if err != nil { + el.AddRecoverable(ctx, clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation)) + continue + } + + sc.stream <- item + progress <- struct{}{} + } } func serializeContent( @@ -315,3 +296,79 @@ func serializeContent( return byteArray, nil } + +func (sc *Collection) handleListItems( + ctx context.Context, + semaphoreCh chan struct{}, + progress chan<- struct{}, + listID string, + el *fault.Bus, + metrics *support.CollectionMetrics, +) { + defer func() { <-semaphoreCh }() + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + var ( + list models.Listable + info *details.SharePointInfo + err error + ) + + list, info, err = sc.getter.GetItemByID(ctx, listID) + if err != nil { + err = clues.WrapWC(ctx, err, "getting list data").Label(fault.LabelForceNoBackupCreation) + el.AddRecoverable(ctx, err) + + return + } + + metrics.Objects++ + + if err := writer.WriteObjectValue("", list); err != nil { + err = clues.WrapWC(ctx, err, "writing list to serializer").Label(fault.LabelForceNoBackupCreation) + el.AddRecoverable(ctx, err) + + return + } + + entryBytes, err := writer.GetSerializedContent() + if err != nil { + err = clues.WrapWC(ctx, err, "serializing list").Label(fault.LabelForceNoBackupCreation) + el.AddRecoverable(ctx, err) + + return + } + + size := int64(len(entryBytes)) + + if size == 0 { + return + } + + metrics.Bytes += size + metrics.Successes++ + + template := "" + if list != nil && list.GetList() != nil { + template = ptr.Val(list.GetList().GetTemplate()) + } + + rc := io.NopCloser(bytes.NewReader(entryBytes)) + itemInfo := details.ItemInfo{ + SharePoint: info, + NotRecoverable: template == api.WebTemplateExtensionsListTemplateName, + } + + item, err := data.NewPrefetchedItemWithInfo(rc, listID, itemInfo) + if err != nil { + err = clues.StackWC(ctx, err).Label(fault.LabelForceNoBackupCreation) + el.AddRecoverable(ctx, err) + + return + } + + sc.stream <- item + progress <- struct{}{} +} diff --git a/src/internal/m365/collection/site/collection_test.go b/src/internal/m365/collection/site/collection_test.go index cca4d9ba3..afcf00880 100644 --- a/src/internal/m365/collection/site/collection_test.go +++ b/src/internal/m365/collection/site/collection_test.go @@ -7,21 +7,22 @@ import ( "github.com/alcionai/clues" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/sites" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/site/mock" betaAPI "github.com/alcionai/corso/src/internal/m365/service/sharepoint/api" spMock "github.com/alcionai/corso/src/internal/m365/service/sharepoint/mock" + "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -76,7 +77,9 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { tables := []struct { name, itemName string + notRecoverable bool scope selectors.SharePointScope + getter getItemByIDer getDir func(t *testing.T) path.Path getItem func(t *testing.T, itemName string) data.Item }{ @@ -84,6 +87,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { name: "List", itemName: "MockListing", scope: sel.Lists(selectors.Any())[0], + getter: &mock.ListHandler{}, getDir: func(t *testing.T) path.Path { dir, err := path.Build( tenant, @@ -107,10 +111,61 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { byteArray, err := ow.GetSerializedContent() require.NoError(t, err, clues.ToCore(err)) + info := &details.SharePointInfo{ + ItemName: name, + } + data, err := data.NewPrefetchedItemWithInfo( io.NopCloser(bytes.NewReader(byteArray)), name, - details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))}) + details.ItemInfo{SharePoint: info}) + require.NoError(t, err, clues.ToCore(err)) + + return data + }, + }, + { + name: "List with wte template", + itemName: "MockListing", + notRecoverable: true, + scope: sel.Lists(selectors.Any())[0], + getter: &mock.ListHandler{}, + getDir: func(t *testing.T) path.Path { + dir, err := path.Build( + tenant, + user, + path.SharePointService, + path.ListsCategory, + false, + dirRoot) + require.NoError(t, err, clues.ToCore(err)) + + return dir + }, + getItem: func(t *testing.T, name string) data.Item { + ow := kioser.NewJsonSerializationWriter() + + listInfo := models.NewListInfo() + listInfo.SetTemplate(ptr.To("webTemplateExtensionsList")) + + listing := spMock.ListDefault(name) + listing.SetDisplayName(&name) + listing.SetList(listInfo) + + err := ow.WriteObjectValue("", listing) + require.NoError(t, err, clues.ToCore(err)) + + byteArray, err := ow.GetSerializedContent() + require.NoError(t, err, clues.ToCore(err)) + + info := &details.SharePointInfo{ + ItemName: name, + } + + data, err := data.NewPrefetchedItemWithInfo( + io.NopCloser(bytes.NewReader(byteArray)), + name, + details.ItemInfo{SharePoint: info, NotRecoverable: true}) require.NoError(t, err, clues.ToCore(err)) return data @@ -120,6 +175,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { name: "Pages", itemName: "MockPages", scope: sel.Pages(selectors.Any())[0], + getter: nil, getDir: func(t *testing.T) path.Path { dir, err := path.Build( tenant, @@ -156,12 +212,13 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { defer flush() col := NewCollection( + test.getter, test.getDir(t), suite.ac, test.scope, nil, control.DefaultOptions()) - col.data <- test.getItem(t, test.itemName) + col.stream <- test.getItem(t, test.itemName) readItems := []data.Item{} @@ -180,68 +237,100 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() { assert.NotNil(t, info) assert.NotNil(t, info.SharePoint) assert.Equal(t, test.itemName, info.SharePoint.ItemName) + assert.Equal(t, test.notRecoverable, info.NotRecoverable) }) } } -// TestRestoreListCollection verifies Graph Restore API for the List Collection -func (suite *SharePointCollectionSuite) TestListCollection_Restore() { - t := suite.T() - // https://github.com/microsoftgraph/msgraph-sdk-go/issues/490 - t.Skip("disabled until upstream issue with list restore is fixed.") - - ctx, flush := tester.NewContext(t) - defer flush() - - service := createTestService(t, suite.creds) - listing := spMock.ListDefault("Mock List") - testName := "MockListing" - listing.SetDisplayName(&testName) - byteArray, err := service.Serialize(listing) - require.NoError(t, err, clues.ToCore(err)) - - listData, err := data.NewPrefetchedItemWithInfo( - io.NopCloser(bytes.NewReader(byteArray)), - testName, - details.ItemInfo{SharePoint: ListToSPInfo(listing, int64(len(byteArray)))}) - require.NoError(t, err, clues.ToCore(err)) - - destName := testdata.DefaultRestoreConfig("").Location - - deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName) - assert.NoError(t, err, clues.ToCore(err)) - t.Logf("List created: %s\n", deets.SharePoint.ItemName) - - // Clean-Up +func (suite *SharePointCollectionSuite) TestCollection_streamItems() { var ( - builder = service.Client().Sites().BySiteId(suite.siteID).Lists() - isFound bool - deleteID string + t = suite.T() + statusUpdater = func(*support.ControllerOperationStatus) {} + tenant = "some" + resource = "siteid" + list = "list" ) - for { - resp, err := builder.Get(ctx, nil) - assert.NoError(t, err, "getting site lists", clues.ToCore(err)) + table := []struct { + name string + category path.CategoryType + items []string + getDir func(t *testing.T) path.Path + }{ + { + name: "no items", + items: []string{}, + category: path.ListsCategory, + getDir: func(t *testing.T) path.Path { + dir, err := path.Build( + tenant, + resource, + path.SharePointService, + path.ListsCategory, + false, + list) + require.NoError(t, err, clues.ToCore(err)) - for _, temp := range resp.GetValue() { - if ptr.Val(temp.GetDisplayName()) == deets.SharePoint.ItemName { - isFound = true - deleteID = ptr.Val(temp.GetId()) + return dir + }, + }, + { + name: "with items", + items: []string{"list1", "list2", "list3"}, + category: path.ListsCategory, + getDir: func(t *testing.T) path.Path { + dir, err := path.Build( + tenant, + resource, + path.SharePointService, + path.ListsCategory, + false, + list) + require.NoError(t, err, clues.ToCore(err)) - break - } - } - // Get Next Link - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsRequestBuilder(link, service.Adapter()) + return dir + }, + }, } + for _, test := range table { + suite.Run(test.name, func() { + t.Log("running test", test) - if isFound { - err := DeleteList(ctx, service, suite.siteID, deleteID) - assert.NoError(t, err, clues.ToCore(err)) + var ( + errs = fault.New(true) + itemCount int + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + col := &Collection{ + fullPath: test.getDir(t), + category: test.category, + items: test.items, + getter: &mock.ListHandler{}, + stream: make(chan data.Item), + statusUpdater: statusUpdater, + } + + itemMap := func(js []string) map[string]struct{} { + m := make(map[string]struct{}) + for _, j := range js { + m[j] = struct{}{} + } + return m + }(test.items) + + go col.streamItems(ctx, errs) + + for item := range col.stream { + itemCount++ + _, ok := itemMap[item.ID()] + assert.True(t, ok, "should fetch item") + } + + assert.NoError(t, errs.Failure()) + assert.Equal(t, len(test.items), itemCount, "should see all expected items") + }) } } diff --git a/src/internal/m365/collection/site/helper_test.go b/src/internal/m365/collection/site/helper_test.go index e9f396a10..344620993 100644 --- a/src/internal/m365/collection/site/helper_test.go +++ b/src/internal/m365/collection/site/helper_test.go @@ -1,16 +1,9 @@ package site import ( - "testing" - - "github.com/alcionai/clues" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/m365/support" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/count" - "github.com/alcionai/corso/src/pkg/services/m365/api/graph" ) // --------------------------------------------------------------------------- @@ -42,18 +35,3 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter { func (ms *MockGraphService) UpdateStatus(*support.ControllerOperationStatus) { } - -// --------------------------------------------------------------------------- -// Helper functions -// --------------------------------------------------------------------------- - -func createTestService(t *testing.T, credentials account.M365Config) *graph.Service { - adapter, err := graph.CreateAdapter( - credentials.AzureTenantID, - credentials.AzureClientID, - credentials.AzureClientSecret, - count.New()) - require.NoError(t, err, "creating microsoft graph service for exchange", clues.ToCore(err)) - - return graph.NewService(adapter) -} diff --git a/src/internal/m365/collection/site/lists.go b/src/internal/m365/collection/site/lists.go deleted file mode 100644 index a46d03e2e..000000000 --- a/src/internal/m365/collection/site/lists.go +++ /dev/null @@ -1,440 +0,0 @@ -package site - -import ( - "context" - "sync" - - "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/sites" - - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/services/m365/api/graph" -) - -// ListToSPInfo translates models.Listable metadata into searchable content -// List Details: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 -func ListToSPInfo(lst models.Listable, size int64) *details.SharePointInfo { - var ( - name = ptr.Val(lst.GetDisplayName()) - webURL = ptr.Val(lst.GetWebUrl()) - created = ptr.Val(lst.GetCreatedDateTime()) - modified = ptr.Val(lst.GetLastModifiedDateTime()) - ) - - return &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: name, - Created: created, - Modified: modified, - WebURL: webURL, - Size: size, - } -} - -type ListTuple struct { - ID string - Name string -} - -func preFetchListOptions() *sites.ItemListsRequestBuilderGetRequestConfiguration { - selecting := []string{"id", "displayName"} - queryOptions := sites.ItemListsRequestBuilderGetQueryParameters{ - Select: selecting, - } - options := &sites.ItemListsRequestBuilderGetRequestConfiguration{ - QueryParameters: &queryOptions, - } - - return options -} - -func PreFetchLists( - ctx context.Context, - gs graph.Servicer, - siteID string, -) ([]ListTuple, error) { - var ( - builder = gs.Client().Sites().BySiteId(siteID).Lists() - options = preFetchListOptions() - listTuples = make([]ListTuple, 0) - ) - - for { - resp, err := builder.Get(ctx, options) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting lists") - } - - for _, entry := range resp.GetValue() { - var ( - id = ptr.Val(entry.GetId()) - name = ptr.Val(entry.GetDisplayName()) - temp = ListTuple{ID: id, Name: name} - ) - - if len(name) == 0 { - temp.Name = id - } - - listTuples = append(listTuples, temp) - } - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsRequestBuilder(link, gs.Adapter()) - } - - return listTuples, nil -} - -// list.go contains additional functions to help retrieve SharePoint List data from M365 -// SharePoint lists represent lists on a site. Inherits additional properties from -// baseItem: https://learn.microsoft.com/en-us/graph/api/resources/baseitem?view=graph-rest-1.0 -// The full details concerning SharePoint Lists can -// be found at: https://learn.microsoft.com/en-us/graph/api/resources/list?view=graph-rest-1.0 -// Note additional calls are required for the relationships that exist outside of the object properties. - -// loadSiteLists is a utility function to populate a collection of SharePoint.List -// objects associated with a given siteID. -// @param siteID the M365 ID that represents the SharePoint Site -// Makes additional calls to retrieve the following relationships: -// - Columns -// - ContentTypes -// - List Items -func loadSiteLists( - ctx context.Context, - gs graph.Servicer, - siteID string, - listIDs []string, - errs *fault.Bus, -) ([]models.Listable, error) { - var ( - results = make([]models.Listable, 0) - semaphoreCh = make(chan struct{}, fetchChannelSize) - el = errs.Local() - wg sync.WaitGroup - m sync.Mutex - ) - - defer close(semaphoreCh) - - updateLists := func(list models.Listable) { - m.Lock() - defer m.Unlock() - - results = append(results, list) - } - - for _, listID := range listIDs { - if el.Failure() != nil { - break - } - - semaphoreCh <- struct{}{} - - wg.Add(1) - - go func(id string) { - defer wg.Done() - defer func() { <-semaphoreCh }() - - var ( - entry models.Listable - err error - ) - - entry, err = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(id).Get(ctx, nil) - if err != nil { - el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting site list")) - return - } - - cols, cTypes, lItems, err := fetchListContents(ctx, gs, siteID, id, errs) - if err != nil { - el.AddRecoverable(ctx, clues.Wrap(err, "getting list contents")) - return - } - - entry.SetColumns(cols) - entry.SetContentTypes(cTypes) - entry.SetItems(lItems) - updateLists(entry) - }(listID) - } - - wg.Wait() - - return results, el.Failure() -} - -// fetchListContents utility function to retrieve associated M365 relationships -// which are not included with the standard List query: -// - Columns, ContentTypes, ListItems -func fetchListContents( - ctx context.Context, - service graph.Servicer, - siteID, listID string, - errs *fault.Bus, -) ( - []models.ColumnDefinitionable, - []models.ContentTypeable, - []models.ListItemable, - error, -) { - cols, err := fetchColumns(ctx, service, siteID, listID, "") - if err != nil { - return nil, nil, nil, err - } - - cTypes, err := fetchContentTypes(ctx, service, siteID, listID, errs) - if err != nil { - return nil, nil, nil, err - } - - lItems, err := fetchListItems(ctx, service, siteID, listID, errs) - if err != nil { - return nil, nil, nil, err - } - - return cols, cTypes, lItems, nil -} - -// fetchListItems utility for retrieving ListItem data and the associated relationship -// data. Additional call append data to the tracked items, and do not create additional collections. -// Additional Call: -// * Fields -func fetchListItems( - ctx context.Context, - gs graph.Servicer, - siteID, listID string, - errs *fault.Bus, -) ([]models.ListItemable, error) { - var ( - prefix = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID) - builder = prefix.Items() - itms = make([]models.ListItemable, 0) - el = errs.Local() - ) - - for { - if errs.Failure() != nil { - break - } - - resp, err := builder.Get(ctx, nil) - if err != nil { - return nil, err - } - - for _, itm := range resp.GetValue() { - if el.Failure() != nil { - break - } - - newPrefix := prefix.Items().ByListItemId(ptr.Val(itm.GetId())) - - fields, err := newPrefix.Fields().Get(ctx, nil) - if err != nil { - el.AddRecoverable(ctx, graph.Wrap(ctx, err, "getting list fields")) - continue - } - - itm.SetFields(fields) - - itms = append(itms, itm) - } - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsItemItemsRequestBuilder(link, gs.Adapter()) - } - - return itms, el.Failure() -} - -// fetchColumns utility function to return columns from a site. -// An additional call required to check for details concerning the SourceColumn. -// For additional details: https://learn.microsoft.com/en-us/graph/api/resources/columndefinition?view=graph-rest-1.0 -// TODO: Refactor on if/else (dadams39) -func fetchColumns( - ctx context.Context, - gs graph.Servicer, - siteID, listID, cTypeID string, -) ([]models.ColumnDefinitionable, error) { - cs := make([]models.ColumnDefinitionable, 0) - - if len(cTypeID) == 0 { - builder := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Columns() - - for { - resp, err := builder.Get(ctx, nil) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting list columns") - } - - cs = append(cs, resp.GetValue()...) - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsItemColumnsRequestBuilder(link, gs.Adapter()) - } - } else { - builder := gs.Client(). - Sites(). - BySiteId(siteID). - Lists(). - ByListId(listID). - ContentTypes(). - ByContentTypeId(cTypeID). - Columns() - - for { - resp, err := builder.Get(ctx, nil) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting content columns") - } - - cs = append(cs, resp.GetValue()...) - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsItemContentTypesItemColumnsRequestBuilder(link, gs.Adapter()) - } - } - - return cs, nil -} - -// fetchContentTypes retrieves all data for content type. Additional queries required -// for the following: -// - ColumnLinks -// - Columns -// Expand queries not used to retrieve the above. Possibly more than 20. -// Known Limitations: https://learn.microsoft.com/en-us/graph/known-issues#query-parameters -func fetchContentTypes( - ctx context.Context, - gs graph.Servicer, - siteID, listID string, - errs *fault.Bus, -) ([]models.ContentTypeable, error) { - var ( - el = errs.Local() - cTypes = make([]models.ContentTypeable, 0) - builder = gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).ContentTypes() - ) - - for { - if errs.Failure() != nil { - break - } - - resp, err := builder.Get(ctx, nil) - if err != nil { - return nil, err - } - - for _, cont := range resp.GetValue() { - if el.Failure() != nil { - break - } - - id := ptr.Val(cont.GetId()) - - links, err := fetchColumnLinks(ctx, gs, siteID, listID, id) - if err != nil { - el.AddRecoverable(ctx, err) - continue - } - - cont.SetColumnLinks(links) - - cs, err := fetchColumns(ctx, gs, siteID, listID, id) - if err != nil { - el.AddRecoverable(ctx, err) - continue - } - - cont.SetColumns(cs) - - cTypes = append(cTypes, cont) - } - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsItemContentTypesRequestBuilder(link, gs.Adapter()) - } - - return cTypes, el.Failure() -} - -func fetchColumnLinks( - ctx context.Context, - gs graph.Servicer, - siteID, listID, cTypeID string, -) ([]models.ColumnLinkable, error) { - var ( - builder = gs.Client(). - Sites(). - BySiteId(siteID). - Lists(). - ByListId(listID). - ContentTypes(). - ByContentTypeId(cTypeID). - ColumnLinks() - links = make([]models.ColumnLinkable, 0) - ) - - for { - resp, err := builder.Get(ctx, nil) - if err != nil { - return nil, graph.Wrap(ctx, err, "getting column links") - } - - links = append(links, resp.GetValue()...) - - link, ok := ptr.ValOK(resp.GetOdataNextLink()) - if !ok { - break - } - - builder = sites.NewItemListsItemContentTypesItemColumnLinksRequestBuilder( - link, - gs.Adapter()) - } - - return links, nil -} - -// DeleteList removes a list object from a site. -// deletes require unique http clients -// https://github.com/alcionai/corso/issues/2707 -func DeleteList( - ctx context.Context, - gs graph.Servicer, - siteID, listID string, -) error { - err := gs.Client().Sites().BySiteId(siteID).Lists().ByListId(listID).Delete(ctx, nil) - if err != nil { - return graph.Wrap(ctx, err, "deleting list") - } - - return nil -} diff --git a/src/internal/m365/collection/site/lists_test.go b/src/internal/m365/collection/site/lists_test.go deleted file mode 100644 index bd58a3e70..000000000 --- a/src/internal/m365/collection/site/lists_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package site - -import ( - "testing" - - "github.com/alcionai/clues" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/internal/tester/tconfig" - "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/services/m365/api/graph" -) - -type ListsUnitSuite struct { - tester.Suite - creds account.M365Config -} - -func (suite *ListsUnitSuite) SetupSuite() { - t := suite.T() - a := tconfig.NewM365Account(t) - m365, err := a.M365Config() - require.NoError(t, err, clues.ToCore(err)) - - suite.creds = m365 - - ctx, flush := tester.NewContext(suite.T()) - defer flush() - - graph.InitializeConcurrencyLimiter(ctx, false, 4) -} - -func TestListsUnitSuite(t *testing.T) { - suite.Run(t, &ListsUnitSuite{ - Suite: tester.NewIntegrationSuite( - t, - [][]string{tconfig.M365AcctCredEnvs}), - }) -} - -// Test LoadList --> Retrieves all data from backStore -// Functions tested: -// - fetchListItems() -// - fetchColumns() -// - fetchContentColumns() -// - fetchContentTypes() -// - fetchColumnLinks -// TODO: upgrade passed github.com/microsoftgraph/msgraph-sdk-go v0.40.0 -// to verify if these 2 calls are valid -// - fetchContentBaseTypes -// - fetchColumnPositions -func (suite *ListsUnitSuite) TestLoadList() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - service := createTestService(t, suite.creds) - tuples, err := PreFetchLists(ctx, service, "root") - require.NoError(t, err, clues.ToCore(err)) - - job := []string{tuples[0].ID} - lists, err := loadSiteLists(ctx, service, "root", job, fault.New(true)) - assert.NoError(t, err, clues.ToCore(err)) - assert.Greater(t, len(lists), 0) - t.Logf("Length: %d\n", len(lists)) -} - -func (suite *ListsUnitSuite) TestSharePointInfo() { - tests := []struct { - name string - listAndDeets func() (models.Listable, *details.SharePointInfo) - }{ - { - name: "Empty List", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - i := &details.SharePointInfo{ItemType: details.SharePointList} - return models.NewList(), i - }, - }, { - name: "Only Name", - listAndDeets: func() (models.Listable, *details.SharePointInfo) { - aTitle := "Whole List" - listing := models.NewList() - listing.SetDisplayName(&aTitle) - i := &details.SharePointInfo{ - ItemType: details.SharePointList, - ItemName: aTitle, - } - - return listing, i - }, - }, - } - for _, test := range tests { - suite.Run(test.name, func() { - t := suite.T() - - list, expected := test.listAndDeets() - info := ListToSPInfo(list, 10) - assert.Equal(t, expected.ItemType, info.ItemType) - assert.Equal(t, expected.ItemName, info.ItemName) - assert.Equal(t, expected.WebURL, info.WebURL) - }) - } -} diff --git a/src/internal/m365/collection/site/restore.go b/src/internal/m365/collection/site/restore.go index 5dd6c238e..a6a57cd66 100644 --- a/src/internal/m365/collection/site/restore.go +++ b/src/internal/m365/collection/site/restore.go @@ -200,7 +200,7 @@ func restoreListItem( } } - dii.SharePoint = ListToSPInfo(restoredList, int64(len(byteArray))) + dii.SharePoint = api.ListToSPInfo(restoredList) return dii, nil } diff --git a/src/internal/m365/service/sharepoint/backup.go b/src/internal/m365/service/sharepoint/backup.go index 9961578a9..d4f82c0bf 100644 --- a/src/internal/m365/service/sharepoint/backup.go +++ b/src/internal/m365/service/sharepoint/backup.go @@ -55,8 +55,11 @@ func ProduceBackupCollections( switch scope.Category().PathType() { case path.ListsCategory: + bh := site.NewListsBackupHandler(bpc.ProtectedResource.ID(), ac.Lists()) + spcs, err = site.CollectLists( ctx, + bh, bpc, ac, creds.AzureTenantID, diff --git a/src/internal/operations/test/m365/sharepoint/sharepoint_test.go b/src/internal/operations/test/m365/sharepoint/sharepoint_test.go index f7949e1fd..520f81990 100644 --- a/src/internal/operations/test/m365/sharepoint/sharepoint_test.go +++ b/src/internal/operations/test/m365/sharepoint/sharepoint_test.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/internal/version" + deeTD "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/count" @@ -61,6 +62,67 @@ func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePoint() { sel.Selector) } +func (suite *SharePointBackupIntgSuite) TestBackup_Run_sharePointList() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + resourceID = suite.its.Site.ID + sel = selectors.NewSharePointBackup([]string{resourceID}) + tenID = tconfig.M365TenantID(t) + mb = evmock.NewBus() + counter = count.New() + ws = deeTD.CategoryFromRepoRef + ) + + sel.Include(selTD.SharePointBackupListsScope(sel)) + + bo, bod := PrepNewTestBackupOp( + t, + ctx, + mb, + sel.Selector, + control.DefaultOptions(), + version.Backup, + counter) + defer bod.Close(t, ctx) + + RunAndCheckBackup(t, ctx, &bo, mb, false) + CheckBackupIsInManifests( + t, + ctx, + bod.KW, + bod.SW, + &bo, + bod.Sel, + bod.Sel.ID(), + path.ListsCategory) + + bID := bo.Results.BackupID + + _, expectDeets := deeTD.GetDeetsInBackup( + t, + ctx, + bID, + tenID, + bod.Sel.ID(), + path.SharePointService, + ws, + bod.KMS, + bod.SSS) + deeTD.CheckBackupDetails( + t, + ctx, + bID, + ws, + bod.KMS, + bod.SSS, + expectDeets, + false) +} + func (suite *SharePointBackupIntgSuite) TestBackup_Run_incrementalSharePoint() { runSharePointIncrementalBackupTests(suite, suite.its, control.DefaultOptions()) } diff --git a/src/pkg/selectors/testdata/sharepoint.go b/src/pkg/selectors/testdata/sharepoint.go index e6c3be58b..23760ed49 100644 --- a/src/pkg/selectors/testdata/sharepoint.go +++ b/src/pkg/selectors/testdata/sharepoint.go @@ -4,8 +4,14 @@ import ( "github.com/alcionai/corso/src/pkg/selectors" ) +const TestListName = "test-list" + // SharePointBackupFolderScope is the standard folder scope that should be used // in integration backups with sharepoint. func SharePointBackupFolderScope(sel *selectors.SharePointBackup) []selectors.SharePointScope { return sel.LibraryFolders([]string{TestFolderName}, selectors.PrefixMatch()) } + +func SharePointBackupListsScope(sel *selectors.SharePointBackup) []selectors.SharePointScope { + return sel.ListItems([]string{TestListName}, selectors.Any()) +}