From 87104ce404ea70719dd1b2535828619947bea560 Mon Sep 17 00:00:00 2001 From: Hitesh Pattanayak <48874082+HiteshRepo@users.noreply.github.com> Date: Mon, 8 Jan 2024 11:27:32 +0530 Subject: [PATCH] exports sharepoint lists (#4959) provision to export sharepoint lists #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) #4752 #### Test Plan - [x] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/m365/collection/site/export.go | 71 +++++++++ .../m365/collection/site/export_test.go | 138 ++++++++++++++++++ .../m365/service/sharepoint/export.go | 58 +++++--- .../m365/service/sharepoint/export_test.go | 117 +++++++++++---- 4 files changed, 337 insertions(+), 47 deletions(-) create mode 100644 src/internal/m365/collection/site/export.go create mode 100644 src/internal/m365/collection/site/export_test.go diff --git a/src/internal/m365/collection/site/export.go b/src/internal/m365/collection/site/export.go new file mode 100644 index 000000000..da535b313 --- /dev/null +++ b/src/internal/m365/collection/site/export.go @@ -0,0 +1,71 @@ +package site + +import ( + "context" + + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/metrics" + "github.com/alcionai/corso/src/pkg/path" +) + +func NewExportCollection( + baseDir string, + backingCollection []data.RestoreCollection, + backupVersion int, + stats *metrics.ExportStats, +) export.Collectioner { + return export.BaseCollection{ + BaseDir: baseDir, + BackingCollection: backingCollection, + BackupVersion: backupVersion, + Stream: streamItems, + Stats: stats, + } +} + +func streamItems( + ctx context.Context, + drc []data.RestoreCollection, + backupVersion int, + config control.ExportConfig, + ch chan<- export.Item, + stats *metrics.ExportStats, +) { + defer close(ch) + + errs := fault.New(false) + + for _, rc := range drc { + for item := range rc.Items(ctx, errs) { + stats.UpdateResourceCount(path.ListsCategory) + body := metrics.ReaderWithStats(item.ToReader(), path.ListsCategory, stats) + + name := item.ID() + ".json" + + ch <- export.Item{ + ID: item.ID(), + Name: name, + Body: body, + } + } + + items, recovered := errs.ItemsAndRecovered() + + // Return all the items that we failed to source from the persistence layer + for _, item := range items { + ch <- export.Item{ + ID: item.ID, + Error: &item, + } + } + + for _, err := range recovered { + ch <- export.Item{ + Error: err, + } + } + } +} diff --git a/src/internal/m365/collection/site/export_test.go b/src/internal/m365/collection/site/export_test.go new file mode 100644 index 000000000..8489c9d71 --- /dev/null +++ b/src/internal/m365/collection/site/export_test.go @@ -0,0 +1,138 @@ +package site + +import ( + "bytes" + "io" + "testing" + + "github.com/alcionai/clues" + kjson "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/data" + dataMock "github.com/alcionai/corso/src/internal/data/mock" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/metrics" +) + +type ExportUnitSuite struct { + tester.Suite +} + +func TestExportUnitSuite(t *testing.T) { + suite.Run(t, &ExportUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ExportUnitSuite) TestStreamItems() { + t := suite.T() + + table := []struct { + name string + backingColl dataMock.Collection + expectName string + expectErr assert.ErrorAssertionFunc + }{ + { + name: "no errors", + backingColl: dataMock.Collection{ + ItemData: []data.Item{ + &dataMock.Item{ + ItemID: "list1", + Reader: makeListJSONReader(t, "list1"), + }, + }, + }, + expectName: "list1.json", + expectErr: assert.NoError, + }, + { + name: "only recoverable errors", + backingColl: dataMock.Collection{ + ItemsRecoverableErrs: []error{ + clues.New("some error"), + }, + }, + expectErr: assert.Error, + }, + { + name: "items and recoverable errors", + backingColl: dataMock.Collection{ + ItemData: []data.Item{ + &dataMock.Item{ + ItemID: "list2", + Reader: makeListJSONReader(t, "list2"), + }, + }, + ItemsRecoverableErrs: []error{ + clues.New("some error"), + }, + }, + expectName: "list2.json", + expectErr: assert.Error, + }, + } + + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ch := make(chan export.Item) + + go streamItems( + ctx, + []data.RestoreCollection{test.backingColl}, + version.NoBackup, + control.DefaultExportConfig(), + ch, + &metrics.ExportStats{}) + + var ( + itm export.Item + err error + ) + + for i := range ch { + if i.Error == nil { + itm = i + } else { + err = i.Error + } + } + + test.expectErr(t, err, clues.ToCore(err)) + + assert.Equal(t, test.expectName, itm.Name, "item name") + }) + } +} + +func makeListJSONReader(t *testing.T, listName string) io.ReadCloser { + listBytes := getListBytes(t, listName) + return io.NopCloser(bytes.NewReader(listBytes)) +} + +func getListBytes(t *testing.T, listName string) []byte { + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + list := models.NewList() + list.SetId(ptr.To(listName)) + + err := writer.WriteObjectValue("", list) + require.NoError(t, err) + + storedListBytes, err := writer.GetSerializedContent() + require.NoError(t, err) + + return storedListBytes +} diff --git a/src/internal/m365/service/sharepoint/export.go b/src/internal/m365/service/sharepoint/export.go index 2ad2ee3de..d314fe38c 100644 --- a/src/internal/m365/service/sharepoint/export.go +++ b/src/internal/m365/service/sharepoint/export.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/m365/collection/site" "github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" @@ -72,30 +73,51 @@ func (h *baseSharePointHandler) ProduceExportCollections( ) for _, dc := range dcs { - drivePath, err := path.ToDrivePath(dc.FullPath()) - if err != nil { - return nil, clues.WrapWC(ctx, err, "transforming path to drive path") - } + cat := dc.FullPath().Category() - driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID) - if !ok { - // This should not happen, but just in case - logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") - driveName = drivePath.DriveID - } + ictx := clues.Add(ctx, "fullpath_category", cat) - baseDir := path.Builder{}. - Append(path.LibrariesCategory.HumanString()). - Append(driveName). - Append(drivePath.Folders...) + switch cat { + case path.LibrariesCategory: + drivePath, err := path.ToDrivePath(dc.FullPath()) + if err != nil { + return nil, clues.WrapWC(ictx, err, "transforming path to drive path") + } - ec = append( - ec, - drive.NewExportCollection( + driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID) + if !ok { + // This should not happen, but just in case + logger.Ctx(ictx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") + driveName = drivePath.DriveID + } + + baseDir := path.Builder{}. + Append(path.LibrariesCategory.HumanString()). + Append(driveName). + Append(drivePath.Folders...) + + coll := drive.NewExportCollection( baseDir.String(), []data.RestoreCollection{dc}, backupVersion, - stats)) + stats) + + ec = append(ec, coll) + case path.ListsCategory: + folders := dc.FullPath().Folders() + pth := path.Builder{}.Append(path.ListsCategory.HumanString()).Append(folders...) + + ec = append( + ec, + site.NewExportCollection( + pth.String(), + []data.RestoreCollection{dc}, + backupVersion, + stats)) + default: + return nil, clues.NewWC(ctx, "data category not supported"). + With("category", cat) + } } return ec, el.Failure() diff --git a/src/internal/m365/service/sharepoint/export_test.go b/src/internal/m365/service/sharepoint/export_test.go index 366dfa77f..eb680b6d5 100644 --- a/src/internal/m365/service/sharepoint/export_test.go +++ b/src/internal/m365/service/sharepoint/export_test.go @@ -60,51 +60,110 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { defer flush() var ( - driveID = "driveID1" - driveName = "driveName1" - itemName = "name1" - exportCfg = control.ExportConfig{} - dpb = odConsts.DriveFolderPrefixBuilder(driveID) - expectedPath = path.LibrariesCategory.HumanString() + "/" + driveName - expectedItems = []export.Item{ - { - ID: "id1.data", - Name: itemName, - Body: io.NopCloser((bytes.NewBufferString("body1"))), - }, - } + driveID = "driveID1" + driveName = "driveName1" + exportCfg = control.ExportConfig{} + dpb = odConsts.DriveFolderPrefixBuilder(driveID) ) - p, err := dpb.ToDataLayerSharePointPath("t", "u", path.LibrariesCategory, false) - assert.NoError(t, err, "build path") - table := []struct { - name string - itemInfo details.ItemInfo + name string + itemName string + itemID string + itemInfo details.ItemInfo + getCollPath func(t *testing.T) path.Path + statsCat path.CategoryType + expectedItems []export.Item + expectedPath string }{ { - name: "OneDriveLegacyItemInfo", + name: "OneDriveLegacyItemInfo", + itemName: "name1", + itemID: "id1.data", itemInfo: details.ItemInfo{ OneDrive: &details.OneDriveInfo{ ItemType: details.OneDriveItem, - ItemName: itemName, + ItemName: "name1", Size: 1, DriveName: driveName, DriveID: driveID, }, }, + getCollPath: func(t *testing.T) path.Path { + p, err := dpb.ToDataLayerSharePointPath("t", "u", path.LibrariesCategory, false) + assert.NoError(t, err, "build path") + + return p + }, + statsCat: path.FilesCategory, + expectedPath: path.LibrariesCategory.HumanString() + "/" + driveName, + expectedItems: []export.Item{ + { + ID: "id1.data", + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, }, { - name: "SharePointItemInfo", + name: "SharePointItemInfo, Libraries Category", + itemName: "name1", + itemID: "id1.data", itemInfo: details.ItemInfo{ SharePoint: &details.SharePointInfo{ ItemType: details.SharePointLibrary, - ItemName: itemName, + ItemName: "name1", Size: 1, DriveName: driveName, DriveID: driveID, }, }, + getCollPath: func(t *testing.T) path.Path { + p, err := dpb.ToDataLayerSharePointPath("t", "u", path.LibrariesCategory, false) + assert.NoError(t, err, "build path") + + return p + }, + statsCat: path.FilesCategory, + expectedPath: path.LibrariesCategory.HumanString() + "/" + driveName, + expectedItems: []export.Item{ + { + ID: "id1.data", + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, + }, + { + name: "SharePointItemInfo, Lists Category", + itemName: "list1", + itemID: "listid1", + itemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointList, + List: &details.ListInfo{ + Name: "list1", + ItemCount: 10, + }, + }, + }, + getCollPath: func(t *testing.T) path.Path { + p, err := path.Elements{"listid1"}. + Builder(). + ToDataLayerSharePointListPath("t", "u", path.ListsCategory, false) + assert.NoError(t, err, "build path") + + return p + }, + statsCat: path.ListsCategory, + expectedPath: path.ListsCategory.HumanString() + "/listid1", + expectedItems: []export.Item{ + { + ID: "listid1", + Name: "listid1.json", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + }, }, } @@ -115,15 +174,15 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { dcs := []data.RestoreCollection{ data.FetchRestoreCollection{ Collection: dataMock.Collection{ - Path: p, + Path: test.getCollPath(t), ItemData: []data.Item{ &dataMock.Item{ - ItemID: "id1.data", + ItemID: test.itemID, Reader: io.NopCloser(bytes.NewBufferString("body1")), }, }, }, - FetchItemByNamer: finD{id: "id1.meta", name: itemName}, + FetchItemByNamer: finD{id: "id1.meta", name: test.itemName}, }, } @@ -142,7 +201,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") - assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir") + assert.Equal(t, test.expectedPath, ecs[0].BasePath(), "base dir") fitems := []export.Item{} size := 0 @@ -159,11 +218,11 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { fitems = append(fitems, item) } - assert.Equal(t, expectedItems, fitems, "items") + assert.Equal(t, test.expectedItems, fitems, "items") expectedStats := metrics.ExportStats{} - expectedStats.UpdateBytes(path.FilesCategory, int64(size)) - expectedStats.UpdateResourceCount(path.FilesCategory) + expectedStats.UpdateBytes(test.statsCat, int64(size)) + expectedStats.UpdateResourceCount(test.statsCat) assert.Equal(t, expectedStats, stats, "stats") }) }