From 1389b53eafa55b9467e40e6afad0ceabb60f2f70 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Wed, 20 Sep 2023 12:12:42 +0530 Subject: [PATCH] Add export support for Group SharePoint (#4260) Ideally goes in after https://github.com/alcionai/corso/pull/4257 . We could merge this instead of https://github.com/alcionai/corso/pull/4258 but I thought we could skip for now as this as been tested much less. But outside of me being paranoid, I think this should be in a good shape. If this do go in, we can update the CHANGELOG entry to say that we do support export for SP libs. This will also need more e2e(sanity) tests which I'll add in a follow up PR. --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [x] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * closes https://github.com/alcionai/corso/issues/4259 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/m365/collection/drive/export.go | 2 +- src/internal/m365/controller.go | 14 ++- src/internal/m365/controller_test.go | 59 ++++++++---- src/internal/m365/export.go | 2 + src/internal/m365/service/groups/export.go | 64 +++++++++++-- .../m365/service/groups/export_test.go | 96 ++++++++++++++++++- 6 files changed, 204 insertions(+), 33 deletions(-) diff --git a/src/internal/m365/collection/drive/export.go b/src/internal/m365/collection/drive/export.go index 60af17d2b..6c2200854 100644 --- a/src/internal/m365/collection/drive/export.go +++ b/src/internal/m365/collection/drive/export.go @@ -76,7 +76,7 @@ func streamItems( // isMetadataFile is used to determine if a path corresponds to a // metadata file. This is OneDrive specific logic and depends on the -// version of the backup unlike metadata.IsMetadataFile which only has +// version of the backup unlike metadata.isMetadataFile which only has // to be concerned about the current version. func isMetadataFile(id string, backupVersion int) bool { if backupVersion < version.OneDrive1DataAndMetaFiles { diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index e9f8161dc..e9e8b2228 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -50,10 +50,15 @@ type Controller struct { mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status - // backupDriveIDNames is populated on restore. It maps the backup's - // drive names to their id. Primarily for use when creating or looking - // up a new drive. + // backupDriveIDNames is populated on restore and export. It maps + // the backup's drive names to their id. Primarily for use when + // creating or looking up a new drive. backupDriveIDNames idname.CacheBuilder + + // backupSiteIDWebURL is populated on restore and export. It maps + // the backup's site names to their id. Primarily for use in + // exports for groups. + backupSiteIDWebURL idname.CacheBuilder } func NewController( @@ -99,6 +104,7 @@ func NewController( tenant: acct.ID(), wg: &sync.WaitGroup{}, backupDriveIDNames: idname.NewCache(nil), + backupSiteIDWebURL: idname.NewCache(nil), } return &ctrl, nil @@ -163,10 +169,12 @@ func (ctrl *Controller) incrementAwaitingMessages() { func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { if dii.Groups != nil { ctrl.backupDriveIDNames.Add(dii.Groups.DriveID, dii.Groups.DriveName) + ctrl.backupSiteIDWebURL.Add(dii.Groups.SiteID, dii.Groups.WebURL) } if dii.SharePoint != nil { ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + ctrl.backupSiteIDWebURL.Add(dii.SharePoint.SiteID, dii.SharePoint.WebURL) } if dii.OneDrive != nil { diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 991aa7955..72d9e4478 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -270,6 +270,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { odname = "od-name" spid = "sp-id" spname = "sp-name" + spsid = "sp-sid" + spurl = "sp-url" gpid = "gp-id" gpname = "gp-name" // intentionally declared outside the test loop @@ -277,32 +279,35 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { wg: &sync.WaitGroup{}, region: &trace.Region{}, backupDriveIDNames: idname.NewCache(nil), + backupSiteIDWebURL: idname.NewCache(nil), } ) table := []struct { - name string - service path.ServiceType - cat path.CategoryType - dii details.ItemInfo - expectID string - expectName string + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectDriveID string + expectDriveName string + expectSiteID string + expectSiteWebURL string }{ { name: "exchange", dii: details.ItemInfo{ Exchange: &details.ExchangeInfo{}, }, - expectID: "", - expectName: "", + expectDriveID: "", + expectDriveName: "", }, { name: "folder", dii: details.ItemInfo{ Folder: &details.FolderInfo{}, }, - expectID: "", - expectName: "", + expectDriveID: "", + expectDriveName: "", }, { name: "onedrive", @@ -312,8 +317,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { DriveName: odname, }, }, - expectID: odid, - expectName: odname, + expectDriveID: odid, + expectDriveName: odname, }, { name: "sharepoint", @@ -321,10 +326,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { SharePoint: &details.SharePointInfo{ DriveID: spid, DriveName: spname, + SiteID: spsid, + WebURL: spurl, }, }, - expectID: spid, - expectName: spname, + expectDriveID: spid, + expectDriveName: spname, + expectSiteID: spsid, + expectSiteWebURL: spurl, }, { name: "groups", @@ -332,10 +341,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { Groups: &details.GroupsInfo{ DriveID: gpid, DriveName: gpname, + SiteID: spsid, + WebURL: spurl, }, }, - expectID: gpid, - expectName: gpname, + expectDriveID: gpid, + expectDriveName: gpname, + expectSiteID: spsid, + expectSiteWebURL: spurl, }, } for _, test := range table { @@ -344,11 +357,17 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { ctrl.CacheItemInfo(test.dii) - name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) - assert.Equal(t, test.expectName, name) + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectDriveID) + assert.Equal(t, test.expectDriveName, name) - id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) - assert.Equal(t, test.expectID, id) + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectDriveName) + assert.Equal(t, test.expectDriveID, id) + + url, _ := ctrl.backupSiteIDWebURL.NameOf(test.expectSiteID) + assert.Equal(t, test.expectSiteWebURL, url) + + sid, _ := ctrl.backupSiteIDWebURL.IDOf(test.expectSiteWebURL) + assert.Equal(t, test.expectSiteID, sid) }) } } diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 94e00f295..ab7a94ceb 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -69,6 +69,8 @@ func (ctrl *Controller) ProduceExportCollections( exportCfg, opts, dcs, + ctrl.backupDriveIDNames, + ctrl.backupSiteIDWebURL, deets, errs) diff --git a/src/internal/m365/service/groups/export.go b/src/internal/m365/service/groups/export.go index 005fd3ee2..be433493f 100644 --- a/src/internal/m365/service/groups/export.go +++ b/src/internal/m365/service/groups/export.go @@ -2,13 +2,19 @@ package groups import ( "context" + stdlibpath "path" + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/collection/groups" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) @@ -20,6 +26,8 @@ func ProduceExportCollections( exportCfg control.ExportConfig, opts control.Options, dcs []data.RestoreCollection, + backupDriveIDNames idname.Cacher, + backupSiteIDWebURL idname.Cacher, deets *details.Builder, errs *fault.Bus, ) ([]export.Collectioner, error) { @@ -33,21 +41,65 @@ func ProduceExportCollections( fp = restoreColl.FullPath() cat = fp.Category() folders = []string{cat.String()} + coll export.Collectioner ) switch cat { case path.ChannelMessagesCategory: folders = append(folders, fp.Folders()...) + + coll = groups.NewExportCollection( + path.Builder{}.Append(folders...).String(), + []data.RestoreCollection{restoreColl}, + backupVersion, + exportCfg) + case path.LibrariesCategory: + drivePath, err := path.ToDrivePath(restoreColl.FullPath()) + if err != nil { + return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) + } + + driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID) + if !ok { + // This should not happen, but just in case + logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") + driveName = drivePath.DriveID + } + + folders := restoreColl.FullPath().Folders() + siteName := folders[1] // use siteID by default + + webURL, ok := backupSiteIDWebURL.NameOf(siteName) + if !ok { + // This should not happen, but just in case + logger.Ctx(ctx).With("site_id", folders[1]).Info("site weburl not found, using site id") + } + + if len(webURL) != 0 { + // We can't use the actual name anyways as it might + // contain invalid characters. This should also avoid + // possibility of name collisions. + siteName = stdlibpath.Base(webURL) + } + + baseDir := path.Builder{}. + Append("Libraries"). + Append(siteName). + Append(driveName). + Append(drivePath.Folders...) + + coll = drive.NewExportCollection( + baseDir.String(), + []data.RestoreCollection{restoreColl}, + backupVersion) default: + el.AddRecoverable( + ctx, + clues.New("unsupported category for export").With("category", cat)) + continue } - coll := groups.NewExportCollection( - path.Builder{}.Append(folders...).String(), - []data.RestoreCollection{restoreColl}, - backupVersion, - exportCfg) - ec = append(ec, coll) } diff --git a/src/internal/m365/service/groups/export_test.go b/src/internal/m365/service/groups/export_test.go index ffcf13036..a621760ac 100644 --- a/src/internal/m365/service/groups/export_test.go +++ b/src/internal/m365/service/groups/export_test.go @@ -4,14 +4,18 @@ import ( "bytes" "context" "io" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" @@ -30,6 +34,7 @@ func TestExportUnitSuite(t *testing.T) { type finD struct { id string + key string name string err error } @@ -42,14 +47,14 @@ func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Item, err if name == fd.id { return &dataMock.Item{ ItemID: fd.id, - Reader: io.NopCloser(bytes.NewBufferString(`{"displayname": "` + fd.name + `"}`)), + Reader: io.NopCloser(bytes.NewBufferString(`{"` + fd.key + `": "` + fd.name + `"}`)), }, nil } return nil, assert.AnError } -func (suite *ExportUnitSuite) TestExportRestoreCollections() { +func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -87,7 +92,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { }, }, }, - FetchItemByNamer: finD{id: itemID, name: dii.Groups.ItemName}, + FetchItemByNamer: finD{id: itemID, key: "displayname", name: dii.Groups.ItemName}, }, } @@ -98,6 +103,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { control.DefaultOptions(), dcs, nil, + nil, + nil, fault.New(true)) assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") @@ -115,3 +122,86 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { assert.Equal(t, expectedItems, fitems, "items") } + +func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + siteID = "siteID1" + siteEscapedName = "siteName1" + siteWebURL = "https://site1.sharepoint.com/sites/" + siteEscapedName + driveID = "driveID1" + driveName = "driveName1" + exportCfg = control.ExportConfig{} + dpb = odConsts.DriveFolderPrefixBuilder(driveID) + driveNameCache = idname.NewCache( + // Cache check with lowercased ids + map[string]string{strings.ToLower(driveID): driveName}) + siteWebURLCache = idname.NewCache( + // Cache check with lowercased ids + map[string]string{strings.ToLower(siteID): siteWebURL}) + dii = odStub.DriveItemInfo() + expectedPath = "Libraries/" + siteEscapedName + "/" + driveName + expectedItems = []export.Item{ + { + ID: "id1.data", + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + } + ) + + dii.OneDrive.ItemName = "name1" + + p, err := dpb.ToDataLayerPath( + "t", + "u", + path.GroupsService, + path.LibrariesCategory, + false, + odConsts.SitesPathDir, + siteID) + assert.NoError(t, err, "build path") + + dcs := []data.RestoreCollection{ + data.FetchRestoreCollection{ + Collection: dataMock.Collection{ + Path: p, + ItemData: []data.Item{ + &dataMock.Item{ + ItemID: "id1.data", + Reader: io.NopCloser(bytes.NewBufferString("body1")), + ItemInfo: dii, + }, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", key: "filename", name: "name1"}, + }, + } + + ecs, err := ProduceExportCollections( + ctx, + int(version.Backup), + exportCfg, + control.DefaultOptions(), + dcs, + driveNameCache, + siteWebURLCache, + nil, + fault.New(true)) + assert.NoError(t, err, "export collections error") + assert.Len(t, ecs, 1, "num of collections") + + assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir") + + fitems := []export.Item{} + + for item := range ecs[0].Items(ctx) { + fitems = append(fitems, item) + } + + assert.Equal(t, expectedItems, fitems, "items") +}