larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add export support for Group SharePoint (#4260)

Browse Source 
<!-- PR description-->

Ideally goes in after https://github.com/alcionai/corso/pull/4257 .
We could merge this instead of https://github.com/alcionai/corso/pull/4258 but I thought we could skip for now as this as been tested much less. But outside of me being paranoid, I think this should be in a good shape.

If this do go in, we can update the CHANGELOG entry to say that we do support export for SP libs.

This will also need more e2e(sanity) tests which I'll add in a follow up PR.

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes https://github.com/alcionai/corso/issues/4259

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2023-09-20 12:12:42 +05:30 committed by GitHub
parent 4814154928
commit 1389b53eaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 204 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/internal/m365/collection/drive/export.go
View File

@ -76,7 +76,7 @@ func streamItems(
// isMetadataFile is used to determine if a path corresponds to a
// metadata file. This is OneDrive specific logic and depends on the
// version of the backup unlike metadata.IsMetadataFile which only has
// version of the backup unlike metadata.isMetadataFile which only has
// to be concerned about the current version.
func isMetadataFile(id string, backupVersion int) bool {
if backupVersion < version.OneDrive1DataAndMetaFiles {

14
src/internal/m365/controller.go
View File

@ -50,10 +50,15 @@ type Controller struct {
mu sync.Mutex
status support.ControllerOperationStatus // contains the status of the last run status
// backupDriveIDNames is populated on restore. It maps the backup's
// drive names to their id. Primarily for use when creating or looking
// up a new drive.
// backupDriveIDNames is populated on restore and export. It maps
// the backup's drive names to their id. Primarily for use when
// creating or looking up a new drive.
backupDriveIDNames idname.CacheBuilder
// backupSiteIDWebURL is populated on restore and export. It maps
// the backup's site names to their id. Primarily for use in
// exports for groups.
backupSiteIDWebURL idname.CacheBuilder
}
func NewController(
@ -99,6 +104,7 @@ func NewController(
tenant: acct.ID(),
wg: &sync.WaitGroup{},
backupDriveIDNames: idname.NewCache(nil),
backupSiteIDWebURL: idname.NewCache(nil),
}
return &ctrl, nil
@ -163,10 +169,12 @@ func (ctrl *Controller) incrementAwaitingMessages() {
func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) {
if dii.Groups != nil {
ctrl.backupDriveIDNames.Add(dii.Groups.DriveID, dii.Groups.DriveName)
ctrl.backupSiteIDWebURL.Add(dii.Groups.SiteID, dii.Groups.WebURL)
}
if dii.SharePoint != nil {
ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName)
ctrl.backupSiteIDWebURL.Add(dii.SharePoint.SiteID, dii.SharePoint.WebURL)
}
if dii.OneDrive != nil {

59
src/internal/m365/controller_test.go
View File

@ -270,6 +270,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
odname = "od-name"
spid = "sp-id"
spname = "sp-name"
spsid = "sp-sid"
spurl = "sp-url"
gpid = "gp-id"
gpname = "gp-name"
// intentionally declared outside the test loop
@ -277,32 +279,35 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
wg: &sync.WaitGroup{},
region: &trace.Region{},
backupDriveIDNames: idname.NewCache(nil),
backupSiteIDWebURL: idname.NewCache(nil),
}
)
table := []struct {
name string
service path.ServiceType
cat path.CategoryType
dii details.ItemInfo
expectID string
expectName string
name string
service path.ServiceType
cat path.CategoryType
dii details.ItemInfo
expectDriveID string
expectDriveName string
expectSiteID string
expectSiteWebURL string
}{
{
name: "exchange",
dii: details.ItemInfo{
Exchange: &details.ExchangeInfo{},
},
expectID: "",
expectName: "",
expectDriveID: "",
expectDriveName: "",
},
{
name: "folder",
dii: details.ItemInfo{
Folder: &details.FolderInfo{},
},
expectID: "",
expectName: "",
expectDriveID: "",
expectDriveName: "",
},
{
name: "onedrive",
@ -312,8 +317,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
DriveName: odname,
},
},
expectID: odid,
expectName: odname,
expectDriveID: odid,
expectDriveName: odname,
},
{
name: "sharepoint",
@ -321,10 +326,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
SharePoint: &details.SharePointInfo{
DriveID: spid,
DriveName: spname,
SiteID: spsid,
WebURL: spurl,
},
},
expectID: spid,
expectName: spname,
expectDriveID: spid,
expectDriveName: spname,
expectSiteID: spsid,
expectSiteWebURL: spurl,
},
{
name: "groups",
@ -332,10 +341,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
Groups: &details.GroupsInfo{
DriveID: gpid,
DriveName: gpname,
SiteID: spsid,
WebURL: spurl,
},
},
expectID: gpid,
expectName: gpname,
expectDriveID: gpid,
expectDriveName: gpname,
expectSiteID: spsid,
expectSiteWebURL: spurl,
},
}
for _, test := range table {
@ -344,11 +357,17 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
ctrl.CacheItemInfo(test.dii)
name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID)
assert.Equal(t, test.expectName, name)
name, _ := ctrl.backupDriveIDNames.NameOf(test.expectDriveID)
assert.Equal(t, test.expectDriveName, name)
id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName)
assert.Equal(t, test.expectID, id)
id, _ := ctrl.backupDriveIDNames.IDOf(test.expectDriveName)
assert.Equal(t, test.expectDriveID, id)
url, _ := ctrl.backupSiteIDWebURL.NameOf(test.expectSiteID)
assert.Equal(t, test.expectSiteWebURL, url)
sid, _ := ctrl.backupSiteIDWebURL.IDOf(test.expectSiteWebURL)
assert.Equal(t, test.expectSiteID, sid)
})
}
}

2
src/internal/m365/export.go
View File

@ -69,6 +69,8 @@ func (ctrl *Controller) ProduceExportCollections(
exportCfg,
opts,
dcs,
ctrl.backupDriveIDNames,
ctrl.backupSiteIDWebURL,
deets,
errs)

64
src/internal/m365/service/groups/export.go
View File

@ -2,13 +2,19 @@ package groups
import (
"context"
stdlibpath "path"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
"github.com/alcionai/corso/src/internal/m365/collection/groups"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -20,6 +26,8 @@ func ProduceExportCollections(
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
backupDriveIDNames idname.Cacher,
backupSiteIDWebURL idname.Cacher,
deets *details.Builder,
errs *fault.Bus,
) ([]export.Collectioner, error) {
@ -33,21 +41,65 @@ func ProduceExportCollections(
fp = restoreColl.FullPath()
cat = fp.Category()
folders = []string{cat.String()}
coll export.Collectioner
)
switch cat {
case path.ChannelMessagesCategory:
folders = append(folders, fp.Folders()...)
coll = groups.NewExportCollection(
path.Builder{}.Append(folders...).String(),
[]data.RestoreCollection{restoreColl},
backupVersion,
exportCfg)
case path.LibrariesCategory:
drivePath, err := path.ToDrivePath(restoreColl.FullPath())
if err != nil {
return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx)
}
driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID)
if !ok {
// This should not happen, but just in case
logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id")
driveName = drivePath.DriveID
}
folders := restoreColl.FullPath().Folders()
siteName := folders[1] // use siteID by default
webURL, ok := backupSiteIDWebURL.NameOf(siteName)
if !ok {
// This should not happen, but just in case
logger.Ctx(ctx).With("site_id", folders[1]).Info("site weburl not found, using site id")
}
if len(webURL) != 0 {
// We can't use the actual name anyways as it might
// contain invalid characters. This should also avoid
// possibility of name collisions.
siteName = stdlibpath.Base(webURL)
}
baseDir := path.Builder{}.
Append("Libraries").
Append(siteName).
Append(driveName).
Append(drivePath.Folders...)
coll = drive.NewExportCollection(
baseDir.String(),
[]data.RestoreCollection{restoreColl},
backupVersion)
default:
el.AddRecoverable(
ctx,
clues.New("unsupported category for export").With("category", cat))
continue
}
coll := groups.NewExportCollection(
path.Builder{}.Append(folders...).String(),
[]data.RestoreCollection{restoreColl},
backupVersion,
exportCfg)
ec = append(ec, coll)
}

96
src/internal/m365/service/groups/export_test.go
View File

@ -4,14 +4,18 @@ import (
"bytes"
"context"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control"
@ -30,6 +34,7 @@ func TestExportUnitSuite(t *testing.T) {
type finD struct {
id string
key string
name string
err error
}
@ -42,14 +47,14 @@ func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Item, err
if name == fd.id {
return &dataMock.Item{
ItemID: fd.id,
Reader: io.NopCloser(bytes.NewBufferString(`{"displayname": "` + fd.name + `"}`)),
Reader: io.NopCloser(bytes.NewBufferString(`{"` + fd.key + `": "` + fd.name + `"}`)),
}, nil
}
return nil, assert.AnError
}
func (suite *ExportUnitSuite) TestExportRestoreCollections() {
func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() {
t := suite.T()
ctx, flush := tester.NewContext(t)
@ -87,7 +92,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
},
},
},
FetchItemByNamer: finD{id: itemID, name: dii.Groups.ItemName},
FetchItemByNamer: finD{id: itemID, key: "displayname", name: dii.Groups.ItemName},
},
}
@ -98,6 +103,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
control.DefaultOptions(),
dcs,
nil,
nil,
nil,
fault.New(true))
assert.NoError(t, err, "export collections error")
assert.Len(t, ecs, 1, "num of collections")
@ -115,3 +122,86 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
assert.Equal(t, expectedItems, fitems, "items")
}
func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
var (
siteID = "siteID1"
siteEscapedName = "siteName1"
siteWebURL = "https://site1.sharepoint.com/sites/" + siteEscapedName
driveID = "driveID1"
driveName = "driveName1"
exportCfg = control.ExportConfig{}
dpb = odConsts.DriveFolderPrefixBuilder(driveID)
driveNameCache = idname.NewCache(
// Cache check with lowercased ids
map[string]string{strings.ToLower(driveID): driveName})
siteWebURLCache = idname.NewCache(
// Cache check with lowercased ids
map[string]string{strings.ToLower(siteID): siteWebURL})
dii = odStub.DriveItemInfo()
expectedPath = "Libraries/" + siteEscapedName + "/" + driveName
expectedItems = []export.Item{
{
ID: "id1.data",
Name: "name1",
Body: io.NopCloser((bytes.NewBufferString("body1"))),
},
}
)
dii.OneDrive.ItemName = "name1"
p, err := dpb.ToDataLayerPath(
"t",
"u",
path.GroupsService,
path.LibrariesCategory,
false,
odConsts.SitesPathDir,
siteID)
assert.NoError(t, err, "build path")
dcs := []data.RestoreCollection{
data.FetchRestoreCollection{
Collection: dataMock.Collection{
Path: p,
ItemData: []data.Item{
&dataMock.Item{
ItemID: "id1.data",
Reader: io.NopCloser(bytes.NewBufferString("body1")),
ItemInfo: dii,
},
},
},
FetchItemByNamer: finD{id: "id1.meta", key: "filename", name: "name1"},
},
}
ecs, err := ProduceExportCollections(
ctx,
int(version.Backup),
exportCfg,
control.DefaultOptions(),
dcs,
driveNameCache,
siteWebURLCache,
nil,
fault.New(true))
assert.NoError(t, err, "export collections error")
assert.Len(t, ecs, 1, "num of collections")
assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir")
fitems := []export.Item{}
for item := range ecs[0].Items(ctx) {
fitems = append(fitems, item)
}
assert.Equal(t, expectedItems, fitems, "items")
}