Add export support for Group SharePoint (#4260)

<!-- PR description-->

Ideally goes in after https://github.com/alcionai/corso/pull/4257 .
We could merge this instead of https://github.com/alcionai/corso/pull/4258 but I thought we could skip for now as this as been tested much less. But outside of me being paranoid, I think this should be in a good shape.

If this do go in, we can update the CHANGELOG entry to say that we do support export for SP libs.

This will also need more e2e(sanity) tests which I'll add in a follow up PR.

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes https://github.com/alcionai/corso/issues/4259

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

src/internal/m365/collection/drive/export.go

@@ -76,7 +76,7 @@ func streamItems(
 
 // isMetadataFile is used to determine if a path corresponds to a
 // metadata file.  This is OneDrive specific logic and depends on the
-// version of the backup unlike metadata.IsMetadataFile which only has
+// version of the backup unlike metadata.isMetadataFile which only has
 // to be concerned about the current version.
 func isMetadataFile(id string, backupVersion int) bool {
 	if backupVersion < version.OneDrive1DataAndMetaFiles {

src/internal/m365/controller.go

@@ -50,10 +50,15 @@ type Controller struct {
 	mu     sync.Mutex
 	status support.ControllerOperationStatus // contains the status of the last run status
 
-	// backupDriveIDNames is populated on restore.  It maps the backup's
+	// backupDriveIDNames is populated on restore and export.  It maps
-	// drive names to their id. Primarily for use when creating or looking
+	// the backup's drive names to their id. Primarily for use when
-	// up a new drive.
+	// creating or looking up a new drive.
 	backupDriveIDNames idname.CacheBuilder
+
+	// backupSiteIDWebURL is populated on restore and export. It maps
+	// the backup's site names to their id. Primarily for use in
+	// exports for groups.
+	backupSiteIDWebURL idname.CacheBuilder
 }
 
 func NewController(
@@ -99,6 +104,7 @@ func NewController(
 		tenant:             acct.ID(),
 		wg:                 &sync.WaitGroup{},
 		backupDriveIDNames: idname.NewCache(nil),
+		backupSiteIDWebURL: idname.NewCache(nil),
 	}
 
 	return &ctrl, nil
@@ -163,10 +169,12 @@ func (ctrl *Controller) incrementAwaitingMessages() {
 func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) {
 	if dii.Groups != nil {
 		ctrl.backupDriveIDNames.Add(dii.Groups.DriveID, dii.Groups.DriveName)
+		ctrl.backupSiteIDWebURL.Add(dii.Groups.SiteID, dii.Groups.WebURL)
 	}
 
 	if dii.SharePoint != nil {
 		ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName)
+		ctrl.backupSiteIDWebURL.Add(dii.SharePoint.SiteID, dii.SharePoint.WebURL)
 	}
 
 	if dii.OneDrive != nil {

src/internal/m365/controller_test.go

@@ -270,6 +270,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 		odname = "od-name"
 		spid   = "sp-id"
 		spname = "sp-name"
+		spsid  = "sp-sid"
+		spurl  = "sp-url"
 		gpid   = "gp-id"
 		gpname = "gp-name"
 		// intentionally declared outside the test loop
@@ -277,32 +279,35 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 			wg:                 &sync.WaitGroup{},
 			region:             &trace.Region{},
 			backupDriveIDNames: idname.NewCache(nil),
+			backupSiteIDWebURL: idname.NewCache(nil),
 		}
 	)
 
 	table := []struct {
-		name       string
+		name             string
-		service    path.ServiceType
+		service          path.ServiceType
-		cat        path.CategoryType
+		cat              path.CategoryType
-		dii        details.ItemInfo
+		dii              details.ItemInfo
-		expectID   string
+		expectDriveID    string
-		expectName string
+		expectDriveName  string
+		expectSiteID     string
+		expectSiteWebURL string
 	}{
 		{
 			name: "exchange",
 			dii: details.ItemInfo{
 				Exchange: &details.ExchangeInfo{},
 			},
-			expectID:   "",
+			expectDriveID:   "",
-			expectName: "",
+			expectDriveName: "",
 		},
 		{
 			name: "folder",
 			dii: details.ItemInfo{
 				Folder: &details.FolderInfo{},
 			},
-			expectID:   "",
+			expectDriveID:   "",
-			expectName: "",
+			expectDriveName: "",
 		},
 		{
 			name: "onedrive",
@@ -312,8 +317,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 					DriveName: odname,
 				},
 			},
-			expectID:   odid,
+			expectDriveID:   odid,
-			expectName: odname,
+			expectDriveName: odname,
 		},
 		{
 			name: "sharepoint",
@@ -321,10 +326,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 				SharePoint: &details.SharePointInfo{
 					DriveID:   spid,
 					DriveName: spname,
+					SiteID:    spsid,
+					WebURL:    spurl,
 				},
 			},
-			expectID:   spid,
+			expectDriveID:    spid,
-			expectName: spname,
+			expectDriveName:  spname,
+			expectSiteID:     spsid,
+			expectSiteWebURL: spurl,
 		},
 		{
 			name: "groups",
@@ -332,10 +341,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 				Groups: &details.GroupsInfo{
 					DriveID:   gpid,
 					DriveName: gpname,
+					SiteID:    spsid,
+					WebURL:    spurl,
 				},
 			},
-			expectID:   gpid,
+			expectDriveID:    gpid,
-			expectName: gpname,
+			expectDriveName:  gpname,
+			expectSiteID:     spsid,
+			expectSiteWebURL: spurl,
 		},
 	}
 	for _, test := range table {
@@ -344,11 +357,17 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() {
 
 			ctrl.CacheItemInfo(test.dii)
 
-			name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID)
+			name, _ := ctrl.backupDriveIDNames.NameOf(test.expectDriveID)
-			assert.Equal(t, test.expectName, name)
+			assert.Equal(t, test.expectDriveName, name)
 
-			id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName)
+			id, _ := ctrl.backupDriveIDNames.IDOf(test.expectDriveName)
-			assert.Equal(t, test.expectID, id)
+			assert.Equal(t, test.expectDriveID, id)
+
+			url, _ := ctrl.backupSiteIDWebURL.NameOf(test.expectSiteID)
+			assert.Equal(t, test.expectSiteWebURL, url)
+
+			sid, _ := ctrl.backupSiteIDWebURL.IDOf(test.expectSiteWebURL)
+			assert.Equal(t, test.expectSiteID, sid)
 		})
 	}
 }

src/internal/m365/export.go

@@ -69,6 +69,8 @@ func (ctrl *Controller) ProduceExportCollections(
 			exportCfg,
 			opts,
 			dcs,
+			ctrl.backupDriveIDNames,
+			ctrl.backupSiteIDWebURL,
 			deets,
 			errs)
 

src/internal/m365/service/groups/export.go

@@ -2,13 +2,19 @@ package groups
 
 import (
 	"context"
+	stdlibpath "path"
 
+	"github.com/alcionai/clues"
+
+	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/internal/m365/collection/drive"
 	"github.com/alcionai/corso/src/internal/m365/collection/groups"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/export"
 	"github.com/alcionai/corso/src/pkg/fault"
+	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
@@ -20,6 +26,8 @@ func ProduceExportCollections(
 	exportCfg control.ExportConfig,
 	opts control.Options,
 	dcs []data.RestoreCollection,
+	backupDriveIDNames idname.Cacher,
+	backupSiteIDWebURL idname.Cacher,
 	deets *details.Builder,
 	errs *fault.Bus,
 ) ([]export.Collectioner, error) {
@@ -33,21 +41,65 @@ func ProduceExportCollections(
 			fp      = restoreColl.FullPath()
 			cat     = fp.Category()
 			folders = []string{cat.String()}
+			coll    export.Collectioner
 		)
 
 		switch cat {
 		case path.ChannelMessagesCategory:
 			folders = append(folders, fp.Folders()...)
+
+			coll = groups.NewExportCollection(
+				path.Builder{}.Append(folders...).String(),
+				[]data.RestoreCollection{restoreColl},
+				backupVersion,
+				exportCfg)
+		case path.LibrariesCategory:
+			drivePath, err := path.ToDrivePath(restoreColl.FullPath())
+			if err != nil {
+				return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx)
+			}
+
+			driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID)
+			if !ok {
+				// This should not happen, but just in case
+				logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id")
+				driveName = drivePath.DriveID
+			}
+
+			folders := restoreColl.FullPath().Folders()
+			siteName := folders[1] // use siteID by default
+
+			webURL, ok := backupSiteIDWebURL.NameOf(siteName)
+			if !ok {
+				// This should not happen, but just in case
+				logger.Ctx(ctx).With("site_id", folders[1]).Info("site weburl not found, using site id")
+			}
+
+			if len(webURL) != 0 {
+				// We can't use the actual name anyways as it might
+				// contain invalid characters. This should also avoid
+				// possibility of name collisions.
+				siteName = stdlibpath.Base(webURL)
+			}
+
+			baseDir := path.Builder{}.
+				Append("Libraries").
+				Append(siteName).
+				Append(driveName).
+				Append(drivePath.Folders...)
+
+			coll = drive.NewExportCollection(
+				baseDir.String(),
+				[]data.RestoreCollection{restoreColl},
+				backupVersion)
 		default:
+			el.AddRecoverable(
+				ctx,
+				clues.New("unsupported category for export").With("category", cat))
+
 			continue
 		}
 
-		coll := groups.NewExportCollection(
-			path.Builder{}.Append(folders...).String(),
-			[]data.RestoreCollection{restoreColl},
-			backupVersion,
-			exportCfg)
-
 		ec = append(ec, coll)
 	}
 

src/internal/m365/service/groups/export_test.go

@@ -4,14 +4,18 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/alcionai/corso/src/internal/common/idname"
 	"github.com/alcionai/corso/src/internal/data"
 	dataMock "github.com/alcionai/corso/src/internal/data/mock"
 	groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock"
+	odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
+	odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/internal/version"
 	"github.com/alcionai/corso/src/pkg/control"
@@ -30,6 +34,7 @@ func TestExportUnitSuite(t *testing.T) {
 
 type finD struct {
 	id   string
+	key  string
 	name string
 	err  error
 }
@@ -42,14 +47,14 @@ func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Item, err
 	if name == fd.id {
 		return &dataMock.Item{
 			ItemID: fd.id,
-			Reader: io.NopCloser(bytes.NewBufferString(`{"displayname": "` + fd.name + `"}`)),
+			Reader: io.NopCloser(bytes.NewBufferString(`{"` + fd.key + `": "` + fd.name + `"}`)),
 		}, nil
 	}
 
 	return nil, assert.AnError
 }
 
-func (suite *ExportUnitSuite) TestExportRestoreCollections() {
+func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() {
 	t := suite.T()
 
 	ctx, flush := tester.NewContext(t)
@@ -87,7 +92,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
 					},
 				},
 			},
-			FetchItemByNamer: finD{id: itemID, name: dii.Groups.ItemName},
+			FetchItemByNamer: finD{id: itemID, key: "displayname", name: dii.Groups.ItemName},
 		},
 	}
 
@@ -98,6 +103,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
 		control.DefaultOptions(),
 		dcs,
 		nil,
+		nil,
+		nil,
 		fault.New(true))
 	assert.NoError(t, err, "export collections error")
 	assert.Len(t, ecs, 1, "num of collections")
@@ -115,3 +122,86 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
 
 	assert.Equal(t, expectedItems, fitems, "items")
 }
+
+func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	var (
+		siteID          = "siteID1"
+		siteEscapedName = "siteName1"
+		siteWebURL      = "https://site1.sharepoint.com/sites/" + siteEscapedName
+		driveID         = "driveID1"
+		driveName       = "driveName1"
+		exportCfg       = control.ExportConfig{}
+		dpb             = odConsts.DriveFolderPrefixBuilder(driveID)
+		driveNameCache  = idname.NewCache(
+			// Cache check with lowercased ids
+			map[string]string{strings.ToLower(driveID): driveName})
+		siteWebURLCache = idname.NewCache(
+			// Cache check with lowercased ids
+			map[string]string{strings.ToLower(siteID): siteWebURL})
+		dii           = odStub.DriveItemInfo()
+		expectedPath  = "Libraries/" + siteEscapedName + "/" + driveName
+		expectedItems = []export.Item{
+			{
+				ID:   "id1.data",
+				Name: "name1",
+				Body: io.NopCloser((bytes.NewBufferString("body1"))),
+			},
+		}
+	)
+
+	dii.OneDrive.ItemName = "name1"
+
+	p, err := dpb.ToDataLayerPath(
+		"t",
+		"u",
+		path.GroupsService,
+		path.LibrariesCategory,
+		false,
+		odConsts.SitesPathDir,
+		siteID)
+	assert.NoError(t, err, "build path")
+
+	dcs := []data.RestoreCollection{
+		data.FetchRestoreCollection{
+			Collection: dataMock.Collection{
+				Path: p,
+				ItemData: []data.Item{
+					&dataMock.Item{
+						ItemID:   "id1.data",
+						Reader:   io.NopCloser(bytes.NewBufferString("body1")),
+						ItemInfo: dii,
+					},
+				},
+			},
+			FetchItemByNamer: finD{id: "id1.meta", key: "filename", name: "name1"},
+		},
+	}
+
+	ecs, err := ProduceExportCollections(
+		ctx,
+		int(version.Backup),
+		exportCfg,
+		control.DefaultOptions(),
+		dcs,
+		driveNameCache,
+		siteWebURLCache,
+		nil,
+		fault.New(true))
+	assert.NoError(t, err, "export collections error")
+	assert.Len(t, ecs, 1, "num of collections")
+
+	assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir")
+
+	fitems := []export.Item{}
+
+	for item := range ecs[0].Items(ctx) {
+		fitems = append(fitems, item)
+	}
+
+	assert.Equal(t, expectedItems, fitems, "items")
+}