diff --git a/src/internal/m365/collection/drive/export.go b/src/internal/m365/collection/drive/export.go index 60af17d2b..6c2200854 100644 --- a/src/internal/m365/collection/drive/export.go +++ b/src/internal/m365/collection/drive/export.go @@ -76,7 +76,7 @@ func streamItems( // isMetadataFile is used to determine if a path corresponds to a // metadata file. This is OneDrive specific logic and depends on the -// version of the backup unlike metadata.IsMetadataFile which only has +// version of the backup unlike metadata.isMetadataFile which only has // to be concerned about the current version. func isMetadataFile(id string, backupVersion int) bool { if backupVersion < version.OneDrive1DataAndMetaFiles { diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index e9f8161dc..e9e8b2228 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -50,10 +50,15 @@ type Controller struct { mu sync.Mutex status support.ControllerOperationStatus // contains the status of the last run status - // backupDriveIDNames is populated on restore. It maps the backup's - // drive names to their id. Primarily for use when creating or looking - // up a new drive. + // backupDriveIDNames is populated on restore and export. It maps + // the backup's drive names to their id. Primarily for use when + // creating or looking up a new drive. backupDriveIDNames idname.CacheBuilder + + // backupSiteIDWebURL is populated on restore and export. It maps + // the backup's site names to their id. Primarily for use in + // exports for groups. + backupSiteIDWebURL idname.CacheBuilder } func NewController( @@ -99,6 +104,7 @@ func NewController( tenant: acct.ID(), wg: &sync.WaitGroup{}, backupDriveIDNames: idname.NewCache(nil), + backupSiteIDWebURL: idname.NewCache(nil), } return &ctrl, nil @@ -163,10 +169,12 @@ func (ctrl *Controller) incrementAwaitingMessages() { func (ctrl *Controller) CacheItemInfo(dii details.ItemInfo) { if dii.Groups != nil { ctrl.backupDriveIDNames.Add(dii.Groups.DriveID, dii.Groups.DriveName) + ctrl.backupSiteIDWebURL.Add(dii.Groups.SiteID, dii.Groups.WebURL) } if dii.SharePoint != nil { ctrl.backupDriveIDNames.Add(dii.SharePoint.DriveID, dii.SharePoint.DriveName) + ctrl.backupSiteIDWebURL.Add(dii.SharePoint.SiteID, dii.SharePoint.WebURL) } if dii.OneDrive != nil { diff --git a/src/internal/m365/controller_test.go b/src/internal/m365/controller_test.go index 991aa7955..72d9e4478 100644 --- a/src/internal/m365/controller_test.go +++ b/src/internal/m365/controller_test.go @@ -270,6 +270,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { odname = "od-name" spid = "sp-id" spname = "sp-name" + spsid = "sp-sid" + spurl = "sp-url" gpid = "gp-id" gpname = "gp-name" // intentionally declared outside the test loop @@ -277,32 +279,35 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { wg: &sync.WaitGroup{}, region: &trace.Region{}, backupDriveIDNames: idname.NewCache(nil), + backupSiteIDWebURL: idname.NewCache(nil), } ) table := []struct { - name string - service path.ServiceType - cat path.CategoryType - dii details.ItemInfo - expectID string - expectName string + name string + service path.ServiceType + cat path.CategoryType + dii details.ItemInfo + expectDriveID string + expectDriveName string + expectSiteID string + expectSiteWebURL string }{ { name: "exchange", dii: details.ItemInfo{ Exchange: &details.ExchangeInfo{}, }, - expectID: "", - expectName: "", + expectDriveID: "", + expectDriveName: "", }, { name: "folder", dii: details.ItemInfo{ Folder: &details.FolderInfo{}, }, - expectID: "", - expectName: "", + expectDriveID: "", + expectDriveName: "", }, { name: "onedrive", @@ -312,8 +317,8 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { DriveName: odname, }, }, - expectID: odid, - expectName: odname, + expectDriveID: odid, + expectDriveName: odname, }, { name: "sharepoint", @@ -321,10 +326,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { SharePoint: &details.SharePointInfo{ DriveID: spid, DriveName: spname, + SiteID: spsid, + WebURL: spurl, }, }, - expectID: spid, - expectName: spname, + expectDriveID: spid, + expectDriveName: spname, + expectSiteID: spsid, + expectSiteWebURL: spurl, }, { name: "groups", @@ -332,10 +341,14 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { Groups: &details.GroupsInfo{ DriveID: gpid, DriveName: gpname, + SiteID: spsid, + WebURL: spurl, }, }, - expectID: gpid, - expectName: gpname, + expectDriveID: gpid, + expectDriveName: gpname, + expectSiteID: spsid, + expectSiteWebURL: spurl, }, } for _, test := range table { @@ -344,11 +357,17 @@ func (suite *ControllerUnitSuite) TestController_CacheItemInfo() { ctrl.CacheItemInfo(test.dii) - name, _ := ctrl.backupDriveIDNames.NameOf(test.expectID) - assert.Equal(t, test.expectName, name) + name, _ := ctrl.backupDriveIDNames.NameOf(test.expectDriveID) + assert.Equal(t, test.expectDriveName, name) - id, _ := ctrl.backupDriveIDNames.IDOf(test.expectName) - assert.Equal(t, test.expectID, id) + id, _ := ctrl.backupDriveIDNames.IDOf(test.expectDriveName) + assert.Equal(t, test.expectDriveID, id) + + url, _ := ctrl.backupSiteIDWebURL.NameOf(test.expectSiteID) + assert.Equal(t, test.expectSiteWebURL, url) + + sid, _ := ctrl.backupSiteIDWebURL.IDOf(test.expectSiteWebURL) + assert.Equal(t, test.expectSiteID, sid) }) } } diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 94e00f295..ab7a94ceb 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -69,6 +69,8 @@ func (ctrl *Controller) ProduceExportCollections( exportCfg, opts, dcs, + ctrl.backupDriveIDNames, + ctrl.backupSiteIDWebURL, deets, errs) diff --git a/src/internal/m365/service/groups/export.go b/src/internal/m365/service/groups/export.go index 005fd3ee2..be433493f 100644 --- a/src/internal/m365/service/groups/export.go +++ b/src/internal/m365/service/groups/export.go @@ -2,13 +2,19 @@ package groups import ( "context" + stdlibpath "path" + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/collection/groups" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) @@ -20,6 +26,8 @@ func ProduceExportCollections( exportCfg control.ExportConfig, opts control.Options, dcs []data.RestoreCollection, + backupDriveIDNames idname.Cacher, + backupSiteIDWebURL idname.Cacher, deets *details.Builder, errs *fault.Bus, ) ([]export.Collectioner, error) { @@ -33,21 +41,65 @@ func ProduceExportCollections( fp = restoreColl.FullPath() cat = fp.Category() folders = []string{cat.String()} + coll export.Collectioner ) switch cat { case path.ChannelMessagesCategory: folders = append(folders, fp.Folders()...) + + coll = groups.NewExportCollection( + path.Builder{}.Append(folders...).String(), + []data.RestoreCollection{restoreColl}, + backupVersion, + exportCfg) + case path.LibrariesCategory: + drivePath, err := path.ToDrivePath(restoreColl.FullPath()) + if err != nil { + return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) + } + + driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID) + if !ok { + // This should not happen, but just in case + logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") + driveName = drivePath.DriveID + } + + folders := restoreColl.FullPath().Folders() + siteName := folders[1] // use siteID by default + + webURL, ok := backupSiteIDWebURL.NameOf(siteName) + if !ok { + // This should not happen, but just in case + logger.Ctx(ctx).With("site_id", folders[1]).Info("site weburl not found, using site id") + } + + if len(webURL) != 0 { + // We can't use the actual name anyways as it might + // contain invalid characters. This should also avoid + // possibility of name collisions. + siteName = stdlibpath.Base(webURL) + } + + baseDir := path.Builder{}. + Append("Libraries"). + Append(siteName). + Append(driveName). + Append(drivePath.Folders...) + + coll = drive.NewExportCollection( + baseDir.String(), + []data.RestoreCollection{restoreColl}, + backupVersion) default: + el.AddRecoverable( + ctx, + clues.New("unsupported category for export").With("category", cat)) + continue } - coll := groups.NewExportCollection( - path.Builder{}.Append(folders...).String(), - []data.RestoreCollection{restoreColl}, - backupVersion, - exportCfg) - ec = append(ec, coll) } diff --git a/src/internal/m365/service/groups/export_test.go b/src/internal/m365/service/groups/export_test.go index ffcf13036..a621760ac 100644 --- a/src/internal/m365/service/groups/export_test.go +++ b/src/internal/m365/service/groups/export_test.go @@ -4,14 +4,18 @@ import ( "bytes" "context" "io" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock" + odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" + odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" @@ -30,6 +34,7 @@ func TestExportUnitSuite(t *testing.T) { type finD struct { id string + key string name string err error } @@ -42,14 +47,14 @@ func (fd finD) FetchItemByName(ctx context.Context, name string) (data.Item, err if name == fd.id { return &dataMock.Item{ ItemID: fd.id, - Reader: io.NopCloser(bytes.NewBufferString(`{"displayname": "` + fd.name + `"}`)), + Reader: io.NopCloser(bytes.NewBufferString(`{"` + fd.key + `": "` + fd.name + `"}`)), }, nil } return nil, assert.AnError } -func (suite *ExportUnitSuite) TestExportRestoreCollections() { +func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() { t := suite.T() ctx, flush := tester.NewContext(t) @@ -87,7 +92,7 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { }, }, }, - FetchItemByNamer: finD{id: itemID, name: dii.Groups.ItemName}, + FetchItemByNamer: finD{id: itemID, key: "displayname", name: dii.Groups.ItemName}, }, } @@ -98,6 +103,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { control.DefaultOptions(), dcs, nil, + nil, + nil, fault.New(true)) assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") @@ -115,3 +122,86 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { assert.Equal(t, expectedItems, fitems, "items") } + +func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + var ( + siteID = "siteID1" + siteEscapedName = "siteName1" + siteWebURL = "https://site1.sharepoint.com/sites/" + siteEscapedName + driveID = "driveID1" + driveName = "driveName1" + exportCfg = control.ExportConfig{} + dpb = odConsts.DriveFolderPrefixBuilder(driveID) + driveNameCache = idname.NewCache( + // Cache check with lowercased ids + map[string]string{strings.ToLower(driveID): driveName}) + siteWebURLCache = idname.NewCache( + // Cache check with lowercased ids + map[string]string{strings.ToLower(siteID): siteWebURL}) + dii = odStub.DriveItemInfo() + expectedPath = "Libraries/" + siteEscapedName + "/" + driveName + expectedItems = []export.Item{ + { + ID: "id1.data", + Name: "name1", + Body: io.NopCloser((bytes.NewBufferString("body1"))), + }, + } + ) + + dii.OneDrive.ItemName = "name1" + + p, err := dpb.ToDataLayerPath( + "t", + "u", + path.GroupsService, + path.LibrariesCategory, + false, + odConsts.SitesPathDir, + siteID) + assert.NoError(t, err, "build path") + + dcs := []data.RestoreCollection{ + data.FetchRestoreCollection{ + Collection: dataMock.Collection{ + Path: p, + ItemData: []data.Item{ + &dataMock.Item{ + ItemID: "id1.data", + Reader: io.NopCloser(bytes.NewBufferString("body1")), + ItemInfo: dii, + }, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", key: "filename", name: "name1"}, + }, + } + + ecs, err := ProduceExportCollections( + ctx, + int(version.Backup), + exportCfg, + control.DefaultOptions(), + dcs, + driveNameCache, + siteWebURLCache, + nil, + fault.New(true)) + assert.NoError(t, err, "export collections error") + assert.Len(t, ecs, 1, "num of collections") + + assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir") + + fitems := []export.Item{} + + for item := range ecs[0].Items(ctx) { + fitems = append(fitems, item) + } + + assert.Equal(t, expectedItems, fitems, "items") +}