From 74cf0ab737ebd2a6648128cc89e7d96682273172 Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Wed, 1 Nov 2023 08:55:30 -0700 Subject: [PATCH] Create service specific handlers that know how to run an export (#4491) First step in reducing the number of places we have to check the service type manually. Create a way to get a handle to a service specific handler and implement exports for those handlers --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [ ] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #4254 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- src/internal/m365/controller.go | 6 +- src/internal/m365/export.go | 88 ++--------- src/internal/m365/mock/connector.go | 3 - src/internal/m365/service/groups/export.go | 40 ++++- .../m365/service/groups/export_test.go | 64 ++++---- src/internal/m365/service/onedrive/export.go | 21 ++- .../m365/service/onedrive/export_test.go | 26 ++-- .../m365/service/sharepoint/export.go | 35 ++++- .../m365/service/sharepoint/export_test.go | 140 +++++++++++------- src/internal/operations/export.go | 11 +- src/internal/operations/export_test.go | 2 +- src/internal/operations/inject/inject.go | 18 ++- src/pkg/repository/data_providers.go | 3 +- src/pkg/repository/exports.go | 9 +- 14 files changed, 253 insertions(+), 213 deletions(-) diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 90dc6b841..d5b8d629a 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -25,9 +25,9 @@ var ErrNoResourceLookup = clues.New("missing resource lookup client") // must comply with BackupProducer and RestoreConsumer var ( - _ inject.BackupProducer = &Controller{} - _ inject.RestoreConsumer = &Controller{} - _ inject.ExportConsumer = &Controller{} + _ inject.BackupProducer = &Controller{} + _ inject.RestoreConsumer = &Controller{} + _ inject.ToServiceHandler = &Controller{} ) // Controller is a struct used to wrap the GraphServiceClient and diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index ddf512611..c7136f221 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -1,89 +1,33 @@ package m365 import ( - "context" - "github.com/alcionai/clues" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/diagnostics" - "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/internal/m365/service/groups" "github.com/alcionai/corso/src/internal/m365/service/onedrive" "github.com/alcionai/corso/src/internal/m365/service/sharepoint" - "github.com/alcionai/corso/src/internal/m365/support" - "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/export" - "github.com/alcionai/corso/src/pkg/fault" - "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/path" ) -// ProduceExportCollections exports data from the specified collections -func (ctrl *Controller) ProduceExportCollections( - ctx context.Context, - backupVersion int, - sels selectors.Selector, - exportCfg control.ExportConfig, +// NewServiceHandler returns an instance of a struct capable of running various +// operations for a given service. +func (ctrl *Controller) NewServiceHandler( opts control.Options, - dcs []data.RestoreCollection, - stats *data.ExportStats, - errs *fault.Bus, -) ([]export.Collectioner, error) { - ctx, end := diagnostics.Span(ctx, "m365:export") - defer end() + service path.ServiceType, +) (inject.ServiceHandler, error) { + switch service { + case path.OneDriveService: + return onedrive.NewOneDriveHandler(opts), nil - ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()}) - ctx = clues.Add(ctx, "export_config", exportCfg) + case path.SharePointService: + return sharepoint.NewSharePointHandler(opts), nil - var ( - expCollections []export.Collectioner - status *support.ControllerOperationStatus - deets = &details.Builder{} - err error - ) - - switch sels.Service { - case selectors.ServiceOneDrive: - expCollections, err = onedrive.ProduceExportCollections( - ctx, - backupVersion, - exportCfg, - opts, - dcs, - deets, - stats, - errs) - case selectors.ServiceSharePoint: - expCollections, err = sharepoint.ProduceExportCollections( - ctx, - backupVersion, - exportCfg, - opts, - dcs, - ctrl.backupDriveIDNames, - deets, - stats, - errs) - case selectors.ServiceGroups: - expCollections, err = groups.ProduceExportCollections( - ctx, - backupVersion, - exportCfg, - opts, - dcs, - ctrl.backupDriveIDNames, - ctrl.backupSiteIDWebURL, - deets, - stats, - errs) - - default: - err = clues.Wrap(clues.New(sels.Service.String()), "service not supported") + case path.GroupsService: + return groups.NewGroupsHandler(opts), nil } - ctrl.incrementAwaitingMessages() - ctrl.UpdateStatus(status) - - return expCollections, err + return nil, clues.New("unrecognized service"). + With("service_type", service.String()) } diff --git a/src/internal/m365/mock/connector.go b/src/internal/m365/mock/connector.go index a6db1d5e0..a45530bae 100644 --- a/src/internal/m365/mock/connector.go +++ b/src/internal/m365/mock/connector.go @@ -17,7 +17,6 @@ import ( "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" ) var _ inject.BackupProducer = &Controller{} @@ -87,9 +86,7 @@ func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {} func (ctrl Controller) ProduceExportCollections( _ context.Context, _ int, - _ selectors.Selector, _ control.ExportConfig, - _ control.Options, _ []data.RestoreCollection, _ *data.ExportStats, _ *fault.Bus, diff --git a/src/internal/m365/service/groups/export.go b/src/internal/m365/service/groups/export.go index 09b0fbf92..82ee4548a 100644 --- a/src/internal/m365/service/groups/export.go +++ b/src/internal/m365/service/groups/export.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/drive" "github.com/alcionai/corso/src/internal/m365/collection/groups" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" @@ -18,17 +19,41 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +var _ inject.ServiceHandler = &groupsHandler{} + +func NewGroupsHandler( + opts control.Options, +) *groupsHandler { + return &groupsHandler{ + opts: opts, + backupDriveIDNames: idname.NewCache(nil), + backupSiteIDWebURL: idname.NewCache(nil), + } +} + +type groupsHandler struct { + opts control.Options + + backupDriveIDNames idname.CacheBuilder + backupSiteIDWebURL idname.CacheBuilder +} + +func (h *groupsHandler) CacheItemInfo(v details.ItemInfo) { + if v.Groups == nil { + return + } + + h.backupDriveIDNames.Add(v.Groups.DriveID, v.Groups.DriveName) + h.backupSiteIDWebURL.Add(v.Groups.SiteID, v.Groups.WebURL) +} + // ProduceExportCollections will create the export collections for the // given restore collections. -func ProduceExportCollections( +func (h *groupsHandler) ProduceExportCollections( ctx context.Context, backupVersion int, exportCfg control.ExportConfig, - opts control.Options, dcs []data.RestoreCollection, - backupDriveIDNames idname.Cacher, - backupSiteIDWebURL idname.Cacher, - deets *details.Builder, stats *data.ExportStats, errs *fault.Bus, ) ([]export.Collectioner, error) { @@ -55,13 +80,14 @@ func ProduceExportCollections( backupVersion, exportCfg, stats) + case path.LibrariesCategory: drivePath, err := path.ToDrivePath(restoreColl.FullPath()) if err != nil { return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) } - driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID) + driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID) if !ok { // This should not happen, but just in case logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") @@ -71,7 +97,7 @@ func ProduceExportCollections( rfds := restoreColl.FullPath().Folders() siteName := rfds[1] // use siteID by default - webURL, ok := backupSiteIDWebURL.NameOf(siteName) + webURL, ok := h.backupSiteIDWebURL.NameOf(siteName) if !ok { // This should not happen, but just in case logger.Ctx(ctx).With("site_id", rfds[1]).Info("site weburl not found, using site id") diff --git a/src/internal/m365/service/groups/export_test.go b/src/internal/m365/service/groups/export_test.go index bc633cde3..35d4356fd 100644 --- a/src/internal/m365/service/groups/export_test.go +++ b/src/internal/m365/service/groups/export_test.go @@ -4,21 +4,19 @@ import ( "bytes" "context" "io" - "strings" "testing" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" - odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" @@ -87,9 +85,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() { Path: p, ItemData: []data.Item{ &dataMock.Item{ - ItemID: itemID, - Reader: body, - ItemInfo: dii, + ItemID: itemID, + Reader: body, }, }, }, @@ -99,17 +96,14 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() { stats := data.ExportStats{} - ecs, err := ProduceExportCollections( - ctx, - int(version.Backup), - exportCfg, - control.DefaultOptions(), - dcs, - nil, - nil, - nil, - &stats, - fault.New(true)) + ecs, err := NewGroupsHandler(control.DefaultOptions()). + ProduceExportCollections( + ctx, + int(version.Backup), + exportCfg, + dcs, + &stats, + fault.New(true)) assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") @@ -154,13 +148,19 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { driveName = "driveName1" exportCfg = control.ExportConfig{} dpb = odConsts.DriveFolderPrefixBuilder(driveID) - driveNameCache = idname.NewCache( - // Cache check with lowercased ids - map[string]string{strings.ToLower(driveID): driveName}) - siteWebURLCache = idname.NewCache( - // Cache check with lowercased ids - map[string]string{strings.ToLower(siteID): siteWebURL}) - dii = odStub.DriveItemInfo() + + dii = details.ItemInfo{ + Groups: &details.GroupsInfo{ + ItemType: details.SharePointLibrary, + ItemName: "name1", + Size: 1, + DriveName: driveName, + DriveID: driveID, + SiteID: siteID, + WebURL: siteWebURL, + }, + } + expectedPath = "Libraries/" + siteEscapedName + "/" + driveName expectedItems = []export.Item{ { @@ -171,8 +171,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { } ) - dii.OneDrive.ItemName = "name1" - p, err := dpb.ToDataLayerPath( "t", "u", @@ -189,9 +187,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { Path: p, ItemData: []data.Item{ &dataMock.Item{ - ItemID: "id1.data", - Reader: io.NopCloser(bytes.NewBufferString("body1")), - ItemInfo: dii, + ItemID: "id1.data", + Reader: io.NopCloser(bytes.NewBufferString("body1")), }, }, }, @@ -199,17 +196,16 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() { }, } + handler := NewGroupsHandler(control.DefaultOptions()) + handler.CacheItemInfo(dii) + stats := data.ExportStats{} - ecs, err := ProduceExportCollections( + ecs, err := handler.ProduceExportCollections( ctx, int(version.Backup), exportCfg, - control.DefaultOptions(), dcs, - driveNameCache, - siteWebURLCache, - nil, &stats, fault.New(true)) assert.NoError(t, err, "export collections error") diff --git a/src/internal/m365/service/onedrive/export.go b/src/internal/m365/service/onedrive/export.go index 9b985b608..93a268644 100644 --- a/src/internal/m365/service/onedrive/export.go +++ b/src/internal/m365/service/onedrive/export.go @@ -7,6 +7,7 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" @@ -14,15 +15,29 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +var _ inject.ServiceHandler = &onedriveHandler{} + +func NewOneDriveHandler( + opts control.Options, +) *onedriveHandler { + return &onedriveHandler{ + opts: opts, + } +} + +type onedriveHandler struct { + opts control.Options +} + +func (h *onedriveHandler) CacheItemInfo(v details.ItemInfo) {} + // ProduceExportCollections will create the export collections for the // given restore collections. -func ProduceExportCollections( +func (h *onedriveHandler) ProduceExportCollections( ctx context.Context, backupVersion int, exportCfg control.ExportConfig, - opts control.Options, dcs []data.RestoreCollection, - deets *details.Builder, stats *data.ExportStats, errs *fault.Bus, ) ([]export.Collectioner, error) { diff --git a/src/internal/m365/service/onedrive/export_test.go b/src/internal/m365/service/onedrive/export_test.go index 9d941cf3b..bf2cd5cfc 100644 --- a/src/internal/m365/service/onedrive/export_test.go +++ b/src/internal/m365/service/onedrive/export_test.go @@ -14,7 +14,6 @@ import ( dataMock "github.com/alcionai/corso/src/internal/data/mock" "github.com/alcionai/corso/src/internal/m365/collection/drive" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" - odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" @@ -313,7 +312,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { var ( exportCfg = control.ExportConfig{} dpb = odConsts.DriveFolderPrefixBuilder("driveID1") - dii = odStub.DriveItemInfo() expectedItems = []export.Item{ { ID: "id1.data", @@ -323,8 +321,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { } ) - dii.OneDrive.ItemName = "name1" - p, err := dpb.ToDataLayerOneDrivePath("t", "u", false) assert.NoError(t, err, "build path") @@ -334,9 +330,8 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { Path: p, ItemData: []data.Item{ &dataMock.Item{ - ItemID: "id1.data", - Reader: io.NopCloser(bytes.NewBufferString("body1")), - ItemInfo: dii, + ItemID: "id1.data", + Reader: io.NopCloser(bytes.NewBufferString("body1")), }, }, }, @@ -346,15 +341,14 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { stats := data.ExportStats{} - ecs, err := ProduceExportCollections( - ctx, - int(version.Backup), - exportCfg, - control.DefaultOptions(), - dcs, - nil, - &stats, - fault.New(true)) + ecs, err := NewOneDriveHandler(control.DefaultOptions()). + ProduceExportCollections( + ctx, + int(version.Backup), + exportCfg, + dcs, + &stats, + fault.New(true)) assert.NoError(t, err, "export collections error") assert.Len(t, ecs, 1, "num of collections") diff --git a/src/internal/m365/service/sharepoint/export.go b/src/internal/m365/service/sharepoint/export.go index eb52647cd..461c6256c 100644 --- a/src/internal/m365/service/sharepoint/export.go +++ b/src/internal/m365/service/sharepoint/export.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/collection/drive" + "github.com/alcionai/corso/src/internal/operations/inject" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" @@ -16,16 +17,40 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +var _ inject.ServiceHandler = &sharepointHandler{} + +func NewSharePointHandler( + opts control.Options, +) *sharepointHandler { + return &sharepointHandler{ + opts: opts, + backupDriveIDNames: idname.NewCache(nil), + } +} + +type sharepointHandler struct { + opts control.Options + backupDriveIDNames idname.CacheBuilder +} + +func (h *sharepointHandler) CacheItemInfo(v details.ItemInfo) { + // Old versions would store SharePoint data as OneDrive. + switch { + case v.SharePoint != nil: + h.backupDriveIDNames.Add(v.SharePoint.DriveID, v.SharePoint.DriveName) + + case v.OneDrive != nil: + h.backupDriveIDNames.Add(v.OneDrive.DriveID, v.OneDrive.DriveName) + } +} + // ProduceExportCollections will create the export collections for the // given restore collections. -func ProduceExportCollections( +func (h *sharepointHandler) ProduceExportCollections( ctx context.Context, backupVersion int, exportCfg control.ExportConfig, - opts control.Options, dcs []data.RestoreCollection, - backupDriveIDNames idname.CacheBuilder, - deets *details.Builder, stats *data.ExportStats, errs *fault.Bus, ) ([]export.Collectioner, error) { @@ -40,7 +65,7 @@ func ProduceExportCollections( return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx) } - driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID) + driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID) if !ok { // This should not happen, but just in case logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id") diff --git a/src/internal/m365/service/sharepoint/export_test.go b/src/internal/m365/service/sharepoint/export_test.go index 6de83ab7f..4b2ea2521 100644 --- a/src/internal/m365/service/sharepoint/export_test.go +++ b/src/internal/m365/service/sharepoint/export_test.go @@ -4,20 +4,18 @@ import ( "bytes" "context" "io" - "strings" "testing" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" dataMock "github.com/alcionai/corso/src/internal/data/mock" odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts" - odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" @@ -60,81 +58,111 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() { defer flush() var ( - driveID = "driveID1" - driveName = "driveName1" - exportCfg = control.ExportConfig{} - dpb = odConsts.DriveFolderPrefixBuilder(driveID) - cache = idname.NewCache( - // Cache check with lowercased ids - map[string]string{strings.ToLower(driveID): driveName}) - dii = odStub.DriveItemInfo() + driveID = "driveID1" + driveName = "driveName1" + itemName = "name1" + exportCfg = control.ExportConfig{} + dpb = odConsts.DriveFolderPrefixBuilder(driveID) expectedPath = path.LibrariesCategory.HumanString() + "/" + driveName expectedItems = []export.Item{ { ID: "id1.data", - Name: "name1", + Name: itemName, Body: io.NopCloser((bytes.NewBufferString("body1"))), }, } ) - dii.OneDrive.ItemName = "name1" - p, err := dpb.ToDataLayerSharePointPath("t", "u", path.LibrariesCategory, false) assert.NoError(t, err, "build path") - dcs := []data.RestoreCollection{ - data.FetchRestoreCollection{ - Collection: dataMock.Collection{ - Path: p, - ItemData: []data.Item{ - &dataMock.Item{ - ItemID: "id1.data", - Reader: io.NopCloser(bytes.NewBufferString("body1")), - ItemInfo: dii, - }, + table := []struct { + name string + itemInfo details.ItemInfo + }{ + { + name: "OneDriveLegacyItemInfo", + itemInfo: details.ItemInfo{ + OneDrive: &details.OneDriveInfo{ + ItemType: details.OneDriveItem, + ItemName: itemName, + Size: 1, + DriveName: driveName, + DriveID: driveID, + }, + }, + }, + { + name: "SharePointItemInfo", + itemInfo: details.ItemInfo{ + SharePoint: &details.SharePointInfo{ + ItemType: details.SharePointLibrary, + ItemName: itemName, + Size: 1, + DriveName: driveName, + DriveID: driveID, }, }, - FetchItemByNamer: finD{id: "id1.meta", name: "name1"}, }, } - stats := data.ExportStats{} + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() - ecs, err := ProduceExportCollections( - ctx, - int(version.Backup), - exportCfg, - control.DefaultOptions(), - dcs, - cache, - nil, - &stats, - fault.New(true)) - assert.NoError(t, err, "export collections error") - assert.Len(t, ecs, 1, "num of collections") + dcs := []data.RestoreCollection{ + data.FetchRestoreCollection{ + Collection: dataMock.Collection{ + Path: p, + ItemData: []data.Item{ + &dataMock.Item{ + ItemID: "id1.data", + Reader: io.NopCloser(bytes.NewBufferString("body1")), + }, + }, + }, + FetchItemByNamer: finD{id: "id1.meta", name: itemName}, + }, + } - assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir") + handler := NewSharePointHandler(control.DefaultOptions()) + handler.CacheItemInfo(test.itemInfo) - fitems := []export.Item{} - size := 0 + stats := data.ExportStats{} - for item := range ecs[0].Items(ctx) { - // unwrap the body from stats reader - b, err := io.ReadAll(item.Body) - assert.NoError(t, err, clues.ToCore(err)) + ecs, err := handler.ProduceExportCollections( + ctx, + int(version.Backup), + exportCfg, + dcs, + &stats, + fault.New(true)) + assert.NoError(t, err, "export collections error") + assert.Len(t, ecs, 1, "num of collections") - size += len(b) - bitem := io.NopCloser(bytes.NewBuffer(b)) - item.Body = bitem + assert.Equal(t, expectedPath, ecs[0].BasePath(), "base dir") - fitems = append(fitems, item) + fitems := []export.Item{} + size := 0 + + for item := range ecs[0].Items(ctx) { + // unwrap the body from stats reader + b, err := io.ReadAll(item.Body) + assert.NoError(t, err, clues.ToCore(err)) + + size += len(b) + bitem := io.NopCloser(bytes.NewBuffer(b)) + item.Body = bitem + + fitems = append(fitems, item) + } + + assert.Equal(t, expectedItems, fitems, "items") + + expectedStats := data.ExportStats{} + expectedStats.UpdateBytes(path.FilesCategory, int64(size)) + expectedStats.UpdateResourceCount(path.FilesCategory) + assert.Equal(t, expectedStats, stats, "stats") + }) } - - assert.Equal(t, expectedItems, fitems, "items") - - expectedStats := data.ExportStats{} - expectedStats.UpdateBytes(path.FilesCategory, int64(size)) - expectedStats.UpdateResourceCount(path.FilesCategory) - assert.Equal(t, expectedStats, stats, "stats") } diff --git a/src/internal/operations/export.go b/src/internal/operations/export.go index ac7202512..efa7b84e5 100644 --- a/src/internal/operations/export.go +++ b/src/internal/operations/export.go @@ -261,9 +261,7 @@ func (op *ExportOperation) do( ctx, op.ec, bup.Version, - op.Selectors, op.ExportCfg, - op.Options, dcs, // We also have opStats, but that tracks different data. // Maybe we can look into merging them some time in the future. @@ -329,9 +327,7 @@ func produceExportCollections( ctx context.Context, ec inject.ExportConsumer, backupVersion int, - sel selectors.Selector, exportCfg control.ExportConfig, - opts control.Options, dcs []data.RestoreCollection, exportStats *data.ExportStats, errs *fault.Bus, @@ -342,12 +338,15 @@ func produceExportCollections( close(complete) }() + ctx, end := diagnostics.Span(ctx, "m365:export") + defer end() + + ctx = clues.Add(ctx, "export_config", exportCfg) + expCollections, err := ec.ProduceExportCollections( ctx, backupVersion, - sel, exportCfg, - opts, dcs, exportStats, errs) diff --git a/src/internal/operations/export_test.go b/src/internal/operations/export_test.go index f1cda2c20..f4ac6515a 100644 --- a/src/internal/operations/export_test.go +++ b/src/internal/operations/export_test.go @@ -37,7 +37,7 @@ func TestExportUnitSuite(t *testing.T) { suite.Run(t, &ExportUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *ExportUnitSuite) TestExportOperation_PersistResults() { +func (suite *ExportUnitSuite) TestExportOperation_Export() { var ( kw = &kopia.Wrapper{} sw = store.NewWrapper(&kopia.ModelStore{}) diff --git a/src/internal/operations/inject/inject.go b/src/internal/operations/inject/inject.go index fb9e13528..0f449bf81 100644 --- a/src/internal/operations/inject/inject.go +++ b/src/internal/operations/inject/inject.go @@ -15,7 +15,6 @@ import ( "github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" ) type ( @@ -85,16 +84,12 @@ type ( ProduceExportCollections( ctx context.Context, backupVersion int, - selector selectors.Selector, exportCfg control.ExportConfig, - opts control.Options, dcs []data.RestoreCollection, stats *data.ExportStats, errs *fault.Bus, ) ([]export.Collectioner, error) - Wait() *data.CollectionStats - CacheItemInfoer } @@ -117,4 +112,17 @@ type ( RepoMaintenancer interface { RepoMaintenance(ctx context.Context, opts repository.Maintenance) error } + + // ServiceHandler contains the set of functions required to implement all + // service-specific functionality for backups, restores, and exports. + ServiceHandler interface { + ExportConsumer + } + + ToServiceHandler interface { + NewServiceHandler( + opts control.Options, + service path.ServiceType, + ) (ServiceHandler, error) + } ) diff --git a/src/pkg/repository/data_providers.go b/src/pkg/repository/data_providers.go index 8f219b4bb..f57b70235 100644 --- a/src/pkg/repository/data_providers.go +++ b/src/pkg/repository/data_providers.go @@ -15,9 +15,10 @@ import ( type DataProvider interface { inject.BackupProducer - inject.ExportConsumer inject.RestoreConsumer + inject.ToServiceHandler + VerifyAccess(ctx context.Context) error } diff --git a/src/pkg/repository/exports.go b/src/pkg/repository/exports.go index 2aadd2bfb..c8634ce61 100644 --- a/src/pkg/repository/exports.go +++ b/src/pkg/repository/exports.go @@ -3,6 +3,8 @@ package repository import ( "context" + "github.com/alcionai/clues" + "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/operations" "github.com/alcionai/corso/src/pkg/control" @@ -26,12 +28,17 @@ func (r repository) NewExport( sel selectors.Selector, exportCfg control.ExportConfig, ) (operations.ExportOperation, error) { + handler, err := r.Provider.NewServiceHandler(r.Opts, sel.PathService()) + if err != nil { + return operations.ExportOperation{}, clues.Stack(err) + } + return operations.NewExportOperation( ctx, r.Opts, r.dataLayer, store.NewWrapper(r.modelStore), - r.Provider, + handler, r.Account, model.StableID(backupID), sel,