Create service specific handlers that know how to run an export (#4491)

First step in reducing the number of places we have to check the service type manually. Create a way to get a handle to a service specific handler and implement exports for those handlers

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #4254

#### Test Plan

- [ ] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
ashmrtn 2023-11-01 08:55:30 -07:00 committed by GitHub
parent 3df3a44c7b
commit 74cf0ab737
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 253 additions and 213 deletions

View File

@ -27,7 +27,7 @@ var ErrNoResourceLookup = clues.New("missing resource lookup client")
var (
_ inject.BackupProducer = &Controller{}
_ inject.RestoreConsumer = &Controller{}
_ inject.ExportConsumer = &Controller{}
_ inject.ToServiceHandler = &Controller{}
)
// Controller is a struct used to wrap the GraphServiceClient and

View File

@ -1,89 +1,33 @@
package m365
import (
"context"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/m365/service/groups"
"github.com/alcionai/corso/src/internal/m365/service/onedrive"
"github.com/alcionai/corso/src/internal/m365/service/sharepoint"
"github.com/alcionai/corso/src/internal/m365/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/path"
)
// ProduceExportCollections exports data from the specified collections
func (ctrl *Controller) ProduceExportCollections(
ctx context.Context,
backupVersion int,
sels selectors.Selector,
exportCfg control.ExportConfig,
// NewServiceHandler returns an instance of a struct capable of running various
// operations for a given service.
func (ctrl *Controller) NewServiceHandler(
opts control.Options,
dcs []data.RestoreCollection,
stats *data.ExportStats,
errs *fault.Bus,
) ([]export.Collectioner, error) {
ctx, end := diagnostics.Span(ctx, "m365:export")
defer end()
service path.ServiceType,
) (inject.ServiceHandler, error) {
switch service {
case path.OneDriveService:
return onedrive.NewOneDriveHandler(opts), nil
ctx = graph.BindRateLimiterConfig(ctx, graph.LimiterCfg{Service: sels.PathService()})
ctx = clues.Add(ctx, "export_config", exportCfg)
case path.SharePointService:
return sharepoint.NewSharePointHandler(opts), nil
var (
expCollections []export.Collectioner
status *support.ControllerOperationStatus
deets = &details.Builder{}
err error
)
switch sels.Service {
case selectors.ServiceOneDrive:
expCollections, err = onedrive.ProduceExportCollections(
ctx,
backupVersion,
exportCfg,
opts,
dcs,
deets,
stats,
errs)
case selectors.ServiceSharePoint:
expCollections, err = sharepoint.ProduceExportCollections(
ctx,
backupVersion,
exportCfg,
opts,
dcs,
ctrl.backupDriveIDNames,
deets,
stats,
errs)
case selectors.ServiceGroups:
expCollections, err = groups.ProduceExportCollections(
ctx,
backupVersion,
exportCfg,
opts,
dcs,
ctrl.backupDriveIDNames,
ctrl.backupSiteIDWebURL,
deets,
stats,
errs)
default:
err = clues.Wrap(clues.New(sels.Service.String()), "service not supported")
case path.GroupsService:
return groups.NewGroupsHandler(opts), nil
}
ctrl.incrementAwaitingMessages()
ctrl.UpdateStatus(status)
return expCollections, err
return nil, clues.New("unrecognized service").
With("service_type", service.String())
}

View File

@ -17,7 +17,6 @@ import (
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
var _ inject.BackupProducer = &Controller{}
@ -87,9 +86,7 @@ func (ctrl Controller) CacheItemInfo(dii details.ItemInfo) {}
func (ctrl Controller) ProduceExportCollections(
_ context.Context,
_ int,
_ selectors.Selector,
_ control.ExportConfig,
_ control.Options,
_ []data.RestoreCollection,
_ *data.ExportStats,
_ *fault.Bus,

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
"github.com/alcionai/corso/src/internal/m365/collection/groups"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
@ -18,17 +19,41 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ inject.ServiceHandler = &groupsHandler{}
func NewGroupsHandler(
opts control.Options,
) *groupsHandler {
return &groupsHandler{
opts: opts,
backupDriveIDNames: idname.NewCache(nil),
backupSiteIDWebURL: idname.NewCache(nil),
}
}
type groupsHandler struct {
opts control.Options
backupDriveIDNames idname.CacheBuilder
backupSiteIDWebURL idname.CacheBuilder
}
func (h *groupsHandler) CacheItemInfo(v details.ItemInfo) {
if v.Groups == nil {
return
}
h.backupDriveIDNames.Add(v.Groups.DriveID, v.Groups.DriveName)
h.backupSiteIDWebURL.Add(v.Groups.SiteID, v.Groups.WebURL)
}
// ProduceExportCollections will create the export collections for the
// given restore collections.
func ProduceExportCollections(
func (h *groupsHandler) ProduceExportCollections(
ctx context.Context,
backupVersion int,
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
backupDriveIDNames idname.Cacher,
backupSiteIDWebURL idname.Cacher,
deets *details.Builder,
stats *data.ExportStats,
errs *fault.Bus,
) ([]export.Collectioner, error) {
@ -55,13 +80,14 @@ func ProduceExportCollections(
backupVersion,
exportCfg,
stats)
case path.LibrariesCategory:
drivePath, err := path.ToDrivePath(restoreColl.FullPath())
if err != nil {
return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx)
}
driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID)
driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID)
if !ok {
// This should not happen, but just in case
logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id")
@ -71,7 +97,7 @@ func ProduceExportCollections(
rfds := restoreColl.FullPath().Folders()
siteName := rfds[1] // use siteID by default
webURL, ok := backupSiteIDWebURL.NameOf(siteName)
webURL, ok := h.backupSiteIDWebURL.NameOf(siteName)
if !ok {
// This should not happen, but just in case
logger.Ctx(ctx).With("site_id", rfds[1]).Info("site weburl not found, using site id")

View File

@ -4,21 +4,19 @@ import (
"bytes"
"context"
"io"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
groupMock "github.com/alcionai/corso/src/internal/m365/service/groups/mock"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
@ -89,7 +87,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() {
&dataMock.Item{
ItemID: itemID,
Reader: body,
ItemInfo: dii,
},
},
},
@ -99,15 +96,12 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_messages() {
stats := data.ExportStats{}
ecs, err := ProduceExportCollections(
ecs, err := NewGroupsHandler(control.DefaultOptions()).
ProduceExportCollections(
ctx,
int(version.Backup),
exportCfg,
control.DefaultOptions(),
dcs,
nil,
nil,
nil,
&stats,
fault.New(true))
assert.NoError(t, err, "export collections error")
@ -154,13 +148,19 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
driveName = "driveName1"
exportCfg = control.ExportConfig{}
dpb = odConsts.DriveFolderPrefixBuilder(driveID)
driveNameCache = idname.NewCache(
// Cache check with lowercased ids
map[string]string{strings.ToLower(driveID): driveName})
siteWebURLCache = idname.NewCache(
// Cache check with lowercased ids
map[string]string{strings.ToLower(siteID): siteWebURL})
dii = odStub.DriveItemInfo()
dii = details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.SharePointLibrary,
ItemName: "name1",
Size: 1,
DriveName: driveName,
DriveID: driveID,
SiteID: siteID,
WebURL: siteWebURL,
},
}
expectedPath = "Libraries/" + siteEscapedName + "/" + driveName
expectedItems = []export.Item{
{
@ -171,8 +171,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
}
)
dii.OneDrive.ItemName = "name1"
p, err := dpb.ToDataLayerPath(
"t",
"u",
@ -191,7 +189,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
&dataMock.Item{
ItemID: "id1.data",
Reader: io.NopCloser(bytes.NewBufferString("body1")),
ItemInfo: dii,
},
},
},
@ -199,17 +196,16 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections_libraries() {
},
}
handler := NewGroupsHandler(control.DefaultOptions())
handler.CacheItemInfo(dii)
stats := data.ExportStats{}
ecs, err := ProduceExportCollections(
ecs, err := handler.ProduceExportCollections(
ctx,
int(version.Backup),
exportCfg,
control.DefaultOptions(),
dcs,
driveNameCache,
siteWebURLCache,
nil,
&stats,
fault.New(true))
assert.NoError(t, err, "export collections error")

View File

@ -7,6 +7,7 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
@ -14,15 +15,29 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ inject.ServiceHandler = &onedriveHandler{}
func NewOneDriveHandler(
opts control.Options,
) *onedriveHandler {
return &onedriveHandler{
opts: opts,
}
}
type onedriveHandler struct {
opts control.Options
}
func (h *onedriveHandler) CacheItemInfo(v details.ItemInfo) {}
// ProduceExportCollections will create the export collections for the
// given restore collections.
func ProduceExportCollections(
func (h *onedriveHandler) ProduceExportCollections(
ctx context.Context,
backupVersion int,
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
deets *details.Builder,
stats *data.ExportStats,
errs *fault.Bus,
) ([]export.Collectioner, error) {

View File

@ -14,7 +14,6 @@ import (
dataMock "github.com/alcionai/corso/src/internal/data/mock"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control"
@ -313,7 +312,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
var (
exportCfg = control.ExportConfig{}
dpb = odConsts.DriveFolderPrefixBuilder("driveID1")
dii = odStub.DriveItemInfo()
expectedItems = []export.Item{
{
ID: "id1.data",
@ -323,8 +321,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
}
)
dii.OneDrive.ItemName = "name1"
p, err := dpb.ToDataLayerOneDrivePath("t", "u", false)
assert.NoError(t, err, "build path")
@ -336,7 +332,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
&dataMock.Item{
ItemID: "id1.data",
Reader: io.NopCloser(bytes.NewBufferString("body1")),
ItemInfo: dii,
},
},
},
@ -346,13 +341,12 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
stats := data.ExportStats{}
ecs, err := ProduceExportCollections(
ecs, err := NewOneDriveHandler(control.DefaultOptions()).
ProduceExportCollections(
ctx,
int(version.Backup),
exportCfg,
control.DefaultOptions(),
dcs,
nil,
&stats,
fault.New(true))
assert.NoError(t, err, "export collections error")

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
@ -16,16 +17,40 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ inject.ServiceHandler = &sharepointHandler{}
func NewSharePointHandler(
opts control.Options,
) *sharepointHandler {
return &sharepointHandler{
opts: opts,
backupDriveIDNames: idname.NewCache(nil),
}
}
type sharepointHandler struct {
opts control.Options
backupDriveIDNames idname.CacheBuilder
}
func (h *sharepointHandler) CacheItemInfo(v details.ItemInfo) {
// Old versions would store SharePoint data as OneDrive.
switch {
case v.SharePoint != nil:
h.backupDriveIDNames.Add(v.SharePoint.DriveID, v.SharePoint.DriveName)
case v.OneDrive != nil:
h.backupDriveIDNames.Add(v.OneDrive.DriveID, v.OneDrive.DriveName)
}
}
// ProduceExportCollections will create the export collections for the
// given restore collections.
func ProduceExportCollections(
func (h *sharepointHandler) ProduceExportCollections(
ctx context.Context,
backupVersion int,
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
backupDriveIDNames idname.CacheBuilder,
deets *details.Builder,
stats *data.ExportStats,
errs *fault.Bus,
) ([]export.Collectioner, error) {
@ -40,7 +65,7 @@ func ProduceExportCollections(
return nil, clues.Wrap(err, "transforming path to drive path").WithClues(ctx)
}
driveName, ok := backupDriveIDNames.NameOf(drivePath.DriveID)
driveName, ok := h.backupDriveIDNames.NameOf(drivePath.DriveID)
if !ok {
// This should not happen, but just in case
logger.Ctx(ctx).With("drive_id", drivePath.DriveID).Info("drive name not found, using drive id")

View File

@ -4,20 +4,18 @@ import (
"bytes"
"context"
"io"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
dataMock "github.com/alcionai/corso/src/internal/data/mock"
odConsts "github.com/alcionai/corso/src/internal/m365/service/onedrive/consts"
odStub "github.com/alcionai/corso/src/internal/m365/service/onedrive/stub"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
@ -62,27 +60,56 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
var (
driveID = "driveID1"
driveName = "driveName1"
itemName = "name1"
exportCfg = control.ExportConfig{}
dpb = odConsts.DriveFolderPrefixBuilder(driveID)
cache = idname.NewCache(
// Cache check with lowercased ids
map[string]string{strings.ToLower(driveID): driveName})
dii = odStub.DriveItemInfo()
expectedPath = path.LibrariesCategory.HumanString() + "/" + driveName
expectedItems = []export.Item{
{
ID: "id1.data",
Name: "name1",
Name: itemName,
Body: io.NopCloser((bytes.NewBufferString("body1"))),
},
}
)
dii.OneDrive.ItemName = "name1"
p, err := dpb.ToDataLayerSharePointPath("t", "u", path.LibrariesCategory, false)
assert.NoError(t, err, "build path")
table := []struct {
name string
itemInfo details.ItemInfo
}{
{
name: "OneDriveLegacyItemInfo",
itemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
ItemName: itemName,
Size: 1,
DriveName: driveName,
DriveID: driveID,
},
},
},
{
name: "SharePointItemInfo",
itemInfo: details.ItemInfo{
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointLibrary,
ItemName: itemName,
Size: 1,
DriveName: driveName,
DriveID: driveID,
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
dcs := []data.RestoreCollection{
data.FetchRestoreCollection{
Collection: dataMock.Collection{
@ -91,24 +118,23 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
&dataMock.Item{
ItemID: "id1.data",
Reader: io.NopCloser(bytes.NewBufferString("body1")),
ItemInfo: dii,
},
},
},
FetchItemByNamer: finD{id: "id1.meta", name: "name1"},
FetchItemByNamer: finD{id: "id1.meta", name: itemName},
},
}
handler := NewSharePointHandler(control.DefaultOptions())
handler.CacheItemInfo(test.itemInfo)
stats := data.ExportStats{}
ecs, err := ProduceExportCollections(
ecs, err := handler.ProduceExportCollections(
ctx,
int(version.Backup),
exportCfg,
control.DefaultOptions(),
dcs,
cache,
nil,
&stats,
fault.New(true))
assert.NoError(t, err, "export collections error")
@ -137,4 +163,6 @@ func (suite *ExportUnitSuite) TestExportRestoreCollections() {
expectedStats.UpdateBytes(path.FilesCategory, int64(size))
expectedStats.UpdateResourceCount(path.FilesCategory)
assert.Equal(t, expectedStats, stats, "stats")
})
}
}

View File

@ -261,9 +261,7 @@ func (op *ExportOperation) do(
ctx,
op.ec,
bup.Version,
op.Selectors,
op.ExportCfg,
op.Options,
dcs,
// We also have opStats, but that tracks different data.
// Maybe we can look into merging them some time in the future.
@ -329,9 +327,7 @@ func produceExportCollections(
ctx context.Context,
ec inject.ExportConsumer,
backupVersion int,
sel selectors.Selector,
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
exportStats *data.ExportStats,
errs *fault.Bus,
@ -342,12 +338,15 @@ func produceExportCollections(
close(complete)
}()
ctx, end := diagnostics.Span(ctx, "m365:export")
defer end()
ctx = clues.Add(ctx, "export_config", exportCfg)
expCollections, err := ec.ProduceExportCollections(
ctx,
backupVersion,
sel,
exportCfg,
opts,
dcs,
exportStats,
errs)

View File

@ -37,7 +37,7 @@ func TestExportUnitSuite(t *testing.T) {
suite.Run(t, &ExportUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ExportUnitSuite) TestExportOperation_PersistResults() {
func (suite *ExportUnitSuite) TestExportOperation_Export() {
var (
kw = &kopia.Wrapper{}
sw = store.NewWrapper(&kopia.ModelStore{})

View File

@ -15,7 +15,6 @@ import (
"github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
type (
@ -85,16 +84,12 @@ type (
ProduceExportCollections(
ctx context.Context,
backupVersion int,
selector selectors.Selector,
exportCfg control.ExportConfig,
opts control.Options,
dcs []data.RestoreCollection,
stats *data.ExportStats,
errs *fault.Bus,
) ([]export.Collectioner, error)
Wait() *data.CollectionStats
CacheItemInfoer
}
@ -117,4 +112,17 @@ type (
RepoMaintenancer interface {
RepoMaintenance(ctx context.Context, opts repository.Maintenance) error
}
// ServiceHandler contains the set of functions required to implement all
// service-specific functionality for backups, restores, and exports.
ServiceHandler interface {
ExportConsumer
}
ToServiceHandler interface {
NewServiceHandler(
opts control.Options,
service path.ServiceType,
) (ServiceHandler, error)
}
)

View File

@ -15,9 +15,10 @@ import (
type DataProvider interface {
inject.BackupProducer
inject.ExportConsumer
inject.RestoreConsumer
inject.ToServiceHandler
VerifyAccess(ctx context.Context) error
}

View File

@ -3,6 +3,8 @@ package repository
import (
"context"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/pkg/control"
@ -26,12 +28,17 @@ func (r repository) NewExport(
sel selectors.Selector,
exportCfg control.ExportConfig,
) (operations.ExportOperation, error) {
handler, err := r.Provider.NewServiceHandler(r.Opts, sel.PathService())
if err != nil {
return operations.ExportOperation{}, clues.Stack(err)
}
return operations.NewExportOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
r.Provider,
handler,
r.Account,
model.StableID(backupID),
sel,