generic drive retrieval for sharepoint (#1536)

## Description

Adapts the graph onedrive library to handle
access to drive data across both onedrive and
sharepoint services.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1506

## Test Plan

- [x] 💪 Manual
- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-11-22 11:05:47 -07:00 committed by GitHub
parent 5f82edb783
commit 10acf0ccf6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 901 additions and 335 deletions

View File

@ -29,7 +29,7 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se
ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String())) ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
defer end() defer end()
err := verifyBackupInputs(sels, gc.Users) err := verifyBackupInputs(sels, gc.GetUsers(), gc.GetSiteIds())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -40,52 +40,59 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se
case selectors.ServiceOneDrive: case selectors.ServiceOneDrive:
return gc.OneDriveDataCollections(ctx, sels) return gc.OneDriveDataCollections(ctx, sels)
case selectors.ServiceSharePoint: case selectors.ServiceSharePoint:
return gc.SharePointDataCollections(ctx, sels) colls, err := sharepoint.DataCollections(ctx, sels, gc.GetSiteIds(), gc.credentials.AzureTenantID, gc)
if err != nil {
return nil, err
}
for range colls {
gc.incrementAwaitingMessages()
}
return colls, nil
default: default:
return nil, errors.Errorf("service %s not supported", sels) return nil, errors.Errorf("service %s not supported", sels.Service.String())
} }
} }
func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) error { func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) error {
var personnel []string var ids []string
// retrieve users from selectors resourceOwners, err := sels.ResourceOwners()
switch sel.Service { if err != nil {
case selectors.ServiceExchange: return errors.Wrap(err, "invalid backup inputs")
backup, err := sel.ToExchangeBackup()
if err != nil {
return err
}
for _, scope := range backup.Scopes() {
temp := scope.Get(selectors.ExchangeUser)
personnel = append(personnel, temp...)
}
case selectors.ServiceOneDrive:
backup, err := sel.ToOneDriveBackup()
if err != nil {
return err
}
for _, user := range backup.Scopes() {
temp := user.Get(selectors.OneDriveUser)
personnel = append(personnel, temp...)
}
default:
return errors.New("service %s not supported")
} }
// verify personnel switch sels.Service {
normUsers := map[string]struct{}{} case selectors.ServiceExchange, selectors.ServiceOneDrive:
ids = userPNs
for k := range mapOfUsers { case selectors.ServiceSharePoint:
normUsers[strings.ToLower(k)] = struct{}{} ids = siteIDs
} }
for _, user := range personnel { // verify resourceOwners
if _, ok := normUsers[strings.ToLower(user)]; !ok { normROs := map[string]struct{}{}
return fmt.Errorf("%s user not found within tenant", user)
for _, id := range ids {
normROs[strings.ToLower(id)] = struct{}{}
}
for _, ro := range resourceOwners.Includes {
if _, ok := normROs[strings.ToLower(ro)]; !ok {
return fmt.Errorf("included resource owner %s not found within tenant", ro)
}
}
for _, ro := range resourceOwners.Excludes {
if _, ok := normROs[strings.ToLower(ro)]; !ok {
return fmt.Errorf("excluded resource owner %s not found within tenant", ro)
}
}
for _, ro := range resourceOwners.Filters {
if _, ok := normROs[strings.ToLower(ro)]; !ok {
return fmt.Errorf("filtered resource owner %s not found within tenant", ro)
} }
} }
@ -193,6 +200,18 @@ func (gc *GraphConnector) ExchangeDataCollection(
// OneDrive // OneDrive
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type odFolderMatcher struct {
scope selectors.OneDriveScope
}
func (fm odFolderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.OneDriveFolder)
}
func (fm odFolderMatcher) Matches(dir string) bool {
return fm.scope.Matches(selectors.OneDriveFolder, dir)
}
// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data // OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
// for the specified user // for the specified user
func (gc *GraphConnector) OneDriveDataCollections( func (gc *GraphConnector) OneDriveDataCollections(
@ -218,7 +237,8 @@ func (gc *GraphConnector) OneDriveDataCollections(
odcs, err := onedrive.NewCollections( odcs, err := onedrive.NewCollections(
gc.credentials.AzureTenantID, gc.credentials.AzureTenantID,
user, user,
scope, onedrive.OneDriveSource,
odFolderMatcher{scope},
&gc.graphService, &gc.graphService,
gc.UpdateStatus, gc.UpdateStatus,
).Get(ctx) ).Get(ctx)
@ -236,104 +256,3 @@ func (gc *GraphConnector) OneDriveDataCollections(
return collections, errs return collections, errs
} }
// ---------------------------------------------------------------------------
// SharePoint
// ---------------------------------------------------------------------------
// createSharePointCollections - utility function that retrieves M365
// IDs through Microsoft Graph API. The selectors.SharePointScope
// determines the type of collections that are retrieved.
func (gc *GraphConnector) createSharePointCollections(
ctx context.Context,
scope selectors.SharePointScope,
) ([]*sharepoint.Collection, error) {
var (
errs *multierror.Error
sites = scope.Get(selectors.SharePointSite)
colls = make([]*sharepoint.Collection, 0)
)
// Create collection of ExchangeDataCollection
for _, site := range sites {
collections := make(map[string]*sharepoint.Collection)
qp := graph.QueryParams{
Category: scope.Category().PathType(),
ResourceOwner: site,
FailFast: gc.failFast,
Credentials: gc.credentials,
}
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, site))
defer closer()
defer close(foldersComplete)
// resolver, err := exchange.PopulateExchangeContainerResolver(
// ctx,
// qp,
// qp.Scope.Category().PathType(),
// )
// if err != nil {
// return nil, errors.Wrap(err, "getting folder cache")
// }
// err = sharepoint.FilterContainersAndFillCollections(
// ctx,
// qp,
// collections,
// gc.UpdateStatus,
// resolver)
// if err != nil {
// return nil, errors.Wrap(err, "filling collections")
// }
foldersComplete <- struct{}{}
for _, collection := range collections {
gc.incrementAwaitingMessages()
colls = append(colls, collection)
}
}
return colls, errs.ErrorOrNil()
}
// SharePointDataCollections returns a set of DataCollection which represents the SharePoint data
// for the specified user
func (gc *GraphConnector) SharePointDataCollections(
ctx context.Context,
selector selectors.Selector,
) ([]data.Collection, error) {
b, err := selector.ToSharePointBackup()
if err != nil {
return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector")
}
var (
scopes = b.DiscreteScopes(gc.GetSites())
collections = []data.Collection{}
errs error
)
// for each scope that includes oneDrive items, get all
for _, scope := range scopes {
// Creates a map of collections based on scope
dcs, err := gc.createSharePointCollections(ctx, scope)
if err != nil {
return nil, support.WrapAndAppend(scope.Get(selectors.SharePointSite)[0], err, errs)
}
for _, collection := range dcs {
collections = append(collections, collection)
}
}
for range collections {
gc.incrementAwaitingMessages()
}
return collections, errs
}

View File

@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
@ -162,7 +163,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
connector := loadConnector(ctx, suite.T(), Users) connector := loadConnector(ctx, suite.T(), Sites)
tests := []struct { tests := []struct {
name string name string
getSelector func(t *testing.T) selectors.Selector getSelector func(t *testing.T) selectors.Selector
@ -180,24 +181,28 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
for _, test := range tests { for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
_, err := connector.SharePointDataCollections(ctx, test.getSelector(t)) collection, err := sharepoint.DataCollections(
ctx,
test.getSelector(t),
[]string{suite.site},
connector.credentials.AzureTenantID,
connector)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, 1, len(collection))
// TODO: Implementation // the test only reads the firstt collection
// assert.Equal(t, len(collection), 1) connector.incrementAwaitingMessages()
// channel := collection[0].Items() for object := range collection[0].Items() {
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(object.ToReader())
assert.NoError(t, err, "received a buf.Read error")
}
// for object := range channel { status := connector.AwaitStatus()
// buf := &bytes.Buffer{} assert.NotZero(t, status.Successful)
// _, err := buf.ReadFrom(object.ToReader())
// assert.NoError(t, err, "received a buf.Read error")
// }
// status := connector.AwaitStatus() t.Log(status.String())
// assert.NotZero(t, status.Successful)
// t.Log(status.String())
}) })
} }
} }
@ -499,16 +504,17 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
var ( var (
t = suite.T() t = suite.T()
userID = tester.M365UserID(t) siteID = tester.M365SiteID(t)
gc = loadConnector(ctx, t, Sites)
sel = selectors.NewSharePointBackup()
) )
gc := loadConnector(ctx, t, Sites) sel.Include(sel.Folders(
scope := selectors.NewSharePointBackup().Folders( []string{siteID},
[]string{userID},
[]string{"foo"}, []string{"foo"},
selectors.PrefixMatch(), selectors.PrefixMatch(),
)[0] ))
_, err := gc.createSharePointCollections(ctx, scope) _, err := gc.DataCollections(ctx, sel.Selector)
require.NoError(t, err) require.NoError(t, err)
} }

View File

@ -18,7 +18,7 @@ import (
// --------------------------------------------------------------- // ---------------------------------------------------------------
// Disconnected Test Section // Disconnected Test Section
// ------------------------- // ---------------------------------------------------------------
type DisconnectedGraphConnectorSuite struct { type DisconnectedGraphConnectorSuite struct {
suite.Suite suite.Suite
} }
@ -206,12 +206,13 @@ func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() {
} }
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() { func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
users := make(map[string]string) users := []string{
users["elliotReid@someHospital.org"] = "" "elliotReid@someHospital.org",
users["chrisTurk@someHospital.org"] = "" "chrisTurk@someHospital.org",
users["carlaEspinosa@someHospital.org"] = "" "carlaEspinosa@someHospital.org",
users["bobKelso@someHospital.org"] = "" "bobKelso@someHospital.org",
users["johnDorian@someHospital.org"] = "" "johnDorian@someHospital.org",
}
tests := []struct { tests := []struct {
name string name string
@ -219,12 +220,10 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
checkError assert.ErrorAssertionFunc checkError assert.ErrorAssertionFunc
}{ }{
{ {
name: "Invalid User", name: "No scopes",
checkError: assert.Error, checkError: assert.NoError,
getSelector: func(t *testing.T) selectors.Selector { getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup() return selectors.NewExchangeBackup().Selector
sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
return sel.Selector
}, },
}, },
{ {
@ -260,7 +259,108 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
for _, test := range tests { for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
err := verifyBackupInputs(test.getSelector(t), users) err := verifyBackupInputs(test.getSelector(t), users, nil)
test.checkError(t, err)
})
}
}
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices() {
users := []string{"elliotReid@someHospital.org"}
sites := []string{"abc.site.foo", "bar.site.baz"}
tests := []struct {
name string
excludes func(t *testing.T) selectors.Selector
filters func(t *testing.T) selectors.Selector
includes func(t *testing.T) selectors.Selector
checkError assert.ErrorAssertionFunc
}{
{
name: "Valid User",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Exclude(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Filter(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Include(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
return sel.Selector
},
},
{
name: "Invalid User",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Exclude(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Filter(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup()
sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
return sel.Selector
},
},
{
name: "valid sites",
checkError: assert.NoError,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Exclude(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Filter(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Include(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
return sel.Selector
},
},
{
name: "invalid sites",
checkError: assert.Error,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Exclude(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
return sel.Selector
},
filters: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Filter(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
return sel.Selector
},
includes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Include(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
return sel.Selector
},
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
err := verifyBackupInputs(test.excludes(t), users, sites)
test.checkError(t, err)
err = verifyBackupInputs(test.filters(t), users, sites)
test.checkError(t, err)
err = verifyBackupInputs(test.includes(t), users, sites)
test.checkError(t, err) test.checkError(t, err)
}) })
} }

View File

@ -19,6 +19,7 @@ import (
// 6. subject // 6. subject
// 7. hasAttachments // 7. hasAttachments
// 8. attachments // 8. attachments
//
//nolint:lll //nolint:lll
const ( const (
eventTmpl = `{ eventTmpl = `{

View File

@ -60,7 +60,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
collStatus := support.ConnectorOperationStatus{} collStatus := support.ConnectorOperationStatus{}
folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user") folderPath, err := GetCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user", OneDriveSource)
require.NoError(t, err) require.NoError(t, err)
driveFolderPath, err := getDriveFolderPath(folderPath) driveFolderPath, err := getDriveFolderPath(folderPath)
require.NoError(t, err) require.NoError(t, err)
@ -117,7 +117,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
wg.Add(1) wg.Add(1)
folderPath, err := getCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user") folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", OneDriveSource)
require.NoError(t, err) require.NoError(t, err)
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus)) coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))

View File

@ -14,79 +14,172 @@ import (
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
) )
// Collections is used to retrieve OneDrive data for a type driveSource int
// specified user
const (
unknownDriveSource driveSource = iota
OneDriveSource
SharePointSource
)
type folderMatcher interface {
IsAny() bool
Matches(string) bool
}
// Collections is used to retrieve drive data for a
// resource owner, which can be either a user or a sharepoint site.
type Collections struct { type Collections struct {
tenant string tenant string
user string resourceOwner string
scope selectors.OneDriveScope source driveSource
// collectionMap allows lookup of the data.Collection matcher folderMatcher
// for a OneDrive folder
collectionMap map[string]data.Collection
service graph.Service service graph.Service
statusUpdater support.StatusUpdater statusUpdater support.StatusUpdater
// collectionMap allows lookup of the data.Collection
// for a OneDrive folder
CollectionMap map[string]data.Collection
// Track stats from drive enumeration. Represents the items backed up. // Track stats from drive enumeration. Represents the items backed up.
numItems int NumItems int
numFiles int NumFiles int
numContainers int NumContainers int
} }
func NewCollections( func NewCollections(
tenant string, tenant string,
user string, resourceOwner string,
scope selectors.OneDriveScope, source driveSource,
matcher folderMatcher,
service graph.Service, service graph.Service,
statusUpdater support.StatusUpdater, statusUpdater support.StatusUpdater,
) *Collections { ) *Collections {
return &Collections{ return &Collections{
tenant: tenant, tenant: tenant,
user: user, resourceOwner: resourceOwner,
scope: scope, source: source,
collectionMap: map[string]data.Collection{}, matcher: matcher,
CollectionMap: map[string]data.Collection{},
service: service, service: service,
statusUpdater: statusUpdater, statusUpdater: statusUpdater,
} }
} }
// Retrieves OneDrive data as set of `data.Collections` // Retrieves drive data as set of `data.Collections`
func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
// Enumerate drives for the specified user // Enumerate drives for the specified resourceOwner
drives, err := drives(ctx, c.service, c.user) drives, err := drives(ctx, c.service, c.resourceOwner, c.source)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Update the collection map with items from each drive // Update the collection map with items from each drive
for _, d := range drives { for _, d := range drives {
err = collectItems(ctx, c.service, *d.GetId(), c.updateCollections) err = collectItems(ctx, c.service, *d.GetId(), c.UpdateCollections)
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
observe.Message(fmt.Sprintf("Discovered %d items to backup", c.numItems)) observe.Message(fmt.Sprintf("Discovered %d items to backup", c.NumItems))
collections := make([]data.Collection, 0, len(c.collectionMap)) collections := make([]data.Collection, 0, len(c.CollectionMap))
for _, coll := range c.collectionMap { for _, coll := range c.CollectionMap {
collections = append(collections, coll) collections = append(collections, coll)
} }
return collections, nil return collections, nil
} }
func getCanonicalPath(p, tenant, user string) (path.Path, error) { // UpdateCollections initializes and adds the provided drive items to Collections
pathBuilder := path.Builder{}.Append(strings.Split(p, "/")...) // A new collection is created for every drive folder (or package)
func (c *Collections) UpdateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {
for _, item := range items {
if item.GetRoot() != nil {
// Skip the root item
continue
}
if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil {
return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName())
}
// Create a collection for the parent of this item
collectionPath, err := GetCanonicalPath(
*item.GetParentReference().GetPath(),
c.tenant,
c.resourceOwner,
c.source,
)
if err != nil {
return err
}
// Skip items that don't match the folder selectors we were given.
if !includePath(ctx, c.matcher, collectionPath) {
logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
continue
}
switch {
case item.GetFolder() != nil, item.GetPackage() != nil:
// Leave this here so we don't fall into the default case.
// TODO: This is where we might create a "special file" to represent these in the backup repository
// e.g. a ".folderMetadataFile"
case item.GetFile() != nil:
col, found := c.CollectionMap[collectionPath.String()]
if !found {
col = NewCollection(
collectionPath,
driveID,
c.service,
c.statusUpdater,
)
c.CollectionMap[collectionPath.String()] = col
c.NumContainers++
c.NumItems++
}
collection := col.(*Collection)
collection.Add(*item.GetId())
c.NumFiles++
c.NumItems++
default:
return errors.Errorf("item type not supported. item name : %s", *item.GetName())
}
}
return nil
}
// GetCanonicalPath constructs the standard path for the given source.
func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) {
var (
pathBuilder = path.Builder{}.Append(strings.Split(p, "/")...)
result path.Path
err error
)
switch source {
case OneDriveSource:
result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false)
case SharePointSource:
result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false)
default:
return nil, errors.Errorf("unrecognized drive data source")
}
res, err := pathBuilder.ToDataLayerOneDrivePath(tenant, user, false)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "converting to canonical path") return nil, errors.Wrap(err, "converting to canonical path")
} }
return res, nil return result, nil
} }
// Returns the path to the folder within the drive (i.e. under `root:`) // Returns the path to the folder within the drive (i.e. under `root:`)
@ -99,70 +192,7 @@ func getDriveFolderPath(p path.Path) (string, error) {
return path.Builder{}.Append(drivePath.folders...).String(), nil return path.Builder{}.Append(drivePath.folders...).String(), nil
} }
// updateCollections initializes and adds the provided OneDrive items to Collections func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) bool {
// A new collection is created for every OneDrive folder (or package)
func (c *Collections) updateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {
for _, item := range items {
if item.GetRoot() != nil {
// Skip the root item
continue
}
if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil {
return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName())
}
// Create a collection for the parent of this item
collectionPath, err := getCanonicalPath(
*item.GetParentReference().GetPath(),
c.tenant,
c.user,
)
if err != nil {
return err
}
// Skip items that don't match the folder selectors we were given.
if !includePath(ctx, c.scope, collectionPath) {
logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
continue
}
switch {
case item.GetFolder() != nil, item.GetPackage() != nil:
// Leave this here so we don't fall into the default case.
// TODO: This is where we might create a "special file" to represent these in the backup repository
// e.g. a ".folderMetadataFile"
case item.GetFile() != nil:
col, found := c.collectionMap[collectionPath.String()]
if !found {
col = NewCollection(
collectionPath,
driveID,
c.service,
c.statusUpdater,
)
c.collectionMap[collectionPath.String()] = col
c.numContainers++
c.numItems++
}
collection := col.(*Collection)
collection.Add(*item.GetId())
c.numFiles++
c.numItems++
default:
return errors.Errorf("item type not supported. item name : %s", *item.GetName())
}
}
return nil
}
func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath path.Path) bool {
// Check if the folder is allowed by the scope. // Check if the folder is allowed by the scope.
folderPathString, err := getDriveFolderPath(folderPath) folderPathString, err := getDriveFolderPath(folderPath)
if err != nil { if err != nil {
@ -172,9 +202,9 @@ func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath
// Hack for the edge case where we're looking at the root folder and can // Hack for the edge case where we're looking at the root folder and can
// select any folder. Right now the root folder has an empty folder path. // select any folder. Right now the root folder has an empty folder path.
if len(folderPathString) == 0 && scope.IsAny(selectors.OneDriveFolder) { if len(folderPathString) == 0 && m.IsAny() {
return true return true
} }
return scope.Matches(selectors.OneDriveFolder, folderPathString) return m.Matches(folderPathString)
} }

View File

@ -1,6 +1,7 @@
package onedrive package onedrive
import ( import (
"strings"
"testing" "testing"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -20,7 +21,7 @@ func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []st
res := make([]string, 0, len(rest)) res := make([]string, 0, len(rest))
for _, r := range rest { for _, r := range rest {
p, err := getCanonicalPath(r, tenant, user) p, err := GetCanonicalPath(r, tenant, user, OneDriveSource)
require.NoError(t, err) require.NoError(t, err)
res = append(res, p.String()) res = append(res, p.String())
@ -37,6 +38,49 @@ func TestOneDriveCollectionsSuite(t *testing.T) {
suite.Run(t, new(OneDriveCollectionsSuite)) suite.Run(t, new(OneDriveCollectionsSuite))
} }
func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() {
tenant, resourceOwner := "tenant", "resourceOwner"
table := []struct {
name string
source driveSource
dir []string
expect string
expectErr assert.ErrorAssertionFunc
}{
{
name: "onedrive",
source: OneDriveSource,
dir: []string{"onedrive"},
expect: "tenant/onedrive/resourceOwner/files/onedrive",
expectErr: assert.NoError,
},
{
name: "sharepoint",
source: SharePointSource,
dir: []string{"sharepoint"},
expect: "tenant/sharepoint/resourceOwner/files/sharepoint",
expectErr: assert.NoError,
},
{
name: "unknown",
source: unknownDriveSource,
dir: []string{"unknown"},
expectErr: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
p := strings.Join(test.dir, "/")
result, err := GetCanonicalPath(p, tenant, resourceOwner, test.source)
test.expectErr(t, err)
if result != nil {
assert.Equal(t, test.expect, result.String())
}
})
}
}
func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0] anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0]
@ -211,15 +255,15 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil) c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil)
err := c.updateCollections(ctx, "driveID", tt.items) err := c.UpdateCollections(ctx, "driveID", tt.items)
tt.expect(t, err) tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths") assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
assert.Equal(t, tt.expectedItemCount, c.numItems, "item count") assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, tt.expectedFileCount, c.numFiles, "file count") assert.Equal(t, tt.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, tt.expectedContainerCount, c.numContainers, "container count") assert.Equal(t, tt.expectedContainerCount, c.NumContainers, "container count")
for _, collPath := range tt.expectedCollectionPaths { for _, collPath := range tt.expectedCollectionPaths {
assert.Contains(t, c.collectionMap, collPath) assert.Contains(t, c.CollectionMap, collPath)
} }
}) })
} }

View File

@ -67,7 +67,33 @@ const (
) )
// Enumerates the drives for the specified user // Enumerates the drives for the specified user
func drives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) { func drives(
ctx context.Context,
service graph.Service,
resourceOwner string,
source driveSource,
) ([]models.Driveable, error) {
switch source {
case OneDriveSource:
return userDrives(ctx, service, resourceOwner)
case SharePointSource:
return siteDrives(ctx, service, resourceOwner)
default:
return nil, errors.Errorf("unrecognized drive data source")
}
}
func siteDrives(ctx context.Context, service graph.Service, site string) ([]models.Driveable, error) {
r, err := service.Client().SitesById(site).Drives().Get(ctx, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to retrieve site drives. site: %s, details: %s",
site, support.ConnectorStackErrorTrace(err))
}
return r.GetValue(), nil
}
func userDrives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) {
var hasDrive bool var hasDrive bool
hasDrive, err := hasDriveLicense(ctx, service, user) hasDrive, err := hasDriveLicense(ctx, service, user)
@ -237,7 +263,7 @@ func GetAllFolders(
userID string, userID string,
prefix string, prefix string,
) ([]*Displayable, error) { ) ([]*Displayable, error) {
drives, err := drives(ctx, gs, userID) drives, err := drives(ctx, gs, userID, OneDriveSource)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "getting OneDrive folders") return nil, errors.Wrap(err, "getting OneDrive folders")
} }
@ -321,7 +347,7 @@ func hasDriveLicense(
cb := func(pageItem any) bool { cb := func(pageItem any) bool {
entry, ok := pageItem.(models.LicenseDetailsable) entry, ok := pageItem.(models.LicenseDetailsable)
if !ok { if !ok {
err = errors.New("casting item to models.MailFolderable") err = errors.New("casting item to models.LicenseDetailsable")
return false return false
} }

View File

@ -43,7 +43,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
folderElements := []string{folderName1} folderElements := []string{folderName1}
gs := loadTestService(t) gs := loadTestService(t)
drives, err := drives(ctx, gs, suite.userID) drives, err := drives(ctx, gs, suite.userID, OneDriveSource)
require.NoError(t, err) require.NoError(t, err)
require.NotEmpty(t, drives) require.NotEmpty(t, drives)
@ -100,6 +100,18 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
} }
} }
type testFolderMatcher struct {
scope selectors.OneDriveScope
}
func (fm testFolderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.OneDriveFolder)
}
func (fm testFolderMatcher) Matches(path string) bool {
return fm.scope.Matches(selectors.OneDriveFolder, path)
}
func (suite *OneDriveSuite) TestOneDriveNewCollections() { func (suite *OneDriveSuite) TestOneDriveNewCollections() {
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
defer flush() defer flush()
@ -129,7 +141,8 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() {
odcs, err := NewCollections( odcs, err := NewCollections(
creds.AzureTenantID, creds.AzureTenantID,
test.user, test.user,
scope, OneDriveSource,
testFolderMatcher{scope},
service, service,
service.updateStatus, service.updateStatus,
).Get(ctx) ).Get(ctx)

View File

@ -67,7 +67,7 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
suite.user = tester.SecondaryM365UserID(suite.T()) suite.user = tester.SecondaryM365UserID(suite.T())
drives, err := drives(ctx, suite, suite.user) drives, err := drives(ctx, suite, suite.user, OneDriveSource)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
// Test Requirement 1: Need a drive // Test Requirement 1: Need a drive
require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user) require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user)

View File

@ -0,0 +1,124 @@
package sharepoint
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
type statusUpdater interface {
UpdateStatus(status *support.ConnectorOperationStatus)
}
type connector interface {
statusUpdater
Service() graph.Service
}
// DataCollections returns a set of DataCollection which represents the SharePoint data
// for the specified user
func DataCollections(
ctx context.Context,
selector selectors.Selector,
siteIDs []string,
tenantID string,
con connector,
) ([]data.Collection, error) {
b, err := selector.ToSharePointBackup()
if err != nil {
return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector")
}
var (
scopes = b.DiscreteScopes(siteIDs)
collections = []data.Collection{}
serv = con.Service()
errs error
)
for _, scope := range scopes {
// due to DiscreteScopes(siteIDs), each range should only contain one site.
for _, site := range scope.Get(selectors.SharePointSite) {
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf(
"∙ %s - %s:",
scope.Category().PathType(), site))
defer closer()
defer close(foldersComplete)
switch scope.Category().PathType() {
case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory
spcs, err := collectLibraries(
ctx,
serv,
tenantID,
site,
scope,
con)
if err != nil {
return nil, support.WrapAndAppend(site, err, errs)
}
collections = append(collections, spcs...)
}
foldersComplete <- struct{}{}
}
}
return collections, errs
}
// collectLibraries constructs a onedrive Collections struct and Get()s
// all the drives associated with the site.
func collectLibraries(
ctx context.Context,
serv graph.Service,
tenantID, siteID string,
scope selectors.SharePointScope,
updater statusUpdater,
) ([]data.Collection, error) {
var (
collections = []data.Collection{}
errs error
)
logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Library collections")
colls := onedrive.NewCollections(
tenantID,
siteID,
onedrive.SharePointSource,
folderMatcher{scope},
serv,
updater.UpdateStatus)
odcs, err := colls.Get(ctx)
if err != nil {
return nil, support.WrapAndAppend(siteID, err, errs)
}
return append(collections, odcs...), errs
}
type folderMatcher struct {
scope selectors.SharePointScope
}
func (fm folderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.SharePointFolder)
}
func (fm folderMatcher) Matches(dir string) bool {
return fm.scope.Matches(selectors.SharePointFolder, dir)
}

View File

@ -0,0 +1,158 @@
package sharepoint_test
import (
"context"
"testing"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
// consts, mocks
// ---------------------------------------------------------------------------
const (
testBaseDrivePath = "drive/driveID1/root:"
)
type testFolderMatcher struct {
scope selectors.SharePointScope
}
func (fm testFolderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.SharePointFolder)
}
func (fm testFolderMatcher) Matches(path string) bool {
return fm.scope.Matches(selectors.SharePointFolder, path)
}
type MockGraphService struct{}
func (ms *MockGraphService) Client() *msgraphsdk.GraphServiceClient {
return nil
}
func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
return nil
}
func (ms *MockGraphService) ErrPolicy() bool {
return false
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
type SharePointLibrariesSuite struct {
suite.Suite
ctx context.Context
}
func TestSharePointLibrariesSuite(t *testing.T) {
suite.Run(t, new(SharePointLibrariesSuite))
}
func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0]
const (
tenant = "tenant"
site = "site"
)
tests := []struct {
testCase string
items []models.DriveItemable
scope selectors.SharePointScope
expect assert.ErrorAssertionFunc
expectedCollectionPaths []string
expectedItemCount int
expectedContainerCount int
expectedFileCount int
}{
{
testCase: "Single File",
items: []models.DriveItemable{
driveItem("file", testBaseDrivePath, true),
},
scope: anyFolder,
expect: assert.NoError,
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
tenant,
site,
testBaseDrivePath,
),
expectedItemCount: 2,
expectedFileCount: 1,
expectedContainerCount: 1,
},
}
for _, test := range tests {
suite.T().Run(test.testCase, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
c := onedrive.NewCollections(
tenant,
site,
onedrive.SharePointSource,
testFolderMatcher{test.scope},
&MockGraphService{},
nil)
err := c.UpdateCollections(ctx, "driveID", test.items)
test.expect(t, err)
assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
for _, collPath := range test.expectedCollectionPaths {
assert.Contains(t, c.CollectionMap, collPath)
}
})
}
}
func driveItem(name string, path string, isFile bool) models.DriveItemable {
item := models.NewDriveItem()
item.SetName(&name)
item.SetId(&name)
parentReference := models.NewItemReference()
parentReference.SetPath(&path)
item.SetParentReference(parentReference)
if isFile {
item.SetFile(models.NewFile())
}
return item
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {
res := make([]string, 0, len(rest))
for _, r := range rest {
p, err := onedrive.GetCanonicalPath(r, tenant, user, onedrive.SharePointSource)
require.NoError(t, err)
res = append(res, p.String())
}
return res
}

View File

@ -23,6 +23,7 @@ const (
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS" CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS" CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS" CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"
CorsoGraphConnectorSharePointTests = "CORSO_GRAPH_CONNECTOR_SHAREPOINT_TESTS"
CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS" CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS"
CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS" CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS"
CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS" CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS"

View File

@ -241,7 +241,7 @@ func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
} }
if len(resourceOwner) == 0 { if len(resourceOwner) == 0 {
return errors.Wrap(errMissingSegment, "user") return errors.Wrap(errMissingSegment, "resourceOwner")
} }
if len(pb.elements) == 0 { if len(pb.elements) == 0 {

View File

@ -36,7 +36,11 @@ type (
} }
) )
var _ Reducer = &ExchangeRestore{} var (
_ Reducer = &ExchangeRestore{}
_ printabler = &ExchangeRestore{}
_ resourceOwnerer = &ExchangeRestore{}
)
// NewExchange produces a new Selector with the service set to ServiceExchange. // NewExchange produces a new Selector with the service set to ServiceExchange.
func NewExchangeBackup() *ExchangeBackup { func NewExchangeBackup() *ExchangeBackup {
@ -89,6 +93,16 @@ func (s exchange) Printable() Printable {
return toPrintable[ExchangeScope](s.Selector) return toPrintable[ExchangeScope](s.Selector)
} }
// ResourceOwners produces the aggregation of discrete users described by each type of scope.
// Any and None values are omitted.
func (s exchange) ResourceOwners() selectorResourceOwners {
return selectorResourceOwners{
Excludes: resourceOwnersIn(s.Excludes, ExchangeUser.String()),
Filters: resourceOwnersIn(s.Filters, ExchangeUser.String()),
Includes: resourceOwnersIn(s.Includes, ExchangeUser.String()),
}
}
// ------------------- // -------------------
// Exclude/Includes // Exclude/Includes

View File

@ -35,7 +35,11 @@ type (
} }
) )
var _ Reducer = &OneDriveRestore{} var (
_ Reducer = &OneDriveRestore{}
_ printabler = &OneDriveRestore{}
_ resourceOwnerer = &OneDriveRestore{}
)
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive. // NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
func NewOneDriveBackup() *OneDriveBackup { func NewOneDriveBackup() *OneDriveBackup {
@ -88,6 +92,16 @@ func (s oneDrive) Printable() Printable {
return toPrintable[OneDriveScope](s.Selector) return toPrintable[OneDriveScope](s.Selector)
} }
// ResourceOwners produces the aggregation of discrete users described by each type of scope.
// Any and None values are omitted.
func (s oneDrive) ResourceOwners() selectorResourceOwners {
return selectorResourceOwners{
Excludes: resourceOwnersIn(s.Excludes, OneDriveUser.String()),
Filters: resourceOwnersIn(s.Filters, OneDriveUser.String()),
Includes: resourceOwnersIn(s.Includes, OneDriveUser.String()),
}
}
// ------------------- // -------------------
// Scope Factories // Scope Factories

View File

@ -54,13 +54,13 @@ const (
// None() to any selector will force all matches() checks on that // None() to any selector will force all matches() checks on that
// selector to fail. // selector to fail.
// Ex: {user: u1, events: NoneTgt} => matches nothing. // Ex: {user: u1, events: NoneTgt} => matches nothing.
NoneTgt = "" NoneTgt = ""
delimiter = ","
) )
var ( var (
passAny = filters.Pass() delimiter = fmt.Sprint(0x1F)
failAny = filters.Fail() passAny = filters.Pass()
failAny = filters.Fail()
) )
// All is the resource name that gets output when the resource is AnyTgt. // All is the resource name that gets output when the resource is AnyTgt.
@ -71,6 +71,19 @@ type Reducer interface {
Reduce(context.Context, *details.Details) *details.Details Reduce(context.Context, *details.Details) *details.Details
} }
// selectorResourceOwners aggregates all discrete resource owner ids described
// in the selector. Any and None values are ignored. ResourceOwner sets are
// grouped by their scope type (includes, excludes, filters).
type selectorResourceOwners struct {
Includes []string
Excludes []string
Filters []string
}
type resourceOwnerer interface {
ResourceOwners() selectorResourceOwners
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Selector // Selector
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -184,22 +197,7 @@ func (s Selector) PathService() path.ServiceType {
// than have the caller make that interpretation. Returns an error if the // than have the caller make that interpretation. Returns an error if the
// service is unsupported. // service is unsupported.
func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.Details, error) { func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.Details, error) {
var ( r, err := selectorAsIface[Reducer](s)
r Reducer
err error
)
switch s.Service {
case ServiceExchange:
r, err = s.ToExchangeRestore()
case ServiceOneDrive:
r, err = s.ToOneDriveRestore()
case ServiceSharePoint:
r, err = s.ToSharePointRestore()
default:
return nil, errors.New("service not supported: " + s.Service.String())
}
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -207,6 +205,40 @@ func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.
return r.Reduce(ctx, deets), nil return r.Reduce(ctx, deets), nil
} }
func (s Selector) ResourceOwners() (selectorResourceOwners, error) {
ro, err := selectorAsIface[resourceOwnerer](s)
if err != nil {
return selectorResourceOwners{}, err
}
return ro.ResourceOwners(), nil
}
// transformer for arbitrary selector interfaces
func selectorAsIface[T any](s Selector) (T, error) {
var (
a any
t T
err error
)
switch s.Service {
case ServiceExchange:
a, err = func() (any, error) { return s.ToExchangeRestore() }()
t = a.(T)
case ServiceOneDrive:
a, err = func() (any, error) { return s.ToOneDriveRestore() }()
t = a.(T)
case ServiceSharePoint:
a, err = func() (any, error) { return s.ToSharePointRestore() }()
t = a.(T)
default:
err = errors.New("service not supported: " + s.Service.String())
}
return t, err
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Printing Selectors for Human Reading // Printing Selectors for Human Reading
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -218,35 +250,18 @@ type Printable struct {
Includes map[string][]string `json:"includes,omitempty"` Includes map[string][]string `json:"includes,omitempty"`
} }
type printabler interface {
Printable() Printable
}
// ToPrintable creates the minimized display of a selector, formatted for human readability. // ToPrintable creates the minimized display of a selector, formatted for human readability.
func (s Selector) ToPrintable() Printable { func (s Selector) ToPrintable() Printable {
switch s.Service { p, err := selectorAsIface[printabler](s)
case ServiceExchange: if err != nil {
r, err := s.ToExchangeRestore() return Printable{}
if err != nil {
return Printable{}
}
return r.Printable()
case ServiceOneDrive:
r, err := s.ToOneDriveBackup()
if err != nil {
return Printable{}
}
return r.Printable()
case ServiceSharePoint:
r, err := s.ToSharePointBackup()
if err != nil {
return Printable{}
}
return r.Printable()
} }
return Printable{} return p.Printable()
} }
// toPrintable creates the minimized display of a selector, formatted for human readability. // toPrintable creates the minimized display of a selector, formatted for human readability.
@ -351,6 +366,32 @@ func addToSet(set []string, v []string) []string {
// helpers // helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// produces the discrete set of resource owners in the slice of scopes.
// Any and None values are discarded.
func resourceOwnersIn(s []scope, rootCat string) []string {
rm := map[string]struct{}{}
for _, sc := range s {
for _, v := range split(sc[rootCat].Target) {
rm[v] = struct{}{}
}
}
rs := []string{}
for k := range rm {
if k != AnyTgt && k != NoneTgt {
rs = append(rs, k)
}
}
return rs
}
// ---------------------------------------------------------------------------
// scope helpers
// ---------------------------------------------------------------------------
type scopeConfig struct { type scopeConfig struct {
usePathFilter bool usePathFilter bool
usePrefixFilter bool usePrefixFilter bool
@ -382,10 +423,6 @@ func pathType() option {
} }
} }
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
func badCastErr(cast, is service) error { func badCastErr(cast, is service) error {
return errors.Wrapf(ErrorBadSelectorCast, "%s service is not %s", cast, is) return errors.Wrapf(ErrorBadSelectorCast, "%s service is not %s", cast, is)
} }

View File

@ -6,6 +6,8 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/pkg/filters"
) )
type SelectorSuite struct { type SelectorSuite struct {
@ -143,6 +145,69 @@ func (suite *SelectorSuite) TestToResourceTypeMap() {
} }
} }
func (suite *SelectorSuite) TestResourceOwnersIn() {
rootCat := rootCatStub.String()
table := []struct {
name string
input []scope
expect []string
}{
{
name: "nil",
input: nil,
expect: []string{},
},
{
name: "empty",
input: []scope{},
expect: []string{},
},
{
name: "single",
input: []scope{{rootCat: filters.Identity("foo")}},
expect: []string{"foo"},
},
{
name: "multiple values",
input: []scope{{rootCat: filters.Identity(join("foo", "bar"))}},
expect: []string{"foo", "bar"},
},
{
name: "with any",
input: []scope{{rootCat: filters.Identity(join("foo", "bar", AnyTgt))}},
expect: []string{"foo", "bar"},
},
{
name: "with none",
input: []scope{{rootCat: filters.Identity(join("foo", "bar", NoneTgt))}},
expect: []string{"foo", "bar"},
},
{
name: "multiple scopes",
input: []scope{
{rootCat: filters.Identity(join("foo", "bar"))},
{rootCat: filters.Identity(join("baz"))},
},
expect: []string{"foo", "bar", "baz"},
},
{
name: "multiple scopes with duplicates",
input: []scope{
{rootCat: filters.Identity(join("foo", "bar"))},
{rootCat: filters.Identity(join("baz", "foo"))},
},
expect: []string{"foo", "bar", "baz"},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
result := resourceOwnersIn(test.input, rootCat)
assert.ElementsMatch(t, test.expect, result)
})
}
}
func (suite *SelectorSuite) TestContains() { func (suite *SelectorSuite) TestContains() {
t := suite.T() t := suite.T()
key := rootCatStub key := rootCatStub

View File

@ -34,7 +34,11 @@ type (
} }
) )
var _ Reducer = &SharePointRestore{} var (
_ Reducer = &SharePointRestore{}
_ printabler = &SharePointRestore{}
_ resourceOwnerer = &SharePointRestore{}
)
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint. // NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
func NewSharePointBackup() *SharePointBackup { func NewSharePointBackup() *SharePointBackup {
@ -87,6 +91,16 @@ func (s sharePoint) Printable() Printable {
return toPrintable[SharePointScope](s.Selector) return toPrintable[SharePointScope](s.Selector)
} }
// ResourceOwners produces the aggregation of discrete sitets described by each type of scope.
// Any and None values are omitted.
func (s sharePoint) ResourceOwners() selectorResourceOwners {
return selectorResourceOwners{
Excludes: resourceOwnersIn(s.Excludes, SharePointSite.String()),
Filters: resourceOwnersIn(s.Filters, SharePointSite.String()),
Includes: resourceOwnersIn(s.Includes, SharePointSite.String()),
}
}
// ------------------- // -------------------
// Scope Factories // Scope Factories