generic drive retrieval for sharepoint (#1536)
## Description Adapts the graph onedrive library to handle access to drive data across both onedrive and sharepoint services. ## Type of change - [x] 🌻 Feature ## Issue(s) * #1506 ## Test Plan - [x] 💪 Manual - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
5f82edb783
commit
10acf0ccf6
@ -29,7 +29,7 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se
|
|||||||
ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
|
ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
|
||||||
defer end()
|
defer end()
|
||||||
|
|
||||||
err := verifyBackupInputs(sels, gc.Users)
|
err := verifyBackupInputs(sels, gc.GetUsers(), gc.GetSiteIds())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -40,52 +40,59 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se
|
|||||||
case selectors.ServiceOneDrive:
|
case selectors.ServiceOneDrive:
|
||||||
return gc.OneDriveDataCollections(ctx, sels)
|
return gc.OneDriveDataCollections(ctx, sels)
|
||||||
case selectors.ServiceSharePoint:
|
case selectors.ServiceSharePoint:
|
||||||
return gc.SharePointDataCollections(ctx, sels)
|
colls, err := sharepoint.DataCollections(ctx, sels, gc.GetSiteIds(), gc.credentials.AzureTenantID, gc)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for range colls {
|
||||||
|
gc.incrementAwaitingMessages()
|
||||||
|
}
|
||||||
|
|
||||||
|
return colls, nil
|
||||||
default:
|
default:
|
||||||
return nil, errors.Errorf("service %s not supported", sels)
|
return nil, errors.Errorf("service %s not supported", sels.Service.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) error {
|
func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) error {
|
||||||
var personnel []string
|
var ids []string
|
||||||
|
|
||||||
// retrieve users from selectors
|
resourceOwners, err := sels.ResourceOwners()
|
||||||
switch sel.Service {
|
if err != nil {
|
||||||
case selectors.ServiceExchange:
|
return errors.Wrap(err, "invalid backup inputs")
|
||||||
backup, err := sel.ToExchangeBackup()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, scope := range backup.Scopes() {
|
|
||||||
temp := scope.Get(selectors.ExchangeUser)
|
|
||||||
personnel = append(personnel, temp...)
|
|
||||||
}
|
|
||||||
case selectors.ServiceOneDrive:
|
|
||||||
backup, err := sel.ToOneDriveBackup()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, user := range backup.Scopes() {
|
|
||||||
temp := user.Get(selectors.OneDriveUser)
|
|
||||||
personnel = append(personnel, temp...)
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
return errors.New("service %s not supported")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// verify personnel
|
switch sels.Service {
|
||||||
normUsers := map[string]struct{}{}
|
case selectors.ServiceExchange, selectors.ServiceOneDrive:
|
||||||
|
ids = userPNs
|
||||||
|
|
||||||
for k := range mapOfUsers {
|
case selectors.ServiceSharePoint:
|
||||||
normUsers[strings.ToLower(k)] = struct{}{}
|
ids = siteIDs
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, user := range personnel {
|
// verify resourceOwners
|
||||||
if _, ok := normUsers[strings.ToLower(user)]; !ok {
|
normROs := map[string]struct{}{}
|
||||||
return fmt.Errorf("%s user not found within tenant", user)
|
|
||||||
|
for _, id := range ids {
|
||||||
|
normROs[strings.ToLower(id)] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ro := range resourceOwners.Includes {
|
||||||
|
if _, ok := normROs[strings.ToLower(ro)]; !ok {
|
||||||
|
return fmt.Errorf("included resource owner %s not found within tenant", ro)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ro := range resourceOwners.Excludes {
|
||||||
|
if _, ok := normROs[strings.ToLower(ro)]; !ok {
|
||||||
|
return fmt.Errorf("excluded resource owner %s not found within tenant", ro)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ro := range resourceOwners.Filters {
|
||||||
|
if _, ok := normROs[strings.ToLower(ro)]; !ok {
|
||||||
|
return fmt.Errorf("filtered resource owner %s not found within tenant", ro)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,6 +200,18 @@ func (gc *GraphConnector) ExchangeDataCollection(
|
|||||||
// OneDrive
|
// OneDrive
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type odFolderMatcher struct {
|
||||||
|
scope selectors.OneDriveScope
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm odFolderMatcher) IsAny() bool {
|
||||||
|
return fm.scope.IsAny(selectors.OneDriveFolder)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm odFolderMatcher) Matches(dir string) bool {
|
||||||
|
return fm.scope.Matches(selectors.OneDriveFolder, dir)
|
||||||
|
}
|
||||||
|
|
||||||
// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
|
// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
|
||||||
// for the specified user
|
// for the specified user
|
||||||
func (gc *GraphConnector) OneDriveDataCollections(
|
func (gc *GraphConnector) OneDriveDataCollections(
|
||||||
@ -218,7 +237,8 @@ func (gc *GraphConnector) OneDriveDataCollections(
|
|||||||
odcs, err := onedrive.NewCollections(
|
odcs, err := onedrive.NewCollections(
|
||||||
gc.credentials.AzureTenantID,
|
gc.credentials.AzureTenantID,
|
||||||
user,
|
user,
|
||||||
scope,
|
onedrive.OneDriveSource,
|
||||||
|
odFolderMatcher{scope},
|
||||||
&gc.graphService,
|
&gc.graphService,
|
||||||
gc.UpdateStatus,
|
gc.UpdateStatus,
|
||||||
).Get(ctx)
|
).Get(ctx)
|
||||||
@ -236,104 +256,3 @@ func (gc *GraphConnector) OneDriveDataCollections(
|
|||||||
|
|
||||||
return collections, errs
|
return collections, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// SharePoint
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
// createSharePointCollections - utility function that retrieves M365
|
|
||||||
// IDs through Microsoft Graph API. The selectors.SharePointScope
|
|
||||||
// determines the type of collections that are retrieved.
|
|
||||||
func (gc *GraphConnector) createSharePointCollections(
|
|
||||||
ctx context.Context,
|
|
||||||
scope selectors.SharePointScope,
|
|
||||||
) ([]*sharepoint.Collection, error) {
|
|
||||||
var (
|
|
||||||
errs *multierror.Error
|
|
||||||
sites = scope.Get(selectors.SharePointSite)
|
|
||||||
colls = make([]*sharepoint.Collection, 0)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Create collection of ExchangeDataCollection
|
|
||||||
for _, site := range sites {
|
|
||||||
collections := make(map[string]*sharepoint.Collection)
|
|
||||||
|
|
||||||
qp := graph.QueryParams{
|
|
||||||
Category: scope.Category().PathType(),
|
|
||||||
ResourceOwner: site,
|
|
||||||
FailFast: gc.failFast,
|
|
||||||
Credentials: gc.credentials,
|
|
||||||
}
|
|
||||||
|
|
||||||
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, site))
|
|
||||||
defer closer()
|
|
||||||
defer close(foldersComplete)
|
|
||||||
|
|
||||||
// resolver, err := exchange.PopulateExchangeContainerResolver(
|
|
||||||
// ctx,
|
|
||||||
// qp,
|
|
||||||
// qp.Scope.Category().PathType(),
|
|
||||||
// )
|
|
||||||
// if err != nil {
|
|
||||||
// return nil, errors.Wrap(err, "getting folder cache")
|
|
||||||
// }
|
|
||||||
|
|
||||||
// err = sharepoint.FilterContainersAndFillCollections(
|
|
||||||
// ctx,
|
|
||||||
// qp,
|
|
||||||
// collections,
|
|
||||||
// gc.UpdateStatus,
|
|
||||||
// resolver)
|
|
||||||
|
|
||||||
// if err != nil {
|
|
||||||
// return nil, errors.Wrap(err, "filling collections")
|
|
||||||
// }
|
|
||||||
|
|
||||||
foldersComplete <- struct{}{}
|
|
||||||
|
|
||||||
for _, collection := range collections {
|
|
||||||
gc.incrementAwaitingMessages()
|
|
||||||
|
|
||||||
colls = append(colls, collection)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return colls, errs.ErrorOrNil()
|
|
||||||
}
|
|
||||||
|
|
||||||
// SharePointDataCollections returns a set of DataCollection which represents the SharePoint data
|
|
||||||
// for the specified user
|
|
||||||
func (gc *GraphConnector) SharePointDataCollections(
|
|
||||||
ctx context.Context,
|
|
||||||
selector selectors.Selector,
|
|
||||||
) ([]data.Collection, error) {
|
|
||||||
b, err := selector.ToSharePointBackup()
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector")
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
scopes = b.DiscreteScopes(gc.GetSites())
|
|
||||||
collections = []data.Collection{}
|
|
||||||
errs error
|
|
||||||
)
|
|
||||||
|
|
||||||
// for each scope that includes oneDrive items, get all
|
|
||||||
for _, scope := range scopes {
|
|
||||||
// Creates a map of collections based on scope
|
|
||||||
dcs, err := gc.createSharePointCollections(ctx, scope)
|
|
||||||
if err != nil {
|
|
||||||
return nil, support.WrapAndAppend(scope.Get(selectors.SharePointSite)[0], err, errs)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, collection := range dcs {
|
|
||||||
collections = append(collections, collection)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for range collections {
|
|
||||||
gc.incrementAwaitingMessages()
|
|
||||||
}
|
|
||||||
|
|
||||||
return collections, errs
|
|
||||||
}
|
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/connector/exchange"
|
"github.com/alcionai/corso/src/internal/connector/exchange"
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/sharepoint"
|
||||||
"github.com/alcionai/corso/src/internal/connector/support"
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
"github.com/alcionai/corso/src/internal/tester"
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
@ -162,7 +163,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
|
|||||||
ctx, flush := tester.NewContext()
|
ctx, flush := tester.NewContext()
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
connector := loadConnector(ctx, suite.T(), Users)
|
connector := loadConnector(ctx, suite.T(), Sites)
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
getSelector func(t *testing.T) selectors.Selector
|
getSelector func(t *testing.T) selectors.Selector
|
||||||
@ -180,24 +181,28 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
suite.T().Run(test.name, func(t *testing.T) {
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
_, err := connector.SharePointDataCollections(ctx, test.getSelector(t))
|
collection, err := sharepoint.DataCollections(
|
||||||
|
ctx,
|
||||||
|
test.getSelector(t),
|
||||||
|
[]string{suite.site},
|
||||||
|
connector.credentials.AzureTenantID,
|
||||||
|
connector)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, len(collection))
|
||||||
|
|
||||||
// TODO: Implementation
|
// the test only reads the firstt collection
|
||||||
// assert.Equal(t, len(collection), 1)
|
connector.incrementAwaitingMessages()
|
||||||
|
|
||||||
// channel := collection[0].Items()
|
for object := range collection[0].Items() {
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
_, err := buf.ReadFrom(object.ToReader())
|
||||||
|
assert.NoError(t, err, "received a buf.Read error")
|
||||||
|
}
|
||||||
|
|
||||||
// for object := range channel {
|
status := connector.AwaitStatus()
|
||||||
// buf := &bytes.Buffer{}
|
assert.NotZero(t, status.Successful)
|
||||||
// _, err := buf.ReadFrom(object.ToReader())
|
|
||||||
// assert.NoError(t, err, "received a buf.Read error")
|
|
||||||
// }
|
|
||||||
|
|
||||||
// status := connector.AwaitStatus()
|
t.Log(status.String())
|
||||||
// assert.NotZero(t, status.Successful)
|
|
||||||
|
|
||||||
// t.Log(status.String())
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -499,16 +504,17 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
t = suite.T()
|
t = suite.T()
|
||||||
userID = tester.M365UserID(t)
|
siteID = tester.M365SiteID(t)
|
||||||
|
gc = loadConnector(ctx, t, Sites)
|
||||||
|
sel = selectors.NewSharePointBackup()
|
||||||
)
|
)
|
||||||
|
|
||||||
gc := loadConnector(ctx, t, Sites)
|
sel.Include(sel.Folders(
|
||||||
scope := selectors.NewSharePointBackup().Folders(
|
[]string{siteID},
|
||||||
[]string{userID},
|
|
||||||
[]string{"foo"},
|
[]string{"foo"},
|
||||||
selectors.PrefixMatch(),
|
selectors.PrefixMatch(),
|
||||||
)[0]
|
))
|
||||||
|
|
||||||
_, err := gc.createSharePointCollections(ctx, scope)
|
_, err := gc.DataCollections(ctx, sel.Selector)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
// ---------------------------------------------------------------
|
// ---------------------------------------------------------------
|
||||||
// Disconnected Test Section
|
// Disconnected Test Section
|
||||||
// -------------------------
|
// ---------------------------------------------------------------
|
||||||
type DisconnectedGraphConnectorSuite struct {
|
type DisconnectedGraphConnectorSuite struct {
|
||||||
suite.Suite
|
suite.Suite
|
||||||
}
|
}
|
||||||
@ -206,12 +206,13 @@ func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
|
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
|
||||||
users := make(map[string]string)
|
users := []string{
|
||||||
users["elliotReid@someHospital.org"] = ""
|
"elliotReid@someHospital.org",
|
||||||
users["chrisTurk@someHospital.org"] = ""
|
"chrisTurk@someHospital.org",
|
||||||
users["carlaEspinosa@someHospital.org"] = ""
|
"carlaEspinosa@someHospital.org",
|
||||||
users["bobKelso@someHospital.org"] = ""
|
"bobKelso@someHospital.org",
|
||||||
users["johnDorian@someHospital.org"] = ""
|
"johnDorian@someHospital.org",
|
||||||
|
}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@ -219,12 +220,10 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
|
|||||||
checkError assert.ErrorAssertionFunc
|
checkError assert.ErrorAssertionFunc
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Invalid User",
|
name: "No scopes",
|
||||||
checkError: assert.Error,
|
checkError: assert.NoError,
|
||||||
getSelector: func(t *testing.T) selectors.Selector {
|
getSelector: func(t *testing.T) selectors.Selector {
|
||||||
sel := selectors.NewOneDriveBackup()
|
return selectors.NewExchangeBackup().Selector
|
||||||
sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
|
|
||||||
return sel.Selector
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -260,7 +259,108 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
suite.T().Run(test.name, func(t *testing.T) {
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
err := verifyBackupInputs(test.getSelector(t), users)
|
err := verifyBackupInputs(test.getSelector(t), users, nil)
|
||||||
|
test.checkError(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices() {
|
||||||
|
users := []string{"elliotReid@someHospital.org"}
|
||||||
|
sites := []string{"abc.site.foo", "bar.site.baz"}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
excludes func(t *testing.T) selectors.Selector
|
||||||
|
filters func(t *testing.T) selectors.Selector
|
||||||
|
includes func(t *testing.T) selectors.Selector
|
||||||
|
checkError assert.ErrorAssertionFunc
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Valid User",
|
||||||
|
checkError: assert.NoError,
|
||||||
|
excludes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Exclude(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
filters: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Filter(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
includes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Include(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Invalid User",
|
||||||
|
checkError: assert.Error,
|
||||||
|
excludes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Exclude(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
filters: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Filter(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
includes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewOneDriveBackup()
|
||||||
|
sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any()))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid sites",
|
||||||
|
checkError: assert.NoError,
|
||||||
|
excludes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Exclude(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
filters: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Filter(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
includes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Include(sel.Sites([]string{"abc.site.foo", "bar.site.baz"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid sites",
|
||||||
|
checkError: assert.Error,
|
||||||
|
excludes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Exclude(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
filters: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Filter(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
includes: func(t *testing.T) selectors.Selector {
|
||||||
|
sel := selectors.NewSharePointBackup()
|
||||||
|
sel.Include(sel.Sites([]string{"fnords.smarfs.brawnhilda"}))
|
||||||
|
return sel.Selector
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
|
err := verifyBackupInputs(test.excludes(t), users, sites)
|
||||||
|
test.checkError(t, err)
|
||||||
|
err = verifyBackupInputs(test.filters(t), users, sites)
|
||||||
|
test.checkError(t, err)
|
||||||
|
err = verifyBackupInputs(test.includes(t), users, sites)
|
||||||
test.checkError(t, err)
|
test.checkError(t, err)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -19,6 +19,7 @@ import (
|
|||||||
// 6. subject
|
// 6. subject
|
||||||
// 7. hasAttachments
|
// 7. hasAttachments
|
||||||
// 8. attachments
|
// 8. attachments
|
||||||
|
//
|
||||||
//nolint:lll
|
//nolint:lll
|
||||||
const (
|
const (
|
||||||
eventTmpl = `{
|
eventTmpl = `{
|
||||||
|
|||||||
@ -60,7 +60,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
|
|||||||
wg := sync.WaitGroup{}
|
wg := sync.WaitGroup{}
|
||||||
collStatus := support.ConnectorOperationStatus{}
|
collStatus := support.ConnectorOperationStatus{}
|
||||||
|
|
||||||
folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user")
|
folderPath, err := GetCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user", OneDriveSource)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
driveFolderPath, err := getDriveFolderPath(folderPath)
|
driveFolderPath, err := getDriveFolderPath(folderPath)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@ -117,7 +117,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
|
|||||||
wg := sync.WaitGroup{}
|
wg := sync.WaitGroup{}
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
|
|
||||||
folderPath, err := getCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user")
|
folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", OneDriveSource)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
|
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
|
||||||
|
|||||||
@ -14,79 +14,172 @@ import (
|
|||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
"github.com/alcionai/corso/src/pkg/logger"
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
"github.com/alcionai/corso/src/pkg/path"
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
"github.com/alcionai/corso/src/pkg/selectors"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Collections is used to retrieve OneDrive data for a
|
type driveSource int
|
||||||
// specified user
|
|
||||||
|
const (
|
||||||
|
unknownDriveSource driveSource = iota
|
||||||
|
OneDriveSource
|
||||||
|
SharePointSource
|
||||||
|
)
|
||||||
|
|
||||||
|
type folderMatcher interface {
|
||||||
|
IsAny() bool
|
||||||
|
Matches(string) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collections is used to retrieve drive data for a
|
||||||
|
// resource owner, which can be either a user or a sharepoint site.
|
||||||
type Collections struct {
|
type Collections struct {
|
||||||
tenant string
|
tenant string
|
||||||
user string
|
resourceOwner string
|
||||||
scope selectors.OneDriveScope
|
source driveSource
|
||||||
// collectionMap allows lookup of the data.Collection
|
matcher folderMatcher
|
||||||
// for a OneDrive folder
|
|
||||||
collectionMap map[string]data.Collection
|
|
||||||
service graph.Service
|
service graph.Service
|
||||||
statusUpdater support.StatusUpdater
|
statusUpdater support.StatusUpdater
|
||||||
|
|
||||||
|
// collectionMap allows lookup of the data.Collection
|
||||||
|
// for a OneDrive folder
|
||||||
|
CollectionMap map[string]data.Collection
|
||||||
|
|
||||||
// Track stats from drive enumeration. Represents the items backed up.
|
// Track stats from drive enumeration. Represents the items backed up.
|
||||||
numItems int
|
NumItems int
|
||||||
numFiles int
|
NumFiles int
|
||||||
numContainers int
|
NumContainers int
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewCollections(
|
func NewCollections(
|
||||||
tenant string,
|
tenant string,
|
||||||
user string,
|
resourceOwner string,
|
||||||
scope selectors.OneDriveScope,
|
source driveSource,
|
||||||
|
matcher folderMatcher,
|
||||||
service graph.Service,
|
service graph.Service,
|
||||||
statusUpdater support.StatusUpdater,
|
statusUpdater support.StatusUpdater,
|
||||||
) *Collections {
|
) *Collections {
|
||||||
return &Collections{
|
return &Collections{
|
||||||
tenant: tenant,
|
tenant: tenant,
|
||||||
user: user,
|
resourceOwner: resourceOwner,
|
||||||
scope: scope,
|
source: source,
|
||||||
collectionMap: map[string]data.Collection{},
|
matcher: matcher,
|
||||||
|
CollectionMap: map[string]data.Collection{},
|
||||||
service: service,
|
service: service,
|
||||||
statusUpdater: statusUpdater,
|
statusUpdater: statusUpdater,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Retrieves OneDrive data as set of `data.Collections`
|
// Retrieves drive data as set of `data.Collections`
|
||||||
func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
|
func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
|
||||||
// Enumerate drives for the specified user
|
// Enumerate drives for the specified resourceOwner
|
||||||
drives, err := drives(ctx, c.service, c.user)
|
drives, err := drives(ctx, c.service, c.resourceOwner, c.source)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update the collection map with items from each drive
|
// Update the collection map with items from each drive
|
||||||
for _, d := range drives {
|
for _, d := range drives {
|
||||||
err = collectItems(ctx, c.service, *d.GetId(), c.updateCollections)
|
err = collectItems(ctx, c.service, *d.GetId(), c.UpdateCollections)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
observe.Message(fmt.Sprintf("Discovered %d items to backup", c.numItems))
|
observe.Message(fmt.Sprintf("Discovered %d items to backup", c.NumItems))
|
||||||
|
|
||||||
collections := make([]data.Collection, 0, len(c.collectionMap))
|
collections := make([]data.Collection, 0, len(c.CollectionMap))
|
||||||
for _, coll := range c.collectionMap {
|
for _, coll := range c.CollectionMap {
|
||||||
collections = append(collections, coll)
|
collections = append(collections, coll)
|
||||||
}
|
}
|
||||||
|
|
||||||
return collections, nil
|
return collections, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getCanonicalPath(p, tenant, user string) (path.Path, error) {
|
// UpdateCollections initializes and adds the provided drive items to Collections
|
||||||
pathBuilder := path.Builder{}.Append(strings.Split(p, "/")...)
|
// A new collection is created for every drive folder (or package)
|
||||||
|
func (c *Collections) UpdateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {
|
||||||
|
for _, item := range items {
|
||||||
|
if item.GetRoot() != nil {
|
||||||
|
// Skip the root item
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil {
|
||||||
|
return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a collection for the parent of this item
|
||||||
|
collectionPath, err := GetCanonicalPath(
|
||||||
|
*item.GetParentReference().GetPath(),
|
||||||
|
c.tenant,
|
||||||
|
c.resourceOwner,
|
||||||
|
c.source,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip items that don't match the folder selectors we were given.
|
||||||
|
if !includePath(ctx, c.matcher, collectionPath) {
|
||||||
|
logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case item.GetFolder() != nil, item.GetPackage() != nil:
|
||||||
|
// Leave this here so we don't fall into the default case.
|
||||||
|
// TODO: This is where we might create a "special file" to represent these in the backup repository
|
||||||
|
// e.g. a ".folderMetadataFile"
|
||||||
|
|
||||||
|
case item.GetFile() != nil:
|
||||||
|
col, found := c.CollectionMap[collectionPath.String()]
|
||||||
|
if !found {
|
||||||
|
col = NewCollection(
|
||||||
|
collectionPath,
|
||||||
|
driveID,
|
||||||
|
c.service,
|
||||||
|
c.statusUpdater,
|
||||||
|
)
|
||||||
|
|
||||||
|
c.CollectionMap[collectionPath.String()] = col
|
||||||
|
c.NumContainers++
|
||||||
|
c.NumItems++
|
||||||
|
}
|
||||||
|
|
||||||
|
collection := col.(*Collection)
|
||||||
|
collection.Add(*item.GetId())
|
||||||
|
c.NumFiles++
|
||||||
|
c.NumItems++
|
||||||
|
|
||||||
|
default:
|
||||||
|
return errors.Errorf("item type not supported. item name : %s", *item.GetName())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCanonicalPath constructs the standard path for the given source.
|
||||||
|
func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) {
|
||||||
|
var (
|
||||||
|
pathBuilder = path.Builder{}.Append(strings.Split(p, "/")...)
|
||||||
|
result path.Path
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
switch source {
|
||||||
|
case OneDriveSource:
|
||||||
|
result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false)
|
||||||
|
case SharePointSource:
|
||||||
|
result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false)
|
||||||
|
default:
|
||||||
|
return nil, errors.Errorf("unrecognized drive data source")
|
||||||
|
}
|
||||||
|
|
||||||
res, err := pathBuilder.ToDataLayerOneDrivePath(tenant, user, false)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "converting to canonical path")
|
return nil, errors.Wrap(err, "converting to canonical path")
|
||||||
}
|
}
|
||||||
|
|
||||||
return res, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the path to the folder within the drive (i.e. under `root:`)
|
// Returns the path to the folder within the drive (i.e. under `root:`)
|
||||||
@ -99,70 +192,7 @@ func getDriveFolderPath(p path.Path) (string, error) {
|
|||||||
return path.Builder{}.Append(drivePath.folders...).String(), nil
|
return path.Builder{}.Append(drivePath.folders...).String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateCollections initializes and adds the provided OneDrive items to Collections
|
func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) bool {
|
||||||
// A new collection is created for every OneDrive folder (or package)
|
|
||||||
func (c *Collections) updateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {
|
|
||||||
for _, item := range items {
|
|
||||||
if item.GetRoot() != nil {
|
|
||||||
// Skip the root item
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil {
|
|
||||||
return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a collection for the parent of this item
|
|
||||||
collectionPath, err := getCanonicalPath(
|
|
||||||
*item.GetParentReference().GetPath(),
|
|
||||||
c.tenant,
|
|
||||||
c.user,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip items that don't match the folder selectors we were given.
|
|
||||||
if !includePath(ctx, c.scope, collectionPath) {
|
|
||||||
logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case item.GetFolder() != nil, item.GetPackage() != nil:
|
|
||||||
// Leave this here so we don't fall into the default case.
|
|
||||||
// TODO: This is where we might create a "special file" to represent these in the backup repository
|
|
||||||
// e.g. a ".folderMetadataFile"
|
|
||||||
|
|
||||||
case item.GetFile() != nil:
|
|
||||||
col, found := c.collectionMap[collectionPath.String()]
|
|
||||||
if !found {
|
|
||||||
col = NewCollection(
|
|
||||||
collectionPath,
|
|
||||||
driveID,
|
|
||||||
c.service,
|
|
||||||
c.statusUpdater,
|
|
||||||
)
|
|
||||||
|
|
||||||
c.collectionMap[collectionPath.String()] = col
|
|
||||||
c.numContainers++
|
|
||||||
c.numItems++
|
|
||||||
}
|
|
||||||
|
|
||||||
collection := col.(*Collection)
|
|
||||||
collection.Add(*item.GetId())
|
|
||||||
c.numFiles++
|
|
||||||
c.numItems++
|
|
||||||
|
|
||||||
default:
|
|
||||||
return errors.Errorf("item type not supported. item name : %s", *item.GetName())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath path.Path) bool {
|
|
||||||
// Check if the folder is allowed by the scope.
|
// Check if the folder is allowed by the scope.
|
||||||
folderPathString, err := getDriveFolderPath(folderPath)
|
folderPathString, err := getDriveFolderPath(folderPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -172,9 +202,9 @@ func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath
|
|||||||
|
|
||||||
// Hack for the edge case where we're looking at the root folder and can
|
// Hack for the edge case where we're looking at the root folder and can
|
||||||
// select any folder. Right now the root folder has an empty folder path.
|
// select any folder. Right now the root folder has an empty folder path.
|
||||||
if len(folderPathString) == 0 && scope.IsAny(selectors.OneDriveFolder) {
|
if len(folderPathString) == 0 && m.IsAny() {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
return scope.Matches(selectors.OneDriveFolder, folderPathString)
|
return m.Matches(folderPathString)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package onedrive
|
package onedrive
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
@ -20,7 +21,7 @@ func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []st
|
|||||||
res := make([]string, 0, len(rest))
|
res := make([]string, 0, len(rest))
|
||||||
|
|
||||||
for _, r := range rest {
|
for _, r := range rest {
|
||||||
p, err := getCanonicalPath(r, tenant, user)
|
p, err := GetCanonicalPath(r, tenant, user, OneDriveSource)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
res = append(res, p.String())
|
res = append(res, p.String())
|
||||||
@ -37,6 +38,49 @@ func TestOneDriveCollectionsSuite(t *testing.T) {
|
|||||||
suite.Run(t, new(OneDriveCollectionsSuite))
|
suite.Run(t, new(OneDriveCollectionsSuite))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() {
|
||||||
|
tenant, resourceOwner := "tenant", "resourceOwner"
|
||||||
|
|
||||||
|
table := []struct {
|
||||||
|
name string
|
||||||
|
source driveSource
|
||||||
|
dir []string
|
||||||
|
expect string
|
||||||
|
expectErr assert.ErrorAssertionFunc
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "onedrive",
|
||||||
|
source: OneDriveSource,
|
||||||
|
dir: []string{"onedrive"},
|
||||||
|
expect: "tenant/onedrive/resourceOwner/files/onedrive",
|
||||||
|
expectErr: assert.NoError,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "sharepoint",
|
||||||
|
source: SharePointSource,
|
||||||
|
dir: []string{"sharepoint"},
|
||||||
|
expect: "tenant/sharepoint/resourceOwner/files/sharepoint",
|
||||||
|
expectErr: assert.NoError,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown",
|
||||||
|
source: unknownDriveSource,
|
||||||
|
dir: []string{"unknown"},
|
||||||
|
expectErr: assert.Error,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range table {
|
||||||
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
|
p := strings.Join(test.dir, "/")
|
||||||
|
result, err := GetCanonicalPath(p, tenant, resourceOwner, test.source)
|
||||||
|
test.expectErr(t, err)
|
||||||
|
if result != nil {
|
||||||
|
assert.Equal(t, test.expect, result.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
|
func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
|
||||||
anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0]
|
anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0]
|
||||||
|
|
||||||
@ -211,15 +255,15 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
|
|||||||
ctx, flush := tester.NewContext()
|
ctx, flush := tester.NewContext()
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil)
|
c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil)
|
||||||
err := c.updateCollections(ctx, "driveID", tt.items)
|
err := c.UpdateCollections(ctx, "driveID", tt.items)
|
||||||
tt.expect(t, err)
|
tt.expect(t, err)
|
||||||
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths")
|
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
|
||||||
assert.Equal(t, tt.expectedItemCount, c.numItems, "item count")
|
assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
|
||||||
assert.Equal(t, tt.expectedFileCount, c.numFiles, "file count")
|
assert.Equal(t, tt.expectedFileCount, c.NumFiles, "file count")
|
||||||
assert.Equal(t, tt.expectedContainerCount, c.numContainers, "container count")
|
assert.Equal(t, tt.expectedContainerCount, c.NumContainers, "container count")
|
||||||
for _, collPath := range tt.expectedCollectionPaths {
|
for _, collPath := range tt.expectedCollectionPaths {
|
||||||
assert.Contains(t, c.collectionMap, collPath)
|
assert.Contains(t, c.CollectionMap, collPath)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -67,7 +67,33 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Enumerates the drives for the specified user
|
// Enumerates the drives for the specified user
|
||||||
func drives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) {
|
func drives(
|
||||||
|
ctx context.Context,
|
||||||
|
service graph.Service,
|
||||||
|
resourceOwner string,
|
||||||
|
source driveSource,
|
||||||
|
) ([]models.Driveable, error) {
|
||||||
|
switch source {
|
||||||
|
case OneDriveSource:
|
||||||
|
return userDrives(ctx, service, resourceOwner)
|
||||||
|
case SharePointSource:
|
||||||
|
return siteDrives(ctx, service, resourceOwner)
|
||||||
|
default:
|
||||||
|
return nil, errors.Errorf("unrecognized drive data source")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func siteDrives(ctx context.Context, service graph.Service, site string) ([]models.Driveable, error) {
|
||||||
|
r, err := service.Client().SitesById(site).Drives().Get(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "failed to retrieve site drives. site: %s, details: %s",
|
||||||
|
site, support.ConnectorStackErrorTrace(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.GetValue(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func userDrives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) {
|
||||||
var hasDrive bool
|
var hasDrive bool
|
||||||
|
|
||||||
hasDrive, err := hasDriveLicense(ctx, service, user)
|
hasDrive, err := hasDriveLicense(ctx, service, user)
|
||||||
@ -237,7 +263,7 @@ func GetAllFolders(
|
|||||||
userID string,
|
userID string,
|
||||||
prefix string,
|
prefix string,
|
||||||
) ([]*Displayable, error) {
|
) ([]*Displayable, error) {
|
||||||
drives, err := drives(ctx, gs, userID)
|
drives, err := drives(ctx, gs, userID, OneDriveSource)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "getting OneDrive folders")
|
return nil, errors.Wrap(err, "getting OneDrive folders")
|
||||||
}
|
}
|
||||||
@ -321,7 +347,7 @@ func hasDriveLicense(
|
|||||||
cb := func(pageItem any) bool {
|
cb := func(pageItem any) bool {
|
||||||
entry, ok := pageItem.(models.LicenseDetailsable)
|
entry, ok := pageItem.(models.LicenseDetailsable)
|
||||||
if !ok {
|
if !ok {
|
||||||
err = errors.New("casting item to models.MailFolderable")
|
err = errors.New("casting item to models.LicenseDetailsable")
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -43,7 +43,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
|
|||||||
folderElements := []string{folderName1}
|
folderElements := []string{folderName1}
|
||||||
gs := loadTestService(t)
|
gs := loadTestService(t)
|
||||||
|
|
||||||
drives, err := drives(ctx, gs, suite.userID)
|
drives, err := drives(ctx, gs, suite.userID, OneDriveSource)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotEmpty(t, drives)
|
require.NotEmpty(t, drives)
|
||||||
|
|
||||||
@ -100,6 +100,18 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type testFolderMatcher struct {
|
||||||
|
scope selectors.OneDriveScope
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm testFolderMatcher) IsAny() bool {
|
||||||
|
return fm.scope.IsAny(selectors.OneDriveFolder)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm testFolderMatcher) Matches(path string) bool {
|
||||||
|
return fm.scope.Matches(selectors.OneDriveFolder, path)
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *OneDriveSuite) TestOneDriveNewCollections() {
|
func (suite *OneDriveSuite) TestOneDriveNewCollections() {
|
||||||
ctx, flush := tester.NewContext()
|
ctx, flush := tester.NewContext()
|
||||||
defer flush()
|
defer flush()
|
||||||
@ -129,7 +141,8 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() {
|
|||||||
odcs, err := NewCollections(
|
odcs, err := NewCollections(
|
||||||
creds.AzureTenantID,
|
creds.AzureTenantID,
|
||||||
test.user,
|
test.user,
|
||||||
scope,
|
OneDriveSource,
|
||||||
|
testFolderMatcher{scope},
|
||||||
service,
|
service,
|
||||||
service.updateStatus,
|
service.updateStatus,
|
||||||
).Get(ctx)
|
).Get(ctx)
|
||||||
|
|||||||
@ -67,7 +67,7 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
|
|||||||
|
|
||||||
suite.user = tester.SecondaryM365UserID(suite.T())
|
suite.user = tester.SecondaryM365UserID(suite.T())
|
||||||
|
|
||||||
drives, err := drives(ctx, suite, suite.user)
|
drives, err := drives(ctx, suite, suite.user, OneDriveSource)
|
||||||
require.NoError(suite.T(), err)
|
require.NoError(suite.T(), err)
|
||||||
// Test Requirement 1: Need a drive
|
// Test Requirement 1: Need a drive
|
||||||
require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user)
|
require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user)
|
||||||
|
|||||||
124
src/internal/connector/sharepoint/data_collections.go
Normal file
124
src/internal/connector/sharepoint/data_collections.go
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
package sharepoint
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/support"
|
||||||
|
"github.com/alcionai/corso/src/internal/data"
|
||||||
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
|
"github.com/alcionai/corso/src/pkg/path"
|
||||||
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
|
)
|
||||||
|
|
||||||
|
type statusUpdater interface {
|
||||||
|
UpdateStatus(status *support.ConnectorOperationStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
type connector interface {
|
||||||
|
statusUpdater
|
||||||
|
|
||||||
|
Service() graph.Service
|
||||||
|
}
|
||||||
|
|
||||||
|
// DataCollections returns a set of DataCollection which represents the SharePoint data
|
||||||
|
// for the specified user
|
||||||
|
func DataCollections(
|
||||||
|
ctx context.Context,
|
||||||
|
selector selectors.Selector,
|
||||||
|
siteIDs []string,
|
||||||
|
tenantID string,
|
||||||
|
con connector,
|
||||||
|
) ([]data.Collection, error) {
|
||||||
|
b, err := selector.ToSharePointBackup()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector")
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
scopes = b.DiscreteScopes(siteIDs)
|
||||||
|
collections = []data.Collection{}
|
||||||
|
serv = con.Service()
|
||||||
|
errs error
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, scope := range scopes {
|
||||||
|
// due to DiscreteScopes(siteIDs), each range should only contain one site.
|
||||||
|
for _, site := range scope.Get(selectors.SharePointSite) {
|
||||||
|
foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf(
|
||||||
|
"∙ %s - %s:",
|
||||||
|
scope.Category().PathType(), site))
|
||||||
|
defer closer()
|
||||||
|
defer close(foldersComplete)
|
||||||
|
|
||||||
|
switch scope.Category().PathType() {
|
||||||
|
case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory
|
||||||
|
spcs, err := collectLibraries(
|
||||||
|
ctx,
|
||||||
|
serv,
|
||||||
|
tenantID,
|
||||||
|
site,
|
||||||
|
scope,
|
||||||
|
con)
|
||||||
|
if err != nil {
|
||||||
|
return nil, support.WrapAndAppend(site, err, errs)
|
||||||
|
}
|
||||||
|
|
||||||
|
collections = append(collections, spcs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
foldersComplete <- struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return collections, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectLibraries constructs a onedrive Collections struct and Get()s
|
||||||
|
// all the drives associated with the site.
|
||||||
|
func collectLibraries(
|
||||||
|
ctx context.Context,
|
||||||
|
serv graph.Service,
|
||||||
|
tenantID, siteID string,
|
||||||
|
scope selectors.SharePointScope,
|
||||||
|
updater statusUpdater,
|
||||||
|
) ([]data.Collection, error) {
|
||||||
|
var (
|
||||||
|
collections = []data.Collection{}
|
||||||
|
errs error
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Library collections")
|
||||||
|
|
||||||
|
colls := onedrive.NewCollections(
|
||||||
|
tenantID,
|
||||||
|
siteID,
|
||||||
|
onedrive.SharePointSource,
|
||||||
|
folderMatcher{scope},
|
||||||
|
serv,
|
||||||
|
updater.UpdateStatus)
|
||||||
|
|
||||||
|
odcs, err := colls.Get(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, support.WrapAndAppend(siteID, err, errs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return append(collections, odcs...), errs
|
||||||
|
}
|
||||||
|
|
||||||
|
type folderMatcher struct {
|
||||||
|
scope selectors.SharePointScope
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm folderMatcher) IsAny() bool {
|
||||||
|
return fm.scope.IsAny(selectors.SharePointFolder)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm folderMatcher) Matches(dir string) bool {
|
||||||
|
return fm.scope.Matches(selectors.SharePointFolder, dir)
|
||||||
|
}
|
||||||
158
src/internal/connector/sharepoint/data_collections_test.go
Normal file
158
src/internal/connector/sharepoint/data_collections_test.go
Normal file
@ -0,0 +1,158 @@
|
|||||||
|
package sharepoint_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
|
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/connector/onedrive"
|
||||||
|
"github.com/alcionai/corso/src/internal/tester"
|
||||||
|
"github.com/alcionai/corso/src/pkg/selectors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// consts, mocks
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const (
|
||||||
|
testBaseDrivePath = "drive/driveID1/root:"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testFolderMatcher struct {
|
||||||
|
scope selectors.SharePointScope
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm testFolderMatcher) IsAny() bool {
|
||||||
|
return fm.scope.IsAny(selectors.SharePointFolder)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fm testFolderMatcher) Matches(path string) bool {
|
||||||
|
return fm.scope.Matches(selectors.SharePointFolder, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
type MockGraphService struct{}
|
||||||
|
|
||||||
|
func (ms *MockGraphService) Client() *msgraphsdk.GraphServiceClient {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ms *MockGraphService) ErrPolicy() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// tests
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type SharePointLibrariesSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSharePointLibrariesSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(SharePointLibrariesSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
|
||||||
|
anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0]
|
||||||
|
|
||||||
|
const (
|
||||||
|
tenant = "tenant"
|
||||||
|
site = "site"
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
testCase string
|
||||||
|
items []models.DriveItemable
|
||||||
|
scope selectors.SharePointScope
|
||||||
|
expect assert.ErrorAssertionFunc
|
||||||
|
expectedCollectionPaths []string
|
||||||
|
expectedItemCount int
|
||||||
|
expectedContainerCount int
|
||||||
|
expectedFileCount int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
testCase: "Single File",
|
||||||
|
items: []models.DriveItemable{
|
||||||
|
driveItem("file", testBaseDrivePath, true),
|
||||||
|
},
|
||||||
|
scope: anyFolder,
|
||||||
|
expect: assert.NoError,
|
||||||
|
expectedCollectionPaths: expectedPathAsSlice(
|
||||||
|
suite.T(),
|
||||||
|
tenant,
|
||||||
|
site,
|
||||||
|
testBaseDrivePath,
|
||||||
|
),
|
||||||
|
expectedItemCount: 2,
|
||||||
|
expectedFileCount: 1,
|
||||||
|
expectedContainerCount: 1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
suite.T().Run(test.testCase, func(t *testing.T) {
|
||||||
|
ctx, flush := tester.NewContext()
|
||||||
|
defer flush()
|
||||||
|
|
||||||
|
c := onedrive.NewCollections(
|
||||||
|
tenant,
|
||||||
|
site,
|
||||||
|
onedrive.SharePointSource,
|
||||||
|
testFolderMatcher{test.scope},
|
||||||
|
&MockGraphService{},
|
||||||
|
nil)
|
||||||
|
err := c.UpdateCollections(ctx, "driveID", test.items)
|
||||||
|
test.expect(t, err)
|
||||||
|
assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
|
||||||
|
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
|
||||||
|
assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count")
|
||||||
|
assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count")
|
||||||
|
for _, collPath := range test.expectedCollectionPaths {
|
||||||
|
assert.Contains(t, c.CollectionMap, collPath)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func driveItem(name string, path string, isFile bool) models.DriveItemable {
|
||||||
|
item := models.NewDriveItem()
|
||||||
|
item.SetName(&name)
|
||||||
|
item.SetId(&name)
|
||||||
|
|
||||||
|
parentReference := models.NewItemReference()
|
||||||
|
parentReference.SetPath(&path)
|
||||||
|
item.SetParentReference(parentReference)
|
||||||
|
|
||||||
|
if isFile {
|
||||||
|
item.SetFile(models.NewFile())
|
||||||
|
}
|
||||||
|
|
||||||
|
return item
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string {
|
||||||
|
res := make([]string, 0, len(rest))
|
||||||
|
|
||||||
|
for _, r := range rest {
|
||||||
|
p, err := onedrive.GetCanonicalPath(r, tenant, user, onedrive.SharePointSource)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
res = append(res, p.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
}
|
||||||
@ -23,6 +23,7 @@ const (
|
|||||||
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
|
CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS"
|
||||||
CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
|
CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
|
||||||
CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"
|
CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"
|
||||||
|
CorsoGraphConnectorSharePointTests = "CORSO_GRAPH_CONNECTOR_SHAREPOINT_TESTS"
|
||||||
CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS"
|
CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS"
|
||||||
CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS"
|
CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS"
|
||||||
CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS"
|
CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS"
|
||||||
|
|||||||
@ -241,7 +241,7 @@ func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(resourceOwner) == 0 {
|
if len(resourceOwner) == 0 {
|
||||||
return errors.Wrap(errMissingSegment, "user")
|
return errors.Wrap(errMissingSegment, "resourceOwner")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(pb.elements) == 0 {
|
if len(pb.elements) == 0 {
|
||||||
|
|||||||
@ -36,7 +36,11 @@ type (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ Reducer = &ExchangeRestore{}
|
var (
|
||||||
|
_ Reducer = &ExchangeRestore{}
|
||||||
|
_ printabler = &ExchangeRestore{}
|
||||||
|
_ resourceOwnerer = &ExchangeRestore{}
|
||||||
|
)
|
||||||
|
|
||||||
// NewExchange produces a new Selector with the service set to ServiceExchange.
|
// NewExchange produces a new Selector with the service set to ServiceExchange.
|
||||||
func NewExchangeBackup() *ExchangeBackup {
|
func NewExchangeBackup() *ExchangeBackup {
|
||||||
@ -89,6 +93,16 @@ func (s exchange) Printable() Printable {
|
|||||||
return toPrintable[ExchangeScope](s.Selector)
|
return toPrintable[ExchangeScope](s.Selector)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResourceOwners produces the aggregation of discrete users described by each type of scope.
|
||||||
|
// Any and None values are omitted.
|
||||||
|
func (s exchange) ResourceOwners() selectorResourceOwners {
|
||||||
|
return selectorResourceOwners{
|
||||||
|
Excludes: resourceOwnersIn(s.Excludes, ExchangeUser.String()),
|
||||||
|
Filters: resourceOwnersIn(s.Filters, ExchangeUser.String()),
|
||||||
|
Includes: resourceOwnersIn(s.Includes, ExchangeUser.String()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -------------------
|
// -------------------
|
||||||
// Exclude/Includes
|
// Exclude/Includes
|
||||||
|
|
||||||
|
|||||||
@ -35,7 +35,11 @@ type (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ Reducer = &OneDriveRestore{}
|
var (
|
||||||
|
_ Reducer = &OneDriveRestore{}
|
||||||
|
_ printabler = &OneDriveRestore{}
|
||||||
|
_ resourceOwnerer = &OneDriveRestore{}
|
||||||
|
)
|
||||||
|
|
||||||
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
|
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
|
||||||
func NewOneDriveBackup() *OneDriveBackup {
|
func NewOneDriveBackup() *OneDriveBackup {
|
||||||
@ -88,6 +92,16 @@ func (s oneDrive) Printable() Printable {
|
|||||||
return toPrintable[OneDriveScope](s.Selector)
|
return toPrintable[OneDriveScope](s.Selector)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResourceOwners produces the aggregation of discrete users described by each type of scope.
|
||||||
|
// Any and None values are omitted.
|
||||||
|
func (s oneDrive) ResourceOwners() selectorResourceOwners {
|
||||||
|
return selectorResourceOwners{
|
||||||
|
Excludes: resourceOwnersIn(s.Excludes, OneDriveUser.String()),
|
||||||
|
Filters: resourceOwnersIn(s.Filters, OneDriveUser.String()),
|
||||||
|
Includes: resourceOwnersIn(s.Includes, OneDriveUser.String()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -------------------
|
// -------------------
|
||||||
// Scope Factories
|
// Scope Factories
|
||||||
|
|
||||||
|
|||||||
@ -54,13 +54,13 @@ const (
|
|||||||
// None() to any selector will force all matches() checks on that
|
// None() to any selector will force all matches() checks on that
|
||||||
// selector to fail.
|
// selector to fail.
|
||||||
// Ex: {user: u1, events: NoneTgt} => matches nothing.
|
// Ex: {user: u1, events: NoneTgt} => matches nothing.
|
||||||
NoneTgt = ""
|
NoneTgt = ""
|
||||||
delimiter = ","
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
passAny = filters.Pass()
|
delimiter = fmt.Sprint(0x1F)
|
||||||
failAny = filters.Fail()
|
passAny = filters.Pass()
|
||||||
|
failAny = filters.Fail()
|
||||||
)
|
)
|
||||||
|
|
||||||
// All is the resource name that gets output when the resource is AnyTgt.
|
// All is the resource name that gets output when the resource is AnyTgt.
|
||||||
@ -71,6 +71,19 @@ type Reducer interface {
|
|||||||
Reduce(context.Context, *details.Details) *details.Details
|
Reduce(context.Context, *details.Details) *details.Details
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// selectorResourceOwners aggregates all discrete resource owner ids described
|
||||||
|
// in the selector. Any and None values are ignored. ResourceOwner sets are
|
||||||
|
// grouped by their scope type (includes, excludes, filters).
|
||||||
|
type selectorResourceOwners struct {
|
||||||
|
Includes []string
|
||||||
|
Excludes []string
|
||||||
|
Filters []string
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceOwnerer interface {
|
||||||
|
ResourceOwners() selectorResourceOwners
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Selector
|
// Selector
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -184,22 +197,7 @@ func (s Selector) PathService() path.ServiceType {
|
|||||||
// than have the caller make that interpretation. Returns an error if the
|
// than have the caller make that interpretation. Returns an error if the
|
||||||
// service is unsupported.
|
// service is unsupported.
|
||||||
func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.Details, error) {
|
func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.Details, error) {
|
||||||
var (
|
r, err := selectorAsIface[Reducer](s)
|
||||||
r Reducer
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
|
|
||||||
switch s.Service {
|
|
||||||
case ServiceExchange:
|
|
||||||
r, err = s.ToExchangeRestore()
|
|
||||||
case ServiceOneDrive:
|
|
||||||
r, err = s.ToOneDriveRestore()
|
|
||||||
case ServiceSharePoint:
|
|
||||||
r, err = s.ToSharePointRestore()
|
|
||||||
default:
|
|
||||||
return nil, errors.New("service not supported: " + s.Service.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -207,6 +205,40 @@ func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.
|
|||||||
return r.Reduce(ctx, deets), nil
|
return r.Reduce(ctx, deets), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s Selector) ResourceOwners() (selectorResourceOwners, error) {
|
||||||
|
ro, err := selectorAsIface[resourceOwnerer](s)
|
||||||
|
if err != nil {
|
||||||
|
return selectorResourceOwners{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ro.ResourceOwners(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// transformer for arbitrary selector interfaces
|
||||||
|
func selectorAsIface[T any](s Selector) (T, error) {
|
||||||
|
var (
|
||||||
|
a any
|
||||||
|
t T
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
switch s.Service {
|
||||||
|
case ServiceExchange:
|
||||||
|
a, err = func() (any, error) { return s.ToExchangeRestore() }()
|
||||||
|
t = a.(T)
|
||||||
|
case ServiceOneDrive:
|
||||||
|
a, err = func() (any, error) { return s.ToOneDriveRestore() }()
|
||||||
|
t = a.(T)
|
||||||
|
case ServiceSharePoint:
|
||||||
|
a, err = func() (any, error) { return s.ToSharePointRestore() }()
|
||||||
|
t = a.(T)
|
||||||
|
default:
|
||||||
|
err = errors.New("service not supported: " + s.Service.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return t, err
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Printing Selectors for Human Reading
|
// Printing Selectors for Human Reading
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@ -218,35 +250,18 @@ type Printable struct {
|
|||||||
Includes map[string][]string `json:"includes,omitempty"`
|
Includes map[string][]string `json:"includes,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type printabler interface {
|
||||||
|
Printable() Printable
|
||||||
|
}
|
||||||
|
|
||||||
// ToPrintable creates the minimized display of a selector, formatted for human readability.
|
// ToPrintable creates the minimized display of a selector, formatted for human readability.
|
||||||
func (s Selector) ToPrintable() Printable {
|
func (s Selector) ToPrintable() Printable {
|
||||||
switch s.Service {
|
p, err := selectorAsIface[printabler](s)
|
||||||
case ServiceExchange:
|
if err != nil {
|
||||||
r, err := s.ToExchangeRestore()
|
return Printable{}
|
||||||
if err != nil {
|
|
||||||
return Printable{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.Printable()
|
|
||||||
|
|
||||||
case ServiceOneDrive:
|
|
||||||
r, err := s.ToOneDriveBackup()
|
|
||||||
if err != nil {
|
|
||||||
return Printable{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.Printable()
|
|
||||||
|
|
||||||
case ServiceSharePoint:
|
|
||||||
r, err := s.ToSharePointBackup()
|
|
||||||
if err != nil {
|
|
||||||
return Printable{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return r.Printable()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Printable{}
|
return p.Printable()
|
||||||
}
|
}
|
||||||
|
|
||||||
// toPrintable creates the minimized display of a selector, formatted for human readability.
|
// toPrintable creates the minimized display of a selector, formatted for human readability.
|
||||||
@ -351,6 +366,32 @@ func addToSet(set []string, v []string) []string {
|
|||||||
// helpers
|
// helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// produces the discrete set of resource owners in the slice of scopes.
|
||||||
|
// Any and None values are discarded.
|
||||||
|
func resourceOwnersIn(s []scope, rootCat string) []string {
|
||||||
|
rm := map[string]struct{}{}
|
||||||
|
|
||||||
|
for _, sc := range s {
|
||||||
|
for _, v := range split(sc[rootCat].Target) {
|
||||||
|
rm[v] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rs := []string{}
|
||||||
|
|
||||||
|
for k := range rm {
|
||||||
|
if k != AnyTgt && k != NoneTgt {
|
||||||
|
rs = append(rs, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rs
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// scope helpers
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
type scopeConfig struct {
|
type scopeConfig struct {
|
||||||
usePathFilter bool
|
usePathFilter bool
|
||||||
usePrefixFilter bool
|
usePrefixFilter bool
|
||||||
@ -382,10 +423,6 @@ func pathType() option {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// helpers
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
func badCastErr(cast, is service) error {
|
func badCastErr(cast, is service) error {
|
||||||
return errors.Wrapf(ErrorBadSelectorCast, "%s service is not %s", cast, is)
|
return errors.Wrapf(ErrorBadSelectorCast, "%s service is not %s", cast, is)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/pkg/filters"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SelectorSuite struct {
|
type SelectorSuite struct {
|
||||||
@ -143,6 +145,69 @@ func (suite *SelectorSuite) TestToResourceTypeMap() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *SelectorSuite) TestResourceOwnersIn() {
|
||||||
|
rootCat := rootCatStub.String()
|
||||||
|
|
||||||
|
table := []struct {
|
||||||
|
name string
|
||||||
|
input []scope
|
||||||
|
expect []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "nil",
|
||||||
|
input: nil,
|
||||||
|
expect: []string{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty",
|
||||||
|
input: []scope{},
|
||||||
|
expect: []string{},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single",
|
||||||
|
input: []scope{{rootCat: filters.Identity("foo")}},
|
||||||
|
expect: []string{"foo"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple values",
|
||||||
|
input: []scope{{rootCat: filters.Identity(join("foo", "bar"))}},
|
||||||
|
expect: []string{"foo", "bar"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with any",
|
||||||
|
input: []scope{{rootCat: filters.Identity(join("foo", "bar", AnyTgt))}},
|
||||||
|
expect: []string{"foo", "bar"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "with none",
|
||||||
|
input: []scope{{rootCat: filters.Identity(join("foo", "bar", NoneTgt))}},
|
||||||
|
expect: []string{"foo", "bar"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple scopes",
|
||||||
|
input: []scope{
|
||||||
|
{rootCat: filters.Identity(join("foo", "bar"))},
|
||||||
|
{rootCat: filters.Identity(join("baz"))},
|
||||||
|
},
|
||||||
|
expect: []string{"foo", "bar", "baz"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiple scopes with duplicates",
|
||||||
|
input: []scope{
|
||||||
|
{rootCat: filters.Identity(join("foo", "bar"))},
|
||||||
|
{rootCat: filters.Identity(join("baz", "foo"))},
|
||||||
|
},
|
||||||
|
expect: []string{"foo", "bar", "baz"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, test := range table {
|
||||||
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
|
result := resourceOwnersIn(test.input, rootCat)
|
||||||
|
assert.ElementsMatch(t, test.expect, result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *SelectorSuite) TestContains() {
|
func (suite *SelectorSuite) TestContains() {
|
||||||
t := suite.T()
|
t := suite.T()
|
||||||
key := rootCatStub
|
key := rootCatStub
|
||||||
|
|||||||
@ -34,7 +34,11 @@ type (
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ Reducer = &SharePointRestore{}
|
var (
|
||||||
|
_ Reducer = &SharePointRestore{}
|
||||||
|
_ printabler = &SharePointRestore{}
|
||||||
|
_ resourceOwnerer = &SharePointRestore{}
|
||||||
|
)
|
||||||
|
|
||||||
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
|
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
|
||||||
func NewSharePointBackup() *SharePointBackup {
|
func NewSharePointBackup() *SharePointBackup {
|
||||||
@ -87,6 +91,16 @@ func (s sharePoint) Printable() Printable {
|
|||||||
return toPrintable[SharePointScope](s.Selector)
|
return toPrintable[SharePointScope](s.Selector)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResourceOwners produces the aggregation of discrete sitets described by each type of scope.
|
||||||
|
// Any and None values are omitted.
|
||||||
|
func (s sharePoint) ResourceOwners() selectorResourceOwners {
|
||||||
|
return selectorResourceOwners{
|
||||||
|
Excludes: resourceOwnersIn(s.Excludes, SharePointSite.String()),
|
||||||
|
Filters: resourceOwnersIn(s.Filters, SharePointSite.String()),
|
||||||
|
Includes: resourceOwnersIn(s.Includes, SharePointSite.String()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// -------------------
|
// -------------------
|
||||||
// Scope Factories
|
// Scope Factories
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user