diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 833c1f977..36d05b731 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -29,7 +29,7 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String())) defer end() - err := verifyBackupInputs(sels, gc.Users) + err := verifyBackupInputs(sels, gc.GetUsers(), gc.GetSiteIds()) if err != nil { return nil, err } @@ -40,52 +40,59 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se case selectors.ServiceOneDrive: return gc.OneDriveDataCollections(ctx, sels) case selectors.ServiceSharePoint: - return gc.SharePointDataCollections(ctx, sels) + colls, err := sharepoint.DataCollections(ctx, sels, gc.GetSiteIds(), gc.credentials.AzureTenantID, gc) + if err != nil { + return nil, err + } + + for range colls { + gc.incrementAwaitingMessages() + } + + return colls, nil default: - return nil, errors.Errorf("service %s not supported", sels) + return nil, errors.Errorf("service %s not supported", sels.Service.String()) } } -func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) error { - var personnel []string +func verifyBackupInputs(sels selectors.Selector, userPNs, siteIDs []string) error { + var ids []string - // retrieve users from selectors - switch sel.Service { - case selectors.ServiceExchange: - backup, err := sel.ToExchangeBackup() - if err != nil { - return err - } - - for _, scope := range backup.Scopes() { - temp := scope.Get(selectors.ExchangeUser) - personnel = append(personnel, temp...) - } - case selectors.ServiceOneDrive: - backup, err := sel.ToOneDriveBackup() - if err != nil { - return err - } - - for _, user := range backup.Scopes() { - temp := user.Get(selectors.OneDriveUser) - personnel = append(personnel, temp...) - } - - default: - return errors.New("service %s not supported") + resourceOwners, err := sels.ResourceOwners() + if err != nil { + return errors.Wrap(err, "invalid backup inputs") } - // verify personnel - normUsers := map[string]struct{}{} + switch sels.Service { + case selectors.ServiceExchange, selectors.ServiceOneDrive: + ids = userPNs - for k := range mapOfUsers { - normUsers[strings.ToLower(k)] = struct{}{} + case selectors.ServiceSharePoint: + ids = siteIDs } - for _, user := range personnel { - if _, ok := normUsers[strings.ToLower(user)]; !ok { - return fmt.Errorf("%s user not found within tenant", user) + // verify resourceOwners + normROs := map[string]struct{}{} + + for _, id := range ids { + normROs[strings.ToLower(id)] = struct{}{} + } + + for _, ro := range resourceOwners.Includes { + if _, ok := normROs[strings.ToLower(ro)]; !ok { + return fmt.Errorf("included resource owner %s not found within tenant", ro) + } + } + + for _, ro := range resourceOwners.Excludes { + if _, ok := normROs[strings.ToLower(ro)]; !ok { + return fmt.Errorf("excluded resource owner %s not found within tenant", ro) + } + } + + for _, ro := range resourceOwners.Filters { + if _, ok := normROs[strings.ToLower(ro)]; !ok { + return fmt.Errorf("filtered resource owner %s not found within tenant", ro) } } @@ -193,6 +200,18 @@ func (gc *GraphConnector) ExchangeDataCollection( // OneDrive // --------------------------------------------------------------------------- +type odFolderMatcher struct { + scope selectors.OneDriveScope +} + +func (fm odFolderMatcher) IsAny() bool { + return fm.scope.IsAny(selectors.OneDriveFolder) +} + +func (fm odFolderMatcher) Matches(dir string) bool { + return fm.scope.Matches(selectors.OneDriveFolder, dir) +} + // OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data // for the specified user func (gc *GraphConnector) OneDriveDataCollections( @@ -218,7 +237,8 @@ func (gc *GraphConnector) OneDriveDataCollections( odcs, err := onedrive.NewCollections( gc.credentials.AzureTenantID, user, - scope, + onedrive.OneDriveSource, + odFolderMatcher{scope}, &gc.graphService, gc.UpdateStatus, ).Get(ctx) @@ -236,104 +256,3 @@ func (gc *GraphConnector) OneDriveDataCollections( return collections, errs } - -// --------------------------------------------------------------------------- -// SharePoint -// --------------------------------------------------------------------------- - -// createSharePointCollections - utility function that retrieves M365 -// IDs through Microsoft Graph API. The selectors.SharePointScope -// determines the type of collections that are retrieved. -func (gc *GraphConnector) createSharePointCollections( - ctx context.Context, - scope selectors.SharePointScope, -) ([]*sharepoint.Collection, error) { - var ( - errs *multierror.Error - sites = scope.Get(selectors.SharePointSite) - colls = make([]*sharepoint.Collection, 0) - ) - - // Create collection of ExchangeDataCollection - for _, site := range sites { - collections := make(map[string]*sharepoint.Collection) - - qp := graph.QueryParams{ - Category: scope.Category().PathType(), - ResourceOwner: site, - FailFast: gc.failFast, - Credentials: gc.credentials, - } - - foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, site)) - defer closer() - defer close(foldersComplete) - - // resolver, err := exchange.PopulateExchangeContainerResolver( - // ctx, - // qp, - // qp.Scope.Category().PathType(), - // ) - // if err != nil { - // return nil, errors.Wrap(err, "getting folder cache") - // } - - // err = sharepoint.FilterContainersAndFillCollections( - // ctx, - // qp, - // collections, - // gc.UpdateStatus, - // resolver) - - // if err != nil { - // return nil, errors.Wrap(err, "filling collections") - // } - - foldersComplete <- struct{}{} - - for _, collection := range collections { - gc.incrementAwaitingMessages() - - colls = append(colls, collection) - } - } - - return colls, errs.ErrorOrNil() -} - -// SharePointDataCollections returns a set of DataCollection which represents the SharePoint data -// for the specified user -func (gc *GraphConnector) SharePointDataCollections( - ctx context.Context, - selector selectors.Selector, -) ([]data.Collection, error) { - b, err := selector.ToSharePointBackup() - if err != nil { - return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector") - } - - var ( - scopes = b.DiscreteScopes(gc.GetSites()) - collections = []data.Collection{} - errs error - ) - - // for each scope that includes oneDrive items, get all - for _, scope := range scopes { - // Creates a map of collections based on scope - dcs, err := gc.createSharePointCollections(ctx, scope) - if err != nil { - return nil, support.WrapAndAppend(scope.Get(selectors.SharePointSite)[0], err, errs) - } - - for _, collection := range dcs { - collections = append(collections, collection) - } - } - - for range collections { - gc.incrementAwaitingMessages() - } - - return collections, errs -} diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 296fd3d4d..9ee159340 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/exchange" + "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/selectors" @@ -162,7 +163,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti ctx, flush := tester.NewContext() defer flush() - connector := loadConnector(ctx, suite.T(), Users) + connector := loadConnector(ctx, suite.T(), Sites) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -180,24 +181,28 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - _, err := connector.SharePointDataCollections(ctx, test.getSelector(t)) + collection, err := sharepoint.DataCollections( + ctx, + test.getSelector(t), + []string{suite.site}, + connector.credentials.AzureTenantID, + connector) require.NoError(t, err) + assert.Equal(t, 1, len(collection)) - // TODO: Implementation - // assert.Equal(t, len(collection), 1) + // the test only reads the firstt collection + connector.incrementAwaitingMessages() - // channel := collection[0].Items() + for object := range collection[0].Items() { + buf := &bytes.Buffer{} + _, err := buf.ReadFrom(object.ToReader()) + assert.NoError(t, err, "received a buf.Read error") + } - // for object := range channel { - // buf := &bytes.Buffer{} - // _, err := buf.ReadFrom(object.ToReader()) - // assert.NoError(t, err, "received a buf.Read error") - // } + status := connector.AwaitStatus() + assert.NotZero(t, status.Successful) - // status := connector.AwaitStatus() - // assert.NotZero(t, status.Successful) - - // t.Log(status.String()) + t.Log(status.String()) }) } } @@ -499,16 +504,17 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar var ( t = suite.T() - userID = tester.M365UserID(t) + siteID = tester.M365SiteID(t) + gc = loadConnector(ctx, t, Sites) + sel = selectors.NewSharePointBackup() ) - gc := loadConnector(ctx, t, Sites) - scope := selectors.NewSharePointBackup().Folders( - []string{userID}, + sel.Include(sel.Folders( + []string{siteID}, []string{"foo"}, selectors.PrefixMatch(), - )[0] + )) - _, err := gc.createSharePointCollections(ctx, scope) + _, err := gc.DataCollections(ctx, sel.Selector) require.NoError(t, err) } diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 8e791db0c..53cce1012 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -18,7 +18,7 @@ import ( // --------------------------------------------------------------- // Disconnected Test Section -// ------------------------- +// --------------------------------------------------------------- type DisconnectedGraphConnectorSuite struct { suite.Suite } @@ -206,12 +206,13 @@ func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() { } func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() { - users := make(map[string]string) - users["elliotReid@someHospital.org"] = "" - users["chrisTurk@someHospital.org"] = "" - users["carlaEspinosa@someHospital.org"] = "" - users["bobKelso@someHospital.org"] = "" - users["johnDorian@someHospital.org"] = "" + users := []string{ + "elliotReid@someHospital.org", + "chrisTurk@someHospital.org", + "carlaEspinosa@someHospital.org", + "bobKelso@someHospital.org", + "johnDorian@someHospital.org", + } tests := []struct { name string @@ -219,12 +220,10 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() { checkError assert.ErrorAssertionFunc }{ { - name: "Invalid User", - checkError: assert.Error, + name: "No scopes", + checkError: assert.NoError, getSelector: func(t *testing.T) selectors.Selector { - sel := selectors.NewOneDriveBackup() - sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any())) - return sel.Selector + return selectors.NewExchangeBackup().Selector }, }, { @@ -260,7 +259,108 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() { for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - err := verifyBackupInputs(test.getSelector(t), users) + err := verifyBackupInputs(test.getSelector(t), users, nil) + test.checkError(t, err) + }) + } +} + +func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices() { + users := []string{"elliotReid@someHospital.org"} + sites := []string{"abc.site.foo", "bar.site.baz"} + + tests := []struct { + name string + excludes func(t *testing.T) selectors.Selector + filters func(t *testing.T) selectors.Selector + includes func(t *testing.T) selectors.Selector + checkError assert.ErrorAssertionFunc + }{ + { + name: "Valid User", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Exclude(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any())) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Filter(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any())) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Include(sel.Folders([]string{"elliotReid@someHospital.org"}, selectors.Any())) + return sel.Selector + }, + }, + { + name: "Invalid User", + checkError: assert.Error, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Exclude(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any())) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Filter(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any())) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewOneDriveBackup() + sel.Include(sel.Folders([]string{"foo@SomeCompany.org"}, selectors.Any())) + return sel.Selector + }, + }, + { + name: "valid sites", + checkError: assert.NoError, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Exclude(sel.Sites([]string{"abc.site.foo", "bar.site.baz"})) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Filter(sel.Sites([]string{"abc.site.foo", "bar.site.baz"})) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Include(sel.Sites([]string{"abc.site.foo", "bar.site.baz"})) + return sel.Selector + }, + }, + { + name: "invalid sites", + checkError: assert.Error, + excludes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Exclude(sel.Sites([]string{"fnords.smarfs.brawnhilda"})) + return sel.Selector + }, + filters: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Filter(sel.Sites([]string{"fnords.smarfs.brawnhilda"})) + return sel.Selector + }, + includes: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Include(sel.Sites([]string{"fnords.smarfs.brawnhilda"})) + return sel.Selector + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + err := verifyBackupInputs(test.excludes(t), users, sites) + test.checkError(t, err) + err = verifyBackupInputs(test.filters(t), users, sites) + test.checkError(t, err) + err = verifyBackupInputs(test.includes(t), users, sites) test.checkError(t, err) }) } diff --git a/src/internal/connector/mockconnector/mock_data_event.go b/src/internal/connector/mockconnector/mock_data_event.go index d45bcbff7..237355cb5 100644 --- a/src/internal/connector/mockconnector/mock_data_event.go +++ b/src/internal/connector/mockconnector/mock_data_event.go @@ -19,6 +19,7 @@ import ( // 6. subject // 7. hasAttachments // 8. attachments +// //nolint:lll const ( eventTmpl = `{ diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index 27183e1d6..2f7533eb3 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -60,7 +60,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() { wg := sync.WaitGroup{} collStatus := support.ConnectorOperationStatus{} - folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user") + folderPath, err := GetCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user", OneDriveSource) require.NoError(t, err) driveFolderPath, err := getDriveFolderPath(folderPath) require.NoError(t, err) @@ -117,7 +117,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() { wg := sync.WaitGroup{} wg.Add(1) - folderPath, err := getCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user") + folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", OneDriveSource) require.NoError(t, err) coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus)) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index cc8ccaec1..39c25e296 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -14,79 +14,172 @@ import ( "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" ) -// Collections is used to retrieve OneDrive data for a -// specified user +type driveSource int + +const ( + unknownDriveSource driveSource = iota + OneDriveSource + SharePointSource +) + +type folderMatcher interface { + IsAny() bool + Matches(string) bool +} + +// Collections is used to retrieve drive data for a +// resource owner, which can be either a user or a sharepoint site. type Collections struct { - tenant string - user string - scope selectors.OneDriveScope - // collectionMap allows lookup of the data.Collection - // for a OneDrive folder - collectionMap map[string]data.Collection + tenant string + resourceOwner string + source driveSource + matcher folderMatcher service graph.Service statusUpdater support.StatusUpdater + // collectionMap allows lookup of the data.Collection + // for a OneDrive folder + CollectionMap map[string]data.Collection + // Track stats from drive enumeration. Represents the items backed up. - numItems int - numFiles int - numContainers int + NumItems int + NumFiles int + NumContainers int } func NewCollections( tenant string, - user string, - scope selectors.OneDriveScope, + resourceOwner string, + source driveSource, + matcher folderMatcher, service graph.Service, statusUpdater support.StatusUpdater, ) *Collections { return &Collections{ tenant: tenant, - user: user, - scope: scope, - collectionMap: map[string]data.Collection{}, + resourceOwner: resourceOwner, + source: source, + matcher: matcher, + CollectionMap: map[string]data.Collection{}, service: service, statusUpdater: statusUpdater, } } -// Retrieves OneDrive data as set of `data.Collections` +// Retrieves drive data as set of `data.Collections` func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) { - // Enumerate drives for the specified user - drives, err := drives(ctx, c.service, c.user) + // Enumerate drives for the specified resourceOwner + drives, err := drives(ctx, c.service, c.resourceOwner, c.source) if err != nil { return nil, err } // Update the collection map with items from each drive for _, d := range drives { - err = collectItems(ctx, c.service, *d.GetId(), c.updateCollections) + err = collectItems(ctx, c.service, *d.GetId(), c.UpdateCollections) if err != nil { return nil, err } } - observe.Message(fmt.Sprintf("Discovered %d items to backup", c.numItems)) + observe.Message(fmt.Sprintf("Discovered %d items to backup", c.NumItems)) - collections := make([]data.Collection, 0, len(c.collectionMap)) - for _, coll := range c.collectionMap { + collections := make([]data.Collection, 0, len(c.CollectionMap)) + for _, coll := range c.CollectionMap { collections = append(collections, coll) } return collections, nil } -func getCanonicalPath(p, tenant, user string) (path.Path, error) { - pathBuilder := path.Builder{}.Append(strings.Split(p, "/")...) +// UpdateCollections initializes and adds the provided drive items to Collections +// A new collection is created for every drive folder (or package) +func (c *Collections) UpdateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error { + for _, item := range items { + if item.GetRoot() != nil { + // Skip the root item + continue + } + + if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil { + return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName()) + } + + // Create a collection for the parent of this item + collectionPath, err := GetCanonicalPath( + *item.GetParentReference().GetPath(), + c.tenant, + c.resourceOwner, + c.source, + ) + if err != nil { + return err + } + + // Skip items that don't match the folder selectors we were given. + if !includePath(ctx, c.matcher, collectionPath) { + logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String()) + continue + } + + switch { + case item.GetFolder() != nil, item.GetPackage() != nil: + // Leave this here so we don't fall into the default case. + // TODO: This is where we might create a "special file" to represent these in the backup repository + // e.g. a ".folderMetadataFile" + + case item.GetFile() != nil: + col, found := c.CollectionMap[collectionPath.String()] + if !found { + col = NewCollection( + collectionPath, + driveID, + c.service, + c.statusUpdater, + ) + + c.CollectionMap[collectionPath.String()] = col + c.NumContainers++ + c.NumItems++ + } + + collection := col.(*Collection) + collection.Add(*item.GetId()) + c.NumFiles++ + c.NumItems++ + + default: + return errors.Errorf("item type not supported. item name : %s", *item.GetName()) + } + } + + return nil +} + +// GetCanonicalPath constructs the standard path for the given source. +func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) { + var ( + pathBuilder = path.Builder{}.Append(strings.Split(p, "/")...) + result path.Path + err error + ) + + switch source { + case OneDriveSource: + result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false) + case SharePointSource: + result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false) + default: + return nil, errors.Errorf("unrecognized drive data source") + } - res, err := pathBuilder.ToDataLayerOneDrivePath(tenant, user, false) if err != nil { return nil, errors.Wrap(err, "converting to canonical path") } - return res, nil + return result, nil } // Returns the path to the folder within the drive (i.e. under `root:`) @@ -99,70 +192,7 @@ func getDriveFolderPath(p path.Path) (string, error) { return path.Builder{}.Append(drivePath.folders...).String(), nil } -// updateCollections initializes and adds the provided OneDrive items to Collections -// A new collection is created for every OneDrive folder (or package) -func (c *Collections) updateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error { - for _, item := range items { - if item.GetRoot() != nil { - // Skip the root item - continue - } - - if item.GetParentReference() == nil || item.GetParentReference().GetPath() == nil { - return errors.Errorf("item does not have a parent reference. item name : %s", *item.GetName()) - } - - // Create a collection for the parent of this item - collectionPath, err := getCanonicalPath( - *item.GetParentReference().GetPath(), - c.tenant, - c.user, - ) - if err != nil { - return err - } - - // Skip items that don't match the folder selectors we were given. - if !includePath(ctx, c.scope, collectionPath) { - logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String()) - continue - } - - switch { - case item.GetFolder() != nil, item.GetPackage() != nil: - // Leave this here so we don't fall into the default case. - // TODO: This is where we might create a "special file" to represent these in the backup repository - // e.g. a ".folderMetadataFile" - - case item.GetFile() != nil: - col, found := c.collectionMap[collectionPath.String()] - if !found { - col = NewCollection( - collectionPath, - driveID, - c.service, - c.statusUpdater, - ) - - c.collectionMap[collectionPath.String()] = col - c.numContainers++ - c.numItems++ - } - - collection := col.(*Collection) - collection.Add(*item.GetId()) - c.numFiles++ - c.numItems++ - - default: - return errors.Errorf("item type not supported. item name : %s", *item.GetName()) - } - } - - return nil -} - -func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath path.Path) bool { +func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) bool { // Check if the folder is allowed by the scope. folderPathString, err := getDriveFolderPath(folderPath) if err != nil { @@ -172,9 +202,9 @@ func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath // Hack for the edge case where we're looking at the root folder and can // select any folder. Right now the root folder has an empty folder path. - if len(folderPathString) == 0 && scope.IsAny(selectors.OneDriveFolder) { + if len(folderPathString) == 0 && m.IsAny() { return true } - return scope.Matches(selectors.OneDriveFolder, folderPathString) + return m.Matches(folderPathString) } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 206f1b0e1..a1a00c4ca 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -1,6 +1,7 @@ package onedrive import ( + "strings" "testing" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -20,7 +21,7 @@ func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []st res := make([]string, 0, len(rest)) for _, r := range rest { - p, err := getCanonicalPath(r, tenant, user) + p, err := GetCanonicalPath(r, tenant, user, OneDriveSource) require.NoError(t, err) res = append(res, p.String()) @@ -37,6 +38,49 @@ func TestOneDriveCollectionsSuite(t *testing.T) { suite.Run(t, new(OneDriveCollectionsSuite)) } +func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() { + tenant, resourceOwner := "tenant", "resourceOwner" + + table := []struct { + name string + source driveSource + dir []string + expect string + expectErr assert.ErrorAssertionFunc + }{ + { + name: "onedrive", + source: OneDriveSource, + dir: []string{"onedrive"}, + expect: "tenant/onedrive/resourceOwner/files/onedrive", + expectErr: assert.NoError, + }, + { + name: "sharepoint", + source: SharePointSource, + dir: []string{"sharepoint"}, + expect: "tenant/sharepoint/resourceOwner/files/sharepoint", + expectErr: assert.NoError, + }, + { + name: "unknown", + source: unknownDriveSource, + dir: []string{"unknown"}, + expectErr: assert.Error, + }, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + p := strings.Join(test.dir, "/") + result, err := GetCanonicalPath(p, tenant, resourceOwner, test.source) + test.expectErr(t, err) + if result != nil { + assert.Equal(t, test.expect, result.String()) + } + }) + } +} + func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0] @@ -211,15 +255,15 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { ctx, flush := tester.NewContext() defer flush() - c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil) - err := c.updateCollections(ctx, "driveID", tt.items) + c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil) + err := c.UpdateCollections(ctx, "driveID", tt.items) tt.expect(t, err) - assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths") - assert.Equal(t, tt.expectedItemCount, c.numItems, "item count") - assert.Equal(t, tt.expectedFileCount, c.numFiles, "file count") - assert.Equal(t, tt.expectedContainerCount, c.numContainers, "container count") + assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") + assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count") + assert.Equal(t, tt.expectedFileCount, c.NumFiles, "file count") + assert.Equal(t, tt.expectedContainerCount, c.NumContainers, "container count") for _, collPath := range tt.expectedCollectionPaths { - assert.Contains(t, c.collectionMap, collPath) + assert.Contains(t, c.CollectionMap, collPath) } }) } diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index e0db895d0..2c491b6b4 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -67,7 +67,33 @@ const ( ) // Enumerates the drives for the specified user -func drives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) { +func drives( + ctx context.Context, + service graph.Service, + resourceOwner string, + source driveSource, +) ([]models.Driveable, error) { + switch source { + case OneDriveSource: + return userDrives(ctx, service, resourceOwner) + case SharePointSource: + return siteDrives(ctx, service, resourceOwner) + default: + return nil, errors.Errorf("unrecognized drive data source") + } +} + +func siteDrives(ctx context.Context, service graph.Service, site string) ([]models.Driveable, error) { + r, err := service.Client().SitesById(site).Drives().Get(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to retrieve site drives. site: %s, details: %s", + site, support.ConnectorStackErrorTrace(err)) + } + + return r.GetValue(), nil +} + +func userDrives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) { var hasDrive bool hasDrive, err := hasDriveLicense(ctx, service, user) @@ -237,7 +263,7 @@ func GetAllFolders( userID string, prefix string, ) ([]*Displayable, error) { - drives, err := drives(ctx, gs, userID) + drives, err := drives(ctx, gs, userID, OneDriveSource) if err != nil { return nil, errors.Wrap(err, "getting OneDrive folders") } @@ -321,7 +347,7 @@ func hasDriveLicense( cb := func(pageItem any) bool { entry, ok := pageItem.(models.LicenseDetailsable) if !ok { - err = errors.New("casting item to models.MailFolderable") + err = errors.New("casting item to models.LicenseDetailsable") return false } diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index a8d25d12e..2ab50f555 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -43,7 +43,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() { folderElements := []string{folderName1} gs := loadTestService(t) - drives, err := drives(ctx, gs, suite.userID) + drives, err := drives(ctx, gs, suite.userID, OneDriveSource) require.NoError(t, err) require.NotEmpty(t, drives) @@ -100,6 +100,18 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() { } } +type testFolderMatcher struct { + scope selectors.OneDriveScope +} + +func (fm testFolderMatcher) IsAny() bool { + return fm.scope.IsAny(selectors.OneDriveFolder) +} + +func (fm testFolderMatcher) Matches(path string) bool { + return fm.scope.Matches(selectors.OneDriveFolder, path) +} + func (suite *OneDriveSuite) TestOneDriveNewCollections() { ctx, flush := tester.NewContext() defer flush() @@ -129,7 +141,8 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() { odcs, err := NewCollections( creds.AzureTenantID, test.user, - scope, + OneDriveSource, + testFolderMatcher{scope}, service, service.updateStatus, ).Get(ctx) diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index dedb5a508..df87955cd 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -67,7 +67,7 @@ func (suite *ItemIntegrationSuite) SetupSuite() { suite.user = tester.SecondaryM365UserID(suite.T()) - drives, err := drives(ctx, suite, suite.user) + drives, err := drives(ctx, suite, suite.user, OneDriveSource) require.NoError(suite.T(), err) // Test Requirement 1: Need a drive require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user) diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go new file mode 100644 index 000000000..c1b7e1b3f --- /dev/null +++ b/src/internal/connector/sharepoint/data_collections.go @@ -0,0 +1,124 @@ +package sharepoint + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" +) + +type statusUpdater interface { + UpdateStatus(status *support.ConnectorOperationStatus) +} + +type connector interface { + statusUpdater + + Service() graph.Service +} + +// DataCollections returns a set of DataCollection which represents the SharePoint data +// for the specified user +func DataCollections( + ctx context.Context, + selector selectors.Selector, + siteIDs []string, + tenantID string, + con connector, +) ([]data.Collection, error) { + b, err := selector.ToSharePointBackup() + if err != nil { + return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector") + } + + var ( + scopes = b.DiscreteScopes(siteIDs) + collections = []data.Collection{} + serv = con.Service() + errs error + ) + + for _, scope := range scopes { + // due to DiscreteScopes(siteIDs), each range should only contain one site. + for _, site := range scope.Get(selectors.SharePointSite) { + foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf( + "∙ %s - %s:", + scope.Category().PathType(), site)) + defer closer() + defer close(foldersComplete) + + switch scope.Category().PathType() { + case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory + spcs, err := collectLibraries( + ctx, + serv, + tenantID, + site, + scope, + con) + if err != nil { + return nil, support.WrapAndAppend(site, err, errs) + } + + collections = append(collections, spcs...) + } + + foldersComplete <- struct{}{} + } + } + + return collections, errs +} + +// collectLibraries constructs a onedrive Collections struct and Get()s +// all the drives associated with the site. +func collectLibraries( + ctx context.Context, + serv graph.Service, + tenantID, siteID string, + scope selectors.SharePointScope, + updater statusUpdater, +) ([]data.Collection, error) { + var ( + collections = []data.Collection{} + errs error + ) + + logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Library collections") + + colls := onedrive.NewCollections( + tenantID, + siteID, + onedrive.SharePointSource, + folderMatcher{scope}, + serv, + updater.UpdateStatus) + + odcs, err := colls.Get(ctx) + if err != nil { + return nil, support.WrapAndAppend(siteID, err, errs) + } + + return append(collections, odcs...), errs +} + +type folderMatcher struct { + scope selectors.SharePointScope +} + +func (fm folderMatcher) IsAny() bool { + return fm.scope.IsAny(selectors.SharePointFolder) +} + +func (fm folderMatcher) Matches(dir string) bool { + return fm.scope.Matches(selectors.SharePointFolder, dir) +} diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go new file mode 100644 index 000000000..15d0007e3 --- /dev/null +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -0,0 +1,158 @@ +package sharepoint_test + +import ( + "context" + "testing" + + msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/selectors" +) + +// --------------------------------------------------------------------------- +// consts, mocks +// --------------------------------------------------------------------------- + +const ( + testBaseDrivePath = "drive/driveID1/root:" +) + +type testFolderMatcher struct { + scope selectors.SharePointScope +} + +func (fm testFolderMatcher) IsAny() bool { + return fm.scope.IsAny(selectors.SharePointFolder) +} + +func (fm testFolderMatcher) Matches(path string) bool { + return fm.scope.Matches(selectors.SharePointFolder, path) +} + +type MockGraphService struct{} + +func (ms *MockGraphService) Client() *msgraphsdk.GraphServiceClient { + return nil +} + +func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter { + return nil +} + +func (ms *MockGraphService) ErrPolicy() bool { + return false +} + +// --------------------------------------------------------------------------- +// tests +// --------------------------------------------------------------------------- + +type SharePointLibrariesSuite struct { + suite.Suite + ctx context.Context +} + +func TestSharePointLibrariesSuite(t *testing.T) { + suite.Run(t, new(SharePointLibrariesSuite)) +} + +func (suite *SharePointLibrariesSuite) TestUpdateCollections() { + anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0] + + const ( + tenant = "tenant" + site = "site" + ) + + tests := []struct { + testCase string + items []models.DriveItemable + scope selectors.SharePointScope + expect assert.ErrorAssertionFunc + expectedCollectionPaths []string + expectedItemCount int + expectedContainerCount int + expectedFileCount int + }{ + { + testCase: "Single File", + items: []models.DriveItemable{ + driveItem("file", testBaseDrivePath, true), + }, + scope: anyFolder, + expect: assert.NoError, + expectedCollectionPaths: expectedPathAsSlice( + suite.T(), + tenant, + site, + testBaseDrivePath, + ), + expectedItemCount: 2, + expectedFileCount: 1, + expectedContainerCount: 1, + }, + } + + for _, test := range tests { + suite.T().Run(test.testCase, func(t *testing.T) { + ctx, flush := tester.NewContext() + defer flush() + + c := onedrive.NewCollections( + tenant, + site, + onedrive.SharePointSource, + testFolderMatcher{test.scope}, + &MockGraphService{}, + nil) + err := c.UpdateCollections(ctx, "driveID", test.items) + test.expect(t, err) + assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths") + assert.Equal(t, test.expectedItemCount, c.NumItems, "item count") + assert.Equal(t, test.expectedFileCount, c.NumFiles, "file count") + assert.Equal(t, test.expectedContainerCount, c.NumContainers, "container count") + for _, collPath := range test.expectedCollectionPaths { + assert.Contains(t, c.CollectionMap, collPath) + } + }) + } +} + +func driveItem(name string, path string, isFile bool) models.DriveItemable { + item := models.NewDriveItem() + item.SetName(&name) + item.SetId(&name) + + parentReference := models.NewItemReference() + parentReference.SetPath(&path) + item.SetParentReference(parentReference) + + if isFile { + item.SetFile(models.NewFile()) + } + + return item +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string { + res := make([]string, 0, len(rest)) + + for _, r := range rest { + p, err := onedrive.GetCanonicalPath(r, tenant, user, onedrive.SharePointSource) + require.NoError(t, err) + + res = append(res, p.String()) + } + + return res +} diff --git a/src/internal/tester/integration_runners.go b/src/internal/tester/integration_runners.go index 0215e0a9c..cd3256dff 100644 --- a/src/internal/tester/integration_runners.go +++ b/src/internal/tester/integration_runners.go @@ -23,6 +23,7 @@ const ( CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS" CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS" CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS" + CorsoGraphConnectorSharePointTests = "CORSO_GRAPH_CONNECTOR_SHAREPOINT_TESTS" CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS" CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS" CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS" diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index aefb833b5..a0e9b426f 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -241,7 +241,7 @@ func (pb Builder) verifyPrefix(tenant, resourceOwner string) error { } if len(resourceOwner) == 0 { - return errors.Wrap(errMissingSegment, "user") + return errors.Wrap(errMissingSegment, "resourceOwner") } if len(pb.elements) == 0 { diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index 442e8c1d5..ca98fefe5 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -36,7 +36,11 @@ type ( } ) -var _ Reducer = &ExchangeRestore{} +var ( + _ Reducer = &ExchangeRestore{} + _ printabler = &ExchangeRestore{} + _ resourceOwnerer = &ExchangeRestore{} +) // NewExchange produces a new Selector with the service set to ServiceExchange. func NewExchangeBackup() *ExchangeBackup { @@ -89,6 +93,16 @@ func (s exchange) Printable() Printable { return toPrintable[ExchangeScope](s.Selector) } +// ResourceOwners produces the aggregation of discrete users described by each type of scope. +// Any and None values are omitted. +func (s exchange) ResourceOwners() selectorResourceOwners { + return selectorResourceOwners{ + Excludes: resourceOwnersIn(s.Excludes, ExchangeUser.String()), + Filters: resourceOwnersIn(s.Filters, ExchangeUser.String()), + Includes: resourceOwnersIn(s.Includes, ExchangeUser.String()), + } +} + // ------------------- // Exclude/Includes diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index 02bd4f05a..752ebc85f 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -35,7 +35,11 @@ type ( } ) -var _ Reducer = &OneDriveRestore{} +var ( + _ Reducer = &OneDriveRestore{} + _ printabler = &OneDriveRestore{} + _ resourceOwnerer = &OneDriveRestore{} +) // NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive. func NewOneDriveBackup() *OneDriveBackup { @@ -88,6 +92,16 @@ func (s oneDrive) Printable() Printable { return toPrintable[OneDriveScope](s.Selector) } +// ResourceOwners produces the aggregation of discrete users described by each type of scope. +// Any and None values are omitted. +func (s oneDrive) ResourceOwners() selectorResourceOwners { + return selectorResourceOwners{ + Excludes: resourceOwnersIn(s.Excludes, OneDriveUser.String()), + Filters: resourceOwnersIn(s.Filters, OneDriveUser.String()), + Includes: resourceOwnersIn(s.Includes, OneDriveUser.String()), + } +} + // ------------------- // Scope Factories diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 836a2ae7f..38ae1fa15 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -54,13 +54,13 @@ const ( // None() to any selector will force all matches() checks on that // selector to fail. // Ex: {user: u1, events: NoneTgt} => matches nothing. - NoneTgt = "" - delimiter = "," + NoneTgt = "" ) var ( - passAny = filters.Pass() - failAny = filters.Fail() + delimiter = fmt.Sprint(0x1F) + passAny = filters.Pass() + failAny = filters.Fail() ) // All is the resource name that gets output when the resource is AnyTgt. @@ -71,6 +71,19 @@ type Reducer interface { Reduce(context.Context, *details.Details) *details.Details } +// selectorResourceOwners aggregates all discrete resource owner ids described +// in the selector. Any and None values are ignored. ResourceOwner sets are +// grouped by their scope type (includes, excludes, filters). +type selectorResourceOwners struct { + Includes []string + Excludes []string + Filters []string +} + +type resourceOwnerer interface { + ResourceOwners() selectorResourceOwners +} + // --------------------------------------------------------------------------- // Selector // --------------------------------------------------------------------------- @@ -184,22 +197,7 @@ func (s Selector) PathService() path.ServiceType { // than have the caller make that interpretation. Returns an error if the // service is unsupported. func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details.Details, error) { - var ( - r Reducer - err error - ) - - switch s.Service { - case ServiceExchange: - r, err = s.ToExchangeRestore() - case ServiceOneDrive: - r, err = s.ToOneDriveRestore() - case ServiceSharePoint: - r, err = s.ToSharePointRestore() - default: - return nil, errors.New("service not supported: " + s.Service.String()) - } - + r, err := selectorAsIface[Reducer](s) if err != nil { return nil, err } @@ -207,6 +205,40 @@ func (s Selector) Reduce(ctx context.Context, deets *details.Details) (*details. return r.Reduce(ctx, deets), nil } +func (s Selector) ResourceOwners() (selectorResourceOwners, error) { + ro, err := selectorAsIface[resourceOwnerer](s) + if err != nil { + return selectorResourceOwners{}, err + } + + return ro.ResourceOwners(), nil +} + +// transformer for arbitrary selector interfaces +func selectorAsIface[T any](s Selector) (T, error) { + var ( + a any + t T + err error + ) + + switch s.Service { + case ServiceExchange: + a, err = func() (any, error) { return s.ToExchangeRestore() }() + t = a.(T) + case ServiceOneDrive: + a, err = func() (any, error) { return s.ToOneDriveRestore() }() + t = a.(T) + case ServiceSharePoint: + a, err = func() (any, error) { return s.ToSharePointRestore() }() + t = a.(T) + default: + err = errors.New("service not supported: " + s.Service.String()) + } + + return t, err +} + // --------------------------------------------------------------------------- // Printing Selectors for Human Reading // --------------------------------------------------------------------------- @@ -218,35 +250,18 @@ type Printable struct { Includes map[string][]string `json:"includes,omitempty"` } +type printabler interface { + Printable() Printable +} + // ToPrintable creates the minimized display of a selector, formatted for human readability. func (s Selector) ToPrintable() Printable { - switch s.Service { - case ServiceExchange: - r, err := s.ToExchangeRestore() - if err != nil { - return Printable{} - } - - return r.Printable() - - case ServiceOneDrive: - r, err := s.ToOneDriveBackup() - if err != nil { - return Printable{} - } - - return r.Printable() - - case ServiceSharePoint: - r, err := s.ToSharePointBackup() - if err != nil { - return Printable{} - } - - return r.Printable() + p, err := selectorAsIface[printabler](s) + if err != nil { + return Printable{} } - return Printable{} + return p.Printable() } // toPrintable creates the minimized display of a selector, formatted for human readability. @@ -351,6 +366,32 @@ func addToSet(set []string, v []string) []string { // helpers // --------------------------------------------------------------------------- +// produces the discrete set of resource owners in the slice of scopes. +// Any and None values are discarded. +func resourceOwnersIn(s []scope, rootCat string) []string { + rm := map[string]struct{}{} + + for _, sc := range s { + for _, v := range split(sc[rootCat].Target) { + rm[v] = struct{}{} + } + } + + rs := []string{} + + for k := range rm { + if k != AnyTgt && k != NoneTgt { + rs = append(rs, k) + } + } + + return rs +} + +// --------------------------------------------------------------------------- +// scope helpers +// --------------------------------------------------------------------------- + type scopeConfig struct { usePathFilter bool usePrefixFilter bool @@ -382,10 +423,6 @@ func pathType() option { } } -// --------------------------------------------------------------------------- -// helpers -// --------------------------------------------------------------------------- - func badCastErr(cast, is service) error { return errors.Wrapf(ErrorBadSelectorCast, "%s service is not %s", cast, is) } diff --git a/src/pkg/selectors/selectors_test.go b/src/pkg/selectors/selectors_test.go index 5311f14f8..9668e4fb9 100644 --- a/src/pkg/selectors/selectors_test.go +++ b/src/pkg/selectors/selectors_test.go @@ -6,6 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/pkg/filters" ) type SelectorSuite struct { @@ -143,6 +145,69 @@ func (suite *SelectorSuite) TestToResourceTypeMap() { } } +func (suite *SelectorSuite) TestResourceOwnersIn() { + rootCat := rootCatStub.String() + + table := []struct { + name string + input []scope + expect []string + }{ + { + name: "nil", + input: nil, + expect: []string{}, + }, + { + name: "empty", + input: []scope{}, + expect: []string{}, + }, + { + name: "single", + input: []scope{{rootCat: filters.Identity("foo")}}, + expect: []string{"foo"}, + }, + { + name: "multiple values", + input: []scope{{rootCat: filters.Identity(join("foo", "bar"))}}, + expect: []string{"foo", "bar"}, + }, + { + name: "with any", + input: []scope{{rootCat: filters.Identity(join("foo", "bar", AnyTgt))}}, + expect: []string{"foo", "bar"}, + }, + { + name: "with none", + input: []scope{{rootCat: filters.Identity(join("foo", "bar", NoneTgt))}}, + expect: []string{"foo", "bar"}, + }, + { + name: "multiple scopes", + input: []scope{ + {rootCat: filters.Identity(join("foo", "bar"))}, + {rootCat: filters.Identity(join("baz"))}, + }, + expect: []string{"foo", "bar", "baz"}, + }, + { + name: "multiple scopes with duplicates", + input: []scope{ + {rootCat: filters.Identity(join("foo", "bar"))}, + {rootCat: filters.Identity(join("baz", "foo"))}, + }, + expect: []string{"foo", "bar", "baz"}, + }, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + result := resourceOwnersIn(test.input, rootCat) + assert.ElementsMatch(t, test.expect, result) + }) + } +} + func (suite *SelectorSuite) TestContains() { t := suite.T() key := rootCatStub diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 57950c72f..7aecab5e3 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -34,7 +34,11 @@ type ( } ) -var _ Reducer = &SharePointRestore{} +var ( + _ Reducer = &SharePointRestore{} + _ printabler = &SharePointRestore{} + _ resourceOwnerer = &SharePointRestore{} +) // NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint. func NewSharePointBackup() *SharePointBackup { @@ -87,6 +91,16 @@ func (s sharePoint) Printable() Printable { return toPrintable[SharePointScope](s.Selector) } +// ResourceOwners produces the aggregation of discrete sitets described by each type of scope. +// Any and None values are omitted. +func (s sharePoint) ResourceOwners() selectorResourceOwners { + return selectorResourceOwners{ + Excludes: resourceOwnersIn(s.Excludes, SharePointSite.String()), + Filters: resourceOwnersIn(s.Filters, SharePointSite.String()), + Includes: resourceOwnersIn(s.Includes, SharePointSite.String()), + } +} + // ------------------- // Scope Factories