generic drive retrieval for sharepoint

Adapts the graph onedrive library to handle
access to drive data across both onedrive and
sharepoint services.

src/cmd/purge/purge.go

@@ -240,12 +240,12 @@ func purgeMailFolders(
 	uid string,
 ) error {
 	getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
-		params, err := exchangeQueryParamFactory(uid, path.EmailCategory)
+		params, scope, err := exchangeQueryParamFactory(uid, path.EmailCategory)
 		if err != nil {
 			return nil, err
 		}
 
-		allFolders, err := exchange.GetAllMailFolders(ctx, *params, gs)
+		allFolders, err := exchange.GetAllMailFolders(ctx, *params, gs, scope)
 		if err != nil {
 			return nil, err
 		}
@@ -276,12 +276,12 @@ func purgeCalendarFolders(
 	uid string,
 ) error {
 	getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
-		params, err := exchangeQueryParamFactory(uid, path.EventsCategory)
+		params, scope, err := exchangeQueryParamFactory(uid, path.EventsCategory)
 		if err != nil {
 			return nil, err
 		}
 
-		allCalendars, err := exchange.GetAllCalendars(ctx, *params, gs)
+		allCalendars, err := exchange.GetAllCalendars(ctx, *params, gs, scope)
 		if err != nil {
 			return nil, err
 		}
@@ -312,12 +312,12 @@ func purgeContactFolders(
 	uid string,
 ) error {
 	getter := func(gs graph.Service, uid, prefix string) ([]purgable, error) {
-		params, err := exchangeQueryParamFactory(uid, path.ContactsCategory)
+		params, scope, err := exchangeQueryParamFactory(uid, path.ContactsCategory)
 		if err != nil {
 			return nil, err
 		}
 
-		allContainers, err := exchange.GetAllContactFolders(ctx, *params, gs)
+		allContainers, err := exchange.GetAllContactFolders(ctx, *params, gs, scope)
 		if err != nil {
 			return nil, err
 		}
@@ -518,7 +518,10 @@ func containerFilter(nameContains string, containers []graph.CachedContainer) []
 	return result
 }
 
-func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.QueryParams, error) {
+func exchangeQueryParamFactory(
+	user string,
+	category path.CategoryType,
+) (*graph.QueryParams, selectors.ExchangeScope, error) {
 	var scope selectors.ExchangeScope
 
 	switch category {
@@ -529,12 +532,11 @@ func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.
 	case path.EventsCategory:
 		scope = selectors.NewExchangeBackup().EventCalendars([]string{user}, selectors.Any())[0]
 	default:
-		return nil, fmt.Errorf("category %s not supported", category)
+		return nil, scope, fmt.Errorf("category %s not supported", category)
 	}
 
 	params := &graph.QueryParams{
-		User:     user,
+		ResourceOwner: user,
-		Scope:    scope,
 		FailFast:      false,
 		Credentials: account.M365Config{
 			M365:          credentials.GetM365(),
@@ -542,5 +544,5 @@ func exchangeQueryParamFactory(user string, category path.CategoryType) (*graph.
 		},
 	}
 
-	return params, nil
+	return params, scope, nil
 }

src/internal/connector/data_collections.go

@@ -0,0 +1,341 @@
+package connector
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+
+	"github.com/alcionai/corso/src/internal/connector/exchange"
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/onedrive"
+	"github.com/alcionai/corso/src/internal/connector/sharepoint"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/internal/data"
+	D "github.com/alcionai/corso/src/internal/diagnostics"
+	"github.com/alcionai/corso/src/internal/observe"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// ---------------------------------------------------------------------------
+// Data Collections
+// ---------------------------------------------------------------------------
+
+// DataCollections utility function to launch backup operations for exchange and onedrive
+func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Selector) ([]data.Collection, error) {
+	ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
+	defer end()
+
+	err := verifyBackupInputs(sels, gc.Users)
+	if err != nil {
+		return nil, err
+	}
+
+	switch sels.Service {
+	case selectors.ServiceExchange:
+		return gc.ExchangeDataCollection(ctx, sels)
+	case selectors.ServiceOneDrive:
+		return gc.OneDriveDataCollections(ctx, sels)
+	case selectors.ServiceSharePoint:
+		return gc.SharePointDataCollections(ctx, sels)
+	default:
+		return nil, errors.Errorf("service %s not supported", sels)
+	}
+}
+
+func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) error {
+	var personnel []string
+
+	// retrieve users from selectors
+	switch sel.Service {
+	case selectors.ServiceExchange:
+		backup, err := sel.ToExchangeBackup()
+		if err != nil {
+			return err
+		}
+
+		for _, scope := range backup.Scopes() {
+			temp := scope.Get(selectors.ExchangeUser)
+			personnel = append(personnel, temp...)
+		}
+	case selectors.ServiceOneDrive:
+		backup, err := sel.ToOneDriveBackup()
+		if err != nil {
+			return err
+		}
+
+		for _, user := range backup.Scopes() {
+			temp := user.Get(selectors.OneDriveUser)
+			personnel = append(personnel, temp...)
+		}
+
+	default:
+		return errors.New("service %s not supported")
+	}
+
+	// verify personnel
+	normUsers := map[string]struct{}{}
+
+	for k := range mapOfUsers {
+		normUsers[strings.ToLower(k)] = struct{}{}
+	}
+
+	for _, user := range personnel {
+		if _, ok := normUsers[strings.ToLower(user)]; !ok {
+			return fmt.Errorf("%s user not found within tenant", user)
+		}
+	}
+
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// Exchange
+// ---------------------------------------------------------------------------
+
+// createExchangeCollections - utility function that retrieves M365
+// IDs through Microsoft Graph API. The selectors.ExchangeScope
+// determines the type of collections that are retrieved.
+func (gc *GraphConnector) createExchangeCollections(
+	ctx context.Context,
+	scope selectors.ExchangeScope,
+) ([]*exchange.Collection, error) {
+	var (
+		errs           *multierror.Error
+		users          = scope.Get(selectors.ExchangeUser)
+		allCollections = make([]*exchange.Collection, 0)
+	)
+
+	// Create collection of ExchangeDataCollection
+	for _, user := range users {
+		collections := make(map[string]*exchange.Collection)
+
+		qp := graph.QueryParams{
+			Category:      scope.Category().PathType(),
+			ResourceOwner: user,
+			FailFast:      gc.failFast,
+			Credentials:   gc.credentials,
+		}
+
+		foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", qp.Category, user))
+		defer closer()
+		defer close(foldersComplete)
+
+		resolver, err := exchange.PopulateExchangeContainerResolver(ctx, qp)
+		if err != nil {
+			return nil, errors.Wrap(err, "getting folder cache")
+		}
+
+		err = exchange.FilterContainersAndFillCollections(
+			ctx,
+			qp,
+			collections,
+			gc.UpdateStatus,
+			resolver,
+			scope)
+
+		if err != nil {
+			return nil, errors.Wrap(err, "filling collections")
+		}
+
+		foldersComplete <- struct{}{}
+
+		for _, collection := range collections {
+			gc.incrementAwaitingMessages()
+
+			allCollections = append(allCollections, collection)
+		}
+	}
+
+	return allCollections, errs.ErrorOrNil()
+}
+
+// ExchangeDataCollections returns a DataCollection which the caller can
+// use to read mailbox data out for the specified user
+// Assumption: User exists
+//
+//	Add iota to this call -> mail, contacts, calendar,  etc.
+func (gc *GraphConnector) ExchangeDataCollection(
+	ctx context.Context,
+	selector selectors.Selector,
+) ([]data.Collection, error) {
+	eb, err := selector.ToExchangeBackup()
+	if err != nil {
+		return nil, errors.Wrap(err, "exchangeDataCollection: parsing selector")
+	}
+
+	var (
+		scopes      = eb.DiscreteScopes(gc.GetUsers())
+		collections = []data.Collection{}
+		errs        error
+	)
+
+	for _, scope := range scopes {
+		// Creates a map of collections based on scope
+		dcs, err := gc.createExchangeCollections(ctx, scope)
+		if err != nil {
+			user := scope.Get(selectors.ExchangeUser)
+			return nil, support.WrapAndAppend(user[0], err, errs)
+		}
+
+		for _, collection := range dcs {
+			collections = append(collections, collection)
+		}
+	}
+
+	return collections, errs
+}
+
+// ---------------------------------------------------------------------------
+// OneDrive
+// ---------------------------------------------------------------------------
+
+type odFolderMatcher struct {
+	scope selectors.OneDriveScope
+}
+
+func (fm odFolderMatcher) IsAny() bool {
+	return fm.scope.IsAny(selectors.OneDriveFolder)
+}
+
+func (fm odFolderMatcher) Matches(path string) bool {
+	return fm.scope.Matches(selectors.OneDriveFolder, path)
+}
+
+// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
+// for the specified user
+func (gc *GraphConnector) OneDriveDataCollections(
+	ctx context.Context,
+	selector selectors.Selector,
+) ([]data.Collection, error) {
+	odb, err := selector.ToOneDriveBackup()
+	if err != nil {
+		return nil, errors.Wrap(err, "oneDriveDataCollection: parsing selector")
+	}
+
+	var (
+		scopes      = odb.DiscreteScopes(gc.GetUsers())
+		collections = []data.Collection{}
+		errs        error
+	)
+
+	// for each scope that includes oneDrive items, get all
+	for _, scope := range scopes {
+		for _, user := range scope.Get(selectors.OneDriveUser) {
+			logger.Ctx(ctx).With("user", user).Debug("Creating OneDrive collections")
+
+			odcs, err := onedrive.NewCollections(
+				gc.credentials.AzureTenantID,
+				user,
+				onedrive.OneDriveSource,
+				odFolderMatcher{scope},
+				&gc.graphService,
+				gc.UpdateStatus,
+			).Get(ctx)
+			if err != nil {
+				return nil, support.WrapAndAppend(user, err, errs)
+			}
+
+			collections = append(collections, odcs...)
+		}
+	}
+
+	for range collections {
+		gc.incrementAwaitingMessages()
+	}
+
+	return collections, errs
+}
+
+// ---------------------------------------------------------------------------
+// SharePoint
+// ---------------------------------------------------------------------------
+
+// createSharePointCollections - utility function that retrieves M365
+// IDs through Microsoft Graph API. The selectors.SharePointScope
+// determines the type of collections that are retrieved.
+func (gc *GraphConnector) createSharePointCollections(
+	ctx context.Context,
+	scope selectors.SharePointScope,
+) ([]data.Collection, error) {
+	var (
+		errs        *multierror.Error
+		sites       = scope.Get(selectors.SharePointSite)
+		category    = scope.Category().PathType()
+		collections = make([]data.Collection, 0)
+	)
+
+	// Create collection of ExchangeDataCollection
+	for _, site := range sites {
+
+		foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", category, site))
+		defer closer()
+		defer close(foldersComplete)
+
+		switch category {
+		case path.FilesCategory: // TODO: better category for drives
+			spcs, err := sharepoint.CollectLibraries(
+				ctx,
+				gc.Service(),
+				gc.credentials.AzureTenantID,
+				gc.GetSiteIds(),
+				scope,
+				gc.UpdateStatus,
+				gc.incrementAwaitingMessages,
+			)
+			if err != nil {
+				return nil, support.WrapAndAppend(site, err, errs)
+			}
+
+			collections = append(collections, spcs...)
+
+			// case path.UnknownCategory: // TODO: ListsCategory
+			// 	// get lists
+		}
+
+		foldersComplete <- struct{}{}
+	}
+
+	return collections, errs.ErrorOrNil()
+}
+
+// SharePointDataCollections returns a set of DataCollection which represents the SharePoint data
+// for the specified user
+func (gc *GraphConnector) SharePointDataCollections(
+	ctx context.Context,
+	selector selectors.Selector,
+) ([]data.Collection, error) {
+	b, err := selector.ToSharePointBackup()
+	if err != nil {
+		return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector")
+	}
+
+	var (
+		scopes      = b.DiscreteScopes(gc.GetSites())
+		collections = []data.Collection{}
+		errs        error
+	)
+
+	// for each scope that includes oneDrive items, get all
+	for _, scope := range scopes {
+		// Creates a slice of collections based on scope
+		dcs, err := gc.createSharePointCollections(ctx, scope)
+		if err != nil {
+			return nil, support.WrapAndAppend(scope.Get(selectors.SharePointSite)[0], err, errs)
+		}
+
+		for _, collection := range dcs {
+			collections = append(collections, collection)
+		}
+	}
+
+	for range collections {
+		gc.incrementAwaitingMessages()
+	}
+
+	return collections, errs
+}

src/internal/connector/data_collections_test.go

@@ -0,0 +1,510 @@
+package connector
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/connector/exchange"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// ---------------------------------------------------------------------------
+// DataCollection tests
+// ---------------------------------------------------------------------------
+
+type ConnectorDataCollectionIntegrationSuite struct {
+	suite.Suite
+	connector *GraphConnector
+	user      string
+	site      string
+}
+
+func TestConnectorDataCollectionIntegrationSuite(t *testing.T) {
+	if err := tester.RunOnAny(
+		tester.CorsoCITests,
+		tester.CorsoConnectorDataCollectionTests,
+	); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(ConnectorDataCollectionIntegrationSuite))
+}
+
+func (suite *ConnectorDataCollectionIntegrationSuite) SetupSuite() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	_, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...)
+	require.NoError(suite.T(), err)
+	suite.connector = loadConnector(ctx, suite.T())
+	suite.user = tester.M365UserID(suite.T())
+	suite.site = tester.M365SiteID(suite.T())
+	tester.LogTimeOfTest(suite.T())
+}
+
+// TestExchangeDataCollection verifies interface between operation and
+// GraphConnector remains stable to receive a non-zero amount of Collections
+// for the Exchange Package. Enabled exchange applications:
+// - mail
+// - contacts
+// - events
+func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	connector := loadConnector(ctx, suite.T())
+	tests := []struct {
+		name        string
+		getSelector func(t *testing.T) selectors.Selector
+	}{
+		{
+			name: suite.user + " Email",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
+
+				return sel.Selector
+			},
+		},
+		{
+			name: suite.user + " Contacts",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.ContactFolders(
+					[]string{suite.user},
+					[]string{exchange.DefaultContactFolder},
+					selectors.PrefixMatch()))
+
+				return sel.Selector
+			},
+		},
+		{
+			name: suite.user + " Events",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
+
+				return sel.Selector
+			},
+		},
+	}
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			collection, err := connector.ExchangeDataCollection(ctx, test.getSelector(t))
+			require.NoError(t, err)
+			assert.Equal(t, len(collection), 1)
+			channel := collection[0].Items()
+			for object := range channel {
+				buf := &bytes.Buffer{}
+				_, err := buf.ReadFrom(object.ToReader())
+				assert.NoError(t, err, "received a buf.Read error")
+			}
+			status := connector.AwaitStatus()
+			assert.NotZero(t, status.Successful)
+			t.Log(status.String())
+		})
+	}
+}
+
+// TestInvalidUserForDataCollections ensures verification process for users
+func (suite *ConnectorDataCollectionIntegrationSuite) TestInvalidUserForDataCollections() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	invalidUser := "foo@example.com"
+	connector := loadConnector(ctx, suite.T())
+	tests := []struct {
+		name        string
+		getSelector func(t *testing.T) selectors.Selector
+	}{
+		{
+			name: "invalid exchange backup user",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.MailFolders([]string{invalidUser}, selectors.Any()))
+				return sel.Selector
+			},
+		},
+		{
+			name: "Invalid onedrive backup user",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewOneDriveBackup()
+				sel.Include(sel.Folders([]string{invalidUser}, selectors.Any()))
+				return sel.Selector
+			},
+		},
+	}
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			collections, err := connector.DataCollections(ctx, test.getSelector(t))
+			assert.Error(t, err)
+			assert.Empty(t, collections)
+		})
+	}
+}
+
+// TestSharePointDataCollection verifies interface between operation and
+// GraphConnector remains stable to receive a non-zero amount of Collections
+// for the SharePoint Package.
+func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollection() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	connector := loadConnector(ctx, suite.T())
+	tests := []struct {
+		name        string
+		getSelector func(t *testing.T) selectors.Selector
+	}{
+		{
+			name: "Items - TODO: actual sharepoint categories",
+			getSelector: func(t *testing.T) selectors.Selector {
+				sel := selectors.NewSharePointBackup()
+				sel.Include(sel.Folders([]string{suite.site}, selectors.Any()))
+
+				return sel.Selector
+			},
+		},
+	}
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			_, err := connector.SharePointDataCollections(ctx, test.getSelector(t))
+			require.NoError(t, err)
+
+			// TODO: Implementation
+			// assert.Equal(t, len(collection), 1)
+
+			// channel := collection[0].Items()
+
+			// for object := range channel {
+			// 	buf := &bytes.Buffer{}
+			// 	_, err := buf.ReadFrom(object.ToReader())
+			// 	assert.NoError(t, err, "received a buf.Read error")
+			// }
+
+			// status := connector.AwaitStatus()
+			// assert.NotZero(t, status.Successful)
+
+			// t.Log(status.String())
+		})
+	}
+}
+
+// ---------------------------------------------------------------------------
+// CreateExchangeCollection tests
+// ---------------------------------------------------------------------------
+
+type ConnectorCreateExchangeCollectionIntegrationSuite struct {
+	suite.Suite
+	connector *GraphConnector
+	user      string
+	site      string
+}
+
+func TestConnectorCreateExchangeCollectionIntegrationSuite(t *testing.T) {
+	if err := tester.RunOnAny(
+		tester.CorsoCITests,
+		tester.CorsoConnectorCreateExchangeCollectionTests,
+	); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(ConnectorCreateExchangeCollectionIntegrationSuite))
+}
+
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) SetupSuite() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	_, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...)
+	require.NoError(suite.T(), err)
+	suite.connector = loadConnector(ctx, suite.T())
+	suite.user = tester.M365UserID(suite.T())
+	suite.site = tester.M365SiteID(suite.T())
+	tester.LogTimeOfTest(suite.T())
+}
+
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailFetch() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	var (
+		t      = suite.T()
+		userID = tester.M365UserID(t)
+	)
+
+	tests := []struct {
+		name        string
+		scope       selectors.ExchangeScope
+		folderNames map[string]struct{}
+	}{
+		{
+			name: "Folder Iterative Check Mail",
+			scope: selectors.NewExchangeBackup().MailFolders(
+				[]string{userID},
+				[]string{exchange.DefaultMailFolder},
+				selectors.PrefixMatch(),
+			)[0],
+			folderNames: map[string]struct{}{
+				exchange.DefaultMailFolder: {},
+			},
+		},
+	}
+
+	gc := loadConnector(ctx, t)
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			collections, err := gc.createExchangeCollections(ctx, test.scope)
+			require.NoError(t, err)
+
+			for _, c := range collections {
+				require.NotEmpty(t, c.FullPath().Folder())
+				folder := c.FullPath().Folder()
+
+				if _, ok := test.folderNames[folder]; ok {
+					delete(test.folderNames, folder)
+				}
+			}
+
+			assert.Empty(t, test.folderNames)
+		})
+	}
+}
+
+// TestMailSerializationRegression verifies that all mail data stored in the
+// test account can be successfully downloaded into bytes and restored into
+// M365 mail objects
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailSerializationRegression() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	t := suite.T()
+	connector := loadConnector(ctx, t)
+	sel := selectors.NewExchangeBackup()
+	sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
+	collection, err := connector.createExchangeCollections(ctx, sel.Scopes()[0])
+	require.NoError(t, err)
+
+	for _, edc := range collection {
+		suite.T().Run(edc.FullPath().String(), func(t *testing.T) {
+			streamChannel := edc.Items()
+			// Verify that each message can be restored
+			for stream := range streamChannel {
+				buf := &bytes.Buffer{}
+				read, err := buf.ReadFrom(stream.ToReader())
+				assert.NoError(t, err)
+				assert.NotZero(t, read)
+				message, err := support.CreateMessageFromBytes(buf.Bytes())
+				assert.NotNil(t, message)
+				assert.NoError(t, err)
+			}
+		})
+	}
+
+	status := connector.AwaitStatus()
+	suite.NotNil(status)
+	suite.Equal(status.ObjectCount, status.Successful)
+}
+
+// TestContactSerializationRegression verifies ability to query contact items
+// and to store contact within Collection. Downloaded contacts are run through
+// a regression test to ensure that downloaded items can be uploaded.
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestContactSerializationRegression() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	connector := loadConnector(ctx, suite.T())
+
+	tests := []struct {
+		name          string
+		getCollection func(t *testing.T) []*exchange.Collection
+	}{
+		{
+			name: "Default Contact Folder",
+			getCollection: func(t *testing.T) []*exchange.Collection {
+				scope := selectors.
+					NewExchangeBackup().
+					ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch())[0]
+				collections, err := connector.createExchangeCollections(ctx, scope)
+				require.NoError(t, err)
+
+				return collections
+			},
+		},
+	}
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			edcs := test.getCollection(t)
+			require.Equal(t, len(edcs), 1)
+			edc := edcs[0]
+			assert.Equal(t, edc.FullPath().Folder(), exchange.DefaultContactFolder)
+			streamChannel := edc.Items()
+			count := 0
+			for stream := range streamChannel {
+				buf := &bytes.Buffer{}
+				read, err := buf.ReadFrom(stream.ToReader())
+				assert.NoError(t, err)
+				assert.NotZero(t, read)
+				contact, err := support.CreateContactFromBytes(buf.Bytes())
+				assert.NotNil(t, contact)
+				assert.NoError(t, err, "error on converting contact bytes: "+string(buf.Bytes()))
+				count++
+			}
+			assert.NotZero(t, count)
+
+			status := connector.AwaitStatus()
+			suite.NotNil(status)
+			suite.Equal(status.ObjectCount, status.Successful)
+		})
+	}
+}
+
+// TestEventsSerializationRegression ensures functionality of createCollections
+// to be able to successfully query, download and restore event objects
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestEventsSerializationRegression() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	connector := loadConnector(ctx, suite.T())
+
+	tests := []struct {
+		name, expected string
+		getCollection  func(t *testing.T) []*exchange.Collection
+	}{
+		{
+			name:     "Default Event Calendar",
+			expected: exchange.DefaultCalendar,
+			getCollection: func(t *testing.T) []*exchange.Collection {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
+				collections, err := connector.createExchangeCollections(ctx, sel.Scopes()[0])
+				require.NoError(t, err)
+
+				return collections
+			},
+		},
+		{
+			name:     "Birthday Calendar",
+			expected: "Birthdays",
+			getCollection: func(t *testing.T) []*exchange.Collection {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}, selectors.PrefixMatch()))
+				collections, err := connector.createExchangeCollections(ctx, sel.Scopes()[0])
+				require.NoError(t, err)
+
+				return collections
+			},
+		},
+	}
+
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			collections := test.getCollection(t)
+			require.Equal(t, len(collections), 1)
+			edc := collections[0]
+			assert.Equal(t, edc.FullPath().Folder(), test.expected)
+			streamChannel := edc.Items()
+
+			for stream := range streamChannel {
+				buf := &bytes.Buffer{}
+				read, err := buf.ReadFrom(stream.ToReader())
+				assert.NoError(t, err)
+				assert.NotZero(t, read)
+				event, err := support.CreateEventFromBytes(buf.Bytes())
+				assert.NotNil(t, event)
+				assert.NoError(t, err, "experienced error parsing event bytes: "+string(buf.Bytes()))
+			}
+
+			status := connector.AwaitStatus()
+			suite.NotNil(status)
+			suite.Equal(status.ObjectCount, status.Successful)
+		})
+	}
+}
+
+// TestAccessOfInboxAllUsers verifies that GraphConnector can
+// support `--users *` for backup operations. Selector.DiscreteScopes
+// returns all of the users within one scope. Only users who have
+// messages in their inbox will have a collection returned.
+// The final test insures that more than a 75% of the user collections are
+// returned. If an error was experienced, the test will fail overall
+func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestAccessOfInboxAllUsers() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	t := suite.T()
+	connector := loadConnector(ctx, t)
+	sel := selectors.NewExchangeBackup()
+	sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
+	scopes := sel.DiscreteScopes(connector.GetUsers())
+
+	for _, scope := range scopes {
+		users := scope.Get(selectors.ExchangeUser)
+		standard := (len(users) / 4) * 3
+		collections, err := connector.createExchangeCollections(ctx, scope)
+		require.NoError(t, err)
+		suite.Greater(len(collections), standard)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// CreateSharePointCollection tests
+// ---------------------------------------------------------------------------
+
+type ConnectorCreateSharePointCollectionIntegrationSuite struct {
+	suite.Suite
+	connector *GraphConnector
+	user      string
+}
+
+func TestConnectorCreateSharePointCollectionIntegrationSuite(t *testing.T) {
+	if err := tester.RunOnAny(
+		tester.CorsoCITests,
+		tester.CorsoConnectorCreateSharePointCollectionTests,
+	); err != nil {
+		t.Skip(err)
+	}
+
+	suite.Run(t, new(ConnectorCreateSharePointCollectionIntegrationSuite))
+}
+
+func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) SetupSuite() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	_, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...)
+	require.NoError(suite.T(), err)
+	suite.connector = loadConnector(ctx, suite.T())
+	suite.user = tester.M365UserID(suite.T())
+	tester.LogTimeOfTest(suite.T())
+}
+
+func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateSharePointCollection() {
+	ctx, flush := tester.NewContext()
+	defer flush()
+
+	var (
+		t      = suite.T()
+		userID = tester.M365UserID(t)
+	)
+
+	gc := loadConnector(ctx, t)
+	scope := selectors.NewSharePointBackup().Folders(
+		[]string{userID},
+		[]string{"foo"},
+		selectors.PrefixMatch(),
+	)[0]
+
+	_, err := gc.createSharePointCollections(ctx, scope)
+	require.NoError(t, err)
+}

src/internal/connector/exchange/cache_container.go

@@ -2,66 +2,8 @@ package exchange
 
 import (
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/pkg/errors"
-
-	"github.com/alcionai/corso/src/internal/connector/graph"
-	"github.com/alcionai/corso/src/pkg/path"
 )
 
-// checkIDAndName is a helper function to ensure that
-// the ID and name pointers are set prior to being called.
-func checkIDAndName(c graph.Container) error {
-	idPtr := c.GetId()
-	if idPtr == nil || len(*idPtr) == 0 {
-		return errors.New("folder without ID")
-	}
-
-	ptr := c.GetDisplayName()
-	if ptr == nil || len(*ptr) == 0 {
-		return errors.Errorf("folder %s without display name", *idPtr)
-	}
-
-	return nil
-}
-
-// checkRequiredValues is a helper function to ensure that
-// all the pointers are set prior to being called.
-func checkRequiredValues(c graph.Container) error {
-	if err := checkIDAndName(c); err != nil {
-		return err
-	}
-
-	ptr := c.GetParentFolderId()
-	if ptr == nil || len(*ptr) == 0 {
-		return errors.Errorf("folder %s without parent ID", *c.GetId())
-	}
-
-	return nil
-}
-
-//======================================
-// cachedContainer Implementations
-//======================
-
-var _ graph.CachedContainer = &cacheFolder{}
-
-type cacheFolder struct {
-	graph.Container
-	p *path.Builder
-}
-
-//=========================================
-// Required Functions to satisfy interfaces
-//=====================================
-
-func (cf cacheFolder) Path() *path.Builder {
-	return cf.p
-}
-
-func (cf *cacheFolder) SetPath(newPath *path.Builder) {
-	cf.p = newPath
-}
-
 // CalendarDisplayable is a transformative struct that aligns
 // models.Calendarable interface with the container interface.
 // Calendars do not have a parentFolderID. Therefore,

src/internal/connector/exchange/contact_folder_cache.go

@@ -12,10 +12,10 @@ import (
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
-var _ graph.ContainerResolver = &contactFolderCache{}
+var _ graph.ContainerPopulater = &contactFolderCache{}
 
 type contactFolderCache struct {
-	*containerResolver
+	*graph.ContainerCache
 	gs     graph.Service
 	userID string
 }
@@ -44,12 +44,11 @@ func (cfc *contactFolderCache) populateContactRoot(
 			"fetching root contact folder: "+support.ConnectorStackErrorTrace(err))
 	}
 
-	temp := cacheFolder{
+	temp := graph.NewCacheFolder(
-		Container: f,
+		f,
-		p:         path.Builder{}.Append(baseContainerPath...),
+		path.Builder{}.Append(baseContainerPath...))
-	}
 
-	if err := cfc.addFolder(temp); err != nil {
+	if err := cfc.ContainerCache.AddFolder(temp); err != nil {
 		return errors.Wrap(err, "adding cache root")
 	}
 
@@ -105,11 +104,11 @@ func (cfc *contactFolderCache) Populate(
 	}
 
 	for _, entry := range containers {
-		temp := cacheFolder{
+		temp := graph.CacheFolder{
 			Container: entry,
 		}
 
-		err = cfc.addFolder(temp)
+		err = cfc.ContainerCache.AddFolder(temp)
 		if err != nil {
 			errs = support.WrapAndAppend(
 				"cache build in cfc.Populate",
@@ -118,7 +117,7 @@ func (cfc *contactFolderCache) Populate(
 		}
 	}
 
-	if err := cfc.populatePaths(ctx); err != nil {
+	if err := cfc.ContainerCache.PopulatePaths(ctx); err != nil {
 		errs = support.WrapAndAppend(
 			"contacts resolver",
 			err,
@@ -138,8 +137,8 @@ func (cfc *contactFolderCache) init(
 		return errors.New("m365 folderID required for base folder")
 	}
 
-	if cfc.containerResolver == nil {
+	if cfc.ContainerCache == nil {
-		cfc.containerResolver = newContainerResolver()
+		cfc.ContainerCache = graph.NewContainerCache()
 	}
 
 	return cfc.populateContactRoot(ctx, baseNode, baseContainerPath)

src/internal/connector/exchange/event_calendar_cache.go

@@ -12,10 +12,10 @@ import (
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
-var _ graph.ContainerResolver = &eventCalendarCache{}
+var _ graph.ContainerPopulater = &eventCalendarCache{}
 
 type eventCalendarCache struct {
-	*containerResolver
+	*graph.ContainerCache
 	gs     graph.Service
 	userID string
 }
@@ -28,8 +28,8 @@ func (ecc *eventCalendarCache) Populate(
 	baseID string,
 	baseContainerPath ...string,
 ) error {
-	if ecc.containerResolver == nil {
+	if ecc.ContainerCache == nil {
-		ecc.containerResolver = newContainerResolver()
+		ecc.ContainerCache = graph.NewContainerCache()
 	}
 
 	options, err := optionsForCalendars([]string{"name"})
@@ -76,7 +76,7 @@ func (ecc *eventCalendarCache) Populate(
 	}
 
 	for _, container := range directories {
-		if err := checkIDAndName(container); err != nil {
+		if err := graph.CheckIDAndName(container); err != nil {
 			iterateErr = support.WrapAndAppend(
 				"adding folder to cache",
 				err,
@@ -86,12 +86,11 @@ func (ecc *eventCalendarCache) Populate(
 			continue
 		}
 
-		temp := cacheFolder{
+		temp := graph.NewCacheFolder(
-			Container: container,
+			container,
-			p:         path.Builder{}.Append(*container.GetDisplayName()),
+			path.Builder{}.Append(*container.GetDisplayName()))
-		}
 
-		if err := ecc.addFolder(temp); err != nil {
+		if err := ecc.ContainerCache.AddFolder(temp); err != nil {
 			iterateErr = support.WrapAndAppend(
 				"failure adding "+*container.GetDisplayName(),
 				err,
@@ -104,23 +103,22 @@ func (ecc *eventCalendarCache) Populate(
 
 // AddToCache adds container to map in field 'cache'
 // @returns error iff the required values are not accessible.
-func (ecc *eventCalendarCache) AddToCache(ctx context.Context, f graph.Container) error {
+func (ecc *eventCalendarCache) AddToCache(ctx context.Context, c graph.Container) error {
-	if err := checkIDAndName(f); err != nil {
+	if err := graph.CheckIDAndName(c); err != nil {
 		return errors.Wrap(err, "adding cache folder")
 	}
 
-	temp := cacheFolder{
+	temp := graph.NewCacheFolder(
-		Container: f,
+		c,
-		p:         path.Builder{}.Append(*f.GetDisplayName()),
+		path.Builder{}.Append(*c.GetDisplayName()))
-	}
 
-	if err := ecc.addFolder(temp); err != nil {
+	if err := ecc.ContainerCache.AddFolder(temp); err != nil {
 		return errors.Wrap(err, "adding cache folder")
 	}
 
 	// Populate the path for this entry so calls to PathInCache succeed no matter
 	// when they're made.
-	_, err := ecc.IDToPath(ctx, *f.GetId())
+	_, err := ecc.ContainerCache.IDToPath(ctx, *c.GetId())
 	if err != nil {
 		return errors.Wrap(err, "adding cache entry")
 	}

src/internal/connector/exchange/exchange_service_test.go

@@ -494,7 +494,7 @@ func (suite *ExchangeServiceSuite) TestGetContainerIDFromCache() {
 		t               = suite.T()
 		user            = tester.M365UserID(t)
 		connector       = loadService(t)
-		directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
+		directoryCaches = make(map[path.CategoryType]graph.ContainerPopulater)
 		folderName      = tester.DefaultTestRestoreDestination().ContainerName
 		tests           = []struct {
 			name      string

src/internal/connector/exchange/folder_resolver_test.go

@@ -50,14 +50,14 @@ func (suite *CacheResolverSuite) TestPopulate() {
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	eventFunc := func(t *testing.T) graph.ContainerResolver {
+	eventFunc := func(t *testing.T) graph.ContainerPopulater {
 		return &eventCalendarCache{
 			userID: tester.M365UserID(t),
 			gs:     suite.gs,
 		}
 	}
 
-	contactFunc := func(t *testing.T) graph.ContainerResolver {
+	contactFunc := func(t *testing.T) graph.ContainerPopulater {
 		return &contactFolderCache{
 			userID: tester.M365UserID(t),
 			gs:     suite.gs,
@@ -66,7 +66,7 @@ func (suite *CacheResolverSuite) TestPopulate() {
 
 	tests := []struct {
 		name, folderName, root, basePath string
-		resolverFunc                     func(t *testing.T) graph.ContainerResolver
+		resolverFunc                     func(t *testing.T) graph.ContainerPopulater
 		canFind                          assert.BoolAssertionFunc
 	}{
 		{

src/internal/connector/exchange/mail_folder_cache.go

@@ -12,13 +12,13 @@ import (
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
-var _ graph.ContainerResolver = &mailFolderCache{}
+var _ graph.ContainerPopulater = &mailFolderCache{}
 
 // mailFolderCache struct used to improve lookup of directories within exchange.Mail
 // cache map of cachedContainers where the  key =  M365ID
 // nameLookup map: Key: DisplayName Value: ID
 type mailFolderCache struct {
-	*containerResolver
+	*graph.ContainerCache
 	gs     graph.Service
 	userID string
 }
@@ -51,12 +51,11 @@ func (mc *mailFolderCache) populateMailRoot(
 		return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err))
 	}
 
-	temp := cacheFolder{
+	temp := graph.NewCacheFolder(
-		Container: f,
+		f,
-		p:         path.Builder{}.Append(baseContainerPath...),
+		path.Builder{}.Append(baseContainerPath...))
-	}
 
-	if err := mc.addFolder(temp); err != nil {
+	if err := mc.ContainerCache.AddFolder(temp); err != nil {
 		return errors.Wrap(err, "initializing mail resolver")
 	}
 
@@ -95,14 +94,14 @@ func (mc *mailFolderCache) Populate(
 		}
 
 		for _, f := range resp.GetValue() {
-			temp := cacheFolder{
+			temp := graph.CacheFolder{
 				Container: f,
 			}
 
 			// Use addFolder instead of AddToCache to be conservative about path
 			// population. The fetch order of the folders could cause failures while
 			// trying to resolve paths, so put it off until we've gotten all folders.
-			if err := mc.addFolder(temp); err != nil {
+			if err := mc.ContainerCache.AddFolder(temp); err != nil {
 				errs = multierror.Append(errs, errors.Wrap(err, "delta fetch"))
 				continue
 			}
@@ -119,7 +118,7 @@ func (mc *mailFolderCache) Populate(
 		query = msfolderdelta.NewDeltaRequestBuilder(link, mc.gs.Adapter())
 	}
 
-	if err := mc.populatePaths(ctx); err != nil {
+	if err := mc.ContainerCache.PopulatePaths(ctx); err != nil {
 		errs = multierror.Append(errs, errors.Wrap(err, "mail resolver"))
 	}
 
@@ -138,8 +137,8 @@ func (mc *mailFolderCache) init(
 		return errors.New("m365 folder ID required for base folder")
 	}
 
-	if mc.containerResolver == nil {
+	if mc.ContainerCache == nil {
-		mc.containerResolver = newContainerResolver()
+		mc.ContainerCache = graph.NewContainerCache()
 	}
 
 	return mc.populateMailRoot(ctx, baseNode, baseContainerPath)

src/internal/connector/exchange/service_functions.go

@@ -26,6 +26,7 @@ type exchangeService struct {
 ///------------------------------------------------------------
 // Functions to comply with graph.Service Interface
 //-------------------------------------------------------
+
 func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient {
 	return &es.client
 }
@@ -137,10 +138,11 @@ func GetAllMailFolders(
 	ctx context.Context,
 	qp graph.QueryParams,
 	gs graph.Service,
+	scope selectors.ExchangeScope,
 ) ([]graph.CachedContainer, error) {
 	containers := make([]graph.CachedContainer, 0)
 
-	resolver, err := PopulateExchangeContainerResolver(ctx, qp, path.EmailCategory)
+	resolver, err := PopulateExchangeContainerResolver(ctx, qp)
 	if err != nil {
 		return nil, errors.Wrap(err, "building directory resolver in GetAllMailFolders")
 	}
@@ -151,7 +153,7 @@ func GetAllMailFolders(
 			continue
 		}
 
-		if qp.Scope.Matches(selectors.ExchangeMailFolder, directory) {
+		if scope.Matches(selectors.ExchangeMailFolder, directory) {
 			containers = append(containers, c)
 		}
 	}
@@ -166,10 +168,11 @@ func GetAllCalendars(
 	ctx context.Context,
 	qp graph.QueryParams,
 	gs graph.Service,
+	scope selectors.ExchangeScope,
 ) ([]graph.CachedContainer, error) {
 	containers := make([]graph.CachedContainer, 0)
 
-	resolver, err := PopulateExchangeContainerResolver(ctx, qp, path.EventsCategory)
+	resolver, err := PopulateExchangeContainerResolver(ctx, qp)
 	if err != nil {
 		return nil, errors.Wrap(err, "building calendar resolver in GetAllCalendars")
 	}
@@ -177,7 +180,7 @@ func GetAllCalendars(
 	for _, c := range resolver.Items() {
 		directory := c.Path().String()
 
-		if qp.Scope.Matches(selectors.ExchangeEventCalendar, directory) {
+		if scope.Matches(selectors.ExchangeEventCalendar, directory) {
 			containers = append(containers, c)
 		}
 	}
@@ -193,12 +196,13 @@ func GetAllContactFolders(
 	ctx context.Context,
 	qp graph.QueryParams,
 	gs graph.Service,
+	scope selectors.ExchangeScope,
 ) ([]graph.CachedContainer, error) {
 	var query string
 
 	containers := make([]graph.CachedContainer, 0)
 
-	resolver, err := PopulateExchangeContainerResolver(ctx, qp, path.ContactsCategory)
+	resolver, err := PopulateExchangeContainerResolver(ctx, qp)
 	if err != nil {
 		return nil, errors.Wrap(err, "building directory resolver in GetAllContactFolders")
 	}
@@ -212,7 +216,7 @@ func GetAllContactFolders(
 			query = directory
 		}
 
-		if qp.Scope.Matches(selectors.ExchangeContactFolder, query) {
+		if scope.Matches(selectors.ExchangeContactFolder, query) {
 			containers = append(containers, c)
 		}
 	}
@@ -220,25 +224,6 @@ func GetAllContactFolders(
 	return containers, err
 }
 
-func GetContainers(
-	ctx context.Context,
-	qp graph.QueryParams,
-	gs graph.Service,
-) ([]graph.CachedContainer, error) {
-	category := qp.Scope.Category().PathType()
-
-	switch category {
-	case path.ContactsCategory:
-		return GetAllContactFolders(ctx, qp, gs)
-	case path.EmailCategory:
-		return GetAllMailFolders(ctx, qp, gs)
-	case path.EventsCategory:
-		return GetAllCalendars(ctx, qp, gs)
-	default:
-		return nil, fmt.Errorf("path.Category %s not supported", category)
-	}
-}
-
 // PopulateExchangeContainerResolver gets a folder resolver if one is available for
 // this category of data. If one is not available, returns nil so that other
 // logic in the caller can complete as long as they check if the resolver is not
@@ -246,10 +231,9 @@ func GetContainers(
 func PopulateExchangeContainerResolver(
 	ctx context.Context,
 	qp graph.QueryParams,
-	category path.CategoryType,
 ) (graph.ContainerResolver, error) {
 	var (
-		res          graph.ContainerResolver
+		res          graph.ContainerPopulater
 		cacheRoot    string
 		service, err = createService(qp.Credentials, qp.FailFast)
 	)
@@ -258,30 +242,30 @@ func PopulateExchangeContainerResolver(
 		return nil, err
 	}
 
-	switch category {
+	switch qp.Category {
 	case path.EmailCategory:
 		res = &mailFolderCache{
-			userID: qp.User,
+			userID: qp.ResourceOwner,
 			gs:     service,
 		}
 		cacheRoot = rootFolderAlias
 
 	case path.ContactsCategory:
 		res = &contactFolderCache{
-			userID: qp.User,
+			userID: qp.ResourceOwner,
 			gs:     service,
 		}
 		cacheRoot = DefaultContactFolder
 
 	case path.EventsCategory:
 		res = &eventCalendarCache{
-			userID: qp.User,
+			userID: qp.ResourceOwner,
 			gs:     service,
 		}
 		cacheRoot = DefaultCalendar
 
 	default:
-		return nil, fmt.Errorf("ContainerResolver not present for %s type", category)
+		return nil, fmt.Errorf("ContainerResolver not present for %s type", qp.Category)
 	}
 
 	if err := res.Populate(ctx, cacheRoot); err != nil {
@@ -291,8 +275,13 @@ func PopulateExchangeContainerResolver(
 	return res, nil
 }
 
-func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.CachedContainer) (path.Path, bool) {
+func pathAndMatch(
+	qp graph.QueryParams,
+	c graph.CachedContainer,
+	scope selectors.ExchangeScope,
+) (path.Path, bool) {
 	var (
+		category  = scope.Category().PathType()
 		directory string
 		pb        = c.Path()
 	)
@@ -304,7 +293,7 @@ func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.Cach
 
 	dirPath, err := pb.ToDataLayerExchangePathForCategory(
 		qp.Credentials.AzureTenantID,
-		qp.User,
+		qp.ResourceOwner,
 		category,
 		false,
 	)
@@ -320,11 +309,11 @@ func pathAndMatch(qp graph.QueryParams, category path.CategoryType, c graph.Cach
 
 	switch category {
 	case path.EmailCategory:
-		return dirPath, qp.Scope.Matches(selectors.ExchangeMailFolder, directory)
+		return dirPath, scope.Matches(selectors.ExchangeMailFolder, directory)
 	case path.ContactsCategory:
-		return dirPath, qp.Scope.Matches(selectors.ExchangeContactFolder, directory)
+		return dirPath, scope.Matches(selectors.ExchangeContactFolder, directory)
 	case path.EventsCategory:
-		return dirPath, qp.Scope.Matches(selectors.ExchangeEventCalendar, directory)
+		return dirPath, scope.Matches(selectors.ExchangeEventCalendar, directory)
 	default:
 		return nil, false
 	}

src/internal/connector/exchange/service_functions_test.go

@@ -115,13 +115,14 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
+			scope := test.getScope(t)
 			params := graph.QueryParams{
-				User:        test.user,
+				Category:      scope.Category().PathType(),
-				Scope:       test.getScope(t),
+				ResourceOwner: test.user,
 				FailFast:      false,
 				Credentials:   suite.creds,
 			}
-			cals, err := GetAllCalendars(ctx, params, gs)
+			cals, err := GetAllCalendars(ctx, params, gs, scope)
 			test.expectErr(t, err)
 			test.expectCount(t, len(cals), 0)
 		})
@@ -199,13 +200,14 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
+			scope := test.getScope(t)
 			params := graph.QueryParams{
-				User:        test.user,
+				Category:      scope.Category().PathType(),
-				Scope:       test.getScope(t),
+				ResourceOwner: test.user,
 				FailFast:      false,
 				Credentials:   suite.creds,
 			}
-			cals, err := GetAllContactFolders(ctx, params, gs)
+			cals, err := GetAllContactFolders(ctx, params, gs, scope)
 			test.expectErr(t, err)
 			test.expectCount(t, len(cals), 0)
 		})
@@ -283,84 +285,16 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
+			scope := test.getScope(t)
 			params := graph.QueryParams{
-				User:        test.user,
+				Category:      scope.Category().PathType(),
-				Scope:       test.getScope(t),
+				ResourceOwner: test.user,
 				FailFast:      false,
 				Credentials:   suite.creds,
 			}
-			cals, err := GetAllMailFolders(ctx, params, gs)
+			cals, err := GetAllMailFolders(ctx, params, gs, scope)
 			test.expectErr(t, err)
 			test.expectCount(t, len(cals), 0)
 		})
 	}
 }
-
-func (suite *ServiceFunctionsIntegrationSuite) TestCollectContainers() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	failFast := false
-	containerCount := 1
-	t := suite.T()
-	user := tester.M365UserID(t)
-	a := tester.NewM365Account(t)
-	service := loadService(t)
-	credentials, err := a.M365Config()
-	require.NoError(t, err)
-
-	tests := []struct {
-		name, contains string
-		getScope       func() selectors.ExchangeScope
-		expectedCount  assert.ComparisonAssertionFunc
-	}{
-		{
-			name:          "All Events",
-			contains:      "Birthdays",
-			expectedCount: assert.Greater,
-			getScope: func() selectors.ExchangeScope {
-				return selectors.
-					NewExchangeBackup().
-					EventCalendars([]string{user}, selectors.Any())[0]
-			},
-		}, {
-			name:          "Default Calendar",
-			contains:      DefaultCalendar,
-			expectedCount: assert.Equal,
-			getScope: func() selectors.ExchangeScope {
-				return selectors.
-					NewExchangeBackup().
-					EventCalendars([]string{user}, []string{DefaultCalendar}, selectors.PrefixMatch())[0]
-			},
-		}, {
-			name:          "Default Mail",
-			contains:      DefaultMailFolder,
-			expectedCount: assert.Equal,
-			getScope: func() selectors.ExchangeScope {
-				return selectors.
-					NewExchangeBackup().
-					MailFolders([]string{user}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0]
-			},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			qp := graph.QueryParams{
-				User:        user,
-				Scope:       test.getScope(),
-				FailFast:    failFast,
-				Credentials: credentials,
-			}
-			collections, err := GetContainers(ctx, qp, service)
-			assert.NoError(t, err)
-			test.expectedCount(t, len(collections), containerCount)
-
-			keys := make([]string, 0, len(collections))
-			for _, k := range collections {
-				keys = append(keys, *k.GetDisplayName())
-			}
-			assert.Contains(t, keys, test.contains)
-		})
-	}
-}

src/internal/connector/exchange/service_iterators.go

@@ -13,6 +13,7 @@ import (
 	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 // FilterContainersAndFillCollections is a utility function
@@ -26,21 +27,21 @@ func FilterContainersAndFillCollections(
 	collections map[string]*Collection,
 	statusUpdater support.StatusUpdater,
 	resolver graph.ContainerResolver,
+	scope selectors.ExchangeScope,
 ) error {
 	var (
-		category       = qp.Scope.Category().PathType()
+		collectionType = CategoryToOptionIdentifier(scope.Category().PathType())
-		collectionType = CategoryToOptionIdentifier(category)
 		errs           error
 	)
 
 	for _, c := range resolver.Items() {
-		dirPath, ok := pathAndMatch(qp, category, c)
+		dirPath, ok := pathAndMatch(qp, c, scope)
 		if ok {
 			// Create only those that match
 			service, err := createService(qp.Credentials, qp.FailFast)
 			if err != nil {
 				errs = support.WrapAndAppend(
-					qp.User+" FilterContainerAndFillCollection",
+					qp.ResourceOwner+" FilterContainerAndFillCollection",
 					err,
 					errs)
 
@@ -50,7 +51,7 @@ func FilterContainersAndFillCollections(
 			}
 
 			edc := NewCollection(
-				qp.User,
+				qp.ResourceOwner,
 				dirPath,
 				collectionType,
 				service,
@@ -61,10 +62,10 @@ func FilterContainersAndFillCollections(
 	}
 
 	for directoryID, col := range collections {
-		fetchFunc, err := getFetchIDFunc(category)
+		fetchFunc, err := getFetchIDFunc(scope.Category().PathType())
 		if err != nil {
 			errs = support.WrapAndAppend(
-				qp.User,
+				qp.ResourceOwner,
 				err,
 				errs)
 
@@ -75,10 +76,10 @@ func FilterContainersAndFillCollections(
 			continue
 		}
 
-		jobs, err := fetchFunc(ctx, col.service, qp.User, directoryID)
+		jobs, err := fetchFunc(ctx, col.service, qp.ResourceOwner, directoryID)
 		if err != nil {
 			errs = support.WrapAndAppend(
-				qp.User,
+				qp.ResourceOwner,
 				err,
 				errs,
 			)

src/internal/connector/exchange/service_restore.go

@@ -292,7 +292,7 @@ func RestoreExchangeDataCollections(
 ) (*support.ConnectorOperationStatus, error) {
 	var (
 		// map of caches... but not yet...
-		directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerResolver)
+		directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerPopulater)
 		metrics         support.CollectionMetrics
 		errs            error
 		// TODO policy to be updated from external source after completion of refactoring
@@ -308,7 +308,7 @@ func RestoreExchangeDataCollections(
 
 		userCaches := directoryCaches[userID]
 		if userCaches == nil {
-			directoryCaches[userID] = make(map[path.CategoryType]graph.ContainerResolver)
+			directoryCaches[userID] = make(map[path.CategoryType]graph.ContainerPopulater)
 			userCaches = directoryCaches[userID]
 		}
 
@@ -432,7 +432,7 @@ func GetContainerIDFromCache(
 	gs graph.Service,
 	directory path.Path,
 	destination string,
-	caches map[path.CategoryType]graph.ContainerResolver,
+	caches map[path.CategoryType]graph.ContainerPopulater,
 ) (string, error) {
 	var (
 		newCache       = false
@@ -512,7 +512,7 @@ func GetContainerIDFromCache(
 func establishMailRestoreLocation(
 	ctx context.Context,
 	folders []string,
-	mfc graph.ContainerResolver,
+	mfc graph.ContainerPopulater,
 	user string,
 	service graph.Service,
 	isNewCache bool,
@@ -569,7 +569,7 @@ func establishMailRestoreLocation(
 func establishContactsRestoreLocation(
 	ctx context.Context,
 	folders []string,
-	cfc graph.ContainerResolver,
+	cfc graph.ContainerPopulater,
 	user string,
 	gs graph.Service,
 	isNewCache bool,
@@ -602,7 +602,7 @@ func establishContactsRestoreLocation(
 func establishEventsRestoreLocation(
 	ctx context.Context,
 	folders []string,
-	ecc graph.ContainerResolver, // eventCalendarCache
+	ecc graph.ContainerPopulater, // eventCalendarCache
 	user string,
 	gs graph.Service,
 	isNewCache bool,

src/internal/connector/graph/cache_container.go

@@ -1,7 +1,6 @@
 package graph
 
 import (
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/src/pkg/path"
@@ -16,9 +15,9 @@ type CachedContainer interface {
 	SetPath(*path.Builder)
 }
 
-// checkRequiredValues is a helper function to ensure that
+// CheckIDAndName is a helper function to ensure that
-// all the pointers are set prior to being called.
+// the ID and name pointers are set prior to being called.
-func CheckRequiredValues(c Container) error {
+func CheckIDAndName(c Container) error {
 	idPtr := c.GetId()
 	if idPtr == nil || len(*idPtr) == 0 {
 		return errors.New("folder without ID")
@@ -29,9 +28,19 @@ func CheckRequiredValues(c Container) error {
 		return errors.Errorf("folder %s without display name", *idPtr)
 	}
 
-	ptr = c.GetParentFolderId()
+	return nil
+}
+
+// CheckRequiredValues is a helper function to ensure that
+// all the pointers are set prior to being called.
+func CheckRequiredValues(c Container) error {
+	if err := CheckIDAndName(c); err != nil {
+		return err
+	}
+
+	ptr := c.GetParentFolderId()
 	if ptr == nil || len(*ptr) == 0 {
-		return errors.Errorf("folder %s without parent ID", *idPtr)
+		return errors.Errorf("folder %s without parent ID", *c.GetId())
 	}
 
 	return nil
@@ -58,10 +67,6 @@ func NewCacheFolder(c Container, pb *path.Builder) CacheFolder {
 	return cf
 }
 
-//=========================================
-// Required Functions to satisfy interfaces
-//=========================================
-
 func (cf CacheFolder) Path() *path.Builder {
 	return cf.p
 }
@@ -69,41 +74,3 @@ func (cf CacheFolder) Path() *path.Builder {
 func (cf *CacheFolder) SetPath(newPath *path.Builder) {
 	cf.p = newPath
 }
-
-// CalendarDisplayable is a transformative struct that aligns
-// models.Calendarable interface with the container interface.
-// Calendars do not have the 2 of the
-type CalendarDisplayable struct {
-	models.Calendarable
-	parentID string
-}
-
-// GetDisplayName returns the *string of the calendar name
-func (c CalendarDisplayable) GetDisplayName() *string {
-	return c.GetName()
-}
-
-// GetParentFolderId returns the default calendar name address
-// EventCalendars have a flat hierarchy and Calendars are rooted
-// at the default
-//nolint:revive
-func (c CalendarDisplayable) GetParentFolderId() *string {
-	return &c.parentID
-}
-
-// CreateCalendarDisplayable helper function to create the
-// calendarDisplayable during msgraph-sdk-go iterative process
-// @param entry is the input supplied by pageIterator.Iterate()
-// @param parentID of Calendar sets. Only populate when used with
-// EventCalendarCache
-func CreateCalendarDisplayable(entry any, parentID string) *CalendarDisplayable {
-	calendar, ok := entry.(models.Calendarable)
-	if !ok {
-		return nil
-	}
-
-	return &CalendarDisplayable{
-		Calendarable: calendar,
-		parentID:     parentID,
-	}
-}

src/internal/connector/graph/cache_resolver.go

@@ -1,4 +1,4 @@
-package exchange
+package graph
 
 import (
 	"context"
@@ -6,21 +6,28 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 
-	"github.com/alcionai/corso/src/internal/connector/graph"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
-func newContainerResolver() *containerResolver {
+var _ ContainerResolver = &ContainerCache{}
-	return &containerResolver{
+
-		cache: map[string]graph.CachedContainer{},
+type populatorFunc func(
+	ctx context.Context,
+	baseID string,
+	baseContainerPath ...string,
+)
+
+type ContainerCache struct {
+	cache map[string]CachedContainer
+}
+
+func NewContainerCache() *ContainerCache {
+	return &ContainerCache{
+		cache: map[string]CachedContainer{},
 	}
 }
 
-type containerResolver struct {
+func (cr *ContainerCache) IDToPath(
-	cache map[string]graph.CachedContainer
-}
-
-func (cr *containerResolver) IDToPath(
 	ctx context.Context,
 	folderID string,
 ) (*path.Builder, error) {
@@ -48,7 +55,7 @@ func (cr *containerResolver) IDToPath(
 // PathInCache utility function to return m365ID of folder if the pathString
 // matches the path of a container within the cache. A boolean function
 // accompanies the call to indicate whether the lookup was successful.
-func (cr *containerResolver) PathInCache(pathString string) (string, bool) {
+func (cr *ContainerCache) PathInCache(pathString string) (string, bool) {
 	if len(pathString) == 0 || cr == nil {
 		return "", false
 	}
@@ -66,17 +73,17 @@ func (cr *containerResolver) PathInCache(pathString string) (string, bool) {
 	return "", false
 }
 
-// addFolder adds a folder to the cache with the given ID. If the item is
+// AddFolder adds a folder to the cache with the given ID. If the item is
 // already in the cache does nothing. The path for the item is not modified.
-func (cr *containerResolver) addFolder(cf cacheFolder) error {
+func (cr *ContainerCache) AddFolder(cf CacheFolder) error {
 	// Only require a non-nil non-empty parent if the path isn't already
 	// populated.
 	if cf.p != nil {
-		if err := checkIDAndName(cf.Container); err != nil {
+		if err := CheckIDAndName(cf.Container); err != nil {
 			return errors.Wrap(err, "adding item to cache")
 		}
 	} else {
-		if err := checkRequiredValues(cf.Container); err != nil {
+		if err := CheckRequiredValues(cf.Container); err != nil {
 			return errors.Wrap(err, "adding item to cache")
 		}
 	}
@@ -90,8 +97,9 @@ func (cr *containerResolver) addFolder(cf cacheFolder) error {
 	return nil
 }
 
-func (cr *containerResolver) Items() []graph.CachedContainer {
+// Items returns the list of Containers in the cache.
-	res := make([]graph.CachedContainer, 0, len(cr.cache))
+func (cr *ContainerCache) Items() []CachedContainer {
+	res := make([]CachedContainer, 0, len(cr.cache))
 
 	for _, c := range cr.cache {
 		res = append(res, c)
@@ -102,12 +110,12 @@ func (cr *containerResolver) Items() []graph.CachedContainer {
 
 // AddToCache adds container to map in field 'cache'
 // @returns error iff the required values are not accessible.
-func (cr *containerResolver) AddToCache(ctx context.Context, f graph.Container) error {
+func (cr *ContainerCache) AddToCache(ctx context.Context, f Container) error {
-	temp := cacheFolder{
+	temp := CacheFolder{
 		Container: f,
 	}
 
-	if err := cr.addFolder(temp); err != nil {
+	if err := cr.AddFolder(temp); err != nil {
 		return errors.Wrap(err, "adding cache folder")
 	}
 
@@ -121,10 +129,10 @@ func (cr *containerResolver) AddToCache(ctx context.Context, f graph.Container)
 	return nil
 }
 
-func (cr *containerResolver) populatePaths(ctx context.Context) error {
+// PopulatePaths ensures that all items in the cache can construct valid paths.
+func (cr *ContainerCache) PopulatePaths(ctx context.Context) error {
 	var errs *multierror.Error
 
-	// Populate all folder paths.
 	for _, f := range cr.Items() {
 		_, err := cr.IDToPath(ctx, *f.GetId())
 		if err != nil {

src/internal/connector/graph/service.go

@@ -7,12 +7,11 @@ import (
 
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 type QueryParams struct {
-	User        string
+	Category      path.CategoryType
-	Scope       selectors.ExchangeScope
+	ResourceOwner string
 	Credentials   account.M365Config
 	FailFast      bool
 }
@@ -61,11 +60,6 @@ type ContainerResolver interface {
 	// to that container. The path has a similar format to paths on the local
 	// file system.
 	IDToPath(ctx context.Context, m365ID string) (*path.Builder, error)
-	// Populate performs initialization steps for the resolver
-	// @param ctx is necessary param for Graph API tracing
-	// @param baseFolderID represents the M365ID base that the resolver will
-	// conclude its search. Default input is "".
-	Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
 
 	// PathInCache performs a look up of a path reprensentation
 	// and returns the m365ID of directory iff the pathString
@@ -78,3 +72,16 @@ type ContainerResolver interface {
 	// Items returns the containers in the cache.
 	Items() []CachedContainer
 }
+
+// ContainerPopulater houses functions for populating and retrieving info
+// about containers from remote APIs (i.e. resolve folder paths with Graph
+// API). Populaters may cache information about containers.
+type ContainerPopulater interface {
+	ContainerResolver
+
+	// Populate performs initialization steps for the populater
+	// @param ctx is necessary param for Graph API tracing
+	// @param baseFolderID represents the M365ID base that the
+	// populater will conclude its search. Default input is "".
+	Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
+}

src/internal/connector/graph_connector.go

@@ -6,10 +6,8 @@ import (
 	"context"
 	"fmt"
 	"runtime/trace"
-	"strings"
 	"sync"
 
-	"github.com/hashicorp/go-multierror"
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
 	msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
@@ -21,14 +19,16 @@ import (
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
 	D "github.com/alcionai/corso/src/internal/diagnostics"
-	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
-	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
+// ---------------------------------------------------------------------------
+// Graph Connector
+// ---------------------------------------------------------------------------
+
 // GraphConnector is a struct used to wrap the GraphServiceClient and
 // GraphRequestAdapter from the msgraph-sdk-go. Additional fields are for
 // bookkeeping and interfacing with other component.
@@ -36,6 +36,7 @@ type GraphConnector struct {
 	graphService
 	tenant      string
 	Users       map[string]string // key<email> value<id>
+	Sites       map[string]string // key<???> value<???>
 	credentials account.M365Config
 
 	// wg is used to track completion of GC tasks
@@ -97,6 +98,12 @@ func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnect
 		return nil, errors.Wrap(err, "retrieving tenant user list")
 	}
 
+	// TODO: users or sites, one or the other, not both.
+	err = gc.setTenantSites(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "retrieveing tenant site list")
+	}
+
 	return &gc, nil
 }
 
@@ -126,7 +133,7 @@ func (gs *graphService) EnableFailFast() {
 
 // setTenantUsers queries the M365 to identify the users in the
 // workspace. The users field is updated during this method
-// iff the return value is true
+// iff the return value is nil
 func (gc *GraphConnector) setTenantUsers(ctx context.Context) error {
 	ctx, end := D.Span(ctx, "gc:setTenantUsers")
 	defer end()
@@ -191,6 +198,79 @@ func (gc *GraphConnector) GetUsersIds() []string {
 	return buildFromMap(false, gc.Users)
 }
 
+// setTenantSites queries the M365 to identify the sites in the
+// workspace. The sitets field is updated during this method
+// iff the return value is nil
+func (gc *GraphConnector) setTenantSites(ctx context.Context) error {
+	// TODO
+	gc.Sites = map[string]string{}
+
+	// ctx, end := D.Span(ctx, "gc:setTenantSites")
+	// defer end()
+
+	// response, err := exchange.GetAllUsersForTenant(ctx, gc.graphService, "")
+	// if err != nil {
+	// 	return errors.Wrapf(
+	// 		err,
+	// 		"tenant %s M365 query: %s",
+	// 		gc.tenant,
+	// 		support.ConnectorStackErrorTrace(err),
+	// 	)
+	// }
+
+	// userIterator, err := msgraphgocore.NewPageIterator(
+	// 	response,
+	// 	&gc.graphService.adapter,
+	// 	models.CreateUserCollectionResponseFromDiscriminatorValue,
+	// )
+	// if err != nil {
+	// 	return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
+	// }
+
+	// callbackFunc := func(userItem interface{}) bool {
+	// 	user, ok := userItem.(models.Userable)
+	// 	if !ok {
+	// 		err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(),
+	//  errors.New("received non-User on iteration"), err)
+	// 		return true
+	// 	}
+
+	// 	if user.GetUserPrincipalName() == nil {
+	// 		err = support.WrapAndAppend(
+	// 			gc.graphService.adapter.GetBaseUrl(),
+	// 			fmt.Errorf("no email address for User: %s", *user.GetId()),
+	// 			err,
+	// 		)
+
+	// 		return true
+	// 	}
+
+	// 	// *user.GetId() is populated for every M365 entityable object by M365 backstore
+	// 	gc.Users[*user.GetUserPrincipalName()] = *user.GetId()
+
+	// 	return true
+	// }
+
+	// iterateError := userIterator.Iterate(ctx, callbackFunc)
+	// if iterateError != nil {
+	// 	err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err)
+	// }
+
+	// return err
+
+	return nil
+}
+
+// GetSites returns the siteIDs of sharepoint sites within tenant.
+func (gc *GraphConnector) GetSites() []string {
+	return buildFromMap(true, gc.Sites)
+}
+
+// GetSiteIds returns the M365 id for the user
+func (gc *GraphConnector) GetSiteIds() []string {
+	return buildFromMap(false, gc.Sites)
+}
+
 // buildFromMap helper function for returning []string from map.
 // Returns list of keys iff true; otherwise returns a list of values
 func buildFromMap(isKey bool, mapping map[string]string) []string {
@@ -209,42 +289,6 @@ func buildFromMap(isKey bool, mapping map[string]string) []string {
 	return returnString
 }
 
-// ExchangeDataStream returns a DataCollection which the caller can
-// use to read mailbox data out for the specified user
-// Assumption: User exists
-//
-//	Add iota to this call -> mail, contacts, calendar,  etc.
-func (gc *GraphConnector) ExchangeDataCollection(
-	ctx context.Context,
-	selector selectors.Selector,
-) ([]data.Collection, error) {
-	eb, err := selector.ToExchangeBackup()
-	if err != nil {
-		return nil, errors.Wrap(err, "exchangeDataCollection: parsing selector")
-	}
-
-	var (
-		scopes      = eb.DiscreteScopes(gc.GetUsers())
-		collections = []data.Collection{}
-		errs        error
-	)
-
-	for _, scope := range scopes {
-		// Creates a map of collections based on scope
-		dcs, err := gc.createCollections(ctx, scope)
-		if err != nil {
-			user := scope.Get(selectors.ExchangeUser)
-			return nil, support.WrapAndAppend(user[0], err, errs)
-		}
-
-		for _, collection := range dcs {
-			collections = append(collections, collection)
-		}
-	}
-
-	return collections, errs
-}
-
 // RestoreDataCollections restores data from the specified collections
 // into M365 using the GraphAPI.
 // SideEffect: gc.status is updated at the completion of operation
@@ -278,67 +322,6 @@ func (gc *GraphConnector) RestoreDataCollections(
 	return deets, err
 }
 
-// createCollections - utility function that retrieves M365
-// IDs through Microsoft Graph API. The selectors.ExchangeScope
-// determines the type of collections that are stored.
-// to the GraphConnector struct.
-func (gc *GraphConnector) createCollections(
-	ctx context.Context,
-	scope selectors.ExchangeScope,
-) ([]*exchange.Collection, error) {
-	var errs *multierror.Error
-
-	users := scope.Get(selectors.ExchangeUser)
-	allCollections := make([]*exchange.Collection, 0)
-	// Create collection of ExchangeDataCollection
-	for _, user := range users {
-		collections := make(map[string]*exchange.Collection)
-
-		qp := graph.QueryParams{
-			User:        user,
-			Scope:       scope,
-			FailFast:    gc.failFast,
-			Credentials: gc.credentials,
-		}
-
-		itemCategory := qp.Scope.Category().PathType()
-
-		foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", itemCategory.String(), user))
-		defer closer()
-		defer close(foldersComplete)
-
-		resolver, err := exchange.PopulateExchangeContainerResolver(
-			ctx,
-			qp,
-			qp.Scope.Category().PathType(),
-		)
-		if err != nil {
-			return nil, errors.Wrap(err, "getting folder cache")
-		}
-
-		err = exchange.FilterContainersAndFillCollections(
-			ctx,
-			qp,
-			collections,
-			gc.UpdateStatus,
-			resolver)
-
-		if err != nil {
-			return nil, errors.Wrap(err, "filling collections")
-		}
-
-		foldersComplete <- struct{}{}
-
-		for _, collection := range collections {
-			gc.incrementAwaitingMessages()
-
-			allCollections = append(allCollections, collection)
-		}
-	}
-
-	return allCollections, errs.ErrorOrNil()
-}
-
 // AwaitStatus waits for all gc tasks to complete and then returns status
 func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
 	defer func() {
@@ -378,6 +361,10 @@ func (gc *GraphConnector) incrementAwaitingMessages() {
 	gc.wg.Add(1)
 }
 
+// ---------------------------------------------------------------------------
+// Helper Funcs
+// ---------------------------------------------------------------------------
+
 // IsRecoverableError returns true iff error is a RecoverableGCEerror
 func IsRecoverableError(e error) bool {
 	var recoverable support.RecoverableGCError
@@ -389,113 +376,3 @@ func IsNonRecoverableError(e error) bool {
 	var nonRecoverable support.NonRecoverableGCError
 	return errors.As(e, &nonRecoverable)
 }
-
-// DataCollections utility function to launch backup operations for exchange and onedrive
-func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Selector) ([]data.Collection, error) {
-	ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
-	defer end()
-
-	err := verifyBackupInputs(sels, gc.Users)
-	if err != nil {
-		return nil, err
-	}
-
-	switch sels.Service {
-	case selectors.ServiceExchange:
-		return gc.ExchangeDataCollection(ctx, sels)
-	case selectors.ServiceOneDrive:
-		return gc.OneDriveDataCollections(ctx, sels)
-	default:
-		return nil, errors.Errorf("service %s not supported", sels)
-	}
-}
-
-// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
-// for the specified user
-func (gc *GraphConnector) OneDriveDataCollections(
-	ctx context.Context,
-	selector selectors.Selector,
-) ([]data.Collection, error) {
-	odb, err := selector.ToOneDriveBackup()
-	if err != nil {
-		return nil, errors.Wrap(err, "oneDriveDataCollection: parsing selector")
-	}
-
-	collections := []data.Collection{}
-
-	scopes := odb.DiscreteScopes(gc.GetUsers())
-
-	var errs error
-
-	// for each scope that includes oneDrive items, get all
-	for _, scope := range scopes {
-		for _, user := range scope.Get(selectors.OneDriveUser) {
-			logger.Ctx(ctx).With("user", user).Debug("Creating OneDrive collections")
-
-			odcs, err := onedrive.NewCollections(
-				gc.credentials.AzureTenantID,
-				user,
-				scope,
-				&gc.graphService,
-				gc.UpdateStatus,
-			).Get(ctx)
-			if err != nil {
-				return nil, support.WrapAndAppend(user, err, errs)
-			}
-
-			collections = append(collections, odcs...)
-		}
-	}
-
-	for range collections {
-		gc.incrementAwaitingMessages()
-	}
-
-	return collections, errs
-}
-
-func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) error {
-	var personnel []string
-
-	// retrieve users from selectors
-	switch sel.Service {
-	case selectors.ServiceExchange:
-		backup, err := sel.ToExchangeBackup()
-		if err != nil {
-			return err
-		}
-
-		for _, scope := range backup.Scopes() {
-			temp := scope.Get(selectors.ExchangeUser)
-			personnel = append(personnel, temp...)
-		}
-	case selectors.ServiceOneDrive:
-		backup, err := sel.ToOneDriveBackup()
-		if err != nil {
-			return err
-		}
-
-		for _, user := range backup.Scopes() {
-			temp := user.Get(selectors.OneDriveUser)
-			personnel = append(personnel, temp...)
-		}
-
-	default:
-		return errors.New("service %s not supported")
-	}
-
-	// verify personnel
-	normUsers := map[string]struct{}{}
-
-	for k := range mapOfUsers {
-		normUsers[strings.ToLower(k)] = struct{}{}
-	}
-
-	for _, user := range personnel {
-		if _, ok := normUsers[strings.ToLower(user)]; !ok {
-			return fmt.Errorf("%s user not found within tenant", user)
-		}
-	}
-
-	return nil
-}

src/internal/connector/graph_connector_helper_test.go

@@ -1,6 +1,7 @@
 package connector
 
 import (
+	"context"
 	"io"
 	"reflect"
 	"testing"
@@ -13,6 +14,7 @@ import (
 	"github.com/alcionai/corso/src/internal/connector/mockconnector"
 	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/selectors"
@@ -878,3 +880,11 @@ func getSelectorWith(service path.ServiceType) selectors.Selector {
 		Service: s,
 	}
 }
+
+func loadConnector(ctx context.Context, t *testing.T) *GraphConnector {
+	a := tester.NewM365Account(t)
+	connector, err := NewGraphConnector(ctx, a)
+	require.NoError(t, err)
+
+	return connector
+}

src/internal/connector/graph_connector_test.go

@@ -1,8 +1,6 @@
 package connector
 
 import (
-	"bytes"
-	"context"
 	"testing"
 	"time"
 
@@ -10,9 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
-	"github.com/alcionai/corso/src/internal/connector/exchange"
 	"github.com/alcionai/corso/src/internal/connector/mockconnector"
-	"github.com/alcionai/corso/src/internal/connector/support"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/pkg/control"
@@ -26,14 +22,6 @@ type GraphConnectorIntegrationSuite struct {
 	user      string
 }
 
-func loadConnector(ctx context.Context, t *testing.T) *GraphConnector {
-	a := tester.NewM365Account(t)
-	connector, err := NewGraphConnector(ctx, a)
-	require.NoError(t, err)
-
-	return connector
-}
-
 func TestGraphConnectorIntegrationSuite(t *testing.T) {
 	if err := tester.RunOnAny(
 		tester.CorsoCITests,
@@ -80,338 +68,31 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
 	suite.Greater(len(newConnector.Users), 0)
 }
 
-// TestInvalidUserForDataCollections ensures verification process for users
+// TestSetTenantUsers verifies GraphConnector's ability to query
-func (suite *GraphConnectorIntegrationSuite) TestInvalidUserForDataCollections() {
+// the sites associated with the credentials
+func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() {
+	newConnector := GraphConnector{
+		tenant:      "test_tenant",
+		Sites:       make(map[string]string, 0),
+		credentials: suite.connector.credentials,
+	}
+
 	ctx, flush := tester.NewContext()
 	defer flush()
 
-	invalidUser := "foo@example.com"
+	service, err := newConnector.createService(false)
-	connector := loadConnector(ctx, suite.T())
+	require.NoError(suite.T(), err)
-	tests := []struct {
+
-		name        string
+	newConnector.graphService = *service
-		getSelector func(t *testing.T) selectors.Selector
+
-	}{
+	suite.Equal(0, len(newConnector.Sites))
-		{
+	err = newConnector.setTenantSites(ctx)
-			name: "invalid exchange backup user",
+	assert.NoError(suite.T(), err)
-			getSelector: func(t *testing.T) selectors.Selector {
+	// TODO: should be non-zero once implemented.
-				sel := selectors.NewExchangeBackup()
+	// suite.Greater(len(newConnector.Users), 0)
-				sel.Include(sel.MailFolders([]string{invalidUser}, selectors.Any()))
+	suite.Equal(0, len(newConnector.Sites))
-				return sel.Selector
-			},
-		},
-		{
-			name: "Invalid onedrive backup user",
-			getSelector: func(t *testing.T) selectors.Selector {
-				sel := selectors.NewOneDriveBackup()
-				sel.Include(sel.Folders([]string{invalidUser}, selectors.Any()))
-				return sel.Selector
-			},
-		},
 }
 
-	for _, test := range tests {
-		suite.T().Run(test.name, func(t *testing.T) {
-			collections, err := connector.DataCollections(ctx, test.getSelector(t))
-			assert.Error(t, err)
-			assert.Empty(t, collections)
-		})
-	}
-}
-
-// TestExchangeDataCollection verifies interface between operation and
-// GraphConnector remains stable to receive a non-zero amount of Collections
-// for the Exchange Package. Enabled exchange applications:
-// - mail
-// - contacts
-// - events
-func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	connector := loadConnector(ctx, suite.T())
-	tests := []struct {
-		name        string
-		getSelector func(t *testing.T) selectors.Selector
-	}{
-		{
-			name: suite.user + " Email",
-			getSelector: func(t *testing.T) selectors.Selector {
-				sel := selectors.NewExchangeBackup()
-				sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
-
-				return sel.Selector
-			},
-		},
-		{
-			name: suite.user + " Contacts",
-			getSelector: func(t *testing.T) selectors.Selector {
-				sel := selectors.NewExchangeBackup()
-				sel.Include(sel.ContactFolders(
-					[]string{suite.user},
-					[]string{exchange.DefaultContactFolder},
-					selectors.PrefixMatch()))
-
-				return sel.Selector
-			},
-		},
-		{
-			name: suite.user + " Events",
-			getSelector: func(t *testing.T) selectors.Selector {
-				sel := selectors.NewExchangeBackup()
-				sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
-
-				return sel.Selector
-			},
-		},
-	}
-
-	for _, test := range tests {
-		suite.T().Run(test.name, func(t *testing.T) {
-			collection, err := connector.ExchangeDataCollection(ctx, test.getSelector(t))
-			require.NoError(t, err)
-			assert.Equal(t, len(collection), 1)
-			channel := collection[0].Items()
-			for object := range channel {
-				buf := &bytes.Buffer{}
-				_, err := buf.ReadFrom(object.ToReader())
-				assert.NoError(t, err, "received a buf.Read error")
-			}
-			status := connector.AwaitStatus()
-			assert.NotZero(t, status.Successful)
-			t.Log(status.String())
-		})
-	}
-}
-
-// TestMailSerializationRegression verifies that all mail data stored in the
-// test account can be successfully downloaded into bytes and restored into
-// M365 mail objects
-func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	t := suite.T()
-	connector := loadConnector(ctx, t)
-	sel := selectors.NewExchangeBackup()
-	sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
-	collection, err := connector.createCollections(ctx, sel.Scopes()[0])
-	require.NoError(t, err)
-
-	for _, edc := range collection {
-		suite.T().Run(edc.FullPath().String(), func(t *testing.T) {
-			streamChannel := edc.Items()
-			// Verify that each message can be restored
-			for stream := range streamChannel {
-				buf := &bytes.Buffer{}
-				read, err := buf.ReadFrom(stream.ToReader())
-				assert.NoError(t, err)
-				assert.NotZero(t, read)
-				message, err := support.CreateMessageFromBytes(buf.Bytes())
-				assert.NotNil(t, message)
-				assert.NoError(t, err)
-			}
-		})
-	}
-
-	status := connector.AwaitStatus()
-	suite.NotNil(status)
-	suite.Equal(status.ObjectCount, status.Successful)
-}
-
-// TestContactSerializationRegression verifies ability to query contact items
-// and to store contact within Collection. Downloaded contacts are run through
-// a regression test to ensure that downloaded items can be uploaded.
-func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	connector := loadConnector(ctx, suite.T())
-
-	tests := []struct {
-		name          string
-		getCollection func(t *testing.T) []*exchange.Collection
-	}{
-		{
-			name: "Default Contact Folder",
-			getCollection: func(t *testing.T) []*exchange.Collection {
-				scope := selectors.
-					NewExchangeBackup().
-					ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch())[0]
-				collections, err := connector.createCollections(ctx, scope)
-				require.NoError(t, err)
-
-				return collections
-			},
-		},
-	}
-
-	for _, test := range tests {
-		suite.T().Run(test.name, func(t *testing.T) {
-			edcs := test.getCollection(t)
-			require.Equal(t, len(edcs), 1)
-			edc := edcs[0]
-			assert.Equal(t, edc.FullPath().Folder(), exchange.DefaultContactFolder)
-			streamChannel := edc.Items()
-			count := 0
-			for stream := range streamChannel {
-				buf := &bytes.Buffer{}
-				read, err := buf.ReadFrom(stream.ToReader())
-				assert.NoError(t, err)
-				assert.NotZero(t, read)
-				contact, err := support.CreateContactFromBytes(buf.Bytes())
-				assert.NotNil(t, contact)
-				assert.NoError(t, err, "error on converting contact bytes: "+string(buf.Bytes()))
-				count++
-			}
-			assert.NotZero(t, count)
-
-			status := connector.AwaitStatus()
-			suite.NotNil(status)
-			suite.Equal(status.ObjectCount, status.Successful)
-		})
-	}
-}
-
-// TestEventsSerializationRegression ensures functionality of createCollections
-// to be able to successfully query, download and restore event objects
-func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	connector := loadConnector(ctx, suite.T())
-
-	tests := []struct {
-		name, expected string
-		getCollection  func(t *testing.T) []*exchange.Collection
-	}{
-		{
-			name:     "Default Event Calendar",
-			expected: exchange.DefaultCalendar,
-			getCollection: func(t *testing.T) []*exchange.Collection {
-				sel := selectors.NewExchangeBackup()
-				sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
-				collections, err := connector.createCollections(ctx, sel.Scopes()[0])
-				require.NoError(t, err)
-
-				return collections
-			},
-		},
-		{
-			name:     "Birthday Calendar",
-			expected: "Birthdays",
-			getCollection: func(t *testing.T) []*exchange.Collection {
-				sel := selectors.NewExchangeBackup()
-				sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}, selectors.PrefixMatch()))
-				collections, err := connector.createCollections(ctx, sel.Scopes()[0])
-				require.NoError(t, err)
-
-				return collections
-			},
-		},
-	}
-
-	for _, test := range tests {
-		suite.T().Run(test.name, func(t *testing.T) {
-			collections := test.getCollection(t)
-			require.Equal(t, len(collections), 1)
-			edc := collections[0]
-			assert.Equal(t, edc.FullPath().Folder(), test.expected)
-			streamChannel := edc.Items()
-
-			for stream := range streamChannel {
-				buf := &bytes.Buffer{}
-				read, err := buf.ReadFrom(stream.ToReader())
-				assert.NoError(t, err)
-				assert.NotZero(t, read)
-				event, err := support.CreateEventFromBytes(buf.Bytes())
-				assert.NotNil(t, event)
-				assert.NoError(t, err, "experienced error parsing event bytes: "+string(buf.Bytes()))
-			}
-
-			status := connector.AwaitStatus()
-			suite.NotNil(status)
-			suite.Equal(status.ObjectCount, status.Successful)
-		})
-	}
-}
-
-// TestAccessOfInboxAllUsers verifies that GraphConnector can
-// support `--users *` for backup operations. Selector.DiscreteScopes
-// returns all of the users within one scope. Only users who have
-// messages in their inbox will have a collection returned.
-// The final test insures that more than a 75% of the user collections are
-// returned. If an error was experienced, the test will fail overall
-func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	t := suite.T()
-	connector := loadConnector(ctx, t)
-	sel := selectors.NewExchangeBackup()
-	sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
-	scopes := sel.DiscreteScopes(connector.GetUsers())
-
-	for _, scope := range scopes {
-		users := scope.Get(selectors.ExchangeUser)
-		standard := (len(users) / 4) * 3
-		collections, err := connector.createCollections(ctx, scope)
-		require.NoError(t, err)
-		suite.Greater(len(collections), standard)
-	}
-}
-
-func (suite *GraphConnectorIntegrationSuite) TestMailFetch() {
-	ctx, flush := tester.NewContext()
-	defer flush()
-
-	var (
-		t      = suite.T()
-		userID = tester.M365UserID(t)
-	)
-
-	tests := []struct {
-		name        string
-		scope       selectors.ExchangeScope
-		folderNames map[string]struct{}
-	}{
-		{
-			name: "Folder Iterative Check Mail",
-			scope: selectors.NewExchangeBackup().MailFolders(
-				[]string{userID},
-				[]string{exchange.DefaultMailFolder},
-				selectors.PrefixMatch(),
-			)[0],
-			folderNames: map[string]struct{}{
-				exchange.DefaultMailFolder: {},
-			},
-		},
-	}
-
-	gc := loadConnector(ctx, t)
-
-	for _, test := range tests {
-		suite.T().Run(test.name, func(t *testing.T) {
-			collections, err := gc.createCollections(ctx, test.scope)
-			require.NoError(t, err)
-
-			for _, c := range collections {
-				require.NotEmpty(t, c.FullPath().Folder())
-				folder := c.FullPath().Folder()
-
-				if _, ok := test.folderNames[folder]; ok {
-					delete(test.folderNames, folder)
-				}
-			}
-
-			assert.Empty(t, test.folderNames)
-		})
-	}
-}
-
-///------------------------------------------------------------
-// Exchange Functions
-//-------------------------------------------------------
-
 func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
 	dest := tester.DefaultTestRestoreDestination()
 	table := []struct {
@@ -447,6 +128,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
 				Service: selectors.ServiceOneDrive,
 			},
 		},
+		// TODO: SharePoint
 	}
 
 	for _, test := range table {
@@ -466,6 +148,10 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
 	}
 }
 
+//-------------------------------------------------------------
+// Exchange Functions
+//-------------------------------------------------------------
+
 func runRestoreBackupTest(
 	t *testing.T,
 	test restoreBackupInfo,

src/internal/connector/onedrive/collection_test.go

@@ -60,7 +60,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
 	wg := sync.WaitGroup{}
 	collStatus := support.ConnectorOperationStatus{}
 
-	folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user")
+	folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user", OneDriveSource)
 	require.NoError(t, err)
 	driveFolderPath, err := getDriveFolderPath(folderPath)
 	require.NoError(t, err)
@@ -117,7 +117,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 
-	folderPath, err := getCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user")
+	folderPath, err := getCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", OneDriveSource)
 	require.NoError(t, err)
 
 	coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))

src/internal/connector/onedrive/collections.go

@@ -14,20 +14,34 @@ import (
 	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
-	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
-// Collections is used to retrieve OneDrive data for a
+type driveSource int
-// specified user
+
+const (
+	unknownDriveSource = iota
+	OneDriveSource
+	SharePointSource
+)
+
+type isAnyMatcher interface {
+	IsAny() bool
+	Matches(path string) bool
+}
+
+// Collections is used to retrieve drive data for a
+// resource owner, which can be either a user or a sharepoint site.
 type Collections struct {
 	tenant        string
-	user   string
+	resourceOwner string
-	scope  selectors.OneDriveScope
+	source        driveSource
+	matcher       isAnyMatcher
+	service       graph.Service
+	statusUpdater support.StatusUpdater
+
 	// collectionMap allows lookup of the data.Collection
 	// for a OneDrive folder
 	collectionMap map[string]data.Collection
-	service       graph.Service
-	statusUpdater support.StatusUpdater
 
 	// Track stats from drive enumeration. Represents the items backed up.
 	numItems      int
@@ -37,25 +51,27 @@ type Collections struct {
 
 func NewCollections(
 	tenant string,
-	user string,
+	resourceOwner string,
-	scope selectors.OneDriveScope,
+	source driveSource,
+	matcher isAnyMatcher,
 	service graph.Service,
 	statusUpdater support.StatusUpdater,
 ) *Collections {
 	return &Collections{
 		tenant:        tenant,
-		user:          user,
+		resourceOwner: resourceOwner,
-		scope:         scope,
+		source:        source,
+		matcher:       matcher,
 		collectionMap: map[string]data.Collection{},
 		service:       service,
 		statusUpdater: statusUpdater,
 	}
 }
 
-// Retrieves OneDrive data as set of `data.Collections`
+// Retrieves drive data as set of `data.Collections`
 func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
-	// Enumerate drives for the specified user
+	// Enumerate drives for the specified resourceOwner
-	drives, err := drives(ctx, c.service, c.user)
+	drives, err := drives(ctx, c.service, c.resourceOwner, c.source)
 	if err != nil {
 		return nil, err
 	}
@@ -78,29 +94,8 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
 	return collections, nil
 }
 
-func getCanonicalPath(p, tenant, user string) (path.Path, error) {
+// updateCollections initializes and adds the provided drive items to Collections
-	pathBuilder := path.Builder{}.Append(strings.Split(p, "/")...)
+// A new collection is created for every drive folder (or package)
-
-	res, err := pathBuilder.ToDataLayerOneDrivePath(tenant, user, false)
-	if err != nil {
-		return nil, errors.Wrap(err, "converting to canonical path")
-	}
-
-	return res, nil
-}
-
-// Returns the path to the folder within the drive (i.e. under `root:`)
-func getDriveFolderPath(p path.Path) (string, error) {
-	drivePath, err := toOneDrivePath(p)
-	if err != nil {
-		return "", err
-	}
-
-	return path.Builder{}.Append(drivePath.folders...).String(), nil
-}
-
-// updateCollections initializes and adds the provided OneDrive items to Collections
-// A new collection is created for every OneDrive folder (or package)
 func (c *Collections) updateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {
 	for _, item := range items {
 		if item.GetRoot() != nil {
@@ -116,14 +111,15 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite
 		collectionPath, err := getCanonicalPath(
 			*item.GetParentReference().GetPath(),
 			c.tenant,
-			c.user,
+			c.resourceOwner,
+			c.source,
 		)
 		if err != nil {
 			return err
 		}
 
 		// Skip items that don't match the folder selectors we were given.
-		if !includePath(ctx, c.scope, collectionPath) {
+		if !includePath(ctx, c.matcher, collectionPath) {
 			logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
 			continue
 		}
@@ -162,7 +158,40 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite
 	return nil
 }
 
-func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath path.Path) bool {
+func getCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) {
+	var (
+		pathBuilder = path.Builder{}.Append(strings.Split(p, "/")...)
+		result      path.Path
+		err         error
+	)
+
+	switch source {
+	case OneDriveSource:
+		result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false)
+	case SharePointSource:
+		result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false)
+	default:
+		return nil, errors.Errorf("unrecognized drive data source")
+	}
+
+	if err != nil {
+		return nil, errors.Wrap(err, "converting to canonical path")
+	}
+
+	return result, nil
+}
+
+// Returns the path to the folder within the drive (i.e. under `root:`)
+func getDriveFolderPath(p path.Path) (string, error) {
+	drivePath, err := toOneDrivePath(p)
+	if err != nil {
+		return "", err
+	}
+
+	return path.Builder{}.Append(drivePath.folders...).String(), nil
+}
+
+func includePath(ctx context.Context, m isAnyMatcher, folderPath path.Path) bool {
 	// Check if the folder is allowed by the scope.
 	folderPathString, err := getDriveFolderPath(folderPath)
 	if err != nil {
@@ -172,9 +201,9 @@ func includePath(ctx context.Context, scope selectors.OneDriveScope, folderPath
 
 	// Hack for the edge case where we're looking at the root folder and can
 	// select any folder. Right now the root folder has an empty folder path.
-	if len(folderPathString) == 0 && scope.IsAny(selectors.OneDriveFolder) {
+	if len(folderPathString) == 0 && m.IsAny() {
 		return true
 	}
 
-	return scope.Matches(selectors.OneDriveFolder, folderPathString)
+	return m.Matches(folderPathString)
 }

src/internal/connector/onedrive/collections_test.go

@@ -20,7 +20,7 @@ func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []st
 	res := make([]string, 0, len(rest))
 
 	for _, r := range rest {
-		p, err := getCanonicalPath(r, tenant, user)
+		p, err := getCanonicalPath(r, tenant, user, OneDriveSource)
 		require.NoError(t, err)
 
 		res = append(res, p.String())
@@ -211,7 +211,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 			ctx, flush := tester.NewContext()
 			defer flush()
 
-			c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil)
+			c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil)
 			err := c.updateCollections(ctx, "driveID", tt.items)
 			tt.expect(t, err)
 			assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths")

src/internal/connector/onedrive/drive.go

@@ -67,7 +67,33 @@ const (
 )
 
 // Enumerates the drives for the specified user
-func drives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) {
+func drives(
+	ctx context.Context,
+	service graph.Service,
+	resourceOwner string,
+	source driveSource,
+) ([]models.Driveable, error) {
+	switch source {
+	case OneDriveSource:
+		return userDrives(ctx, service, resourceOwner)
+	case SharePointSource:
+		return siteDrives(ctx, service, resourceOwner)
+	default:
+		return nil, errors.Errorf("unrecognized drive data source")
+	}
+}
+
+func siteDrives(ctx context.Context, service graph.Service, site string) ([]models.Driveable, error) {
+	r, err := service.Client().SitesById(site).Drives().Get(ctx, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to retrieve site drives. site: %s, details: %s",
+			site, support.ConnectorStackErrorTrace(err))
+	}
+
+	return r.GetValue(), nil
+}
+
+func userDrives(ctx context.Context, service graph.Service, user string) ([]models.Driveable, error) {
 	var hasDrive bool
 
 	hasDrive, err := hasDriveLicense(ctx, service, user)
@@ -237,7 +263,7 @@ func GetAllFolders(
 	userID string,
 	prefix string,
 ) ([]*Displayable, error) {
-	drives, err := drives(ctx, gs, userID)
+	drives, err := drives(ctx, gs, userID, OneDriveSource)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting OneDrive folders")
 	}
@@ -321,7 +347,7 @@ func hasDriveLicense(
 	cb := func(pageItem any) bool {
 		entry, ok := pageItem.(models.LicenseDetailsable)
 		if !ok {
-			err = errors.New("casting item to models.MailFolderable")
+			err = errors.New("casting item to models.LicenseDetailsable")
 			return false
 		}
 

src/internal/connector/onedrive/drive_test.go

@@ -43,7 +43,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
 	folderElements := []string{folderName1}
 	gs := loadTestService(t)
 
-	drives, err := drives(ctx, gs, suite.userID)
+	drives, err := drives(ctx, gs, suite.userID, OneDriveSource)
 	require.NoError(t, err)
 	require.NotEmpty(t, drives)
 
@@ -100,6 +100,18 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
 	}
 }
 
+type testFolderMatcher struct {
+	scope selectors.OneDriveScope
+}
+
+func (fm testFolderMatcher) IsAny() bool {
+	return fm.scope.IsAny(selectors.OneDriveFolder)
+}
+
+func (fm testFolderMatcher) Matches(path string) bool {
+	return fm.scope.Matches(selectors.OneDriveFolder, path)
+}
+
 func (suite *OneDriveSuite) TestOneDriveNewCollections() {
 	ctx, flush := tester.NewContext()
 	defer flush()
@@ -129,7 +141,8 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() {
 			odcs, err := NewCollections(
 				creds.AzureTenantID,
 				test.user,
-				scope,
+				OneDriveSource,
+				testFolderMatcher{scope},
 				service,
 				service.updateStatus,
 			).Get(ctx)

src/internal/connector/onedrive/item_test.go

@@ -67,7 +67,7 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
 
 	suite.user = tester.SecondaryM365UserID(suite.T())
 
-	drives, err := drives(ctx, suite, suite.user)
+	drives, err := drives(ctx, suite, suite.user, OneDriveSource)
 	require.NoError(suite.T(), err)
 	// Test Requirement 1: Need a drive
 	require.Greaterf(suite.T(), len(drives), 0, "user %s does not have a drive", suite.user)

src/internal/connector/sharepoint/libraries.go

@@ -0,0 +1,65 @@
+package sharepoint
+
+import (
+	"context"
+
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/onedrive"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+func CollectLibraries(
+	ctx context.Context,
+	serv graph.Service,
+	tenantID string,
+	siteIDs []string,
+	scope selectors.SharePointScope,
+	updater support.StatusUpdater,
+	incrementWaitCount func(),
+) ([]data.Collection, error) {
+	var (
+		collections = []data.Collection{}
+		errs        error
+	)
+
+	for _, site := range scope.Get(selectors.SharePointSite) {
+		logger.Ctx(ctx).With("site", site).Debug("Creating SharePoint Libary collections")
+
+		colls := onedrive.NewCollections(
+			tenantID,
+			site,
+			onedrive.SharePointSource,
+			folderMatcher{scope},
+			serv,
+			updater,
+		)
+
+		odcs, err := colls.Get(ctx)
+		if err != nil {
+			return nil, support.WrapAndAppend(site, err, errs)
+		}
+
+		collections = append(collections, odcs...)
+	}
+
+	for range collections {
+		incrementWaitCount()
+	}
+
+	return collections, errs
+}
+
+type folderMatcher struct {
+	scope selectors.SharePointScope
+}
+
+func (fm folderMatcher) IsAny() bool {
+	return fm.scope.IsAny(selectors.SharePointFolder)
+}
+
+func (fm folderMatcher) Matches(path string) bool {
+	return fm.scope.Matches(selectors.SharePointFolder, path)
+}

src/internal/connector/sharepoint/resolver.go

@@ -0,0 +1,95 @@
+package sharepoint
+
+import (
+	"context"
+
+	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
+	"github.com/pkg/errors"
+
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/pkg/account"
+)
+
+type sharePointService struct {
+	client      msgraphsdk.GraphServiceClient
+	adapter     msgraphsdk.GraphRequestAdapter
+	failFast    bool // if true service will exit sequence upon encountering an error
+	credentials account.M365Config
+}
+
+///------------------------------------------------------------
+// Functions to comply with graph.Service Interface
+//-------------------------------------------------------
+
+func (es *sharePointService) Client() *msgraphsdk.GraphServiceClient {
+	return &es.client
+}
+
+func (es *sharePointService) Adapter() *msgraphsdk.GraphRequestAdapter {
+	return &es.adapter
+}
+
+func (es *sharePointService) ErrPolicy() bool {
+	return es.failFast
+}
+
+// createService internal constructor for sharePointService struct returns an error
+// iff the params for the entry are incorrect (e.g. len(TenantID) == 0, etc.)
+// NOTE: Incorrect account information will result in errors on subsequent queries.
+func createService(credentials account.M365Config, shouldFailFast bool) (*sharePointService, error) {
+	adapter, err := graph.CreateAdapter(
+		credentials.AzureTenantID,
+		credentials.AzureClientID,
+		credentials.AzureClientSecret,
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating microsoft graph service")
+	}
+
+	service := sharePointService{
+		adapter:     *adapter,
+		client:      *msgraphsdk.NewGraphServiceClient(adapter),
+		failFast:    shouldFailFast,
+		credentials: credentials,
+	}
+
+	return &service, nil
+}
+
+// PopulateContainerResolver gets a container resolver if one is available for
+// this category of data. If one is not available, returns nil so that other
+// logic in the caller can complete as long as they check if the resolver is not
+// nil. If an error occurs populating the resolver, returns an error.
+func PopulateContainerResolver(
+	ctx context.Context,
+	qp graph.QueryParams,
+) (graph.ContainerResolver, error) {
+	return nil, nil
+	// var (
+	// 	c            graph.ContainerPopulater
+	// 	service, err = createService(qp.Credentials, qp.FailFast)
+	// 	cacheRoot    string
+	// )
+
+	// if err != nil {
+	// 	return nil, err
+	// }
+
+	// switch qp.Category {
+	// case path.FilesCategory:
+	// 	c = &driveCache{
+	// 		siteID: qp.ResourceOwner,
+	// 		gs:     service,
+	// 	}
+	// 	cacheRoot = "root"
+
+	// default:
+	// 	return nil, fmt.Errorf("ContainerResolver not present for %s type", qp.Category)
+	// }
+
+	// if err := c.Populate(ctx, cacheRoot); err != nil {
+	// 	return nil, errors.Wrap(err, "populating container resolver")
+	// }
+
+	// return c, nil
+}

src/internal/connector/sharepoint/service_iterators.go

@@ -0,0 +1,88 @@
+package sharepoint
+
+import (
+	"context"
+
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/pkg/selectors"
+)
+
+// FilterContainersAndFillCollections is a utility function
+// that places the M365 object ids belonging to specific directories
+// into a Collection. Items outside of those directories are omitted.
+// @param collection is filled with during this function.
+func FilterContainersAndFillCollections(
+	ctx context.Context,
+	qp graph.QueryParams,
+	collections map[string]*Collection,
+	statusUpdater support.StatusUpdater,
+	resolver graph.ContainerResolver,
+	scope selectors.SharePointScope,
+) error {
+	return nil
+}
+
+// code previously within the function, moved here to make the linter happy
+
+// var (
+// 	category       = qp.Scope.Category().PathType()
+// 	collectionType = CategoryToOptionIdentifier(category)
+// 	errs           error
+// )
+
+// for _, c := range resolver.Items() {
+// 	dirPath, ok := pathAndMatch(qp, category, c)
+// 	if ok {
+// 		// Create only those that match
+// 		service, err := createService(qp.Credentials, qp.FailFast)
+// 		if err != nil {
+// 			errs = support.WrapAndAppend(
+// 				qp.User+" FilterContainerAndFillCollection",
+// 				err,
+// 				errs)
+
+// 			if qp.FailFast {
+// 				return errs
+// 			}
+// 		}
+
+// 		edc := NewCollection(
+// 			qp.User,
+// 			dirPath,
+// 			collectionType,
+// 			service,
+// 			statusUpdater,
+// 		)
+// 		collections[*c.GetId()] = &edc
+// 	}
+// }
+
+// for directoryID, col := range collections {
+// 	fetchFunc, err := getFetchIDFunc(category)
+// 	if err != nil {
+// 		errs = support.WrapAndAppend(
+// 			qp.User,
+// 			err,
+// 			errs)
+
+// 		if qp.FailFast {
+// 			return errs
+// 		}
+
+// 		continue
+// 	}
+
+// 	jobs, err := fetchFunc(ctx, col.service, qp.User, directoryID)
+// 	if err != nil {
+// 		errs = support.WrapAndAppend(
+// 			qp.User,
+// 			err,
+// 			errs,
+// 		)
+// 	}
+
+// 	col.jobs = append(col.jobs, jobs...)
+// }
+
+// return errs

src/internal/tester/config.go

@@ -21,6 +21,7 @@ const (
 
 	// M365 config
 	TestCfgAzureTenantID    = "azure_tenantid"
+	TestCfgSiteID           = "m365siteid"
 	TestCfgUserID           = "m365userid"
 	TestCfgSecondaryUserID  = "secondarym365userid"
 	TestCfgLoadTestUserID   = "loadtestm365userid"
@@ -30,6 +31,7 @@ const (
 
 // test specific env vars
 const (
+	EnvCorsoM365TestSiteID          = "CORSO_M365_TEST_SITE_ID"
 	EnvCorsoM365TestUserID          = "CORSO_M365_TEST_USER_ID"
 	EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID"
 	EnvCorsoM365LoadTestUserID      = "CORSO_M365_LOAD_TEST_USER_ID"
@@ -137,6 +139,13 @@ func readTestConfig() (map[string]string, error) {
 		vpr.GetString(TestCfgLoadTestOrgUsers),
 		"lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com",
 	)
+	fallbackTo(
+		testEnv,
+		TestCfgSiteID,
+		os.Getenv(EnvCorsoM365TestSiteID),
+		vpr.GetString(TestCfgSiteID),
+		"8qzvrj.sharepoint.com,1c9ef309-f47c-4e69-832b-a83edd69fa7f,c57f6e0e-3e4b-472c-b528-b56a2ccd0507",
+	)
 
 	testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath)
 	testConfig = testEnv

src/internal/tester/integration_runners.go

@@ -16,6 +16,9 @@ const (
 	CorsoCLIRepoTests                             = "CORSO_COMMAND_LINE_REPO_TESTS"
 	CorsoCLIRestoreTests                          = "CORSO_COMMAND_LINE_RESTORE_TESTS"
 	CorsoCLITests                                 = "CORSO_COMMAND_LINE_TESTS"
+	CorsoConnectorCreateExchangeCollectionTests   = "CORSO_CONNECTOR_CREATE_EXCHANGE_COLLECTION_TESTS"
+	CorsoConnectorCreateSharePointCollectionTests = "CORSO_CONNECTOR_CREATE_SHAREPOINT_COLLECTION_TESTS"
+	CorsoConnectorDataCollectionTests             = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS"
 	CorsoGraphConnectorTests                      = "CORSO_GRAPH_CONNECTOR_TESTS"
 	CorsoGraphConnectorExchangeTests              = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS"
 	CorsoGraphConnectorOneDriveTests              = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS"

src/internal/tester/resource_owners.go

@@ -57,3 +57,14 @@ func LoadTestM365OrgUsers(t *testing.T) []string {
 
 	return strings.Split(users, ",")
 }
+
+// M365SiteID returns a siteID string representing the m365SiteID described
+// by either the env var CORSO_M365_TEST_SITE_ID, the corso_test.toml config
+// file or the default value (in that order of priority).  The default is a
+// last-attempt fallback that will only work on alcion's testing org.
+func M365SiteID(t *testing.T) string {
+	cfg, err := readTestConfig()
+	require.NoError(t, err, "retrieving m365 site id from test configuration")
+
+	return cfg[TestCfgSiteID]
+}

src/pkg/path/path.go

@@ -241,7 +241,7 @@ func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
 	}
 
 	if len(resourceOwner) == 0 {
-		return errors.Wrap(errMissingSegment, "user")
+		return errors.Wrap(errMissingSegment, "resourceOwner")
 	}
 
 	if len(pb.elements) == 0 {