diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index ef6a6c923..fa4087015 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" D "github.com/alcionai/corso/src/internal/diagnostics" @@ -38,6 +39,8 @@ func (gc *GraphConnector) DataCollections(ctx context.Context, sels selectors.Se return gc.ExchangeDataCollection(ctx, sels) case selectors.ServiceOneDrive: return gc.OneDriveDataCollections(ctx, sels) + case selectors.ServiceSharePoint: + return gc.SharePointDataCollections(ctx, sels) default: return nil, errors.Errorf("service %s not supported", sels) } @@ -89,18 +92,23 @@ func verifyBackupInputs(sel selectors.Selector, mapOfUsers map[string]string) er return nil } -// createCollections - utility function that retrieves M365 +// --------------------------------------------------------------------------- +// Exchange +// --------------------------------------------------------------------------- + +// createExchangeCollections - utility function that retrieves M365 // IDs through Microsoft Graph API. The selectors.ExchangeScope -// determines the type of collections that are stored. -// to the GraphConnector struct. -func (gc *GraphConnector) createCollections( +// determines the type of collections that are retrieved. +func (gc *GraphConnector) createExchangeCollections( ctx context.Context, scope selectors.ExchangeScope, ) ([]*exchange.Collection, error) { - var errs *multierror.Error + var ( + errs *multierror.Error + users = scope.Get(selectors.ExchangeUser) + allCollections = make([]*exchange.Collection, 0) + ) - users := scope.Get(selectors.ExchangeUser) - allCollections := make([]*exchange.Collection, 0) // Create collection of ExchangeDataCollection for _, user := range users { collections := make(map[string]*exchange.Collection) @@ -172,7 +180,7 @@ func (gc *GraphConnector) ExchangeDataCollection( for _, scope := range scopes { // Creates a map of collections based on scope - dcs, err := gc.createCollections(ctx, scope) + dcs, err := gc.createExchangeCollections(ctx, scope) if err != nil { user := scope.Get(selectors.ExchangeUser) return nil, support.WrapAndAppend(user[0], err, errs) @@ -186,6 +194,10 @@ func (gc *GraphConnector) ExchangeDataCollection( return collections, errs } +// --------------------------------------------------------------------------- +// OneDrive +// --------------------------------------------------------------------------- + // OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data // for the specified user func (gc *GraphConnector) OneDriveDataCollections( @@ -197,11 +209,11 @@ func (gc *GraphConnector) OneDriveDataCollections( return nil, errors.Wrap(err, "oneDriveDataCollection: parsing selector") } - collections := []data.Collection{} - - scopes := odb.DiscreteScopes(gc.GetUsers()) - - var errs error + var ( + scopes = odb.DiscreteScopes(gc.GetUsers()) + collections = []data.Collection{} + errs error + ) // for each scope that includes oneDrive items, get all for _, scope := range scopes { @@ -229,3 +241,109 @@ func (gc *GraphConnector) OneDriveDataCollections( return collections, errs } + +// --------------------------------------------------------------------------- +// SharePoint +// --------------------------------------------------------------------------- + +// createSharePointCollections - utility function that retrieves M365 +// IDs through Microsoft Graph API. The selectors.SharePointScope +// determines the type of collections that are retrieved. +func (gc *GraphConnector) createSharePointCollections( + ctx context.Context, + scope selectors.SharePointScope, +) ([]*sharepoint.Collection, error) { + var ( + errs *multierror.Error + sites = scope.Get(selectors.SharePointSite) + colls = make([]*sharepoint.Collection, 0) + ) + + // Create collection of ExchangeDataCollection + for _, site := range sites { + collections := make(map[string]*sharepoint.Collection) + + qp := graph.QueryParams{ + // TODO: Resource owner, not user/site. + User: site, + // TODO: generic scope handling in query params. + // - or, break scope out of QP. + // Scope: scope, + FailFast: gc.failFast, + Credentials: gc.credentials, + } + + itemCategory := qp.Scope.Category().PathType() + + foldersComplete, closer := observe.MessageWithCompletion(fmt.Sprintf("∙ %s - %s:", itemCategory.String(), site)) + defer closer() + defer close(foldersComplete) + + // resolver, err := exchange.PopulateExchangeContainerResolver( + // ctx, + // qp, + // qp.Scope.Category().PathType(), + // ) + // if err != nil { + // return nil, errors.Wrap(err, "getting folder cache") + // } + + // err = sharepoint.FilterContainersAndFillCollections( + // ctx, + // qp, + // collections, + // gc.UpdateStatus, + // resolver) + + // if err != nil { + // return nil, errors.Wrap(err, "filling collections") + // } + + foldersComplete <- struct{}{} + + for _, collection := range collections { + gc.incrementAwaitingMessages() + + colls = append(colls, collection) + } + } + + return colls, errs.ErrorOrNil() +} + +// SharePointDataCollections returns a set of DataCollection which represents the SharePoint data +// for the specified user +func (gc *GraphConnector) SharePointDataCollections( + ctx context.Context, + selector selectors.Selector, +) ([]data.Collection, error) { + b, err := selector.ToSharePointBackup() + if err != nil { + return nil, errors.Wrap(err, "sharePointDataCollection: parsing selector") + } + + var ( + scopes = b.DiscreteScopes(gc.GetSites()) + collections = []data.Collection{} + errs error + ) + + // for each scope that includes oneDrive items, get all + for _, scope := range scopes { + // Creates a map of collections based on scope + dcs, err := gc.createSharePointCollections(ctx, scope) + if err != nil { + return nil, support.WrapAndAppend(scope.Get(selectors.SharePointSite)[0], err, errs) + } + + for _, collection := range dcs { + collections = append(collections, collection) + } + } + + for range collections { + gc.incrementAwaitingMessages() + } + + return collections, errs +} diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index ea721a3d9..661ce934f 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -10,7 +10,6 @@ import ( "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/support" - "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -23,6 +22,7 @@ type ConnectorDataCollectionIntegrationSuite struct { suite.Suite connector *GraphConnector user string + site string } func TestConnectorDataCollectionIntegrationSuite(t *testing.T) { @@ -44,6 +44,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) SetupSuite() { require.NoError(suite.T(), err) suite.connector = loadConnector(ctx, suite.T()) suite.user = tester.M365UserID(suite.T()) + suite.site = tester.M365SiteID(suite.T()) tester.LogTimeOfTest(suite.T()) } @@ -154,82 +155,76 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestInvalidUserForDataColl } } -func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { - dest := tester.DefaultTestRestoreDestination() - table := []struct { - name string - col []data.Collection - sel selectors.Selector +// TestSharePointDataCollection verifies interface between operation and +// GraphConnector remains stable to receive a non-zero amount of Collections +// for the SharePoint Package. +func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollection() { + ctx, flush := tester.NewContext() + defer flush() + + connector := loadConnector(ctx, suite.T()) + tests := []struct { + name string + getSelector func(t *testing.T) selectors.Selector }{ { - name: "ExchangeNil", - col: nil, - sel: selectors.Selector{ - Service: selectors.ServiceExchange, - }, - }, - { - name: "ExchangeEmpty", - col: []data.Collection{}, - sel: selectors.Selector{ - Service: selectors.ServiceExchange, - }, - }, - { - name: "OneDriveNil", - col: nil, - sel: selectors.Selector{ - Service: selectors.ServiceOneDrive, - }, - }, - { - name: "OneDriveEmpty", - col: []data.Collection{}, - sel: selectors.Selector{ - Service: selectors.ServiceOneDrive, + name: "Items - TODO: actual sharepoint categories", + getSelector: func(t *testing.T) selectors.Selector { + sel := selectors.NewSharePointBackup() + sel.Include(sel.Folders([]string{suite.site}, selectors.Any())) + + return sel.Selector }, }, } - for _, test := range table { + for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - ctx, flush := tester.NewContext() - defer flush() - - deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col) + _, err := connector.SharePointDataCollections(ctx, test.getSelector(t)) require.NoError(t, err) - assert.NotNil(t, deets) - stats := suite.connector.AwaitStatus() - assert.Zero(t, stats.ObjectCount) - assert.Zero(t, stats.FolderCount) - assert.Zero(t, stats.Successful) + // TODO: Implementation + // assert.Equal(t, len(collection), 1) + + // channel := collection[0].Items() + + // for object := range channel { + // buf := &bytes.Buffer{} + // _, err := buf.ReadFrom(object.ToReader()) + // assert.NoError(t, err, "received a buf.Read error") + // } + + // status := connector.AwaitStatus() + // assert.NotZero(t, status.Successful) + + // t.Log(status.String()) }) } } // --------------------------------------------------------------------------- -// CreateCollection tests +// CreateExchangeCollection tests // --------------------------------------------------------------------------- -type ConnectorCreateCollectionIntegrationSuite struct { +type ConnectorCreateExchangeCollectionIntegrationSuite struct { suite.Suite connector *GraphConnector user string + site string } -func TestConnectorCreateCollectionIntegrationSuite(t *testing.T) { +func TestConnectorCreateExchangeCollectionIntegrationSuite(t *testing.T) { if err := tester.RunOnAny( tester.CorsoCITests, - tester.CorsoConnectorCreateCollectionTests, + tester.CorsoConnectorCreateExchangeCollectionTests, ); err != nil { t.Skip(err) } - suite.Run(t, new(ConnectorCreateCollectionIntegrationSuite)) + suite.Run(t, new(ConnectorCreateExchangeCollectionIntegrationSuite)) } -func (suite *ConnectorCreateCollectionIntegrationSuite) SetupSuite() { +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) SetupSuite() { ctx, flush := tester.NewContext() defer flush() @@ -237,13 +232,62 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) SetupSuite() { require.NoError(suite.T(), err) suite.connector = loadConnector(ctx, suite.T()) suite.user = tester.M365UserID(suite.T()) + suite.site = tester.M365SiteID(suite.T()) tester.LogTimeOfTest(suite.T()) } +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailFetch() { + ctx, flush := tester.NewContext() + defer flush() + + var ( + t = suite.T() + userID = tester.M365UserID(t) + ) + + tests := []struct { + name string + scope selectors.ExchangeScope + folderNames map[string]struct{} + }{ + { + name: "Folder Iterative Check Mail", + scope: selectors.NewExchangeBackup().MailFolders( + []string{userID}, + []string{exchange.DefaultMailFolder}, + selectors.PrefixMatch(), + )[0], + folderNames: map[string]struct{}{ + exchange.DefaultMailFolder: {}, + }, + }, + } + + gc := loadConnector(ctx, t) + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + collections, err := gc.createExchangeCollections(ctx, test.scope) + require.NoError(t, err) + + for _, c := range collections { + require.NotEmpty(t, c.FullPath().Folder()) + folder := c.FullPath().Folder() + + if _, ok := test.folderNames[folder]; ok { + delete(test.folderNames, folder) + } + } + + assert.Empty(t, test.folderNames) + }) + } +} + // TestMailSerializationRegression verifies that all mail data stored in the // test account can be successfully downloaded into bytes and restored into // M365 mail objects -func (suite *ConnectorCreateCollectionIntegrationSuite) TestMailSerializationRegression() { +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailSerializationRegression() { ctx, flush := tester.NewContext() defer flush() @@ -251,7 +295,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestMailSerializationReg connector := loadConnector(ctx, t) sel := selectors.NewExchangeBackup() sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) - collection, err := connector.createCollections(ctx, sel.Scopes()[0]) + collection, err := connector.createExchangeCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) for _, edc := range collection { @@ -278,7 +322,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestMailSerializationReg // TestContactSerializationRegression verifies ability to query contact items // and to store contact within Collection. Downloaded contacts are run through // a regression test to ensure that downloaded items can be uploaded. -func (suite *ConnectorCreateCollectionIntegrationSuite) TestContactSerializationRegression() { +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestContactSerializationRegression() { ctx, flush := tester.NewContext() defer flush() @@ -294,7 +338,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestContactSerialization scope := selectors. NewExchangeBackup(). ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch())[0] - collections, err := connector.createCollections(ctx, scope) + collections, err := connector.createExchangeCollections(ctx, scope) require.NoError(t, err) return collections @@ -331,7 +375,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestContactSerialization // TestEventsSerializationRegression ensures functionality of createCollections // to be able to successfully query, download and restore event objects -func (suite *ConnectorCreateCollectionIntegrationSuite) TestEventsSerializationRegression() { +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestEventsSerializationRegression() { ctx, flush := tester.NewContext() defer flush() @@ -347,7 +391,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestEventsSerializationR getCollection: func(t *testing.T) []*exchange.Collection { sel := selectors.NewExchangeBackup() sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) - collections, err := connector.createCollections(ctx, sel.Scopes()[0]) + collections, err := connector.createExchangeCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) return collections @@ -359,7 +403,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestEventsSerializationR getCollection: func(t *testing.T) []*exchange.Collection { sel := selectors.NewExchangeBackup() sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}, selectors.PrefixMatch())) - collections, err := connector.createCollections(ctx, sel.Scopes()[0]) + collections, err := connector.createExchangeCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) return collections @@ -398,7 +442,7 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestEventsSerializationR // messages in their inbox will have a collection returned. // The final test insures that more than a 75% of the user collections are // returned. If an error was experienced, the test will fail overall -func (suite *ConnectorCreateCollectionIntegrationSuite) TestAccessOfInboxAllUsers() { +func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestAccessOfInboxAllUsers() { ctx, flush := tester.NewContext() defer flush() @@ -411,8 +455,60 @@ func (suite *ConnectorCreateCollectionIntegrationSuite) TestAccessOfInboxAllUser for _, scope := range scopes { users := scope.Get(selectors.ExchangeUser) standard := (len(users) / 4) * 3 - collections, err := connector.createCollections(ctx, scope) + collections, err := connector.createExchangeCollections(ctx, scope) require.NoError(t, err) suite.Greater(len(collections), standard) } } + +// --------------------------------------------------------------------------- +// CreateSharePointCollection tests +// --------------------------------------------------------------------------- + +type ConnectorCreateSharePointCollectionIntegrationSuite struct { + suite.Suite + connector *GraphConnector + user string +} + +func TestConnectorCreateSharePointCollectionIntegrationSuite(t *testing.T) { + if err := tester.RunOnAny( + tester.CorsoCITests, + tester.CorsoConnectorCreateSharePointCollectionTests, + ); err != nil { + t.Skip(err) + } + + suite.Run(t, new(ConnectorCreateSharePointCollectionIntegrationSuite)) +} + +func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) SetupSuite() { + ctx, flush := tester.NewContext() + defer flush() + + _, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...) + require.NoError(suite.T(), err) + suite.connector = loadConnector(ctx, suite.T()) + suite.user = tester.M365UserID(suite.T()) + tester.LogTimeOfTest(suite.T()) +} + +func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateSharePointCollection() { + ctx, flush := tester.NewContext() + defer flush() + + var ( + t = suite.T() + userID = tester.M365UserID(t) + ) + + gc := loadConnector(ctx, t) + scope := selectors.NewSharePointBackup().Folders( + []string{userID}, + []string{"foo"}, + selectors.PrefixMatch(), + )[0] + + _, err := gc.createSharePointCollections(ctx, scope) + require.NoError(t, err) +} diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 528006bd1..0eb7bdc0e 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -36,6 +36,7 @@ type GraphConnector struct { graphService tenant string Users map[string]string // key value + Sites map[string]string // key value credentials account.M365Config // wg is used to track completion of GC tasks @@ -97,6 +98,12 @@ func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnect return nil, errors.Wrap(err, "retrieving tenant user list") } + // TODO: users or sites, one or the other, not both. + err = gc.setTenantSites(ctx) + if err != nil { + return nil, errors.Wrap(err, "retrieveing tenant site list") + } + return &gc, nil } @@ -126,7 +133,7 @@ func (gs *graphService) EnableFailFast() { // setTenantUsers queries the M365 to identify the users in the // workspace. The users field is updated during this method -// iff the return value is true +// iff the return value is nil func (gc *GraphConnector) setTenantUsers(ctx context.Context) error { ctx, end := D.Span(ctx, "gc:setTenantUsers") defer end() @@ -191,6 +198,79 @@ func (gc *GraphConnector) GetUsersIds() []string { return buildFromMap(false, gc.Users) } +// setTenantSites queries the M365 to identify the sites in the +// workspace. The sitets field is updated during this method +// iff the return value is nil +func (gc *GraphConnector) setTenantSites(ctx context.Context) error { + // TODO + gc.Sites = map[string]string{} + + // ctx, end := D.Span(ctx, "gc:setTenantSites") + // defer end() + + // response, err := exchange.GetAllUsersForTenant(ctx, gc.graphService, "") + // if err != nil { + // return errors.Wrapf( + // err, + // "tenant %s M365 query: %s", + // gc.tenant, + // support.ConnectorStackErrorTrace(err), + // ) + // } + + // userIterator, err := msgraphgocore.NewPageIterator( + // response, + // &gc.graphService.adapter, + // models.CreateUserCollectionResponseFromDiscriminatorValue, + // ) + // if err != nil { + // return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + // } + + // callbackFunc := func(userItem interface{}) bool { + // user, ok := userItem.(models.Userable) + // if !ok { + // err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), + // errors.New("received non-User on iteration"), err) + // return true + // } + + // if user.GetUserPrincipalName() == nil { + // err = support.WrapAndAppend( + // gc.graphService.adapter.GetBaseUrl(), + // fmt.Errorf("no email address for User: %s", *user.GetId()), + // err, + // ) + + // return true + // } + + // // *user.GetId() is populated for every M365 entityable object by M365 backstore + // gc.Users[*user.GetUserPrincipalName()] = *user.GetId() + + // return true + // } + + // iterateError := userIterator.Iterate(ctx, callbackFunc) + // if iterateError != nil { + // err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err) + // } + + // return err + + return nil +} + +// GetSites returns the siteIDs of sharepoint sites within tenant. +func (gc *GraphConnector) GetSites() []string { + return buildFromMap(true, gc.Sites) +} + +// GetSiteIds returns the M365 id for the user +func (gc *GraphConnector) GetSiteIds() []string { + return buildFromMap(false, gc.Sites) +} + // buildFromMap helper function for returning []string from map. // Returns list of keys iff true; otherwise returns a list of values func buildFromMap(isKey bool, mapping map[string]string) []string { diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 3538d12d2..af6a6fea4 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" @@ -69,50 +68,82 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { suite.Greater(len(newConnector.Users), 0) } -func (suite *GraphConnectorIntegrationSuite) TestMailFetch() { +// TestSetTenantUsers verifies GraphConnector's ability to query +// the sites associated with the credentials +func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() { + newConnector := GraphConnector{ + tenant: "test_tenant", + Sites: make(map[string]string, 0), + credentials: suite.connector.credentials, + } + ctx, flush := tester.NewContext() defer flush() - var ( - t = suite.T() - userID = tester.M365UserID(t) - ) + service, err := newConnector.createService(false) + require.NoError(suite.T(), err) - tests := []struct { - name string - scope selectors.ExchangeScope - folderNames map[string]struct{} + newConnector.graphService = *service + + suite.Equal(0, len(newConnector.Sites)) + err = newConnector.setTenantSites(ctx) + assert.NoError(suite.T(), err) + // TODO: should be non-zero once implemented. + // suite.Greater(len(newConnector.Users), 0) + suite.Equal(0, len(newConnector.Sites)) +} + +func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { + dest := tester.DefaultTestRestoreDestination() + table := []struct { + name string + col []data.Collection + sel selectors.Selector }{ { - name: "Folder Iterative Check Mail", - scope: selectors.NewExchangeBackup().MailFolders( - []string{userID}, - []string{exchange.DefaultMailFolder}, - selectors.PrefixMatch(), - )[0], - folderNames: map[string]struct{}{ - exchange.DefaultMailFolder: {}, + name: "ExchangeNil", + col: nil, + sel: selectors.Selector{ + Service: selectors.ServiceExchange, }, }, + { + name: "ExchangeEmpty", + col: []data.Collection{}, + sel: selectors.Selector{ + Service: selectors.ServiceExchange, + }, + }, + { + name: "OneDriveNil", + col: nil, + sel: selectors.Selector{ + Service: selectors.ServiceOneDrive, + }, + }, + { + name: "OneDriveEmpty", + col: []data.Collection{}, + sel: selectors.Selector{ + Service: selectors.ServiceOneDrive, + }, + }, + // TODO: SharePoint } - gc := loadConnector(ctx, t) - - for _, test := range tests { + for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - collections, err := gc.createCollections(ctx, test.scope) + ctx, flush := tester.NewContext() + defer flush() + + deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col) require.NoError(t, err) + assert.NotNil(t, deets) - for _, c := range collections { - require.NotEmpty(t, c.FullPath().Folder()) - folder := c.FullPath().Folder() - - if _, ok := test.folderNames[folder]; ok { - delete(test.folderNames, folder) - } - } - - assert.Empty(t, test.folderNames) + stats := suite.connector.AwaitStatus() + assert.Zero(t, stats.ObjectCount) + assert.Zero(t, stats.FolderCount) + assert.Zero(t, stats.Successful) }) } } diff --git a/src/internal/connector/sharepoint/service_iterators.go b/src/internal/connector/sharepoint/service_iterators.go new file mode 100644 index 000000000..a1f682c26 --- /dev/null +++ b/src/internal/connector/sharepoint/service_iterators.go @@ -0,0 +1,86 @@ +package sharepoint + +import ( + "context" + + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" +) + +// FilterContainersAndFillCollections is a utility function +// that places the M365 object ids belonging to specific directories +// into a Collection. Items outside of those directories are omitted. +// @param collection is filled with during this function. +func FilterContainersAndFillCollections( + ctx context.Context, + qp graph.QueryParams, + collections map[string]*Collection, + statusUpdater support.StatusUpdater, + resolver graph.ContainerResolver, +) error { + return nil +} + +// code previously within the function, moved here to make the linter happy + +// var ( +// category = qp.Scope.Category().PathType() +// collectionType = CategoryToOptionIdentifier(category) +// errs error +// ) + +// for _, c := range resolver.Items() { +// dirPath, ok := pathAndMatch(qp, category, c) +// if ok { +// // Create only those that match +// service, err := createService(qp.Credentials, qp.FailFast) +// if err != nil { +// errs = support.WrapAndAppend( +// qp.User+" FilterContainerAndFillCollection", +// err, +// errs) + +// if qp.FailFast { +// return errs +// } +// } + +// edc := NewCollection( +// qp.User, +// dirPath, +// collectionType, +// service, +// statusUpdater, +// ) +// collections[*c.GetId()] = &edc +// } +// } + +// for directoryID, col := range collections { +// fetchFunc, err := getFetchIDFunc(category) +// if err != nil { +// errs = support.WrapAndAppend( +// qp.User, +// err, +// errs) + +// if qp.FailFast { +// return errs +// } + +// continue +// } + +// jobs, err := fetchFunc(ctx, col.service, qp.User, directoryID) +// if err != nil { +// errs = support.WrapAndAppend( +// qp.User, +// err, +// errs, +// ) +// } + +// col.jobs = append(col.jobs, jobs...) +// } + +// return errs diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 6c4b9cce5..078fcfadc 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -21,6 +21,7 @@ const ( // M365 config TestCfgAzureTenantID = "azure_tenantid" + TestCfgSiteID = "m365siteid" TestCfgUserID = "m365userid" TestCfgSecondaryUserID = "secondarym365userid" TestCfgLoadTestUserID = "loadtestm365userid" @@ -30,6 +31,7 @@ const ( // test specific env vars const ( + EnvCorsoM365TestSiteID = "CORSO_M365_TEST_SITE_ID" EnvCorsoM365TestUserID = "CORSO_M365_TEST_USER_ID" EnvCorsoSecondaryM365TestUserID = "CORSO_SECONDARY_M365_TEST_USER_ID" EnvCorsoM365LoadTestUserID = "CORSO_M365_LOAD_TEST_USER_ID" @@ -137,6 +139,13 @@ func readTestConfig() (map[string]string, error) { vpr.GetString(TestCfgLoadTestOrgUsers), "lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com", ) + fallbackTo( + testEnv, + TestCfgSiteID, + os.Getenv(EnvCorsoM365TestSiteID), + vpr.GetString(TestCfgSiteID), + "8qzvrj.sharepoint.com,1c9ef309-f47c-4e69-832b-a83edd69fa7f,c57f6e0e-3e4b-472c-b528-b56a2ccd0507", + ) testEnv[EnvCorsoTestConfigFilePath] = os.Getenv(EnvCorsoTestConfigFilePath) testConfig = testEnv diff --git a/src/internal/tester/integration_runners.go b/src/internal/tester/integration_runners.go index c0b295b9d..cfc2b2623 100644 --- a/src/internal/tester/integration_runners.go +++ b/src/internal/tester/integration_runners.go @@ -9,23 +9,24 @@ import ( ) const ( - CorsoLoadTests = "CORSO_LOAD_TESTS" - CorsoCITests = "CORSO_CI_TESTS" - CorsoCLIBackupTests = "CORSO_COMMAND_LINE_BACKUP_TESTS" - CorsoCLIConfigTests = "CORSO_COMMAND_LINE_CONFIG_TESTS" - CorsoCLIRepoTests = "CORSO_COMMAND_LINE_REPO_TESTS" - CorsoCLIRestoreTests = "CORSO_COMMAND_LINE_RESTORE_TESTS" - CorsoCLITests = "CORSO_COMMAND_LINE_TESTS" - CorsoConnectorCreateCollectionTests = "CORSO_CONNECTOR_CREATE_COLLECTION_TESTS" - CorsoConnectorDataCollectionTests = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS" - CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS" - CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS" - CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS" - CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS" - CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS" - CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS" - CorsoOperationTests = "CORSO_OPERATION_TESTS" - CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS" + CorsoLoadTests = "CORSO_LOAD_TESTS" + CorsoCITests = "CORSO_CI_TESTS" + CorsoCLIBackupTests = "CORSO_COMMAND_LINE_BACKUP_TESTS" + CorsoCLIConfigTests = "CORSO_COMMAND_LINE_CONFIG_TESTS" + CorsoCLIRepoTests = "CORSO_COMMAND_LINE_REPO_TESTS" + CorsoCLIRestoreTests = "CORSO_COMMAND_LINE_RESTORE_TESTS" + CorsoCLITests = "CORSO_COMMAND_LINE_TESTS" + CorsoConnectorCreateExchangeCollectionTests = "CORSO_CONNECTOR_CREATE_EXCHANGE_COLLECTION_TESTS" + CorsoConnectorCreateSharePointCollectionTests = "CORSO_CONNECTOR_CREATE_SHAREPOINT_COLLECTION_TESTS" + CorsoConnectorDataCollectionTests = "CORSO_CONNECTOR_DATA_COLLECTION_TESTS" + CorsoGraphConnectorTests = "CORSO_GRAPH_CONNECTOR_TESTS" + CorsoGraphConnectorExchangeTests = "CORSO_GRAPH_CONNECTOR_EXCHANGE_TESTS" + CorsoGraphConnectorOneDriveTests = "CORSO_GRAPH_CONNECTOR_ONE_DRIVE_TESTS" + CorsoKopiaWrapperTests = "CORSO_KOPIA_WRAPPER_TESTS" + CorsoModelStoreTests = "CORSO_MODEL_STORE_TESTS" + CorsoOneDriveTests = "CORSO_ONE_DRIVE_TESTS" + CorsoOperationTests = "CORSO_OPERATION_TESTS" + CorsoRepositoryTests = "CORSO_REPOSITORY_TESTS" ) // File needs to be a single message .json diff --git a/src/internal/tester/users.go b/src/internal/tester/resource_owners.go similarity index 82% rename from src/internal/tester/users.go rename to src/internal/tester/resource_owners.go index 7e1ff41f1..797941091 100644 --- a/src/internal/tester/users.go +++ b/src/internal/tester/resource_owners.go @@ -57,3 +57,14 @@ func LoadTestM365OrgUsers(t *testing.T) []string { return strings.Split(users, ",") } + +// M365SiteID returns a siteID string representing the m365SiteID described +// by either the env var CORSO_M365_TEST_SITE_ID, the corso_test.toml config +// file or the default value (in that order of priority). The default is a +// last-attempt fallback that will only work on alcion's testing org. +func M365SiteID(t *testing.T) string { + cfg, err := readTestConfig() + require.NoError(t, err, "retrieving m365 site id from test configuration") + + return cfg[TestCfgSiteID] +}