GC: Accept selectors.BackupExchange exchange.Event and exchange.Contacts (#605)

GraphConnector suite expanded to support backup of Mail, Contacts, and Events. e2e testing suite expanded for all 3 applications.

src/cli/backup/exchange_integration_test.go

@@ -159,8 +159,8 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
 
 	// some tests require an existing backup
 	sel := selectors.NewExchangeBackup()
-	// TODO: only backup the inbox
+	sel.Include(sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"}))
-	sel.Include(sel.Users([]string{suite.m365UserID}))
+
 	suite.backupOp, err = suite.repo.NewBackup(
 		ctx,
 		sel.Selector,

src/internal/connector/graph_connector.go

@@ -206,10 +206,6 @@ func (gc *GraphConnector) ExchangeDataCollection(
 
 	// for each scope that includes mail messages, get all
 	for _, scope := range scopes {
-		if !scope.IncludesCategory(selectors.ExchangeMail) {
-			continue
-		}
-
 		for _, user := range scope.Get(selectors.ExchangeUser) {
 			// TODO: handle "get mail for all users"
 			// this would probably no-op without this check,
@@ -355,8 +351,8 @@ func (gc *GraphConnector) createCollections(
 // AwaitStatus updates status field based on item within statusChannel.
 func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
 	if gc.awaitingMessages > 0 {
-		gc.status = <-gc.statusCh
 		atomic.AddInt32(&gc.awaitingMessages, -1)
+		gc.status = <-gc.statusCh
 	}
 	return gc.status
 }

src/internal/connector/graph_connector_test.go

@@ -87,18 +87,32 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
 	sel.Include(sel.Users([]string{suite.user}))
 	collectionList, err := connector.ExchangeDataCollection(context.Background(), sel.Selector)
 	assert.NotNil(t, collectionList, "collection list")
-	assert.Nil(t, err)
+	assert.NoError(t, err)
 	assert.True(t, connector.awaitingMessages > 0)
 	assert.Nil(t, connector.status)
+	streams := make(map[string]<-chan data.Stream)
 	// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
-	channel := collectionList[0].Items()
+	for _, collection := range collectionList {
-	for object := range channel {
+		temp := collection.Items()
-		buf := &bytes.Buffer{}
+		testName := collection.FullPath()[2]
-		_, err := buf.ReadFrom(object.ToReader())
+		streams[testName] = temp
-		assert.Nil(suite.T(), err, "received a buf.Read error")
+	}
+
+	for i := 0; i < int(connector.awaitingMessages); i++ {
+		status := connector.AwaitStatus()
+		assert.NotNil(t, status)
+	}
+
+	for name, channel := range streams {
+		suite.T().Run(name, func(t *testing.T) {
+			t.Logf("Test: %s\t Items: %d", name, len(channel))
+			for object := range channel {
+				buf := &bytes.Buffer{}
+				_, err := buf.ReadFrom(object.ToReader())
+				assert.NoError(t, err, "received a buf.Read error")
+			}
+		})
 	}
-	status := connector.AwaitStatus()
-	assert.NotNil(t, status, "status not blocking on async call")
 	exchangeData := collectionList[0]
 	suite.Greater(len(exchangeData.FullPath()), 2)
 }
@@ -110,11 +124,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
 	t := suite.T()
 	connector := loadConnector(t)
 	sel := selectors.NewExchangeBackup()
-	sel.Include(sel.MailFolders([]string{suite.user}, []string{selectors.AnyTgt}))
+	sel.Include(sel.MailFolders([]string{suite.user}, selectors.Any()))
 	eb, err := sel.ToExchangeBackup()
 	require.NoError(t, err)
 	scopes := eb.Scopes()
-	suite.Equal(1, len(scopes))
+	suite.Len(scopes, 1)
 	mailScope := scopes[0]
 	collection, err := connector.createCollections(context.Background(), mailScope)
 	require.NoError(t, err)
@@ -126,11 +140,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
 			for stream := range streamChannel {
 				buf := &bytes.Buffer{}
 				read, err := buf.ReadFrom(stream.ToReader())
-				suite.NoError(err)
+				assert.NoError(t, err)
-				suite.NotZero(read)
+				assert.NotZero(t, read)
 				message, err := support.CreateMessageFromBytes(buf.Bytes())
-				suite.NotNil(message)
+				assert.NotNil(t, message)
-				suite.NoError(err)
+				assert.NoError(t, err)
 			}
 		})
 	}
@@ -145,12 +159,12 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
 func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression() {
 	t := suite.T()
 	sel := selectors.NewExchangeBackup()
-	sel.Include(sel.ContactFolders([]string{suite.user}, []string{selectors.AnyTgt}))
+	sel.Include(sel.ContactFolders([]string{suite.user}, selectors.Any()))
 	eb, err := sel.ToExchangeBackup()
 	require.NoError(t, err)
 	scopes := eb.Scopes()
 	connector := loadConnector(t)
-	suite.Equal(1, len(scopes))
+	suite.Len(scopes, 1)
 	contactsOnly := scopes[0]
 	collections, err := connector.createCollections(context.Background(), contactsOnly)
 	assert.NoError(t, err)
@@ -162,8 +176,8 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
 			for stream := range streamChannel {
 				buf := &bytes.Buffer{}
 				read, err := buf.ReadFrom(stream.ToReader())
-				suite.NoError(err)
+				assert.NoError(t, err)
-				suite.NotZero(read)
+				assert.NotZero(t, read)
 				contact, err := support.CreateContactFromBytes(buf.Bytes())
 				assert.NotNil(t, contact)
 				assert.NoError(t, err)
@@ -183,7 +197,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
 	t := suite.T()
 	connector := loadConnector(t)
 	sel := selectors.NewExchangeBackup()
-	sel.Include(sel.Events([]string{suite.user}, []string{selectors.AnyTgt}))
+	sel.Include(sel.Events([]string{suite.user}, selectors.Any()))
 	scopes := sel.Scopes()
 	suite.Equal(len(scopes), 1)
 	collections, err := connector.createCollections(context.Background(), scopes[0])
@@ -196,8 +210,8 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
 			suite.T().Run(testName, func(t *testing.T) {
 				buf := &bytes.Buffer{}
 				read, err := buf.ReadFrom(stream.ToReader())
-				suite.NoError(err)
+				assert.NoError(t, err)
-				suite.NotZero(read)
+				assert.NotZero(t, read)
 				event, err := support.CreateEventFromBytes(buf.Bytes())
 				assert.NotNil(t, event)
 				assert.NoError(t, err)
@@ -241,11 +255,11 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder
 				suite.T().Run(testName, func(t *testing.T) {
 					buf := &bytes.Buffer{}
 					read, err := buf.ReadFrom(stream.ToReader())
-					suite.NoError(err)
+					assert.NoError(t, err)
-					suite.NotZero(read)
+					assert.NotZero(t, read)
 					message, err := support.CreateMessageFromBytes(buf.Bytes())
-					suite.NotNil(message)
+					assert.NotNil(t, message)
-					suite.NoError(err)
+					assert.NoError(t, err)
 					number++
 				})
 			}

src/internal/connector/support/m365Support_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/internal/connector/mockconnector"
-	"github.com/alcionai/corso/internal/tester"
 )
 
 type DataSupportSuite struct {
@@ -15,10 +14,6 @@ type DataSupportSuite struct {
 }
 
 func TestDataSupportSuite(t *testing.T) {
-	err := tester.RunOnAny(tester.CorsoGraphConnectorTestSupportFile)
-	if err != nil {
-		t.Skipf("Skipping: %v\n", err)
-	}
 	suite.Run(t, new(DataSupportSuite))
 }
 

src/internal/connector/support/m365Transform_test.go

@@ -1,13 +1,12 @@
 package support
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
-	"github.com/alcionai/corso/internal/tester"
+	"github.com/alcionai/corso/internal/connector/mockconnector"
 )
 
 type SupportTestSuite struct {
@@ -15,23 +14,11 @@ type SupportTestSuite struct {
 }
 
 func TestSupportTestSuite(t *testing.T) {
-	evs, err := tester.GetRequiredEnvVars(tester.CorsoGraphConnectorTestSupportFile)
-	if err != nil {
-		t.Skipf("Env not configured: %v\n", err)
-	}
-	_, err = os.Stat(evs[tester.CorsoGraphConnectorTestSupportFile])
-	if err != nil {
-		t.Skip("Test object not available: Module Skipped")
-	}
 	suite.Run(t, new(SupportTestSuite))
 }
 
 func (suite *SupportTestSuite) TestToMessage() {
-	bytes, err := tester.LoadAFile(os.Getenv(tester.CorsoGraphConnectorTestSupportFile))
+	bytes := mockconnector.GetMockMessageBytes("m365 mail support test")
-	if err != nil {
-		suite.T().Errorf("Failed with %v\n", err)
-	}
-	require.NoError(suite.T(), err)
 	message, err := CreateMessageFromBytes(bytes)
 	require.NoError(suite.T(), err)
 	clone := ToMessage(message)

src/internal/operations/backup_test.go

@@ -124,6 +124,8 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
 	}
 }
 
+// TestBackup_Run ensures that Integration Testing works
+// for the following scopes: Contacts, Events, and Mail
 func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
 	t := suite.T()
 	ctx := context.Background()
@@ -131,44 +133,77 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
 	m365UserID := tester.M365UserID(t)
 	acct := tester.NewM365Account(t)
 
-	// need to initialize the repository before we can test connecting to it.
+	tests := []struct {
-	st := tester.NewPrefixedS3Storage(t)
+		name       string
+		selectFunc func() *selectors.Selector
+	}{
+		{
+			name: "Integration Exchange.Mail",
+			selectFunc: func() *selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.MailFolders([]string{m365UserID}, []string{"Inbox"}))
+				return &sel.Selector
+			},
+		},
 
-	k := kopia.NewConn(st)
+		{
-	require.NoError(t, k.Initialize(ctx))
+			name: "Integration Exchange.Contacts",
+			selectFunc: func() *selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.ContactFolders([]string{m365UserID}, selectors.Any()))
+				return &sel.Selector
+			},
+		},
+		{
+			name: "Integration Exchange.Events",
+			selectFunc: func() *selectors.Selector {
+				sel := selectors.NewExchangeBackup()
+				sel.Include(sel.Events([]string{m365UserID}, selectors.Any()))
 
-	// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
+				return &sel.Selector
-	// to close here.
+			},
-	defer k.Close(ctx)
+		},
+	}
 
-	kw, err := kopia.NewWrapper(k)
+	for _, test := range tests {
-	require.NoError(t, err)
+		suite.T().Run(test.name, func(t *testing.T) {
-	defer kw.Close(ctx)
+			// need to initialize the repository before we can test connecting to it.
+			st := tester.NewPrefixedS3Storage(t)
+			k := kopia.NewConn(st)
+			require.NoError(t, k.Initialize(ctx))
 
-	ms, err := kopia.NewModelStore(k)
+			// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
-	require.NoError(t, err)
+			// to close here.
-	defer ms.Close(ctx)
+			defer k.Close(ctx)
 
-	sw := store.NewKopiaStore(ms)
+			kw, err := kopia.NewWrapper(k)
+			require.NoError(t, err)
+			defer kw.Close(ctx)
 
-	sel := selectors.NewExchangeBackup()
+			ms, err := kopia.NewModelStore(k)
-	sel.Include(sel.Users([]string{m365UserID}))
+			require.NoError(t, err)
+			defer ms.Close(ctx)
 
-	bo, err := NewBackupOperation(
+			sw := store.NewKopiaStore(ms)
-		ctx,
+			selected := test.selectFunc()
-		control.Options{},
+			bo, err := NewBackupOperation(
-		kw,
+				ctx,
-		sw,
+				control.Options{},
-		acct,
+				kw,
-		sel.Selector)
+				sw,
-	require.NoError(t, err)
+				acct,
+				*selected,
+			)
+			require.NoError(t, err)
 
-	require.NoError(t, bo.Run(ctx))
+			require.NoError(t, bo.Run(ctx))
-	require.NotEmpty(t, bo.Results)
+			require.NotEmpty(t, bo.Results)
-	require.NotEmpty(t, bo.Results.BackupID)
+			require.NotEmpty(t, bo.Results.BackupID)
-	assert.Equal(t, bo.Status, Completed)
+			assert.Equal(t, bo.Status, Completed)
-	assert.Greater(t, bo.Results.ItemsRead, 0)
+			assert.Greater(t, bo.Results.ItemsRead, 0)
-	assert.Greater(t, bo.Results.ItemsWritten, 0)
+			assert.Greater(t, bo.Results.ItemsWritten, 0)
-	assert.Zero(t, bo.Results.ReadErrors)
+			assert.Zero(t, bo.Results.ReadErrors)
-	assert.Zero(t, bo.Results.WriteErrors)
+			assert.Zero(t, bo.Results.WriteErrors)
+		})
+	}
 }