larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: Accept selectors.BackupExchange exchange.Event and exchange.Contacts (#605)

Browse Source 
GraphConnector suite expanded to support backup of Mail, Contacts, and Events. e2e testing suite expanded for all 3 applications.
This commit is contained in:
Danny 2022-08-22 16:52:01 -04:00 committed by GitHub
parent d954c68216
commit b573882746
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 111 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli/backup/exchange_integration_test.go
View File

@ -159,8 +159,8 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
// some tests require an existing backup
sel := selectors.NewExchangeBackup()
// TODO: only backup the inbox
sel.Include(sel.Users([]string{suite.m365UserID}))
sel.Include(sel.MailFolders([]string{suite.m365UserID}, []string{"Inbox"}))
suite.backupOp, err = suite.repo.NewBackup(
ctx,
sel.Selector,

6
src/internal/connector/graph_connector.go
View File

@ -206,10 +206,6 @@ func (gc *GraphConnector) ExchangeDataCollection(
// for each scope that includes mail messages, get all
for _, scope := range scopes {
if !scope.IncludesCategory(selectors.ExchangeMail) {
continue
}
for _, user := range scope.Get(selectors.ExchangeUser) {
// TODO: handle "get mail for all users"
// this would probably no-op without this check,
@ -355,8 +351,8 @@ func (gc *GraphConnector) createCollections(
// AwaitStatus updates status field based on item within statusChannel.
func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
if gc.awaitingMessages > 0 {
gc.status = <-gc.statusCh
atomic.AddInt32(&gc.awaitingMessages, -1)
gc.status = <-gc.statusCh
}
return gc.status
}

64
src/internal/connector/graph_connector_test.go
View File

@ -87,18 +87,32 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
sel.Include(sel.Users([]string{suite.user}))
collectionList, err := connector.ExchangeDataCollection(context.Background(), sel.Selector)
assert.NotNil(t, collectionList, "collection list")
assert.Nil(t, err)
assert.NoError(t, err)
assert.True(t, connector.awaitingMessages > 0)
assert.Nil(t, connector.status)
streams := make(map[string]<-chan data.Stream)
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
channel := collectionList[0].Items()
for object := range channel {
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(object.ToReader())
assert.Nil(suite.T(), err, "received a buf.Read error")
for _, collection := range collectionList {
temp := collection.Items()
testName := collection.FullPath()[2]
streams[testName] = temp
}
for i := 0; i < int(connector.awaitingMessages); i++ {
status := connector.AwaitStatus()
assert.NotNil(t, status)
}
for name, channel := range streams {
suite.T().Run(name, func(t *testing.T) {
t.Logf("Test: %s\t Items: %d", name, len(channel))
for object := range channel {
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(object.ToReader())
assert.NoError(t, err, "received a buf.Read error")
}
})
}
status := connector.AwaitStatus()
assert.NotNil(t, status, "status not blocking on async call")
exchangeData := collectionList[0]
suite.Greater(len(exchangeData.FullPath()), 2)
}
@ -110,11 +124,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
t := suite.T()
connector := loadConnector(t)
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{suite.user}, []string{selectors.AnyTgt}))
sel.Include(sel.MailFolders([]string{suite.user}, selectors.Any()))
eb, err := sel.ToExchangeBackup()
require.NoError(t, err)
scopes := eb.Scopes()
suite.Equal(1, len(scopes))
suite.Len(scopes, 1)
mailScope := scopes[0]
collection, err := connector.createCollections(context.Background(), mailScope)
require.NoError(t, err)
@ -126,11 +140,11 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
for stream := range streamChannel {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
suite.NoError(err)
suite.NotZero(read)
assert.NoError(t, err)
assert.NotZero(t, read)
message, err := support.CreateMessageFromBytes(buf.Bytes())
suite.NotNil(message)
suite.NoError(err)
assert.NotNil(t, message)
assert.NoError(t, err)
}
})
}
@ -145,12 +159,12 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression() {
t := suite.T()
sel := selectors.NewExchangeBackup()
sel.Include(sel.ContactFolders([]string{suite.user}, []string{selectors.AnyTgt}))
sel.Include(sel.ContactFolders([]string{suite.user}, selectors.Any()))
eb, err := sel.ToExchangeBackup()
require.NoError(t, err)
scopes := eb.Scopes()
connector := loadConnector(t)
suite.Equal(1, len(scopes))
suite.Len(scopes, 1)
contactsOnly := scopes[0]
collections, err := connector.createCollections(context.Background(), contactsOnly)
assert.NoError(t, err)
@ -162,8 +176,8 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
for stream := range streamChannel {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
suite.NoError(err)
suite.NotZero(read)
assert.NoError(t, err)
assert.NotZero(t, read)
contact, err := support.CreateContactFromBytes(buf.Bytes())
assert.NotNil(t, contact)
assert.NoError(t, err)
@ -183,7 +197,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
t := suite.T()
connector := loadConnector(t)
sel := selectors.NewExchangeBackup()
sel.Include(sel.Events([]string{suite.user}, []string{selectors.AnyTgt}))
sel.Include(sel.Events([]string{suite.user}, selectors.Any()))
scopes := sel.Scopes()
suite.Equal(len(scopes), 1)
collections, err := connector.createCollections(context.Background(), scopes[0])
@ -196,8 +210,8 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
suite.T().Run(testName, func(t *testing.T) {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
suite.NoError(err)
suite.NotZero(read)
assert.NoError(t, err)
assert.NotZero(t, read)
event, err := support.CreateEventFromBytes(buf.Bytes())
assert.NotNil(t, event)
assert.NoError(t, err)
@ -241,11 +255,11 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolder
suite.T().Run(testName, func(t *testing.T) {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
suite.NoError(err)
suite.NotZero(read)
assert.NoError(t, err)
assert.NotZero(t, read)
message, err := support.CreateMessageFromBytes(buf.Bytes())
suite.NotNil(message)
suite.NoError(err)
assert.NotNil(t, message)
assert.NoError(t, err)
number++
})
}

5
src/internal/connector/support/m365Support_test.go
View File

@ -7,7 +7,6 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/tester"
)
type DataSupportSuite struct {
@ -15,10 +14,6 @@ type DataSupportSuite struct {
}
func TestDataSupportSuite(t *testing.T) {
err := tester.RunOnAny(tester.CorsoGraphConnectorTestSupportFile)
if err != nil {
t.Skipf("Skipping: %v\n", err)
}
suite.Run(t, new(DataSupportSuite))
}

17
src/internal/connector/support/m365Transform_test.go
View File

@ -1,13 +1,12 @@
package support
import (
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/internal/tester"
"github.com/alcionai/corso/internal/connector/mockconnector"
)
type SupportTestSuite struct {
@ -15,23 +14,11 @@ type SupportTestSuite struct {
}
func TestSupportTestSuite(t *testing.T) {
evs, err := tester.GetRequiredEnvVars(tester.CorsoGraphConnectorTestSupportFile)
if err != nil {
t.Skipf("Env not configured: %v\n", err)
}
_, err = os.Stat(evs[tester.CorsoGraphConnectorTestSupportFile])
if err != nil {
t.Skip("Test object not available: Module Skipped")
}
suite.Run(t, new(SupportTestSuite))
}
func (suite *SupportTestSuite) TestToMessage() {
bytes, err := tester.LoadAFile(os.Getenv(tester.CorsoGraphConnectorTestSupportFile))
if err != nil {
suite.T().Errorf("Failed with %v\n", err)
}
require.NoError(suite.T(), err)
bytes := mockconnector.GetMockMessageBytes("m365 mail support test")
message, err := CreateMessageFromBytes(bytes)
require.NoError(suite.T(), err)
clone := ToMessage(message)

99
src/internal/operations/backup_test.go
View File

@ -124,6 +124,8 @@ func (suite *BackupOpIntegrationSuite) TestNewBackupOperation() {
}
}
// TestBackup_Run ensures that Integration Testing works
// for the following scopes: Contacts, Events, and Mail
func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
t := suite.T()
ctx := context.Background()
@ -131,44 +133,77 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
m365UserID := tester.M365UserID(t)
acct := tester.NewM365Account(t)
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
tests := []struct {
name string
selectFunc func() *selectors.Selector
}{
{
name: "Integration Exchange.Mail",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{m365UserID}, []string{"Inbox"}))
return &sel.Selector
},
},
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx))
{
name: "Integration Exchange.Contacts",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.ContactFolders([]string{m365UserID}, selectors.Any()))
return &sel.Selector
},
},
{
name: "Integration Exchange.Events",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.Events([]string{m365UserID}, selectors.Any()))
// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
// to close here.
defer k.Close(ctx)
return &sel.Selector
},
},
}
kw, err := kopia.NewWrapper(k)
require.NoError(t, err)
defer kw.Close(ctx)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
// need to initialize the repository before we can test connecting to it.
st := tester.NewPrefixedS3Storage(t)
k := kopia.NewConn(st)
require.NoError(t, k.Initialize(ctx))
ms, err := kopia.NewModelStore(k)
require.NoError(t, err)
defer ms.Close(ctx)
// kopiaRef comes with a count of 1 and Wrapper bumps it again so safe
// to close here.
defer k.Close(ctx)
sw := store.NewKopiaStore(ms)
kw, err := kopia.NewWrapper(k)
require.NoError(t, err)
defer kw.Close(ctx)
sel := selectors.NewExchangeBackup()
sel.Include(sel.Users([]string{m365UserID}))
ms, err := kopia.NewModelStore(k)
require.NoError(t, err)
defer ms.Close(ctx)
bo, err := NewBackupOperation(
ctx,
control.Options{},
kw,
sw,
acct,
sel.Selector)
require.NoError(t, err)
sw := store.NewKopiaStore(ms)
selected := test.selectFunc()
bo, err := NewBackupOperation(
ctx,
control.Options{},
kw,
sw,
acct,
*selected,
)
require.NoError(t, err)
require.NoError(t, bo.Run(ctx))
require.NotEmpty(t, bo.Results)
require.NotEmpty(t, bo.Results.BackupID)
assert.Equal(t, bo.Status, Completed)
assert.Greater(t, bo.Results.ItemsRead, 0)
assert.Greater(t, bo.Results.ItemsWritten, 0)
assert.Zero(t, bo.Results.ReadErrors)
assert.Zero(t, bo.Results.WriteErrors)
require.NoError(t, bo.Run(ctx))
require.NotEmpty(t, bo.Results)
require.NotEmpty(t, bo.Results.BackupID)
assert.Equal(t, bo.Status, Completed)
assert.Greater(t, bo.Results.ItemsRead, 0)
assert.Greater(t, bo.Results.ItemsWritten, 0)
assert.Zero(t, bo.Results.ReadErrors)
assert.Zero(t, bo.Results.WriteErrors)
})
}
}