larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: Backup all users flag support for exchange (#631)

Browse Source 
All-users flag support enabled for Graph-Connector
This commit is contained in:
Danny 2022-08-26 17:05:14 -04:00 committed by GitHub
parent 2ceafd8d62
commit ab92358ea6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 80 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/internal/connector/exchange/exchange_service_test.go
View File

@ -353,6 +353,7 @@ func (suite *ExchangeServiceSuite) TestIterativeFunctions() {
// with corresponding item IDs. New collections are created for each directory
callbackFunc := test.iterativeFunction(
"testingTenant",
userID,
test.scope,
errs, false,
suite.es.credentials,

2
src/internal/connector/exchange/service_functions.go
View File

@ -227,7 +227,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
nil
}
if scope.IncludesCategory(selectors.ExchangeContactFolder) {
if scope.IncludesCategory(selectors.ExchangeContact) {
return models.CreateContactFromDiscriminatorValue,
GetAllContactsForUser,
IterateAllContactsForCollection,

19
src/internal/connector/exchange/service_iterators.go
View File

@ -36,6 +36,7 @@ type displayable interface {
// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function
type GraphIterateFunc func(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -50,6 +51,7 @@ type GraphIterateFunc func(
// placed into a Collection based on the parent folder
func IterateSelectAllMessagesForCollections(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -95,6 +97,7 @@ func IterateSelectAllMessagesForCollections(
// the calendarID which originates from M365.
func IterateSelectAllEventsForCollections(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -103,8 +106,6 @@ func IterateSelectAllEventsForCollections(
statusCh chan<- *support.ConnectorOperationStatus,
) func(any) bool {
return func(eventItem any) bool {
user := scope.Get(selectors.ExchangeUser)[0]
event, ok := eventItem.(models.Eventable)
if !ok {
errs = support.WrapAndAppend(
@ -173,6 +174,7 @@ func IterateSelectAllEventsForCollections(
// Contacts Ids are placed into a collection based upon the parent folder
func IterateAllContactsForCollection(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -181,8 +183,6 @@ func IterateAllContactsForCollection(
statusCh chan<- *support.ConnectorOperationStatus,
) func(any) bool {
return func(contactsItem any) bool {
user := scope.Get(selectors.ExchangeUser)[0]
contact, ok := contactsItem.(models.Contactable)
if !ok {
errs = support.WrapAndAppend(user, errors.New("contact iteration failure"), errs)
@ -209,8 +209,12 @@ func IterateAllContactsForCollection(
}
}
// IterateAndFilterMessagesForCollections is a filtering GraphIterateFunc
// that places exchange mail message ids belonging to specific directories
// into a Collection. Messages outside of those directories are omitted.
func IterateAndFilterMessagesForCollections(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -220,7 +224,6 @@ func IterateAndFilterMessagesForCollections(
) func(any) bool {
var isFilterSet bool
return func(messageItem any) bool {
user := scope.Get(selectors.ExchangeUser)[0]
if !isFilterSet {
err := CollectMailFolders(
@ -256,6 +259,7 @@ func IterateAndFilterMessagesForCollections(
func IterateFilterFolderDirectoriesForCollections(
tenant string,
user string,
scope selectors.ExchangeScope,
errs error,
failFast bool,
@ -268,7 +272,6 @@ func IterateFilterFolderDirectoriesForCollections(
err error
)
return func(folderItem any) bool {
user := scope.Get(selectors.ExchangeUser)[0]
folder, ok := folderItem.(models.MailFolderable)
if !ok {
errs = support.WrapAndAppend(
@ -279,6 +282,10 @@ func IterateFilterFolderDirectoriesForCollections(
return true
}
// Continue to iterate if folder name is empty
if folder.GetDisplayName() == nil {
return true
}
if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) {
return true
}

1
src/internal/connector/exchange/service_query.go
View File

@ -133,6 +133,7 @@ func CollectMailFolders(
callbackFunc := IterateFilterFolderDirectoriesForCollections(
tenant,
user,
scope,
err,
failFast,

56
src/internal/connector/graph_connector.go
View File

@ -201,39 +201,23 @@ func (gc *GraphConnector) ExchangeDataCollection(
) ([]data.Collection, error) {
eb, err := selector.ToExchangeBackup()
if err != nil {
return nil, errors.Wrap(err, "collecting exchange data")
return nil, errors.Wrap(err, "exchangeDataCollection: unable to parse selector")
}
scopes := eb.DiscreteScopes(gc.GetUsers())
collections := []data.Collection{}
scopes := eb.Scopes()
var errs error
// for each scope that includes mail messages, get all
for _, scope := range scopes {
for _, user := range scope.Get(selectors.ExchangeUser) {
// TODO: handle "get mail for all users"
// this would probably no-op without this check,
// but we want it made obvious that we're punting.
if user == selectors.AnyTgt {
errs = support.WrapAndAppend(
"all-users",
errors.New("all users selector currently not handled"),
errs)
continue
}
// Creates a map of collections based on scope
dcs, err := gc.createCollections(ctx, scope)
if err != nil {
return nil, support.WrapAndAppend(user, err, errs)
user := scope.Get(selectors.ExchangeUser)
return nil, support.WrapAndAppend(user[0], err, errs)
}
if len(dcs) > 0 {
for _, collection := range dcs {
collections = append(collections, collection)
}
}
}
}
return collections, errs
}
@ -307,17 +291,22 @@ func (gc *GraphConnector) RestoreMessages(ctx context.Context, dcs []data.Collec
func (gc *GraphConnector) createCollections(
ctx context.Context,
scope selectors.ExchangeScope,
) (map[string]*exchange.Collection, error) {
) ([]*exchange.Collection, error) {
var (
transformer absser.ParsableFactory
query exchange.GraphQuery
gIter exchange.GraphIterateFunc
errs error
)
user := scope.Get(selectors.ExchangeUser)[0]
transformer, query, gIter, err := exchange.SetupExchangeCollectionVars(scope)
if err != nil {
return nil, support.WrapAndAppend(user, err, nil)
return nil, support.WrapAndAppend(gc.Service().Adapter().GetBaseUrl(), err, nil)
}
users := scope.Get(selectors.ExchangeUser)
allCollections := make([]*exchange.Collection, 0)
// Create collection of ExchangeDataCollection
for _, user := range users {
collections := make(map[string]*exchange.Collection)
response, err := query(&gc.graphService, user)
if err != nil {
return nil, errors.Wrapf(
@ -330,12 +319,11 @@ func (gc *GraphConnector) createCollections(
if err != nil {
return nil, err
}
// Create collection of ExchangeDataCollection and create data Holder
collections := make(map[string]*exchange.Collection)
var errs error
// callbackFunc iterates through all models.Messageable and fills exchange.Collection.jobs[]
// with corresponding item IDs. New collections are created for each directory
callbackFunc := gIter(gc.tenant, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh)
// callbackFunc iterates through all M365 object target and fills exchange.Collection.jobs[]
// with corresponding item M365IDs. New collections are created for each directory.
// Each directory used the M365 Identifier. The use of ID stops collisions betweens users
callbackFunc := gIter(gc.tenant, user, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh)
iterateError := pageIterator.Iterate(callbackFunc)
if iterateError != nil {
errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs)
@ -343,12 +331,12 @@ func (gc *GraphConnector) createCollections(
if errs != nil {
return nil, errs // return error if snapshot is incomplete
}
for range collections {
for _, collection := range collections {
gc.incrementAwaitingMessages()
allCollections = append(allCollections, collection)
}
return collections, errs
}
return allCollections, errs
}
// AwaitStatus updates status field based on item within statusChannel.

2
src/internal/connector/graph_connector_disconnected_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/data"
"github.com/alcionai/corso/internal/tester"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/credentials"
)
@ -24,6 +25,7 @@ type DisconnectedGraphConnectorSuite struct {
}
func TestDisconnectedGraphSuite(t *testing.T) {
tester.LogTimeOfTest(t)
suite.Run(t, new(DisconnectedGraphConnectorSuite))
}

41
src/internal/connector/graph_connector_test.go
View File

@ -80,6 +80,8 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
// GraphConnector remains stable to receive a non-zero amount of Collections
// for the Exchange Package. Enabled exchange applications:
// - mail
// - contacts
// - events
func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
t := suite.T()
connector := loadConnector(t)
@ -248,35 +250,24 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() {
assert.Equal(t, status.FolderCount, 1)
}
// TestGraphConnector_SingleMailFolderCollectionQuery verifies single folder support
// for Backup operation
func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_SingleMailFolderCollectionQuery() {
// TestAccessOfInboxAllUsers verifies that GraphConnector can
// support `--all-users` for backup operations. Selector.DiscreteScopes
// returns all of the users within one scope. Only users who have
// messages in their inbox will have a collection returned.
// The final test insures that more than a 75% of the user collections are
// returned. If an error was experienced, the test will fail overall
func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() {
t := suite.T()
connector := loadConnector(t)
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{suite.user}, []string{"Inbox"}))
scopes := sel.Scopes()
sel.Include(sel.MailFolders(selectors.Any(), []string{"Inbox"}))
scopes := sel.DiscreteScopes(connector.GetUsers())
for _, scope := range scopes {
collections, err := suite.connector.createCollections(context.Background(), scope)
users := scope.Get(selectors.ExchangeUser)
standard := (len(users) / 4) * 3
collections, err := connector.createCollections(context.Background(), scope)
require.NoError(t, err)
suite.Equal(len(collections), 1)
for _, edc := range collections {
number := 0
streamChannel := edc.Items()
// Verify that each message can be restored
for stream := range streamChannel {
testName := fmt.Sprintf("%s_InboxMessage_%d", edc.FullPath()[1], number)
suite.T().Run(testName, func(t *testing.T) {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
assert.NoError(t, err)
assert.NotZero(t, read)
message, err := support.CreateMessageFromBytes(buf.Bytes())
assert.NotNil(t, message)
assert.NoError(t, err)
number++
})
}
}
suite.Greater(len(collections), standard)
}
}