larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: Backup: Contacts: Ability to filter on folder [FEATURE] (#891)

Browse Source 
## Description
PR enables Filtering for `exchange.Contact` items wherein if an `exchange.ContactFolder` is requested specifically, only the requested folders are backed up. 
<!-- Insert PR description-->

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature


## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
*closes  #811<issue>

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Danny 2022-09-19 13:42:12 -04:00 committed by GitHub
parent 423b6e19f7
commit f250665dd6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 172 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/internal/connector/exchange/exchange_vars.go
View File

@ -26,6 +26,12 @@ const (
// MailReceiveDateTimeOverrideProperty allows receive date time to be updated.
// Section: 2.789 PidTagMessageDeliveryTime
MailReceiveDateTimeOverriveProperty = "SystemTime 0x0E06"
//----------------------------------
// Default Folder Names
//------------------------
// Mail Definitions: https://docs.microsoft.com/en-us/graph/api/resources/mailfolder?view=graph-rest-1.0
DefaultContactFolder = "Contacts"
)
// descendable represents objects that implement msgraph-sdk-go/models.entityable

8
src/internal/connector/exchange/iterators_test.go
View File

@ -143,7 +143,7 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() {
scope: eventScope,
transformer: models.CreateCalendarCollectionResponseFromDiscriminatorValue,
}, {
name: "Folder Iterative Check",
name: "Folder Iterative Check Mail",
queryFunction: GetAllFolderNamesForUser,
iterativeFunction: IterateFilterFolderDirectoriesForCollections,
scope: mailScope,
@ -153,6 +153,12 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() {
"Sent Items": {},
"Deleted Items": {},
},
}, {
name: "Folder Iterative Check Contacts",
queryFunction: GetAllContactFolderNamesForUser,
iterativeFunction: IterateFilterFolderDirectoriesForCollections,
scope: contactScope,
transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
},
}
for _, test := range tests {

14
src/internal/connector/exchange/query_options.go
View File

@ -16,6 +16,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
//-----------------------------------------------------------------------
@ -115,6 +116,19 @@ func categoryToOptionIdentifier(category path.CategoryType) optionIdentifier {
}
}
func scopeToOptionIdentifier(selector selectors.ExchangeScope) optionIdentifier {
switch selector.Category() {
case selectors.ExchangeMailFolder, selectors.ExchangeMail:
return messages
case selectors.ExchangeContactFolder, selectors.ExchangeContact:
return contacts
case selectors.ExchangeEventCalendar, selectors.ExchangeEvent:
return events
default:
return unknown
}
}
//---------------------------------------------------------------------------
// exchange.Query Option Section
// These functions can be used to filter a response on M365

16
src/internal/connector/exchange/service_functions.go
View File

@ -335,14 +335,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
return models.CreateMessageCollectionResponseFromDiscriminatorValue,
GetAllMessagesForUser,
IterateAndFilterMessagesForCollections,
nil
}
if scope.IncludesCategory(selectors.ExchangeEvent) {
return models.CreateCalendarCollectionResponseFromDiscriminatorValue,
GetAllCalendarNamesForUser,
IterateSelectAllEventsFromCalendars,
IterateAndFilterDescendablesForCollections,
nil
}
@ -353,6 +346,13 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
nil
}
if scope.IncludesCategory(selectors.ExchangeEvent) {
return models.CreateCalendarCollectionResponseFromDiscriminatorValue,
GetAllCalendarNamesForUser,
IterateSelectAllEventsFromCalendars,
nil
}
return nil, nil, nil, errors.New("exchange scope option not supported")
}

208
src/internal/connector/exchange/service_iterators.go
View File

@ -206,10 +206,10 @@ func IterateSelectAllEventsFromCalendars(
}
}
// IterateAndFilterMessagesForCollections is a filtering GraphIterateFunc
// that places exchange mail message ids belonging to specific directories
// IterateAndFilterDescendablesForCollections is a filtering GraphIterateFunc
// that places exchange objectsids belonging to specific directories
// into a Collection. Messages outside of those directories are omitted.
func IterateAndFilterMessagesForCollections(
func IterateAndFilterDescendablesForCollections(
ctx context.Context,
qp graph.QueryParams,
errUpdater func(string, error),
@ -218,9 +218,9 @@ func IterateAndFilterMessagesForCollections(
) func(any) bool {
var isFilterSet bool
return func(messageItem any) bool {
return func(descendItem any) bool {
if !isFilterSet {
err := CollectMailFolders(
err := CollectFolders(
ctx,
qp,
collections,
@ -234,7 +234,7 @@ func IterateAndFilterMessagesForCollections(
isFilterSet = true
}
message, ok := messageItem.(descendable)
message, ok := descendItem.(descendable)
if !ok {
errUpdater(qp.User, errors.New("casting messageItem to descendable"))
return true
@ -259,38 +259,66 @@ func IterateFilterFolderDirectoriesForCollections(
statusUpdater support.StatusUpdater,
) func(any) bool {
var (
service graph.Service
err error
resolver graph.ContainerResolver
isSet bool
collectPath string
err error
option optionIdentifier
category path.CategoryType
validate func(string) bool
)
resolver, err := maybeGetAndPopulateFolderResolver(ctx, qp, path.EmailCategory)
if err != nil {
errUpdater("getting folder resolver for category email", err)
}
return func(folderItem any) bool {
folder, ok := folderItem.(displayable)
if !ok {
errUpdater(qp.User, errors.New("casting folderItem to displayable"))
return true
}
if !isSet {
option = scopeToOptionIdentifier(qp.Scope)
switch option {
case messages:
category = path.EmailCategory
validate = func(name string) bool {
return !qp.Scope.Matches(selectors.ExchangeMailFolder, name)
}
case contacts:
category = path.ContactsCategory
validate = func(name string) bool {
return !qp.Scope.Matches(selectors.ExchangeContactFolder, name)
}
}
resolver, err = maybeGetAndPopulateFolderResolver(ctx, qp, category)
if err != nil {
errUpdater("getting folder resolver for category "+category.String(), err)
}
isSet = true
}
// Continue to iterate if folder name is empty
if folder.GetDisplayName() == nil {
return true
}
if !qp.Scope.Matches(selectors.ExchangeMailFolder, *folder.GetDisplayName()) {
if validate(*folder.GetDisplayName()) {
return true
}
directory := *folder.GetId()
if option == contacts {
collectPath = *folder.GetDisplayName()
} else {
collectPath = *folder.GetId()
}
dirPath, err := getCollectionPath(
ctx,
qp,
resolver,
directory,
path.EmailCategory,
collectPath,
category,
)
if err != nil {
errUpdater(
@ -301,7 +329,7 @@ func IterateFilterFolderDirectoriesForCollections(
return true
}
service, err = createService(qp.Credentials, qp.FailFast)
service, err := createService(qp.Credentials, qp.FailFast)
if err != nil {
errUpdater(
*folder.GetDisplayName(),
@ -313,11 +341,11 @@ func IterateFilterFolderDirectoriesForCollections(
temp := NewCollection(
qp.User,
dirPath,
messages,
option,
service,
statusUpdater,
)
collections[directory] = &temp
collections[*folder.GetId()] = &temp
return true
}
@ -330,7 +358,10 @@ func IterateSelectAllContactsForCollections(
collections map[string]*Collection,
statusUpdater support.StatusUpdater,
) func(any) bool {
var isPrimarySet bool
var (
isPrimarySet bool
service graph.Service
)
return func(folderItem any) bool {
folder, ok := folderItem.(models.ContactFolderable)
@ -342,7 +373,18 @@ func IterateSelectAllContactsForCollections(
}
if !isPrimarySet && folder.GetParentFolderId() != nil {
service, err := createService(qp.Credentials, qp.FailFast)
err := CollectFolders(
ctx,
qp,
collections,
statusUpdater,
)
if err != nil {
errUpdater(qp.User, err)
return false
}
service, err = createService(qp.Credentials, qp.FailFast)
if err != nil {
errUpdater(
qp.User,
@ -352,44 +394,58 @@ func IterateSelectAllContactsForCollections(
return true
}
contactIDS, err := ReturnContactIDsFromDirectory(service, qp.User, *folder.GetParentFolderId())
if err != nil {
errUpdater(
// Create and Populate Default Contacts folder Collection if true
if qp.Scope.Matches(selectors.ExchangeContactFolder, DefaultContactFolder) {
dirPath, err := path.Builder{}.Append(DefaultContactFolder).ToDataLayerExchangePathForCategory(
qp.Credentials.TenantID,
qp.User,
err,
path.ContactsCategory,
false,
)
if err != nil {
errUpdater(
qp.User,
err,
)
return false
}
edc := NewCollection(
qp.User,
dirPath,
contacts,
service,
statusUpdater,
)
return true
listOfIDs, err := ReturnContactIDsFromDirectory(service, qp.User, *folder.GetParentFolderId())
if err != nil {
errUpdater(
qp.User,
err,
)
return false
}
edc.jobs = append(edc.jobs, listOfIDs...)
collections[DefaultContactFolder] = &edc
isPrimarySet = true
}
dirPath, err := path.Builder{}.Append(*folder.GetParentFolderId()).ToDataLayerExchangePathForCategory(
qp.Credentials.TenantID,
qp.User,
path.ContactsCategory,
false,
)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
edc := NewCollection(
qp.User,
dirPath,
contacts,
service,
statusUpdater,
)
edc.jobs = append(edc.jobs, contactIDS...)
collections["Contacts"] = &edc
isPrimarySet = true
}
service, err := createService(qp.Credentials, qp.FailFast)
if folder.GetDisplayName() == nil {
// This should never happen. Skipping to avoid kernel panic
return true
}
collection, ok := collections[*folder.GetDisplayName()]
if !ok {
return true // Not included
}
listOfIDs, err := ReturnContactIDsFromDirectory(service, qp.User, *folder.GetId())
if err != nil {
errUpdater(
qp.User,
@ -399,49 +455,7 @@ func IterateSelectAllContactsForCollections(
return true
}
folderID := *folder.GetId()
listOfIDs, err := ReturnContactIDsFromDirectory(service, qp.User, folderID)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
if folder.GetDisplayName() == nil ||
listOfIDs == nil {
return true // Invalid state TODO: How should this be named
}
directory := *folder.GetDisplayName()
dirPath, err := path.Builder{}.Append(directory).ToDataLayerExchangePathForCategory(
qp.Credentials.TenantID,
qp.User,
path.ContactsCategory,
false,
)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
edc := NewCollection(
qp.User,
dirPath,
contacts,
service,
statusUpdater,
)
edc.jobs = append(edc.jobs, listOfIDs...)
collections[directory] = &edc
collection.jobs = append(collection.jobs, listOfIDs...)
return true
}

32
src/internal/connector/exchange/service_query.go
View File

@ -122,18 +122,38 @@ func RetrieveMessageDataForUser(gs graph.Service, user, m365ID string) (absser.P
return gs.Client().UsersById(user).MessagesById(m365ID).Get()
}
func CollectMailFolders(
func CollectFolders(
ctx context.Context,
qp graph.QueryParams,
collections map[string]*Collection,
statusUpdater support.StatusUpdater,
) error {
queryService, err := createService(qp.Credentials, qp.FailFast)
var (
query GraphQuery
transformer absser.ParsableFactory
queryService, err = createService(qp.Credentials, qp.FailFast)
)
if err != nil {
return errors.New("unable to create a mail folder query service for " + qp.User)
return errors.Wrapf(
err,
"unable to create graph.Service within CollectFolders service for "+qp.User,
)
}
query, err := GetAllFolderNamesForUser(queryService, qp.User)
option := scopeToOptionIdentifier(qp.Scope)
switch option {
case messages:
query = GetAllFolderNamesForUser
transformer = models.CreateMailFolderCollectionResponseFromDiscriminatorValue
case contacts:
query = GetAllContactFolderNamesForUser
transformer = models.CreateContactFolderCollectionResponseFromDiscriminatorValue
default:
return fmt.Errorf("unsupported option %s used in CollectFolders", option)
}
response, err := query(queryService, qp.User)
if err != nil {
return fmt.Errorf(
"unable to query mail folder for %s: details: %s",
@ -145,9 +165,9 @@ func CollectMailFolders(
// Iterator required to ensure all potential folders are inspected
// when the breadth of the folder space is large
pageIterator, err := msgraphgocore.NewPageIterator(
query,
response,
&queryService.adapter,
models.CreateMailFolderCollectionResponseFromDiscriminatorValue)
transformer)
if err != nil {
return errors.Wrap(err, "unable to create iterator during mail folder query service")
}