larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GC: Backup: Contacts: Retrieve IDs from all Contact Folders (#830)

Browse Source 
## Description

Adds the ability to retrieve M365 IDs of contacts not in the default folder. 

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature


## Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes #804 
* closes #767<issue>
* closes #669

## Test Plan
- [x] ⚡ Unit test
This commit is contained in:
Danny 2022-09-15 17:38:52 -04:00 committed by GitHub
parent 6d5540cdf6
commit 87014a132b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 281 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
src/internal/connector/exchange/exchange_service_test.go
View File

@ -5,7 +5,6 @@ import (
"testing"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -305,117 +304,11 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
}
}
// TestParseCalendarIDFromEvent verifies that parse function
// works on the current accepted reference format of
// additional data["calendar@odata.associationLink"]
func (suite *ExchangeServiceSuite) TestParseCalendarIDFromEvent() {
tests := []struct {
name string
input string
checkError assert.ErrorAssertionFunc
}{
{
name: "Empty string",
input: "",
checkError: assert.Error,
},
{
name: "Invalid string",
input: "https://github.com/whyNot/calendarNot Used",
checkError: assert.Error,
},
{
name: "Missing calendarID not found",
input: "https://graph.microsoft.com/v1.0/users" +
"('invalid@onmicrosoft.com')/calendars(" +
"'')/$ref",
checkError: assert.Error,
},
{
name: "Valid string",
input: "https://graph.microsoft.com/v1.0/users" +
"('valid@onmicrosoft.com')/calendars(" +
"'AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAA" +
"DCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEGAADSEBNbUIB9RL6ePDeF3FIYAAAZkDq1AAA=')/$ref",
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := parseCalendarIDFromEvent(test.input)
test.checkError(t, err)
})
}
}
// TestRetrievalFunctions ensures that utility functions used
// to transform work within the current version of GraphAPI.
func (suite *ExchangeServiceSuite) TestRetrievalFunctions() {
var (
userID = tester.M365UserID(suite.T())
objectID string
)
tests := []struct {
name string
query GraphQuery
retrieveFunc GraphRetrievalFunc
}{
{
name: "Test Retrieve Message Function",
query: GetAllMessagesForUser,
retrieveFunc: RetrieveMessageDataForUser,
},
{
name: "Test Retrieve Contact Function",
query: GetAllContactsForUser,
retrieveFunc: RetrieveContactDataForUser,
},
{
name: "Test Retrieve Event Function",
query: GetAllEventsForUser,
retrieveFunc: RetrieveEventDataForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
output, err := test.query(suite.es, userID)
require.NoError(t, err)
switch v := output.(type) {
case *models.MessageCollectionResponse:
transform := output.(models.MessageCollectionResponseable)
response := transform.GetValue()
require.Greater(t, len(response), 0)
objectID = *response[0].GetId()
case *models.ContactCollectionResponse:
transform := output.(models.ContactCollectionResponseable)
response := transform.GetValue()
require.Greater(t, len(response), 0)
objectID = *response[0].GetId()
case *models.EventCollectionResponse:
transform := output.(models.EventCollectionResponseable)
response := transform.GetValue()
require.Greater(t, len(response), 0)
objectID = *response[0].GetId()
default:
t.Logf("What is this type: %T\n", v)
}
require.NotEmpty(t, objectID)
retrieved, err := test.retrieveFunc(suite.es, userID, objectID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
assert.NotNil(t, retrieved)
})
}
}
// TestGetMailFolderID verifies the ability to retrieve folder ID of folders
// at the top level of the file tree
func (suite *ExchangeServiceSuite) TestGetContainerID() {
userID := tester.M365UserID(suite.T())
ctx := context.Background()
tests := []struct {
name string
containerName string
@ -464,6 +357,7 @@ func (suite *ExchangeServiceSuite) TestGetContainerID() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := GetContainerID(
ctx,
suite.es,
test.containerName,
userID,
@ -549,6 +443,7 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() {
// TestGetRestoreContainer checks the ability to Create a "container" for the
// GraphConnector's Restore Workflow based on OptionIdentifier.
func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
ctx := context.Background()
tests := []struct {
name string
option path.CategoryType
@ -591,7 +486,7 @@ func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
containerID, err := GetRestoreContainer(suite.es, userID, test.option)
containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option)
require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err)))
if test.cleanupFunc != nil {

14
src/internal/connector/exchange/exchange_vars.go
View File

@ -27,3 +27,17 @@ const (
// Section: 2.789 PidTagMessageDeliveryTime
MailReceiveDateTimeOverriveProperty = "SystemTime 0x0E06"
)
// descendable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a "parent folder".
type descendable interface {
GetId() *string
GetParentFolderId() *string
}
// displayable represents objects that implement msgraph-sdk-fo/models.entityable
// and have the concept of a display name.
type displayable interface {
GetId() *string
GetDisplayName() *string
}

6
src/internal/connector/exchange/iterators_test.go
View File

@ -130,6 +130,12 @@ func (suite *ExchangeIteratorSuite) TestIterativeFunctions() {
iterativeFunction: IterateSelectAllDescendablesForCollections,
scope: contactScope,
transformer: models.CreateContactFromDiscriminatorValue,
}, {
name: "Contact Folder Traversal",
queryFunction: GetAllContactFolderNamesForUser,
iterativeFunction: IterateSelectAllContactsForCollections,
scope: contactScope,
transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
}, {
name: "Events Iterative Check",
queryFunction: GetAllCalendarNamesForUser,

23
src/internal/connector/exchange/query_options.go
View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
mscalendars "github.com/microsoftgraph/msgraph-sdk-go/users/item/calendars"
mscontactfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders"
mscontactfolderitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders/item/contacts"
mscontacts "github.com/microsoftgraph/msgraph-sdk-go/users/item/contacts"
msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events"
msfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/mailfolders"
@ -242,6 +243,26 @@ func optionsForMailFoldersItem(
return options, nil
}
// optionsForContactFoldersItem is the same as optionsForContacts.
// TODO: Remove after Issue #828; requires updating msgraph to v0.34
func optionsForContactFoldersItem(
moreOps []string,
) (*mscontactfolderitem.ContactsRequestBuilderGetRequestConfiguration, error) {
selecting, err := buildOptions(moreOps, contacts)
if err != nil {
return nil, err
}
requestParameters := &mscontactfolderitem.ContactsRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &mscontactfolderitem.ContactsRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForEvents ensures valid option inputs for exchange.Events
// @return is first call in Events().GetWithRequestConfigurationAndResponseHandler(options, handler)
func optionsForEvents(moreOps []string) (*msevents.EventsRequestBuilderGetRequestConfiguration, error) {
@ -325,7 +346,7 @@ func buildOptions(options []string, optID optionIdentifier) ([]string, error) {
for _, entry := range options {
_, ok := allowedOptions[entry]
if !ok {
return nil, fmt.Errorf("unsupported option: %v", entry)
return nil, fmt.Errorf("unsupported element passed to buildOptions: %v", entry)
}
returnedOptions = append(returnedOptions, entry)

49
src/internal/connector/exchange/service_functions.go
View File

@ -1,6 +1,7 @@
package exchange
import (
"context"
"fmt"
"strings"
@ -100,7 +101,8 @@ func DeleteCalendar(gs graph.Service, user, calendarID string) error {
// If successful, returns the created folder object.
func CreateContactFolder(gs graph.Service, user, folderName string) (models.ContactFolderable, error) {
requestBody := models.NewContactFolder()
requestBody.SetDisplayName(&folderName)
temp := folderName
requestBody.SetDisplayName(&temp)
return gs.Client().UsersById(user).ContactFolders().Post(requestBody)
}
@ -244,13 +246,22 @@ func GetAllContactFolders(gs graph.Service, user, nameContains string) ([]models
// @param containerName is the target's name, user-readable and case sensitive
// @param category switches query and iteration to support multiple exchange applications
// @returns a *string if the folder exists. If the folder does not exist returns nil, error-> folder not found
func GetContainerID(service graph.Service, containerName, user string, category optionIdentifier) (*string, error) {
func GetContainerID(
ctx context.Context,
service graph.Service,
containerName,
user string,
category optionIdentifier,
) (*string, error) {
var (
errs error
targetID *string
query GraphQuery
transform absser.ParsableFactory
isCalendar bool
errUpdater = func(id string, err error) {
errs = support.WrapAndAppend(id, err, errs)
}
)
switch category {
@ -291,7 +302,7 @@ func GetContainerID(service graph.Service, containerName, user string, category
containerName,
service.Adapter().GetBaseUrl(),
isCalendar,
errs,
errUpdater,
)
if err := pageIterator.Iterate(callbackFunc); err != nil {
@ -305,32 +316,6 @@ func GetContainerID(service graph.Service, containerName, user string, category
return targetID, errs
}
// parseCalendarIDFromEvent returns the M365 ID for a calendar
// @param reference: string from additionalData map of an event
// References should follow the form `https://... calendars('ID')/$ref`
// If the reference does not follow form an error is returned
func parseCalendarIDFromEvent(reference string) (string, error) {
stringArray := strings.Split(reference, "calendars('")
if len(stringArray) < 2 {
return "", errors.New("calendarID not found")
}
temp := stringArray[1]
stringArray = strings.Split(temp, "')/$ref")
if len(stringArray) < 2 {
return "", errors.New("calendarID not found")
}
calendarID := stringArray[0]
if len(calendarID) == 0 {
return "", errors.New("calendarID empty")
}
return calendarID, nil
}
// SetupExchangeCollectionVars is a helper function returns a sets
// Exchange.Type specific functions based on scope
func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
@ -361,9 +346,9 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
}
if scope.IncludesCategory(selectors.ExchangeContact) {
return models.CreateContactFromDiscriminatorValue,
GetAllContactsForUser,
IterateSelectAllDescendablesForCollections,
return models.CreateContactFolderCollectionResponseFromDiscriminatorValue,
GetAllContactFolderNamesForUser,
IterateSelectAllContactsForCollections,
nil
}

194
src/internal/connector/exchange/service_iterators.go
View File

@ -3,6 +3,7 @@ package exchange
import (
"context"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
@ -14,20 +15,6 @@ import (
var errNilResolver = errors.New("nil resolver")
// descendable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a "parent folder".
type descendable interface {
GetId() *string
GetParentFolderId() *string
}
// displayable represents objects that implement msgraph-sdk-fo/models.entityable
// and have the concept of a display name.
type displayable interface {
GetId() *string
GetDisplayName() *string
}
// GraphIterateFuncs are iterate functions to be used with the M365 iterators (e.g. msgraphgocore.NewPageIterator)
// @returns a callback func that works with msgraphgocore.PageIterator.Iterate function
type GraphIterateFunc func(
@ -420,6 +407,128 @@ func IterateFilterFolderDirectoriesForCollections(
}
}
func IterateSelectAllContactsForCollections(
ctx context.Context,
qp graph.QueryParams,
errUpdater func(string, error),
collections map[string]*Collection,
statusUpdater support.StatusUpdater,
) func(any) bool {
var isPrimarySet bool
return func(folderItem any) bool {
folder, ok := folderItem.(models.ContactFolderable)
if !ok {
errUpdater(
qp.User,
errors.New("casting folderItem to models.ContactFolderable"),
)
}
if !isPrimarySet && folder.GetParentFolderId() != nil {
service, err := createService(qp.Credentials, qp.FailFast)
if err != nil {
errUpdater(
qp.User,
errors.Wrap(err, "unable to create service during IterateSelectAllContactsForCollections"),
)
return true
}
contactIDS, err := ReturnContactIDsFromDirectory(service, qp.User, *folder.GetParentFolderId())
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
dirPath, err := path.Builder{}.Append(*folder.GetParentFolderId()).ToDataLayerExchangePathForCategory(
qp.Credentials.TenantID,
qp.User,
path.ContactsCategory,
false,
)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
edc := NewCollection(
qp.User,
dirPath,
contacts,
service,
statusUpdater,
)
edc.jobs = append(edc.jobs, contactIDS...)
collections["Contacts"] = &edc
isPrimarySet = true
}
service, err := createService(qp.Credentials, qp.FailFast)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
folderID := *folder.GetId()
listOfIDs, err := ReturnContactIDsFromDirectory(service, qp.User, folderID)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
if folder.GetDisplayName() == nil ||
listOfIDs == nil {
return true // Invalid state TODO: How should this be named
}
dirPath, err := path.Builder{}.Append(*folder.GetId()).ToDataLayerExchangePathForCategory(
qp.Credentials.TenantID,
qp.User,
path.ContactsCategory,
false,
)
if err != nil {
errUpdater(
qp.User,
err,
)
return true
}
edc := NewCollection(
qp.User,
dirPath,
contacts,
service,
statusUpdater,
)
edc.jobs = append(edc.jobs, listOfIDs...)
collections[*folder.GetId()] = &edc
return true
}
}
// iterateFindContainerID is a utility function that supports finding
// M365 folders objects that matches the folderName. Iterator callback function
// will work on folderCollection responses whose objects implement
@ -431,7 +540,7 @@ func iterateFindContainerID(
containerID **string,
containerName, errorIdentifier string,
isCalendar bool,
errs error,
errUpdater func(string, error),
) func(any) bool {
return func(entry any) bool {
if isCalendar {
@ -446,10 +555,9 @@ func iterateFindContainerID(
folder, ok := entry.(displayable)
if !ok {
errs = support.WrapAndAppend(
errUpdater(
errorIdentifier,
errors.New("struct does not implement displayable"),
errs,
)
return true
@ -473,3 +581,55 @@ func iterateFindContainerID(
return true
}
}
// IDistFunc collection of helper functions which return a list of strings
// from a response.
type IDListFunc func(gs graph.Service, user, m365ID string) ([]string, error)
// ReturnContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts
// of the targeted directory
func ReturnContactIDsFromDirectory(gs graph.Service, user, directoryID string) ([]string, error) {
options, err := optionsForContactFoldersItem([]string{"parentFolderId"})
if err != nil {
return nil, err
}
stringArray := []string{}
response, err := gs.Client().
UsersById(user).
ContactFoldersById(directoryID).
Contacts().
GetWithRequestConfigurationAndResponseHandler(options, nil)
if err != nil {
return nil, err
}
pageIterator, err := msgraphgocore.NewPageIterator(
response,
gs.Adapter(),
models.CreateContactCollectionResponseFromDiscriminatorValue,
)
callbackFunc := func(pageItem any) bool {
entry, ok := pageItem.(models.Contactable)
if !ok {
err = errors.New("casting pageItem to models.Contactable")
return false
}
stringArray = append(stringArray, *entry.GetId())
return true
}
if iterateErr := pageIterator.Iterate(callbackFunc); iterateErr != nil {
return nil, iterateErr
}
if err != nil {
return nil, err
}
return stringArray, nil
}

2
src/internal/connector/exchange/service_query.go
View File

@ -67,7 +67,7 @@ func GetAllCalendarNamesForUser(gs graph.Service, user string) (absser.Parsable,
// and display names for contacts. All other information is omitted.
// Does not return the primary Contact Folder
func GetAllContactFolderNamesForUser(gs graph.Service, user string) (absser.Parsable, error) {
options, err := optionsForContactFolders([]string{"displayName"})
options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, err
}

41
src/internal/connector/exchange/service_restore.go
View File

@ -20,6 +20,7 @@ import (
// @param category: input from fullPath()[2]
// that defines the application the folder is created in.
func GetRestoreContainer(
ctx context.Context,
service graph.Service,
user string,
category path.CategoryType,
@ -27,39 +28,39 @@ func GetRestoreContainer(
name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat))
option := categoryToOptionIdentifier(category)
folderID, err := GetContainerID(service, name, user, option)
folderID, err := GetContainerID(ctx, service, name, user, option)
if err == nil {
return *folderID, nil
}
// Experienced error other than folder does not exist
if !errors.Is(err, ErrFolderNotFound) {
return "", support.WrapAndAppend(user, err, err)
return "", support.WrapAndAppend(user+": lookup failue during GetContainerID", err, err)
}
switch option {
case messages:
fold, err := CreateMailFolder(service, user, name)
if err != nil {
return "", support.WrapAndAppend(user, err, err)
return "", support.WrapAndAppend(fmt.Sprintf("creating folder %s for user %s", name, user), err, err)
}
return *fold.GetId(), nil
case contacts:
fold, err := CreateContactFolder(service, user, name)
if err != nil {
return "", support.WrapAndAppend(user, err, err)
return "", support.WrapAndAppend(user+"failure during CreateContactFolder during restore Contact", err, err)
}
return *fold.GetId(), nil
case events:
calendar, err := CreateCalendar(service, user, name)
if err != nil {
return "", support.WrapAndAppend(user, err, err)
return "", support.WrapAndAppend(user+"failure during CreateCalendar during restore Event", err, err)
}
return *calendar.GetId(), nil
default:
return "", fmt.Errorf("category: %s not supported for folder creation", option)
return "", fmt.Errorf("category: %s not supported for folder creation: GetRestoreContainer", option)
}
}
@ -75,7 +76,7 @@ func RestoreExchangeObject(
destination, user string,
) error {
if policy != control.Copy {
return fmt.Errorf("restore policy: %s not supported", policy)
return fmt.Errorf("restore policy: %s not supported for RestoreExchangeObject", policy)
}
setting := categoryToOptionIdentifier(category)
@ -88,7 +89,7 @@ func RestoreExchangeObject(
case events:
return RestoreExchangeEvent(ctx, bits, service, control.Copy, destination, user)
default:
return fmt.Errorf("type: %s not supported for exchange restore", category)
return fmt.Errorf("type: %s not supported for RestoreExchangeObject", category)
}
}
@ -107,12 +108,16 @@ func RestoreExchangeContact(
) error {
contact, err := support.CreateContactFromBytes(bits)
if err != nil {
return err
return errors.Wrap(err, "failure to create contact from bytes: RestoreExchangeContact")
}
response, err := service.Client().UsersById(user).ContactFoldersById(destination).Contacts().Post(contact)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return errors.Wrap(
err,
"failure to create Contact during RestoreExchangeContact: "+
support.ConnectorStackErrorTrace(err),
)
}
if response == nil {
@ -142,7 +147,11 @@ func RestoreExchangeEvent(
response, err := service.Client().UsersById(user).CalendarsById(destination).Events().Post(event)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return errors.Wrap(err,
fmt.Sprintf(
"failure to event creation failure during RestoreExchangeEvent: %s",
support.ConnectorStackErrorTrace(err)),
)
}
if response == nil {
@ -202,7 +211,7 @@ func RestoreMailMessage(
// Switch workflow based on collision policy
switch cp {
default:
logger.Ctx(ctx).DPanicw("unrecognized restore policy; defaulting to copy",
logger.Ctx(ctx).DPanicw("restoreMailMessage received unrecognized restore policy; defaulting to copy",
"policy", cp)
fallthrough
case control.Copy:
@ -217,16 +226,14 @@ func RestoreMailMessage(
func SendMailToBackStore(service graph.Service, user, destination string, message models.Messageable) error {
sentMessage, err := service.Client().UsersById(user).MailFoldersById(destination).Messages().Post(message)
if err != nil {
return support.WrapAndAppend(": "+support.ConnectorStackErrorTrace(err), err, nil)
return errors.Wrap(err,
*message.GetId()+": failure sendMailAPI: "+support.ConnectorStackErrorTrace(err),
)
}
if sentMessage == nil {
return errors.New("message not Sent: blocked by server")
}
if err != nil {
return support.WrapAndAppend(": "+support.ConnectorStackErrorTrace(err), err, nil)
}
return nil
}

20
src/internal/connector/graph_connector.go
View File

@ -279,10 +279,11 @@ func (gc *GraphConnector) RestoreDataCollections(
switch service {
case path.ExchangeService:
folderID, errs = exchange.GetRestoreContainer(&gc.graphService, user, category)
folderID, errs = exchange.GetRestoreContainer(ctx, &gc.graphService, user, category)
}
if errs != nil {
fmt.Println("RestoreContainer Failed")
return errs
}
}
@ -294,7 +295,7 @@ func (gc *GraphConnector) RestoreDataCollections(
case itemData, ok := <-items:
if !ok {
exit = true
break
continue
}
attempts++
@ -302,7 +303,12 @@ func (gc *GraphConnector) RestoreDataCollections(
_, err := buf.ReadFrom(itemData.ToReader())
if err != nil {
errs = support.WrapAndAppend(itemData.UUID(), err, errs)
errs = support.WrapAndAppend(
itemData.UUID()+": byteReadError during RestoreDataCollection",
err,
errs,
)
continue
}
@ -312,7 +318,13 @@ func (gc *GraphConnector) RestoreDataCollections(
}
if err != nil {
errs = support.WrapAndAppend(itemData.UUID(), err, errs)
// More information to be here
errs = support.WrapAndAppend(
itemData.UUID()+": failed to upload RestoreExchangeObject: "+service.String()+"-"+category.String(),
err,
errs,
)
continue
}
successes++