larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add fault & clues to exchange api (#2492)

Browse Source 
## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-02-15 10:01:04 -07:00 committed by GitHub
parent e70bf25f6e
commit 26d286138b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 338 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/cmd/getM365/getItem.go
View File

@ -81,7 +81,7 @@ func handleGetCommand(cmd *cobra.Command, args []string) error {
return err
}
err = runDisplayM365JSON(ctx, creds, user, m365ID)
err = runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true))
if err != nil {
return Only(ctx, errors.Wrapf(err, "unable to create mock from M365: %s", m365ID))
}
@ -93,6 +93,7 @@ func runDisplayM365JSON(
ctx context.Context,
creds account.M365Config,
user, itemID string,
errs *fault.Errors,
) error {
var (
bs []byte
@ -108,11 +109,11 @@ func runDisplayM365JSON(
switch cat {
case path.EmailCategory:
bs, err = getItem(ctx, ac.Mail(), user, itemID)
bs, err = getItem(ctx, ac.Mail(), user, itemID, errs)
case path.EventsCategory:
bs, err = getItem(ctx, ac.Events(), user, itemID)
bs, err = getItem(ctx, ac.Events(), user, itemID, errs)
case path.ContactsCategory:
bs, err = getItem(ctx, ac.Contacts(), user, itemID)
bs, err = getItem(ctx, ac.Contacts(), user, itemID, errs)
default:
return fmt.Errorf("unable to process category: %s", cat)
}
@ -142,6 +143,7 @@ type itemer interface {
GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error)
Serialize(
ctx context.Context,
@ -154,8 +156,9 @@ func getItem(
ctx context.Context,
itm itemer,
user, itemID string,
errs *fault.Errors,
) ([]byte, error) {
sp, _, err := itm.GetItem(ctx, user, itemID)
sp, _, err := itm.GetItem(ctx, user, itemID, errs)
if err != nil {
return nil, errors.Wrap(err, "getting item")
}

24
src/internal/common/ptr/pointer.go
View File

@ -1,5 +1,7 @@
package ptr
import "time"
// ptr package is a common package used for pointer
// access and deserialization.
@ -19,3 +21,25 @@ func Val[T any](ptr *T) T {
return *ptr
}
// ValOK behaves the same as Val, except it also gives
// a boolean response for whether the pointer was nil
// (false) or non-nil (true).
func ValOK[T any](ptr *T) (T, bool) {
if ptr == nil {
return *new(T), false
}
return *ptr, true
}
// OrNow returns the value of the provided time, if the
// parameter is non-nil. Otherwise it returns the current
// time in UTC.
func OrNow(t *time.Time) time.Time {
if t == nil {
return time.Now().UTC()
}
return *t
}

29
src/internal/connector/exchange/api/api.go
View File

@ -3,12 +3,13 @@ package api
import (
"context"
"strings"
"time"
"github.com/alcionai/clues"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
)
@ -17,8 +18,6 @@ import (
// common types and consts
// ---------------------------------------------------------------------------
const numberOfRetries = 3
// DeltaUpdate holds the results of a current delta token. It normally
// gets produced when aggregating the addition and removal of items in
// a delta-queriable folder.
@ -122,34 +121,24 @@ func newLargeItemService(creds account.M365Config) (*graph.Service, error) {
// checkIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func checkIDAndName(c graph.Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
id := ptr.Val(c.GetId())
if len(id) == 0 {
return errors.New("container missing ID")
}
ptr := c.GetDisplayName()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without display name", *idPtr)
dn := ptr.Val(c.GetDisplayName())
if len(dn) == 0 {
return clues.New("container missing display name").With("container_id", id)
}
return nil
}
func orNow(t *time.Time) time.Time {
if t == nil {
return time.Now().UTC()
}
return *t
}
func HasAttachments(body models.ItemBodyable) bool {
if body.GetContent() == nil || body.GetContentType() == nil ||
*body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 {
return false
}
content := *body.GetContent()
return strings.Contains(content, "src=\"cid:")
return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:")
}

127
src/internal/connector/exchange/api/contacts.go
View File

@ -5,18 +5,16 @@ import (
"fmt"
"github.com/alcionai/clues"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -47,7 +45,12 @@ func (c Contacts) CreateContactFolder(
temp := folderName
requestBody.SetDisplayName(&temp)
return c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
mdl, err := c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return mdl, nil
}
// DeleteContainer deletes the ContactFolder associated with the M365 ID if permissions are valid.
@ -55,22 +58,23 @@ func (c Contacts) DeleteContainer(
ctx context.Context,
user, folderID string,
) error {
return c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
err := c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return nil
}
// GetItem retrieves a Contactable item.
func (c Contacts) GetItem(
ctx context.Context,
user, itemID string,
_ *fault.Errors, // no attachments to iterate over, so this goes unused
) (serialization.Parsable, *details.ExchangeInfo, error) {
var (
cont models.Contactable
err error
)
cont, err = c.stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil)
cont, err := c.stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, err
return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return cont, ContactInfo(cont), nil
@ -82,14 +86,15 @@ func (c Contacts) GetContainerByID(
) (graph.Container, error) {
ofcf, err := optionsForContactFolderByID([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, errors.Wrap(err, "options for contact folder")
return nil, clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var resp models.ContactFolderable
resp, err := c.stable.Client().UsersById(userID).ContactFoldersById(dirID).Get(ctx, ofcf)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resp, err = c.stable.Client().UsersById(userID).ContactFoldersById(dirID).Get(ctx, ofcf)
return resp, err
return resp, nil
}
// EnumerateContainers iterates through all of the users current
@ -102,21 +107,21 @@ func (c Contacts) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
) error {
service, err := c.service()
if err != nil {
return err
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var (
errs *multierror.Error
resp models.ContactFolderCollectionResponseable
fields = []string{"displayName", "parentFolderId"}
)
fields := []string{"displayName", "parentFolderId"}
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return errors.Wrapf(err, "options for contact child folders: %v", fields)
return clues.Wrap(err, "setting contact child folder options").
WithClues(ctx).
WithAll(graph.ErrData(err)...).
With("options_fields", fields)
}
builder := service.Client().
@ -125,32 +130,42 @@ func (c Contacts) EnumerateContainers(
ChildFolders()
for {
resp, err = builder.Get(ctx, ofcf)
resp, err := builder.Get(ctx, ofcf)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
for _, fold := range resp.GetValue() {
if errs.Err() != nil {
return errs.Err()
}
if err := checkIDAndName(fold); err != nil {
errs = multierror.Append(err, errs)
errs.Add(clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...))
continue
}
fctx := clues.AddAll(
ctx,
"container_id", ptr.Val(fold.GetId()),
"container_display_name", ptr.Val(fold.GetDisplayName()))
temp := graph.NewCacheFolder(fold, nil, nil)
if err := fn(temp); err != nil {
errs = multierror.Append(err, errs)
errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...))
continue
}
}
if resp.GetOdataNextLink() == nil {
link, ok := ptr.ValOK(resp.GetOdataNextLink())
if !ok {
break
}
builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), service.Adapter())
builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(link, service.Adapter())
}
return errs.ErrorOrNil()
return errs.Err()
}
// ---------------------------------------------------------------------------
@ -166,14 +181,12 @@ type contactPager struct {
}
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
var (
resp api.DeltaPageLinker
err error
)
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resp, err = p.builder.Get(ctx, p.options)
return resp, err
return resp, nil
}
func (p *contactPager) setNext(nextLink string) {
@ -190,41 +203,43 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
) ([]string, []string, DeltaUpdate, error) {
service, err := c.service()
if err != nil {
return nil, nil, DeltaUpdate{}, err
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var (
errs *multierror.Error
resetDelta bool
)
var resetDelta bool
ctx = clues.AddAll(
ctx,
"category", selectors.ExchangeContact,
"folder_id", directoryID)
"container_id", directoryID)
options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
return nil,
nil,
DeltaUpdate{},
clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
if len(oldDelta) > 0 {
builder := users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr := &contactPager{service, builder, options}
var (
builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr = &contactPager{service, builder, options}
)
added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
// note: happy path, not the error condition
if err == nil {
return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, false}, err
}
// only return on error if it is NOT a delta issue.
// on bad deltas we retry the call with the regular builder
if !graph.IsErrInvalidDelta(err) {
return nil, nil, DeltaUpdate{}, err
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resetDelta = true
errs = nil
}
builder := service.Client().UsersById(user).ContactFoldersById(directoryID).Contacts().Delta()
@ -235,7 +250,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
return nil, nil, DeltaUpdate{}, err
}
return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil
}
// ---------------------------------------------------------------------------
@ -250,10 +265,10 @@ func (c Contacts) Serialize(
) ([]byte, error) {
contact, ok := item.(models.Contactable)
if !ok {
return nil, fmt.Errorf("expected Contactable, got %T", item)
return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not a Contactable")
}
ctx = clues.Add(ctx, "item_id", *contact.GetId())
ctx = clues.Add(ctx, "item_id", ptr.Val(contact.GetId()))
var (
err error
@ -263,12 +278,12 @@ func (c Contacts) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", contact); err != nil {
return nil, clues.Stack(err).WithClues(ctx)
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, errors.Wrap(err, "serializing contact")
return nil, clues.Wrap(err, "serializing contact").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return bs, nil
@ -286,6 +301,6 @@ func ContactInfo(contact models.Contactable) *details.ExchangeInfo {
ItemType: details.ExchangeContact,
ContactName: name,
Created: created,
Modified: orNow(contact.GetLastModifiedDateTime()),
Modified: ptr.OrNow(contact.GetLastModifiedDateTime()),
}
}

164
src/internal/connector/exchange/api/events.go
View File

@ -6,22 +6,18 @@ import (
"time"
"github.com/alcionai/clues"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
@ -50,7 +46,12 @@ func (c Events) CreateCalendar(
requestbody := models.NewCalendar()
requestbody.SetName(&calendarName)
return c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
mdl, err := c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return mdl, nil
}
// DeleteContainer removes a calendar from user's M365 account
@ -59,7 +60,12 @@ func (c Events) DeleteContainer(
ctx context.Context,
user, calendarID string,
) error {
return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
err := c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return nil
}
func (c Events) GetContainerByID(
@ -68,20 +74,17 @@ func (c Events) GetContainerByID(
) (graph.Container, error) {
service, err := c.service()
if err != nil {
return nil, err
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
ofc, err := optionsForCalendarsByID([]string{"name", "owner"})
if err != nil {
return nil, errors.Wrap(err, "options for event calendar")
return nil, clues.Wrap(err, "setting event calendar options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var cal models.Calendarable
cal, err = service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc)
cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc)
if err != nil {
return nil, err
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return graph.CalendarDisplayable{Calendarable: cal}, nil
@ -91,47 +94,36 @@ func (c Events) GetContainerByID(
func (c Events) GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error) {
var (
event models.Eventable
err error
event models.Eventable
)
event, err = c.stable.Client().UsersById(user).EventsById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, err
return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var (
errs *multierror.Error
options = &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{
if *event.GetHasAttachments() || HasAttachments(event.GetBody()) {
options := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"},
},
}
)
if *event.GetHasAttachments() || HasAttachments(event.GetBody()) {
for count := 0; count < numberOfRetries; count++ {
attached, err := c.largeItem.
Client().
UsersById(user).
EventsById(itemID).
Attachments().
Get(ctx, options)
if err == nil {
event.SetAttachments(attached.GetValue())
break
}
logger.Ctx(ctx).Debugw("retrying event attachment download", "err", err)
errs = multierror.Append(errs, err)
}
attached, err := c.largeItem.
Client().
UsersById(user).
EventsById(itemID).
Attachments().
Get(ctx, options)
if err != nil {
logger.Ctx(ctx).Errorw("event attachment download exceeded maximum retries", "err", errs)
return nil, nil, support.WrapAndAppend(itemID, errors.Wrap(err, "download event attachment"), nil)
return nil, nil, clues.Wrap(err, "event attachment download").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
event.SetAttachments(attached.GetValue())
}
return event, EventInfo(event), nil
@ -147,57 +139,57 @@ func (c Events) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
) error {
service, err := c.service()
if err != nil {
return err
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var (
resp models.CalendarCollectionResponseable
errs *multierror.Error
)
ofc, err := optionsForCalendars([]string{"name"})
if err != nil {
return errors.Wrapf(err, "options for event calendars")
return clues.Wrap(err, "setting calendar options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
builder := service.Client().UsersById(userID).Calendars()
for {
var err error
resp, err = builder.Get(ctx, ofc)
resp, err := builder.Get(ctx, ofc)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
for _, cal := range resp.GetValue() {
cd := CalendarDisplayable{Calendarable: cal}
if err := checkIDAndName(cd); err != nil {
errs = multierror.Append(err, errs)
errs.Add(clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...))
continue
}
fctx := clues.AddAll(
ctx,
"container_id", ptr.Val(cal.GetId()),
"container_name", ptr.Val(cal.GetName()))
temp := graph.NewCacheFolder(
cd,
path.Builder{}.Append(*cd.GetId()), // storage path
path.Builder{}.Append(*cd.GetDisplayName())) // display location
path.Builder{}.Append(ptr.Val(cd.GetId())), // storage path
path.Builder{}.Append(ptr.Val(cd.GetDisplayName()))) // display location
if err := fn(temp); err != nil {
errs = multierror.Append(err, errs)
errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...))
continue
}
}
if resp.GetOdataNextLink() == nil {
link, ok := ptr.ValOK(resp.GetOdataNextLink())
if !ok {
break
}
builder = users.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), service.Adapter())
builder = users.NewItemCalendarsRequestBuilder(link, service.Adapter())
}
return errs.ErrorOrNil()
return errs.Err()
}
// ---------------------------------------------------------------------------
@ -217,14 +209,12 @@ type eventPager struct {
}
func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
var (
resp api.DeltaPageLinker
err error
)
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resp, err = p.builder.Get(ctx, p.options)
return resp, err
return resp, nil
}
func (p *eventPager) setNext(nextLink string) {
@ -244,33 +234,30 @@ func (c Events) GetAddedAndRemovedItemIDs(
return nil, nil, DeltaUpdate{}, err
}
var (
resetDelta bool
errs *multierror.Error
)
var resetDelta bool
ctx = clues.AddAll(
ctx,
"category", selectors.ExchangeEvent,
"calendar_id", calendarID)
if len(oldDelta) > 0 {
builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr := &eventPager{service, builder, nil}
var (
builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
pgr = &eventPager{service, builder, nil}
)
added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
// note: happy path, not the error condition
if err == nil {
return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, false}, nil
}
// only return on error if it is NOT a delta issue.
// on bad deltas we retry the call with the regular builder
if !graph.IsErrInvalidDelta(err) {
return nil, nil, DeltaUpdate{}, err
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resetDelta = true
errs = nil
}
// Graph SDK only supports delta queries against events on the beta version, so we're
@ -291,7 +278,7 @@ func (c Events) GetAddedAndRemovedItemIDs(
}
// Events don't have a delta endpoint so just return an empty string.
return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil
}
// ---------------------------------------------------------------------------
@ -306,10 +293,10 @@ func (c Events) Serialize(
) ([]byte, error) {
event, ok := item.(models.Eventable)
if !ok {
return nil, fmt.Errorf("expected Eventable, got %T", item)
return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not an Eventable")
}
ctx = clues.Add(ctx, "item_id", *event.GetId())
ctx = clues.Add(ctx, "item_id", ptr.Val(event.GetId()))
var (
err error
@ -319,12 +306,12 @@ func (c Events) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", event); err != nil {
return nil, clues.Stack(err).WithClues(ctx)
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, errors.Wrap(err, "serializing calendar event")
return nil, clues.Wrap(err, "serializing event").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return bs, nil
@ -368,22 +355,18 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo {
)
if evt.GetOrganizer() != nil &&
evt.GetOrganizer().GetEmailAddress() != nil &&
evt.GetOrganizer().GetEmailAddress().GetAddress() != nil {
organizer = *evt.GetOrganizer().
GetEmailAddress().
GetAddress()
evt.GetOrganizer().GetEmailAddress() != nil {
organizer = ptr.Val(evt.GetOrganizer().GetEmailAddress().GetAddress())
}
if evt.GetRecurrence() != nil {
recurs = true
}
if evt.GetStart() != nil &&
evt.GetStart().GetDateTime() != nil {
if evt.GetStart() != nil && len(ptr.Val(evt.GetStart().GetDateTime())) > 0 {
// timeString has 'Z' literal added to ensure the stored
// DateTime is not: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)
startTime := *evt.GetStart().GetDateTime() + "Z"
startTime := ptr.Val(evt.GetStart().GetDateTime()) + "Z"
output, err := common.ParseTime(startTime)
if err == nil {
@ -391,11 +374,10 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo {
}
}
if evt.GetEnd() != nil &&
evt.GetEnd().GetDateTime() != nil {
if evt.GetEnd() != nil && len(ptr.Val(evt.GetEnd().GetDateTime())) > 0 {
// timeString has 'Z' literal added to ensure the stored
// DateTime is not: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)
endTime := *evt.GetEnd().GetDateTime() + "Z"
endTime := ptr.Val(evt.GetEnd().GetDateTime()) + "Z"
output, err := common.ParseTime(endTime)
if err == nil {
@ -411,6 +393,6 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo {
EventEnd: end,
EventRecurs: recurs,
Created: created,
Modified: orNow(evt.GetLastModifiedDateTime()),
Modified: ptr.OrNow(evt.GetLastModifiedDateTime()),
}
}

159
src/internal/connector/exchange/api/mail.go
View File

@ -5,19 +5,16 @@ import (
"fmt"
"github.com/alcionai/clues"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -49,7 +46,12 @@ func (c Mail) CreateMailFolder(
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
mdl, err := c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return mdl, nil
}
func (c Mail) CreateMailFolderWithParent(
@ -58,7 +60,7 @@ func (c Mail) CreateMailFolderWithParent(
) (models.MailFolderable, error) {
service, err := c.service()
if err != nil {
return nil, err
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
isHidden := false
@ -66,12 +68,17 @@ func (c Mail) CreateMailFolderWithParent(
requestBody.SetDisplayName(&folder)
requestBody.SetIsHidden(&isHidden)
return service.
mdl, err := service.
Client().
UsersById(user).
MailFoldersById(parentID).
ChildFolders().
Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return mdl, nil
}
// DeleteContainer removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account
@ -80,7 +87,12 @@ func (c Mail) DeleteContainer(
ctx context.Context,
user, folderID string,
) error {
return c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
err := c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return nil
}
func (c Mail) GetContainerByID(
@ -89,19 +101,20 @@ func (c Mail) GetContainerByID(
) (graph.Container, error) {
service, err := c.service()
if err != nil {
return nil, err
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
ofmf, err := optionsForMailFoldersItem([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, errors.Wrap(err, "options for mail folder")
return nil, clues.Wrap(err, "setting mail folder options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var resp graph.Container
resp, err := service.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
resp, err = service.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf)
return resp, err
return resp, nil
}
// GetItem retrieves a Messageable item. If the item contains an attachment, that
@ -109,45 +122,31 @@ func (c Mail) GetContainerByID(
func (c Mail) GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error) {
var (
mail models.Messageable
err error
)
mail, err = c.stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil)
mail, err := c.stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, err
return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var errs *multierror.Error
if *mail.GetHasAttachments() || HasAttachments(mail.GetBody()) {
options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{
Expand: []string{"microsoft.graph.itemattachment/item"},
},
}
for count := 0; count < numberOfRetries; count++ {
attached, err := c.largeItem.
Client().
UsersById(user).
MessagesById(itemID).
Attachments().
Get(ctx, options)
if err == nil {
mail.SetAttachments(attached.GetValue())
break
}
logger.Ctx(ctx).Debugw("retrying mail attachment download", "err", err)
errs = multierror.Append(errs, err)
}
attached, err := c.largeItem.
Client().
UsersById(user).
MessagesById(itemID).
Attachments().
Get(ctx, options)
if err != nil {
logger.Ctx(ctx).Errorw("mail attachment download exceeded maximum retries", "err", errs)
return nil, nil, support.WrapAndAppend(itemID, errors.Wrap(err, "downloading mail attachment"), nil)
return nil, nil, clues.Wrap(err, "mail attachment download").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
mail.SetAttachments(attached.GetValue())
}
return mail, MailInfo(mail), nil
@ -163,46 +162,46 @@ func (c Mail) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
) error {
service, err := c.service()
if err != nil {
return err
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
var (
resp users.ItemMailFoldersDeltaResponseable
errs *multierror.Error
builder = service.Client().
UsersById(userID).
MailFolders().
Delta()
)
builder := service.Client().
UsersById(userID).
MailFolders().
Delta()
for {
var err error
resp, err = builder.Get(ctx, nil)
resp, err := builder.Get(ctx, nil)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
for _, v := range resp.GetValue() {
fctx := clues.AddAll(
ctx,
"container_id", ptr.Val(v.GetId()),
"container_name", ptr.Val(v.GetDisplayName()))
temp := graph.NewCacheFolder(v, nil, nil)
if err := fn(temp); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "iterating mail folders delta"))
errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...))
continue
}
}
link := resp.GetOdataNextLink()
if link == nil {
link, ok := ptr.ValOK(resp.GetOdataNextLink())
if !ok {
break
}
builder = users.NewItemMailFoldersDeltaRequestBuilder(*link, service.Adapter())
builder = users.NewItemMailFoldersDeltaRequestBuilder(link, service.Adapter())
}
return errs.ErrorOrNil()
return errs.Err()
}
// ---------------------------------------------------------------------------
@ -218,14 +217,12 @@ type mailPager struct {
}
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
var (
page api.DeltaPageLinker
err error
)
page, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
page, err = p.builder.Get(ctx, p.options)
return page, err
return page, nil
}
func (p *mailPager) setNext(nextLink string) {
@ -246,7 +243,6 @@ func (c Mail) GetAddedAndRemovedItemIDs(
}
var (
errs *multierror.Error
deltaURL string
resetDelta bool
)
@ -258,17 +254,22 @@ func (c Mail) GetAddedAndRemovedItemIDs(
options, err := optionsForFolderMessagesDelta([]string{"isRead"})
if err != nil {
return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options")
return nil,
nil,
DeltaUpdate{},
clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
if len(oldDelta) > 0 {
builder := users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter())
pgr := &mailPager{service, builder, options}
var (
builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter())
pgr = &mailPager{service, builder, options}
)
added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
// note: happy path, not the error condition
if err == nil {
return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, false}, err
}
// only return on error if it is NOT a delta issue.
// on bad deltas we retry the call with the regular builder
@ -277,7 +278,6 @@ func (c Mail) GetAddedAndRemovedItemIDs(
}
resetDelta = true
errs = nil
}
builder := service.Client().UsersById(user).MailFoldersById(directoryID).Messages().Delta()
@ -288,7 +288,7 @@ func (c Mail) GetAddedAndRemovedItemIDs(
return nil, nil, DeltaUpdate{}, err
}
return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil()
return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil
}
// ---------------------------------------------------------------------------
@ -303,10 +303,10 @@ func (c Mail) Serialize(
) ([]byte, error) {
msg, ok := item.(models.Messageable)
if !ok {
return nil, fmt.Errorf("expected Messageable, got %T", item)
return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not a Messageable")
}
ctx = clues.Add(ctx, "item_id", *msg.GetId())
ctx = clues.Add(ctx, "item_id", ptr.Val(msg.GetId()))
var (
err error
@ -316,12 +316,12 @@ func (c Mail) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", msg); err != nil {
return nil, clues.Stack(err).WithClues(ctx)
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, errors.Wrap(err, "serializing email")
return nil, clues.Wrap(err, "serializing email").WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return bs, nil
@ -338,9 +338,8 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo {
created := ptr.Val(msg.GetCreatedDateTime())
if msg.GetSender() != nil &&
msg.GetSender().GetEmailAddress() != nil &&
msg.GetSender().GetEmailAddress().GetAddress() != nil {
sender = *msg.GetSender().GetEmailAddress().GetAddress()
msg.GetSender().GetEmailAddress() != nil {
sender = ptr.Val(msg.GetSender().GetEmailAddress().GetAddress())
}
return &details.ExchangeInfo{
@ -349,6 +348,6 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo {
Subject: subject,
Received: received,
Created: created,
Modified: orNow(msg.GetLastModifiedDateTime()),
Modified: ptr.OrNow(msg.GetLastModifiedDateTime()),
}
}

21
src/internal/connector/exchange/api/shared.go
View File

@ -2,12 +2,13 @@ package api
import (
"context"
"fmt"
"github.com/pkg/errors"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/logger"
)
@ -34,7 +35,7 @@ type getIDAndAddtler interface {
func toValues[T any](a any) ([]getIDAndAddtler, error) {
gv, ok := a.(interface{ GetValue() []T })
if !ok {
return nil, errors.Errorf("response of type [%T] does not comply with the GetValue() interface", a)
return nil, clues.Wrap(fmt.Errorf("%T", a), "does not comply with the GetValue() interface")
}
items := gv.GetValue()
@ -45,7 +46,7 @@ func toValues[T any](a any) ([]getIDAndAddtler, error) {
ri, ok := a.(getIDAndAddtler)
if !ok {
return nil, errors.Errorf("item of type [%T] does not comply with the getIDAndAddtler interface", item)
return nil, clues.Wrap(fmt.Errorf("%T", item), "does not comply with the getIDAndAddtler interface")
}
r = append(r, ri)
@ -72,18 +73,14 @@ func getItemsAddedAndRemovedFromContainer(
// get the next page of data, check for standard errors
resp, err := pager.getPage(ctx)
if err != nil {
if graph.IsErrDeletedInFlight(err) || graph.IsErrInvalidDelta(err) {
return nil, nil, deltaURL, err
}
return nil, nil, deltaURL, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return nil, nil, deltaURL, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
// each category type responds with a different interface, but all
// of them comply with GetValue, which is where we'll get our item data.
items, err := pager.valuesIn(resp)
if err != nil {
return nil, nil, "", err
return nil, nil, "", clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
itemCount += len(items)
@ -100,9 +97,9 @@ func getItemsAddedAndRemovedFromContainer(
// be 'changed' or 'deleted'. We don't really care about the cause: both
// cases are handled the same way in storage.
if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
addedIDs = append(addedIDs, *item.GetId())
addedIDs = append(addedIDs, ptr.Val(item.GetId()))
} else {
removedIDs = append(removedIDs, *item.GetId())
removedIDs = append(removedIDs, ptr.Val(item.GetId()))
}
}

4
src/internal/connector/exchange/contact_folder_cache.go
View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -46,6 +47,7 @@ func (cfc *contactFolderCache) populateContactRoot(
// as of (Oct-07-2022)
func (cfc *contactFolderCache) Populate(
ctx context.Context,
errs *fault.Errors,
baseID string,
baseContainerPather ...string,
) error {
@ -53,7 +55,7 @@ func (cfc *contactFolderCache) Populate(
return errors.Wrap(err, "initializing")
}
err := cfc.enumer.EnumerateContainers(ctx, cfc.userID, baseID, cfc.addFolder)
err := cfc.enumer.EnumerateContainers(ctx, cfc.userID, baseID, cfc.addFolder, errs)
if err != nil {
return errors.Wrap(err, "enumerating containers")
}

2
src/internal/connector/exchange/container_resolver.go
View File

@ -8,6 +8,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -27,6 +28,7 @@ type containersEnumerator interface {
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
) error
}

7
src/internal/connector/exchange/container_resolver_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -657,7 +658,8 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
m365,
test.pathFunc1(t),
folderName,
directoryCaches)
directoryCaches,
fault.New(true))
require.NoError(t, err)
resolver := directoryCaches[test.category]
@ -675,7 +677,8 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
m365,
test.pathFunc2(t),
parentContainer,
directoryCaches)
directoryCaches,
fault.New(true))
require.NoError(t, err)
_, _, err = resolver.IDToPath(ctx, secondID, test.useIDForPath)

2
src/internal/connector/exchange/data_collections.go
View File

@ -264,7 +264,7 @@ func createCollections(
defer closer()
defer close(foldersComplete)
resolver, err := PopulateExchangeContainerResolver(ctx, qp)
resolver, err := PopulateExchangeContainerResolver(ctx, qp, errs)
if err != nil {
return nil, errors.Wrap(err, "populating container cache")
}

2
src/internal/connector/exchange/data_collections_test.go
View File

@ -554,7 +554,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
return nil
}
require.NoError(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn))
require.NoError(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn, fault.New(true)))
tests := []struct {
name, expected string

5
src/internal/connector/exchange/event_calendar_cache.go
View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -61,6 +62,7 @@ func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error {
// @param baseID: ignored. Present to conform to interface
func (ecc *eventCalendarCache) Populate(
ctx context.Context,
errs *fault.Errors,
baseID string,
baseContainerPath ...string,
) error {
@ -72,7 +74,8 @@ func (ecc *eventCalendarCache) Populate(
ctx,
ecc.userID,
"",
ecc.addFolder)
ecc.addFolder,
errs)
if err != nil {
return errors.Wrap(err, "enumerating containers")
}

12
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -43,6 +44,7 @@ type itemer interface {
GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error)
Serialize(
ctx context.Context,
@ -250,7 +252,12 @@ func (col *Collection) streamItems(ctx context.Context) {
err error
)
item, info, err = getItemWithRetries(ctx, user, id, col.items)
item, info, err = getItemWithRetries(
ctx,
user,
id,
col.items,
fault.New(true)) // temporary way to force a failFast error
if err != nil {
// Don't report errors for deleted items as there's no way for us to
// back up data that is gone. Record it as a "success", since there's
@ -298,6 +305,7 @@ func getItemWithRetries(
ctx context.Context,
userID, itemID string,
items itemer,
errs *fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error) {
var (
item serialization.Parsable
@ -306,7 +314,7 @@ func getItemWithRetries(
)
for i := 1; i <= numberOfRetries; i++ {
item, info, err = items.GetItem(ctx, userID, itemID)
item, info, err = items.GetItem(ctx, userID, itemID, errs)
if err == nil {
break
}

10
src/internal/connector/exchange/exchange_data_collection_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -29,12 +30,17 @@ type mockItemer struct {
func (mi *mockItemer) GetItem(
context.Context,
string, string,
*fault.Errors,
) (serialization.Parsable, *details.ExchangeInfo, error) {
mi.getCount++
return nil, nil, mi.getErr
}
func (mi *mockItemer) Serialize(context.Context, serialization.Parsable, string, string) ([]byte, error) {
func (mi *mockItemer) Serialize(
context.Context,
serialization.Parsable,
string, string,
) ([]byte, error) {
mi.serializeCount++
return nil, mi.serializeErr
}
@ -224,7 +230,7 @@ func (suite *ExchangeDataCollectionSuite) TestGetItemWithRetries() {
defer flush()
// itemer is mocked, so only the errors are configured atm.
_, _, err := getItemWithRetries(ctx, "userID", "itemID", test.items)
_, _, err := getItemWithRetries(ctx, "userID", "itemID", test.items, fault.New(true))
test.expectErr(t, err)
})
}

3
src/internal/connector/exchange/folder_resolver_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
type CacheResolverSuite struct {
@ -119,7 +120,7 @@ func (suite *CacheResolverSuite) TestPopulate() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
resolver := test.resolverFunc(t)
require.NoError(t, resolver.Populate(ctx, test.root, test.basePath))
require.NoError(t, resolver.Populate(ctx, fault.New(true), test.root, test.basePath))
_, isFound := resolver.PathInCache(test.folderInCache)
test.canFind(t, isFound)

4
src/internal/connector/exchange/mail_folder_cache.go
View File

@ -7,6 +7,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -71,6 +72,7 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error {
// for the base container in the cache.
func (mc *mailFolderCache) Populate(
ctx context.Context,
errs *fault.Errors,
baseID string,
baseContainerPath ...string,
) error {
@ -78,7 +80,7 @@ func (mc *mailFolderCache) Populate(
return errors.Wrap(err, "initializing")
}
err := mc.enumer.EnumerateContainers(ctx, mc.userID, "", mc.addFolder)
err := mc.enumer.EnumerateContainers(ctx, mc.userID, "", mc.addFolder, errs)
if err != nil {
return errors.Wrap(err, "enumerating containers")
}

3
src/internal/connector/exchange/mail_folder_cache_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
const (
@ -92,7 +93,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
getter: acm,
}
require.NoError(t, mfc.Populate(ctx, test.root, test.path...))
require.NoError(t, mfc.Populate(ctx, fault.New(true), test.root, test.path...))
p, l, err := mfc.IDToPath(ctx, testFolderID, true)
require.NoError(t, err)

4
src/internal/connector/exchange/service_functions.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -35,6 +36,7 @@ func createService(credentials account.M365Config) (*graph.Service, error) {
func PopulateExchangeContainerResolver(
ctx context.Context,
qp graph.QueryParams,
errs *fault.Errors,
) (graph.ContainerResolver, error) {
var (
res graph.ContainerResolver
@ -78,7 +80,7 @@ func PopulateExchangeContainerResolver(
return nil, clues.New("no container resolver registered for category").WithClues(ctx)
}
if err := res.Populate(ctx, cacheRoot); err != nil {
if err := res.Populate(ctx, errs, cacheRoot); err != nil {
return nil, clues.Wrap(err, "populating directory resolver").WithClues(ctx)
}

4
src/internal/connector/exchange/service_iterators_test.go
View File

@ -91,8 +91,8 @@ func (m mockResolver) DestinationNameToID(dest string) string { return m.added[d
func (m mockResolver) IDToPath(context.Context, string, bool) (*path.Builder, *path.Builder, error) {
return nil, nil, nil
}
func (m mockResolver) PathInCache(string) (string, bool) { return "", false }
func (m mockResolver) Populate(context.Context, string, ...string) error { return nil }
func (m mockResolver) PathInCache(string) (string, bool) { return "", false }
func (m mockResolver) Populate(context.Context, *fault.Errors, string, ...string) error { return nil }
// ---------------------------------------------------------------------------
// tests

22
src/internal/connector/exchange/service_restore.go
View File

@ -330,7 +330,8 @@ func RestoreExchangeDataCollections(
creds,
dc.FullPath(),
dest.ContainerName,
userCaches)
userCaches,
errs)
if err != nil {
errs.Add(clues.Wrap(err, "creating destination").WithClues(ctx))
continue
@ -471,6 +472,7 @@ func CreateContainerDestination(
directory path.Path,
destination string,
caches map[path.CategoryType]graph.ContainerResolver,
errs *fault.Errors,
) (string, error) {
var (
newCache = false
@ -508,7 +510,8 @@ func CreateContainerDestination(
folders,
directoryCache,
user,
newCache)
newCache,
errs)
case path.ContactsCategory:
folders := append([]string{destination}, directory.Folders()...)
@ -531,7 +534,8 @@ func CreateContainerDestination(
folders,
directoryCache,
user,
newCache)
newCache,
errs)
case path.EventsCategory:
dest := destination
@ -561,7 +565,8 @@ func CreateContainerDestination(
folders,
directoryCache,
user,
newCache)
newCache,
errs)
default:
return "", clues.Wrap(fmt.Errorf("%T", category), "not support for exchange cache").WithClues(ctx)
@ -580,6 +585,7 @@ func establishMailRestoreLocation(
mfc graph.ContainerResolver,
user string,
isNewCache bool,
errs *fault.Errors,
) (string, error) {
// Process starts with the root folder in order to recreate
// the top-level folder with the same tactic
@ -609,7 +615,7 @@ func establishMailRestoreLocation(
// newCache to false in this we'll only try to populate it once per function
// call even if we make a new cache.
if isNewCache {
if err := mfc.Populate(ctx, rootFolderAlias); err != nil {
if err := mfc.Populate(ctx, errs, rootFolderAlias); err != nil {
return "", errors.Wrap(err, "populating folder cache")
}
@ -638,6 +644,7 @@ func establishContactsRestoreLocation(
cfc graph.ContainerResolver,
user string,
isNewCache bool,
errs *fault.Errors,
) (string, error) {
cached, ok := cfc.PathInCache(folders[0])
if ok {
@ -654,7 +661,7 @@ func establishContactsRestoreLocation(
folderID := *temp.GetId()
if isNewCache {
if err := cfc.Populate(ctx, folderID, folders[0]); err != nil {
if err := cfc.Populate(ctx, errs, folderID, folders[0]); err != nil {
return "", errors.Wrap(err, "populating contact cache")
}
@ -673,6 +680,7 @@ func establishEventsRestoreLocation(
ecc graph.ContainerResolver, // eventCalendarCache
user string,
isNewCache bool,
errs *fault.Errors,
) (string, error) {
// Need to prefix with the "Other Calendars" folder so lookup happens properly.
cached, ok := ecc.PathInCache(folders[0])
@ -690,7 +698,7 @@ func establishEventsRestoreLocation(
folderID := *temp.GetId()
if isNewCache {
if err = ecc.Populate(ctx, folderID, folders[0]); err != nil {
if err = ecc.Populate(ctx, errs, folderID, folders[0]); err != nil {
return "", errors.Wrap(err, "populating event cache")
}

3
src/internal/connector/graph/cache_container.go
View File

@ -6,6 +6,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -62,7 +63,7 @@ type ContainerResolver interface {
// @param ctx is necessary param for Graph API tracing
// @param baseFolderID represents the M365ID base that the resolver will
// conclude its search. Default input is "".
Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
Populate(ctx context.Context, errs *fault.Errors, baseFolderID string, baseContainerPather ...string) error
// PathInCache performs a look up of a path reprensentation
// and returns the m365ID of directory iff the pathString

4
src/internal/operations/backup_integration_test.go
View File

@ -771,7 +771,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
ResourceOwner: suite.user,
Credentials: m365,
}
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp)
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
require.NoError(t, err, "populating %s container resolver", category)
for destName, dest := range gen.dests {
@ -890,7 +890,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
ResourceOwner: suite.user,
Credentials: m365,
}
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp)
cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true))
require.NoError(t, err, "populating %s container resolver", category)
p, err := path.FromDataLayerPath(deets.Entries[0].RepoRef, true)