From 26d286138b8126d5d96e710754deaba3f3a06350 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 15 Feb 2023 10:01:04 -0700 Subject: [PATCH] add fault & clues to exchange api (#2492) ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :broom: Tech Debt/Cleanup ## Issue(s) * #1970 ## Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/getM365/getItem.go | 13 +- src/internal/common/ptr/pointer.go | 24 +++ src/internal/connector/exchange/api/api.go | 29 +--- .../connector/exchange/api/contacts.go | 127 ++++++++------ src/internal/connector/exchange/api/events.go | 164 ++++++++---------- src/internal/connector/exchange/api/mail.go | 159 +++++++++-------- src/internal/connector/exchange/api/shared.go | 21 +-- .../exchange/contact_folder_cache.go | 4 +- .../connector/exchange/container_resolver.go | 2 + .../exchange/container_resolver_test.go | 7 +- .../connector/exchange/data_collections.go | 2 +- .../exchange/data_collections_test.go | 2 +- .../exchange/event_calendar_cache.go | 5 +- .../exchange/exchange_data_collection.go | 12 +- .../exchange/exchange_data_collection_test.go | 10 +- .../exchange/folder_resolver_test.go | 3 +- .../connector/exchange/mail_folder_cache.go | 4 +- .../exchange/mail_folder_cache_test.go | 3 +- .../connector/exchange/service_functions.go | 4 +- .../exchange/service_iterators_test.go | 4 +- .../connector/exchange/service_restore.go | 22 ++- .../connector/graph/cache_container.go | 3 +- .../operations/backup_integration_test.go | 4 +- 23 files changed, 338 insertions(+), 290 deletions(-) diff --git a/src/cmd/getM365/getItem.go b/src/cmd/getM365/getItem.go index 7f1936206..8507d65b2 100644 --- a/src/cmd/getM365/getItem.go +++ b/src/cmd/getM365/getItem.go @@ -81,7 +81,7 @@ func handleGetCommand(cmd *cobra.Command, args []string) error { return err } - err = runDisplayM365JSON(ctx, creds, user, m365ID) + err = runDisplayM365JSON(ctx, creds, user, m365ID, fault.New(true)) if err != nil { return Only(ctx, errors.Wrapf(err, "unable to create mock from M365: %s", m365ID)) } @@ -93,6 +93,7 @@ func runDisplayM365JSON( ctx context.Context, creds account.M365Config, user, itemID string, + errs *fault.Errors, ) error { var ( bs []byte @@ -108,11 +109,11 @@ func runDisplayM365JSON( switch cat { case path.EmailCategory: - bs, err = getItem(ctx, ac.Mail(), user, itemID) + bs, err = getItem(ctx, ac.Mail(), user, itemID, errs) case path.EventsCategory: - bs, err = getItem(ctx, ac.Events(), user, itemID) + bs, err = getItem(ctx, ac.Events(), user, itemID, errs) case path.ContactsCategory: - bs, err = getItem(ctx, ac.Contacts(), user, itemID) + bs, err = getItem(ctx, ac.Contacts(), user, itemID, errs) default: return fmt.Errorf("unable to process category: %s", cat) } @@ -142,6 +143,7 @@ type itemer interface { GetItem( ctx context.Context, user, itemID string, + errs *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) Serialize( ctx context.Context, @@ -154,8 +156,9 @@ func getItem( ctx context.Context, itm itemer, user, itemID string, + errs *fault.Errors, ) ([]byte, error) { - sp, _, err := itm.GetItem(ctx, user, itemID) + sp, _, err := itm.GetItem(ctx, user, itemID, errs) if err != nil { return nil, errors.Wrap(err, "getting item") } diff --git a/src/internal/common/ptr/pointer.go b/src/internal/common/ptr/pointer.go index 7dbf9052f..a8f9a02b9 100644 --- a/src/internal/common/ptr/pointer.go +++ b/src/internal/common/ptr/pointer.go @@ -1,5 +1,7 @@ package ptr +import "time" + // ptr package is a common package used for pointer // access and deserialization. @@ -19,3 +21,25 @@ func Val[T any](ptr *T) T { return *ptr } + +// ValOK behaves the same as Val, except it also gives +// a boolean response for whether the pointer was nil +// (false) or non-nil (true). +func ValOK[T any](ptr *T) (T, bool) { + if ptr == nil { + return *new(T), false + } + + return *ptr, true +} + +// OrNow returns the value of the provided time, if the +// parameter is non-nil. Otherwise it returns the current +// time in UTC. +func OrNow(t *time.Time) time.Time { + if t == nil { + return time.Now().UTC() + } + + return *t +} diff --git a/src/internal/connector/exchange/api/api.go b/src/internal/connector/exchange/api/api.go index c4858c5c1..1b4ebe2b4 100644 --- a/src/internal/connector/exchange/api/api.go +++ b/src/internal/connector/exchange/api/api.go @@ -3,12 +3,13 @@ package api import ( "context" "strings" - "time" + "github.com/alcionai/clues" "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" ) @@ -17,8 +18,6 @@ import ( // common types and consts // --------------------------------------------------------------------------- -const numberOfRetries = 3 - // DeltaUpdate holds the results of a current delta token. It normally // gets produced when aggregating the addition and removal of items in // a delta-queriable folder. @@ -122,34 +121,24 @@ func newLargeItemService(creds account.M365Config) (*graph.Service, error) { // checkIDAndName is a helper function to ensure that // the ID and name pointers are set prior to being called. func checkIDAndName(c graph.Container) error { - idPtr := c.GetId() - if idPtr == nil || len(*idPtr) == 0 { - return errors.New("folder without ID") + id := ptr.Val(c.GetId()) + if len(id) == 0 { + return errors.New("container missing ID") } - ptr := c.GetDisplayName() - if ptr == nil || len(*ptr) == 0 { - return errors.Errorf("folder %s without display name", *idPtr) + dn := ptr.Val(c.GetDisplayName()) + if len(dn) == 0 { + return clues.New("container missing display name").With("container_id", id) } return nil } -func orNow(t *time.Time) time.Time { - if t == nil { - return time.Now().UTC() - } - - return *t -} - func HasAttachments(body models.ItemBodyable) bool { if body.GetContent() == nil || body.GetContentType() == nil || *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { return false } - content := *body.GetContent() - - return strings.Contains(content, "src=\"cid:") + return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:") } diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go index 5de33e916..ab680355d 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/internal/connector/exchange/api/contacts.go @@ -5,18 +5,16 @@ import ( "fmt" "github.com/alcionai/clues" - "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -47,7 +45,12 @@ func (c Contacts) CreateContactFolder( temp := folderName requestBody.SetDisplayName(&temp) - return c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil) + mdl, err := c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return mdl, nil } // DeleteContainer deletes the ContactFolder associated with the M365 ID if permissions are valid. @@ -55,22 +58,23 @@ func (c Contacts) DeleteContainer( ctx context.Context, user, folderID string, ) error { - return c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil) + err := c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil) + if err != nil { + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return nil } // GetItem retrieves a Contactable item. func (c Contacts) GetItem( ctx context.Context, user, itemID string, + _ *fault.Errors, // no attachments to iterate over, so this goes unused ) (serialization.Parsable, *details.ExchangeInfo, error) { - var ( - cont models.Contactable - err error - ) - - cont, err = c.stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil) + cont, err := c.stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil) if err != nil { - return nil, nil, err + return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } return cont, ContactInfo(cont), nil @@ -82,14 +86,15 @@ func (c Contacts) GetContainerByID( ) (graph.Container, error) { ofcf, err := optionsForContactFolderByID([]string{"displayName", "parentFolderId"}) if err != nil { - return nil, errors.Wrap(err, "options for contact folder") + return nil, clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...) } - var resp models.ContactFolderable + resp, err := c.stable.Client().UsersById(userID).ContactFoldersById(dirID).Get(ctx, ofcf) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } - resp, err = c.stable.Client().UsersById(userID).ContactFoldersById(dirID).Get(ctx, ofcf) - - return resp, err + return resp, nil } // EnumerateContainers iterates through all of the users current @@ -102,21 +107,21 @@ func (c Contacts) EnumerateContainers( ctx context.Context, userID, baseDirID string, fn func(graph.CacheFolder) error, + errs *fault.Errors, ) error { service, err := c.service() if err != nil { - return err + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var ( - errs *multierror.Error - resp models.ContactFolderCollectionResponseable - fields = []string{"displayName", "parentFolderId"} - ) + fields := []string{"displayName", "parentFolderId"} ofcf, err := optionsForContactChildFolders(fields) if err != nil { - return errors.Wrapf(err, "options for contact child folders: %v", fields) + return clues.Wrap(err, "setting contact child folder options"). + WithClues(ctx). + WithAll(graph.ErrData(err)...). + With("options_fields", fields) } builder := service.Client(). @@ -125,32 +130,42 @@ func (c Contacts) EnumerateContainers( ChildFolders() for { - resp, err = builder.Get(ctx, ofcf) + resp, err := builder.Get(ctx, ofcf) if err != nil { - return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } for _, fold := range resp.GetValue() { + if errs.Err() != nil { + return errs.Err() + } + if err := checkIDAndName(fold); err != nil { - errs = multierror.Append(err, errs) + errs.Add(clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)) continue } + fctx := clues.AddAll( + ctx, + "container_id", ptr.Val(fold.GetId()), + "container_display_name", ptr.Val(fold.GetDisplayName())) + temp := graph.NewCacheFolder(fold, nil, nil) if err := fn(temp); err != nil { - errs = multierror.Append(err, errs) + errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...)) continue } } - if resp.GetOdataNextLink() == nil { + link, ok := ptr.ValOK(resp.GetOdataNextLink()) + if !ok { break } - builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), service.Adapter()) + builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(link, service.Adapter()) } - return errs.ErrorOrNil() + return errs.Err() } // --------------------------------------------------------------------------- @@ -166,14 +181,12 @@ type contactPager struct { } func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { - var ( - resp api.DeltaPageLinker - err error - ) + resp, err := p.builder.Get(ctx, p.options) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } - resp, err = p.builder.Get(ctx, p.options) - - return resp, err + return resp, nil } func (p *contactPager) setNext(nextLink string) { @@ -190,41 +203,43 @@ func (c Contacts) GetAddedAndRemovedItemIDs( ) ([]string, []string, DeltaUpdate, error) { service, err := c.service() if err != nil { - return nil, nil, DeltaUpdate{}, err + return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var ( - errs *multierror.Error - resetDelta bool - ) + var resetDelta bool ctx = clues.AddAll( ctx, "category", selectors.ExchangeContact, - "folder_id", directoryID) + "container_id", directoryID) options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"}) if err != nil { - return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options") + return nil, + nil, + DeltaUpdate{}, + clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...) } if len(oldDelta) > 0 { - builder := users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &contactPager{service, builder, options} + var ( + builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter()) + pgr = &contactPager{service, builder, options} + ) added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition if err == nil { - return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, false}, err } + // only return on error if it is NOT a delta issue. // on bad deltas we retry the call with the regular builder if !graph.IsErrInvalidDelta(err) { - return nil, nil, DeltaUpdate{}, err + return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } resetDelta = true - errs = nil } builder := service.Client().UsersById(user).ContactFoldersById(directoryID).Contacts().Delta() @@ -235,7 +250,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs( return nil, nil, DeltaUpdate{}, err } - return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil } // --------------------------------------------------------------------------- @@ -250,10 +265,10 @@ func (c Contacts) Serialize( ) ([]byte, error) { contact, ok := item.(models.Contactable) if !ok { - return nil, fmt.Errorf("expected Contactable, got %T", item) + return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not a Contactable") } - ctx = clues.Add(ctx, "item_id", *contact.GetId()) + ctx = clues.Add(ctx, "item_id", ptr.Val(contact.GetId())) var ( err error @@ -263,12 +278,12 @@ func (c Contacts) Serialize( defer writer.Close() if err = writer.WriteObjectValue("", contact); err != nil { - return nil, clues.Stack(err).WithClues(ctx) + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } bs, err := writer.GetSerializedContent() if err != nil { - return nil, errors.Wrap(err, "serializing contact") + return nil, clues.Wrap(err, "serializing contact").WithClues(ctx).WithAll(graph.ErrData(err)...) } return bs, nil @@ -286,6 +301,6 @@ func ContactInfo(contact models.Contactable) *details.ExchangeInfo { ItemType: details.ExchangeContact, ContactName: name, Created: created, - Modified: orNow(contact.GetLastModifiedDateTime()), + Modified: ptr.OrNow(contact.GetLastModifiedDateTime()), } } diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index a998203b8..d6cfc1bbf 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -6,22 +6,18 @@ import ( "time" "github.com/alcionai/clues" - "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" ) // --------------------------------------------------------------------------- @@ -50,7 +46,12 @@ func (c Events) CreateCalendar( requestbody := models.NewCalendar() requestbody.SetName(&calendarName) - return c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil) + mdl, err := c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return mdl, nil } // DeleteContainer removes a calendar from user's M365 account @@ -59,7 +60,12 @@ func (c Events) DeleteContainer( ctx context.Context, user, calendarID string, ) error { - return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) + err := c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) + if err != nil { + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return nil } func (c Events) GetContainerByID( @@ -68,20 +74,17 @@ func (c Events) GetContainerByID( ) (graph.Container, error) { service, err := c.service() if err != nil { - return nil, err + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } ofc, err := optionsForCalendarsByID([]string{"name", "owner"}) if err != nil { - return nil, errors.Wrap(err, "options for event calendar") + return nil, clues.Wrap(err, "setting event calendar options").WithClues(ctx).WithAll(graph.ErrData(err)...) } - var cal models.Calendarable - - cal, err = service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc) - + cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc) if err != nil { - return nil, err + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } return graph.CalendarDisplayable{Calendarable: cal}, nil @@ -91,47 +94,36 @@ func (c Events) GetContainerByID( func (c Events) GetItem( ctx context.Context, user, itemID string, + errs *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) { var ( - event models.Eventable err error + event models.Eventable ) event, err = c.stable.Client().UsersById(user).EventsById(itemID).Get(ctx, nil) if err != nil { - return nil, nil, err + return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var ( - errs *multierror.Error - options = &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{ + if *event.GetHasAttachments() || HasAttachments(event.GetBody()) { + options := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{ QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{ Expand: []string{"microsoft.graph.itemattachment/item"}, }, } - ) - - if *event.GetHasAttachments() || HasAttachments(event.GetBody()) { - for count := 0; count < numberOfRetries; count++ { - attached, err := c.largeItem. - Client(). - UsersById(user). - EventsById(itemID). - Attachments(). - Get(ctx, options) - if err == nil { - event.SetAttachments(attached.GetValue()) - break - } - - logger.Ctx(ctx).Debugw("retrying event attachment download", "err", err) - errs = multierror.Append(errs, err) - } + attached, err := c.largeItem. + Client(). + UsersById(user). + EventsById(itemID). + Attachments(). + Get(ctx, options) if err != nil { - logger.Ctx(ctx).Errorw("event attachment download exceeded maximum retries", "err", errs) - return nil, nil, support.WrapAndAppend(itemID, errors.Wrap(err, "download event attachment"), nil) + return nil, nil, clues.Wrap(err, "event attachment download").WithClues(ctx).WithAll(graph.ErrData(err)...) } + + event.SetAttachments(attached.GetValue()) } return event, EventInfo(event), nil @@ -147,57 +139,57 @@ func (c Events) EnumerateContainers( ctx context.Context, userID, baseDirID string, fn func(graph.CacheFolder) error, + errs *fault.Errors, ) error { service, err := c.service() if err != nil { - return err + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var ( - resp models.CalendarCollectionResponseable - errs *multierror.Error - ) - ofc, err := optionsForCalendars([]string{"name"}) if err != nil { - return errors.Wrapf(err, "options for event calendars") + return clues.Wrap(err, "setting calendar options").WithClues(ctx).WithAll(graph.ErrData(err)...) } builder := service.Client().UsersById(userID).Calendars() for { - var err error - - resp, err = builder.Get(ctx, ofc) + resp, err := builder.Get(ctx, ofc) if err != nil { - return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } for _, cal := range resp.GetValue() { cd := CalendarDisplayable{Calendarable: cal} if err := checkIDAndName(cd); err != nil { - errs = multierror.Append(err, errs) + errs.Add(clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)) continue } + fctx := clues.AddAll( + ctx, + "container_id", ptr.Val(cal.GetId()), + "container_name", ptr.Val(cal.GetName())) + temp := graph.NewCacheFolder( cd, - path.Builder{}.Append(*cd.GetId()), // storage path - path.Builder{}.Append(*cd.GetDisplayName())) // display location + path.Builder{}.Append(ptr.Val(cd.GetId())), // storage path + path.Builder{}.Append(ptr.Val(cd.GetDisplayName()))) // display location if err := fn(temp); err != nil { - errs = multierror.Append(err, errs) + errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...)) continue } } - if resp.GetOdataNextLink() == nil { + link, ok := ptr.ValOK(resp.GetOdataNextLink()) + if !ok { break } - builder = users.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), service.Adapter()) + builder = users.NewItemCalendarsRequestBuilder(link, service.Adapter()) } - return errs.ErrorOrNil() + return errs.Err() } // --------------------------------------------------------------------------- @@ -217,14 +209,12 @@ type eventPager struct { } func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { - var ( - resp api.DeltaPageLinker - err error - ) + resp, err := p.builder.Get(ctx, p.options) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } - resp, err = p.builder.Get(ctx, p.options) - - return resp, err + return resp, nil } func (p *eventPager) setNext(nextLink string) { @@ -244,33 +234,30 @@ func (c Events) GetAddedAndRemovedItemIDs( return nil, nil, DeltaUpdate{}, err } - var ( - resetDelta bool - errs *multierror.Error - ) + var resetDelta bool ctx = clues.AddAll( ctx, - "category", selectors.ExchangeEvent, "calendar_id", calendarID) if len(oldDelta) > 0 { - builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &eventPager{service, builder, nil} + var ( + builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter()) + pgr = &eventPager{service, builder, nil} + ) added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition if err == nil { - return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, false}, nil } // only return on error if it is NOT a delta issue. // on bad deltas we retry the call with the regular builder if !graph.IsErrInvalidDelta(err) { - return nil, nil, DeltaUpdate{}, err + return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } resetDelta = true - errs = nil } // Graph SDK only supports delta queries against events on the beta version, so we're @@ -291,7 +278,7 @@ func (c Events) GetAddedAndRemovedItemIDs( } // Events don't have a delta endpoint so just return an empty string. - return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil } // --------------------------------------------------------------------------- @@ -306,10 +293,10 @@ func (c Events) Serialize( ) ([]byte, error) { event, ok := item.(models.Eventable) if !ok { - return nil, fmt.Errorf("expected Eventable, got %T", item) + return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not an Eventable") } - ctx = clues.Add(ctx, "item_id", *event.GetId()) + ctx = clues.Add(ctx, "item_id", ptr.Val(event.GetId())) var ( err error @@ -319,12 +306,12 @@ func (c Events) Serialize( defer writer.Close() if err = writer.WriteObjectValue("", event); err != nil { - return nil, clues.Stack(err).WithClues(ctx) + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } bs, err := writer.GetSerializedContent() if err != nil { - return nil, errors.Wrap(err, "serializing calendar event") + return nil, clues.Wrap(err, "serializing event").WithClues(ctx).WithAll(graph.ErrData(err)...) } return bs, nil @@ -368,22 +355,18 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo { ) if evt.GetOrganizer() != nil && - evt.GetOrganizer().GetEmailAddress() != nil && - evt.GetOrganizer().GetEmailAddress().GetAddress() != nil { - organizer = *evt.GetOrganizer(). - GetEmailAddress(). - GetAddress() + evt.GetOrganizer().GetEmailAddress() != nil { + organizer = ptr.Val(evt.GetOrganizer().GetEmailAddress().GetAddress()) } if evt.GetRecurrence() != nil { recurs = true } - if evt.GetStart() != nil && - evt.GetStart().GetDateTime() != nil { + if evt.GetStart() != nil && len(ptr.Val(evt.GetStart().GetDateTime())) > 0 { // timeString has 'Z' literal added to ensure the stored // DateTime is not: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) - startTime := *evt.GetStart().GetDateTime() + "Z" + startTime := ptr.Val(evt.GetStart().GetDateTime()) + "Z" output, err := common.ParseTime(startTime) if err == nil { @@ -391,11 +374,10 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo { } } - if evt.GetEnd() != nil && - evt.GetEnd().GetDateTime() != nil { + if evt.GetEnd() != nil && len(ptr.Val(evt.GetEnd().GetDateTime())) > 0 { // timeString has 'Z' literal added to ensure the stored // DateTime is not: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) - endTime := *evt.GetEnd().GetDateTime() + "Z" + endTime := ptr.Val(evt.GetEnd().GetDateTime()) + "Z" output, err := common.ParseTime(endTime) if err == nil { @@ -411,6 +393,6 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo { EventEnd: end, EventRecurs: recurs, Created: created, - Modified: orNow(evt.GetLastModifiedDateTime()), + Modified: ptr.OrNow(evt.GetLastModifiedDateTime()), } } diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index 443b7d3e1..ed768459a 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -5,19 +5,16 @@ import ( "fmt" "github.com/alcionai/clues" - "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/users" - "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" - "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -49,7 +46,12 @@ func (c Mail) CreateMailFolder( requestBody.SetDisplayName(&folder) requestBody.SetIsHidden(&isHidden) - return c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil) + mdl, err := c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return mdl, nil } func (c Mail) CreateMailFolderWithParent( @@ -58,7 +60,7 @@ func (c Mail) CreateMailFolderWithParent( ) (models.MailFolderable, error) { service, err := c.service() if err != nil { - return nil, err + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } isHidden := false @@ -66,12 +68,17 @@ func (c Mail) CreateMailFolderWithParent( requestBody.SetDisplayName(&folder) requestBody.SetIsHidden(&isHidden) - return service. + mdl, err := service. Client(). UsersById(user). MailFoldersById(parentID). ChildFolders(). Post(ctx, requestBody, nil) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return mdl, nil } // DeleteContainer removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account @@ -80,7 +87,12 @@ func (c Mail) DeleteContainer( ctx context.Context, user, folderID string, ) error { - return c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil) + err := c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil) + if err != nil { + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } + + return nil } func (c Mail) GetContainerByID( @@ -89,19 +101,20 @@ func (c Mail) GetContainerByID( ) (graph.Container, error) { service, err := c.service() if err != nil { - return nil, err + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } ofmf, err := optionsForMailFoldersItem([]string{"displayName", "parentFolderId"}) if err != nil { - return nil, errors.Wrap(err, "options for mail folder") + return nil, clues.Wrap(err, "setting mail folder options").WithClues(ctx).WithAll(graph.ErrData(err)...) } - var resp graph.Container + resp, err := service.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } - resp, err = service.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf) - - return resp, err + return resp, nil } // GetItem retrieves a Messageable item. If the item contains an attachment, that @@ -109,45 +122,31 @@ func (c Mail) GetContainerByID( func (c Mail) GetItem( ctx context.Context, user, itemID string, + errs *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) { - var ( - mail models.Messageable - err error - ) - - mail, err = c.stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil) + mail, err := c.stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil) if err != nil { - return nil, nil, err + return nil, nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var errs *multierror.Error - if *mail.GetHasAttachments() || HasAttachments(mail.GetBody()) { options := &users.ItemMessagesItemAttachmentsRequestBuilderGetRequestConfiguration{ QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{ Expand: []string{"microsoft.graph.itemattachment/item"}, }, } - for count := 0; count < numberOfRetries; count++ { - attached, err := c.largeItem. - Client(). - UsersById(user). - MessagesById(itemID). - Attachments(). - Get(ctx, options) - if err == nil { - mail.SetAttachments(attached.GetValue()) - break - } - - logger.Ctx(ctx).Debugw("retrying mail attachment download", "err", err) - errs = multierror.Append(errs, err) - } + attached, err := c.largeItem. + Client(). + UsersById(user). + MessagesById(itemID). + Attachments(). + Get(ctx, options) if err != nil { - logger.Ctx(ctx).Errorw("mail attachment download exceeded maximum retries", "err", errs) - return nil, nil, support.WrapAndAppend(itemID, errors.Wrap(err, "downloading mail attachment"), nil) + return nil, nil, clues.Wrap(err, "mail attachment download").WithClues(ctx).WithAll(graph.ErrData(err)...) } + + mail.SetAttachments(attached.GetValue()) } return mail, MailInfo(mail), nil @@ -163,46 +162,46 @@ func (c Mail) EnumerateContainers( ctx context.Context, userID, baseDirID string, fn func(graph.CacheFolder) error, + errs *fault.Errors, ) error { service, err := c.service() if err != nil { - return err + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } - var ( - resp users.ItemMailFoldersDeltaResponseable - errs *multierror.Error - builder = service.Client(). - UsersById(userID). - MailFolders(). - Delta() - ) + builder := service.Client(). + UsersById(userID). + MailFolders(). + Delta() for { - var err error - - resp, err = builder.Get(ctx, nil) + resp, err := builder.Get(ctx, nil) if err != nil { - return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + return clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } for _, v := range resp.GetValue() { + fctx := clues.AddAll( + ctx, + "container_id", ptr.Val(v.GetId()), + "container_name", ptr.Val(v.GetDisplayName())) + temp := graph.NewCacheFolder(v, nil, nil) if err := fn(temp); err != nil { - errs = multierror.Append(errs, errors.Wrap(err, "iterating mail folders delta")) + errs.Add(clues.Stack(err).WithClues(fctx).WithAll(graph.ErrData(err)...)) continue } } - link := resp.GetOdataNextLink() - if link == nil { + link, ok := ptr.ValOK(resp.GetOdataNextLink()) + if !ok { break } - builder = users.NewItemMailFoldersDeltaRequestBuilder(*link, service.Adapter()) + builder = users.NewItemMailFoldersDeltaRequestBuilder(link, service.Adapter()) } - return errs.ErrorOrNil() + return errs.Err() } // --------------------------------------------------------------------------- @@ -218,14 +217,12 @@ type mailPager struct { } func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) { - var ( - page api.DeltaPageLinker - err error - ) + page, err := p.builder.Get(ctx, p.options) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) + } - page, err = p.builder.Get(ctx, p.options) - - return page, err + return page, nil } func (p *mailPager) setNext(nextLink string) { @@ -246,7 +243,6 @@ func (c Mail) GetAddedAndRemovedItemIDs( } var ( - errs *multierror.Error deltaURL string resetDelta bool ) @@ -258,17 +254,22 @@ func (c Mail) GetAddedAndRemovedItemIDs( options, err := optionsForFolderMessagesDelta([]string{"isRead"}) if err != nil { - return nil, nil, DeltaUpdate{}, errors.Wrap(err, "getting query options") + return nil, + nil, + DeltaUpdate{}, + clues.Wrap(err, "setting contact folder options").WithClues(ctx).WithAll(graph.ErrData(err)...) } if len(oldDelta) > 0 { - builder := users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &mailPager{service, builder, options} + var ( + builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter()) + pgr = &mailPager{service, builder, options} + ) added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition if err == nil { - return added, removed, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, false}, err } // only return on error if it is NOT a delta issue. // on bad deltas we retry the call with the regular builder @@ -277,7 +278,6 @@ func (c Mail) GetAddedAndRemovedItemIDs( } resetDelta = true - errs = nil } builder := service.Client().UsersById(user).MailFoldersById(directoryID).Messages().Delta() @@ -288,7 +288,7 @@ func (c Mail) GetAddedAndRemovedItemIDs( return nil, nil, DeltaUpdate{}, err } - return added, removed, DeltaUpdate{deltaURL, resetDelta}, errs.ErrorOrNil() + return added, removed, DeltaUpdate{deltaURL, resetDelta}, nil } // --------------------------------------------------------------------------- @@ -303,10 +303,10 @@ func (c Mail) Serialize( ) ([]byte, error) { msg, ok := item.(models.Messageable) if !ok { - return nil, fmt.Errorf("expected Messageable, got %T", item) + return nil, clues.Wrap(fmt.Errorf("parseable type: %T", item), "parsable is not a Messageable") } - ctx = clues.Add(ctx, "item_id", *msg.GetId()) + ctx = clues.Add(ctx, "item_id", ptr.Val(msg.GetId())) var ( err error @@ -316,12 +316,12 @@ func (c Mail) Serialize( defer writer.Close() if err = writer.WriteObjectValue("", msg); err != nil { - return nil, clues.Stack(err).WithClues(ctx) + return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } bs, err := writer.GetSerializedContent() if err != nil { - return nil, errors.Wrap(err, "serializing email") + return nil, clues.Wrap(err, "serializing email").WithClues(ctx).WithAll(graph.ErrData(err)...) } return bs, nil @@ -338,9 +338,8 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo { created := ptr.Val(msg.GetCreatedDateTime()) if msg.GetSender() != nil && - msg.GetSender().GetEmailAddress() != nil && - msg.GetSender().GetEmailAddress().GetAddress() != nil { - sender = *msg.GetSender().GetEmailAddress().GetAddress() + msg.GetSender().GetEmailAddress() != nil { + sender = ptr.Val(msg.GetSender().GetEmailAddress().GetAddress()) } return &details.ExchangeInfo{ @@ -349,6 +348,6 @@ func MailInfo(msg models.Messageable) *details.ExchangeInfo { Subject: subject, Received: received, Created: created, - Modified: orNow(msg.GetLastModifiedDateTime()), + Modified: ptr.OrNow(msg.GetLastModifiedDateTime()), } } diff --git a/src/internal/connector/exchange/api/shared.go b/src/internal/connector/exchange/api/shared.go index e4d563e90..6e8a64f34 100644 --- a/src/internal/connector/exchange/api/shared.go +++ b/src/internal/connector/exchange/api/shared.go @@ -2,12 +2,13 @@ package api import ( "context" + "fmt" - "github.com/pkg/errors" + "github.com/alcionai/clues" + "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/logger" ) @@ -34,7 +35,7 @@ type getIDAndAddtler interface { func toValues[T any](a any) ([]getIDAndAddtler, error) { gv, ok := a.(interface{ GetValue() []T }) if !ok { - return nil, errors.Errorf("response of type [%T] does not comply with the GetValue() interface", a) + return nil, clues.Wrap(fmt.Errorf("%T", a), "does not comply with the GetValue() interface") } items := gv.GetValue() @@ -45,7 +46,7 @@ func toValues[T any](a any) ([]getIDAndAddtler, error) { ri, ok := a.(getIDAndAddtler) if !ok { - return nil, errors.Errorf("item of type [%T] does not comply with the getIDAndAddtler interface", item) + return nil, clues.Wrap(fmt.Errorf("%T", item), "does not comply with the getIDAndAddtler interface") } r = append(r, ri) @@ -72,18 +73,14 @@ func getItemsAddedAndRemovedFromContainer( // get the next page of data, check for standard errors resp, err := pager.getPage(ctx) if err != nil { - if graph.IsErrDeletedInFlight(err) || graph.IsErrInvalidDelta(err) { - return nil, nil, deltaURL, err - } - - return nil, nil, deltaURL, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + return nil, nil, deltaURL, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } // each category type responds with a different interface, but all // of them comply with GetValue, which is where we'll get our item data. items, err := pager.valuesIn(resp) if err != nil { - return nil, nil, "", err + return nil, nil, "", clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...) } itemCount += len(items) @@ -100,9 +97,9 @@ func getItemsAddedAndRemovedFromContainer( // be 'changed' or 'deleted'. We don't really care about the cause: both // cases are handled the same way in storage. if item.GetAdditionalData()[graph.AddtlDataRemoved] == nil { - addedIDs = append(addedIDs, *item.GetId()) + addedIDs = append(addedIDs, ptr.Val(item.GetId())) } else { - removedIDs = append(removedIDs, *item.GetId()) + removedIDs = append(removedIDs, ptr.Val(item.GetId())) } } diff --git a/src/internal/connector/exchange/contact_folder_cache.go b/src/internal/connector/exchange/contact_folder_cache.go index e7ef18292..6b6d01513 100644 --- a/src/internal/connector/exchange/contact_folder_cache.go +++ b/src/internal/connector/exchange/contact_folder_cache.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -46,6 +47,7 @@ func (cfc *contactFolderCache) populateContactRoot( // as of (Oct-07-2022) func (cfc *contactFolderCache) Populate( ctx context.Context, + errs *fault.Errors, baseID string, baseContainerPather ...string, ) error { @@ -53,7 +55,7 @@ func (cfc *contactFolderCache) Populate( return errors.Wrap(err, "initializing") } - err := cfc.enumer.EnumerateContainers(ctx, cfc.userID, baseID, cfc.addFolder) + err := cfc.enumer.EnumerateContainers(ctx, cfc.userID, baseID, cfc.addFolder, errs) if err != nil { return errors.Wrap(err, "enumerating containers") } diff --git a/src/internal/connector/exchange/container_resolver.go b/src/internal/connector/exchange/container_resolver.go index 3270940ed..706e4784b 100644 --- a/src/internal/connector/exchange/container_resolver.go +++ b/src/internal/connector/exchange/container_resolver.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -27,6 +28,7 @@ type containersEnumerator interface { ctx context.Context, userID, baseDirID string, fn func(graph.CacheFolder) error, + errs *fault.Errors, ) error } diff --git a/src/internal/connector/exchange/container_resolver_test.go b/src/internal/connector/exchange/container_resolver_test.go index 5bc4fe317..601fec0b2 100644 --- a/src/internal/connector/exchange/container_resolver_test.go +++ b/src/internal/connector/exchange/container_resolver_test.go @@ -13,6 +13,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -657,7 +658,8 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { m365, test.pathFunc1(t), folderName, - directoryCaches) + directoryCaches, + fault.New(true)) require.NoError(t, err) resolver := directoryCaches[test.category] @@ -675,7 +677,8 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { m365, test.pathFunc2(t), parentContainer, - directoryCaches) + directoryCaches, + fault.New(true)) require.NoError(t, err) _, _, err = resolver.IDToPath(ctx, secondID, test.useIDForPath) diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index fd41418f8..fcfa8ab8a 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -264,7 +264,7 @@ func createCollections( defer closer() defer close(foldersComplete) - resolver, err := PopulateExchangeContainerResolver(ctx, qp) + resolver, err := PopulateExchangeContainerResolver(ctx, qp, errs) if err != nil { return nil, errors.Wrap(err, "populating container cache") } diff --git a/src/internal/connector/exchange/data_collections_test.go b/src/internal/connector/exchange/data_collections_test.go index 8eea6bcba..7b8674b58 100644 --- a/src/internal/connector/exchange/data_collections_test.go +++ b/src/internal/connector/exchange/data_collections_test.go @@ -554,7 +554,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression( return nil } - require.NoError(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn)) + require.NoError(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn, fault.New(true))) tests := []struct { name, expected string diff --git a/src/internal/connector/exchange/event_calendar_cache.go b/src/internal/connector/exchange/event_calendar_cache.go index c49e19e14..b7dd1ded4 100644 --- a/src/internal/connector/exchange/event_calendar_cache.go +++ b/src/internal/connector/exchange/event_calendar_cache.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -61,6 +62,7 @@ func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error { // @param baseID: ignored. Present to conform to interface func (ecc *eventCalendarCache) Populate( ctx context.Context, + errs *fault.Errors, baseID string, baseContainerPath ...string, ) error { @@ -72,7 +74,8 @@ func (ecc *eventCalendarCache) Populate( ctx, ecc.userID, "", - ecc.addFolder) + ecc.addFolder, + errs) if err != nil { return errors.Wrap(err, "enumerating containers") } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index fd37a1f1f..a7d07ae66 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" ) @@ -43,6 +44,7 @@ type itemer interface { GetItem( ctx context.Context, user, itemID string, + errs *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) Serialize( ctx context.Context, @@ -250,7 +252,12 @@ func (col *Collection) streamItems(ctx context.Context) { err error ) - item, info, err = getItemWithRetries(ctx, user, id, col.items) + item, info, err = getItemWithRetries( + ctx, + user, + id, + col.items, + fault.New(true)) // temporary way to force a failFast error if err != nil { // Don't report errors for deleted items as there's no way for us to // back up data that is gone. Record it as a "success", since there's @@ -298,6 +305,7 @@ func getItemWithRetries( ctx context.Context, userID, itemID string, items itemer, + errs *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) { var ( item serialization.Parsable @@ -306,7 +314,7 @@ func getItemWithRetries( ) for i := 1; i <= numberOfRetries; i++ { - item, info, err = items.GetItem(ctx, userID, itemID) + item, info, err = items.GetItem(ctx, userID, itemID, errs) if err == nil { break } diff --git a/src/internal/connector/exchange/exchange_data_collection_test.go b/src/internal/connector/exchange/exchange_data_collection_test.go index 8644a0523..7e460ad18 100644 --- a/src/internal/connector/exchange/exchange_data_collection_test.go +++ b/src/internal/connector/exchange/exchange_data_collection_test.go @@ -16,6 +16,7 @@ import ( "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -29,12 +30,17 @@ type mockItemer struct { func (mi *mockItemer) GetItem( context.Context, string, string, + *fault.Errors, ) (serialization.Parsable, *details.ExchangeInfo, error) { mi.getCount++ return nil, nil, mi.getErr } -func (mi *mockItemer) Serialize(context.Context, serialization.Parsable, string, string) ([]byte, error) { +func (mi *mockItemer) Serialize( + context.Context, + serialization.Parsable, + string, string, +) ([]byte, error) { mi.serializeCount++ return nil, mi.serializeErr } @@ -224,7 +230,7 @@ func (suite *ExchangeDataCollectionSuite) TestGetItemWithRetries() { defer flush() // itemer is mocked, so only the errors are configured atm. - _, _, err := getItemWithRetries(ctx, "userID", "itemID", test.items) + _, _, err := getItemWithRetries(ctx, "userID", "itemID", test.items, fault.New(true)) test.expectErr(t, err) }) } diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 8caab5c87..5f9cc8c25 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" ) type CacheResolverSuite struct { @@ -119,7 +120,7 @@ func (suite *CacheResolverSuite) TestPopulate() { for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { resolver := test.resolverFunc(t) - require.NoError(t, resolver.Populate(ctx, test.root, test.basePath)) + require.NoError(t, resolver.Populate(ctx, fault.New(true), test.root, test.basePath)) _, isFound := resolver.PathInCache(test.folderInCache) test.canFind(t, isFound) diff --git a/src/internal/connector/exchange/mail_folder_cache.go b/src/internal/connector/exchange/mail_folder_cache.go index 4bf31460c..90a070919 100644 --- a/src/internal/connector/exchange/mail_folder_cache.go +++ b/src/internal/connector/exchange/mail_folder_cache.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -71,6 +72,7 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error { // for the base container in the cache. func (mc *mailFolderCache) Populate( ctx context.Context, + errs *fault.Errors, baseID string, baseContainerPath ...string, ) error { @@ -78,7 +80,7 @@ func (mc *mailFolderCache) Populate( return errors.Wrap(err, "initializing") } - err := mc.enumer.EnumerateContainers(ctx, mc.userID, "", mc.addFolder) + err := mc.enumer.EnumerateContainers(ctx, mc.userID, "", mc.addFolder, errs) if err != nil { return errors.Wrap(err, "enumerating containers") } diff --git a/src/internal/connector/exchange/mail_folder_cache_test.go b/src/internal/connector/exchange/mail_folder_cache_test.go index e671a1d3b..72e08d6c4 100644 --- a/src/internal/connector/exchange/mail_folder_cache_test.go +++ b/src/internal/connector/exchange/mail_folder_cache_test.go @@ -11,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" ) const ( @@ -92,7 +93,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { getter: acm, } - require.NoError(t, mfc.Populate(ctx, test.root, test.path...)) + require.NoError(t, mfc.Populate(ctx, fault.New(true), test.root, test.path...)) p, l, err := mfc.IDToPath(ctx, testFolderID, true) require.NoError(t, err) diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index f3fb69462..875ed7e66 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -35,6 +36,7 @@ func createService(credentials account.M365Config) (*graph.Service, error) { func PopulateExchangeContainerResolver( ctx context.Context, qp graph.QueryParams, + errs *fault.Errors, ) (graph.ContainerResolver, error) { var ( res graph.ContainerResolver @@ -78,7 +80,7 @@ func PopulateExchangeContainerResolver( return nil, clues.New("no container resolver registered for category").WithClues(ctx) } - if err := res.Populate(ctx, cacheRoot); err != nil { + if err := res.Populate(ctx, errs, cacheRoot); err != nil { return nil, clues.Wrap(err, "populating directory resolver").WithClues(ctx) } diff --git a/src/internal/connector/exchange/service_iterators_test.go b/src/internal/connector/exchange/service_iterators_test.go index 81d9ebd99..70c9a5b37 100644 --- a/src/internal/connector/exchange/service_iterators_test.go +++ b/src/internal/connector/exchange/service_iterators_test.go @@ -91,8 +91,8 @@ func (m mockResolver) DestinationNameToID(dest string) string { return m.added[d func (m mockResolver) IDToPath(context.Context, string, bool) (*path.Builder, *path.Builder, error) { return nil, nil, nil } -func (m mockResolver) PathInCache(string) (string, bool) { return "", false } -func (m mockResolver) Populate(context.Context, string, ...string) error { return nil } +func (m mockResolver) PathInCache(string) (string, bool) { return "", false } +func (m mockResolver) Populate(context.Context, *fault.Errors, string, ...string) error { return nil } // --------------------------------------------------------------------------- // tests diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 6b5b06440..3d11d358f 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -330,7 +330,8 @@ func RestoreExchangeDataCollections( creds, dc.FullPath(), dest.ContainerName, - userCaches) + userCaches, + errs) if err != nil { errs.Add(clues.Wrap(err, "creating destination").WithClues(ctx)) continue @@ -471,6 +472,7 @@ func CreateContainerDestination( directory path.Path, destination string, caches map[path.CategoryType]graph.ContainerResolver, + errs *fault.Errors, ) (string, error) { var ( newCache = false @@ -508,7 +510,8 @@ func CreateContainerDestination( folders, directoryCache, user, - newCache) + newCache, + errs) case path.ContactsCategory: folders := append([]string{destination}, directory.Folders()...) @@ -531,7 +534,8 @@ func CreateContainerDestination( folders, directoryCache, user, - newCache) + newCache, + errs) case path.EventsCategory: dest := destination @@ -561,7 +565,8 @@ func CreateContainerDestination( folders, directoryCache, user, - newCache) + newCache, + errs) default: return "", clues.Wrap(fmt.Errorf("%T", category), "not support for exchange cache").WithClues(ctx) @@ -580,6 +585,7 @@ func establishMailRestoreLocation( mfc graph.ContainerResolver, user string, isNewCache bool, + errs *fault.Errors, ) (string, error) { // Process starts with the root folder in order to recreate // the top-level folder with the same tactic @@ -609,7 +615,7 @@ func establishMailRestoreLocation( // newCache to false in this we'll only try to populate it once per function // call even if we make a new cache. if isNewCache { - if err := mfc.Populate(ctx, rootFolderAlias); err != nil { + if err := mfc.Populate(ctx, errs, rootFolderAlias); err != nil { return "", errors.Wrap(err, "populating folder cache") } @@ -638,6 +644,7 @@ func establishContactsRestoreLocation( cfc graph.ContainerResolver, user string, isNewCache bool, + errs *fault.Errors, ) (string, error) { cached, ok := cfc.PathInCache(folders[0]) if ok { @@ -654,7 +661,7 @@ func establishContactsRestoreLocation( folderID := *temp.GetId() if isNewCache { - if err := cfc.Populate(ctx, folderID, folders[0]); err != nil { + if err := cfc.Populate(ctx, errs, folderID, folders[0]); err != nil { return "", errors.Wrap(err, "populating contact cache") } @@ -673,6 +680,7 @@ func establishEventsRestoreLocation( ecc graph.ContainerResolver, // eventCalendarCache user string, isNewCache bool, + errs *fault.Errors, ) (string, error) { // Need to prefix with the "Other Calendars" folder so lookup happens properly. cached, ok := ecc.PathInCache(folders[0]) @@ -690,7 +698,7 @@ func establishEventsRestoreLocation( folderID := *temp.GetId() if isNewCache { - if err = ecc.Populate(ctx, folderID, folders[0]); err != nil { + if err = ecc.Populate(ctx, errs, folderID, folders[0]); err != nil { return "", errors.Wrap(err, "populating event cache") } diff --git a/src/internal/connector/graph/cache_container.go b/src/internal/connector/graph/cache_container.go index 8de403867..89210a728 100644 --- a/src/internal/connector/graph/cache_container.go +++ b/src/internal/connector/graph/cache_container.go @@ -6,6 +6,7 @@ import ( "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -62,7 +63,7 @@ type ContainerResolver interface { // @param ctx is necessary param for Graph API tracing // @param baseFolderID represents the M365ID base that the resolver will // conclude its search. Default input is "". - Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error + Populate(ctx context.Context, errs *fault.Errors, baseFolderID string, baseContainerPather ...string) error // PathInCache performs a look up of a path reprensentation // and returns the m365ID of directory iff the pathString diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 58a7fb6de..c14090bde 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -771,7 +771,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ResourceOwner: suite.user, Credentials: m365, } - cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp) + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating %s container resolver", category) for destName, dest := range gen.dests { @@ -890,7 +890,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ResourceOwner: suite.user, Credentials: m365, } - cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp) + cr, err := exchange.PopulateExchangeContainerResolver(ctx, qp, fault.New(true)) require.NoError(t, err, "populating %s container resolver", category) p, err := path.FromDataLayerPath(deets.Entries[0].RepoRef, true)