From 60a04c9ca41e4fc2eb0b066243ef04843a4809ff Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Thu, 19 Jan 2023 13:19:07 +0530 Subject: [PATCH 1/9] Handle ResourceNotFound error when fetching drive (#2161) ## Description Follow up for https://github.com/alcionai/corso/pull/2156. We might deice to change where we add the logic to handle this, as in if we should move it to discovery module. ## Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * https://github.com/alcionai/corso/issues/2145 * https://github.com/alcionai/corso/pull/2156 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- CHANGELOG.md | 1 + src/internal/connector/onedrive/drive.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f24db17..b981d778a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Check if the user specified for an exchange backup operation has a mailbox. +- Handle case where user's drive has not been initialized ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index e98e8d35b..18a1b2477 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -76,7 +76,8 @@ const ( itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children" itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s" itemNotFoundErrorCode = "itemNotFound" - userDoesNotHaveDrive = "BadRequest Unable to retrieve user's mysite URL" + userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL" + userMysiteNotFound = "ResourceNotFound User's mysite not found" ) // Enumerates the drives for the specified user @@ -134,7 +135,8 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod r, err = service.Client().UsersById(user).Drives().Get(ctx, nil) if err != nil { detailedError := support.ConnectorStackErrorTrace(err) - if strings.Contains(detailedError, userDoesNotHaveDrive) { + if strings.Contains(detailedError, userMysiteURLNotFound) || + strings.Contains(detailedError, userMysiteNotFound) { logger.Ctx(ctx).Debugf("User %s does not have a drive", user) return make([]models.Driveable, 0), nil // no license } From 01be9fb67f16c4c8c3278609a78145067857b479 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Jan 2023 19:12:31 +0000 Subject: [PATCH 2/9] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aws/?= =?UTF-8?q?aws-sdk-go=20from=201.44.181=20to=201.44.182=20in=20/src=20(#21?= =?UTF-8?q?76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.181 to 1.44.182.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.182 (2023-01-18)

Service Client Updates

  • service/elasticfilesystem: Updates service documentation
    • Documentation updates for EFS access points limit increase
  • service/ivschat: Updates service API
  • service/monitoring: Updates service API and documentation
    • Enable cross-account streams in CloudWatch Metric Streams via Observability Access Manager.
  • service/wafv2: Updates service documentation
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.181&new-version=1.44.182)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index 0ba54fea1..fa987b9b3 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.181 + github.com/aws/aws-sdk-go v1.44.182 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/src/go.sum b/src/go.sum index 222bf48f8..f434af2eb 100644 --- a/src/go.sum +++ b/src/go.sum @@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.181 h1:w4OzE8bwIVo62gUTAp/uEFO2HSsUtf1pjXpSs36cluY= -github.com/aws/aws-sdk-go v1.44.181/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.182 h1:DUEhWpWl4yTPgt142qwUfH1rYeB6KUCHDcpL7lF4+9M= +github.com/aws/aws-sdk-go v1.44.182/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= From 0852de5d6433c4aa8906bee8fef08ee6ea4b2f17 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 19 Jan 2023 12:43:16 -0700 Subject: [PATCH 3/9] set new CI test user default (#2172) ## Description CI Secrets will need to be updated to complete this change. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :robot: Test - [x] :computer: CI/Deployment ## Issue(s) * #2022 ## Test Plan - [x] :muscle: Manual - [x] :green_heart: E2E --- src/internal/tester/config.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 804b217f3..d79d95104 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -111,8 +111,7 @@ func readTestConfig() (map[string]string, error) { TestCfgUserID, os.Getenv(EnvCorsoM365TestUserID), vpr.GetString(TestCfgUserID), - "lynner@8qzvrj.onmicrosoft.com", - //"lidiah@8qzvrj.onmicrosoft.com", + "conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -120,7 +119,6 @@ func readTestConfig() (map[string]string, error) { os.Getenv(EnvCorsoSecondaryM365TestUserID), vpr.GetString(TestCfgSecondaryUserID), "lidiah@8qzvrj.onmicrosoft.com", - //"lynner@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -134,7 +132,7 @@ func readTestConfig() (map[string]string, error) { TestCfgLoadTestOrgUsers, os.Getenv(EnvCorsoM365LoadTestOrgUsers), vpr.GetString(TestCfgLoadTestOrgUsers), - "lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com", + "lidiah@8qzvrj.onmicrosoft.com,conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, From 8df00bd386e43c646396785a2630bf22d93c40de Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 19 Jan 2023 15:30:14 -0700 Subject: [PATCH 4/9] standardizes the calendar resolver interface (#2162) ## Description Adds a little extra process into the calendar resolver so that it mimics the mail and contact resolvers. This will allow us to collapse the three resolvers into a more common handler or interface. We can take this change or drop it. I added the code in exploration of the event test failures, and figured I'd throw it out just in case. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :broom: Tech Debt/Cleanup ## Issue(s) * #2022 ## Test Plan - [x] :green_heart: E2E --- src/internal/connector/exchange/api/events.go | 31 ++++++++--- .../connector/exchange/api/options.go | 54 ++++++++----------- .../exchange/event_calendar_cache.go | 43 ++++++++++++++- .../exchange/folder_resolver_test.go | 1 + .../connector/exchange/mail_folder_cache.go | 30 +++++------ .../connector/exchange/service_functions.go | 4 +- .../connector/exchange/service_restore.go | 4 +- 7 files changed, 108 insertions(+), 59 deletions(-) diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index b4988d28d..0aafaf3f2 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -58,6 +58,28 @@ func (c Events) DeleteCalendar( return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) } +func (c Events) GetContainerByID( + ctx context.Context, + userID, containerID string, +) (graph.Container, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + ofc, err := optionsForCalendarsByID([]string{"name", "owner"}) + if err != nil { + return nil, errors.Wrap(err, "options for event calendar") + } + + cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc) + if err != nil { + return nil, err + } + + return graph.CalendarDisplayable{Calendarable: cal}, nil +} + // GetItem retrieves an Eventable item. func (c Events) GetItem( ctx context.Context, @@ -183,14 +205,9 @@ func (c Events) GetAddedAndRemovedItemIDs( errs *multierror.Error ) - options, err := optionsForEventsByCalendarDelta([]string{"id"}) - if err != nil { - return nil, nil, DeltaUpdate{}, err - } - if len(oldDelta) > 0 { builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition @@ -217,7 +234,7 @@ func (c Events) GetAddedAndRemovedItemIDs( // works as intended (until, at least, we want to _not_ call the beta anymore). rawURL := fmt.Sprintf(eventBetaDeltaURLTemplate, user, calendarID) builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(rawURL, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) if err != nil { diff --git a/src/internal/connector/exchange/api/options.go b/src/internal/connector/exchange/api/options.go index 99d41487c..49debf334 100644 --- a/src/internal/connector/exchange/api/options.go +++ b/src/internal/connector/exchange/api/options.go @@ -21,18 +21,6 @@ var ( "owner": {}, } - fieldsForEvents = map[string]struct{}{ - "calendar": {}, - "end": {}, - "id": {}, - "isOnlineMeeting": {}, - "isReminderOn": {}, - "responseStatus": {}, - "responseRequested": {}, - "showAs": {}, - "subject": {}, - } - fieldsForFolders = map[string]struct{}{ "childFolderCount": {}, "displayName": {}, @@ -112,6 +100,28 @@ func optionsForCalendars(moreOps []string) ( return options, nil } +// optionsForCalendarsByID places allowed options for exchange.Calendar object +// @param moreOps should reflect elements from fieldsForCalendars +// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler +func optionsForCalendarsByID(moreOps []string) ( + *users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration, + error, +) { + selecting, err := buildOptions(moreOps, fieldsForCalendars) + if err != nil { + return nil, err + } + // should be a CalendarsRequestBuilderGetRequestConfiguration + requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ + Select: selecting, + } + options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ + QueryParameters: requestParams, + } + + return options, nil +} + // optionsForContactFolders places allowed options for exchange.ContactFolder object // @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler func optionsForContactFolders(moreOps []string) ( @@ -213,26 +223,6 @@ func optionsForContactFoldersItemDelta( return options, nil } -// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar -func optionsForEventsByCalendarDelta( - moreOps []string, -) (*users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration, error) { - selecting, err := buildOptions(moreOps, fieldsForEvents) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - // optionsForContactChildFolders builds a contacts child folders request. func optionsForContactChildFolders( moreOps []string, diff --git a/src/internal/connector/exchange/event_calendar_cache.go b/src/internal/connector/exchange/event_calendar_cache.go index 2b4e1b22d..e497a272a 100644 --- a/src/internal/connector/exchange/event_calendar_cache.go +++ b/src/internal/connector/exchange/event_calendar_cache.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/path" ) @@ -14,9 +15,43 @@ var _ graph.ContainerResolver = &eventCalendarCache{} type eventCalendarCache struct { *containerResolver enumer containersEnumerator + getter containerGetter userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (ecc *eventCalendarCache) init( + ctx context.Context, +) error { + if ecc.containerResolver == nil { + ecc.containerResolver = newContainerResolver() + } + + return ecc.populateEventRoot(ctx) +} + +// populateEventRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ +// DefaultCalendar is the traditional "Calendar". +// Action ensures that cache will stop at appropriate level. +// @error iff the struct is not properly instantiated +func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error { + container := DefaultCalendar + + f, err := ecc.getter.GetContainerByID(ctx, ecc.userID, container) + if err != nil { + return errors.Wrap(err, "fetching calendar "+support.ConnectorStackErrorTrace(err)) + } + + temp := graph.NewCacheFolder(f, path.Builder{}.Append(container)) + if err := ecc.addFolder(temp); err != nil { + return errors.Wrap(err, "initializing calendar resolver") + } + + return nil +} + // Populate utility function for populating eventCalendarCache. // Executes 1 additional Graph Query // @param baseID: ignored. Present to conform to interface @@ -25,8 +60,8 @@ func (ecc *eventCalendarCache) Populate( baseID string, baseContainerPath ...string, ) error { - if ecc.containerResolver == nil { - ecc.containerResolver = newContainerResolver() + if err := ecc.init(ctx); err != nil { + return errors.Wrap(err, "initializing") } err := ecc.enumer.EnumerateContainers(ctx, ecc.userID, "", ecc.addFolder) @@ -34,6 +69,10 @@ func (ecc *eventCalendarCache) Populate( return errors.Wrap(err, "enumerating containers") } + if err := ecc.populatePaths(ctx); err != nil { + return errors.Wrap(err, "establishing calendar paths") + } + return nil } diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 36bf0ffc5..ea7ccb995 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -51,6 +51,7 @@ func (suite *CacheResolverSuite) TestPopulate() { return &eventCalendarCache{ userID: tester.M365UserID(t), enumer: ac.Events(), + getter: ac.Events(), } } diff --git a/src/internal/connector/exchange/mail_folder_cache.go b/src/internal/connector/exchange/mail_folder_cache.go index 06d4b1285..565f10736 100644 --- a/src/internal/connector/exchange/mail_folder_cache.go +++ b/src/internal/connector/exchange/mail_folder_cache.go @@ -22,14 +22,25 @@ type mailFolderCache struct { userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (mc *mailFolderCache) init( + ctx context.Context, +) error { + if mc.containerResolver == nil { + mc.containerResolver = newContainerResolver() + } + + return mc.populateMailRoot(ctx) +} + // populateMailRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ // rootFolderAlias is the top-level directory for exchange.Mail. // DefaultMailFolder is the traditional "Inbox" for exchange.Mail // Action ensures that cache will stop at appropriate level. // @error iff the struct is not properly instantiated -func (mc *mailFolderCache) populateMailRoot( - ctx context.Context, -) error { +func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error { for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} { var directory string @@ -76,16 +87,3 @@ func (mc *mailFolderCache) Populate( return nil } - -// init ensures that the structure's fields are initialized. -// Fields Initialized when cache == nil: -// [mc.cache] -func (mc *mailFolderCache) init( - ctx context.Context, -) error { - if mc.containerResolver == nil { - mc.containerResolver = newContainerResolver() - } - - return mc.populateMailRoot(ctx) -} diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index d646fb132..9d996f01c 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -66,9 +66,11 @@ func PopulateExchangeContainerResolver( cacheRoot = DefaultContactFolder case path.EventsCategory: + ecc := ac.Events() res = &eventCalendarCache{ userID: qp.ResourceOwner, - enumer: ac.Events(), + getter: ecc, + enumer: ecc, } cacheRoot = DefaultCalendar diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 3f88f6efe..d385dab81 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -507,9 +507,11 @@ func CreateContainerDestinaion( case path.EventsCategory: if directoryCache == nil { + ace := ac.Events() ecc := &eventCalendarCache{ userID: user, - enumer: ac.Events(), + getter: ace, + enumer: ace, } caches[category] = ecc newCache = true From 447321588212266969fa404520e2e2a0e6b5d47b Mon Sep 17 00:00:00 2001 From: Danny Date: Thu, 19 Jan 2023 19:32:18 -0500 Subject: [PATCH 5/9] GC: Backup: [FIX]Embedded image requires download (#2189) ## Description Images copy/pasted into a message were not being saved previously. Verified with Corso binary ## Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, in the ChangeLog. ## Type of change - [x] :bug: Bugfix ## Issue(s) * closes #2163 ## Test Plan - [x] :zap: Unit test --- CHANGELOG.md | 2 +- src/internal/connector/exchange/api/events.go | 2 +- src/internal/connector/exchange/api/mail.go | 2 +- src/internal/connector/support/m365Support.go | 13 +++++ .../connector/support/m365Support_test.go | 54 +++++++++++++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b981d778a..07c061f12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Check if the user specified for an exchange backup operation has a mailbox. - Handle case where user's drive has not been initialized - +- Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163)) ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index 0aafaf3f2..f78aef76b 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -268,7 +268,7 @@ func (c Events) Serialize( defer writer.Close() - if *event.GetHasAttachments() { + if *event.GetHasAttachments() || support.HasAttachments(event.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index b3f67ceb8..59085ba96 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -257,7 +257,7 @@ func (c Mail) Serialize( defer writer.Close() - if *msg.GetHasAttachments() { + if *msg.GetHasAttachments() || support.HasAttachments(msg.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index d7e51e513..99cb95577 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -1,6 +1,8 @@ package support import ( + "strings" + absser "github.com/microsoft/kiota-abstractions-go/serialization" js "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -71,3 +73,14 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { return list, nil } + +func HasAttachments(body models.ItemBodyable) bool { + if body.GetContent() == nil || body.GetContentType() == nil || + *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { + return false + } + + content := *body.GetContent() + + return strings.Contains(content, "src=\"cid:") +} diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index c04c74604..dedde3536 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -3,6 +3,7 @@ package support import ( "testing" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -159,3 +160,56 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { }) } } + +func (suite *DataSupportSuite) TestHasAttachments() { + tests := []struct { + name string + hasAttachment assert.BoolAssertionFunc + getBodyable func(t *testing.T) models.ItemBodyable + }{ + { + name: "Mock w/out attachment", + hasAttachment: assert.False, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMockMessageWithBodyBytes( + "Test", + "This is testing", + "This is testing", + ) + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Mock w/ inline attachment", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy") + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Edge Case", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + //nolint:lll + content := "\r\n
Happy New Year,

In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture? 



Let me know if this meets our culture requirements.

Warm Regards,

Dustin
" + body := models.NewItemBody() + body.SetContent(&content) + cat := models.HTML_BODYTYPE + body.SetContentType(&cat) + return body + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + found := HasAttachments(test.getBodyable(t)) + test.hasAttachment(t, found) + }) + } +} From 298ee35f057d81861790a84b52bca20f310e61d5 Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 19 Jan 2023 20:16:39 -0800 Subject: [PATCH 6/9] Use mutex when appending to error (#2201) ## Description Make error appending thread-safe since multiple goroutines may attempt to add an error at the same time. ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * closes #2197 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/internal/connector/exchange/exchange_data_collection.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 950cf7aaf..734b2fff1 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -197,7 +197,11 @@ func (col *Collection) streamItems(ctx context.Context) { semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize) defer close(semaphoreCh) + updaterMu := sync.Mutex{} errUpdater := func(user string, err error) { + updaterMu.Lock() + defer updaterMu.Unlock() + errs = support.WrapAndAppend(user, err, errs) } From 38b9fabb51cccebeb4fd53cadef9243218bf29ad Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 19 Jan 2023 20:34:48 -0800 Subject: [PATCH 7/9] Check for and ignore item not found errors (#2202) ## Description Item not found errors may appear because an item was moved or deleted between enumerating the items in a folder and fetching the data for the items in the folder. This keeps them from being reported as there is nothing we can do if the item data is no longer available. This applies to all Exchange data categories ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup ## Issue(s) * closes #2198 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/exchange/exchange_data_collection.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 734b2fff1..0d4d53c04 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -262,7 +263,18 @@ func (col *Collection) streamItems(ctx context.Context) { } if err != nil { + // Don't report errors for deleted items as there's no way for us to + // back up data that is gone. Chalk them up as a "success" though since + // there's really nothing we can do and not reporting it will make the + // status code upset cause we won't have the same number of results as + // attempted items. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + return + } + errUpdater(user, err) + return } From 5d20e30062893b5c302bbade2c76ed1de5c9dd7b Mon Sep 17 00:00:00 2001 From: ashmrtn Date: Thu, 19 Jan 2023 21:55:11 -0800 Subject: [PATCH 8/9] Record graph error (#2195) ## Description Record information about the error Graph returned if something went wrong when fetching an item. ## Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No ## Type of change - [ ] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [x] :broom: Tech Debt/Cleanup ## Issue(s) * closes #2194 ## Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../exchange/exchange_data_collection.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 0d4d53c04..e168ed082 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -198,14 +198,6 @@ func (col *Collection) streamItems(ctx context.Context) { semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize) defer close(semaphoreCh) - updaterMu := sync.Mutex{} - errUpdater := func(user string, err error) { - updaterMu.Lock() - defer updaterMu.Unlock() - - errs = support.WrapAndAppend(user, err, errs) - } - // delete all removed items for id := range col.removed { semaphoreCh <- struct{}{} @@ -231,6 +223,14 @@ func (col *Collection) streamItems(ctx context.Context) { }(id) } + updaterMu := sync.Mutex{} + errUpdater := func(user string, err error) { + updaterMu.Lock() + defer updaterMu.Unlock() + + errs = support.WrapAndAppend(user, err, errs) + } + // add any new items for id := range col.added { if col.ctrl.FailFast && errs != nil { @@ -273,7 +273,7 @@ func (col *Collection) streamItems(ctx context.Context) { return } - errUpdater(user, err) + errUpdater(user, support.ConnectorStackErrorTraceWrap(err, "fetching item")) return } From 5923c4a5466605e9b9bfce63b71a0b819c712e85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Jan 2023 08:40:55 +0000 Subject: [PATCH 9/9] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Bump=20github.com/aws/?= =?UTF-8?q?aws-sdk-go=20from=201.44.182=20to=201.44.183=20in=20/src=20(#22?= =?UTF-8?q?03)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.182 to 1.44.183.
Release notes

Sourced from github.com/aws/aws-sdk-go's releases.

Release v1.44.183 (2023-01-19)

Service Client Updates

  • service/appflow: Updates service API and documentation
  • service/codeartifact: Updates service documentation
  • service/connect: Updates service API and documentation
  • service/connectparticipant: Updates service API and documentation
  • service/ec2: Updates service API and documentation
    • Adds SSM Parameter Resource Aliasing support to EC2 Launch Templates. Launch Templates can now store parameter aliases in place of AMI Resource IDs. CreateLaunchTemplateVersion and DescribeLaunchTemplateVersions now support a convenience flag, ResolveAlias, to return the resolved parameter value.
  • service/glue: Updates service API and documentation
    • Release Glue Studio Hudi Data Lake Format for SDK/CLI
  • service/groundstation: Updates service API, documentation, and waiters
  • service/logs: Updates service API and documentation
    • Bug fix - Removed the regex pattern validation from CoralModel to avoid potential security issue.
  • service/medialive: Updates service API and documentation
    • AWS Elemental MediaLive adds support for SCTE 35 preRollMilliSeconds.
  • service/opensearch: Updates service API and documentation
  • service/panorama: Updates service API and documentation
  • service/sagemaker: Updates service API and documentation
    • HyperParameterTuningJobs now allow passing environment variables into the corresponding TrainingJobs
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/aws/aws-sdk-go&package-manager=go_modules&previous-version=1.44.182&new-version=1.44.183)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- src/go.mod | 2 +- src/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go.mod b/src/go.mod index fa987b9b3..fe00918a9 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.182 + github.com/aws/aws-sdk-go v1.44.183 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/src/go.sum b/src/go.sum index f434af2eb..d207d5c07 100644 --- a/src/go.sum +++ b/src/go.sum @@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.182 h1:DUEhWpWl4yTPgt142qwUfH1rYeB6KUCHDcpL7lF4+9M= -github.com/aws/aws-sdk-go v1.44.182/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= +github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=