diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c78699ea..547f6f5ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Beta Libraries are included in package. This can lead to long build times. - +- msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times. +- Handle case where user's drive has not been initialized +- Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163)) ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/src/go.mod b/src/go.mod index d3e795e27..e1e7dac46 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.181 + github.com/aws/aws-sdk-go v1.44.183 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 diff --git a/src/go.sum b/src/go.sum index 6f460c019..2d120f6cd 100644 --- a/src/go.sum +++ b/src/go.sum @@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.181 h1:w4OzE8bwIVo62gUTAp/uEFO2HSsUtf1pjXpSs36cluY= -github.com/aws/aws-sdk-go v1.44.181/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= +github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index 8cde8e241..f9cd5c95f 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -58,6 +58,28 @@ func (c Events) DeleteCalendar( return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) } +func (c Events) GetContainerByID( + ctx context.Context, + userID, containerID string, +) (graph.Container, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + ofc, err := optionsForCalendarsByID([]string{"name", "owner"}) + if err != nil { + return nil, errors.Wrap(err, "options for event calendar") + } + + cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc) + if err != nil { + return nil, err + } + + return graph.CalendarDisplayable{Calendarable: cal}, nil +} + // GetItem retrieves an Eventable item. func (c Events) GetItem( ctx context.Context, @@ -183,14 +205,9 @@ func (c Events) GetAddedAndRemovedItemIDs( errs *multierror.Error ) - options, err := optionsForEventsByCalendarDelta([]string{"id"}) - if err != nil { - return nil, nil, DeltaUpdate{}, err - } - if len(oldDelta) > 0 { builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition @@ -217,7 +234,7 @@ func (c Events) GetAddedAndRemovedItemIDs( // works as intended (until, at least, we want to _not_ call the beta anymore). rawURL := fmt.Sprintf(eventBetaDeltaURLTemplate, user, calendarID) builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(rawURL, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) if err != nil { @@ -251,7 +268,7 @@ func (c Events) Serialize( defer writer.Close() - if *event.GetHasAttachments() { + if *event.GetHasAttachments() || support.HasAttachments(event.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index aac470ad8..a1c9f41b9 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -257,7 +257,7 @@ func (c Mail) Serialize( defer writer.Close() - if *msg.GetHasAttachments() { + if *msg.GetHasAttachments() || support.HasAttachments(msg.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error diff --git a/src/internal/connector/exchange/api/options.go b/src/internal/connector/exchange/api/options.go index 95f55b417..2d5cbfd2d 100644 --- a/src/internal/connector/exchange/api/options.go +++ b/src/internal/connector/exchange/api/options.go @@ -21,18 +21,6 @@ var ( "owner": {}, } - fieldsForEvents = map[string]struct{}{ - "calendar": {}, - "end": {}, - "id": {}, - "isOnlineMeeting": {}, - "isReminderOn": {}, - "responseStatus": {}, - "responseRequested": {}, - "showAs": {}, - "subject": {}, - } - fieldsForFolders = map[string]struct{}{ "childFolderCount": {}, "displayName": {}, @@ -112,6 +100,28 @@ func optionsForCalendars(moreOps []string) ( return options, nil } +// optionsForCalendarsByID places allowed options for exchange.Calendar object +// @param moreOps should reflect elements from fieldsForCalendars +// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler +func optionsForCalendarsByID(moreOps []string) ( + *users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration, + error, +) { + selecting, err := buildOptions(moreOps, fieldsForCalendars) + if err != nil { + return nil, err + } + // should be a CalendarsRequestBuilderGetRequestConfiguration + requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ + Select: selecting, + } + options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ + QueryParameters: requestParams, + } + + return options, nil +} + // optionsForContactFolders places allowed options for exchange.ContactFolder object // @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler func optionsForContactFolders(moreOps []string) ( @@ -213,26 +223,6 @@ func optionsForContactFoldersItemDelta( return options, nil } -// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar -func optionsForEventsByCalendarDelta( - moreOps []string, -) (*users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration, error) { - selecting, err := buildOptions(moreOps, fieldsForEvents) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - // optionsForContactChildFolders builds a contacts child folders request. func optionsForContactChildFolders( moreOps []string, diff --git a/src/internal/connector/exchange/event_calendar_cache.go b/src/internal/connector/exchange/event_calendar_cache.go index 2b4e1b22d..e497a272a 100644 --- a/src/internal/connector/exchange/event_calendar_cache.go +++ b/src/internal/connector/exchange/event_calendar_cache.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/path" ) @@ -14,9 +15,43 @@ var _ graph.ContainerResolver = &eventCalendarCache{} type eventCalendarCache struct { *containerResolver enumer containersEnumerator + getter containerGetter userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (ecc *eventCalendarCache) init( + ctx context.Context, +) error { + if ecc.containerResolver == nil { + ecc.containerResolver = newContainerResolver() + } + + return ecc.populateEventRoot(ctx) +} + +// populateEventRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ +// DefaultCalendar is the traditional "Calendar". +// Action ensures that cache will stop at appropriate level. +// @error iff the struct is not properly instantiated +func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error { + container := DefaultCalendar + + f, err := ecc.getter.GetContainerByID(ctx, ecc.userID, container) + if err != nil { + return errors.Wrap(err, "fetching calendar "+support.ConnectorStackErrorTrace(err)) + } + + temp := graph.NewCacheFolder(f, path.Builder{}.Append(container)) + if err := ecc.addFolder(temp); err != nil { + return errors.Wrap(err, "initializing calendar resolver") + } + + return nil +} + // Populate utility function for populating eventCalendarCache. // Executes 1 additional Graph Query // @param baseID: ignored. Present to conform to interface @@ -25,8 +60,8 @@ func (ecc *eventCalendarCache) Populate( baseID string, baseContainerPath ...string, ) error { - if ecc.containerResolver == nil { - ecc.containerResolver = newContainerResolver() + if err := ecc.init(ctx); err != nil { + return errors.Wrap(err, "initializing") } err := ecc.enumer.EnumerateContainers(ctx, ecc.userID, "", ecc.addFolder) @@ -34,6 +69,10 @@ func (ecc *eventCalendarCache) Populate( return errors.Wrap(err, "enumerating containers") } + if err := ecc.populatePaths(ctx); err != nil { + return errors.Wrap(err, "establishing calendar paths") + } + return nil } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 950cf7aaf..e168ed082 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -197,10 +198,6 @@ func (col *Collection) streamItems(ctx context.Context) { semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize) defer close(semaphoreCh) - errUpdater := func(user string, err error) { - errs = support.WrapAndAppend(user, err, errs) - } - // delete all removed items for id := range col.removed { semaphoreCh <- struct{}{} @@ -226,6 +223,14 @@ func (col *Collection) streamItems(ctx context.Context) { }(id) } + updaterMu := sync.Mutex{} + errUpdater := func(user string, err error) { + updaterMu.Lock() + defer updaterMu.Unlock() + + errs = support.WrapAndAppend(user, err, errs) + } + // add any new items for id := range col.added { if col.ctrl.FailFast && errs != nil { @@ -258,7 +263,18 @@ func (col *Collection) streamItems(ctx context.Context) { } if err != nil { - errUpdater(user, err) + // Don't report errors for deleted items as there's no way for us to + // back up data that is gone. Chalk them up as a "success" though since + // there's really nothing we can do and not reporting it will make the + // status code upset cause we won't have the same number of results as + // attempted items. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + return + } + + errUpdater(user, support.ConnectorStackErrorTraceWrap(err, "fetching item")) + return } diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 36bf0ffc5..ea7ccb995 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -51,6 +51,7 @@ func (suite *CacheResolverSuite) TestPopulate() { return &eventCalendarCache{ userID: tester.M365UserID(t), enumer: ac.Events(), + getter: ac.Events(), } } diff --git a/src/internal/connector/exchange/mail_folder_cache.go b/src/internal/connector/exchange/mail_folder_cache.go index 06d4b1285..565f10736 100644 --- a/src/internal/connector/exchange/mail_folder_cache.go +++ b/src/internal/connector/exchange/mail_folder_cache.go @@ -22,14 +22,25 @@ type mailFolderCache struct { userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (mc *mailFolderCache) init( + ctx context.Context, +) error { + if mc.containerResolver == nil { + mc.containerResolver = newContainerResolver() + } + + return mc.populateMailRoot(ctx) +} + // populateMailRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ // rootFolderAlias is the top-level directory for exchange.Mail. // DefaultMailFolder is the traditional "Inbox" for exchange.Mail // Action ensures that cache will stop at appropriate level. // @error iff the struct is not properly instantiated -func (mc *mailFolderCache) populateMailRoot( - ctx context.Context, -) error { +func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error { for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} { var directory string @@ -76,16 +87,3 @@ func (mc *mailFolderCache) Populate( return nil } - -// init ensures that the structure's fields are initialized. -// Fields Initialized when cache == nil: -// [mc.cache] -func (mc *mailFolderCache) init( - ctx context.Context, -) error { - if mc.containerResolver == nil { - mc.containerResolver = newContainerResolver() - } - - return mc.populateMailRoot(ctx) -} diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index d646fb132..9d996f01c 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -66,9 +66,11 @@ func PopulateExchangeContainerResolver( cacheRoot = DefaultContactFolder case path.EventsCategory: + ecc := ac.Events() res = &eventCalendarCache{ userID: qp.ResourceOwner, - enumer: ac.Events(), + getter: ecc, + enumer: ecc, } cacheRoot = DefaultCalendar diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 7978dd407..c6017f09d 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -507,9 +507,11 @@ func CreateContainerDestinaion( case path.EventsCategory: if directoryCache == nil { + ace := ac.Events() ecc := &eventCalendarCache{ userID: user, - enumer: ac.Events(), + getter: ace, + enumer: ace, } caches[category] = ecc newCache = true diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index 98e321523..07da01521 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -76,7 +76,8 @@ const ( itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children" itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s" itemNotFoundErrorCode = "itemNotFound" - userDoesNotHaveDrive = "BadRequest Unable to retrieve user's mysite URL" + userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL" + userMysiteNotFound = "ResourceNotFound User's mysite not found" ) // Enumerates the drives for the specified user @@ -134,7 +135,8 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod r, err = service.Client().UsersById(user).Drives().Get(ctx, nil) if err != nil { detailedError := support.ConnectorStackErrorTrace(err) - if strings.Contains(detailedError, userDoesNotHaveDrive) { + if strings.Contains(detailedError, userMysiteURLNotFound) || + strings.Contains(detailedError, userMysiteNotFound) { logger.Ctx(ctx).Debugf("User %s does not have a drive", user) return make([]models.Driveable, 0), nil // no license } diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index 98a9a1bb2..c75787045 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -1,6 +1,8 @@ package support import ( + "strings" + absser "github.com/microsoft/kiota-abstractions-go/serialization" js "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-beta-sdk-go/models" @@ -71,3 +73,14 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { return list, nil } + +func HasAttachments(body models.ItemBodyable) bool { + if body.GetContent() == nil || body.GetContentType() == nil || + *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { + return false + } + + content := *body.GetContent() + + return strings.Contains(content, "src=\"cid:") +} diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index c04c74604..dedde3536 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -3,6 +3,7 @@ package support import ( "testing" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -159,3 +160,56 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { }) } } + +func (suite *DataSupportSuite) TestHasAttachments() { + tests := []struct { + name string + hasAttachment assert.BoolAssertionFunc + getBodyable func(t *testing.T) models.ItemBodyable + }{ + { + name: "Mock w/out attachment", + hasAttachment: assert.False, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMockMessageWithBodyBytes( + "Test", + "This is testing", + "This is testing", + ) + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Mock w/ inline attachment", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy") + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Edge Case", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + //nolint:lll + content := "\r\n
Happy New Year,

In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture? 



Let me know if this meets our culture requirements.

Warm Regards,

Dustin
" + body := models.NewItemBody() + body.SetContent(&content) + cat := models.HTML_BODYTYPE + body.SetContentType(&cat) + return body + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + found := HasAttachments(test.getBodyable(t)) + test.hasAttachment(t, found) + }) + } +} diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 804b217f3..d79d95104 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -111,8 +111,7 @@ func readTestConfig() (map[string]string, error) { TestCfgUserID, os.Getenv(EnvCorsoM365TestUserID), vpr.GetString(TestCfgUserID), - "lynner@8qzvrj.onmicrosoft.com", - //"lidiah@8qzvrj.onmicrosoft.com", + "conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -120,7 +119,6 @@ func readTestConfig() (map[string]string, error) { os.Getenv(EnvCorsoSecondaryM365TestUserID), vpr.GetString(TestCfgSecondaryUserID), "lidiah@8qzvrj.onmicrosoft.com", - //"lynner@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -134,7 +132,7 @@ func readTestConfig() (map[string]string, error) { TestCfgLoadTestOrgUsers, os.Getenv(EnvCorsoM365LoadTestOrgUsers), vpr.GetString(TestCfgLoadTestOrgUsers), - "lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com", + "lidiah@8qzvrj.onmicrosoft.com,conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv,