diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 035c3ee4d..ac01fe55b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,12 @@ jobs: working-directory: src steps: - uses: actions/checkout@v3 + + ## increases swap space for beta and v.1.0 + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 12 # single setup and sum cache handling here. # the results will cascade onto both testing and linting. @@ -633,4 +639,4 @@ jobs: steps: - name: Validate docs run: | - curl https://corsobackup.io/docs/quickstart/ | grep https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Linux_x86_64.tar.gz \ No newline at end of file + curl https://corsobackup.io/docs/quickstart/ | grep https://github.com/alcionai/corso/releases/download/${{ env.CORSO_VERSION }}/corso_${{ env.CORSO_VERSION }}_Linux_x86_64.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c78699ea..547f6f5ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Beta Libraries are included in package. This can lead to long build times. - +- msgraph-beta-sdk-go replaces msgraph-sdk-go for new features. This can lead to long build times. +- Handle case where user's drive has not been initialized +- Inline attachments (e.g. copy/paste ) are discovered and backed up correctly ([#2163](https://github.com/alcionai/corso/issues/2163)) ## [v0.1.0] (alpha) - 2023-01-13 diff --git a/src/go.mod b/src/go.mod index 95ea65e06..e1e7dac46 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,7 +6,7 @@ replace github.com/kopia/kopia => github.com/alcionai/kopia v0.10.8-0.2023011220 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/aws/aws-sdk-go v1.44.181 + github.com/aws/aws-sdk-go v1.44.183 github.com/aws/aws-xray-sdk-go v1.8.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 @@ -16,7 +16,6 @@ require ( github.com/microsoft/kiota-http-go v0.13.0 github.com/microsoft/kiota-serialization-json-go v0.7.2 github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0 - github.com/microsoftgraph/msgraph-sdk-go v0.52.0 github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0 github.com/pkg/errors v0.9.1 github.com/rudderlabs/analytics-go v3.3.3+incompatible diff --git a/src/go.sum b/src/go.sum index 1f5efb02d..2d120f6cd 100644 --- a/src/go.sum +++ b/src/go.sum @@ -62,8 +62,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/aws/aws-sdk-go v1.44.181 h1:w4OzE8bwIVo62gUTAp/uEFO2HSsUtf1pjXpSs36cluY= -github.com/aws/aws-sdk-go v1.44.181/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.183 h1:mUk45JZTIMMg9m8GmrbvACCsIOKtKezXRxp06uI5Ahk= +github.com/aws/aws-sdk-go v1.44.183/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-xray-sdk-go v1.8.0 h1:0xncHZ588wB/geLjbM/esoW3FOEThWy2TJyb4VXfLFY= github.com/aws/aws-xray-sdk-go v1.8.0/go.mod h1:7LKe47H+j3evfvS1+q0wzpoaGXGrF3mUsfM+thqVO+A= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -281,8 +281,6 @@ github.com/microsoft/kiota-serialization-text-go v0.6.0 h1:3N2vftYZlwKdog69AN7ha github.com/microsoft/kiota-serialization-text-go v0.6.0/go.mod h1:OUA4dNH+f6afiJUs+rQAatJos7QVF5PJkyrqoD89lx4= github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0 h1:FITWxC0U8u9J82iUnTUgKhFw048OFq6hNiShpIgJx2k= github.com/microsoftgraph/msgraph-beta-sdk-go v0.53.0/go.mod h1:hyea6/iVBicq7jjlU8nzmkFys2ajScSI2q1GX9gjnIQ= -github.com/microsoftgraph/msgraph-sdk-go v0.52.0 h1:hcFSvgxsN6xG9xGC/Y8jFasU+99OleAVZJb8rKBuISg= -github.com/microsoftgraph/msgraph-sdk-go v0.52.0/go.mod h1:RF7cVKAu0EZUmNxXZZ6orJJml02Cf85d1FFOxJ+nbRg= github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0 h1:cDL3ov/IZ2ZarUJdGGPsdR+46ALdd3CRAiDBIylLCoA= github.com/microsoftgraph/msgraph-sdk-go-core v0.33.0/go.mod h1:d0mU3PQAWnN/C4CwPJEZz2QhesrnR5UDnqRu2ODWPkI= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= diff --git a/src/internal/connector/discovery/discovery.go b/src/internal/connector/discovery/discovery.go index a9f2266d1..e9d0474db 100644 --- a/src/internal/connector/discovery/discovery.go +++ b/src/internal/connector/discovery/discovery.go @@ -3,9 +3,9 @@ package discovery import ( "context" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + msuser "github.com/microsoftgraph/msgraph-beta-sdk-go/users" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" - msuser "github.com/microsoftgraph/msgraph-sdk-go/users" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/discovery/discovery_test.go b/src/internal/connector/discovery/discovery_test.go index e9349ba55..15e0f3210 100644 --- a/src/internal/connector/discovery/discovery_test.go +++ b/src/internal/connector/discovery/discovery_test.go @@ -4,7 +4,7 @@ import ( "reflect" "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/suite" ) diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go index e12f4b795..27c21338f 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/internal/connector/exchange/api/contacts.go @@ -8,8 +8,8 @@ import ( "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/users" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/exchange/api/contacts_test.go b/src/internal/connector/exchange/api/contacts_test.go index 411250146..b49f9d920 100644 --- a/src/internal/connector/exchange/api/contacts_test.go +++ b/src/internal/connector/exchange/api/contacts_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index b4988d28d..f9cd5c95f 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -8,8 +8,8 @@ import ( "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/users" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common" @@ -58,6 +58,28 @@ func (c Events) DeleteCalendar( return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) } +func (c Events) GetContainerByID( + ctx context.Context, + userID, containerID string, +) (graph.Container, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + ofc, err := optionsForCalendarsByID([]string{"name", "owner"}) + if err != nil { + return nil, errors.Wrap(err, "options for event calendar") + } + + cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc) + if err != nil { + return nil, err + } + + return graph.CalendarDisplayable{Calendarable: cal}, nil +} + // GetItem retrieves an Eventable item. func (c Events) GetItem( ctx context.Context, @@ -183,14 +205,9 @@ func (c Events) GetAddedAndRemovedItemIDs( errs *multierror.Error ) - options, err := optionsForEventsByCalendarDelta([]string{"id"}) - if err != nil { - return nil, nil, DeltaUpdate{}, err - } - if len(oldDelta) > 0 { builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) // note: happy path, not the error condition @@ -217,7 +234,7 @@ func (c Events) GetAddedAndRemovedItemIDs( // works as intended (until, at least, we want to _not_ call the beta anymore). rawURL := fmt.Sprintf(eventBetaDeltaURLTemplate, user, calendarID) builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(rawURL, service.Adapter()) - pgr := &eventPager{service, builder, options} + pgr := &eventPager{service, builder, nil} added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr) if err != nil { @@ -251,7 +268,7 @@ func (c Events) Serialize( defer writer.Close() - if *event.GetHasAttachments() { + if *event.GetHasAttachments() || support.HasAttachments(event.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error diff --git a/src/internal/connector/exchange/api/events_test.go b/src/internal/connector/exchange/api/events_test.go index a41a48e5a..2cee75525 100644 --- a/src/internal/connector/exchange/api/events_test.go +++ b/src/internal/connector/exchange/api/events_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index b3f67ceb8..a1c9f41b9 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -8,8 +8,8 @@ import ( "github.com/hashicorp/go-multierror" "github.com/microsoft/kiota-abstractions-go/serialization" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/users" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -257,7 +257,7 @@ func (c Mail) Serialize( defer writer.Close() - if *msg.GetHasAttachments() { + if *msg.GetHasAttachments() || support.HasAttachments(msg.GetBody()) { // getting all the attachments might take a couple attempts due to filesize var retriesErr error @@ -278,7 +278,8 @@ func (c Mail) Serialize( if retriesErr != nil { logger.Ctx(ctx).Debug("exceeded maximum retries") - return nil, support.WrapAndAppend(itemID, errors.Wrap(retriesErr, "attachment failed"), nil) + return nil, support.WrapAndAppend(itemID, + support.ConnectorStackErrorTraceWrap(retriesErr, "attachment Failed"), nil) } } diff --git a/src/internal/connector/exchange/api/mail_test.go b/src/internal/connector/exchange/api/mail_test.go index 5611586e2..a3b5158ef 100644 --- a/src/internal/connector/exchange/api/mail_test.go +++ b/src/internal/connector/exchange/api/mail_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/pkg/backup/details" diff --git a/src/internal/connector/exchange/api/options.go b/src/internal/connector/exchange/api/options.go index 99d41487c..2d5cbfd2d 100644 --- a/src/internal/connector/exchange/api/options.go +++ b/src/internal/connector/exchange/api/options.go @@ -3,7 +3,7 @@ package api import ( "fmt" - "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/users" ) // ----------------------------------------------------------------------- @@ -21,18 +21,6 @@ var ( "owner": {}, } - fieldsForEvents = map[string]struct{}{ - "calendar": {}, - "end": {}, - "id": {}, - "isOnlineMeeting": {}, - "isReminderOn": {}, - "responseStatus": {}, - "responseRequested": {}, - "showAs": {}, - "subject": {}, - } - fieldsForFolders = map[string]struct{}{ "childFolderCount": {}, "displayName": {}, @@ -112,6 +100,28 @@ func optionsForCalendars(moreOps []string) ( return options, nil } +// optionsForCalendarsByID places allowed options for exchange.Calendar object +// @param moreOps should reflect elements from fieldsForCalendars +// @return is first call in Calendars().GetWithRequestConfigurationAndResponseHandler +func optionsForCalendarsByID(moreOps []string) ( + *users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration, + error, +) { + selecting, err := buildOptions(moreOps, fieldsForCalendars) + if err != nil { + return nil, err + } + // should be a CalendarsRequestBuilderGetRequestConfiguration + requestParams := &users.ItemCalendarsCalendarItemRequestBuilderGetQueryParameters{ + Select: selecting, + } + options := &users.ItemCalendarsCalendarItemRequestBuilderGetRequestConfiguration{ + QueryParameters: requestParams, + } + + return options, nil +} + // optionsForContactFolders places allowed options for exchange.ContactFolder object // @return is first call in ContactFolders().GetWithRequestConfigurationAndResponseHandler func optionsForContactFolders(moreOps []string) ( @@ -213,26 +223,6 @@ func optionsForContactFoldersItemDelta( return options, nil } -// optionsForEvents ensures a valid option inputs for `exchange.Events` when selected from within a Calendar -func optionsForEventsByCalendarDelta( - moreOps []string, -) (*users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration, error) { - selecting, err := buildOptions(moreOps, fieldsForEvents) - if err != nil { - return nil, err - } - - requestParameters := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetQueryParameters{ - Select: selecting, - } - - options := &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{ - QueryParameters: requestParameters, - } - - return options, nil -} - // optionsForContactChildFolders builds a contacts child folders request. func optionsForContactChildFolders( moreOps []string, diff --git a/src/internal/connector/exchange/attachment.go b/src/internal/connector/exchange/attachment.go index 5cbce271c..757e76ff8 100644 --- a/src/internal/connector/exchange/attachment.go +++ b/src/internal/connector/exchange/attachment.go @@ -5,7 +5,7 @@ import ( "context" "io" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/uploadsession" diff --git a/src/internal/connector/exchange/attachment_uploadable.go b/src/internal/connector/exchange/attachment_uploadable.go index 5369dcfa6..89835aba8 100644 --- a/src/internal/connector/exchange/attachment_uploadable.go +++ b/src/internal/connector/exchange/attachment_uploadable.go @@ -3,8 +3,8 @@ package exchange import ( "context" - "github.com/microsoftgraph/msgraph-sdk-go/models" - msusers "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + msusers "github.com/microsoftgraph/msgraph-beta-sdk-go/users" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/exchange/event_calendar_cache.go b/src/internal/connector/exchange/event_calendar_cache.go index 2b4e1b22d..e497a272a 100644 --- a/src/internal/connector/exchange/event_calendar_cache.go +++ b/src/internal/connector/exchange/event_calendar_cache.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/path" ) @@ -14,9 +15,43 @@ var _ graph.ContainerResolver = &eventCalendarCache{} type eventCalendarCache struct { *containerResolver enumer containersEnumerator + getter containerGetter userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (ecc *eventCalendarCache) init( + ctx context.Context, +) error { + if ecc.containerResolver == nil { + ecc.containerResolver = newContainerResolver() + } + + return ecc.populateEventRoot(ctx) +} + +// populateEventRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ +// DefaultCalendar is the traditional "Calendar". +// Action ensures that cache will stop at appropriate level. +// @error iff the struct is not properly instantiated +func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error { + container := DefaultCalendar + + f, err := ecc.getter.GetContainerByID(ctx, ecc.userID, container) + if err != nil { + return errors.Wrap(err, "fetching calendar "+support.ConnectorStackErrorTrace(err)) + } + + temp := graph.NewCacheFolder(f, path.Builder{}.Append(container)) + if err := ecc.addFolder(temp); err != nil { + return errors.Wrap(err, "initializing calendar resolver") + } + + return nil +} + // Populate utility function for populating eventCalendarCache. // Executes 1 additional Graph Query // @param baseID: ignored. Present to conform to interface @@ -25,8 +60,8 @@ func (ecc *eventCalendarCache) Populate( baseID string, baseContainerPath ...string, ) error { - if ecc.containerResolver == nil { - ecc.containerResolver = newContainerResolver() + if err := ecc.init(ctx); err != nil { + return errors.Wrap(err, "initializing") } err := ecc.enumer.EnumerateContainers(ctx, ecc.userID, "", ecc.addFolder) @@ -34,6 +69,10 @@ func (ecc *eventCalendarCache) Populate( return errors.Wrap(err, "enumerating containers") } + if err := ecc.populatePaths(ctx); err != nil { + return errors.Wrap(err, "establishing calendar paths") + } + return nil } diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 950cf7aaf..e168ed082 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" @@ -197,10 +198,6 @@ func (col *Collection) streamItems(ctx context.Context) { semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize) defer close(semaphoreCh) - errUpdater := func(user string, err error) { - errs = support.WrapAndAppend(user, err, errs) - } - // delete all removed items for id := range col.removed { semaphoreCh <- struct{}{} @@ -226,6 +223,14 @@ func (col *Collection) streamItems(ctx context.Context) { }(id) } + updaterMu := sync.Mutex{} + errUpdater := func(user string, err error) { + updaterMu.Lock() + defer updaterMu.Unlock() + + errs = support.WrapAndAppend(user, err, errs) + } + // add any new items for id := range col.added { if col.ctrl.FailFast && errs != nil { @@ -258,7 +263,18 @@ func (col *Collection) streamItems(ctx context.Context) { } if err != nil { - errUpdater(user, err) + // Don't report errors for deleted items as there's no way for us to + // back up data that is gone. Chalk them up as a "success" though since + // there's really nothing we can do and not reporting it will make the + // status code upset cause we won't have the same number of results as + // attempted items. + if e := graph.IsErrDeletedInFlight(err); e != nil { + atomic.AddInt64(&success, 1) + return + } + + errUpdater(user, support.ConnectorStackErrorTraceWrap(err, "fetching item")) + return } diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 36bf0ffc5..ea7ccb995 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -51,6 +51,7 @@ func (suite *CacheResolverSuite) TestPopulate() { return &eventCalendarCache{ userID: tester.M365UserID(t), enumer: ac.Events(), + getter: ac.Events(), } } diff --git a/src/internal/connector/exchange/mail_folder_cache.go b/src/internal/connector/exchange/mail_folder_cache.go index 06d4b1285..565f10736 100644 --- a/src/internal/connector/exchange/mail_folder_cache.go +++ b/src/internal/connector/exchange/mail_folder_cache.go @@ -22,14 +22,25 @@ type mailFolderCache struct { userID string } +// init ensures that the structure's fields are initialized. +// Fields Initialized when cache == nil: +// [mc.cache] +func (mc *mailFolderCache) init( + ctx context.Context, +) error { + if mc.containerResolver == nil { + mc.containerResolver = newContainerResolver() + } + + return mc.populateMailRoot(ctx) +} + // populateMailRoot manually fetches directories that are not returned during Graph for msgraph-sdk-go v. 40+ // rootFolderAlias is the top-level directory for exchange.Mail. // DefaultMailFolder is the traditional "Inbox" for exchange.Mail // Action ensures that cache will stop at appropriate level. // @error iff the struct is not properly instantiated -func (mc *mailFolderCache) populateMailRoot( - ctx context.Context, -) error { +func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error { for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} { var directory string @@ -76,16 +87,3 @@ func (mc *mailFolderCache) Populate( return nil } - -// init ensures that the structure's fields are initialized. -// Fields Initialized when cache == nil: -// [mc.cache] -func (mc *mailFolderCache) init( - ctx context.Context, -) error { - if mc.containerResolver == nil { - mc.containerResolver = newContainerResolver() - } - - return mc.populateMailRoot(ctx) -} diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index d646fb132..9d996f01c 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -66,9 +66,11 @@ func PopulateExchangeContainerResolver( cacheRoot = DefaultContactFolder case path.EventsCategory: + ecc := ac.Events() res = &eventCalendarCache{ userID: qp.ResourceOwner, - enumer: ac.Events(), + getter: ecc, + enumer: ecc, } cacheRoot = DefaultCalendar diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 3f88f6efe..c6017f09d 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -7,7 +7,7 @@ import ( "reflect" "runtime/trace" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common" @@ -507,9 +507,11 @@ func CreateContainerDestinaion( case path.EventsCategory: if directoryCache == nil { + ace := ac.Events() ecc := &eventCalendarCache{ userID: user, - enumer: ac.Events(), + getter: ace, + enumer: ace, } caches[category] = ecc newCache = true diff --git a/src/internal/connector/graph/cache_container.go b/src/internal/connector/graph/cache_container.go index e792c235e..989698235 100644 --- a/src/internal/connector/graph/cache_container.go +++ b/src/internal/connector/graph/cache_container.go @@ -1,7 +1,7 @@ package graph import ( - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/pkg/path" diff --git a/src/internal/connector/graph/errors.go b/src/internal/connector/graph/errors.go index 86cec64bd..85ae0c874 100644 --- a/src/internal/connector/graph/errors.go +++ b/src/internal/connector/graph/errors.go @@ -5,7 +5,7 @@ import ( "net/url" "os" - "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models/odataerrors" "github.com/pkg/errors" "golang.org/x/exp/slices" diff --git a/src/internal/connector/graph/service.go b/src/internal/connector/graph/service.go index 02ac7eb1e..58a86317a 100644 --- a/src/internal/connector/graph/service.go +++ b/src/internal/connector/graph/service.go @@ -4,8 +4,7 @@ import ( "context" absser "github.com/microsoft/kiota-abstractions-go/serialization" - beta "github.com/microsoftgraph/msgraph-beta-sdk-go" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" "github.com/pkg/errors" "github.com/alcionai/corso/src/pkg/account" @@ -46,28 +45,6 @@ func (s Service) Client() *msgraphsdk.GraphServiceClient { return s.client } -// BetaService provides access to API Calls that are present solely in the Beta library. -// Does not fulfill the interface as the base libraries vary. -type BetaService struct { - adapter *beta.GraphRequestAdapter - client *beta.GraphBaseServiceClient -} - -func (bs BetaService) Adapter() *beta.GraphRequestAdapter { - return bs.adapter -} - -func (bs BetaService) Client() *beta.GraphBaseServiceClient { - return bs.client -} - -func NewBetaService(adapter *beta.GraphRequestAdapter) *BetaService { - return &BetaService{ - adapter: adapter, - client: beta.NewGraphBaseServiceClient(adapter), - } -} - // Seraialize writes an M365 parsable object into a byte array using the built-in // application/json writer within the adapter. func (s Service) Serialize(object absser.Parsable) ([]byte, error) { diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go index f6acdb9d0..b55821b78 100644 --- a/src/internal/connector/graph/service_helper.go +++ b/src/internal/connector/graph/service_helper.go @@ -10,8 +10,7 @@ import ( az "github.com/Azure/azure-sdk-for-go/sdk/azidentity" ka "github.com/microsoft/kiota-authentication-azure-go" khttp "github.com/microsoft/kiota-http-go" - beta "github.com/microsoftgraph/msgraph-beta-sdk-go" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/pkg/errors" @@ -47,25 +46,6 @@ func CreateAdapter(tenant, client, secret string) (*msgraphsdk.GraphRequestAdapt auth, nil, nil, httpClient) } -// CreateBetaAdapter uses the provided credentials with the Kiota Azure Libraries. -// Used to access msgraph-beta-sdk-go specific functionality -func CreateBetaAdapter(tenant, client, secret string) (*beta.GraphRequestAdapter, error) { - cred, err := az.NewClientSecretCredential(tenant, client, secret, nil) - if err != nil { - return nil, errors.Wrap(err, "creating beta m365 client credentials") - } - - auth, err := ka.NewAzureIdentityAuthenticationProviderWithScopes( - cred, - []string{"https://graph.microsoft.com/.default"}, - ) - - httpClient := CreateHTTPClient() - - return beta.NewGraphRequestAdapterWithParseNodeFactoryAndSerializationWriterFactoryAndHttpClient( - auth, nil, nil, httpClient) -} - // CreateHTTPClient creates the httpClient with middlewares and timeout configured func CreateHTTPClient() *http.Client { clientOptions := msgraphsdk.GetDefaultClientOptions() diff --git a/src/internal/connector/graph/service_test.go b/src/internal/connector/graph/service_test.go index c6938a7e3..32e900433 100644 --- a/src/internal/connector/graph/service_test.go +++ b/src/internal/connector/graph/service_test.go @@ -3,7 +3,7 @@ package graph_test import ( "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -43,21 +43,6 @@ func (suite *GraphUnitSuite) TestCreateAdapter() { assert.NotNil(t, adpt) } -func (suite *GraphUnitSuite) TestBetaService() { - t := suite.T() - adpt, err := graph.CreateBetaAdapter( - suite.credentials.AzureTenantID, - suite.credentials.AzureClientID, - suite.credentials.AzureClientSecret, - ) - - assert.NoError(t, err) - require.NotNil(t, adpt) - - serv := graph.NewBetaService(adpt) - assert.NotNil(t, serv) -} - func (suite *GraphUnitSuite) TestSerializationEndPoint() { t := suite.T() adpt, err := graph.CreateAdapter( diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index e047a017b..fd0a8c4e2 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -9,8 +9,8 @@ import ( "sync" "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/pkg/errors" "golang.org/x/exp/maps" diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index c614df05d..953855957 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -6,7 +6,7 @@ import ( "reflect" "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -335,12 +335,12 @@ func checkContact( testEmptyOrEqual(t, expected.GetAssistantName(), got.GetAssistantName(), "AssistantName") testEmptyOrEqual(t, expected.GetBirthday(), got.GetBirthday(), "Birthday") - - assert.Equal(t, expected.GetBusinessAddress(), got.GetBusinessAddress()) - - testEmptyOrEqual(t, expected.GetBusinessHomePage(), got.GetBusinessHomePage(), "BusinessHomePage") - - assert.Equal(t, expected.GetBusinessPhones(), got.GetBusinessPhones()) + // Not present in msgraph-beta-sdk/models + // assert.Equal(t, expected.GetBusinessAddress(), got.GetBusinessAddress()) + // Not present in msgraph-beta-sdk/models + // testEmptyOrEqual(t, expected.GetBusinessHomePage(), got.GetBusinessHomePage(), "BusinessHomePage") + // Not present in msgraph-beta-sdk/models + // assert.Equal(t, expected.GetBusinessPhones(), got.GetBusinessPhones()) assert.Equal(t, expected.GetCategories(), got.GetCategories()) @@ -364,9 +364,10 @@ func checkContact( testEmptyOrEqual(t, expected.GetGivenName(), got.GetGivenName(), "GivenName") - assert.Equal(t, expected.GetHomeAddress(), got.GetHomeAddress()) - - assert.Equal(t, expected.GetHomePhones(), got.GetHomePhones()) + // Not present in msgraph-beta-sdk/models + // assert.Equal(t, expected.GetHomeAddress(), got.GetHomeAddress()) + // Not present in msgraph-beta-sdk/models + // assert.Equal(t, expected.GetHomePhones(), got.GetHomePhones()) // Skip CreatedDateTime as it's tied to this specific instance of the item. @@ -382,13 +383,14 @@ func checkContact( testEmptyOrEqual(t, expected.GetMiddleName(), got.GetMiddleName(), "MiddleName") - testEmptyOrEqual(t, expected.GetMobilePhone(), got.GetMobilePhone(), "MobilePhone") + // Not present in msgraph-beta-sdk/models + // testEmptyOrEqual(t, expected.GetMobilePhone(), got.GetMobilePhone(), "MobilePhone") testEmptyOrEqual(t, expected.GetNickName(), got.GetNickName(), "NickName") testEmptyOrEqual(t, expected.GetOfficeLocation(), got.GetOfficeLocation(), "OfficeLocation") - - assert.Equal(t, expected.GetOtherAddress(), got.GetOtherAddress()) + // Not present in msgraph-beta-sdk/models + // assert.Equal(t, expected.GetOtherAddress(), got.GetOtherAddress()) // Skip ParentFolderId as it's tied to this specific instance of the item. diff --git a/src/internal/connector/mockconnector/mock_data_collection_test.go b/src/internal/connector/mockconnector/mock_data_collection_test.go index 2af9236c4..224969750 100644 --- a/src/internal/connector/mockconnector/mock_data_collection_test.go +++ b/src/internal/connector/mockconnector/mock_data_collection_test.go @@ -6,7 +6,7 @@ import ( "testing" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/mockconnector/mock_data_list.go b/src/internal/connector/mockconnector/mock_data_list.go index 2994dd275..fb60ca87c 100644 --- a/src/internal/connector/mockconnector/mock_data_list.go +++ b/src/internal/connector/mockconnector/mock_data_list.go @@ -6,7 +6,7 @@ import ( "testing" kw "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/data" diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index 4ea9ea9eb..438ffc835 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/spatialcurrent/go-lazy/pkg/lazy" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index a19021ff7..4e8876a92 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 6a59104f1..0cd261b24 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 5a0775edc..b45d6f185 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -4,7 +4,7 @@ import ( "strings" "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index e98e8d35b..07da01521 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -6,12 +6,12 @@ import ( "strings" "time" + msdrive "github.com/microsoftgraph/msgraph-beta-sdk-go/drive" + msdrives "github.com/microsoftgraph/msgraph-beta-sdk-go/drives" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models/odataerrors" + "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" - msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive" - msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives" - "github.com/microsoftgraph/msgraph-sdk-go/models" - "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" - "github.com/microsoftgraph/msgraph-sdk-go/sites" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" @@ -76,7 +76,8 @@ const ( itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children" itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s" itemNotFoundErrorCode = "itemNotFound" - userDoesNotHaveDrive = "BadRequest Unable to retrieve user's mysite URL" + userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL" + userMysiteNotFound = "ResourceNotFound User's mysite not found" ) // Enumerates the drives for the specified user @@ -134,7 +135,8 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod r, err = service.Client().UsersById(user).Drives().Get(ctx, nil) if err != nil { detailedError := support.ConnectorStackErrorTrace(err) - if strings.Contains(detailedError, userDoesNotHaveDrive) { + if strings.Contains(detailedError, userMysiteURLNotFound) || + strings.Contains(detailedError, userMysiteNotFound) { logger.Ctx(ctx).Debugf("User %s does not have a drive", user) return make([]models.Driveable, 0), nil // no license } diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 73391033b..f936de4a1 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -6,8 +6,8 @@ import ( "io" "strings" - msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives" - "github.com/microsoftgraph/msgraph-sdk-go/models" + msdrives "github.com/microsoftgraph/msgraph-beta-sdk-go/drives" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 5c2e8c335..7c94b0ea7 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -6,8 +6,8 @@ import ( "io" "testing" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/onedrive/service_test.go b/src/internal/connector/onedrive/service_test.go index 00a791f1e..dd9a9a6ff 100644 --- a/src/internal/connector/onedrive/service_test.go +++ b/src/internal/connector/onedrive/service_test.go @@ -3,7 +3,7 @@ package onedrive import ( "testing" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/sharepoint/collection.go b/src/internal/connector/sharepoint/collection.go index 4dc39127a..e875e5359 100644 --- a/src/internal/connector/sharepoint/collection.go +++ b/src/internal/connector/sharepoint/collection.go @@ -7,7 +7,7 @@ import ( "time" kw "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" @@ -46,7 +46,6 @@ type Collection struct { jobs []string // M365 IDs of the items of this collection service graph.Servicer - bService graph.BetaService statusUpdater support.StatusUpdater } diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index f049ab26f..3b91b1241 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -6,7 +6,7 @@ import ( "testing" kioser "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/sites" + "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index 71dd24f4e..3e4dfb884 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -168,16 +168,9 @@ func collectPages( ) ([]data.Collection, error) { logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Pages collections") - adpt, err := graph.CreateBetaAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret) - if err != nil { - return nil, support.ConnectorStackErrorTraceWrap(err, "fetching beta adapter") - } - - service := graph.NewBetaService(adpt) - spcs := make([]data.Collection, 0) - tuples, err := fetchPages(ctx, *service, siteID) + tuples, err := fetchPages(ctx, serv, siteID) if err != nil { return nil, err } diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 2934b2fa1..2227eefda 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -3,7 +3,7 @@ package sharepoint import ( "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/sharepoint/helper_test.go b/src/internal/connector/sharepoint/helper_test.go index e716a5bae..f798f8c72 100644 --- a/src/internal/connector/sharepoint/helper_test.go +++ b/src/internal/connector/sharepoint/helper_test.go @@ -3,7 +3,7 @@ package sharepoint import ( "testing" - msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" + msgraphsdk "github.com/microsoftgraph/msgraph-beta-sdk-go" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/src/internal/connector/sharepoint/list.go b/src/internal/connector/sharepoint/list.go index 101de9722..7f8e3523a 100644 --- a/src/internal/connector/sharepoint/list.go +++ b/src/internal/connector/sharepoint/list.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "github.com/microsoftgraph/msgraph-sdk-go/models" - mssite "github.com/microsoftgraph/msgraph-sdk-go/sites" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + mssite "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/sharepoint/listInfo.go b/src/internal/connector/sharepoint/listInfo.go index 3472c915f..bb8e18d62 100644 --- a/src/internal/connector/sharepoint/listInfo.go +++ b/src/internal/connector/sharepoint/listInfo.go @@ -3,7 +3,7 @@ package sharepoint import ( "time" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/alcionai/corso/src/pkg/backup/details" ) diff --git a/src/internal/connector/sharepoint/listInfo_test.go b/src/internal/connector/sharepoint/listInfo_test.go index f56b1a396..0cee792ee 100644 --- a/src/internal/connector/sharepoint/listInfo_test.go +++ b/src/internal/connector/sharepoint/listInfo_test.go @@ -3,7 +3,7 @@ package sharepoint import ( "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" diff --git a/src/internal/connector/sharepoint/pageInfo.go b/src/internal/connector/sharepoint/pageInfo.go index 589df4e06..103beec83 100644 --- a/src/internal/connector/sharepoint/pageInfo.go +++ b/src/internal/connector/sharepoint/pageInfo.go @@ -3,7 +3,7 @@ package sharepoint import ( "time" - bmodel "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/alcionai/corso/src/pkg/backup/details" ) @@ -11,7 +11,7 @@ import ( // sharePointPageInfo propagates metadata from the SharePoint Page data type // into searchable content. // Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta -func sharePointPageInfo(page bmodel.SitePageable, size int64) *details.SharePointInfo { +func sharePointPageInfo(page models.SitePageable, size int64) *details.SharePointInfo { var ( name, webURL string created, modified time.Time diff --git a/src/internal/connector/sharepoint/pageInfo_test.go b/src/internal/connector/sharepoint/pageInfo_test.go index c1bbbb952..16a32c432 100644 --- a/src/internal/connector/sharepoint/pageInfo_test.go +++ b/src/internal/connector/sharepoint/pageInfo_test.go @@ -3,7 +3,7 @@ package sharepoint import ( "testing" - bmodel "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/alcionai/corso/src/pkg/backup/details" @@ -12,20 +12,20 @@ import ( func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() { tests := []struct { name string - pageAndDeets func() (bmodel.SitePageable, *details.SharePointInfo) + pageAndDeets func() (models.SitePageable, *details.SharePointInfo) }{ { name: "Empty Page", - pageAndDeets: func() (bmodel.SitePageable, *details.SharePointInfo) { + pageAndDeets: func() (models.SitePageable, *details.SharePointInfo) { deets := &details.SharePointInfo{ItemType: details.SharePointItem} - return bmodel.NewSitePage(), deets + return models.NewSitePage(), deets }, }, { name: "Only Name", - pageAndDeets: func() (bmodel.SitePageable, *details.SharePointInfo) { + pageAndDeets: func() (models.SitePageable, *details.SharePointInfo) { title := "Blank Page" - sPage := bmodel.NewSitePage() + sPage := models.NewSitePage() sPage.SetTitle(&title) deets := &details.SharePointInfo{ ItemType: details.SharePointItem, diff --git a/src/internal/connector/sharepoint/queries.go b/src/internal/connector/sharepoint/queries.go index 806cf0e97..5e5807974 100644 --- a/src/internal/connector/sharepoint/queries.go +++ b/src/internal/connector/sharepoint/queries.go @@ -4,7 +4,7 @@ import ( "context" absser "github.com/microsoft/kiota-abstractions-go/serialization" - mssite "github.com/microsoftgraph/msgraph-sdk-go/sites" + mssite "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/alcionai/corso/src/internal/connector/graph" ) diff --git a/src/internal/connector/sharepoint/restore.go b/src/internal/connector/sharepoint/restore.go index ef2b940bb..c0c16fd0f 100644 --- a/src/internal/connector/sharepoint/restore.go +++ b/src/internal/connector/sharepoint/restore.go @@ -6,7 +6,7 @@ import ( "io" "runtime/trace" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" diff --git a/src/internal/connector/sharepoint/site_page.go b/src/internal/connector/sharepoint/site_page.go index 1b9671d87..237090b7c 100644 --- a/src/internal/connector/sharepoint/site_page.go +++ b/src/internal/connector/sharepoint/site_page.go @@ -3,32 +3,25 @@ package sharepoint import ( "context" - bmodel "github.com/microsoftgraph/msgraph-beta-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/microsoftgraph/msgraph-beta-sdk-go/sites" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" - "github.com/alcionai/corso/src/pkg/account" ) // GetSitePages retrieves a collection of Pages related to the give Site. // Returns error if error experienced during the call func GetSitePage( ctx context.Context, - creds account.M365Config, + serv graph.Servicer, siteID string, pages []string, -) ([]bmodel.SitePageable, error) { - adpt, err := graph.CreateBetaAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret) - if err != nil { - return nil, support.ConnectorStackErrorTraceWrap(err, "fetching beta adapter") - } - - service := graph.NewBetaService(adpt) - col := make([]bmodel.SitePageable, 0) +) ([]models.SitePageable, error) { + col := make([]models.SitePageable, 0) for _, entry := range pages { - page, err := service.Client().SitesById(siteID).PagesById(entry).Get(ctx, nil) + page, err := serv.Client().SitesById(siteID).PagesById(entry).Get(ctx, nil) if err != nil { return nil, support.ConnectorStackErrorTraceWrap(err, "fetching page: "+entry) } @@ -40,7 +33,7 @@ func GetSitePage( } // fetchPages utility function to return the tuple of item -func fetchPages(ctx context.Context, bs graph.BetaService, siteID string) ([]listTuple, error) { +func fetchPages(ctx context.Context, bs graph.Servicer, siteID string) ([]listTuple, error) { var ( builder = bs.Client().SitesById(siteID).Pages() opts = fetchPageOptions() diff --git a/src/internal/connector/sharepoint/site_pageable.go b/src/internal/connector/sharepoint/site_pageable.go deleted file mode 100644 index d0626321b..000000000 --- a/src/internal/connector/sharepoint/site_pageable.go +++ /dev/null @@ -1,24 +0,0 @@ -package sharepoint - -import ( - "github.com/microsoft/kiota-abstractions-go/serialization" - "github.com/microsoftgraph/msgraph-sdk-go/models" -) - -// SitePageable adjusted from msgraph-beta-sdk-go for temporary testing -type SitePageable interface { - models.BaseItemable - serialization.Parsable - GetContentType() models.ContentTypeInfoable - GetPublishingState() models.PublicationFacetable - GetShowComments() *bool - GetShowRecommendedPages() *bool - GetThumbnailWebUrl() *string - GetTitle() *string - SetContentType(value models.ContentTypeInfoable) - SetPublishingState(value models.PublicationFacetable) - SetShowComments(value *bool) - SetShowRecommendedPages(value *bool) - SetThumbnailWebUrl(value *string) - SetTitle(value *string) -} diff --git a/src/internal/connector/support/attendee.go b/src/internal/connector/support/attendee.go index 9c55d9890..691f3c9e0 100644 --- a/src/internal/connector/support/attendee.go +++ b/src/internal/connector/support/attendee.go @@ -3,7 +3,7 @@ package support import ( "fmt" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" ) type attendee struct { diff --git a/src/internal/connector/support/errors.go b/src/internal/connector/support/errors.go index 8f73ea8fa..e577acb66 100644 --- a/src/internal/connector/support/errors.go +++ b/src/internal/connector/support/errors.go @@ -6,7 +6,7 @@ import ( "strings" multierror "github.com/hashicorp/go-multierror" - msgraph_errors "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors" + msgraph_errors "github.com/microsoftgraph/msgraph-beta-sdk-go/models/odataerrors" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common" diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index d7e51e513..c75787045 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -1,9 +1,11 @@ package support import ( + "strings" + absser "github.com/microsoft/kiota-abstractions-go/serialization" js "github.com/microsoft/kiota-serialization-json-go" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" ) @@ -71,3 +73,14 @@ func CreateListFromBytes(bytes []byte) (models.Listable, error) { return list, nil } + +func HasAttachments(body models.ItemBodyable) bool { + if body.GetContent() == nil || body.GetContentType() == nil || + *body.GetContentType() == models.TEXT_BODYTYPE || len(*body.GetContent()) == 0 { + return false + } + + content := *body.GetContent() + + return strings.Contains(content, "src=\"cid:") +} diff --git a/src/internal/connector/support/m365Support_test.go b/src/internal/connector/support/m365Support_test.go index c04c74604..dedde3536 100644 --- a/src/internal/connector/support/m365Support_test.go +++ b/src/internal/connector/support/m365Support_test.go @@ -3,6 +3,7 @@ package support import ( "testing" + "github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -159,3 +160,56 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() { }) } } + +func (suite *DataSupportSuite) TestHasAttachments() { + tests := []struct { + name string + hasAttachment assert.BoolAssertionFunc + getBodyable func(t *testing.T) models.ItemBodyable + }{ + { + name: "Mock w/out attachment", + hasAttachment: assert.False, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMockMessageWithBodyBytes( + "Test", + "This is testing", + "This is testing", + ) + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Mock w/ inline attachment", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy") + message, err := CreateMessageFromBytes(byteArray) + require.NoError(t, err) + return message.GetBody() + }, + }, + { + name: "Edge Case", + hasAttachment: assert.True, + getBodyable: func(t *testing.T) models.ItemBodyable { + //nolint:lll + content := "\r\n
Happy New Year,

In accordance with TPS report guidelines, there have been questions about how to address our activities SharePoint Cover page. Do you believe this is the best picture? 



Let me know if this meets our culture requirements.

Warm Regards,

Dustin
" + body := models.NewItemBody() + body.SetContent(&content) + cat := models.HTML_BODYTYPE + body.SetContentType(&cat) + return body + }, + }, + } + + for _, test := range tests { + suite.T().Run(test.name, func(t *testing.T) { + found := HasAttachments(test.getBodyable(t)) + test.hasAttachment(t, found) + }) + } +} diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 651689430..27e0215e4 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -3,7 +3,7 @@ package support import ( "strings" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" ) // CloneMessageableFields places data from original data into new message object. diff --git a/src/internal/connector/support/m365Transform_test.go b/src/internal/connector/support/m365Transform_test.go index 90d8e757b..92aae990b 100644 --- a/src/internal/connector/support/m365Transform_test.go +++ b/src/internal/connector/support/m365Transform_test.go @@ -3,7 +3,7 @@ package support import ( "testing" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 3ee5c0230..0c39ced10 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -7,7 +7,7 @@ import ( "time" "github.com/google/uuid" - "github.com/microsoftgraph/msgraph-sdk-go/users" + "github.com/microsoftgraph/msgraph-beta-sdk-go/users" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/src/internal/tester/config.go b/src/internal/tester/config.go index 804b217f3..d79d95104 100644 --- a/src/internal/tester/config.go +++ b/src/internal/tester/config.go @@ -111,8 +111,7 @@ func readTestConfig() (map[string]string, error) { TestCfgUserID, os.Getenv(EnvCorsoM365TestUserID), vpr.GetString(TestCfgUserID), - "lynner@8qzvrj.onmicrosoft.com", - //"lidiah@8qzvrj.onmicrosoft.com", + "conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -120,7 +119,6 @@ func readTestConfig() (map[string]string, error) { os.Getenv(EnvCorsoSecondaryM365TestUserID), vpr.GetString(TestCfgSecondaryUserID), "lidiah@8qzvrj.onmicrosoft.com", - //"lynner@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, @@ -134,7 +132,7 @@ func readTestConfig() (map[string]string, error) { TestCfgLoadTestOrgUsers, os.Getenv(EnvCorsoM365LoadTestOrgUsers), vpr.GetString(TestCfgLoadTestOrgUsers), - "lidiah@8qzvrj.onmicrosoft.com,lynner@8qzvrj.onmicrosoft.com", + "lidiah@8qzvrj.onmicrosoft.com,conneri@8qzvrj.onmicrosoft.com", ) fallbackTo( testEnv, diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index 19da0a21f..4a1ec407b 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -3,7 +3,7 @@ package m365 import ( "context" - "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/microsoftgraph/msgraph-beta-sdk-go/models" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector"