From e64f93a7d37fbbdc7a1c85ae990e6741711c475f Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 16 May 2023 18:21:20 -0600 Subject: [PATCH] migrate exchange api to pkg/services/m365 (#3416) No logic changes, just code movement. Begins the process ofcentralizing the graph package in pkg/services/m365, instead of dividing up the funcs between different connector subpackages. While the ownership of apis in each connector subpackage does make sense, this movement will reduce duplication, centralize import arrangements, and make it easy to export usage of the api to sdk consumers. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #1996 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/getM365/exchange/get_item.go | 2 +- src/cmd/getM365/onedrive/get_item.go | 2 +- src/internal/connector/exchange/api/api.go | 147 ------------------ .../connector/exchange/data_collections.go | 2 +- .../exchange/data_collections_test.go | 2 +- .../exchange/folder_resolver_test.go | 2 +- .../exchange/mail_folder_cache_test.go | 2 +- .../connector/exchange/restore_test.go | 2 +- .../connector/exchange/service_functions.go | 2 +- .../connector/exchange/service_iterators.go | 2 +- .../exchange/service_iterators_test.go | 2 +- .../connector/exchange/service_restore.go | 2 +- src/internal/connector/onedrive/collection.go | 2 +- .../connector/onedrive/collections.go | 2 +- .../connector/onedrive/collections_test.go | 4 +- src/internal/connector/onedrive/drive.go | 2 +- src/internal/connector/onedrive/drive_test.go | 4 +- src/internal/connector/onedrive/item.go | 2 +- src/internal/connector/onedrive/item_test.go | 2 +- src/internal/connector/onedrive/permission.go | 2 +- src/internal/connector/onedrive/restore.go | 2 +- .../operations/backup_integration_test.go | 9 +- src/internal/operations/restore_test.go | 2 +- src/pkg/services/m365/api/api.go | 95 ++++++++--- .../services/m365}/api/api_test.go | 0 .../services/m365}/api/contacts.go | 4 +- .../services/m365}/api/contacts_test.go | 0 .../services/m365}/api/drive.go | 0 .../services/m365}/api/drive_test.go | 2 +- .../services/m365}/api/events.go | 6 +- .../services/m365}/api/events_test.go | 0 src/pkg/services/m365/api/exchange_common.go | 43 +++++ .../services/m365}/api/mail.go | 8 +- .../services/m365}/api/mail_test.go | 4 +- .../services/m365}/api/mock/drive.go | 0 .../services/m365}/api/mock/mail.go | 2 +- .../services/m365}/api/options.go | 0 .../services/m365}/api/shared.go | 0 .../services/m365}/api/shared_test.go | 0 src/pkg/services/m365/api/sites.go | 4 +- src/pkg/services/m365/api/users.go | 10 +- 41 files changed, 164 insertions(+), 216 deletions(-) delete mode 100644 src/internal/connector/exchange/api/api.go rename src/{internal/connector/exchange => pkg/services/m365}/api/api_test.go (100%) rename src/{internal/connector/exchange => pkg/services/m365}/api/contacts.go (99%) rename src/{internal/connector/exchange => pkg/services/m365}/api/contacts_test.go (100%) rename src/{internal/connector/onedrive => pkg/services/m365}/api/drive.go (100%) rename src/{internal/connector/onedrive => pkg/services/m365}/api/drive_test.go (95%) rename src/{internal/connector/exchange => pkg/services/m365}/api/events.go (99%) rename src/{internal/connector/exchange => pkg/services/m365}/api/events_test.go (100%) create mode 100644 src/pkg/services/m365/api/exchange_common.go rename src/{internal/connector/exchange => pkg/services/m365}/api/mail.go (99%) rename src/{internal/connector/exchange => pkg/services/m365}/api/mail_test.go (98%) rename src/{internal/connector/onedrive => pkg/services/m365}/api/mock/drive.go (100%) rename src/{internal/connector/exchange => pkg/services/m365}/api/mock/mail.go (90%) rename src/{internal/connector/exchange => pkg/services/m365}/api/options.go (100%) rename src/{internal/connector/exchange => pkg/services/m365}/api/shared.go (100%) rename src/{internal/connector/exchange => pkg/services/m365}/api/shared_test.go (100%) diff --git a/src/cmd/getM365/exchange/get_item.go b/src/cmd/getM365/exchange/get_item.go index 8196beb34..1d644f97e 100644 --- a/src/cmd/getM365/exchange/get_item.go +++ b/src/cmd/getM365/exchange/get_item.go @@ -16,12 +16,12 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // Required inputs from user for command execution diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go index 9e16ee746..ab1378ab9 100644 --- a/src/cmd/getM365/onedrive/get_item.go +++ b/src/cmd/getM365/onedrive/get_item.go @@ -22,9 +22,9 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const downloadURLKey = "@microsoft.graph.downloadUrl" diff --git a/src/internal/connector/exchange/api/api.go b/src/internal/connector/exchange/api/api.go deleted file mode 100644 index 444251d6b..000000000 --- a/src/internal/connector/exchange/api/api.go +++ /dev/null @@ -1,147 +0,0 @@ -package api - -import ( - "context" - "strings" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/pkg/account" -) - -// --------------------------------------------------------------------------- -// common types and consts -// --------------------------------------------------------------------------- - -// DeltaUpdate holds the results of a current delta token. It normally -// gets produced when aggregating the addition and removal of items in -// a delta-queryable folder. -type DeltaUpdate struct { - // the deltaLink itself - URL string - // true if the old delta was marked as invalid - Reset bool -} - -// GraphQuery represents functions which perform exchange-specific queries -// into M365 backstore. Responses -> returned items will only contain the information -// that is included in the options -// TODO: use selector or path for granularity into specific folders or specific date ranges -type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) - -// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve -// the default associated data of a M365 object. This varies by object. Additional -// Queries must be run to obtain the omitted fields. -type GraphRetrievalFunc func( - ctx context.Context, - user, m365ID string, -) (serialization.Parsable, error) - -// --------------------------------------------------------------------------- -// interfaces -// --------------------------------------------------------------------------- - -// Client is used to fulfill the interface for exchange -// queries that are traditionally backed by GraphAPI. A -// struct is used in this case, instead of deferring to -// pure function wrappers, so that the boundary separates the -// granular implementation of the graphAPI and kiota away -// from the exchange package's broader intents. -type Client struct { - Credentials account.M365Config - - // The Stable service is re-usable for any non-paged request. - // This allows us to maintain performance across async requests. - Stable graph.Servicer - - // The LargeItem graph servicer is configured specifically for - // downloading large items. Specifically for use when handling - // attachments, and for no other use. - LargeItem graph.Servicer -} - -// NewClient produces a new exchange api client. Must be used in -// place of creating an ad-hoc client struct. -func NewClient(creds account.M365Config) (Client, error) { - s, err := NewService(creds) - if err != nil { - return Client{}, err - } - - li, err := newLargeItemService(creds) - if err != nil { - return Client{}, err - } - - return Client{creds, s, li}, nil -} - -// service generates a new service. Used for paged and other long-running -// requests instead of the client's stable service, so that in-flight state -// within the adapter doesn't get clobbered -func (c Client) service() (*graph.Service, error) { - s, err := NewService(c.Credentials) - return s, err -} - -func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) { - a, err := graph.CreateAdapter( - creds.AzureTenantID, - creds.AzureClientID, - creds.AzureClientSecret, - opts...) - if err != nil { - return nil, clues.Wrap(err, "generating graph adapter") - } - - return graph.NewService(a), nil -} - -func newLargeItemService(creds account.M365Config) (*graph.Service, error) { - a, err := NewService(creds, graph.NoTimeout()) - if err != nil { - return nil, clues.Wrap(err, "generating no-timeout graph adapter") - } - - return a, nil -} - -// --------------------------------------------------------------------------- -// helper funcs -// --------------------------------------------------------------------------- - -// checkIDAndName is a helper function to ensure that -// the ID and name pointers are set prior to being called. -func checkIDAndName(c graph.Container) error { - id := ptr.Val(c.GetId()) - if len(id) == 0 { - return clues.New("container missing ID") - } - - dn := ptr.Val(c.GetDisplayName()) - if len(dn) == 0 { - return clues.New("container missing display name").With("container_id", id) - } - - return nil -} - -func HasAttachments(body models.ItemBodyable) bool { - if body == nil { - return false - } - - if ct, ok := ptr.ValOK(body.GetContentType()); !ok || ct == models.TEXT_BODYTYPE { - return false - } - - if body, ok := ptr.ValOK(body.GetContent()); !ok || len(body) == 0 { - return false - } - - return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:") -} diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index 11b2cb0be..d07ee4300 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -8,7 +8,6 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/prefixmatcher" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -18,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // MetadataFileNames produces the category-specific set of filenames used to diff --git a/src/internal/connector/exchange/data_collections_test.go b/src/internal/connector/exchange/data_collections_test.go index f453227af..ef34de5ff 100644 --- a/src/internal/connector/exchange/data_collections_test.go +++ b/src/internal/connector/exchange/data_collections_test.go @@ -12,7 +12,6 @@ import ( inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -21,6 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 76c32a4fd..69e78229d 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type CacheResolverSuite struct { diff --git a/src/internal/connector/exchange/mail_folder_cache_test.go b/src/internal/connector/exchange/mail_folder_cache_test.go index 0ae08ce88..46132125b 100644 --- a/src/internal/connector/exchange/mail_folder_cache_test.go +++ b/src/internal/connector/exchange/mail_folder_cache_test.go @@ -9,10 +9,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index b6ec9168f..de67ab8e6 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" @@ -18,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type ExchangeRestoreSuite struct { diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 52d46ba42..bde0f1ae4 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -6,13 +6,13 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) var ErrFolderNotFound = clues.New("folder not found") diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 0aa6680fb..3d96cf4b5 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -7,7 +7,6 @@ import ( "github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -16,6 +15,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type addedAndRemovedItemIDsGetter interface { diff --git a/src/internal/connector/exchange/service_iterators_test.go b/src/internal/connector/exchange/service_iterators_test.go index 7cc784374..a3b35c7f4 100644 --- a/src/internal/connector/exchange/service_iterators_test.go +++ b/src/internal/connector/exchange/service_iterators_test.go @@ -11,7 +11,6 @@ import ( inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -21,6 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index b0fa94b5f..4ddf369c2 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -11,7 +11,6 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -23,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // RestoreExchangeObject directs restore pipeline towards restore function diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index ea2ef6308..6f7cf3da8 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -15,7 +15,6 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -25,6 +24,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 6b4c1f434..cc27f4fb2 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -14,7 +14,6 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -23,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type driveSource int diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 1c621659d..49c0ad376 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -19,8 +19,6 @@ import ( pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -29,6 +27,8 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type statePath struct { diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index a53c3273b..77460a504 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -13,11 +13,11 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index 519047ead..7bbf186a8 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -17,8 +17,6 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -26,6 +24,8 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) // Unit tests diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 6c90ba7a3..2c9046ebf 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -14,10 +14,10 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // downloadUrlKeys is used to find the download URL in a DriveItem response. diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 91840d84e..8cc4968fa 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -15,10 +15,10 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type ItemIntegrationSuite struct { diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index a7eb3e308..0f1451bb8 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -10,12 +10,12 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) func getParentMetadata( diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 7ebf73367..755496420 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -13,7 +13,6 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -26,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // copyBufferSize is used for chunked upload diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index f6802ae4e..a658a9267 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -23,12 +23,10 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/exchange" - exapi "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" - odapi "github.com/alcionai/corso/src/internal/connector/onedrive/api" odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" @@ -51,6 +49,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/store" ) @@ -763,7 +762,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont m365, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := exapi.NewClient(m365) + ac, err := api.NewClient(m365) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -1333,7 +1332,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalOneDrive() { ctx context.Context, gs graph.Servicer, ) string { - d, err := odapi.GetUsersDrive(ctx, gs, suite.user) + d, err := api.GetUsersDrive(ctx, gs, suite.user) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). With("user", suite.user) @@ -1372,7 +1371,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalSharePoint() { ctx context.Context, gs graph.Servicer, ) string { - d, err := odapi.GetSitesDefaultDrive(ctx, gs, suite.site) + d, err := api.GetSitesDefaultDrive(ctx, gs, suite.site) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). With("site", suite.site) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5eac8852c..e35d47ffc 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -17,7 +17,6 @@ import ( exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -30,6 +29,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/store" ) diff --git a/src/pkg/services/m365/api/api.go b/src/pkg/services/m365/api/api.go index 37b9b48a7..1500840fe 100644 --- a/src/pkg/services/m365/api/api.go +++ b/src/pkg/services/m365/api/api.go @@ -1,7 +1,10 @@ package api import ( + "context" + "github.com/alcionai/clues" + "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" @@ -19,37 +22,87 @@ import ( type Client struct { Credentials account.M365Config - // The stable service is re-usable for any non-paged request. + // The Stable service is re-usable for any non-paged request. // This allows us to maintain performance across async requests. - stable graph.Servicer + Stable graph.Servicer + + // The LargeItem graph servicer is configured specifically for + // downloading large items such as drive item content or outlook + // mail and event attachments. + LargeItem graph.Servicer } -// NewClient produces a new api client. Must be used in +// NewClient produces a new exchange api client. Must be used in // place of creating an ad-hoc client struct. func NewClient(creds account.M365Config) (Client, error) { - s, err := newService(creds) + s, err := NewService(creds) if err != nil { return Client{}, err } - return Client{creds, s}, nil -} - -// service generates a new service. Used for paged and other long-running -// requests instead of the client's stable service, so that in-flight state -// within the adapter doesn't get clobbered -func (c Client) Service() (*graph.Service, error) { - return newService(c.Credentials) -} - -func newService(creds account.M365Config) (*graph.Service, error) { - adapter, err := graph.CreateAdapter( - creds.AzureTenantID, - creds.AzureClientID, - creds.AzureClientSecret) + li, err := newLargeItemService(creds) if err != nil { - return nil, clues.Wrap(err, "generating graph api service client") + return Client{}, err } - return graph.NewService(adapter), nil + return Client{creds, s, li}, nil } + +// Service generates a new graph servicer. New servicers are used for paged +// and other long-running requests instead of the client's stable service, +// so that in-flight state within the adapter doesn't get clobbered. +// Most calls should use the Client.Stable property instead of calling this +// func, unless it is explicitly necessary. +func (c Client) Service() (graph.Servicer, error) { + return NewService(c.Credentials) +} + +func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) { + a, err := graph.CreateAdapter( + creds.AzureTenantID, + creds.AzureClientID, + creds.AzureClientSecret, + opts...) + if err != nil { + return nil, clues.Wrap(err, "generating graph api adapter") + } + + return graph.NewService(a), nil +} + +func newLargeItemService(creds account.M365Config) (*graph.Service, error) { + a, err := NewService(creds, graph.NoTimeout()) + if err != nil { + return nil, clues.Wrap(err, "generating no-timeout graph adapter") + } + + return a, nil +} + +// --------------------------------------------------------------------------- +// common types and consts +// --------------------------------------------------------------------------- + +// DeltaUpdate holds the results of a current delta token. It normally +// gets produced when aggregating the addition and removal of items in +// a delta-queryable folder. +type DeltaUpdate struct { + // the deltaLink itself + URL string + // true if the old delta was marked as invalid + Reset bool +} + +// GraphQuery represents functions which perform exchange-specific queries +// into M365 backstore. Responses -> returned items will only contain the information +// that is included in the options +// TODO: use selector or path for granularity into specific folders or specific date ranges +type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) + +// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve +// the default associated data of a M365 object. This varies by object. Additional +// Queries must be run to obtain the omitted fields. +type GraphRetrievalFunc func( + ctx context.Context, + user, m365ID string, +) (serialization.Parsable, error) diff --git a/src/internal/connector/exchange/api/api_test.go b/src/pkg/services/m365/api/api_test.go similarity index 100% rename from src/internal/connector/exchange/api/api_test.go rename to src/pkg/services/m365/api/api_test.go diff --git a/src/internal/connector/exchange/api/contacts.go b/src/pkg/services/m365/api/contacts.go similarity index 99% rename from src/internal/connector/exchange/api/contacts.go rename to src/pkg/services/m365/api/contacts.go index 48c72025f..2f5395b37 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/pkg/services/m365/api/contacts.go @@ -120,7 +120,7 @@ func (c Contacts) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -330,7 +330,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, graph.Stack(ctx, err) } diff --git a/src/internal/connector/exchange/api/contacts_test.go b/src/pkg/services/m365/api/contacts_test.go similarity index 100% rename from src/internal/connector/exchange/api/contacts_test.go rename to src/pkg/services/m365/api/contacts_test.go diff --git a/src/internal/connector/onedrive/api/drive.go b/src/pkg/services/m365/api/drive.go similarity index 100% rename from src/internal/connector/onedrive/api/drive.go rename to src/pkg/services/m365/api/drive.go diff --git a/src/internal/connector/onedrive/api/drive_test.go b/src/pkg/services/m365/api/drive_test.go similarity index 95% rename from src/internal/connector/onedrive/api/drive_test.go rename to src/pkg/services/m365/api/drive_test.go index 26d189d9d..154615a1b 100644 --- a/src/internal/connector/onedrive/api/drive_test.go +++ b/src/pkg/services/m365/api/drive_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type OneDriveAPISuite struct { diff --git a/src/internal/connector/exchange/api/events.go b/src/pkg/services/m365/api/events.go similarity index 99% rename from src/internal/connector/exchange/api/events.go rename to src/pkg/services/m365/api/events.go index 46583f5b3..3d84a9c53 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/pkg/services/m365/api/events.go @@ -79,7 +79,7 @@ func (c Events) GetContainerByID( ctx context.Context, userID, containerID string, ) (graph.Container, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -194,7 +194,7 @@ func (c Events) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -388,7 +388,7 @@ func (c Events) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, err } diff --git a/src/internal/connector/exchange/api/events_test.go b/src/pkg/services/m365/api/events_test.go similarity index 100% rename from src/internal/connector/exchange/api/events_test.go rename to src/pkg/services/m365/api/events_test.go diff --git a/src/pkg/services/m365/api/exchange_common.go b/src/pkg/services/m365/api/exchange_common.go new file mode 100644 index 000000000..7f4f6afe2 --- /dev/null +++ b/src/pkg/services/m365/api/exchange_common.go @@ -0,0 +1,43 @@ +package api + +import ( + "strings" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/connector/graph" +) + +// checkIDAndName is a helper function to ensure that +// the ID and name pointers are set prior to being called. +func checkIDAndName(c graph.Container) error { + id := ptr.Val(c.GetId()) + if len(id) == 0 { + return clues.New("container missing ID") + } + + dn := ptr.Val(c.GetDisplayName()) + if len(dn) == 0 { + return clues.New("container missing display name").With("container_id", id) + } + + return nil +} + +func HasAttachments(body models.ItemBodyable) bool { + if body == nil { + return false + } + + if ct, ok := ptr.ValOK(body.GetContentType()); !ok || ct == models.TEXT_BODYTYPE { + return false + } + + if body, ok := ptr.ValOK(body.GetContent()); !ok || len(body) == 0 { + return false + } + + return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:") +} diff --git a/src/internal/connector/exchange/api/mail.go b/src/pkg/services/m365/api/mail.go similarity index 99% rename from src/internal/connector/exchange/api/mail.go rename to src/pkg/services/m365/api/mail.go index 3373b0880..2abb889c0 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/pkg/services/m365/api/mail.go @@ -63,7 +63,7 @@ func (c Mail) CreateMailFolderWithParent( ctx context.Context, user, folder, parentID string, ) (models.MailFolderable, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -118,7 +118,7 @@ func (c Mail) GetContainerByID( ctx context.Context, userID, dirID string, ) (graph.Container, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -311,7 +311,7 @@ func (c Mail) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -529,7 +529,7 @@ func (c Mail) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, err } diff --git a/src/internal/connector/exchange/api/mail_test.go b/src/pkg/services/m365/api/mail_test.go similarity index 98% rename from src/internal/connector/exchange/api/mail_test.go rename to src/pkg/services/m365/api/mail_test.go index f98093cf6..5d3641f25 100644 --- a/src/internal/connector/exchange/api/mail_test.go +++ b/src/pkg/services/m365/api/mail_test.go @@ -15,12 +15,12 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" - "github.com/alcionai/corso/src/internal/connector/exchange/api/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type MailAPIUnitSuite struct { diff --git a/src/internal/connector/onedrive/api/mock/drive.go b/src/pkg/services/m365/api/mock/drive.go similarity index 100% rename from src/internal/connector/onedrive/api/mock/drive.go rename to src/pkg/services/m365/api/mock/drive.go diff --git a/src/internal/connector/exchange/api/mock/mail.go b/src/pkg/services/m365/api/mock/mail.go similarity index 90% rename from src/internal/connector/exchange/api/mock/mail.go rename to src/pkg/services/m365/api/mock/mail.go index 6caf47f88..b05cec1a4 100644 --- a/src/internal/connector/exchange/api/mock/mail.go +++ b/src/pkg/services/m365/api/mock/mail.go @@ -1,10 +1,10 @@ package mock import ( - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/mock" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // NewClient produces a new exchange api client that can be diff --git a/src/internal/connector/exchange/api/options.go b/src/pkg/services/m365/api/options.go similarity index 100% rename from src/internal/connector/exchange/api/options.go rename to src/pkg/services/m365/api/options.go diff --git a/src/internal/connector/exchange/api/shared.go b/src/pkg/services/m365/api/shared.go similarity index 100% rename from src/internal/connector/exchange/api/shared.go rename to src/pkg/services/m365/api/shared.go diff --git a/src/internal/connector/exchange/api/shared_test.go b/src/pkg/services/m365/api/shared_test.go similarity index 100% rename from src/internal/connector/exchange/api/shared_test.go rename to src/pkg/services/m365/api/shared_test.go diff --git a/src/pkg/services/m365/api/sites.go b/src/pkg/services/m365/api/sites.go index a7d5efc47..3f8025289 100644 --- a/src/pkg/services/m365/api/sites.go +++ b/src/pkg/services/m365/api/sites.go @@ -109,7 +109,7 @@ func (c Sites) GetByID(ctx context.Context, identifier string) (models.Siteable, ctx = clues.Add(ctx, "given_site_id", identifier) if siteIDRE.MatchString(identifier) { - resp, err = c.stable.Client().Sites().BySiteId(identifier).Get(ctx, nil) + resp, err = c.Stable.Client().Sites().BySiteId(identifier).Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting site by id") } @@ -136,7 +136,7 @@ func (c Sites) GetByID(ctx context.Context, identifier string) (models.Siteable, rawURL := fmt.Sprintf(webURLGetTemplate, u.Host, path) resp, err = sites. - NewItemSitesSiteItemRequestBuilder(rawURL, c.stable.Adapter()). + NewItemSitesSiteItemRequestBuilder(rawURL, c.Stable.Adapter()). Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting site by weburl") diff --git a/src/pkg/services/m365/api/users.go b/src/pkg/services/m365/api/users.go index 9bf72e42b..6bba52de5 100644 --- a/src/pkg/services/m365/api/users.go +++ b/src/pkg/services/m365/api/users.go @@ -214,7 +214,7 @@ func (c Users) GetByID(ctx context.Context, identifier string) (models.Userable, err error ) - resp, err = c.stable.Client().Users().ByUserId(identifier).Get(ctx, nil) + resp, err = c.Stable.Client().Users().ByUserId(identifier).Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting user") @@ -315,7 +315,7 @@ func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) { Top: ptr.To[int32](1), // just one item is enough }, } - _, err = c.stable.Client(). + _, err = c.Stable.Client(). Users(). ByUserId(userID). MailFolders(). @@ -340,7 +340,7 @@ func (c Users) GetMailFolders( userID string, options users.ItemMailFoldersRequestBuilderGetRequestConfiguration, ) (models.MailFolderCollectionResponseable, error) { - mailFolders, err := c.stable.Client().Users().ByUserId(userID).MailFolders().Get(ctx, &options) + mailFolders, err := c.Stable.Client().Users().ByUserId(userID).MailFolders().Get(ctx, &options) if err != nil { return nil, graph.Wrap(ctx, err, "getting MailFolders") } @@ -350,7 +350,7 @@ func (c Users) GetMailFolders( // TODO: remove when drive api goes into this package func (c Users) GetDrives(ctx context.Context, userID string) (models.DriveCollectionResponseable, error) { - drives, err := c.stable.Client().Users().ByUserId(userID).Drives().Get(ctx, nil) + drives, err := c.Stable.Client().Users().ByUserId(userID).Drives().Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting drives") } @@ -364,7 +364,7 @@ func (c Users) getMailboxSettings( ) (MailboxInfo, error) { var ( rawURL = fmt.Sprintf("https://graph.microsoft.com/v1.0/users/%s/mailboxSettings", userID) - adapter = c.stable.Adapter() + adapter = c.Stable.Adapter() mi = MailboxInfo{ ErrGetMailBoxSetting: []error{}, }