diff --git a/src/cmd/getM365/exchange/get_item.go b/src/cmd/getM365/exchange/get_item.go index 8196beb34..1d644f97e 100644 --- a/src/cmd/getM365/exchange/get_item.go +++ b/src/cmd/getM365/exchange/get_item.go @@ -16,12 +16,12 @@ import ( "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/credentials" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // Required inputs from user for command execution diff --git a/src/cmd/getM365/onedrive/get_item.go b/src/cmd/getM365/onedrive/get_item.go index 9e16ee746..ab1378ab9 100644 --- a/src/cmd/getM365/onedrive/get_item.go +++ b/src/cmd/getM365/onedrive/get_item.go @@ -22,9 +22,9 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/credentials" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const downloadURLKey = "@microsoft.graph.downloadUrl" diff --git a/src/internal/connector/exchange/api/api.go b/src/internal/connector/exchange/api/api.go deleted file mode 100644 index 444251d6b..000000000 --- a/src/internal/connector/exchange/api/api.go +++ /dev/null @@ -1,147 +0,0 @@ -package api - -import ( - "context" - "strings" - - "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" - "github.com/microsoftgraph/msgraph-sdk-go/models" - - "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/pkg/account" -) - -// --------------------------------------------------------------------------- -// common types and consts -// --------------------------------------------------------------------------- - -// DeltaUpdate holds the results of a current delta token. It normally -// gets produced when aggregating the addition and removal of items in -// a delta-queryable folder. -type DeltaUpdate struct { - // the deltaLink itself - URL string - // true if the old delta was marked as invalid - Reset bool -} - -// GraphQuery represents functions which perform exchange-specific queries -// into M365 backstore. Responses -> returned items will only contain the information -// that is included in the options -// TODO: use selector or path for granularity into specific folders or specific date ranges -type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) - -// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve -// the default associated data of a M365 object. This varies by object. Additional -// Queries must be run to obtain the omitted fields. -type GraphRetrievalFunc func( - ctx context.Context, - user, m365ID string, -) (serialization.Parsable, error) - -// --------------------------------------------------------------------------- -// interfaces -// --------------------------------------------------------------------------- - -// Client is used to fulfill the interface for exchange -// queries that are traditionally backed by GraphAPI. A -// struct is used in this case, instead of deferring to -// pure function wrappers, so that the boundary separates the -// granular implementation of the graphAPI and kiota away -// from the exchange package's broader intents. -type Client struct { - Credentials account.M365Config - - // The Stable service is re-usable for any non-paged request. - // This allows us to maintain performance across async requests. - Stable graph.Servicer - - // The LargeItem graph servicer is configured specifically for - // downloading large items. Specifically for use when handling - // attachments, and for no other use. - LargeItem graph.Servicer -} - -// NewClient produces a new exchange api client. Must be used in -// place of creating an ad-hoc client struct. -func NewClient(creds account.M365Config) (Client, error) { - s, err := NewService(creds) - if err != nil { - return Client{}, err - } - - li, err := newLargeItemService(creds) - if err != nil { - return Client{}, err - } - - return Client{creds, s, li}, nil -} - -// service generates a new service. Used for paged and other long-running -// requests instead of the client's stable service, so that in-flight state -// within the adapter doesn't get clobbered -func (c Client) service() (*graph.Service, error) { - s, err := NewService(c.Credentials) - return s, err -} - -func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) { - a, err := graph.CreateAdapter( - creds.AzureTenantID, - creds.AzureClientID, - creds.AzureClientSecret, - opts...) - if err != nil { - return nil, clues.Wrap(err, "generating graph adapter") - } - - return graph.NewService(a), nil -} - -func newLargeItemService(creds account.M365Config) (*graph.Service, error) { - a, err := NewService(creds, graph.NoTimeout()) - if err != nil { - return nil, clues.Wrap(err, "generating no-timeout graph adapter") - } - - return a, nil -} - -// --------------------------------------------------------------------------- -// helper funcs -// --------------------------------------------------------------------------- - -// checkIDAndName is a helper function to ensure that -// the ID and name pointers are set prior to being called. -func checkIDAndName(c graph.Container) error { - id := ptr.Val(c.GetId()) - if len(id) == 0 { - return clues.New("container missing ID") - } - - dn := ptr.Val(c.GetDisplayName()) - if len(dn) == 0 { - return clues.New("container missing display name").With("container_id", id) - } - - return nil -} - -func HasAttachments(body models.ItemBodyable) bool { - if body == nil { - return false - } - - if ct, ok := ptr.ValOK(body.GetContentType()); !ok || ct == models.TEXT_BODYTYPE { - return false - } - - if body, ok := ptr.ValOK(body.GetContent()); !ok || len(body) == 0 { - return false - } - - return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:") -} diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index 11b2cb0be..d07ee4300 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -8,7 +8,6 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/prefixmatcher" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -18,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // MetadataFileNames produces the category-specific set of filenames used to diff --git a/src/internal/connector/exchange/data_collections_test.go b/src/internal/connector/exchange/data_collections_test.go index f453227af..ef34de5ff 100644 --- a/src/internal/connector/exchange/data_collections_test.go +++ b/src/internal/connector/exchange/data_collections_test.go @@ -12,7 +12,6 @@ import ( inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -21,6 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 76c32a4fd..69e78229d 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type CacheResolverSuite struct { diff --git a/src/internal/connector/exchange/mail_folder_cache_test.go b/src/internal/connector/exchange/mail_folder_cache_test.go index 0ae08ce88..46132125b 100644 --- a/src/internal/connector/exchange/mail_folder_cache_test.go +++ b/src/internal/connector/exchange/mail_folder_cache_test.go @@ -9,10 +9,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index b6ec9168f..de67ab8e6 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" @@ -18,6 +17,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type ExchangeRestoreSuite struct { diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 52d46ba42..bde0f1ae4 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -6,13 +6,13 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) var ErrFolderNotFound = clues.New("folder not found") diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index 0aa6680fb..3d96cf4b5 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -7,7 +7,6 @@ import ( "github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -16,6 +15,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type addedAndRemovedItemIDsGetter interface { diff --git a/src/internal/connector/exchange/service_iterators_test.go b/src/internal/connector/exchange/service_iterators_test.go index 7cc784374..a3b35c7f4 100644 --- a/src/internal/connector/exchange/service_iterators_test.go +++ b/src/internal/connector/exchange/service_iterators_test.go @@ -11,7 +11,6 @@ import ( inMock "github.com/alcionai/corso/src/internal/common/idname/mock" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -21,6 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // --------------------------------------------------------------------------- diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index b0fa94b5f..4ddf369c2 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -11,7 +11,6 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -23,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // RestoreExchangeObject directs restore pipeline towards restore function diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index ea2ef6308..6f7cf3da8 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -15,7 +15,6 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -25,6 +24,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 6b4c1f434..cc27f4fb2 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -14,7 +14,6 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -23,6 +22,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type driveSource int diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 1c621659d..49c0ad376 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -19,8 +19,6 @@ import ( pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock" "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -29,6 +27,8 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type statePath struct { diff --git a/src/internal/connector/onedrive/drive.go b/src/internal/connector/onedrive/drive.go index a53c3273b..77460a504 100644 --- a/src/internal/connector/onedrive/drive.go +++ b/src/internal/connector/onedrive/drive.go @@ -13,11 +13,11 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" gapi "github.com/alcionai/corso/src/internal/connector/graph/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) const ( diff --git a/src/internal/connector/onedrive/drive_test.go b/src/internal/connector/onedrive/drive_test.go index 519047ead..7bbf186a8 100644 --- a/src/internal/connector/onedrive/drive_test.go +++ b/src/internal/connector/onedrive/drive_test.go @@ -17,8 +17,6 @@ import ( "github.com/alcionai/corso/src/internal/common/prefixmatcher" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" - "github.com/alcionai/corso/src/internal/connector/onedrive/api/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" @@ -26,6 +24,8 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) // Unit tests diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 6c90ba7a3..2c9046ebf 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -14,10 +14,10 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // downloadUrlKeys is used to find the download URL in a DriveItem response. diff --git a/src/internal/connector/onedrive/item_test.go b/src/internal/connector/onedrive/item_test.go index 91840d84e..8cc4968fa 100644 --- a/src/internal/connector/onedrive/item_test.go +++ b/src/internal/connector/onedrive/item_test.go @@ -15,10 +15,10 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type ItemIntegrationSuite struct { diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index a7eb3e308..0f1451bb8 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -10,12 +10,12 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) func getParentMetadata( diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index 7ebf73367..755496420 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -13,7 +13,6 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -26,6 +25,7 @@ import ( "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // copyBufferSize is used for chunked upload diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index f6802ae4e..a658a9267 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -23,12 +23,10 @@ import ( "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/connector" "github.com/alcionai/corso/src/internal/connector/exchange" - exapi "github.com/alcionai/corso/src/internal/connector/exchange/api" exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" "github.com/alcionai/corso/src/internal/connector/onedrive" - odapi "github.com/alcionai/corso/src/internal/connector/onedrive/api" odConsts "github.com/alcionai/corso/src/internal/connector/onedrive/consts" "github.com/alcionai/corso/src/internal/connector/onedrive/metadata" "github.com/alcionai/corso/src/internal/connector/support" @@ -51,6 +49,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" selTD "github.com/alcionai/corso/src/pkg/selectors/testdata" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/store" ) @@ -763,7 +762,7 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont m365, err := acct.M365Config() require.NoError(t, err, clues.ToCore(err)) - ac, err := exapi.NewClient(m365) + ac, err := api.NewClient(m365) require.NoError(t, err, clues.ToCore(err)) // generate 3 new folders with two items each. @@ -1333,7 +1332,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalOneDrive() { ctx context.Context, gs graph.Servicer, ) string { - d, err := odapi.GetUsersDrive(ctx, gs, suite.user) + d, err := api.GetUsersDrive(ctx, gs, suite.user) if err != nil { err = graph.Wrap(ctx, err, "retrieving default user drive"). With("user", suite.user) @@ -1372,7 +1371,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_incrementalSharePoint() { ctx context.Context, gs graph.Servicer, ) string { - d, err := odapi.GetSitesDefaultDrive(ctx, gs, suite.site) + d, err := api.GetSitesDefaultDrive(ctx, gs, suite.site) if err != nil { err = graph.Wrap(ctx, err, "retrieving default site drive"). With("site", suite.site) diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 5eac8852c..e35d47ffc 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -17,7 +17,6 @@ import ( exchMock "github.com/alcionai/corso/src/internal/connector/exchange/mock" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/mock" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" evmock "github.com/alcionai/corso/src/internal/events/mock" @@ -30,6 +29,7 @@ import ( "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/store" ) diff --git a/src/pkg/services/m365/api/api.go b/src/pkg/services/m365/api/api.go index 37b9b48a7..1500840fe 100644 --- a/src/pkg/services/m365/api/api.go +++ b/src/pkg/services/m365/api/api.go @@ -1,7 +1,10 @@ package api import ( + "context" + "github.com/alcionai/clues" + "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" @@ -19,37 +22,87 @@ import ( type Client struct { Credentials account.M365Config - // The stable service is re-usable for any non-paged request. + // The Stable service is re-usable for any non-paged request. // This allows us to maintain performance across async requests. - stable graph.Servicer + Stable graph.Servicer + + // The LargeItem graph servicer is configured specifically for + // downloading large items such as drive item content or outlook + // mail and event attachments. + LargeItem graph.Servicer } -// NewClient produces a new api client. Must be used in +// NewClient produces a new exchange api client. Must be used in // place of creating an ad-hoc client struct. func NewClient(creds account.M365Config) (Client, error) { - s, err := newService(creds) + s, err := NewService(creds) if err != nil { return Client{}, err } - return Client{creds, s}, nil -} - -// service generates a new service. Used for paged and other long-running -// requests instead of the client's stable service, so that in-flight state -// within the adapter doesn't get clobbered -func (c Client) Service() (*graph.Service, error) { - return newService(c.Credentials) -} - -func newService(creds account.M365Config) (*graph.Service, error) { - adapter, err := graph.CreateAdapter( - creds.AzureTenantID, - creds.AzureClientID, - creds.AzureClientSecret) + li, err := newLargeItemService(creds) if err != nil { - return nil, clues.Wrap(err, "generating graph api service client") + return Client{}, err } - return graph.NewService(adapter), nil + return Client{creds, s, li}, nil } + +// Service generates a new graph servicer. New servicers are used for paged +// and other long-running requests instead of the client's stable service, +// so that in-flight state within the adapter doesn't get clobbered. +// Most calls should use the Client.Stable property instead of calling this +// func, unless it is explicitly necessary. +func (c Client) Service() (graph.Servicer, error) { + return NewService(c.Credentials) +} + +func NewService(creds account.M365Config, opts ...graph.Option) (*graph.Service, error) { + a, err := graph.CreateAdapter( + creds.AzureTenantID, + creds.AzureClientID, + creds.AzureClientSecret, + opts...) + if err != nil { + return nil, clues.Wrap(err, "generating graph api adapter") + } + + return graph.NewService(a), nil +} + +func newLargeItemService(creds account.M365Config) (*graph.Service, error) { + a, err := NewService(creds, graph.NoTimeout()) + if err != nil { + return nil, clues.Wrap(err, "generating no-timeout graph adapter") + } + + return a, nil +} + +// --------------------------------------------------------------------------- +// common types and consts +// --------------------------------------------------------------------------- + +// DeltaUpdate holds the results of a current delta token. It normally +// gets produced when aggregating the addition and removal of items in +// a delta-queryable folder. +type DeltaUpdate struct { + // the deltaLink itself + URL string + // true if the old delta was marked as invalid + Reset bool +} + +// GraphQuery represents functions which perform exchange-specific queries +// into M365 backstore. Responses -> returned items will only contain the information +// that is included in the options +// TODO: use selector or path for granularity into specific folders or specific date ranges +type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) + +// GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve +// the default associated data of a M365 object. This varies by object. Additional +// Queries must be run to obtain the omitted fields. +type GraphRetrievalFunc func( + ctx context.Context, + user, m365ID string, +) (serialization.Parsable, error) diff --git a/src/internal/connector/exchange/api/api_test.go b/src/pkg/services/m365/api/api_test.go similarity index 100% rename from src/internal/connector/exchange/api/api_test.go rename to src/pkg/services/m365/api/api_test.go diff --git a/src/internal/connector/exchange/api/contacts.go b/src/pkg/services/m365/api/contacts.go similarity index 99% rename from src/internal/connector/exchange/api/contacts.go rename to src/pkg/services/m365/api/contacts.go index 48c72025f..2f5395b37 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/pkg/services/m365/api/contacts.go @@ -120,7 +120,7 @@ func (c Contacts) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -330,7 +330,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, graph.Stack(ctx, err) } diff --git a/src/internal/connector/exchange/api/contacts_test.go b/src/pkg/services/m365/api/contacts_test.go similarity index 100% rename from src/internal/connector/exchange/api/contacts_test.go rename to src/pkg/services/m365/api/contacts_test.go diff --git a/src/internal/connector/onedrive/api/drive.go b/src/pkg/services/m365/api/drive.go similarity index 100% rename from src/internal/connector/onedrive/api/drive.go rename to src/pkg/services/m365/api/drive.go diff --git a/src/internal/connector/onedrive/api/drive_test.go b/src/pkg/services/m365/api/drive_test.go similarity index 95% rename from src/internal/connector/onedrive/api/drive_test.go rename to src/pkg/services/m365/api/drive_test.go index 26d189d9d..154615a1b 100644 --- a/src/internal/connector/onedrive/api/drive_test.go +++ b/src/pkg/services/m365/api/drive_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/connector/graph" - "github.com/alcionai/corso/src/internal/connector/onedrive/api" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) type OneDriveAPISuite struct { diff --git a/src/internal/connector/exchange/api/events.go b/src/pkg/services/m365/api/events.go similarity index 99% rename from src/internal/connector/exchange/api/events.go rename to src/pkg/services/m365/api/events.go index 46583f5b3..3d84a9c53 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/pkg/services/m365/api/events.go @@ -79,7 +79,7 @@ func (c Events) GetContainerByID( ctx context.Context, userID, containerID string, ) (graph.Container, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -194,7 +194,7 @@ func (c Events) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -388,7 +388,7 @@ func (c Events) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, err } diff --git a/src/internal/connector/exchange/api/events_test.go b/src/pkg/services/m365/api/events_test.go similarity index 100% rename from src/internal/connector/exchange/api/events_test.go rename to src/pkg/services/m365/api/events_test.go diff --git a/src/pkg/services/m365/api/exchange_common.go b/src/pkg/services/m365/api/exchange_common.go new file mode 100644 index 000000000..7f4f6afe2 --- /dev/null +++ b/src/pkg/services/m365/api/exchange_common.go @@ -0,0 +1,43 @@ +package api + +import ( + "strings" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/connector/graph" +) + +// checkIDAndName is a helper function to ensure that +// the ID and name pointers are set prior to being called. +func checkIDAndName(c graph.Container) error { + id := ptr.Val(c.GetId()) + if len(id) == 0 { + return clues.New("container missing ID") + } + + dn := ptr.Val(c.GetDisplayName()) + if len(dn) == 0 { + return clues.New("container missing display name").With("container_id", id) + } + + return nil +} + +func HasAttachments(body models.ItemBodyable) bool { + if body == nil { + return false + } + + if ct, ok := ptr.ValOK(body.GetContentType()); !ok || ct == models.TEXT_BODYTYPE { + return false + } + + if body, ok := ptr.ValOK(body.GetContent()); !ok || len(body) == 0 { + return false + } + + return strings.Contains(ptr.Val(body.GetContent()), "src=\"cid:") +} diff --git a/src/internal/connector/exchange/api/mail.go b/src/pkg/services/m365/api/mail.go similarity index 99% rename from src/internal/connector/exchange/api/mail.go rename to src/pkg/services/m365/api/mail.go index 3373b0880..2abb889c0 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/pkg/services/m365/api/mail.go @@ -63,7 +63,7 @@ func (c Mail) CreateMailFolderWithParent( ctx context.Context, user, folder, parentID string, ) (models.MailFolderable, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -118,7 +118,7 @@ func (c Mail) GetContainerByID( ctx context.Context, userID, dirID string, ) (graph.Container, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, graph.Stack(ctx, err) } @@ -311,7 +311,7 @@ func (c Mail) EnumerateContainers( fn func(graph.CacheFolder) error, errs *fault.Bus, ) error { - service, err := c.service() + service, err := c.Service() if err != nil { return graph.Stack(ctx, err) } @@ -529,7 +529,7 @@ func (c Mail) GetAddedAndRemovedItemIDs( immutableIDs bool, canMakeDeltaQueries bool, ) ([]string, []string, DeltaUpdate, error) { - service, err := c.service() + service, err := c.Service() if err != nil { return nil, nil, DeltaUpdate{}, err } diff --git a/src/internal/connector/exchange/api/mail_test.go b/src/pkg/services/m365/api/mail_test.go similarity index 98% rename from src/internal/connector/exchange/api/mail_test.go rename to src/pkg/services/m365/api/mail_test.go index f98093cf6..5d3641f25 100644 --- a/src/internal/connector/exchange/api/mail_test.go +++ b/src/pkg/services/m365/api/mail_test.go @@ -15,12 +15,12 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/common/ptr" - "github.com/alcionai/corso/src/internal/connector/exchange/api" - "github.com/alcionai/corso/src/internal/connector/exchange/api/mock" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/mock" ) type MailAPIUnitSuite struct { diff --git a/src/internal/connector/onedrive/api/mock/drive.go b/src/pkg/services/m365/api/mock/drive.go similarity index 100% rename from src/internal/connector/onedrive/api/mock/drive.go rename to src/pkg/services/m365/api/mock/drive.go diff --git a/src/internal/connector/exchange/api/mock/mail.go b/src/pkg/services/m365/api/mock/mail.go similarity index 90% rename from src/internal/connector/exchange/api/mock/mail.go rename to src/pkg/services/m365/api/mock/mail.go index 6caf47f88..b05cec1a4 100644 --- a/src/internal/connector/exchange/api/mock/mail.go +++ b/src/pkg/services/m365/api/mock/mail.go @@ -1,10 +1,10 @@ package mock import ( - "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph/mock" "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // NewClient produces a new exchange api client that can be diff --git a/src/internal/connector/exchange/api/options.go b/src/pkg/services/m365/api/options.go similarity index 100% rename from src/internal/connector/exchange/api/options.go rename to src/pkg/services/m365/api/options.go diff --git a/src/internal/connector/exchange/api/shared.go b/src/pkg/services/m365/api/shared.go similarity index 100% rename from src/internal/connector/exchange/api/shared.go rename to src/pkg/services/m365/api/shared.go diff --git a/src/internal/connector/exchange/api/shared_test.go b/src/pkg/services/m365/api/shared_test.go similarity index 100% rename from src/internal/connector/exchange/api/shared_test.go rename to src/pkg/services/m365/api/shared_test.go diff --git a/src/pkg/services/m365/api/sites.go b/src/pkg/services/m365/api/sites.go index a7d5efc47..3f8025289 100644 --- a/src/pkg/services/m365/api/sites.go +++ b/src/pkg/services/m365/api/sites.go @@ -109,7 +109,7 @@ func (c Sites) GetByID(ctx context.Context, identifier string) (models.Siteable, ctx = clues.Add(ctx, "given_site_id", identifier) if siteIDRE.MatchString(identifier) { - resp, err = c.stable.Client().Sites().BySiteId(identifier).Get(ctx, nil) + resp, err = c.Stable.Client().Sites().BySiteId(identifier).Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting site by id") } @@ -136,7 +136,7 @@ func (c Sites) GetByID(ctx context.Context, identifier string) (models.Siteable, rawURL := fmt.Sprintf(webURLGetTemplate, u.Host, path) resp, err = sites. - NewItemSitesSiteItemRequestBuilder(rawURL, c.stable.Adapter()). + NewItemSitesSiteItemRequestBuilder(rawURL, c.Stable.Adapter()). Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting site by weburl") diff --git a/src/pkg/services/m365/api/users.go b/src/pkg/services/m365/api/users.go index 9bf72e42b..6bba52de5 100644 --- a/src/pkg/services/m365/api/users.go +++ b/src/pkg/services/m365/api/users.go @@ -214,7 +214,7 @@ func (c Users) GetByID(ctx context.Context, identifier string) (models.Userable, err error ) - resp, err = c.stable.Client().Users().ByUserId(identifier).Get(ctx, nil) + resp, err = c.Stable.Client().Users().ByUserId(identifier).Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting user") @@ -315,7 +315,7 @@ func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) { Top: ptr.To[int32](1), // just one item is enough }, } - _, err = c.stable.Client(). + _, err = c.Stable.Client(). Users(). ByUserId(userID). MailFolders(). @@ -340,7 +340,7 @@ func (c Users) GetMailFolders( userID string, options users.ItemMailFoldersRequestBuilderGetRequestConfiguration, ) (models.MailFolderCollectionResponseable, error) { - mailFolders, err := c.stable.Client().Users().ByUserId(userID).MailFolders().Get(ctx, &options) + mailFolders, err := c.Stable.Client().Users().ByUserId(userID).MailFolders().Get(ctx, &options) if err != nil { return nil, graph.Wrap(ctx, err, "getting MailFolders") } @@ -350,7 +350,7 @@ func (c Users) GetMailFolders( // TODO: remove when drive api goes into this package func (c Users) GetDrives(ctx context.Context, userID string) (models.DriveCollectionResponseable, error) { - drives, err := c.stable.Client().Users().ByUserId(userID).Drives().Get(ctx, nil) + drives, err := c.Stable.Client().Users().ByUserId(userID).Drives().Get(ctx, nil) if err != nil { return nil, graph.Wrap(ctx, err, "getting drives") } @@ -364,7 +364,7 @@ func (c Users) getMailboxSettings( ) (MailboxInfo, error) { var ( rawURL = fmt.Sprintf("https://graph.microsoft.com/v1.0/users/%s/mailboxSettings", userID) - adapter = c.stable.Adapter() + adapter = c.Stable.Adapter() mi = MailboxInfo{ ErrGetMailBoxSetting: []error{}, }