From 978f9304bf53ec6fae434bb8e12e2ca9beb2c9db Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 4 Jan 2023 10:19:44 -0700 Subject: [PATCH] methodize all api funcs to the Client (#2008) ## Description In order to use the api layer as an interface, we need the functions therein to be methods, so that callers can leverage local interfaces. This change introduces the api.Client, and begins to spread it throughout the exchange package, largely in place of graph servicers. No logic changes have occurred here. The only modifications are what is required to utilize the api client. ## Does this PR need a docs update or release note? - [x] :no_entry: No ## Type of change - [x] :robot: Test ## Issue(s) * #1967 ## Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cmd/factory/exchange.go | 15 +-- src/cmd/factory/factory.go | 17 ++-- src/cmd/getM365/getItem.go | 22 +++-- src/internal/connector/exchange/api/api.go | 46 +++++++++- .../connector/exchange/api/api_test.go | 9 +- .../connector/exchange/api/contacts.go | 56 +++++++----- src/internal/connector/exchange/api/events.go | 61 ++++++++----- src/internal/connector/exchange/api/mail.go | 91 +++++++++++-------- .../exchange/contact_folder_cache.go | 9 +- .../exchange/container_resolver_test.go | 13 +-- .../exchange/event_calendar_cache.go | 6 +- .../exchange/exchange_data_collection.go | 15 +-- .../exchange/exchange_data_collection_test.go | 3 +- .../exchange/folder_resolver_test.go | 16 ++-- .../connector/exchange/iterators_test.go | 21 +++-- .../connector/exchange/mail_folder_cache.go | 11 ++- .../exchange/mail_folder_cache_test.go | 15 +-- .../connector/exchange/restore_test.go | 61 +++++++------ .../connector/exchange/service_functions.go | 13 +-- .../connector/exchange/service_iterators.go | 27 +++--- .../connector/exchange/service_restore.go | 43 +++++---- src/internal/connector/graph_connector.go | 8 +- .../graph_connector_disconnected_test.go | 18 ++-- .../connector/graph_connector_test.go | 24 ++++- .../operations/backup_integration_test.go | 18 +++- src/internal/operations/restore.go | 7 +- 26 files changed, 397 insertions(+), 248 deletions(-) diff --git a/src/cmd/factory/exchange.go b/src/cmd/factory/exchange.go index b7bbb4afa..6a292a8dc 100644 --- a/src/cmd/factory/exchange.go +++ b/src/cmd/factory/exchange.go @@ -47,7 +47,7 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error { return nil } - gc, tenantID, err := getGCAndVerifyUser(ctx, user) + gc, acct, err := getGCAndVerifyUser(ctx, user) if err != nil { return Only(ctx, err) } @@ -55,10 +55,11 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error { deets, err := generateAndRestoreItems( ctx, gc, + acct, service, category, selectors.NewExchangeRestore([]string{user}).Selector, - tenantID, user, destination, + user, destination, count, func(id, now, subject, body string) []byte { return mockconnector.GetMockMessageWith( @@ -87,7 +88,7 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error return nil } - gc, tenantID, err := getGCAndVerifyUser(ctx, user) + gc, acct, err := getGCAndVerifyUser(ctx, user) if err != nil { return Only(ctx, err) } @@ -95,10 +96,11 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error deets, err := generateAndRestoreItems( ctx, gc, + acct, service, category, selectors.NewExchangeRestore([]string{user}).Selector, - tenantID, user, destination, + user, destination, count, func(id, now, subject, body string) []byte { return mockconnector.GetMockEventWith( @@ -126,7 +128,7 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error { return nil } - gc, tenantID, err := getGCAndVerifyUser(ctx, user) + gc, acct, err := getGCAndVerifyUser(ctx, user) if err != nil { return Only(ctx, err) } @@ -134,10 +136,11 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error { deets, err := generateAndRestoreItems( ctx, gc, + acct, service, category, selectors.NewExchangeRestore([]string{user}).Selector, - tenantID, user, destination, + user, destination, count, func(id, now, subject, body string) []byte { given, mid, sur := id[:8], id[9:13], id[len(id)-12:] diff --git a/src/cmd/factory/factory.go b/src/cmd/factory/factory.go index 361c7625c..30474f32c 100644 --- a/src/cmd/factory/factory.go +++ b/src/cmd/factory/factory.go @@ -108,10 +108,11 @@ type dataBuilderFunc func(id, now, subject, body string) []byte func generateAndRestoreItems( ctx context.Context, gc *connector.GraphConnector, + acct account.Account, service path.ServiceType, cat path.CategoryType, sel selectors.Selector, - tenantID, userID, destFldr string, + userID, destFldr string, howMany int, dbf dataBuilderFunc, ) (*details.Details, error) { @@ -144,7 +145,7 @@ func generateAndRestoreItems( dataColls, err := buildCollections( service, - tenantID, userID, + acct.ID(), userID, dest, collections, ) @@ -154,14 +155,14 @@ func generateAndRestoreItems( Infof(ctx, "Generating %d %s items in %s\n", howMany, cat, destination) - return gc.RestoreDataCollections(ctx, sel, dest, dataColls) + return gc.RestoreDataCollections(ctx, acct, sel, dest, dataColls) } // ------------------------------------------------------------------------------------------ // Common Helpers // ------------------------------------------------------------------------------------------ -func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, string, error) { +func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, account.Account, error) { tid := common.First(tenant, os.Getenv(account.AzureTenantID)) // get account info @@ -172,13 +173,13 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon acct, err := account.NewAccount(account.ProviderM365, m365Cfg) if err != nil { - return nil, "", errors.Wrap(err, "finding m365 account details") + return nil, account.Account{}, errors.Wrap(err, "finding m365 account details") } // build a graph connector gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) if err != nil { - return nil, "", errors.Wrap(err, "connecting to graph api") + return nil, account.Account{}, errors.Wrap(err, "connecting to graph api") } normUsers := map[string]struct{}{} @@ -188,10 +189,10 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon } if _, ok := normUsers[strings.ToLower(user)]; !ok { - return nil, "", errors.New("user not found within tenant") + return nil, account.Account{}, errors.New("user not found within tenant") } - return gc, tid, nil + return gc, acct, nil } type item struct { diff --git a/src/cmd/getM365/getItem.go b/src/cmd/getM365/getItem.go index 8f79a92d9..10648006a 100644 --- a/src/cmd/getM365/getItem.go +++ b/src/cmd/getM365/getItem.go @@ -77,12 +77,12 @@ func handleGetCommand(cmd *cobra.Command, args []string) error { return nil } - gc, err := getGC(ctx) + gc, creds, err := getGC(ctx) if err != nil { return err } - err = runDisplayM365JSON(ctx, gc.Service) + err = runDisplayM365JSON(ctx, gc.Service, creds) if err != nil { return Only(ctx, errors.Wrapf(err, "unable to create mock from M365: %s", m365ID)) } @@ -93,6 +93,7 @@ func handleGetCommand(cmd *cobra.Command, args []string) error { func runDisplayM365JSON( ctx context.Context, gs graph.Servicer, + creds account.M365Config, ) error { var ( get api.GraphRetrievalFunc @@ -100,9 +101,14 @@ func runDisplayM365JSON( cat = graph.StringToPathCategory(category) ) + ac, err := api.NewClient(creds) + if err != nil { + return err + } + switch cat { case path.EmailCategory, path.EventsCategory, path.ContactsCategory: - get, serializeFunc = exchange.GetQueryAndSerializeFunc(cat) + get, serializeFunc = exchange.GetQueryAndSerializeFunc(ac, cat) default: return fmt.Errorf("unable to process category: %s", cat) } @@ -111,7 +117,7 @@ func runDisplayM365JSON( sw := kw.NewJsonSerializationWriter() - response, err := get(ctx, gs, user, m365ID) + response, err := get(ctx, user, m365ID) if err != nil { return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } @@ -159,7 +165,7 @@ func runDisplayM365JSON( // Helpers //------------------------------------------------------------------------------- -func getGC(ctx context.Context) (*connector.GraphConnector, error) { +func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config, error) { // get account info m365Cfg := account.M365Config{ M365: credentials.GetM365(), @@ -168,13 +174,13 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) { acct, err := account.NewAccount(account.ProviderM365, m365Cfg) if err != nil { - return nil, Only(ctx, errors.Wrap(err, "finding m365 account details")) + return nil, m365Cfg, Only(ctx, errors.Wrap(err, "finding m365 account details")) } gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) if err != nil { - return nil, Only(ctx, errors.Wrap(err, "connecting to graph API")) + return nil, m365Cfg, Only(ctx, errors.Wrap(err, "connecting to graph API")) } - return gc, nil + return gc, m365Cfg, nil } diff --git a/src/internal/connector/exchange/api/api.go b/src/internal/connector/exchange/api/api.go index b3ee67677..93a25e13d 100644 --- a/src/internal/connector/exchange/api/api.go +++ b/src/internal/connector/exchange/api/api.go @@ -4,8 +4,10 @@ import ( "context" "github.com/microsoft/kiota-abstractions-go/serialization" + "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/account" ) // --------------------------------------------------------------------------- @@ -26,14 +28,13 @@ type DeltaUpdate struct { // into M365 backstore. Responses -> returned items will only contain the information // that is included in the options // TODO: use selector or path for granularity into specific folders or specific date ranges -type GraphQuery func(ctx context.Context, gs graph.Servicer, userID string) (serialization.Parsable, error) +type GraphQuery func(ctx context.Context, userID string) (serialization.Parsable, error) // GraphRetrievalFunctions are functions from the Microsoft Graph API that retrieve // the default associated data of a M365 object. This varies by object. Additional // Queries must be run to obtain the omitted fields. type GraphRetrievalFunc func( ctx context.Context, - gs graph.Servicer, user, m365ID string, ) (serialization.Parsable, error) @@ -41,10 +42,47 @@ type GraphRetrievalFunc func( // interfaces // --------------------------------------------------------------------------- -// API is a struct used to fulfill the interface for exchange +// Client is used to fulfill the interface for exchange // queries that are traditionally backed by GraphAPI. A // struct is used in this case, instead of deferring to // pure function wrappers, so that the boundary separates the // granular implementation of the graphAPI and kiota away // from the exchange package's broader intents. -// type API struct{} +type Client struct { + Credentials account.M365Config + + // The stable service is re-usable for any non-paged request. + // This allows us to maintain performance across async requests. + stable graph.Servicer +} + +// NewClient produces a new exchange api client. Must be used in +// place of creating an ad-hoc client struct. +func NewClient(creds account.M365Config) (Client, error) { + s, err := newService(creds) + if err != nil { + return Client{}, err + } + + return Client{creds, s}, nil +} + +// service generates a new service. Used for paged and other long-running +// requests instead of the client's stable service, so that in-flight state +// within the adapter doesn't get clobbered +func (c Client) service() (*graph.Service, error) { + return newService(c.Credentials) +} + +func newService(creds account.M365Config) (*graph.Service, error) { + adapter, err := graph.CreateAdapter( + creds.AzureTenantID, + creds.AzureClientID, + creds.AzureClientSecret, + ) + if err != nil { + return nil, errors.Wrap(err, "generating graph api service client") + } + + return graph.NewService(adapter), nil +} diff --git a/src/internal/connector/exchange/api/api_test.go b/src/internal/connector/exchange/api/api_test.go index f31bffe5e..00291087f 100644 --- a/src/internal/connector/exchange/api/api_test.go +++ b/src/internal/connector/exchange/api/api_test.go @@ -164,6 +164,9 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() { ctx, flush := tester.NewContext() defer flush() + c, err := NewClient(suite.credentials) + require.NoError(suite.T(), err) + userID := tester.M365UserID(suite.T()) tests := []struct { name string @@ -171,17 +174,17 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() { }{ { name: "GraphQuery: Get All ContactFolders", - function: GetAllContactFolderNamesForUser, + function: c.GetAllContactFolderNamesForUser, }, { name: "GraphQuery: Get All Calendars for User", - function: GetAllCalendarNamesForUser, + function: c.GetAllCalendarNamesForUser, }, } for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - response, err := test.function(ctx, suite.gs, userID) + response, err := test.function(ctx, userID) assert.NoError(t, err) assert.NotNil(t, response) }) diff --git a/src/internal/connector/exchange/api/contacts.go b/src/internal/connector/exchange/api/contacts.go index 48dc4ba15..293671b8e 100644 --- a/src/internal/connector/exchange/api/contacts.go +++ b/src/internal/connector/exchange/api/contacts.go @@ -15,39 +15,39 @@ import ( // CreateContactFolder makes a contact folder with the displayName of folderName. // If successful, returns the created folder object. -func CreateContactFolder( +func (c Client) CreateContactFolder( ctx context.Context, - gs graph.Servicer, user, folderName string, ) (models.ContactFolderable, error) { requestBody := models.NewContactFolder() temp := folderName requestBody.SetDisplayName(&temp) - return gs.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil) + return c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil) } // DeleteContactFolder deletes the ContactFolder associated with the M365 ID if permissions are valid. // Errors returned if the function call was not successful. -func DeleteContactFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error { - return gs.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil) +func (c Client) DeleteContactFolder( + ctx context.Context, + user, folderID string, +) error { + return c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil) } // RetrieveContactDataForUser is a GraphRetrievalFun that returns all associated fields. -func RetrieveContactDataForUser( +func (c Client) RetrieveContactDataForUser( ctx context.Context, - gs graph.Servicer, user, m365ID string, ) (serialization.Parsable, error) { - return gs.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil) + return c.stable.Client().UsersById(user).ContactsById(m365ID).Get(ctx, nil) } // GetAllContactFolderNamesForUser is a GraphQuery function for getting // ContactFolderId and display names for contacts. All other information is omitted. // Does not return the default Contact Folder -func GetAllContactFolderNamesForUser( +func (c Client) GetAllContactFolderNamesForUser( ctx context.Context, - gs graph.Servicer, user string, ) (serialization.Parsable, error) { options, err := optionsForContactFolders([]string{"displayName", "parentFolderId"}) @@ -55,12 +55,11 @@ func GetAllContactFolderNamesForUser( return nil, err } - return gs.Client().UsersById(user).ContactFolders().Get(ctx, options) + return c.stable.Client().UsersById(user).ContactFolders().Get(ctx, options) } -func GetContactFolderByID( +func (c Client) GetContactFolderByID( ctx context.Context, - gs graph.Servicer, userID, dirID string, optionalFields ...string, ) (models.ContactFolderable, error) { @@ -71,7 +70,7 @@ func GetContactFolderByID( return nil, errors.Wrapf(err, "options for contact folder: %v", fields) } - return gs.Client(). + return c.stable.Client(). UsersById(userID). ContactFoldersById(dirID). Get(ctx, ofcf) @@ -79,38 +78,47 @@ func GetContactFolderByID( // TODO: we want this to be the full handler, not only the builder. // but this halfway point minimizes changes for now. -func GetContactChildFoldersBuilder( +func (c Client) GetContactChildFoldersBuilder( ctx context.Context, - gs graph.Servicer, userID, baseDirID string, optionalFields ...string, ) ( *users.ItemContactFoldersItemChildFoldersRequestBuilder, *users.ItemContactFoldersItemChildFoldersRequestBuilderGetRequestConfiguration, + *graph.Service, error, ) { + service, err := c.service() + if err != nil { + return nil, nil, nil, err + } + fields := append([]string{"displayName", "parentFolderId"}, optionalFields...) ofcf, err := optionsForContactChildFolders(fields) if err != nil { - return nil, nil, errors.Wrapf(err, "options for contact child folders: %v", fields) + return nil, nil, nil, errors.Wrapf(err, "options for contact child folders: %v", fields) } - builder := gs.Client(). + builder := service.Client(). UsersById(userID). ContactFoldersById(baseDirID). ChildFolders() - return builder, ofcf, nil + return builder, ofcf, service, nil } // FetchContactIDsFromDirectory function that returns a list of all the m365IDs of the contacts // of the targeted directory -func FetchContactIDsFromDirectory( +func (c Client) FetchContactIDsFromDirectory( ctx context.Context, - gs graph.Servicer, user, directoryID, oldDelta string, ) ([]string, []string, DeltaUpdate, error) { + service, err := c.service() + if err != nil { + return nil, nil, DeltaUpdate{}, err + } + var ( errs *multierror.Error ids []string @@ -167,14 +175,14 @@ func FetchContactIDsFromDirectory( break } - builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, gs.Adapter()) + builder = users.NewItemContactFoldersItemContactsDeltaRequestBuilder(*nextLink, service.Adapter()) } return nil } if len(oldDelta) > 0 { - err := getIDs(users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, gs.Adapter())) + err := getIDs(users.NewItemContactFoldersItemContactsDeltaRequestBuilder(oldDelta, service.Adapter())) // note: happy path, not the error condition if err == nil { return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil() @@ -189,7 +197,7 @@ func FetchContactIDsFromDirectory( errs = nil } - builder := gs.Client(). + builder := service.Client(). UsersById(user). ContactFoldersById(directoryID). Contacts(). diff --git a/src/internal/connector/exchange/api/events.go b/src/internal/connector/exchange/api/events.go index dce5ea0fe..c2a0987ce 100644 --- a/src/internal/connector/exchange/api/events.go +++ b/src/internal/connector/exchange/api/events.go @@ -15,68 +15,82 @@ import ( // CreateCalendar makes an event Calendar with the name in the user's M365 exchange account // Reference: https://docs.microsoft.com/en-us/graph/api/user-post-calendars?view=graph-rest-1.0&tabs=go -func CreateCalendar(ctx context.Context, gs graph.Servicer, user, calendarName string) (models.Calendarable, error) { +func (c Client) CreateCalendar( + ctx context.Context, + user, calendarName string, +) (models.Calendarable, error) { requestbody := models.NewCalendar() requestbody.SetName(&calendarName) - return gs.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil) + return c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil) } // DeleteCalendar removes calendar from user's M365 account // Reference: https://docs.microsoft.com/en-us/graph/api/calendar-delete?view=graph-rest-1.0&tabs=go -func DeleteCalendar(ctx context.Context, gs graph.Servicer, user, calendarID string) error { - return gs.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) +func (c Client) DeleteCalendar( + ctx context.Context, + user, calendarID string, +) error { + return c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil) } // RetrieveEventDataForUser is a GraphRetrievalFunc that returns event data. -// Calendarable and attachment fields are omitted due to size -func RetrieveEventDataForUser( +func (c Client) RetrieveEventDataForUser( ctx context.Context, - gs graph.Servicer, user, m365ID string, ) (serialization.Parsable, error) { - return gs.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil) + return c.stable.Client().UsersById(user).EventsById(m365ID).Get(ctx, nil) } -func GetAllCalendarNamesForUser(ctx context.Context, gs graph.Servicer, user string) (serialization.Parsable, error) { +func (c Client) GetAllCalendarNamesForUser( + ctx context.Context, + user string, +) (serialization.Parsable, error) { options, err := optionsForCalendars([]string{"name", "owner"}) if err != nil { return nil, err } - return gs.Client().UsersById(user).Calendars().Get(ctx, options) + return c.stable.Client().UsersById(user).Calendars().Get(ctx, options) } // TODO: we want this to be the full handler, not only the builder. // but this halfway point minimizes changes for now. -func GetCalendarsBuilder( +func (c Client) GetCalendarsBuilder( ctx context.Context, - gs graph.Servicer, userID string, optionalFields ...string, ) ( *users.ItemCalendarsRequestBuilder, *users.ItemCalendarsRequestBuilderGetRequestConfiguration, + *graph.Service, error, ) { - ofcf, err := optionsForCalendars(optionalFields) + service, err := c.service() if err != nil { - return nil, nil, errors.Wrapf(err, "options for event calendars: %v", optionalFields) + return nil, nil, nil, err } - builder := gs.Client(). - UsersById(userID). - Calendars() + ofcf, err := optionsForCalendars(optionalFields) + if err != nil { + return nil, nil, nil, errors.Wrapf(err, "options for event calendars: %v", optionalFields) + } - return builder, ofcf, nil + builder := service.Client().UsersById(userID).Calendars() + + return builder, ofcf, service, nil } // FetchEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar. -func FetchEventIDsFromCalendar( +func (c Client) FetchEventIDsFromCalendar( ctx context.Context, - gs graph.Servicer, user, calendarID, oldDelta string, ) ([]string, []string, DeltaUpdate, error) { + service, err := c.service() + if err != nil { + return nil, nil, DeltaUpdate{}, err + } + var ( errs *multierror.Error ids []string @@ -87,10 +101,7 @@ func FetchEventIDsFromCalendar( return nil, nil, DeltaUpdate{}, err } - builder := gs.Client(). - UsersById(user). - CalendarsById(calendarID). - Events() + builder := service.Client().UsersById(user).CalendarsById(calendarID).Events() for { resp, err := builder.Get(ctx, options) @@ -121,7 +132,7 @@ func FetchEventIDsFromCalendar( break } - builder = users.NewItemCalendarsItemEventsRequestBuilder(*nextLink, gs.Adapter()) + builder = users.NewItemCalendarsItemEventsRequestBuilder(*nextLink, service.Adapter()) } // Events don't have a delta endpoint so just return an empty string. diff --git a/src/internal/connector/exchange/api/mail.go b/src/internal/connector/exchange/api/mail.go index f7d31fedc..e4a836c83 100644 --- a/src/internal/connector/exchange/api/mail.go +++ b/src/internal/connector/exchange/api/mail.go @@ -15,26 +15,34 @@ import ( // CreateMailFolder makes a mail folder iff a folder of the same name does not exist // Reference: https://docs.microsoft.com/en-us/graph/api/user-post-mailfolders?view=graph-rest-1.0&tabs=http -func CreateMailFolder(ctx context.Context, gs graph.Servicer, user, folder string) (models.MailFolderable, error) { - isHidden := false - requestBody := models.NewMailFolder() - requestBody.SetDisplayName(&folder) - requestBody.SetIsHidden(&isHidden) - - return gs.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil) -} - -func CreateMailFolderWithParent( +func (c Client) CreateMailFolder( ctx context.Context, - gs graph.Servicer, - user, folder, parentID string, + user, folder string, ) (models.MailFolderable, error) { isHidden := false requestBody := models.NewMailFolder() requestBody.SetDisplayName(&folder) requestBody.SetIsHidden(&isHidden) - return gs.Client(). + return c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil) +} + +func (c Client) CreateMailFolderWithParent( + ctx context.Context, + user, folder, parentID string, +) (models.MailFolderable, error) { + service, err := c.service() + if err != nil { + return nil, err + } + + isHidden := false + requestBody := models.NewMailFolder() + requestBody.SetDisplayName(&folder) + requestBody.SetIsHidden(&isHidden) + + return service. + Client(). UsersById(user). MailFoldersById(parentID). ChildFolders(). @@ -43,18 +51,19 @@ func CreateMailFolderWithParent( // DeleteMailFolder removes a mail folder with the corresponding M365 ID from the user's M365 Exchange account // Reference: https://docs.microsoft.com/en-us/graph/api/mailfolder-delete?view=graph-rest-1.0&tabs=http -func DeleteMailFolder(ctx context.Context, gs graph.Servicer, user, folderID string) error { - return gs.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil) +func (c Client) DeleteMailFolder( + ctx context.Context, + user, folderID string, +) error { + return c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil) } // RetrieveMessageDataForUser is a GraphRetrievalFunc that returns message data. -// Attachment field is omitted due to size. -func RetrieveMessageDataForUser( +func (c Client) RetrieveMessageDataForUser( ctx context.Context, - gs graph.Servicer, user, m365ID string, ) (serialization.Parsable, error) { - return gs.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil) + return c.stable.Client().UsersById(user).MessagesById(m365ID).Get(ctx, nil) } // GetMailFoldersBuilder retrieves all of the users current mail folders. @@ -62,20 +71,29 @@ func RetrieveMessageDataForUser( // not contain historical data. // TODO: we want this to be the full handler, not only the builder. // but this halfway point minimizes changes for now. -func GetAllMailFoldersBuilder( +func (c Client) GetAllMailFoldersBuilder( ctx context.Context, - gs graph.Servicer, userID string, -) *users.ItemMailFoldersDeltaRequestBuilder { - return gs.Client(). +) ( + *users.ItemMailFoldersDeltaRequestBuilder, + *graph.Service, + error, +) { + service, err := c.service() + if err != nil { + return nil, nil, err + } + + builder := service.Client(). UsersById(userID). MailFolders(). Delta() + + return builder, service, nil } -func GetMailFolderByID( +func (c Client) GetMailFolderByID( ctx context.Context, - gs graph.Servicer, userID, dirID string, optionalFields ...string, ) (models.MailFolderable, error) { @@ -84,19 +102,20 @@ func GetMailFolderByID( return nil, errors.Wrapf(err, "options for mail folder: %v", optionalFields) } - return gs.Client(). - UsersById(userID). - MailFoldersById(dirID). - Get(ctx, ofmf) + return c.stable.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf) } // FetchMessageIDsFromDirectory function that returns a list of all the m365IDs of the exchange.Mail // of the targeted directory -func FetchMessageIDsFromDirectory( +func (c Client) FetchMessageIDsFromDirectory( ctx context.Context, - gs graph.Servicer, user, directoryID, oldDelta string, ) ([]string, []string, DeltaUpdate, error) { + service, err := c.service() + if err != nil { + return nil, nil, DeltaUpdate{}, err + } + var ( errs *multierror.Error ids []string @@ -153,14 +172,14 @@ func FetchMessageIDsFromDirectory( break } - builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, gs.Adapter()) + builder = users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(*nextLink, service.Adapter()) } return nil } if len(oldDelta) > 0 { - err := getIDs(users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, gs.Adapter())) + err := getIDs(users.NewItemMailFoldersItemMessagesDeltaRequestBuilder(oldDelta, service.Adapter())) // note: happy path, not the error condition if err == nil { return ids, removedIDs, DeltaUpdate{deltaURL, false}, errs.ErrorOrNil() @@ -175,11 +194,7 @@ func FetchMessageIDsFromDirectory( errs = nil } - builder := gs.Client(). - UsersById(user). - MailFoldersById(directoryID). - Messages(). - Delta() + builder := service.Client().UsersById(user).MailFoldersById(directoryID).Messages().Delta() if err := getIDs(builder); err != nil { return nil, nil, DeltaUpdate{}, err diff --git a/src/internal/connector/exchange/contact_folder_cache.go b/src/internal/connector/exchange/contact_folder_cache.go index 29f13e3f5..35afe8e63 100644 --- a/src/internal/connector/exchange/contact_folder_cache.go +++ b/src/internal/connector/exchange/contact_folder_cache.go @@ -16,7 +16,7 @@ var _ graph.ContainerResolver = &contactFolderCache{} type contactFolderCache struct { *containerResolver - gs graph.Servicer + ac api.Client userID string } @@ -25,7 +25,7 @@ func (cfc *contactFolderCache) populateContactRoot( directoryID string, baseContainerPath []string, ) error { - f, err := api.GetContactFolderByID(ctx, cfc.gs, cfc.userID, directoryID) + f, err := cfc.ac.GetContactFolderByID(ctx, cfc.userID, directoryID) if err != nil { return errors.Wrapf( err, @@ -56,9 +56,8 @@ func (cfc *contactFolderCache) Populate( var errs error - builder, options, err := api.GetContactChildFoldersBuilder( + builder, options, servicer, err := cfc.ac.GetContactChildFoldersBuilder( ctx, - cfc.gs, cfc.userID, baseID) if err != nil { @@ -97,7 +96,7 @@ func (cfc *contactFolderCache) Populate( break } - builder = msuser.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), cfc.gs.Adapter()) + builder = msuser.NewItemContactFoldersItemChildFoldersRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter()) } if err := cfc.populatePaths(ctx); err != nil { diff --git a/src/internal/connector/exchange/container_resolver_test.go b/src/internal/connector/exchange/container_resolver_test.go index 8099a7843..f2250c439 100644 --- a/src/internal/connector/exchange/container_resolver_test.go +++ b/src/internal/connector/exchange/container_resolver_test.go @@ -494,9 +494,6 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { m365, err := a.M365Config() require.NoError(suite.T(), err) - connector, err := createService(m365) - require.NoError(suite.T(), err) - var ( user = tester.M365UserID(suite.T()) directoryCaches = make(map[path.CategoryType]graph.ContainerResolver) @@ -596,11 +593,10 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { suite.T().Run(test.name, func(t *testing.T) { folderID, err := CreateContainerDestinaion( ctx, - connector, + m365, test.pathFunc1(t), folderName, - directoryCaches, - ) + directoryCaches) require.NoError(t, err) resolver := directoryCaches[test.category] @@ -609,11 +605,10 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() { secondID, err := CreateContainerDestinaion( ctx, - connector, + m365, test.pathFunc2(t), folderName, - directoryCaches, - ) + directoryCaches) require.NoError(t, err) _, err = resolver.IDToPath(ctx, secondID) diff --git a/src/internal/connector/exchange/event_calendar_cache.go b/src/internal/connector/exchange/event_calendar_cache.go index 362fa902a..2ac251d05 100644 --- a/src/internal/connector/exchange/event_calendar_cache.go +++ b/src/internal/connector/exchange/event_calendar_cache.go @@ -16,7 +16,7 @@ var _ graph.ContainerResolver = &eventCalendarCache{} type eventCalendarCache struct { *containerResolver - gs graph.Servicer + ac api.Client userID string } @@ -32,7 +32,7 @@ func (ecc *eventCalendarCache) Populate( ecc.containerResolver = newContainerResolver() } - builder, options, err := api.GetCalendarsBuilder(ctx, ecc.gs, ecc.userID, "name") + builder, options, servicer, err := ecc.ac.GetCalendarsBuilder(ctx, ecc.userID, "name") if err != nil { return err } @@ -67,7 +67,7 @@ func (ecc *eventCalendarCache) Populate( break } - builder = msuser.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), ecc.gs.Adapter()) + builder = msuser.NewItemCalendarsRequestBuilder(*resp.GetOdataNextLink(), servicer.Adapter()) } for _, container := range directories { diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index b5aca9064..5b1b05b81 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -59,6 +59,7 @@ type Collection struct { // service - client/adapter pair used to access M365 back store service graph.Servicer + ac api.Client category path.CategoryType statusUpdater support.StatusUpdater @@ -88,12 +89,14 @@ func NewCollection( user string, curr, prev path.Path, category path.CategoryType, + ac api.Client, service graph.Servicer, statusUpdater support.StatusUpdater, ctrlOpts control.Options, doNotMergeItems bool, ) Collection { collection := Collection{ + ac: ac, category: category, ctrl: ctrlOpts, data: make(chan data.Stream, collectionChannelBufferSize), @@ -136,14 +139,14 @@ func (col *Collection) Items() <-chan data.Stream { // GetQueryAndSerializeFunc helper function that returns the two functions functions // required to convert M365 identifier into a byte array filled with the serialized data -func GetQueryAndSerializeFunc(category path.CategoryType) (api.GraphRetrievalFunc, GraphSerializeFunc) { +func GetQueryAndSerializeFunc(ac api.Client, category path.CategoryType) (api.GraphRetrievalFunc, GraphSerializeFunc) { switch category { case path.ContactsCategory: - return api.RetrieveContactDataForUser, serializeAndStreamContact + return ac.RetrieveContactDataForUser, serializeAndStreamContact case path.EventsCategory: - return api.RetrieveEventDataForUser, serializeAndStreamEvent + return ac.RetrieveEventDataForUser, serializeAndStreamEvent case path.EmailCategory: - return api.RetrieveMessageDataForUser, serializeAndStreamMessage + return ac.RetrieveMessageDataForUser, serializeAndStreamMessage // Unsupported options returns nil, nil default: return nil, nil @@ -204,7 +207,7 @@ func (col *Collection) streamItems(ctx context.Context) { // get QueryBasedonIdentifier // verify that it is the correct type in called function // serializationFunction - query, serializeFunc := GetQueryAndSerializeFunc(col.category) + query, serializeFunc := GetQueryAndSerializeFunc(col.ac, col.category) if query == nil { errs = fmt.Errorf("unrecognized collection type: %s", col.category) return @@ -263,7 +266,7 @@ func (col *Collection) streamItems(ctx context.Context) { ) for i := 1; i <= numberOfRetries; i++ { - response, err = query(ctx, col.service, user, id) + response, err = query(ctx, user, id) if err == nil { break } diff --git a/src/internal/connector/exchange/exchange_data_collection_test.go b/src/internal/connector/exchange/exchange_data_collection_test.go index 62b1c48f7..79efc59f4 100644 --- a/src/internal/connector/exchange/exchange_data_collection_test.go +++ b/src/internal/connector/exchange/exchange_data_collection_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" @@ -136,7 +137,7 @@ func (suite *ExchangeDataCollectionSuite) TestNewCollection_state() { c := NewCollection( "u", test.curr, test.prev, - 0, nil, nil, control.Options{}, + 0, api.Client{}, nil, nil, control.Options{}, false) assert.Equal(t, test.expect, c.State()) }) diff --git a/src/internal/connector/exchange/folder_resolver_test.go b/src/internal/connector/exchange/folder_resolver_test.go index 95f2ee305..68c17e408 100644 --- a/src/internal/connector/exchange/folder_resolver_test.go +++ b/src/internal/connector/exchange/folder_resolver_test.go @@ -7,13 +7,15 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" ) type CacheResolverSuite struct { suite.Suite - gs graph.Servicer + credentials account.M365Config } func TestCacheResolverIntegrationSuite(t *testing.T) { @@ -35,27 +37,27 @@ func (suite *CacheResolverSuite) SetupSuite() { m365, err := a.M365Config() require.NoError(t, err) - service, err := createService(m365) - require.NoError(t, err) - - suite.gs = service + suite.credentials = m365 } func (suite *CacheResolverSuite) TestPopulate() { ctx, flush := tester.NewContext() defer flush() + ac, err := api.NewClient(suite.credentials) + require.NoError(suite.T(), err) + eventFunc := func(t *testing.T) graph.ContainerResolver { return &eventCalendarCache{ userID: tester.M365UserID(t), - gs: suite.gs, + ac: ac, } } contactFunc := func(t *testing.T) graph.ContainerResolver { return &contactFolderCache{ userID: tester.M365UserID(t), - gs: suite.gs, + ac: ac, } } diff --git a/src/internal/connector/exchange/iterators_test.go b/src/internal/connector/exchange/iterators_test.go index d3f2c8621..2ae9abf2b 100644 --- a/src/internal/connector/exchange/iterators_test.go +++ b/src/internal/connector/exchange/iterators_test.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -84,7 +85,7 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() { tests := []struct { name string - queryFunc api.GraphQuery + queryFunc func(*testing.T, account.M365Config) api.GraphQuery scope selectors.ExchangeScope iterativeFunction func( container map[string]graph.Container, @@ -93,14 +94,22 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() { transformer absser.ParsableFactory }{ { - name: "Contacts Iterative Check", - queryFunc: api.GetAllContactFolderNamesForUser, + name: "Contacts Iterative Check", + queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery { + ac, err := api.NewClient(amc) + require.NoError(t, err) + return ac.GetAllContactFolderNamesForUser + }, transformer: models.CreateContactFolderCollectionResponseFromDiscriminatorValue, iterativeFunction: IterativeCollectContactContainers, }, { - name: "Events Iterative Check", - queryFunc: api.GetAllCalendarNamesForUser, + name: "Events Iterative Check", + queryFunc: func(t *testing.T, amc account.M365Config) api.GraphQuery { + ac, err := api.NewClient(amc) + require.NoError(t, err) + return ac.GetAllCalendarNamesForUser + }, transformer: models.CreateCalendarCollectionResponseFromDiscriminatorValue, iterativeFunction: IterativeCollectCalendarContainers, }, @@ -114,7 +123,7 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() { service, err := createService(m365) require.NoError(t, err) - response, err := test.queryFunc(ctx, service, userID) + response, err := test.queryFunc(t, m365)(ctx, userID) require.NoError(t, err) // Iterator Creation diff --git a/src/internal/connector/exchange/mail_folder_cache.go b/src/internal/connector/exchange/mail_folder_cache.go index 0f09a74f5..11bc6c230 100644 --- a/src/internal/connector/exchange/mail_folder_cache.go +++ b/src/internal/connector/exchange/mail_folder_cache.go @@ -20,7 +20,7 @@ var _ graph.ContainerResolver = &mailFolderCache{} // nameLookup map: Key: DisplayName Value: ID type mailFolderCache struct { *containerResolver - gs graph.Servicer + ac api.Client userID string } @@ -35,7 +35,7 @@ func (mc *mailFolderCache) populateMailRoot( for _, fldr := range []string{rootFolderAlias, DefaultMailFolder} { var directory string - f, err := api.GetMailFolderByID(ctx, mc.gs, mc.userID, fldr, "displayName", "parentFolderId") + f, err := mc.ac.GetMailFolderByID(ctx, mc.userID, fldr, "displayName", "parentFolderId") if err != nil { return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err)) } @@ -67,7 +67,10 @@ func (mc *mailFolderCache) Populate( return err } - query := api.GetAllMailFoldersBuilder(ctx, mc.gs, mc.userID) + query, servicer, err := mc.ac.GetAllMailFoldersBuilder(ctx, mc.userID) + if err != nil { + return err + } var errs *multierror.Error @@ -94,7 +97,7 @@ func (mc *mailFolderCache) Populate( break } - query = msfolderdelta.NewItemMailFoldersDeltaRequestBuilder(*link, mc.gs.Adapter()) + query = msfolderdelta.NewItemMailFoldersDeltaRequestBuilder(*link, servicer.Adapter()) } if err := mc.populatePaths(ctx); err != nil { diff --git a/src/internal/connector/exchange/mail_folder_cache_test.go b/src/internal/connector/exchange/mail_folder_cache_test.go index 1d3c01e75..3cd58be91 100644 --- a/src/internal/connector/exchange/mail_folder_cache_test.go +++ b/src/internal/connector/exchange/mail_folder_cache_test.go @@ -8,8 +8,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" ) const ( @@ -26,7 +27,7 @@ const ( type MailFolderCacheIntegrationSuite struct { suite.Suite - gs graph.Servicer + credentials account.M365Config } func TestMailFolderCacheIntegrationSuite(t *testing.T) { @@ -48,10 +49,7 @@ func (suite *MailFolderCacheIntegrationSuite) SetupSuite() { m365, err := a.M365Config() require.NoError(t, err) - service, err := createService(m365) - require.NoError(t, err) - - suite.gs = service + suite.credentials = m365 } func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { @@ -83,9 +81,12 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() { for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { + ac, err := api.NewClient(suite.credentials) + require.NoError(t, err) + mfc := mailFolderCache{ userID: userID, - gs: suite.gs, + ac: ac, } require.NoError(t, mfc.Populate(ctx, test.root, test.path...)) diff --git a/src/internal/connector/exchange/restore_test.go b/src/internal/connector/exchange/restore_test.go index 6a58d400a..3b739c6bb 100644 --- a/src/internal/connector/exchange/restore_test.go +++ b/src/internal/connector/exchange/restore_test.go @@ -15,13 +15,16 @@ import ( "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" ) type ExchangeRestoreSuite struct { suite.Suite - gs graph.Servicer + gs graph.Servicer + credentials account.M365Config + ac api.Client } func TestExchangeRestoreSuite(t *testing.T) { @@ -41,10 +44,11 @@ func (suite *ExchangeRestoreSuite) SetupSuite() { m365, err := a.M365Config() require.NoError(t, err) - adpt, err := graph.CreateAdapter( - m365.AzureTenantID, - m365.AzureClientID, - m365.AzureClientSecret) + suite.credentials = m365 + suite.ac, err = api.NewClient(m365) + require.NoError(t, err) + + adpt, err := graph.CreateAdapter(m365.AzureTenantID, m365.AzureClientID, m365.AzureClientSecret) require.NoError(t, err) suite.gs = graph.NewService(adpt) @@ -65,18 +69,19 @@ func (suite *ExchangeRestoreSuite) TestRestoreContact() { folderName = "TestRestoreContact: " + common.FormatSimpleDateTime(now) ) - aFolder, err := api.CreateContactFolder(ctx, suite.gs, userID, folderName) + aFolder, err := suite.ac.CreateContactFolder(ctx, userID, folderName) require.NoError(t, err) folderID := *aFolder.GetId() defer func() { // Remove the folder containing contact prior to exiting test - err = api.DeleteContactFolder(ctx, suite.gs, userID, folderID) + err = suite.ac.DeleteContactFolder(ctx, userID, folderID) assert.NoError(t, err) }() - info, err := RestoreExchangeContact(ctx, + info, err := RestoreExchangeContact( + ctx, mockconnector.GetMockContactBytes("Corso TestContact"), suite.gs, control.Copy, @@ -98,14 +103,14 @@ func (suite *ExchangeRestoreSuite) TestRestoreEvent() { name = "TestRestoreEvent: " + common.FormatSimpleDateTime(time.Now()) ) - calendar, err := api.CreateCalendar(ctx, suite.gs, userID, name) + calendar, err := suite.ac.CreateCalendar(ctx, userID, name) require.NoError(t, err) calendarID := *calendar.GetId() defer func() { // Removes calendar containing events created during the test - err = api.DeleteCalendar(ctx, suite.gs, userID, calendarID) + err = suite.ac.DeleteCalendar(ctx, userID, calendarID) assert.NoError(t, err) }() @@ -134,17 +139,17 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name string bytes []byte category path.CategoryType - cleanupFunc func(context.Context, graph.Servicer, string, string) error + cleanupFunc func(context.Context, string, string) error destination func(*testing.T, context.Context) string }{ { name: "Test Mail", bytes: mockconnector.GetMockMessageBytes("Restore Exchange Object"), category: path.EmailCategory, - cleanupFunc: api.DeleteMailFolder, + cleanupFunc: suite.ac.DeleteMailFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -154,10 +159,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Mail: One Direct Attachment", bytes: mockconnector.GetMockMessageWithDirectAttachment("Restore 1 Attachment"), category: path.EmailCategory, - cleanupFunc: api.DeleteMailFolder, + cleanupFunc: suite.ac.DeleteMailFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -167,10 +172,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Mail: One Large Attachment", bytes: mockconnector.GetMockMessageWithLargeAttachment("Restore Large Attachment"), category: path.EmailCategory, - cleanupFunc: api.DeleteMailFolder, + cleanupFunc: suite.ac.DeleteMailFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -180,10 +185,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Mail: Two Attachments", bytes: mockconnector.GetMockMessageWithTwoAttachments("Restore 2 Attachments"), category: path.EmailCategory, - cleanupFunc: api.DeleteMailFolder, + cleanupFunc: suite.ac.DeleteMailFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -193,10 +198,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Mail: Reference(OneDrive) Attachment", bytes: mockconnector.GetMessageWithOneDriveAttachment("Restore Reference(OneDrive) Attachment"), category: path.EmailCategory, - cleanupFunc: api.DeleteMailFolder, + cleanupFunc: suite.ac.DeleteMailFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateMailFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateMailFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -207,10 +212,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Contact", bytes: mockconnector.GetMockContactBytes("Test_Omega"), category: path.ContactsCategory, - cleanupFunc: api.DeleteContactFolder, + cleanupFunc: suite.ac.DeleteContactFolder, destination: func(t *testing.T, ctx context.Context) string { folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now) - folder, err := api.CreateContactFolder(ctx, suite.gs, userID, folderName) + folder, err := suite.ac.CreateContactFolder(ctx, userID, folderName) require.NoError(t, err) return *folder.GetId() @@ -220,10 +225,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Events", bytes: mockconnector.GetDefaultMockEventBytes("Restored Event Object"), category: path.EventsCategory, - cleanupFunc: api.DeleteCalendar, + cleanupFunc: suite.ac.DeleteCalendar, destination: func(t *testing.T, ctx context.Context) string { calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now) - calendar, err := api.CreateCalendar(ctx, suite.gs, userID, calendarName) + calendar, err := suite.ac.CreateCalendar(ctx, userID, calendarName) require.NoError(t, err) return *calendar.GetId() @@ -233,10 +238,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { name: "Test Event with Attachment", bytes: mockconnector.GetMockEventWithAttachment("Restored Event Attachment"), category: path.EventsCategory, - cleanupFunc: api.DeleteCalendar, + cleanupFunc: suite.ac.DeleteCalendar, destination: func(t *testing.T, ctx context.Context) string { calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now) - calendar, err := api.CreateCalendar(ctx, suite.gs, userID, calendarName) + calendar, err := suite.ac.CreateCalendar(ctx, userID, calendarName) require.NoError(t, err) return *calendar.GetId() @@ -262,7 +267,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() { assert.NoError(t, err, support.ConnectorStackErrorTrace(err)) assert.NotNil(t, info, "item info is populated") - cleanupError := test.cleanupFunc(ctx, service, userID, destination) + cleanupError := test.cleanupFunc(ctx, userID, destination) assert.NoError(t, cleanupError) }) } diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index 47d3736df..579a7915c 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/connector/exchange/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/path" @@ -36,11 +37,11 @@ func PopulateExchangeContainerResolver( qp graph.QueryParams, ) (graph.ContainerResolver, error) { var ( - res graph.ContainerResolver - cacheRoot string - service, err = createService(qp.Credentials) + res graph.ContainerResolver + cacheRoot string ) + ac, err := api.NewClient(qp.Credentials) if err != nil { return nil, err } @@ -49,21 +50,21 @@ func PopulateExchangeContainerResolver( case path.EmailCategory: res = &mailFolderCache{ userID: qp.ResourceOwner, - gs: service, + ac: ac, } cacheRoot = rootFolderAlias case path.ContactsCategory: res = &contactFolderCache{ userID: qp.ResourceOwner, - gs: service, + ac: ac, } cacheRoot = DefaultContactFolder case path.EventsCategory: res = &eventCalendarCache{ userID: qp.ResourceOwner, - gs: service, + ac: ac, } cacheRoot = DefaultCalendar diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index c0b54ef91..e8074ae12 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -43,7 +43,13 @@ func filterContainersAndFillCollections( tombstones = makeTombstones(dps) ) - getJobs, err := getFetchIDFunc(qp.Category) + // TODO(rkeepers): pass in the api client instead of generating it here. + ac, err := api.NewClient(qp.Credentials) + if err != nil { + return err + } + + getJobs, err := getFetchIDFunc(ac, qp.Category) if err != nil { return support.WrapAndAppend(qp.ResourceOwner, err, errs) } @@ -85,7 +91,7 @@ func filterContainersAndFillCollections( } } - added, removed, newDelta, err := getJobs(ctx, service, qp.ResourceOwner, cID, prevDelta) + added, removed, newDelta, err := getJobs(ctx, qp.ResourceOwner, cID, prevDelta) if err != nil { if graph.IsErrDeletedInFlight(err) == nil { errs = support.WrapAndAppend(qp.ResourceOwner, err, errs) @@ -110,11 +116,11 @@ func filterContainersAndFillCollections( currPath, prevPath, scope.Category().PathType(), + ac, service, statusUpdater, ctrlOpts, - newDelta.Reset, - ) + newDelta.Reset) collections[cID] = &edc edc.added = append(edc.added, added...) @@ -160,11 +166,11 @@ func filterContainersAndFillCollections( nil, // marks the collection as deleted prevPath, scope.Category().PathType(), + ac, service, statusUpdater, ctrlOpts, - false, - ) + false) collections[id] = &edc } @@ -267,18 +273,17 @@ func IterativeCollectCalendarContainers( // container supports fetching delta records. type FetchIDFunc func( ctx context.Context, - gs graph.Servicer, user, containerID, oldDeltaToken string, ) ([]string, []string, api.DeltaUpdate, error) -func getFetchIDFunc(category path.CategoryType) (FetchIDFunc, error) { +func getFetchIDFunc(ac api.Client, category path.CategoryType) (FetchIDFunc, error) { switch category { case path.EmailCategory: - return api.FetchMessageIDsFromDirectory, nil + return ac.FetchMessageIDsFromDirectory, nil case path.EventsCategory: - return api.FetchEventIDsFromCalendar, nil + return ac.FetchEventIDsFromCalendar, nil case path.ContactsCategory: - return api.FetchContactIDsFromDirectory, nil + return ac.FetchContactIDsFromDirectory, nil default: return nil, fmt.Errorf("category %s not supported by getFetchIDFunc", category) } diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 2c201bdf5..183bd50bf 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/data" D "github.com/alcionai/corso/src/internal/diagnostics" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" @@ -284,6 +285,7 @@ func SendMailToBackStore( // @param dest: container destination to M365 func RestoreExchangeDataCollections( ctx context.Context, + creds account.M365Config, gs graph.Servicer, dest control.RestoreDestination, dcs []data.Collection, @@ -313,7 +315,7 @@ func RestoreExchangeDataCollections( containerID, err := CreateContainerDestinaion( ctx, - gs, + creds, dc.FullPath(), dest.ContainerName, userCaches) @@ -431,7 +433,7 @@ func restoreCollection( // @ returns the container ID of the new destination container. func CreateContainerDestinaion( ctx context.Context, - gs graph.Servicer, + creds account.M365Config, directory path.Path, destination string, caches map[path.CategoryType]graph.ContainerResolver, @@ -444,12 +446,18 @@ func CreateContainerDestinaion( newPathFolders = append([]string{destination}, directory.Folders()...) ) + // TODO(rkeepers): pass the api client into this func, rather than generating one. + ac, err := api.NewClient(creds) + if err != nil { + return "", err + } + switch category { case path.EmailCategory: if directoryCache == nil { mfc := &mailFolderCache{ userID: user, - gs: gs, + ac: ac, } caches[category] = mfc @@ -459,16 +467,17 @@ func CreateContainerDestinaion( return establishMailRestoreLocation( ctx, + ac, newPathFolders, directoryCache, user, - gs, newCache) + case path.ContactsCategory: if directoryCache == nil { cfc := &contactFolderCache{ userID: user, - gs: gs, + ac: ac, } caches[category] = cfc newCache = true @@ -477,16 +486,17 @@ func CreateContainerDestinaion( return establishContactsRestoreLocation( ctx, + ac, newPathFolders, directoryCache, user, - gs, newCache) + case path.EventsCategory: if directoryCache == nil { ecc := &eventCalendarCache{ userID: user, - gs: gs, + ac: ac, } caches[category] = ecc newCache = true @@ -495,10 +505,10 @@ func CreateContainerDestinaion( return establishEventsRestoreLocation( ctx, + ac, newPathFolders, directoryCache, user, - gs, newCache, ) default: @@ -513,10 +523,10 @@ func CreateContainerDestinaion( // @param isNewCache identifies if the cache is created and not populated func establishMailRestoreLocation( ctx context.Context, + ac api.Client, folders []string, mfc graph.ContainerResolver, user string, - service graph.Servicer, isNewCache bool, ) (string, error) { // Process starts with the root folder in order to recreate @@ -533,12 +543,7 @@ func establishMailRestoreLocation( continue } - temp, err := api.CreateMailFolderWithParent( - ctx, - service, - user, - folder, - folderID) + temp, err := ac.CreateMailFolderWithParent(ctx, user, folder, folderID) if err != nil { // Should only error if cache malfunctions or incorrect parameters return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err)) @@ -574,10 +579,10 @@ func establishMailRestoreLocation( // @param isNewCache bool representation of whether Populate function needs to be run func establishContactsRestoreLocation( ctx context.Context, + ac api.Client, folders []string, cfc graph.ContainerResolver, user string, - gs graph.Servicer, isNewCache bool, ) (string, error) { cached, ok := cfc.PathInCache(folders[0]) @@ -585,7 +590,7 @@ func establishContactsRestoreLocation( return cached, nil } - temp, err := api.CreateContactFolder(ctx, gs, user, folders[0]) + temp, err := ac.CreateContactFolder(ctx, user, folders[0]) if err != nil { return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } @@ -607,10 +612,10 @@ func establishContactsRestoreLocation( func establishEventsRestoreLocation( ctx context.Context, + ac api.Client, folders []string, ecc graph.ContainerResolver, // eventCalendarCache user string, - gs graph.Servicer, isNewCache bool, ) (string, error) { cached, ok := ecc.PathInCache(folders[0]) @@ -618,7 +623,7 @@ func establishEventsRestoreLocation( return cached, nil } - temp, err := api.CreateCalendar(ctx, gs, user, folders[0]) + temp, err := ac.CreateCalendar(ctx, user, folders[0]) if err != nil { return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index aeaf8193b..e047a017b 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -252,6 +252,7 @@ func (gc *GraphConnector) UnionSiteIDsAndWebURLs(ctx context.Context, ids, urls // SideEffect: gc.status is updated at the completion of operation func (gc *GraphConnector) RestoreDataCollections( ctx context.Context, + acct account.Account, selector selectors.Selector, dest control.RestoreDestination, dcs []data.Collection, @@ -265,9 +266,14 @@ func (gc *GraphConnector) RestoreDataCollections( deets = &details.Builder{} ) + creds, err := acct.M365Config() + if err != nil { + return nil, errors.Wrap(err, "malformed azure credentials") + } + switch selector.Service { case selectors.ServiceExchange: - status, err = exchange.RestoreExchangeDataCollections(ctx, gc.Service, dest, dcs, deets) + status, err = exchange.RestoreExchangeDataCollections(ctx, creds, gc.Service, dest, dcs, deets) case selectors.ServiceOneDrive: status, err = onedrive.RestoreCollections(ctx, gc.Service, dest, dcs, deets) case selectors.ServiceSharePoint: diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index afefcfad8..f16846acc 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -168,18 +168,20 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_ErrorChecking() } func (suite *DisconnectedGraphConnectorSuite) TestRestoreFailsBadService() { - t := suite.T() - ctx, flush := tester.NewContext() defer flush() - gc := GraphConnector{wg: &sync.WaitGroup{}} - sel := selectors.Selector{ - Service: selectors.ServiceUnknown, - } - dest := tester.DefaultTestRestoreDestination() + var ( + t = suite.T() + acct = tester.NewM365Account(t) + dest = tester.DefaultTestRestoreDestination() + gc = GraphConnector{wg: &sync.WaitGroup{}} + sel = selectors.Selector{ + Service: selectors.ServiceUnknown, + } + ) - deets, err := gc.RestoreDataCollections(ctx, sel, dest, nil) + deets, err := gc.RestoreDataCollections(ctx, acct, sel, dest, nil) assert.Error(t, err) assert.NotNil(t, deets) diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index c788ca07b..1b3f9e925 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -17,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -135,6 +136,7 @@ type GraphConnectorIntegrationSuite struct { suite.Suite connector *GraphConnector user string + acct account.Account } func TestGraphConnectorIntegrationSuite(t *testing.T) { @@ -155,6 +157,7 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() { suite.connector = loadConnector(ctx, suite.T(), Users) suite.user = tester.M365UserID(suite.T()) + suite.acct = tester.NewM365Account(suite.T()) tester.LogTimeOfTest(suite.T()) } @@ -265,7 +268,12 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { ctx, flush := tester.NewContext() defer flush() - deets, err := suite.connector.RestoreDataCollections(ctx, test.sel, dest, test.col) + deets, err := suite.connector.RestoreDataCollections( + ctx, + suite.acct, + test.sel, + dest, + test.col) require.NoError(t, err) assert.NotNil(t, deets) @@ -308,6 +316,7 @@ func mustGetDefaultDriveID( func runRestoreBackupTest( t *testing.T, + acct account.Account, test restoreBackupInfo, tenant string, users []string, @@ -349,7 +358,12 @@ func runRestoreBackupTest( restoreGC := loadConnector(ctx, t, test.resource) restoreSel := getSelectorWith(test.service) - deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + deets, err := restoreGC.RestoreDataCollections( + ctx, + acct, + restoreSel, + dest, + collections) require.NoError(t, err) assert.NotNil(t, deets) @@ -724,7 +738,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - runRestoreBackupTest(t, test, suite.connector.tenant, []string{suite.user}) + runRestoreBackupTest(t, suite.acct, test, suite.connector.tenant, []string{suite.user}) }) } } @@ -833,7 +847,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames ) restoreGC := loadConnector(ctx, t, test.resource) - deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) + deets, err := restoreGC.RestoreDataCollections(ctx, suite.acct, restoreSel, dest, collections) require.NoError(t, err) require.NotNil(t, deets) @@ -977,7 +991,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiuserRestoreAndBackup() { for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - runRestoreBackupTest(t, test, suite.connector.tenant, users) + runRestoreBackupTest(t, suite.acct, test, suite.connector.tenant, users) }) } } diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index cdfaae9f0..05cfb1655 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -294,6 +294,7 @@ func generateContainerOfItems( ctx context.Context, gc *connector.GraphConnector, service path.ServiceType, + acct account.Account, cat path.CategoryType, sel selectors.Selector, tenantID, userID, destFldr string, @@ -330,7 +331,7 @@ func generateContainerOfItems( dest, collections) - deets, err := gc.RestoreDataCollections(ctx, sel, dest, dataColls) + deets, err := gc.RestoreDataCollections(ctx, acct, sel, dest, dataColls) require.NoError(t, err) return deets @@ -651,8 +652,13 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) require.NoError(t, err) - // generate 2 new containers with two items each. - // A third container will be introduced partway through the changes. + ac, err := api.NewClient(m365) + require.NoError(t, err) + + // generate 3 new folders with two items each. + // Only the first two folders will be part of the initial backup and + // incrementals. The third folder will be introduced partway through + // the changes. // This should be enough to cover most delta actions, since moving one // container into another generates a delta for both addition and deletion. type contDeets struct { @@ -705,6 +711,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ctx, gc, path.ExchangeService, + acct, category, selectors.NewExchangeRestore(users).Selector, m365.AzureTenantID, suite.user, destName, @@ -824,6 +831,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { ctx, gc, path.ExchangeService, + acct, category, selectors.NewExchangeRestore(users).Selector, m365.AzureTenantID, suite.user, container3, @@ -925,7 +933,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { switch category { case path.EmailCategory: - ids, _, _, err := api.FetchMessageIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "") + ids, _, _, err := ac.FetchMessageIDsFromDirectory(ctx, suite.user, containerID, "") require.NoError(t, err, "getting message ids") require.NotEmpty(t, ids, "message ids in folder") @@ -933,7 +941,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() { require.NoError(t, err, "deleting email item: %s", support.ConnectorStackErrorTrace(err)) case path.ContactsCategory: - ids, _, _, err := api.FetchContactIDsFromDirectory(ctx, gc.Service, suite.user, containerID, "") + ids, _, _, err := ac.FetchContactIDsFromDirectory(ctx, suite.user, containerID, "") require.NoError(t, err, "getting contact ids") require.NotEmpty(t, ids, "contact ids in folder") diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index c532111a3..80c7b986a 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -188,7 +188,12 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De defer closer() defer close(restoreComplete) - restoreDetails, err = gc.RestoreDataCollections(ctx, op.Selectors, op.Destination, dcs) + restoreDetails, err = gc.RestoreDataCollections( + ctx, + op.account, + op.Selectors, + op.Destination, + dcs) if err != nil { err = errors.Wrap(err, "restoring service data") opStats.writeErr = err