From 07a8d13d1eea55b6c330319859c0ddda06c5bb06 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 17 Nov 2022 17:31:41 -0700 Subject: [PATCH] adds tenant site lookup on new graph conn (#1540) ## Description Adds a lookup for all tenant site ids when a new graph connector is created. Also adds an enum to flag which resource set (users, sites, or all) that the connector should initialize. ## Type of change - [x] :sunflower: Feature ## Issue(s) * #1506 ## Test Plan - [x] :green_heart: E2E --- src/cmd/factory/factory.go | 2 +- src/cmd/getM365/getItem.go | 5 +- src/cmd/purge/purge.go | 2 +- .../connector/data_collections_test.go | 24 +- .../connector/exchange/cache_container.go | 9 +- .../exchange/exchange_service_test.go | 7 +- .../connector/exchange/exchange_vars.go | 11 +- .../connector/exchange/query_options.go | 8 +- .../connector/exchange/service_functions.go | 5 +- .../connector/exchange/service_query.go | 5 +- .../connector/graph/cache_container.go | 9 +- src/internal/connector/graph_connector.go | 230 ++++++++++-------- .../graph_connector_disconnected_test.go | 2 +- .../connector/graph_connector_helper_test.go | 6 +- .../connector/graph_connector_test.go | 55 +++-- src/internal/connector/sharepoint/queries.go | 21 ++ .../connector/support/m365Transform.go | 4 +- src/internal/operations/backup.go | 7 +- src/internal/operations/restore.go | 7 +- src/pkg/services/m365/m365.go | 6 +- 20 files changed, 250 insertions(+), 175 deletions(-) create mode 100644 src/internal/connector/sharepoint/queries.go diff --git a/src/cmd/factory/factory.go b/src/cmd/factory/factory.go index bbde061a2..361c7625c 100644 --- a/src/cmd/factory/factory.go +++ b/src/cmd/factory/factory.go @@ -176,7 +176,7 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon } // build a graph connector - gc, err := connector.NewGraphConnector(ctx, acct) + gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) if err != nil { return nil, "", errors.Wrap(err, "connecting to graph api") } diff --git a/src/cmd/getM365/getItem.go b/src/cmd/getM365/getItem.go index 111185ba8..9b2477006 100644 --- a/src/cmd/getM365/getItem.go +++ b/src/cmd/getM365/getItem.go @@ -44,7 +44,8 @@ var ( // Supports: // - exchange (contacts, email, and events) // Input: go run ./getItem.go --user -// --m365ID --category +// +// --m365ID --category func main() { ctx, _ := logger.SeedLevel(context.Background(), logger.Development) ctx = SetRootCmd(ctx, getCmd) @@ -170,7 +171,7 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) { return nil, Only(ctx, errors.Wrap(err, "finding m365 account details")) } - gc, err := connector.NewGraphConnector(ctx, acct) + gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) if err != nil { return nil, Only(ctx, errors.Wrap(err, "connecting to graph API")) } diff --git a/src/cmd/purge/purge.go b/src/cmd/purge/purge.go index 7a48a5d5b..9b293621f 100644 --- a/src/cmd/purge/purge.go +++ b/src/cmd/purge/purge.go @@ -255,7 +255,7 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) { } // build a graph connector - gc, err := connector.NewGraphConnector(ctx, acct) + gc, err := connector.NewGraphConnector(ctx, acct, connector.Users) if err != nil { return nil, Only(ctx, errors.Wrap(err, "connecting to graph api")) } diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 661ce934f..296fd3d4d 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -42,7 +42,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...) require.NoError(suite.T(), err) - suite.connector = loadConnector(ctx, suite.T()) + suite.connector = loadConnector(ctx, suite.T(), AllResources) suite.user = tester.M365UserID(suite.T()) suite.site = tester.M365SiteID(suite.T()) tester.LogTimeOfTest(suite.T()) @@ -58,7 +58,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection ctx, flush := tester.NewContext() defer flush() - connector := loadConnector(ctx, suite.T()) + connector := loadConnector(ctx, suite.T(), Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -123,7 +123,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestInvalidUserForDataColl defer flush() invalidUser := "foo@example.com" - connector := loadConnector(ctx, suite.T()) + connector := loadConnector(ctx, suite.T(), Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -162,7 +162,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti ctx, flush := tester.NewContext() defer flush() - connector := loadConnector(ctx, suite.T()) + connector := loadConnector(ctx, suite.T(), Users) tests := []struct { name string getSelector func(t *testing.T) selectors.Selector @@ -230,7 +230,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...) require.NoError(suite.T(), err) - suite.connector = loadConnector(ctx, suite.T()) + suite.connector = loadConnector(ctx, suite.T(), Users) suite.user = tester.M365UserID(suite.T()) suite.site = tester.M365SiteID(suite.T()) tester.LogTimeOfTest(suite.T()) @@ -263,7 +263,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailFetch() }, } - gc := loadConnector(ctx, t) + gc := loadConnector(ctx, t, Users) for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { @@ -292,7 +292,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestMailSerializ defer flush() t := suite.T() - connector := loadConnector(ctx, t) + connector := loadConnector(ctx, t, Users) sel := selectors.NewExchangeBackup() sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) collection, err := connector.createExchangeCollections(ctx, sel.Scopes()[0]) @@ -326,7 +326,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestContactSeria ctx, flush := tester.NewContext() defer flush() - connector := loadConnector(ctx, suite.T()) + connector := loadConnector(ctx, suite.T(), Users) tests := []struct { name string @@ -379,7 +379,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestEventsSerial ctx, flush := tester.NewContext() defer flush() - connector := loadConnector(ctx, suite.T()) + connector := loadConnector(ctx, suite.T(), Users) tests := []struct { name, expected string @@ -447,7 +447,7 @@ func (suite *ConnectorCreateExchangeCollectionIntegrationSuite) TestAccessOfInbo defer flush() t := suite.T() - connector := loadConnector(ctx, t) + connector := loadConnector(ctx, t, Users) sel := selectors.NewExchangeBackup() sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) scopes := sel.DiscreteScopes(connector.GetUsers()) @@ -488,7 +488,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...) require.NoError(suite.T(), err) - suite.connector = loadConnector(ctx, suite.T()) + suite.connector = loadConnector(ctx, suite.T(), Sites) suite.user = tester.M365UserID(suite.T()) tester.LogTimeOfTest(suite.T()) } @@ -502,7 +502,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar userID = tester.M365UserID(t) ) - gc := loadConnector(ctx, t) + gc := loadConnector(ctx, t, Sites) scope := selectors.NewSharePointBackup().Folders( []string{userID}, []string{"foo"}, diff --git a/src/internal/connector/exchange/cache_container.go b/src/internal/connector/exchange/cache_container.go index 1c0c12a7b..c0b884113 100644 --- a/src/internal/connector/exchange/cache_container.go +++ b/src/internal/connector/exchange/cache_container.go @@ -39,9 +39,9 @@ func checkRequiredValues(c graph.Container) error { return nil } -//====================================== +// ========================================= // cachedContainer Implementations -//====================== +// ========================================= var _ graph.CachedContainer = &cacheFolder{} @@ -50,9 +50,9 @@ type cacheFolder struct { p *path.Builder } -//========================================= +// ========================================= // Required Functions to satisfy interfaces -//===================================== +// ========================================= func (cf cacheFolder) Path() *path.Builder { return cf.p @@ -79,6 +79,7 @@ func (c CalendarDisplayable) GetDisplayName() *string { // GetParentFolderId returns the default calendar name address // EventCalendars have a flat hierarchy and Calendars are rooted // at the default +// //nolint:revive func (c CalendarDisplayable) GetParentFolderId() *string { return nil diff --git a/src/internal/connector/exchange/exchange_service_test.go b/src/internal/connector/exchange/exchange_service_test.go index e4eae813a..2cef9836e 100644 --- a/src/internal/connector/exchange/exchange_service_test.go +++ b/src/internal/connector/exchange/exchange_service_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + absser "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -252,8 +253,10 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() { function: GetAllFolderNamesForUser, }, { - name: "GraphQuery: Get All Users", - function: GetAllUsersForTenant, + name: "GraphQuery: Get All Users", + function: func(ctx context.Context, gs graph.Service, toss string) (absser.Parsable, error) { + return GetAllUsersForTenant(ctx, gs) + }, }, { name: "GraphQuery: Get All ContactFolders", diff --git a/src/internal/connector/exchange/exchange_vars.go b/src/internal/connector/exchange/exchange_vars.go index 09e44e4ee..60a5c5589 100644 --- a/src/internal/connector/exchange/exchange_vars.go +++ b/src/internal/connector/exchange/exchange_vars.go @@ -6,7 +6,8 @@ package exchange // Legacy Value Tags and constants are used to override certain values within // M365 objects. // Master Property Value Document: -// https://interoperability.blob.core.windows.net/files/MS-OXPROPS/%5bMS-OXPROPS%5d.pdf +// +// https://interoperability.blob.core.windows.net/files/MS-OXPROPS/%5bMS-OXPROPS%5d.pdf const ( // MailRestorePropertyTag inhibits exchange.Mail.Message from being "resent" through the server. // DEFINED: Section 2.791 PidTagMessageFlags @@ -27,9 +28,9 @@ const ( // Section: 2.789 PidTagMessageDeliveryTime MailReceiveDateTimeOverriveProperty = "SystemTime 0x0E06" - //---------------------------------- + // ---------------------------------- // Default Folder Names - //------------------------ + // ---------------------------------- // Mail Definitions: https://docs.microsoft.com/en-us/graph/api/resources/mailfolder?view=graph-rest-1.0 // inbox and root @@ -38,9 +39,9 @@ const ( DefaultContactFolder = "Contacts" DefaultCalendar = "Calendar" - //--------------------- + // ---------------------------------- // Paging - //----------------- + // ---------------------------------- // nextDataLink definition https://docs.microsoft.com/en-us/graph/paging nextDataLink = "@odata.nextLink" ) diff --git a/src/internal/connector/exchange/query_options.go b/src/internal/connector/exchange/query_options.go index d3f6e93d1..2805d1c67 100644 --- a/src/internal/connector/exchange/query_options.go +++ b/src/internal/connector/exchange/query_options.go @@ -21,11 +21,11 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- // Constant Section // Defines the allowable strings that can be passed into // selectors for M365 objects -//------------------------------------------------------------ +// ----------------------------------------------------------------------- var ( fieldsForCalendars = map[string]int{ "changeKey": 1, @@ -118,12 +118,12 @@ func CategoryToOptionIdentifier(category path.CategoryType) optionIdentifier { } } -//--------------------------------------------------------------------------- +// ----------------------------------------------------------------------- // exchange.Query Option Section // These functions can be used to filter a response on M365 // Graph queries and reduce / filter the amount of data returned // which reduces the overall latency of complex calls -//---------------------------------------------------------------- +// ----------------------------------------------------------------------- func optionsForFolderMessages(moreOps []string) (*msmfmessage.MessagesRequestBuilderGetRequestConfiguration, error) { selecting, err := buildOptions(moreOps, messages) diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index b8fcec817..6e4d11426 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -23,9 +23,10 @@ type exchangeService struct { credentials account.M365Config } -// /------------------------------------------------------------ +// ------------------------------------------------------------ // Functions to comply with graph.Service Interface -// ------------------------------------------------------- +// ------------------------------------------------------------ + func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient { return &es.client } diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index 71835d2a8..8d7b952e8 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -74,9 +74,8 @@ func GetAllContactFolderNamesForUser(ctx context.Context, gs graph.Service, user return gs.Client().UsersById(user).ContactFolders().Get(ctx, options) } -// GetAllUsersForTenant is a GraphQuery for retrieving all the UserCollectionResponse with -// that contains the UserID and email for each user. All other information is omitted -func GetAllUsersForTenant(ctx context.Context, gs graph.Service, user string) (absser.Parsable, error) { +// GetAllUsersForTenant makes a GraphQuery request retrieving all the users in the tenant. +func GetAllUsersForTenant(ctx context.Context, gs graph.Service) (absser.Parsable, error) { selecting := []string{"userPrincipalName"} options, err := optionsForUsers(selecting) diff --git a/src/internal/connector/graph/cache_container.go b/src/internal/connector/graph/cache_container.go index 064b8f623..e792c235e 100644 --- a/src/internal/connector/graph/cache_container.go +++ b/src/internal/connector/graph/cache_container.go @@ -37,9 +37,9 @@ func CheckRequiredValues(c Container) error { return nil } -//====================================== +// ====================================== // cachedContainer Implementations -//====================================== +// ====================================== var _ CachedContainer = &CacheFolder{} @@ -58,9 +58,9 @@ func NewCacheFolder(c Container, pb *path.Builder) CacheFolder { return cf } -//========================================= +// ========================================= // Required Functions to satisfy interfaces -//========================================= +// ========================================= func (cf CacheFolder) Path() *path.Builder { return cf.p @@ -86,6 +86,7 @@ func (c CalendarDisplayable) GetDisplayName() *string { // GetParentFolderId returns the default calendar name address // EventCalendars have a flat hierarchy and Calendars are rooted // at the default +// //nolint:revive func (c CalendarDisplayable) GetParentFolderId() *string { return &c.parentID diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 0eb7bdc0e..4f27c6128 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -4,10 +4,11 @@ package connector import ( "context" - "fmt" "runtime/trace" + "strings" "sync" + "github.com/microsoft/kiota-abstractions-go/serialization" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -16,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/exchange" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" D "github.com/alcionai/corso/src/internal/diagnostics" @@ -73,7 +75,16 @@ func (gs graphService) ErrPolicy() bool { return gs.failFast } -func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnector, error) { +type resource int + +const ( + UnknownResource resource = iota + AllResources + Users + Sites +) + +func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (*GraphConnector, error) { m365, err := acct.M365Config() if err != nil { return nil, errors.Wrap(err, "retrieving m365 account configuration") @@ -93,15 +104,16 @@ func NewGraphConnector(ctx context.Context, acct account.Account) (*GraphConnect gc.graphService = *aService - err = gc.setTenantUsers(ctx) - if err != nil { - return nil, errors.Wrap(err, "retrieving tenant user list") + if r == AllResources || r == Users { + if err = gc.setTenantUsers(ctx); err != nil { + return nil, errors.Wrap(err, "retrieving tenant user list") + } } - // TODO: users or sites, one or the other, not both. - err = gc.setTenantSites(ctx) - if err != nil { - return nil, errors.Wrap(err, "retrieveing tenant site list") + if r == AllResources || r == Sites { + if err = gc.setTenantSites(ctx); err != nil { + return nil, errors.Wrap(err, "retrieveing tenant site list") + } } return &gc, nil @@ -133,59 +145,41 @@ func (gs *graphService) EnableFailFast() { // setTenantUsers queries the M365 to identify the users in the // workspace. The users field is updated during this method -// iff the return value is nil +// iff the returned error is nil func (gc *GraphConnector) setTenantUsers(ctx context.Context) error { ctx, end := D.Span(ctx, "gc:setTenantUsers") defer end() - response, err := exchange.GetAllUsersForTenant(ctx, gc.graphService, "") - if err != nil { - return errors.Wrapf( - err, - "tenant %s M365 query: %s", - gc.tenant, - support.ConnectorStackErrorTrace(err), - ) - } - - userIterator, err := msgraphgocore.NewPageIterator( - response, - &gc.graphService.adapter, + users, err := getResources( + ctx, + gc.graphService, + gc.tenant, + exchange.GetAllUsersForTenant, models.CreateUserCollectionResponseFromDiscriminatorValue, + identifyUser, ) if err != nil { - return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + return err } - callbackFunc := func(userItem interface{}) bool { - user, ok := userItem.(models.Userable) - if !ok { - err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), errors.New("received non-User on iteration"), err) - return true - } + gc.Users = users - if user.GetUserPrincipalName() == nil { - err = support.WrapAndAppend( - gc.graphService.adapter.GetBaseUrl(), - fmt.Errorf("no email address for User: %s", *user.GetId()), - err, - ) + return nil +} - return true - } - - // *user.GetId() is populated for every M365 entityable object by M365 backstore - gc.Users[*user.GetUserPrincipalName()] = *user.GetId() - - return true +// Transforms an interface{} into a key,value pair representing +// userPrincipalName:userID. +func identifyUser(item any) (string, string, error) { + m, ok := item.(models.Userable) + if !ok { + return "", "", errors.New("iteration retrieved non-User item") } - iterateError := userIterator.Iterate(ctx, callbackFunc) - if iterateError != nil { - err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err) + if m.GetUserPrincipalName() == nil { + return "", "", errors.Errorf("no principal name for User: %s", *m.GetId()) } - return err + return *m.GetUserPrincipalName(), *m.GetId(), nil } // GetUsers returns the email address of users within tenant. @@ -199,68 +193,53 @@ func (gc *GraphConnector) GetUsersIds() []string { } // setTenantSites queries the M365 to identify the sites in the -// workspace. The sitets field is updated during this method -// iff the return value is nil +// workspace. The sites field is updated during this method +// iff the returned error is nil. func (gc *GraphConnector) setTenantSites(ctx context.Context) error { - // TODO gc.Sites = map[string]string{} - // ctx, end := D.Span(ctx, "gc:setTenantSites") - // defer end() + ctx, end := D.Span(ctx, "gc:setTenantSites") + defer end() - // response, err := exchange.GetAllUsersForTenant(ctx, gc.graphService, "") - // if err != nil { - // return errors.Wrapf( - // err, - // "tenant %s M365 query: %s", - // gc.tenant, - // support.ConnectorStackErrorTrace(err), - // ) - // } + sites, err := getResources( + ctx, + gc.graphService, + gc.tenant, + sharepoint.GetAllSitesForTenant, + models.CreateSiteCollectionResponseFromDiscriminatorValue, + identifySite, + ) + if err != nil { + return err + } - // userIterator, err := msgraphgocore.NewPageIterator( - // response, - // &gc.graphService.adapter, - // models.CreateUserCollectionResponseFromDiscriminatorValue, - // ) - // if err != nil { - // return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) - // } - - // callbackFunc := func(userItem interface{}) bool { - // user, ok := userItem.(models.Userable) - // if !ok { - // err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), - // errors.New("received non-User on iteration"), err) - // return true - // } - - // if user.GetUserPrincipalName() == nil { - // err = support.WrapAndAppend( - // gc.graphService.adapter.GetBaseUrl(), - // fmt.Errorf("no email address for User: %s", *user.GetId()), - // err, - // ) - - // return true - // } - - // // *user.GetId() is populated for every M365 entityable object by M365 backstore - // gc.Users[*user.GetUserPrincipalName()] = *user.GetId() - - // return true - // } - - // iterateError := userIterator.Iterate(ctx, callbackFunc) - // if iterateError != nil { - // err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err) - // } - - // return err + gc.Sites = sites return nil } +var errKnownSkippableCase = errors.New("case is known and skippable") + +// Transforms an interface{} into a key,value pair representing +// siteName:siteID. +func identifySite(item any) (string, string, error) { + m, ok := item.(models.Siteable) + if !ok { + return "", "", errors.New("iteration retrieved non-Site item") + } + + if m.GetName() == nil { + // the built-in site at "htps://{tenant-domain}/search" never has a name. + if m.GetWebUrl() != nil && strings.HasSuffix(*m.GetWebUrl(), "/search") { + return "", "", errKnownSkippableCase + } + + return "", "", errors.Errorf("no name for Site: %s", *m.GetId()) + } + + return *m.GetName(), *m.GetId(), nil +} + // GetSites returns the siteIDs of sharepoint sites within tenant. func (gc *GraphConnector) GetSites() []string { return buildFromMap(true, gc.Sites) @@ -365,6 +344,57 @@ func (gc *GraphConnector) incrementAwaitingMessages() { // Helper Funcs // --------------------------------------------------------------------------- +func getResources( + ctx context.Context, + gs graph.Service, + tenantID string, + query func(context.Context, graph.Service) (serialization.Parsable, error), + parser func(parseNode serialization.ParseNode) (serialization.Parsable, error), + identify func(any) (string, string, error), +) (map[string]string, error) { + resources := map[string]string{} + + response, err := query(ctx, gs) + if err != nil { + return nil, errors.Wrapf( + err, + "retrieving resources for tenant %s: %s", + tenantID, + support.ConnectorStackErrorTrace(err), + ) + } + + iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), parser) + if err != nil { + return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + } + + var iterErrs error + + callbackFunc := func(item any) bool { + k, v, err := identify(item) + if err != nil { + if errors.Is(err, errKnownSkippableCase) { + return true + } + + iterErrs = support.WrapAndAppend(gs.Adapter().GetBaseUrl(), err, iterErrs) + + return true + } + + resources[k] = v + + return true + } + + if err := iter.Iterate(ctx, callbackFunc); err != nil { + return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err)) + } + + return resources, iterErrs +} + // IsRecoverableError returns true iff error is a RecoverableGCEerror func IsRecoverableError(e error) bool { var recoverable support.RecoverableGCError diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 7f6ee0a00..8e791db0c 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -65,7 +65,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() { for _, test := range table { suite.T().Run(test.name, func(t *testing.T) { - gc, err := NewGraphConnector(ctx, test.acct(t)) + gc, err := NewGraphConnector(ctx, test.acct(t), Users) assert.Nil(t, gc, test.name+" failed") assert.NotNil(t, err, test.name+"failed") }) diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 11a6411b3..70520ec1d 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -162,6 +162,7 @@ type restoreBackupInfo struct { name string service path.ServiceType collections []colInfo + resource resource } func attachmentEqual( @@ -858,15 +859,16 @@ func getSelectorWith(service path.ServiceType) selectors.Selector { case path.OneDriveService: s = selectors.ServiceOneDrive } + // TODO: ^ sharepoint return selectors.Selector{ Service: s, } } -func loadConnector(ctx context.Context, t *testing.T) *GraphConnector { +func loadConnector(ctx context.Context, t *testing.T, r resource) *GraphConnector { a := tester.NewM365Account(t) - connector, err := NewGraphConnector(ctx, a) + connector, err := NewGraphConnector(ctx, a, r) require.NoError(t, err) return connector diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 81b530bc2..05396a34b 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -39,7 +39,7 @@ func (suite *GraphConnectorIntegrationSuite) SetupSuite() { _, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...) require.NoError(suite.T(), err) - suite.connector = loadConnector(ctx, suite.T()) + suite.connector = loadConnector(ctx, suite.T(), Users) suite.user = tester.M365UserID(suite.T()) tester.LogTimeOfTest(suite.T()) } @@ -63,8 +63,8 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { suite.Equal(len(newConnector.Users), 0) err = newConnector.setTenantUsers(ctx) - assert.NoError(suite.T(), err) - suite.Greater(len(newConnector.Users), 0) + suite.NoError(err) + suite.Less(0, len(newConnector.Users)) } // TestSetTenantUsers verifies GraphConnector's ability to query @@ -86,10 +86,8 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() { suite.Equal(0, len(newConnector.Sites)) err = newConnector.setTenantSites(ctx) - assert.NoError(suite.T(), err) - // TODO: should be non-zero once implemented. - // suite.Greater(len(newConnector.Users), 0) - suite.Equal(0, len(newConnector.Sites)) + suite.NoError(err) + suite.Less(0, len(newConnector.Sites)) } func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() { @@ -194,7 +192,7 @@ func runRestoreBackupTest( start := time.Now() - restoreGC := loadConnector(ctx, t) + restoreGC := loadConnector(ctx, t, test.resource) restoreSel := getSelectorWith(test.service) deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) require.NoError(t, err) @@ -228,7 +226,7 @@ func runRestoreBackupTest( }) } - backupGC := loadConnector(ctx, t) + backupGC := loadConnector(ctx, t, test.resource) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Logf("Selective backup of %s\n", backupSel) @@ -253,8 +251,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { table := []restoreBackupInfo{ { - name: "EmailsWithAttachments", - service: path.ExchangeService, + name: "EmailsWithAttachments", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Inbox"}, @@ -279,8 +278,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { }, }, { - name: "MultipleEmailsMultipleFolders", - service: path.ExchangeService, + name: "MultipleEmailsMultipleFolders", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Inbox"}, @@ -324,8 +324,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { }, }, { - name: "MultipleContactsSingleFolder", - service: path.ExchangeService, + name: "MultipleContactsSingleFolder", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Contacts"}, @@ -351,8 +352,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { }, }, { - name: "MultipleContactsMutlipleFolders", - service: path.ExchangeService, + name: "MultipleContactsMutlipleFolders", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Work"}, @@ -475,8 +477,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames() { table := []restoreBackupInfo{ { - name: "Contacts", - service: path.ExchangeService, + name: "Contacts", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Work"}, @@ -574,7 +577,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames dest.ContainerName, ) - restoreGC := loadConnector(ctx, t) + restoreGC := loadConnector(ctx, t, test.resource) deets, err := restoreGC.RestoreDataCollections(ctx, restoreSel, dest, collections) require.NoError(t, err) require.NotNil(t, deets) @@ -592,7 +595,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames // Run a backup and compare its output with what we put in. - backupGC := loadConnector(ctx, t) + backupGC := loadConnector(ctx, t, test.resource) backupSel := backupSelectorForExpected(t, test.service, expectedDests) t.Log("Selective backup of", backupSel) @@ -622,8 +625,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiuserRestoreAndBackup() { } table := []restoreBackupInfo{ { - name: "Email", - service: path.ExchangeService, + name: "Email", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Inbox"}, @@ -658,8 +662,9 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiuserRestoreAndBackup() { }, }, { - name: "Contacts", - service: path.ExchangeService, + name: "Contacts", + service: path.ExchangeService, + resource: Users, collections: []colInfo{ { pathElements: []string{"Work"}, diff --git a/src/internal/connector/sharepoint/queries.go b/src/internal/connector/sharepoint/queries.go new file mode 100644 index 000000000..0682bd3d1 --- /dev/null +++ b/src/internal/connector/sharepoint/queries.go @@ -0,0 +1,21 @@ +package sharepoint + +import ( + "context" + + absser "github.com/microsoft/kiota-abstractions-go/serialization" + mssite "github.com/microsoftgraph/msgraph-sdk-go/sites" + + "github.com/alcionai/corso/src/internal/connector/graph" +) + +// GetAllSitesForTenant makes a GraphQuery request retrieving all sites in the tenant. +func GetAllSitesForTenant(ctx context.Context, gs graph.Service) (absser.Parsable, error) { + options := &mssite.SitesRequestBuilderGetRequestConfiguration{ + QueryParameters: &mssite.SitesRequestBuilderGetQueryParameters{ + Select: []string{"id", "name", "weburl"}, + }, + } + + return gs.Client().Sites().Get(ctx, options) +} diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 11044d072..a76ca7d92 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -406,9 +406,9 @@ func SetAdditionalDataToEventMessage( // ToEventSimplified transforms an event to simplifed restore format // To overcome some of the MS Graph API challenges, the event object is modified in the following ways: -// * Instead of adding attendees and generating spurious notifications, +// - Instead of adding attendees and generating spurious notifications, // add a summary of attendees at the beginning to the event before the original body content -// * event.attendees is set to an empty list +// - event.attendees is set to an empty list func ToEventSimplified(orig models.Eventable) models.Eventable { attendees := FormatAttendees(orig, *orig.GetBody().GetContentType() == models.HTML_BODYTYPE) orig.SetAttendees([]models.Attendeeable{}) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 1deb95809..2a5f67205 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -128,7 +128,12 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { defer close(complete) // retrieve data from the producer - gc, err := connector.NewGraphConnector(ctx, op.account) + resource := connector.Users + if op.Selectors.Service == selectors.ServiceSharePoint { + resource = connector.Sites + } + + gc, err := connector.NewGraphConnector(ctx, op.account, resource) if err != nil { err = errors.Wrap(err, "connecting to graph api") opStats.readErr = err diff --git a/src/internal/operations/restore.go b/src/internal/operations/restore.go index 74f1e3c44..7528c5e71 100644 --- a/src/internal/operations/restore.go +++ b/src/internal/operations/restore.go @@ -166,7 +166,12 @@ func (op *RestoreOperation) Run(ctx context.Context) (restoreDetails *details.De defer close(gcComplete) // restore those collections using graph - gc, err := connector.NewGraphConnector(ctx, op.account) + resource := connector.Users + if op.Selectors.Service == selectors.ServiceSharePoint { + resource = connector.Sites + } + + gc, err := connector.NewGraphConnector(ctx, op.account, resource) if err != nil { err = errors.Wrap(err, "connecting to microsoft servers") opStats.writeErr = err diff --git a/src/pkg/services/m365/m365.go b/src/pkg/services/m365/m365.go index bb7c949ee..53ea636be 100644 --- a/src/pkg/services/m365/m365.go +++ b/src/pkg/services/m365/m365.go @@ -12,7 +12,7 @@ import ( // Users returns a list of users in the specified M365 tenant // TODO: Implement paging support func Users(ctx context.Context, m365Account account.Account) ([]string, error) { - gc, err := connector.NewGraphConnector(ctx, m365Account) + gc, err := connector.NewGraphConnector(ctx, m365Account, connector.Users) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } @@ -23,7 +23,7 @@ func Users(ctx context.Context, m365Account account.Account) ([]string, error) { // UserIDs returns a list of user IDs for the specified M365 tenant // TODO: Implement paging support func UserIDs(ctx context.Context, m365Account account.Account) ([]string, error) { - gc, err := connector.NewGraphConnector(ctx, m365Account) + gc, err := connector.NewGraphConnector(ctx, m365Account, connector.Users) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") } @@ -32,7 +32,7 @@ func UserIDs(ctx context.Context, m365Account account.Account) ([]string, error) } func GetEmailAndUserID(ctx context.Context, m365Account account.Account) (map[string]string, error) { - gc, err := connector.NewGraphConnector(ctx, m365Account) + gc, err := connector.NewGraphConnector(ctx, m365Account, connector.Users) if err != nil { return nil, errors.Wrap(err, "could not initialize M365 graph connection") }