From 67bc038c5584af82f7ceae2b0170f63d863ad861 Mon Sep 17 00:00:00 2001 From: Danny Date: Tue, 30 Aug 2022 18:12:38 -0400 Subject: [PATCH] GC: Add WSL Linting Formatting `src/internal/connector` (#694) Lint remaining of `internal/connector` package for wsl --- src/.golangci.yml | 8 --- .../exchange/exchange_service_test.go | 20 +++++- .../connector/exchange/service_functions.go | 24 +++++++ src/internal/connector/graph_connector.go | 71 ++++++++++++++----- .../connector/graph_connector_test.go | 22 ++++++ 5 files changed, 117 insertions(+), 28 deletions(-) diff --git a/src/.golangci.yml b/src/.golangci.yml index bcc3456c9..62e9d9f0e 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -80,13 +80,5 @@ issues: - revive text: "import-shadowing:.*'suite' shadows" # Temporarily skip linting wsl on `connector` package until fixes are merged. - - path: internal/connector/graph_connector_test.go - linters: wsl - - path: internal/connector/graph_connector.go - linters: wsl - - path: internal/connector/exchange/exchange_service_test.go - linters: wsl - - path: internal/connector/exchange/service_functions.go - linters: wsl - path: internal/connector/onedrive linters: wsl diff --git a/src/internal/connector/exchange/exchange_service_test.go b/src/internal/connector/exchange/exchange_service_test.go index 0d015241f..696bf6f39 100644 --- a/src/internal/connector/exchange/exchange_service_test.go +++ b/src/internal/connector/exchange/exchange_service_test.go @@ -33,6 +33,7 @@ func TestExchangeServiceSuite(t *testing.T) { ); err != nil { t.Skip(err) } + suite.Run(t, new(ExchangeServiceSuite)) } @@ -47,6 +48,7 @@ func (suite *ExchangeServiceSuite) SetupSuite() { require.NoError(t, err) service, err := createService(m365, false) require.NoError(t, err) + suite.es = service } @@ -207,6 +209,7 @@ func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() { sel.Include(sel.Users([]string{userID})) eb, err := sel.ToExchangeBackup() require.NoError(suite.T(), err) + scopes := eb.Scopes() for _, test := range scopes { @@ -253,6 +256,7 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() { function: GetAllEventsForUser, }, } + for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { response, err := test.function(suite.es, userID) @@ -341,6 +345,7 @@ func (suite *ExchangeServiceSuite) TestGetFolderID() { checkError: assert.NoError, }, } + for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { _, err := GetFolderID( @@ -356,17 +361,24 @@ func (suite *ExchangeServiceSuite) TestGetFolderID() { // TestIterativeFunctions verifies that GraphQuery to Iterate // functions are valid for current versioning of msgraph-go-sdk func (suite *ExchangeServiceSuite) TestIterativeFunctions() { - userID := tester.M365UserID(suite.T()) - sel := selectors.NewExchangeBackup() + var ( + mailScope, contactScope selectors.ExchangeScope + userID = tester.M365UserID(suite.T()) + sel = selectors.NewExchangeBackup() + ) + sel.Include(sel.Users([]string{userID})) + eb, err := sel.ToExchangeBackup() require.NoError(suite.T(), err) + scopes := eb.Scopes() - var mailScope, contactScope selectors.ExchangeScope + for _, scope := range scopes { if scope.IncludesCategory(selectors.ExchangeContactFolder) { contactScope = scope } + if scope.IncludesCategory(selectors.ExchangeMail) { mailScope = scope } @@ -437,6 +449,7 @@ func (suite *ExchangeServiceSuite) TestRestoreContact() { folderName := "TestRestoreContact: " + common.FormatSimpleDateTime(now) aFolder, err := CreateContactFolder(suite.es, userID, folderName) require.NoError(t, err) + folderID := *aFolder.GetId() err = RestoreExchangeContact(context.Background(), mockconnector.GetMockContactBytes("Corso TestContact"), @@ -487,6 +500,7 @@ func (suite *ExchangeServiceSuite) TestEstablishFolder() { now := time.Now() folderName := "CorsoEstablishFolder" + common.FormatSimpleDateTime(now) userID := tester.M365UserID(suite.T()) + for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { folderID, err := establishFolder(suite.es, folderName, userID, test.option) diff --git a/src/internal/connector/exchange/service_functions.go b/src/internal/connector/exchange/service_functions.go index dca85cec5..224b3616e 100644 --- a/src/internal/connector/exchange/service_functions.go +++ b/src/internal/connector/exchange/service_functions.go @@ -56,12 +56,14 @@ func createService(credentials account.M365Config, shouldFailFast bool) (*exchan if err != nil { return nil, err } + service := exchangeService{ adapter: *adapter, client: *msgraphsdk.NewGraphServiceClient(adapter), failFast: shouldFailFast, credentials: credentials, } + return &service, err } @@ -70,6 +72,7 @@ func createService(credentials account.M365Config, shouldFailFast bool) (*exchan func CreateMailFolder(gs graph.Service, user, folder string) (models.MailFolderable, error) { requestBody := models.NewMailFolder() requestBody.SetDisplayName(&folder) + isHidden := false requestBody.SetIsHidden(&isHidden) @@ -110,6 +113,7 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde mfs = []MailFolder{} err error ) + resp, err := GetAllFolderNamesForUser(gs, user) if err != nil { return nil, err @@ -143,6 +147,7 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde if err := iter.Iterate(cb); err != nil { return nil, err } + return mfs, err } @@ -157,6 +162,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option query GraphQuery transform absser.ParsableFactory ) + switch category { case messages: query = GetAllFolderNamesForUser @@ -176,6 +182,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option user, support.ConnectorStackErrorTrace(err), ) } + pageIterator, err := msgraphgocore.NewPageIterator( response, service.Adapter(), @@ -184,6 +191,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option if err != nil { return nil, err } + callbackFunc := iterateFindFolderID(category, &folderID, folderName, @@ -211,12 +219,16 @@ func parseCalendarIDFromEvent(reference string) (string, error) { if len(stringArray) < 2 { return "", errors.New("calendarID not found") } + temp := stringArray[1] stringArray = strings.Split(temp, "')/$ref") + if len(stringArray) < 2 { return "", errors.New("calendarID not found") } + calendarID := stringArray[0] + if len(calendarID) == 0 { return "", errors.New("calendarID empty") } @@ -245,6 +257,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) ( IterateAndFilterMessagesForCollections, nil } + if scope.IncludesCategory(selectors.ExchangeEvent) { return models.CreateEventCollectionResponseFromDiscriminatorValue, GetAllEventsForUser, @@ -271,6 +284,7 @@ func GetRestoreFolder( user, category string, ) (string, error) { newFolder := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat)) + switch category { case mailCategory, contactsCategory: return establishFolder(service, newFolder, user, categoryToOptionIdentifier(category)) @@ -292,18 +306,21 @@ func establishFolder( if !errors.Is(err, ErrFolderNotFound) { return "", support.WrapAndAppend(user, err, err) } + switch optID { case messages: fold, err := CreateMailFolder(service, user, folderName) if err != nil { return "", support.WrapAndAppend(user, err, err) } + return *fold.GetId(), nil case contacts: fold, err := CreateContactFolder(service, user, folderName) if err != nil { return "", support.WrapAndAppend(user, err, err) } + return *fold.GetId(), nil default: return "", fmt.Errorf("category: %s not supported for folder creation", optID) @@ -319,12 +336,14 @@ func RestoreExchangeObject( destination, user string, ) error { var setting optionIdentifier + switch category { case mailCategory, contactsCategory: setting = categoryToOptionIdentifier(category) default: return fmt.Errorf("type: %s not supported for exchange restore", category) } + if policy != control.Copy { return fmt.Errorf("restore policy: %s not supported", policy) } @@ -361,9 +380,11 @@ func RestoreExchangeContact( if err != nil { return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } + if response == nil { return errors.New("msgraph contact post fail: REST response not received") } + return nil } @@ -395,6 +416,7 @@ func RestoreMailMessage( sv.SetValue(&enableValue) svlep := []models.SingleValueLegacyExtendedPropertyable{sv} clone.SetSingleValueExtendedProperties(svlep) + draft := false clone.SetIsDraft(&draft) @@ -418,8 +440,10 @@ func SendMailToBackStore(service graph.Service, user, destination string, messag if err != nil { return support.WrapAndAppend(": "+support.ConnectorStackErrorTrace(err), err, nil) } + if sentMessage == nil { return errors.New("message not Sent: blocked by server") } + return nil } diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index a6a2046cd..ae25e12f3 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -9,7 +9,6 @@ import ( "strings" "sync/atomic" - absser "github.com/microsoft/kiota-abstractions-go/serialization" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -67,6 +66,7 @@ func NewGraphConnector(acct account.Account) (*GraphConnector, error) { if err != nil { return nil, errors.Wrap(err, "retrieving m356 account configuration") } + gc := GraphConnector{ tenant: m365.TenantID, Users: make(map[string]string, 0), @@ -74,15 +74,19 @@ func NewGraphConnector(acct account.Account) (*GraphConnector, error) { statusCh: make(chan *support.ConnectorOperationStatus), credentials: m365, } + aService, err := gc.createService(false) if err != nil { return nil, err } + gc.graphService = *aService + err = gc.setTenantUsers() if err != nil { return nil, err } + return &gc, nil } @@ -96,11 +100,13 @@ func (gc *GraphConnector) createService(shouldFailFast bool) (*graphService, err if err != nil { return nil, err } + connector := graphService{ adapter: *adapter, client: *msgraphsdk.NewGraphServiceClient(adapter), failFast: shouldFailFast, } + return &connector, err } @@ -121,10 +127,12 @@ func (gc *GraphConnector) setTenantUsers() error { support.ConnectorStackErrorTrace(err), ) } + if response == nil { err = support.WrapAndAppend("general access", errors.New("connector failed: No access"), err) return err } + userIterator, err := msgraphgocore.NewPageIterator( response, &gc.graphService.adapter, @@ -133,31 +141,35 @@ func (gc *GraphConnector) setTenantUsers() error { if err != nil { return errors.Wrap(err, support.ConnectorStackErrorTrace(err)) } - var iterateError error + callbackFunc := func(userItem interface{}) bool { user, ok := userItem.(models.Userable) if !ok { err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), errors.New("user iteration failure"), err) return true } + if user.GetUserPrincipalName() == nil { err = support.WrapAndAppend( gc.graphService.adapter.GetBaseUrl(), fmt.Errorf("no email address for User: %s", *user.GetId()), err, ) + return true } // *user.GetId() is populated for every M365 entityable object by M365 backstore gc.Users[*user.GetUserPrincipalName()] = *user.GetId() + return true } - iterateError = userIterator.Iterate(callbackFunc) + iterateError := userIterator.Iterate(callbackFunc) if iterateError != nil { err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err) } + return err } @@ -175,6 +187,7 @@ func (gc *GraphConnector) GetUsersIds() []string { // Returns list of keys iff true; otherwise returns a list of values func buildFromMap(isKey bool, mapping map[string]string) []string { returnString := make([]string, 0) + if isKey { for k := range mapping { returnString = append(returnString, k) @@ -184,6 +197,7 @@ func buildFromMap(isKey bool, mapping map[string]string) []string { returnString = append(returnString, v) } } + return returnString } @@ -199,9 +213,13 @@ func (gc *GraphConnector) ExchangeDataCollection( if err != nil { return nil, errors.Wrap(err, "exchangeDataCollection: unable to parse selector") } - scopes := eb.DiscreteScopes(gc.GetUsers()) - collections := []data.Collection{} - var errs error + + var ( + scopes = eb.DiscreteScopes(gc.GetUsers()) + collections = []data.Collection{} + errs error + ) + for _, scope := range scopes { // Creates a map of collections based on scope dcs, err := gc.createCollections(ctx, scope) @@ -209,6 +227,7 @@ func (gc *GraphConnector) ExchangeDataCollection( user := scope.Get(selectors.ExchangeUser) return nil, support.WrapAndAppend(user[0], err, errs) } + for _, collection := range dcs { collections = append(collections, collection) } @@ -229,22 +248,28 @@ func (gc *GraphConnector) RestoreExchangeDataCollection( attempts, successes int errs error folderID string + // TODO policy to be updated from external source after completion of refactoring + policy = control.Copy ) - policy := control.Copy // TODO policy to be updated from external source after completion of refactoring for _, dc := range dcs { - directory := strings.Join(dc.FullPath(), "") - user := dc.FullPath()[1] - items := dc.Items() - category := dc.FullPath()[2] + var ( + directory = strings.Join(dc.FullPath(), "") + user = dc.FullPath()[1] + items = dc.Items() + category = dc.FullPath()[2] + exit bool + ) + if _, ok := pathCounter[directory]; !ok { pathCounter[directory] = true folderID, errs = exchange.GetRestoreFolder(&gc.graphService, user, category) + if errs != nil { return errs } } - var exit bool + for !exit { select { case <-ctx.Done(): @@ -257,11 +282,13 @@ func (gc *GraphConnector) RestoreExchangeDataCollection( attempts++ buf := &bytes.Buffer{} + _, err := buf.ReadFrom(itemData.ToReader()) if err != nil { errs = support.WrapAndAppend(itemData.UUID(), err, errs) continue } + err = exchange.RestoreExchangeObject(ctx, buf.Bytes(), category, policy, &gc.graphService, folderID, user) if err != nil { @@ -272,12 +299,15 @@ func (gc *GraphConnector) RestoreExchangeDataCollection( } } } + gc.incrementAwaitingMessages() + status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs) // set the channel asynchronously so that this func doesn't block. go func(cos *support.ConnectorOperationStatus) { gc.statusCh <- cos }(status) + return errs } @@ -290,20 +320,20 @@ func (gc *GraphConnector) createCollections( scope selectors.ExchangeScope, ) ([]*exchange.Collection, error) { var ( - transformer absser.ParsableFactory - query exchange.GraphQuery - gIter exchange.GraphIterateFunc - errs error + errs error + transformer, query, gIter, err = exchange.SetupExchangeCollectionVars(scope) ) - transformer, query, gIter, err := exchange.SetupExchangeCollectionVars(scope) + if err != nil { return nil, support.WrapAndAppend(gc.Service().Adapter().GetBaseUrl(), err, nil) } + users := scope.Get(selectors.ExchangeUser) allCollections := make([]*exchange.Collection, 0) // Create collection of ExchangeDataCollection for _, user := range users { collections := make(map[string]*exchange.Collection) + response, err := query(&gc.graphService, user) if err != nil { return nil, errors.Wrapf( @@ -322,17 +352,22 @@ func (gc *GraphConnector) createCollections( // Each directory used the M365 Identifier. The use of ID stops collisions betweens users callbackFunc := gIter(user, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh) iterateError := pageIterator.Iterate(callbackFunc) + if iterateError != nil { errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs) } + if errs != nil { return nil, errs // return error if snapshot is incomplete } + for _, collection := range collections { gc.incrementAwaitingMessages() + allCollections = append(allCollections, collection) } } + return allCollections, errs } @@ -342,6 +377,7 @@ func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus { atomic.AddInt32(&gc.awaitingMessages, -1) gc.status = <-gc.statusCh } + return gc.status } @@ -355,6 +391,7 @@ func (gc *GraphConnector) PrintableStatus() string { if gc.status == nil { return "" } + return gc.status.String() } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 24a2e6fd8..a1dfc6863 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -31,6 +31,7 @@ func loadConnector(t *testing.T) *GraphConnector { a := tester.NewM365Account(t) connector, err := NewGraphConnector(a) require.NoError(t, err) + return connector } @@ -41,6 +42,7 @@ func TestGraphConnectorIntegrationSuite(t *testing.T) { ); err != nil { t.Skip(err) } + suite.Run(t, new(GraphConnectorIntegrationSuite)) } @@ -66,8 +68,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() { statusCh: make(chan *support.ConnectorOperationStatus), credentials: suite.connector.credentials, } + service, err := newConnector.createService(false) require.NoError(suite.T(), err) + newConnector.graphService = *service suite.Equal(len(newConnector.Users), 0) @@ -92,6 +96,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { assert.NoError(t, err) assert.True(t, connector.awaitingMessages > 0) assert.Nil(t, connector.status) + streams := make(map[string]<-chan data.Stream) // Verify Items() call returns an iterable channel(e.g. a channel that has been closed) for _, collection := range collectionList { @@ -115,6 +120,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { } }) } + exchangeData := collectionList[0] suite.Greater(len(exchangeData.FullPath()), 2) } @@ -129,11 +135,13 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() { sel.Include(sel.MailFolders([]string{suite.user}, selectors.Any())) eb, err := sel.ToExchangeBackup() require.NoError(t, err) + scopes := eb.Scopes() suite.Len(scopes, 1) mailScope := scopes[0] collection, err := connector.createCollections(context.Background(), mailScope) require.NoError(t, err) + for _, edc := range collection { testName := strings.Join(edc.FullPath(), " ") suite.T().Run(testName, func(t *testing.T) { @@ -150,6 +158,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() { } }) } + status := connector.AwaitStatus() suite.NotNil(status) suite.Equal(status.ObjectCount, status.Successful) @@ -164,13 +173,17 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression( sel.Include(sel.ContactFolders([]string{suite.user}, selectors.Any())) eb, err := sel.ToExchangeBackup() require.NoError(t, err) + scopes := eb.Scopes() connector := loadConnector(t) + suite.Len(scopes, 1) contactsOnly := scopes[0] collections, err := connector.createCollections(context.Background(), contactsOnly) assert.NoError(t, err) + number := 0 + for _, edc := range collections { testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath()[1], number) suite.T().Run(testName, func(t *testing.T) { @@ -188,6 +201,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression( number++ }) } + status := connector.AwaitStatus() suite.NotNil(status) suite.Equal(status.ObjectCount, status.Successful) @@ -204,9 +218,11 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() suite.Equal(len(scopes), 1) collections, err := connector.createCollections(context.Background(), scopes[0]) require.NoError(t, err) + for _, edc := range collections { streamChannel := edc.Items() number := 0 + for stream := range streamChannel { testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[2], number) suite.T().Run(testName, func(t *testing.T) { @@ -220,6 +236,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() }) } } + status := connector.AwaitStatus() suite.NotNil(status) suite.Equal(status.ObjectCount, status.Successful) @@ -234,6 +251,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() { category := "mail" connector := loadConnector(t) collection := make([]data.Collection, 0) + for i := 0; i < 3; i++ { mdc := mockconnector.NewMockExchangeCollection( []string{"tenant", suite.user, category, "Inbox"}, @@ -243,6 +261,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() { err := connector.RestoreExchangeDataCollection(context.Background(), collection) assert.NoError(suite.T(), err) + status := connector.AwaitStatus() assert.NotNil(t, status) assert.Equal(t, status.ObjectCount, status.Successful) @@ -261,6 +280,7 @@ func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() { sel := selectors.NewExchangeBackup() sel.Include(sel.MailFolders(selectors.Any(), []string{"Inbox"})) scopes := sel.DiscreteScopes(connector.GetUsers()) + for _, scope := range scopes { users := scope.Get(selectors.ExchangeUser) standard := (len(users) / 4) * 3 @@ -281,6 +301,7 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteMailFolder() { folderName := "TestFolder: " + common.FormatSimpleDateTime(now) aFolder, err := exchange.CreateMailFolder(&suite.connector.graphService, suite.user, folderName) assert.NoError(suite.T(), err, support.ConnectorStackErrorTrace(err)) + if aFolder != nil { err = exchange.DeleteMailFolder(suite.connector.Service(), suite.user, *aFolder.GetId()) assert.NoError(suite.T(), err) @@ -294,6 +315,7 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteContactFolder() folderName := "TestContactFolder: " + common.FormatSimpleDateTime(now) aFolder, err := exchange.CreateContactFolder(suite.connector.Service(), suite.user, folderName) assert.NoError(suite.T(), err) + if aFolder != nil { err = exchange.DeleteContactFolder(suite.connector.Service(), suite.user, *aFolder.GetId()) assert.NoError(suite.T(), err)