From e3abc281d67aa39056884e52f2da4ef11baada1c Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Tue, 30 Aug 2022 13:13:00 -0700 Subject: [PATCH] Enable wsl for most of internal/connector package (#680) * Lint part of internal/connector package with wsl * Cleanup some wsl lint errors in connector package --- src/.golangci.yml | 10 ++- src/internal/connector/exchange/event.go | 4 ++ src/internal/connector/exchange/event_test.go | 1 + .../exchange/exchange_data_collection.go | 29 ++++++++ .../exchange/exchange_data_collection_test.go | 10 ++- .../connector/exchange/iterators_test.go | 2 + src/internal/connector/exchange/message.go | 4 ++ .../connector/exchange/query_options.go | 20 +++++- .../connector/exchange/service_iterators.go | 40 +++++++++-- .../connector/exchange/service_query.go | 5 ++ .../connector/graph/service_helper.go | 3 + .../graph_connector_disconnected_test.go | 3 + .../mockconnector/mock_data_collection.go | 4 ++ .../mock_data_collection_test.go | 2 + src/internal/connector/support/errors.go | 11 ++++ src/internal/connector/support/errors_test.go | 31 ++++++--- src/internal/connector/support/m365Support.go | 7 ++ .../connector/support/m365Transform.go | 66 ++++++++++++++++++- .../connector/support/m365Transform_test.go | 1 + src/internal/connector/support/status.go | 8 ++- src/internal/connector/support/status_test.go | 1 + 21 files changed, 240 insertions(+), 22 deletions(-) diff --git a/src/.golangci.yml b/src/.golangci.yml index 8aca9690f..bcc3456c9 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -80,5 +80,13 @@ issues: - revive text: "import-shadowing:.*'suite' shadows" # Temporarily skip linting wsl on `connector` package until fixes are merged. - - path: internal/connector/ + - path: internal/connector/graph_connector_test.go + linters: wsl + - path: internal/connector/graph_connector.go + linters: wsl + - path: internal/connector/exchange/exchange_service_test.go + linters: wsl + - path: internal/connector/exchange/service_functions.go + linters: wsl + - path: internal/connector/onedrive linters: wsl diff --git a/src/internal/connector/exchange/event.go b/src/internal/connector/exchange/event.go index 9e1b7c003..e39bc46ce 100644 --- a/src/internal/connector/exchange/event.go +++ b/src/internal/connector/exchange/event.go @@ -22,19 +22,23 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo { GetEmailAddress(). GetAddress() } + if evt.GetSubject() != nil { subject = *evt.GetSubject() } + if evt.GetStart() != nil && evt.GetStart().GetDateTime() != nil { // timeString has 'Z' literal added to ensure the stored // DateTime is not: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) timeString := *evt.GetStart().GetDateTime() + "Z" + output, err := common.ParseTime(timeString) if err == nil { start = output } } + return &details.ExchangeInfo{ Organizer: organizer, Subject: subject, diff --git a/src/internal/connector/exchange/event_test.go b/src/internal/connector/exchange/event_test.go index 02b4603d8..5879ff87b 100644 --- a/src/internal/connector/exchange/event_test.go +++ b/src/internal/connector/exchange/event_test.go @@ -29,6 +29,7 @@ func (suite *EventSuite) TestEventInfo() { now := initial.Format(common.StandardTimeFormat) suite.T().Logf("Initial: %v\nFormatted: %v\n", initial, now) + tests := []struct { name string evtAndRP func() (models.Eventable, *details.ExchangeInfo) diff --git a/src/internal/connector/exchange/exchange_data_collection.go b/src/internal/connector/exchange/exchange_data_collection.go index 075023373..43d2edfd6 100644 --- a/src/internal/connector/exchange/exchange_data_collection.go +++ b/src/internal/connector/exchange/exchange_data_collection.go @@ -73,6 +73,7 @@ func NewCollection( fullPath: fullPath, collectionType: collectionType, } + return collection } @@ -118,9 +119,11 @@ func (col *Collection) populateByOptionIdentifier( errs error success int ) + defer func() { col.finishPopulation(ctx, success, errs) }() + user := col.user objectWriter := kw.NewJsonSerializationWriter() // get QueryBasedonIdentifier @@ -140,8 +143,10 @@ func (col *Collection) populateByOptionIdentifier( if col.service.ErrPolicy() { break } + continue } + err = serializeFunc(ctx, col.service.Client(), objectWriter, col.data, response, user) if err != nil { errs = support.WrapAndAppendf(user, err, errs) @@ -149,6 +154,7 @@ func (col *Collection) populateByOptionIdentifier( if col.service.ErrPolicy() { break } + continue } @@ -188,7 +194,9 @@ func eventToDataCollection( user string, ) error { var err error + defer objectWriter.Close() + event, ok := parsable.(models.Eventable) if !ok { return fmt.Errorf("expected Eventable, got %T", parsable) @@ -196,6 +204,7 @@ func eventToDataCollection( if *event.GetHasAttachments() { var retriesErr error + for count := 0; count < numberOfRetries; count++ { attached, err := client. UsersById(user). @@ -203,30 +212,37 @@ func eventToDataCollection( Attachments(). Get() retriesErr = err + if err == nil && attached != nil { event.SetAttachments(attached.GetValue()) break } } + if retriesErr != nil { logger.Ctx(ctx).Debug("exceeded maximum retries") + return support.WrapAndAppend( *event.GetId(), errors.Wrap(retriesErr, "attachment failed"), nil) } } + err = objectWriter.WriteObjectValue("", event) if err != nil { return support.SetNonRecoverableError(errors.Wrap(err, *event.GetId())) } + byteArray, err := objectWriter.GetSerializedContent() if err != nil { return support.WrapAndAppend(*event.GetId(), errors.Wrap(err, "serializing content"), nil) } + if byteArray != nil { dataChannel <- &Stream{id: *event.GetId(), message: byteArray, info: EventInfo(event)} } + return nil } @@ -240,21 +256,26 @@ func contactToDataCollection( user string, ) error { defer objectWriter.Close() + contact, ok := parsable.(models.Contactable) if !ok { return fmt.Errorf("expected Contactable, got %T", parsable) } + err := objectWriter.WriteObjectValue("", contact) if err != nil { return support.SetNonRecoverableError(errors.Wrap(err, *contact.GetId())) } + byteArray, err := objectWriter.GetSerializedContent() if err != nil { return support.WrapAndAppend(*contact.GetId(), err, nil) } + if byteArray != nil { dataChannel <- &Stream{id: *contact.GetId(), message: byteArray, info: ContactInfo(contact)} } + return nil } @@ -268,11 +289,14 @@ func messageToDataCollection( user string, ) error { var err error + defer objectWriter.Close() + aMessage, ok := parsable.(models.Messageable) if !ok { return fmt.Errorf("expected Messageable, got %T", parsable) } + adtl := aMessage.GetAdditionalData() if len(adtl) > 2 { aMessage, err = support.ConvertFromMessageable(adtl, aMessage) @@ -280,9 +304,11 @@ func messageToDataCollection( return err } } + if *aMessage.GetHasAttachments() { // getting all the attachments might take a couple attempts due to filesize var retriesErr error + for count := 0; count < numberOfRetries; count++ { attached, err := client. UsersById(user). @@ -290,11 +316,13 @@ func messageToDataCollection( Attachments(). Get() retriesErr = err + if err == nil { aMessage.SetAttachments(attached.GetValue()) break } } + if retriesErr != nil { logger.Ctx(ctx).Debug("exceeded maximum retries") return support.WrapAndAppend(*aMessage.GetId(), errors.Wrap(retriesErr, "attachment failed"), nil) @@ -313,6 +341,7 @@ func messageToDataCollection( } dataChannel <- &Stream{id: *aMessage.GetId(), message: byteArray, info: MessageInfo(aMessage)} + return nil } diff --git a/src/internal/connector/exchange/exchange_data_collection_test.go b/src/internal/connector/exchange/exchange_data_collection_test.go index ed864eaf4..3305dba42 100644 --- a/src/internal/connector/exchange/exchange_data_collection_test.go +++ b/src/internal/connector/exchange/exchange_data_collection_test.go @@ -30,11 +30,15 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeDataReader_Valid() { } func (suite *ExchangeDataCollectionSuite) TestExchangeDataReader_Empty() { - var empty []byte - expected := int64(0) + var ( + empty []byte + expected int64 + ) + ed := &Stream{message: empty} buf := &bytes.Buffer{} received, err := buf.ReadFrom(ed.ToReader()) + suite.Equal(expected, received) assert.Nil(suite.T(), err, "received buf.Readfrom error ") } @@ -67,9 +71,11 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeCollection_AddJob() { fullPath: []string{"Today", "is", "currently", "different"}, } suite.Zero(len(eoc.jobs)) + shopping := []string{"tomotoes", "potatoes", "pasta", "ice tea"} for _, item := range shopping { eoc.AddJob(item) } + suite.Equal(len(shopping), len(eoc.jobs)) } diff --git a/src/internal/connector/exchange/iterators_test.go b/src/internal/connector/exchange/iterators_test.go index 762d785db..eb91cf393 100644 --- a/src/internal/connector/exchange/iterators_test.go +++ b/src/internal/connector/exchange/iterators_test.go @@ -24,6 +24,7 @@ func (suite *ExchangeIteratorSuite) TestDisplayable() { bytes := mockconnector.GetMockContactBytes("Displayable") contact, err := support.CreateContactFromBytes(bytes) require.NoError(t, err) + aDisplayable, ok := contact.(displayable) assert.True(t, ok) assert.NotNil(t, aDisplayable.GetId()) @@ -35,6 +36,7 @@ func (suite *ExchangeIteratorSuite) TestDescendable() { bytes := mockconnector.GetMockMessageBytes("Descendable") message, err := support.CreateMessageFromBytes(bytes) require.NoError(t, err) + aDescendable, ok := message.(descendable) assert.True(t, ok) assert.NotNil(t, aDescendable.GetId()) diff --git a/src/internal/connector/exchange/message.go b/src/internal/connector/exchange/message.go index 950c443fc..6b0322124 100644 --- a/src/internal/connector/exchange/message.go +++ b/src/internal/connector/exchange/message.go @@ -12,17 +12,21 @@ func MessageInfo(msg models.Messageable) *details.ExchangeInfo { sender := "" subject := "" received := time.Time{} + if msg.GetSender() != nil && msg.GetSender().GetEmailAddress() != nil && msg.GetSender().GetEmailAddress().GetAddress() != nil { sender = *msg.GetSender().GetEmailAddress().GetAddress() } + if msg.GetSubject() != nil { subject = *msg.GetSubject() } + if msg.GetReceivedDateTime() != nil { received = *msg.GetReceivedDateTime() } + return &details.ExchangeInfo{ Sender: sender, Subject: subject, diff --git a/src/internal/connector/exchange/query_options.go b/src/internal/connector/exchange/query_options.go index 306c99bb6..bcddb7e7b 100644 --- a/src/internal/connector/exchange/query_options.go +++ b/src/internal/connector/exchange/query_options.go @@ -116,12 +116,14 @@ func optionsForMessages(moreOps []string) (*msmessage.MessagesRequestBuilderGetR if err != nil { return nil, err } + requestParameters := &msmessage.MessagesRequestBuilderGetQueryParameters{ Select: selecting, } options := &msmessage.MessagesRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } + return options, nil } @@ -133,12 +135,14 @@ func OptionsForSingleMessage(moreOps []string) (*msitem.MessageItemRequestBuilde if err != nil { return nil, err } + requestParams := &msitem.MessageItemRequestBuilderGetQueryParameters{ Select: selecting, } options := &msitem.MessageItemRequestBuilderGetRequestConfiguration{ QueryParameters: requestParams, } + return options, nil } @@ -150,12 +154,14 @@ func optionsForContactFolders(moreOps []string) ( if err != nil { return nil, err } + requestParameters := &mscontactfolder.ContactFoldersRequestBuilderGetQueryParameters{ Select: selecting, } options := &mscontactfolder.ContactFoldersRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } + return options, nil } @@ -174,6 +180,7 @@ func optionsForMailFolders(moreOps []string) (*msfolder.MailFoldersRequestBuilde options := &msfolder.MailFoldersRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } + return options, nil } @@ -184,12 +191,14 @@ func optionsForEvents(moreOps []string) (*msevents.EventsRequestBuilderGetReques if err != nil { return nil, err } + requestParameters := &msevents.EventsRequestBuilderGetQueryParameters{ Select: selecting, } options := &msevents.EventsRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } + return options, nil } @@ -200,12 +209,14 @@ func optionsForContacts(moreOps []string) (*mscontacts.ContactsRequestBuilderGet if err != nil { return nil, err } + requestParameters := &mscontacts.ContactsRequestBuilderGetQueryParameters{ Select: selecting, } options := &mscontacts.ContactsRequestBuilderGetRequestConfiguration{ QueryParameters: requestParameters, } + return options, nil } @@ -214,12 +225,14 @@ func optionsForUsers(moreOps []string) (*msuser.UsersRequestBuilderGetRequestCon if err != nil { return nil, err } + requestParams := &msuser.UsersRequestBuilderGetQueryParameters{ Select: selecting, } options := &msuser.UsersRequestBuilderGetRequestConfiguration{ QueryParameters: requestParams, } + return options, nil } @@ -227,8 +240,10 @@ func optionsForUsers(moreOps []string) (*msuser.UsersRequestBuilderGetRequestCon // @return is a pair. The first is a string literal of allowable options based on the object type, // the second is an error. An error is returned if an unsupported option or optionIdentifier was used func buildOptions(options []string, optID optionIdentifier) ([]string, error) { - var allowedOptions map[string]int - returnedOptions := []string{"id"} + var ( + allowedOptions map[string]int + returnedOptions = []string{"id"} + ) switch optID { case events: @@ -255,5 +270,6 @@ func buildOptions(options []string, optID optionIdentifier) ([]string, error) { returnedOptions = append(returnedOptions, entry) } + return returnedOptions, nil } diff --git a/src/internal/connector/exchange/service_iterators.go b/src/internal/connector/exchange/service_iterators.go index aa7953217..48b73ffa8 100644 --- a/src/internal/connector/exchange/service_iterators.go +++ b/src/internal/connector/exchange/service_iterators.go @@ -51,9 +51,12 @@ func IterateSelectAllDescendablesForCollections( collections map[string]*Collection, statusCh chan<- *support.ConnectorOperationStatus, ) func(any) bool { - var isCategorySet bool - var collectionType optionIdentifier - var category string + var ( + isCategorySet bool + collectionType optionIdentifier + category string + ) + return func(pageItem any) bool { // Defines the type of collection being created within the function if !isCategorySet { @@ -61,10 +64,12 @@ func IterateSelectAllDescendablesForCollections( collectionType = messages category = mailCategory } + if scope.IncludesCategory(selectors.ExchangeContact) { collectionType = contacts category = contactsCategory } + isCategorySet = true } @@ -81,6 +86,7 @@ func IterateSelectAllDescendablesForCollections( errs = support.WrapAndAppend(user, err, errs) return true } + edc := NewCollection( user, []string{credentials.TenantID, user, category, directory}, @@ -90,7 +96,9 @@ func IterateSelectAllDescendablesForCollections( ) collections[directory] = &edc } + collections[directory].AddJob(*entry.GetId()) + return true } } @@ -116,10 +124,12 @@ func IterateSelectAllEventsForCollections( errors.New("event iteration failure"), errs, ) + return true } adtl := event.GetAdditionalData() + value, ok := adtl["calendar@odata.associationLink"] if !ok { errs = support.WrapAndAppend( @@ -127,8 +137,10 @@ func IterateSelectAllEventsForCollections( fmt.Errorf("%s: does not support calendar look up", *event.GetId()), errs, ) + return true } + link, ok := value.(*string) if !ok || link == nil { errs = support.WrapAndAppend( @@ -136,6 +148,7 @@ func IterateSelectAllEventsForCollections( fmt.Errorf("%s: unable to obtain calendar event data", *event.GetId()), errs, ) + return true } // calendars and events are not easily correlated @@ -147,16 +160,17 @@ func IterateSelectAllEventsForCollections( errors.Wrap(err, *event.GetId()), errs, ) + return true } if _, ok := collections[directory]; !ok { - service, err := createService(credentials, failFast) if err != nil { errs = support.WrapAndAppend(user, err, errs) return true } + edc := NewCollection( user, []string{credentials.TenantID, user, eventsCategory, directory}, @@ -168,6 +182,7 @@ func IterateSelectAllEventsForCollections( } collections[directory].AddJob(*event.GetId()) + return true } } @@ -185,9 +200,9 @@ func IterateAndFilterMessagesForCollections( statusCh chan<- *support.ConnectorOperationStatus, ) func(any) bool { var isFilterSet bool + return func(messageItem any) bool { if !isFilterSet { - err := CollectMailFolders( scope, user, @@ -200,6 +215,7 @@ func IterateAndFilterMessagesForCollections( errs = support.WrapAndAppend(user, err, errs) return false } + isFilterSet = true } @@ -213,7 +229,9 @@ func IterateAndFilterMessagesForCollections( if _, ok = collections[directory]; !ok { return true } + collections[directory].AddJob(*message.GetId()) + return true } } @@ -231,6 +249,7 @@ func IterateFilterFolderDirectoriesForCollections( service graph.Service err error ) + return func(folderItem any) bool { folder, ok := folderItem.(displayable) if !ok { @@ -246,10 +265,13 @@ func IterateFilterFolderDirectoriesForCollections( if folder.GetDisplayName() == nil { return true } + if !scope.Contains(selectors.ExchangeMailFolder, *folder.GetDisplayName()) { return true } + directory := *folder.GetId() + service, err = createService(credentials, failFast) if err != nil { errs = support.WrapAndAppend( @@ -260,8 +282,10 @@ func IterateFilterFolderDirectoriesForCollections( ), errs, ) + return true } + temp := NewCollection( user, []string{credentials.TenantID, user, mailCategory, directory}, @@ -296,21 +320,27 @@ func iterateFindFolderID( errors.New("struct does not implement displayable"), errs, ) + return true } // Display name not set on folder if folder.GetDisplayName() == nil { return true } + name := *folder.GetDisplayName() if folderName == name { if folder.GetId() == nil { return true // invalid folder } + *folderID = folder.GetId() + return false } + return true + default: return false } diff --git a/src/internal/connector/exchange/service_query.go b/src/internal/connector/exchange/service_query.go index 80ddc660a..f0a93a8e4 100644 --- a/src/internal/connector/exchange/service_query.go +++ b/src/internal/connector/exchange/service_query.go @@ -35,6 +35,7 @@ func GetAllMessagesForUser(gs graph.Service, user string) (absser.Parsable, erro // GetAllContactsForUser is a GraphQuery function for querying all the contacts in a user's account func GetAllContactsForUser(gs graph.Service, user string) (absser.Parsable, error) { selecting := []string{"parentFolderId"} + options, err := optionsForContacts(selecting) if err != nil { return nil, err @@ -62,6 +63,7 @@ func GetAllContactFolderNamesForUser(gs graph.Service, user string) (absser.Pars if err != nil { return nil, err } + return gs.Client().UsersById(user).ContactFolders().GetWithRequestConfigurationAndResponseHandler(options, nil) } @@ -69,10 +71,12 @@ func GetAllContactFolderNamesForUser(gs graph.Service, user string) (absser.Pars // that contains the UserID and email for each user. All other information is omitted func GetAllUsersForTenant(gs graph.Service, user string) (absser.Parsable, error) { selecting := []string{"userPrincipalName"} + options, err := optionsForUsers(selecting) if err != nil { return nil, err } + return gs.Client().Users().GetWithRequestConfigurationAndResponseHandler(options, nil) } @@ -155,5 +159,6 @@ func CollectMailFolders( if iterateFailure != nil { err = support.WrapAndAppend(user+" iterate failure", iterateFailure, err) } + return err } diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go index 6172ccc8a..097ad6c23 100644 --- a/src/internal/connector/graph/service_helper.go +++ b/src/internal/connector/graph/service_helper.go @@ -14,6 +14,7 @@ func CreateAdapter(tenant, client, secret string) (*msgraphsdk.GraphRequestAdapt if err != nil { return nil, err } + auth, err := ka.NewAzureIdentityAuthenticationProviderWithScopes( cred, []string{"https://graph.microsoft.com/.default"}, @@ -21,6 +22,8 @@ func CreateAdapter(tenant, client, secret string) (*msgraphsdk.GraphRequestAdapt if err != nil { return nil, err } + adapter, err := msgraphsdk.NewGraphRequestAdapter(auth) + return adapter, err } diff --git a/src/internal/connector/graph_connector_disconnected_test.go b/src/internal/connector/graph_connector_disconnected_test.go index 0d38b683a..0b1bb90d3 100644 --- a/src/internal/connector/graph_connector_disconnected_test.go +++ b/src/internal/connector/graph_connector_disconnected_test.go @@ -76,6 +76,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() { names["Axel"] = "Foley" first := buildFromMap(true, names) last := buildFromMap(false, names) + suite.Contains(first, "Al") suite.Contains(first, "Ellen") suite.Contains(first, "Axel") @@ -86,6 +87,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestBuild() { func (suite *DisconnectedGraphConnectorSuite) TestInterfaceAlignment() { var dc data.Collection + concrete := mockconnector.NewMockExchangeCollection([]string{"a", "path"}, 1) dc = concrete assert.NotNil(suite.T(), dc) @@ -97,6 +99,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() { } suite.Equal(len(gc.PrintableStatus()), 0) gc.incrementAwaitingMessages() + go func() { status := support.CreateStatus( context.Background(), diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index 852c08601..c9e762fde 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -41,6 +41,7 @@ func NewMockExchangeCollection(pathRepresentation []string, numMessagesToReturn c.Data = append(c.Data, GetMockMessageBytes("From: NewMockExchangeCollection")) c.Names = append(c.Names, uuid.NewString()) } + return c } @@ -55,6 +56,7 @@ func (medc *MockExchangeDataCollection) Items() <-chan data.Stream { go func() { defer close(res) + for i := 0; i < medc.messageCount; i++ { res <- &MockExchangeData{ ID: medc.Names[i], @@ -114,6 +116,7 @@ func GetMockContactBytes(middleName string) []byte { contact := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEOAADSEBNbUIB9RL6ePDeF3FIYAABS7DZnAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('foobar%408qzvrj.onmicrosoft.com')/contacts/$entity\",\"@odata.etag\":\"W/\\\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\\\"\",\"categories\":[],\"changeKey\":\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\",\"createdDateTime\":\"2019-08-04T06:55:33Z\",\"lastModifiedDateTime\":\"2019-08-04T06:55:33Z\",\"businessAddress\":{},\"businessPhones\":[],\"children\":[],\"displayName\":\"Santiago Quail\",\"emailAddresses\":[],\"fileAs\":\"Quail, Santiago\"," + //nolint:lll "\"givenName\":\"Santiago " + middleName + "\",\"homeAddress\":{},\"homePhones\":[],\"imAddresses\":[],\"otherAddress\":{},\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9FIYAAAAAAEOAAA=\",\"personalNotes\":\"\",\"surname\":\"Quail\"}" + return []byte(contact) } @@ -130,6 +133,7 @@ func GetMockEventBytes(subject string) []byte { ":{\"emailAddress\":{\"address\":\"foobar3@8qzvrj.onmicrosoft.com\",\"name\":\"Anu Pierson\"}},\"originalEndTimeZone\":\"UTC\",\"originalStartTimeZone\":\"UTC\",\"reminderMinutesBeforeStart\":15,\"responseRequested\":true,\"responseStatus\":{\"response\":\"notResponded\",\"time\":\"0001-01-01T00:00:00Z\"},\"sensitivity\":\"normal\",\"showAs\":\"tentative\",\"start\":{\"dateTime\":\"2022-04-28T03:41:58.0000000\",\"timeZone\":\"UTC\"}," + "\"subject\":\" " + subject + "Review + Lunch\",\"type\":\"singleInstance\",\"webLink\":\"https://outlook.office365.com/owa/?itemid=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAADSEBNbUIB9RL6ePDeF3FIYAAAAAG76AAA%3D&exvsurl=1&path=/calendar/item\"}" + return []byte(event) } diff --git a/src/internal/connector/mockconnector/mock_data_collection_test.go b/src/internal/connector/mockconnector/mock_data_collection_test.go index 4370e3354..44cf1035f 100644 --- a/src/internal/connector/mockconnector/mock_data_collection_test.go +++ b/src/internal/connector/mockconnector/mock_data_collection_test.go @@ -33,6 +33,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() { assert.NoError(suite.T(), err) messagesRead++ } + assert.Equal(suite.T(), 2, messagesRead) } @@ -46,6 +47,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchange for stream := range mdc.Items() { _, err := buf.ReadFrom(stream.ToReader()) assert.NoError(t, err) + byteArray := buf.Bytes() something, err := support.CreateFromBytes(byteArray, models.CreateMessageFromDiscriminatorValue) assert.NoError(t, err) diff --git a/src/internal/connector/support/errors.go b/src/internal/connector/support/errors.go index 5a863f9e6..d74589f09 100644 --- a/src/internal/connector/support/errors.go +++ b/src/internal/connector/support/errors.go @@ -47,6 +47,7 @@ func GetNumberOfErrors(err error) int { if err == nil { return 0 } + result, _, wasFound := strings.Cut(err.Error(), " ") if wasFound { aNum, err := strconv.Atoi(result) @@ -54,6 +55,7 @@ func GetNumberOfErrors(err error) int { return aNum } } + return 1 } @@ -62,13 +64,16 @@ func GetNumberOfErrors(err error) int { // depends on ConnectorStackErrorTrace func ListErrors(multi multierror.Error) string { aString := "" + for idx, err := range multi.Errors { detail := ConnectorStackErrorTrace(err) if detail == "" { detail = fmt.Sprintf("%v", err) } + aString = aString + fmt.Sprintf("\n\tErr: %d %v", idx+1, detail) } + return aString } @@ -80,6 +85,7 @@ func concatenateStringFromPointers(orig string, pointers []*string) string { orig = strings.Join([]string{orig, *pointer}, " ") } } + return orig } @@ -87,6 +93,7 @@ func concatenateStringFromPointers(orig string, pointers []*string) string { // stack trace for oDataError types from querying the M365 back store. func ConnectorStackErrorTrace(e error) string { eMessage := "" + if oDataError, ok := e.(msgraph_errors.ODataErrorable); ok { // Get MainError mainErr := oDataError.GetError() @@ -102,10 +109,12 @@ func ConnectorStackErrorTrace(e error) string { inners := mainErr.GetInnererror() eMessage = concatenateStringFromPointers(eMessage, []*string{code, subject, target}) + // Get Error Details // code, message, target if details != nil { eMessage = eMessage + "\nDetails Section:" + for idx, detail := range details { dMessage := fmt.Sprintf("Detail %d:", idx) c := detail.GetCode() @@ -116,6 +125,7 @@ func ConnectorStackErrorTrace(e error) string { eMessage = eMessage + dMessage } } + if inners != nil { eMessage = eMessage + "\nConnector Section:" client := inners.GetClientRequestId() @@ -124,5 +134,6 @@ func ConnectorStackErrorTrace(e error) string { []*string{client, rID}) } } + return eMessage } diff --git a/src/internal/connector/support/errors_test.go b/src/internal/connector/support/errors_test.go index eaaeb8bad..a81d3c10d 100644 --- a/src/internal/connector/support/errors_test.go +++ b/src/internal/connector/support/errors_test.go @@ -25,30 +25,38 @@ func (suite *GraphConnectorErrorSuite) TestWrapAndAppend() { returnErr := WrapAndAppend("arc376", err2, err1) suite.True(strings.Contains(returnErr.Error(), "arc376")) suite.Error(returnErr) + multi := &multierror.Error{Errors: []error{err1, err2}} suite.True(strings.Contains(ListErrors(*multi), "two")) // Does not contain the wrapped information suite.T().Log(ListErrors(*multi)) } func (suite *GraphConnectorErrorSuite) TestWrapAndAppend_OnVar() { - var err1 error - id := "xi2058" + var ( + err1 error + id = "xi2058" + ) + received := WrapAndAppend(id, errors.New("network error"), err1) suite.True(strings.Contains(received.Error(), id)) } func (suite *GraphConnectorErrorSuite) TestAsRecoverableError() { err := assert.AnError - var rcv RecoverableGCError + + rcv := RecoverableGCError{} suite.False(errors.As(err, &rcv)) + aRecoverable := SetRecoverableError(err) suite.True(errors.As(aRecoverable, &rcv)) } func (suite *GraphConnectorErrorSuite) TestAsNonRecoverableError() { err := assert.AnError - var noRecover NonRecoverableGCError + + noRecover := NonRecoverableGCError{} suite.False(errors.As(err, &noRecover)) + nonRecoverable := SetNonRecoverableError(err) suite.True(errors.As(nonRecoverable, &noRecover)) } @@ -70,12 +78,15 @@ func (suite *GraphConnectorErrorSuite) TestWrapAndAppendf() { } func (suite *GraphConnectorErrorSuite) TestConcatenateStringFromPointers() { - var s1, s2, s3 *string - var outString string - v1 := "Corso" - v3 := "remains" - s1 = &v1 - s3 = &v3 + var ( + outString string + v1 = "Corso" + v3 = "remains" + s1 = &v1 + s2 *string + s3 = &v3 + ) + outString = concatenateStringFromPointers(outString, []*string{s1, s2, s3}) suite.True(strings.Contains(outString, v1)) suite.True(strings.Contains(outString, v3)) diff --git a/src/internal/connector/support/m365Support.go b/src/internal/connector/support/m365Support.go index 51c448865..94b75ece9 100644 --- a/src/internal/connector/support/m365Support.go +++ b/src/internal/connector/support/m365Support.go @@ -18,6 +18,7 @@ func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Pa if err != nil { return nil, err } + return anObject, nil } @@ -27,7 +28,9 @@ func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) { if err != nil { return nil, err } + message := aMessage.(models.Messageable) + return message, nil } @@ -38,7 +41,9 @@ func CreateContactFromBytes(bytes []byte) (models.Contactable, error) { if err != nil { return nil, err } + contact := parsable.(models.Contactable) + return contact, nil } @@ -48,6 +53,8 @@ func CreateEventFromBytes(bytes []byte) (models.Eventable, error) { if err != nil { return nil, err } + event := parsable.(models.Eventable) + return event, nil } diff --git a/src/internal/connector/support/m365Transform.go b/src/internal/connector/support/m365Transform.go index 2554f887e..2bdf51e2e 100644 --- a/src/internal/connector/support/m365Transform.go +++ b/src/internal/connector/support/m365Transform.go @@ -44,48 +44,58 @@ func CloneMessageableFields(orig, message models.Messageable) models.Messageable message.SetMultiValueExtendedProperties(orig.GetMultiValueExtendedProperties()) message.SetUniqueBody(orig.GetUniqueBody()) message.SetWebLink(orig.GetWebLink()) + return message } func ToMessage(orig models.Messageable) models.Messageable { message := models.NewMessage() temp := CloneMessageableFields(orig, message) + aMessage, ok := temp.(*models.Message) if !ok { return nil } + return aMessage } func SetEventMessageRequest(orig models.Messageable, adtl map[string]any) (models.EventMessageRequestable, error) { aMessage := models.NewEventMessageRequest() temp := CloneMessageableFields(orig, aMessage) + message, ok := temp.(models.EventMessageRequestable) if !ok { return nil, errors.New(*orig.GetId() + " failed to convert to eventMessageRequestable") } + newMessage, err := SetAdditionalDataToEventMessage(adtl, message) if err != nil { return nil, errors.Wrap(err, *orig.GetId()+" eventMessageRequest could not set additional data") } + additional, err := buildMapFromAdditional(eventRequestableFields, adtl) if err != nil { return nil, errors.Wrap(err, *orig.GetId()+" eventMessageRequest failed on method buildMapFromAdditional") } + message, ok = newMessage.(models.EventMessageRequestable) if !ok { return nil, errors.New(*orig.GetId() + " failed to convert to eventMessageRequestable") } + eventMessage, err := setEventRequestableFields(message, additional) if err != nil { return nil, err } + return eventMessage, nil } func SetEventMessageResponse(orig models.Messageable, adtl map[string]any) (models.EventMessageResponseable, error) { aMessage := models.NewEventMessageResponse() temp := CloneMessageableFields(orig, aMessage) + message, ok := temp.(models.EventMessageResponseable) if !ok { return nil, errors.New(*orig.GetId() + " failed to convert to eventMessageRequestable") @@ -95,14 +105,17 @@ func SetEventMessageResponse(orig models.Messageable, adtl map[string]any) (mode if err != nil { return nil, errors.Wrap(err, *orig.GetId()+" eventMessageResponse could not set additional data") } + message, ok = newMessage.(models.EventMessageResponseable) if !ok { return nil, errors.New("unable to create event message responseable from " + *orig.GetId()) } + additional, err := buildMapFromAdditional(eventResponsableFields, adtl) if err != nil { return nil, errors.Wrap(err, *orig.GetId()+" eventMessageResponse failed on method buildMapFromAdditional") } + for key, val := range additional { switch key { case "responseType": @@ -110,6 +123,7 @@ func SetEventMessageResponse(orig models.Messageable, adtl map[string]any) (mode if err != nil { return nil, errors.Wrap(err, *orig.GetId()+"failure to parse response type") } + rType, ok := temp.(*models.ResponseType) if !ok { return nil, fmt.Errorf( @@ -119,11 +133,14 @@ func SetEventMessageResponse(orig models.Messageable, adtl map[string]any) (mode temp, ) } + message.SetResponseType(rType) + default: return nil, errors.New(key + " not supported for setEventMessageResponse") } } + return message, nil } @@ -131,28 +148,35 @@ func SetEventMessageResponse(orig models.Messageable, adtl map[string]any) (mode // type until upstream can make the appropriate changes func ConvertFromMessageable(adtl map[string]any, orig models.Messageable) (models.EventMessageable, error) { var aType string + aPointer, ok := adtl["@odata.type"] if !ok { return nil, errors.New("unknown data type: no @odata.type field") } + ptr, ok := aPointer.(*string) if !ok { return nil, errors.New("unknown map type encountered") } + aType = *ptr if aType == "#microsoft.graph.eventMessageRequest" { eventRequest, err := SetEventMessageRequest(orig, adtl) if err != nil { return nil, err } + eventRequest.SetId(orig.GetId()) + return eventRequest, err } + if aType == "#microsoft.graph.eventMessageResponse" { eventMessage, err := SetEventMessageResponse(orig, adtl) if err != nil { return nil, err } + eventMessage.SetId(orig.GetId()) return eventMessage, nil @@ -164,24 +188,30 @@ func ConvertFromMessageable(adtl map[string]any, orig models.Messageable) (model // buildMapFromAdditional returns a submap of map[string]*string from map[string]any func buildMapFromAdditional(list []string, adtl map[string]any) (map[string]*string, error) { returnMap := make(map[string]*string) + for _, entry := range list { ptr, ok := adtl[entry] if !ok { continue } + value, ok := ptr.(*string) if !ok { boolConvert, ok := ptr.(*bool) if !ok { return nil, errors.New("unsupported value type: key: " + entry + fmt.Sprintf(" with type: %T", ptr)) } + aBool := *boolConvert boolString := strconv.FormatBool(aBool) returnMap[entry] = &boolString + continue } + returnMap[entry] = value } + return returnMap, nil } @@ -196,25 +226,32 @@ func setEventRequestableFields( if err != nil { return nil, errors.Wrap(err, *em.GetId()+": failed on models.ParseMeetingRequestType") } + rType, ok := temp.(*models.MeetingRequestType) if !ok { return nil, errors.New(*em.GetId() + ": failed to set meeting request type") } + em.SetMeetingRequestType(rType) + case "responseRequested": boolValue, err := strconv.ParseBool(*value) if err != nil { return nil, errors.Wrap(err, *em.GetId()+": failed to set responseRequested") } + em.SetResponseRequested(&boolValue) + case "allowNewTimeProposals": boolValue, err := strconv.ParseBool(*value) if err != nil { return nil, errors.Wrap(err, *em.GetId()+": failed to set allowNewTimeProposals") } + em.SetAllowNewTimeProposals(&boolValue) } } + return em, nil } @@ -226,14 +263,17 @@ func SetAdditionalDataToEventMessage( for key, entry := range adtl { if key == "endDateTime" { dateTime := models.NewDateTimeTimeZone() + mapped, ok := entry.(map[string]*kw.JsonParseNode) if ok { for key, val := range mapped { node := *val + value, err := node.GetStringValue() if err != nil { return nil, err } + switch key { case "dateTime": dateTime.SetDateTime(value) @@ -242,21 +282,27 @@ func SetAdditionalDataToEventMessage( default: return nil, errors.New("key not supported DateTime") } + newMessage.SetEndDateTime(dateTime) } + continue } } + if key == "startDateTime" { dateTime := models.NewDateTimeTimeZone() + mapped, ok := entry.(map[string]*kw.JsonParseNode) if ok { for key, val := range mapped { node := *val + value, err := node.GetStringValue() if err != nil { return nil, err } + switch key { case "dateTime": dateTime.SetDateTime(value) @@ -264,23 +310,28 @@ func SetAdditionalDataToEventMessage( dateTime.SetTimeZone(value) default: return nil, errors.New("key not supported DateTime") - } + newMessage.SetStartDateTime(dateTime) } + continue } } + if key == "location" { aLocation := models.NewLocation() + mapped, ok := entry.(map[string]*kw.JsonParseNode) if ok { for key, val := range mapped { node := *val + value, err := node.GetStringValue() if err != nil { return nil, errors.New("map[string]*JsonParseNode conversion failure") } + switch key { case "displayName": aLocation.SetDisplayName(value) @@ -289,16 +340,20 @@ func SetAdditionalDataToEventMessage( if err != nil { return nil, errors.New("location type parse failure") } + lType, ok := temp.(*models.LocationType) if !ok { return nil, errors.New("location type interface failure") } + aLocation.SetLocationType(lType) } } } + newMessage.SetLocation(aLocation) } + value, ok := entry.(*string) if ok { switch key { @@ -307,31 +362,40 @@ func SetAdditionalDataToEventMessage( if err != nil { return nil, err } + newMessage.SetIsAllDay(&boolValue) + case "isDelegated": boolValue, err := strconv.ParseBool(*value) if err != nil { return nil, err } + newMessage.SetIsDelegated(&boolValue) + case "isOutOfDate": boolValue, err := strconv.ParseBool(*value) if err != nil { return nil, err } + newMessage.SetIsOutOfDate(&boolValue) + case "meetingMessageType": temp, err := models.ParseMeetingMessageType(*value) if err != nil { return nil, err } + mType, ok := temp.(*models.MeetingMessageType) if !ok { return nil, errors.New("failed to create meeting message type") } + newMessage.SetMeetingMessageType(mType) } } } + return newMessage, nil } diff --git a/src/internal/connector/support/m365Transform_test.go b/src/internal/connector/support/m365Transform_test.go index b95aa68e8..20ab6810e 100644 --- a/src/internal/connector/support/m365Transform_test.go +++ b/src/internal/connector/support/m365Transform_test.go @@ -21,6 +21,7 @@ func (suite *SupportTestSuite) TestToMessage() { bytes := mockconnector.GetMockMessageBytes("m365 mail support test") message, err := CreateMessageFromBytes(bytes) require.NoError(suite.T(), err) + clone := ToMessage(message) suite.Equal(message.GetBccRecipients(), clone.GetBccRecipients()) suite.Equal(message.GetSubject(), clone.GetSubject()) diff --git a/src/internal/connector/support/status.go b/src/internal/connector/support/status.go index 8109a41b0..695ae61fd 100644 --- a/src/internal/connector/support/status.go +++ b/src/internal/connector/support/status.go @@ -33,11 +33,13 @@ func CreateStatus( objects, success, folders int, err error, ) *ConnectorOperationStatus { - hasErrors := err != nil var reason string + if err != nil { reason = err.Error() } + + hasErrors := err != nil numErr := GetNumberOfErrors(err) status := ConnectorOperationStatus{ lastOperation: op, @@ -48,6 +50,7 @@ func CreateStatus( incomplete: hasErrors, incompleteReason: reason, } + if status.ObjectCount != status.errorCount+status.Successful { logger.Ctx(ctx).DPanicw( "status object count does not match errors + successes", @@ -55,6 +58,7 @@ func CreateStatus( "successes", success, "errors", numErr) } + return &status } @@ -64,6 +68,8 @@ func (cos *ConnectorOperationStatus) String() string { if cos.incomplete { message += " " + cos.incompleteReason } + message = message + "\n" + return message } diff --git a/src/internal/connector/support/status_test.go b/src/internal/connector/support/status_test.go index 36191a44b..439543a70 100644 --- a/src/internal/connector/support/status_test.go +++ b/src/internal/connector/support/status_test.go @@ -66,6 +66,7 @@ func (suite *GCStatusTestSuite) TestCreateStatus() { func (suite *GCStatusTestSuite) TestCreateStatus_InvalidStatus() { t := suite.T() params := statusParams{Backup, 9, 3, 13, errors.New("invalidcl")} + require.Panics(t, func() { CreateStatus( context.Background(),