GC: Add WSL Linting Formatting src/internal/connector (#694)

Lint remaining of `internal/connector` package for wsl
This commit is contained in:
Danny 2022-08-30 18:12:38 -04:00 committed by GitHub
parent 6f04321a60
commit 67bc038c55
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 117 additions and 28 deletions

View File

@ -80,13 +80,5 @@ issues:
- revive
text: "import-shadowing:.*'suite' shadows"
# Temporarily skip linting wsl on `connector` package until fixes are merged.
- path: internal/connector/graph_connector_test.go
linters: wsl
- path: internal/connector/graph_connector.go
linters: wsl
- path: internal/connector/exchange/exchange_service_test.go
linters: wsl
- path: internal/connector/exchange/service_functions.go
linters: wsl
- path: internal/connector/onedrive
linters: wsl

View File

@ -33,6 +33,7 @@ func TestExchangeServiceSuite(t *testing.T) {
); err != nil {
t.Skip(err)
}
suite.Run(t, new(ExchangeServiceSuite))
}
@ -47,6 +48,7 @@ func (suite *ExchangeServiceSuite) SetupSuite() {
require.NoError(t, err)
service, err := createService(m365, false)
require.NoError(t, err)
suite.es = service
}
@ -207,6 +209,7 @@ func (suite *ExchangeServiceSuite) TestSetupExchangeCollection() {
sel.Include(sel.Users([]string{userID}))
eb, err := sel.ToExchangeBackup()
require.NoError(suite.T(), err)
scopes := eb.Scopes()
for _, test := range scopes {
@ -253,6 +256,7 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
function: GetAllEventsForUser,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
response, err := test.function(suite.es, userID)
@ -341,6 +345,7 @@ func (suite *ExchangeServiceSuite) TestGetFolderID() {
checkError: assert.NoError,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
_, err := GetFolderID(
@ -356,17 +361,24 @@ func (suite *ExchangeServiceSuite) TestGetFolderID() {
// TestIterativeFunctions verifies that GraphQuery to Iterate
// functions are valid for current versioning of msgraph-go-sdk
func (suite *ExchangeServiceSuite) TestIterativeFunctions() {
userID := tester.M365UserID(suite.T())
sel := selectors.NewExchangeBackup()
var (
mailScope, contactScope selectors.ExchangeScope
userID = tester.M365UserID(suite.T())
sel = selectors.NewExchangeBackup()
)
sel.Include(sel.Users([]string{userID}))
eb, err := sel.ToExchangeBackup()
require.NoError(suite.T(), err)
scopes := eb.Scopes()
var mailScope, contactScope selectors.ExchangeScope
for _, scope := range scopes {
if scope.IncludesCategory(selectors.ExchangeContactFolder) {
contactScope = scope
}
if scope.IncludesCategory(selectors.ExchangeMail) {
mailScope = scope
}
@ -437,6 +449,7 @@ func (suite *ExchangeServiceSuite) TestRestoreContact() {
folderName := "TestRestoreContact: " + common.FormatSimpleDateTime(now)
aFolder, err := CreateContactFolder(suite.es, userID, folderName)
require.NoError(t, err)
folderID := *aFolder.GetId()
err = RestoreExchangeContact(context.Background(),
mockconnector.GetMockContactBytes("Corso TestContact"),
@ -487,6 +500,7 @@ func (suite *ExchangeServiceSuite) TestEstablishFolder() {
now := time.Now()
folderName := "CorsoEstablishFolder" + common.FormatSimpleDateTime(now)
userID := tester.M365UserID(suite.T())
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := establishFolder(suite.es, folderName, userID, test.option)

View File

@ -56,12 +56,14 @@ func createService(credentials account.M365Config, shouldFailFast bool) (*exchan
if err != nil {
return nil, err
}
service := exchangeService{
adapter: *adapter,
client: *msgraphsdk.NewGraphServiceClient(adapter),
failFast: shouldFailFast,
credentials: credentials,
}
return &service, err
}
@ -70,6 +72,7 @@ func createService(credentials account.M365Config, shouldFailFast bool) (*exchan
func CreateMailFolder(gs graph.Service, user, folder string) (models.MailFolderable, error) {
requestBody := models.NewMailFolder()
requestBody.SetDisplayName(&folder)
isHidden := false
requestBody.SetIsHidden(&isHidden)
@ -110,6 +113,7 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde
mfs = []MailFolder{}
err error
)
resp, err := GetAllFolderNamesForUser(gs, user)
if err != nil {
return nil, err
@ -143,6 +147,7 @@ func GetAllMailFolders(gs graph.Service, user, nameContains string) ([]MailFolde
if err := iter.Iterate(cb); err != nil {
return nil, err
}
return mfs, err
}
@ -157,6 +162,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option
query GraphQuery
transform absser.ParsableFactory
)
switch category {
case messages:
query = GetAllFolderNamesForUser
@ -176,6 +182,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option
user, support.ConnectorStackErrorTrace(err),
)
}
pageIterator, err := msgraphgocore.NewPageIterator(
response,
service.Adapter(),
@ -184,6 +191,7 @@ func GetFolderID(service graph.Service, folderName, user string, category option
if err != nil {
return nil, err
}
callbackFunc := iterateFindFolderID(category,
&folderID,
folderName,
@ -211,12 +219,16 @@ func parseCalendarIDFromEvent(reference string) (string, error) {
if len(stringArray) < 2 {
return "", errors.New("calendarID not found")
}
temp := stringArray[1]
stringArray = strings.Split(temp, "')/$ref")
if len(stringArray) < 2 {
return "", errors.New("calendarID not found")
}
calendarID := stringArray[0]
if len(calendarID) == 0 {
return "", errors.New("calendarID empty")
}
@ -245,6 +257,7 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
IterateAndFilterMessagesForCollections,
nil
}
if scope.IncludesCategory(selectors.ExchangeEvent) {
return models.CreateEventCollectionResponseFromDiscriminatorValue,
GetAllEventsForUser,
@ -271,6 +284,7 @@ func GetRestoreFolder(
user, category string,
) (string, error) {
newFolder := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat))
switch category {
case mailCategory, contactsCategory:
return establishFolder(service, newFolder, user, categoryToOptionIdentifier(category))
@ -292,18 +306,21 @@ func establishFolder(
if !errors.Is(err, ErrFolderNotFound) {
return "", support.WrapAndAppend(user, err, err)
}
switch optID {
case messages:
fold, err := CreateMailFolder(service, user, folderName)
if err != nil {
return "", support.WrapAndAppend(user, err, err)
}
return *fold.GetId(), nil
case contacts:
fold, err := CreateContactFolder(service, user, folderName)
if err != nil {
return "", support.WrapAndAppend(user, err, err)
}
return *fold.GetId(), nil
default:
return "", fmt.Errorf("category: %s not supported for folder creation", optID)
@ -319,12 +336,14 @@ func RestoreExchangeObject(
destination, user string,
) error {
var setting optionIdentifier
switch category {
case mailCategory, contactsCategory:
setting = categoryToOptionIdentifier(category)
default:
return fmt.Errorf("type: %s not supported for exchange restore", category)
}
if policy != control.Copy {
return fmt.Errorf("restore policy: %s not supported", policy)
}
@ -361,9 +380,11 @@ func RestoreExchangeContact(
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
if response == nil {
return errors.New("msgraph contact post fail: REST response not received")
}
return nil
}
@ -395,6 +416,7 @@ func RestoreMailMessage(
sv.SetValue(&enableValue)
svlep := []models.SingleValueLegacyExtendedPropertyable{sv}
clone.SetSingleValueExtendedProperties(svlep)
draft := false
clone.SetIsDraft(&draft)
@ -418,8 +440,10 @@ func SendMailToBackStore(service graph.Service, user, destination string, messag
if err != nil {
return support.WrapAndAppend(": "+support.ConnectorStackErrorTrace(err), err, nil)
}
if sentMessage == nil {
return errors.New("message not Sent: blocked by server")
}
return nil
}

View File

@ -9,7 +9,6 @@ import (
"strings"
"sync/atomic"
absser "github.com/microsoft/kiota-abstractions-go/serialization"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -67,6 +66,7 @@ func NewGraphConnector(acct account.Account) (*GraphConnector, error) {
if err != nil {
return nil, errors.Wrap(err, "retrieving m356 account configuration")
}
gc := GraphConnector{
tenant: m365.TenantID,
Users: make(map[string]string, 0),
@ -74,15 +74,19 @@ func NewGraphConnector(acct account.Account) (*GraphConnector, error) {
statusCh: make(chan *support.ConnectorOperationStatus),
credentials: m365,
}
aService, err := gc.createService(false)
if err != nil {
return nil, err
}
gc.graphService = *aService
err = gc.setTenantUsers()
if err != nil {
return nil, err
}
return &gc, nil
}
@ -96,11 +100,13 @@ func (gc *GraphConnector) createService(shouldFailFast bool) (*graphService, err
if err != nil {
return nil, err
}
connector := graphService{
adapter: *adapter,
client: *msgraphsdk.NewGraphServiceClient(adapter),
failFast: shouldFailFast,
}
return &connector, err
}
@ -121,10 +127,12 @@ func (gc *GraphConnector) setTenantUsers() error {
support.ConnectorStackErrorTrace(err),
)
}
if response == nil {
err = support.WrapAndAppend("general access", errors.New("connector failed: No access"), err)
return err
}
userIterator, err := msgraphgocore.NewPageIterator(
response,
&gc.graphService.adapter,
@ -133,31 +141,35 @@ func (gc *GraphConnector) setTenantUsers() error {
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
var iterateError error
callbackFunc := func(userItem interface{}) bool {
user, ok := userItem.(models.Userable)
if !ok {
err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), errors.New("user iteration failure"), err)
return true
}
if user.GetUserPrincipalName() == nil {
err = support.WrapAndAppend(
gc.graphService.adapter.GetBaseUrl(),
fmt.Errorf("no email address for User: %s", *user.GetId()),
err,
)
return true
}
// *user.GetId() is populated for every M365 entityable object by M365 backstore
gc.Users[*user.GetUserPrincipalName()] = *user.GetId()
return true
}
iterateError = userIterator.Iterate(callbackFunc)
iterateError := userIterator.Iterate(callbackFunc)
if iterateError != nil {
err = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, err)
}
return err
}
@ -175,6 +187,7 @@ func (gc *GraphConnector) GetUsersIds() []string {
// Returns list of keys iff true; otherwise returns a list of values
func buildFromMap(isKey bool, mapping map[string]string) []string {
returnString := make([]string, 0)
if isKey {
for k := range mapping {
returnString = append(returnString, k)
@ -184,6 +197,7 @@ func buildFromMap(isKey bool, mapping map[string]string) []string {
returnString = append(returnString, v)
}
}
return returnString
}
@ -199,9 +213,13 @@ func (gc *GraphConnector) ExchangeDataCollection(
if err != nil {
return nil, errors.Wrap(err, "exchangeDataCollection: unable to parse selector")
}
scopes := eb.DiscreteScopes(gc.GetUsers())
collections := []data.Collection{}
var errs error
var (
scopes = eb.DiscreteScopes(gc.GetUsers())
collections = []data.Collection{}
errs error
)
for _, scope := range scopes {
// Creates a map of collections based on scope
dcs, err := gc.createCollections(ctx, scope)
@ -209,6 +227,7 @@ func (gc *GraphConnector) ExchangeDataCollection(
user := scope.Get(selectors.ExchangeUser)
return nil, support.WrapAndAppend(user[0], err, errs)
}
for _, collection := range dcs {
collections = append(collections, collection)
}
@ -229,22 +248,28 @@ func (gc *GraphConnector) RestoreExchangeDataCollection(
attempts, successes int
errs error
folderID string
// TODO policy to be updated from external source after completion of refactoring
policy = control.Copy
)
policy := control.Copy // TODO policy to be updated from external source after completion of refactoring
for _, dc := range dcs {
directory := strings.Join(dc.FullPath(), "")
user := dc.FullPath()[1]
items := dc.Items()
category := dc.FullPath()[2]
var (
directory = strings.Join(dc.FullPath(), "")
user = dc.FullPath()[1]
items = dc.Items()
category = dc.FullPath()[2]
exit bool
)
if _, ok := pathCounter[directory]; !ok {
pathCounter[directory] = true
folderID, errs = exchange.GetRestoreFolder(&gc.graphService, user, category)
if errs != nil {
return errs
}
}
var exit bool
for !exit {
select {
case <-ctx.Done():
@ -257,11 +282,13 @@ func (gc *GraphConnector) RestoreExchangeDataCollection(
attempts++
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(itemData.ToReader())
if err != nil {
errs = support.WrapAndAppend(itemData.UUID(), err, errs)
continue
}
err = exchange.RestoreExchangeObject(ctx, buf.Bytes(), category, policy, &gc.graphService, folderID, user)
if err != nil {
@ -272,12 +299,15 @@ func (gc *GraphConnector) RestoreExchangeDataCollection(
}
}
}
gc.incrementAwaitingMessages()
status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs)
// set the channel asynchronously so that this func doesn't block.
go func(cos *support.ConnectorOperationStatus) {
gc.statusCh <- cos
}(status)
return errs
}
@ -290,20 +320,20 @@ func (gc *GraphConnector) createCollections(
scope selectors.ExchangeScope,
) ([]*exchange.Collection, error) {
var (
transformer absser.ParsableFactory
query exchange.GraphQuery
gIter exchange.GraphIterateFunc
errs error
errs error
transformer, query, gIter, err = exchange.SetupExchangeCollectionVars(scope)
)
transformer, query, gIter, err := exchange.SetupExchangeCollectionVars(scope)
if err != nil {
return nil, support.WrapAndAppend(gc.Service().Adapter().GetBaseUrl(), err, nil)
}
users := scope.Get(selectors.ExchangeUser)
allCollections := make([]*exchange.Collection, 0)
// Create collection of ExchangeDataCollection
for _, user := range users {
collections := make(map[string]*exchange.Collection)
response, err := query(&gc.graphService, user)
if err != nil {
return nil, errors.Wrapf(
@ -322,17 +352,22 @@ func (gc *GraphConnector) createCollections(
// Each directory used the M365 Identifier. The use of ID stops collisions betweens users
callbackFunc := gIter(user, scope, errs, gc.failFast, gc.credentials, collections, gc.statusCh)
iterateError := pageIterator.Iterate(callbackFunc)
if iterateError != nil {
errs = support.WrapAndAppend(gc.graphService.adapter.GetBaseUrl(), iterateError, errs)
}
if errs != nil {
return nil, errs // return error if snapshot is incomplete
}
for _, collection := range collections {
gc.incrementAwaitingMessages()
allCollections = append(allCollections, collection)
}
}
return allCollections, errs
}
@ -342,6 +377,7 @@ func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
atomic.AddInt32(&gc.awaitingMessages, -1)
gc.status = <-gc.statusCh
}
return gc.status
}
@ -355,6 +391,7 @@ func (gc *GraphConnector) PrintableStatus() string {
if gc.status == nil {
return ""
}
return gc.status.String()
}

View File

@ -31,6 +31,7 @@ func loadConnector(t *testing.T) *GraphConnector {
a := tester.NewM365Account(t)
connector, err := NewGraphConnector(a)
require.NoError(t, err)
return connector
}
@ -41,6 +42,7 @@ func TestGraphConnectorIntegrationSuite(t *testing.T) {
); err != nil {
t.Skip(err)
}
suite.Run(t, new(GraphConnectorIntegrationSuite))
}
@ -66,8 +68,10 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
statusCh: make(chan *support.ConnectorOperationStatus),
credentials: suite.connector.credentials,
}
service, err := newConnector.createService(false)
require.NoError(suite.T(), err)
newConnector.graphService = *service
suite.Equal(len(newConnector.Users), 0)
@ -92,6 +96,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
assert.NoError(t, err)
assert.True(t, connector.awaitingMessages > 0)
assert.Nil(t, connector.status)
streams := make(map[string]<-chan data.Stream)
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
for _, collection := range collectionList {
@ -115,6 +120,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
}
})
}
exchangeData := collectionList[0]
suite.Greater(len(exchangeData.FullPath()), 2)
}
@ -129,11 +135,13 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
sel.Include(sel.MailFolders([]string{suite.user}, selectors.Any()))
eb, err := sel.ToExchangeBackup()
require.NoError(t, err)
scopes := eb.Scopes()
suite.Len(scopes, 1)
mailScope := scopes[0]
collection, err := connector.createCollections(context.Background(), mailScope)
require.NoError(t, err)
for _, edc := range collection {
testName := strings.Join(edc.FullPath(), " ")
suite.T().Run(testName, func(t *testing.T) {
@ -150,6 +158,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
}
})
}
status := connector.AwaitStatus()
suite.NotNil(status)
suite.Equal(status.ObjectCount, status.Successful)
@ -164,13 +173,17 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
sel.Include(sel.ContactFolders([]string{suite.user}, selectors.Any()))
eb, err := sel.ToExchangeBackup()
require.NoError(t, err)
scopes := eb.Scopes()
connector := loadConnector(t)
suite.Len(scopes, 1)
contactsOnly := scopes[0]
collections, err := connector.createCollections(context.Background(), contactsOnly)
assert.NoError(t, err)
number := 0
for _, edc := range collections {
testName := fmt.Sprintf("%s_ContactFolder_%d", edc.FullPath()[1], number)
suite.T().Run(testName, func(t *testing.T) {
@ -188,6 +201,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
number++
})
}
status := connector.AwaitStatus()
suite.NotNil(status)
suite.Equal(status.ObjectCount, status.Successful)
@ -204,9 +218,11 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
suite.Equal(len(scopes), 1)
collections, err := connector.createCollections(context.Background(), scopes[0])
require.NoError(t, err)
for _, edc := range collections {
streamChannel := edc.Items()
number := 0
for stream := range streamChannel {
testName := fmt.Sprintf("%s_Event_%d", edc.FullPath()[2], number)
suite.T().Run(testName, func(t *testing.T) {
@ -220,6 +236,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
})
}
}
status := connector.AwaitStatus()
suite.NotNil(status)
suite.Equal(status.ObjectCount, status.Successful)
@ -234,6 +251,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() {
category := "mail"
connector := loadConnector(t)
collection := make([]data.Collection, 0)
for i := 0; i < 3; i++ {
mdc := mockconnector.NewMockExchangeCollection(
[]string{"tenant", suite.user, category, "Inbox"},
@ -243,6 +261,7 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() {
err := connector.RestoreExchangeDataCollection(context.Background(), collection)
assert.NoError(suite.T(), err)
status := connector.AwaitStatus()
assert.NotNil(t, status)
assert.Equal(t, status.ObjectCount, status.Successful)
@ -261,6 +280,7 @@ func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() {
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders(selectors.Any(), []string{"Inbox"}))
scopes := sel.DiscreteScopes(connector.GetUsers())
for _, scope := range scopes {
users := scope.Get(selectors.ExchangeUser)
standard := (len(users) / 4) * 3
@ -281,6 +301,7 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteMailFolder() {
folderName := "TestFolder: " + common.FormatSimpleDateTime(now)
aFolder, err := exchange.CreateMailFolder(&suite.connector.graphService, suite.user, folderName)
assert.NoError(suite.T(), err, support.ConnectorStackErrorTrace(err))
if aFolder != nil {
err = exchange.DeleteMailFolder(suite.connector.Service(), suite.user, *aFolder.GetId())
assert.NoError(suite.T(), err)
@ -294,6 +315,7 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteContactFolder()
folderName := "TestContactFolder: " + common.FormatSimpleDateTime(now)
aFolder, err := exchange.CreateContactFolder(suite.connector.Service(), suite.user, folderName)
assert.NoError(suite.T(), err)
if aFolder != nil {
err = exchange.DeleteContactFolder(suite.connector.Service(), suite.user, *aFolder.GetId())
assert.NoError(suite.T(), err)