replace multierror with fault in setSites (#2377)

## Does this PR need a docs update or release note?

- [x]  No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-02-09 11:11:59 -07:00 committed by GitHub
parent 754981d0d2
commit dad6776861
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 133 additions and 67 deletions

View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
@ -272,20 +273,23 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
sel := exchangeBackupCreateSelectors(user, exchangeData)
users, err := m365.UserPNs(ctx, acct)
// TODO: log/print recoverable errors
errs := fault.New(false)
users, err := m365.UserPNs(ctx, acct, errs)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 user(s)"))
}
var (
errs *multierror.Error
bIDs []model.StableID
merrs *multierror.Error
bIDs []model.StableID
)
for _, discSel := range sel.SplitByResourceOwner(users) {
bo, err := r.NewBackup(ctx, discSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to initialize Exchange backup for user %s",
discSel.DiscreteOwner,
@ -296,7 +300,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to run Exchange backup for user %s",
discSel.DiscreteOwner,
@ -316,7 +320,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
if e := merrs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}

View File

@ -16,6 +16,7 @@ import (
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
@ -195,20 +196,23 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
sel := oneDriveBackupCreateSelectors(user)
users, err := m365.UserPNs(ctx, acct)
// TODO: log/print recoverable errors
errs := fault.New(false)
users, err := m365.UserPNs(ctx, acct, errs)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users"))
}
var (
errs *multierror.Error
bIDs []model.StableID
merrs *multierror.Error
bIDs []model.StableID
)
for _, discSel := range sel.SplitByResourceOwner(users) {
bo, err := r.NewBackup(ctx, discSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to initialize OneDrive backup for user %s",
discSel.DiscreteOwner,
@ -219,7 +223,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to run OneDrive backup for user %s",
discSel.DiscreteOwner,
@ -239,7 +243,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
if e := merrs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
@ -210,7 +211,10 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites)
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs"))
}
@ -221,14 +225,14 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
}
var (
errs *multierror.Error
bIDs []model.StableID
merrs *multierror.Error
bIDs []model.StableID
)
for _, discSel := range sel.SplitByResourceOwner(gc.GetSiteIDs()) {
bo, err := r.NewBackup(ctx, discSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to initialize SharePoint backup for site %s",
discSel.DiscreteOwner,
@ -239,7 +243,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
merrs = multierror.Append(merrs, errors.Wrapf(
err,
"Failed to run SharePoint backup for site %s",
discSel.DiscreteOwner,
@ -259,7 +263,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
if e := merrs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}
@ -315,7 +319,10 @@ func sharePointBackupCreateSelectors(
}
}
union, err := gc.UnionSiteIDsAndWebURLs(ctx, sites, weburls)
// TODO: log/print recoverable errors
errs := fault.New(false)
union, err := gc.UnionSiteIDsAndWebURLs(ctx, sites, weburls, errs)
if err != nil {
return nil, err
}

View File

@ -20,6 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -114,7 +115,10 @@ func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphCon
}
// build a graph connector
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users)
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs)
if err != nil {
return nil, account.Account{}, errors.Wrap(err, "connecting to graph api")
}

View File

@ -23,6 +23,7 @@ import (
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -178,7 +179,10 @@ func getGC(ctx context.Context) (*connector.GraphConnector, account.M365Config,
return nil, m365Cfg, Only(ctx, errors.Wrap(err, "finding m365 account details"))
}
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users)
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs)
if err != nil {
return nil, m365Cfg, Only(ctx, errors.Wrap(err, "connecting to graph API"))
}

View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
)
@ -260,7 +261,10 @@ func getGC(ctx context.Context) (*connector.GraphConnector, error) {
}
// build a graph connector
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users)
// TODO: log/print recoverable errors
errs := fault.New(false)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs)
if err != nil {
return nil, Only(ctx, errors.Wrap(err, "connecting to graph api"))
}

View File

@ -11,7 +11,6 @@ import (
"sync"
"github.com/alcionai/clues"
"github.com/hashicorp/go-multierror"
"github.com/microsoft/kiota-abstractions-go/serialization"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -25,6 +24,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
D "github.com/alcionai/corso/src/internal/diagnostics"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
)
@ -68,6 +68,7 @@ func NewGraphConnector(
itemClient *http.Client,
acct account.Account,
r resource,
errs *fault.Errors,
) (*GraphConnector, error) {
m365, err := acct.M365Config()
if err != nil {
@ -103,7 +104,7 @@ func NewGraphConnector(
}
if r == AllResources || r == Sites {
if err = gc.setTenantSites(ctx); err != nil {
if err = gc.setTenantSites(ctx, errs); err != nil {
return nil, errors.Wrap(err, "retrieveing tenant site list")
}
}
@ -159,7 +160,7 @@ func (gc *GraphConnector) GetUsersIds() []string {
// setTenantSites queries the M365 to identify the sites in the
// workspace. The sites field is updated during this method
// iff the returned error is nil.
func (gc *GraphConnector) setTenantSites(ctx context.Context) error {
func (gc *GraphConnector) setTenantSites(ctx context.Context, errs *fault.Errors) error {
gc.Sites = map[string]string{}
ctx, end := D.Span(ctx, "gc:setTenantSites")
@ -171,7 +172,8 @@ func (gc *GraphConnector) setTenantSites(ctx context.Context) error {
gc.tenant,
sharepoint.GetAllSitesForTenant,
models.CreateSiteCollectionResponseFromDiscriminatorValue,
identifySite)
identifySite,
errs)
if err != nil {
return err
}
@ -227,9 +229,13 @@ func (gc *GraphConnector) GetSiteIDs() []string {
// each element in the url must fully match. Ex: the webURL value "foo" will match "www.ex.com/foo",
// but not match "www.ex.com/foobar".
// The returned IDs are reduced to a set of unique values.
func (gc *GraphConnector) UnionSiteIDsAndWebURLs(ctx context.Context, ids, urls []string) ([]string, error) {
func (gc *GraphConnector) UnionSiteIDsAndWebURLs(
ctx context.Context,
ids, urls []string,
errs *fault.Errors,
) ([]string, error) {
if len(gc.Sites) == 0 {
if err := gc.setTenantSites(ctx); err != nil {
if err := gc.setTenantSites(ctx, errs); err != nil {
return nil, err
}
}
@ -308,6 +314,7 @@ func getResources(
query func(context.Context, graph.Servicer) (serialization.Parsable, error),
parser func(parseNode serialization.ParseNode) (serialization.Parsable, error),
identify func(any) (string, string, error),
errs *fault.Errors,
) (map[string]string, error) {
resources := map[string]string{}
@ -318,18 +325,20 @@ func getResources(
WithAll(graph.ErrData(err)...)
}
errs := &multierror.Error{}
iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), parser)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
callbackFunc := func(item any) bool {
if errs.Failed() {
return false
}
k, v, err := identify(item)
if err != nil {
if !errors.Is(err, errKnownSkippableCase) {
errs = multierror.Append(errs, clues.Stack(err).
errs.Add(clues.Stack(err).
WithClues(ctx).
With("query_url", gs.Adapter().GetBaseUrl()))
}
@ -346,5 +355,5 @@ func getResources(
return nil, clues.Stack(err).WithClues(ctx).WithAll(graph.ErrData(err)...)
}
return resources, errs.ErrorOrNil()
return resources, errs.Err()
}

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -66,9 +67,9 @@ func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
gc, err := NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), test.acct(t), Users)
gc, err := NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), test.acct(t), Users, fault.New(true))
assert.Nil(t, gc, test.name+" failed")
assert.NotNil(t, err, test.name+"failed")
assert.NotNil(t, err, test.name+" failed")
})
}
}

View File

@ -22,6 +22,7 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -1107,7 +1108,8 @@ func getSelectorWith(
func loadConnector(ctx context.Context, t *testing.T, itemClient *http.Client, r resource) *GraphConnector {
a := tester.NewM365Account(t)
connector, err := NewGraphConnector(ctx, itemClient, a, r)
connector, err := NewGraphConnector(ctx, itemClient, a, r, fault.New(true))
require.NoError(t, err)
return connector

View File

@ -24,6 +24,7 @@ import (
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -125,9 +126,15 @@ func (suite *GraphConnectorUnitSuite) TestUnionSiteIDsAndWebURLs() {
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
//nolint
result, err := gc.UnionSiteIDsAndWebURLs(context.Background(), test.ids, test.urls)
ctx, flush := tester.NewContext()
defer flush()
errs := fault.New(true)
result, err := gc.UnionSiteIDsAndWebURLs(ctx, test.ids, test.urls, errs)
assert.NoError(t, err)
assert.NoError(t, errs.Err())
assert.Empty(t, errs.Errs())
assert.ElementsMatch(t, test.expect, result)
})
}
@ -204,18 +211,24 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() {
ctx, flush := tester.NewContext()
defer flush()
t := suite.T()
service, err := newConnector.createService()
require.NoError(suite.T(), err)
require.NoError(t, err)
newConnector.Service = service
assert.Equal(t, 0, len(newConnector.Sites))
suite.Equal(0, len(newConnector.Sites))
err = newConnector.setTenantSites(ctx)
suite.NoError(err)
suite.Less(0, len(newConnector.Sites))
errs := fault.New(true)
err = newConnector.setTenantSites(ctx, errs)
assert.NoError(t, err)
assert.NoError(t, errs.Err())
assert.Empty(t, errs.Errs())
assert.Less(t, 0, len(newConnector.Sites))
for _, site := range newConnector.Sites {
suite.NotContains("sharepoint.com/personal/", site)
assert.NotContains(t, "sharepoint.com/personal/", site)
}
}

View File

@ -239,7 +239,7 @@ func (op *BackupOperation) do(
return nil, errors.Wrap(err, "producing manifests and metadata")
}
gc, err := connectToM365(ctx, op.Selectors, op.account)
gc, err := connectToM365(ctx, op.Selectors, op.account, op.Errors)
if err != nil {
return nil, errors.Wrap(err, "connectng to m365")
}

View File

@ -669,7 +669,12 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
m365, err := acct.M365Config()
require.NoError(t, err)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users)
gc, err := connector.NewGraphConnector(
ctx,
graph.HTTPClient(graph.NoTimeout()),
acct,
connector.Users,
fault.New(true))
require.NoError(t, err)
ac, err := api.NewClient(m365)

View File

@ -100,6 +100,7 @@ func connectToM365(
ctx context.Context,
sel selectors.Selector,
acct account.Account,
errs *fault.Errors,
) (*connector.GraphConnector, error) {
complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Connecting to M365"))
defer func() {
@ -114,7 +115,7 @@ func connectToM365(
resource = connector.Sites
}
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, resource)
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, resource, errs)
if err != nil {
return nil, err
}

View File

@ -242,7 +242,7 @@ func (op *RestoreOperation) do(
opStats.resourceCount = 1
opStats.cs = dcs
gc, err := connectToM365(ctx, op.Selectors, op.account)
gc, err := connectToM365(ctx, op.Selectors, op.account, op.Errors)
if err != nil {
return nil, errors.Wrap(err, "connecting to M365")
}

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/discovery"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
type User struct {
@ -21,8 +22,8 @@ type User struct {
// Users returns a list of users in the specified M365 tenant
// TODO: Implement paging support
func Users(ctx context.Context, acct account.Account) ([]*User, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users)
func Users(ctx context.Context, acct account.Account, errs *fault.Errors) ([]*User, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Users, errs)
if err != nil {
return nil, errors.Wrap(err, "initializing M365 graph connection")
}
@ -46,8 +47,8 @@ func Users(ctx context.Context, acct account.Account) ([]*User, error) {
return ret, nil
}
func UserIDs(ctx context.Context, acct account.Account) ([]string, error) {
users, err := Users(ctx, acct)
func UserIDs(ctx context.Context, acct account.Account, errs *fault.Errors) ([]string, error) {
users, err := Users(ctx, acct, errs)
if err != nil {
return nil, err
}
@ -62,8 +63,8 @@ func UserIDs(ctx context.Context, acct account.Account) ([]string, error) {
// UserPNs retrieves all user principleNames in the tenant. Principle Names
// can be used analogous userIDs in graph API queries.
func UserPNs(ctx context.Context, acct account.Account) ([]string, error) {
users, err := Users(ctx, acct)
func UserPNs(ctx context.Context, acct account.Account, errs *fault.Errors) ([]string, error) {
users, err := Users(ctx, acct, errs)
if err != nil {
return nil, err
}
@ -77,8 +78,8 @@ func UserPNs(ctx context.Context, acct account.Account) ([]string, error) {
}
// SiteURLs returns a list of SharePoint site WebURLs in the specified M365 tenant
func SiteURLs(ctx context.Context, acct account.Account) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites)
func SiteURLs(ctx context.Context, acct account.Account, errs *fault.Errors) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return nil, errors.Wrap(err, "initializing M365 graph connection")
}
@ -87,8 +88,8 @@ func SiteURLs(ctx context.Context, acct account.Account) ([]string, error) {
}
// SiteURLs returns a list of SharePoint sites IDs in the specified M365 tenant
func SiteIDs(ctx context.Context, acct account.Account) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites)
func SiteIDs(ctx context.Context, acct account.Account, errs *fault.Errors) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, graph.HTTPClient(graph.NoTimeout()), acct, connector.Sites, errs)
if err != nil {
return nil, errors.Wrap(err, "initializing graph connection")
}

View File

@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/fault"
)
type M365IntegrationSuite struct {
@ -27,18 +28,24 @@ func (suite *M365IntegrationSuite) TestUsers() {
ctx, flush := tester.NewContext()
defer flush()
acct := tester.NewM365Account(suite.T())
var (
t = suite.T()
acct = tester.NewM365Account(suite.T())
errs = fault.New(true)
)
users, err := Users(ctx, acct)
require.NoError(suite.T(), err)
require.NotNil(suite.T(), users)
require.Greater(suite.T(), len(users), 0)
users, err := Users(ctx, acct, errs)
require.NoError(t, err)
require.NoError(t, errs.Err())
require.Empty(t, errs.Errs())
require.NotNil(t, users)
require.Greater(t, len(users), 0)
for _, u := range users {
suite.T().Log(u)
assert.NotEmpty(suite.T(), u.ID)
assert.NotEmpty(suite.T(), u.PrincipalName)
assert.NotEmpty(suite.T(), u.Name)
t.Run("user_"+u.ID, func(t *testing.T) {
assert.NotEmpty(t, u.ID)
assert.NotEmpty(t, u.PrincipalName)
assert.NotEmpty(t, u.Name)
})
}
}