diff --git a/src/internal/connector/data_collections.go b/src/internal/connector/data_collections.go index 4c1bd4461..06b7bec6a 100644 --- a/src/internal/connector/data_collections.go +++ b/src/internal/connector/data_collections.go @@ -19,6 +19,7 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" @@ -38,6 +39,7 @@ func (gc *GraphConnector) DataCollections( sels selectors.Selector, metadata []data.RestoreCollection, ctrlOpts control.Options, + errs *fault.Errors, ) ([]data.BackupCollection, map[string]struct{}, error) { ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String())) defer end() @@ -65,7 +67,8 @@ func (gc *GraphConnector) DataCollections( gc.credentials, // gc.Service, gc.UpdateStatus, - ctrlOpts) + ctrlOpts, + errs) if err != nil { return nil, nil, err } diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 4484a92aa..c2038f006 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -14,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/sharepoint" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -105,7 +106,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection nil, connector.credentials, connector.UpdateStatus, - control.Options{}) + control.Options{}, + fault.New(true)) require.NoError(t, err) assert.Empty(t, excludes) @@ -201,7 +203,12 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestDataCollections_invali for _, test := range tests { suite.T().Run(test.name, func(t *testing.T) { - collections, excludes, err := connector.DataCollections(ctx, test.getSelector(t), nil, control.Options{}) + collections, excludes, err := connector.DataCollections( + ctx, + test.getSelector(t), + nil, + control.Options{}, + fault.New(true)) assert.Error(t, err) assert.Empty(t, collections) assert.Empty(t, excludes) @@ -325,7 +332,12 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar sel := selectors.NewSharePointBackup(siteIDs) sel.Include(sel.Libraries([]string{"foo"}, selectors.PrefixMatch())) - cols, excludes, err := gc.DataCollections(ctx, sel.Selector, nil, control.Options{}) + cols, excludes, err := gc.DataCollections( + ctx, + sel.Selector, + nil, + control.Options{}, + fault.New(true)) require.NoError(t, err) assert.Len(t, cols, 1) // No excludes yet as this isn't an incremental backup. @@ -351,7 +363,12 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar sel := selectors.NewSharePointBackup(siteIDs) sel.Include(sel.Lists(selectors.Any(), selectors.PrefixMatch())) - cols, excludes, err := gc.DataCollections(ctx, sel.Selector, nil, control.Options{}) + cols, excludes, err := gc.DataCollections( + ctx, + sel.Selector, + nil, + control.Options{}, + fault.New(true)) require.NoError(t, err) assert.Less(t, 0, len(cols)) // No excludes yet as this isn't an incremental backup. diff --git a/src/internal/connector/exchange/data_collections.go b/src/internal/connector/exchange/data_collections.go index 92e826f3d..7fd180281 100644 --- a/src/internal/connector/exchange/data_collections.go +++ b/src/internal/connector/exchange/data_collections.go @@ -3,9 +3,8 @@ package exchange import ( "context" "encoding/json" - "fmt" - "github.com/hashicorp/go-multierror" + "github.com/alcionai/clues" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/connector/exchange/api" @@ -15,6 +14,7 @@ import ( "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -90,7 +90,7 @@ func parseMetadataCollections( for { select { case <-ctx.Done(): - return nil, errors.Wrap(ctx.Err(), "parsing collection metadata") + return nil, clues.Wrap(ctx.Err(), "parsing collection metadata").WithClues(ctx) case item, ok := <-items: if !ok { @@ -105,13 +105,13 @@ func parseMetadataCollections( err := json.NewDecoder(item.ToReader()).Decode(&m) if err != nil { - return nil, errors.New("decoding metadata json") + return nil, clues.New("decoding metadata json").WithClues(ctx) } switch item.UUID() { case graph.PreviousPathFileName: if _, ok := found[category]["path"]; ok { - return nil, errors.Errorf("multiple versions of %s path metadata", category) + return nil, clues.Wrap(clues.New(category.String()), "multiple versions of path metadata").WithClues(ctx) } for k, p := range m { @@ -122,7 +122,7 @@ func parseMetadataCollections( case graph.DeltaURLsFileName: if _, ok := found[category]["delta"]; ok { - return nil, errors.Errorf("multiple versions of %s delta metadata", category) + return nil, clues.Wrap(clues.New(category.String()), "multiple versions of delta metadata").WithClues(ctx) } for k, d := range m { @@ -167,16 +167,16 @@ func DataCollections( acct account.M365Config, su support.StatusUpdater, ctrlOpts control.Options, + errs *fault.Errors, ) ([]data.BackupCollection, map[string]struct{}, error) { eb, err := selector.ToExchangeBackup() if err != nil { - return nil, nil, errors.Wrap(err, "exchangeDataCollection: parsing selector") + return nil, nil, clues.Wrap(err, "exchange dataCollection selector").WithClues(ctx) } var ( user = selector.DiscreteOwner collections = []data.BackupCollection{} - errs error ) cdps, err := parseMetadataCollections(ctx, metadata) @@ -185,26 +185,27 @@ func DataCollections( } for _, scope := range eb.Scopes() { - dps := cdps[scope.Category().PathType()] + if errs.Failed() { + break + } dcs, err := createCollections( ctx, acct, user, scope, - dps, + cdps[scope.Category().PathType()], ctrlOpts, su) if err != nil { - return nil, nil, support.WrapAndAppend(user, err, errs) + errs.Add(err) + continue } collections = append(collections, dcs...) } - // Exchange does not require adding items to the global exclude list so always - // return nil. - return collections, nil, errs + return collections, nil, errs.Err() } func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedItemIDsGetter, error) { @@ -216,7 +217,7 @@ func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedIte case path.ContactsCategory: return ac.Contacts(), nil default: - return nil, fmt.Errorf("category %s not supported by getFetchIDFunc", category) + return nil, clues.Wrap(clues.New(category.String()), "category not supported") } } @@ -233,7 +234,6 @@ func createCollections( su support.StatusUpdater, ) ([]data.BackupCollection, error) { var ( - errs *multierror.Error allCollections = make([]data.BackupCollection, 0) ac = api.Client{Credentials: creds} category = scope.Category().PathType() @@ -241,7 +241,7 @@ func createCollections( getter, err := getterByType(ac, category) if err != nil { - return nil, err + return nil, clues.Stack(err).WithClues(ctx) } // Create collection of ExchangeDataCollection @@ -262,7 +262,7 @@ func createCollections( resolver, err := PopulateExchangeContainerResolver(ctx, qp) if err != nil { - return nil, errors.Wrap(err, "getting folder cache") + return nil, errors.Wrap(err, "populating container cache") } err = filterContainersAndFillCollections( @@ -275,7 +275,6 @@ func createCollections( scope, dps, ctrlOpts) - if err != nil { return nil, errors.Wrap(err, "filling collections") } @@ -286,5 +285,5 @@ func createCollections( allCollections = append(allCollections, coll) } - return allCollections, errs.ErrorOrNil() + return allCollections, nil } diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 1840d7473..6a7620dca 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -469,7 +469,7 @@ func runRestoreBackupTest( RestorePermissions: true, ToggleFeatures: control.Toggles{EnablePermissionsBackup: true}, }, - ) + fault.New(true)) require.NoError(t, err) // No excludes yet because this isn't an incremental backup. assert.Empty(t, excludes) @@ -597,7 +597,7 @@ func runRestoreBackupTestVersion0( RestorePermissions: true, ToggleFeatures: control.Toggles{EnablePermissionsBackup: true}, }, - ) + fault.New(true)) require.NoError(t, err) // No excludes yet because this isn't an incremental backup. assert.Empty(t, excludes) @@ -1544,7 +1544,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames RestorePermissions: true, ToggleFeatures: control.Toggles{EnablePermissionsBackup: true}, }, - ) + fault.New(true)) require.NoError(t, err) // No excludes yet because this isn't an incremental backup. assert.Empty(t, excludes) diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 5ffec2456..daaca3260 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -244,7 +244,7 @@ func (op *BackupOperation) do( return nil, errors.Wrap(err, "connectng to m365") } - cs, excludes, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options) + cs, excludes, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options, op.Errors) if err != nil { return nil, errors.Wrap(err, "producing backup data collections") } @@ -313,6 +313,7 @@ func produceBackupDataCollections( sel selectors.Selector, metadata []data.RestoreCollection, ctrlOpts control.Options, + errs *fault.Errors, ) ([]data.BackupCollection, map[string]struct{}, error) { complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Discovering items to backup")) defer func() { @@ -321,9 +322,7 @@ func produceBackupDataCollections( closer() }() - cols, excludes, errs := gc.DataCollections(ctx, sel, metadata, ctrlOpts) - - return cols, excludes, errs + return gc.DataCollections(ctx, sel, metadata, ctrlOpts, errs) } // ---------------------------------------------------------------------------