larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add fault/clues to exchange data_collections (#2381)

Browse Source 
## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🧹 Tech Debt/Cleanup

## Issue(s)

* #1970

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-02-13 11:19:46 -07:00 committed by GitHub
parent 680ec2b751
commit 89fb764526
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 50 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/internal/connector/data_collections.go
View File

@ -19,6 +19,7 @@ import (
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
@ -38,6 +39,7 @@ func (gc *GraphConnector) DataCollections(
sels selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Errors,
) ([]data.BackupCollection, map[string]struct{}, error) {
ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
defer end()
@ -65,7 +67,8 @@ func (gc *GraphConnector) DataCollections(
gc.credentials,
// gc.Service,
gc.UpdateStatus,
ctrlOpts)
ctrlOpts,
errs)
if err != nil {
return nil, nil, err
}

25
src/internal/connector/data_collections_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -105,7 +106,8 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
nil,
connector.credentials,
connector.UpdateStatus,
control.Options{})
control.Options{},
fault.New(true))
require.NoError(t, err)
assert.Empty(t, excludes)
@ -201,7 +203,12 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestDataCollections_invali
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
collections, excludes, err := connector.DataCollections(ctx, test.getSelector(t), nil, control.Options{})
collections, excludes, err := connector.DataCollections(
ctx,
test.getSelector(t),
nil,
control.Options{},
fault.New(true))
assert.Error(t, err)
assert.Empty(t, collections)
assert.Empty(t, excludes)
@ -325,7 +332,12 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
sel := selectors.NewSharePointBackup(siteIDs)
sel.Include(sel.Libraries([]string{"foo"}, selectors.PrefixMatch()))
cols, excludes, err := gc.DataCollections(ctx, sel.Selector, nil, control.Options{})
cols, excludes, err := gc.DataCollections(
ctx,
sel.Selector,
nil,
control.Options{},
fault.New(true))
require.NoError(t, err)
assert.Len(t, cols, 1)
// No excludes yet as this isn't an incremental backup.
@ -351,7 +363,12 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
sel := selectors.NewSharePointBackup(siteIDs)
sel.Include(sel.Lists(selectors.Any(), selectors.PrefixMatch()))
cols, excludes, err := gc.DataCollections(ctx, sel.Selector, nil, control.Options{})
cols, excludes, err := gc.DataCollections(
ctx,
sel.Selector,
nil,
control.Options{},
fault.New(true))
require.NoError(t, err)
assert.Less(t, 0, len(cols))
// No excludes yet as this isn't an incremental backup.

39
src/internal/connector/exchange/data_collections.go
View File

@ -3,9 +3,8 @@ package exchange
import (
"context"
"encoding/json"
"fmt"
"github.com/hashicorp/go-multierror"
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
@ -15,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -90,7 +90,7 @@ func parseMetadataCollections(
for {
select {
case <-ctx.Done():
return nil, errors.Wrap(ctx.Err(), "parsing collection metadata")
return nil, clues.Wrap(ctx.Err(), "parsing collection metadata").WithClues(ctx)
case item, ok := <-items:
if !ok {
@ -105,13 +105,13 @@ func parseMetadataCollections(
err := json.NewDecoder(item.ToReader()).Decode(&m)
if err != nil {
return nil, errors.New("decoding metadata json")
return nil, clues.New("decoding metadata json").WithClues(ctx)
}
switch item.UUID() {
case graph.PreviousPathFileName:
if _, ok := found[category]["path"]; ok {
return nil, errors.Errorf("multiple versions of %s path metadata", category)
return nil, clues.Wrap(clues.New(category.String()), "multiple versions of path metadata").WithClues(ctx)
}
for k, p := range m {
@ -122,7 +122,7 @@ func parseMetadataCollections(
case graph.DeltaURLsFileName:
if _, ok := found[category]["delta"]; ok {
return nil, errors.Errorf("multiple versions of %s delta metadata", category)
return nil, clues.Wrap(clues.New(category.String()), "multiple versions of delta metadata").WithClues(ctx)
}
for k, d := range m {
@ -167,16 +167,16 @@ func DataCollections(
acct account.M365Config,
su support.StatusUpdater,
ctrlOpts control.Options,
errs *fault.Errors,
) ([]data.BackupCollection, map[string]struct{}, error) {
eb, err := selector.ToExchangeBackup()
if err != nil {
return nil, nil, errors.Wrap(err, "exchangeDataCollection: parsing selector")
return nil, nil, clues.Wrap(err, "exchange dataCollection selector").WithClues(ctx)
}
var (
user = selector.DiscreteOwner
collections = []data.BackupCollection{}
errs error
)
cdps, err := parseMetadataCollections(ctx, metadata)
@ -185,26 +185,27 @@ func DataCollections(
}
for _, scope := range eb.Scopes() {
dps := cdps[scope.Category().PathType()]
if errs.Failed() {
break
}
dcs, err := createCollections(
ctx,
acct,
user,
scope,
dps,
cdps[scope.Category().PathType()],
ctrlOpts,
su)
if err != nil {
return nil, nil, support.WrapAndAppend(user, err, errs)
errs.Add(err)
continue
}
collections = append(collections, dcs...)
}
// Exchange does not require adding items to the global exclude list so always
// return nil.
return collections, nil, errs
return collections, nil, errs.Err()
}
func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedItemIDsGetter, error) {
@ -216,7 +217,7 @@ func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedIte
case path.ContactsCategory:
return ac.Contacts(), nil
default:
return nil, fmt.Errorf("category %s not supported by getFetchIDFunc", category)
return nil, clues.Wrap(clues.New(category.String()), "category not supported")
}
}
@ -233,7 +234,6 @@ func createCollections(
su support.StatusUpdater,
) ([]data.BackupCollection, error) {
var (
errs *multierror.Error
allCollections = make([]data.BackupCollection, 0)
ac = api.Client{Credentials: creds}
category = scope.Category().PathType()
@ -241,7 +241,7 @@ func createCollections(
getter, err := getterByType(ac, category)
if err != nil {
return nil, err
return nil, clues.Stack(err).WithClues(ctx)
}
// Create collection of ExchangeDataCollection
@ -262,7 +262,7 @@ func createCollections(
resolver, err := PopulateExchangeContainerResolver(ctx, qp)
if err != nil {
return nil, errors.Wrap(err, "getting folder cache")
return nil, errors.Wrap(err, "populating container cache")
}
err = filterContainersAndFillCollections(
@ -275,7 +275,6 @@ func createCollections(
scope,
dps,
ctrlOpts)
if err != nil {
return nil, errors.Wrap(err, "filling collections")
}
@ -286,5 +285,5 @@ func createCollections(
allCollections = append(allCollections, coll)
}
return allCollections, errs.ErrorOrNil()
return allCollections, nil
}

6
src/internal/connector/graph_connector_test.go
View File

@ -469,7 +469,7 @@ func runRestoreBackupTest(
RestorePermissions: true,
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
},
)
fault.New(true))
require.NoError(t, err)
// No excludes yet because this isn't an incremental backup.
assert.Empty(t, excludes)
@ -597,7 +597,7 @@ func runRestoreBackupTestVersion0(
RestorePermissions: true,
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
},
)
fault.New(true))
require.NoError(t, err)
// No excludes yet because this isn't an incremental backup.
assert.Empty(t, excludes)
@ -1544,7 +1544,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
RestorePermissions: true,
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
},
)
fault.New(true))
require.NoError(t, err)
// No excludes yet because this isn't an incremental backup.
assert.Empty(t, excludes)

7
src/internal/operations/backup.go
View File

@ -244,7 +244,7 @@ func (op *BackupOperation) do(
return nil, errors.Wrap(err, "connectng to m365")
}
cs, excludes, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options)
cs, excludes, err := produceBackupDataCollections(ctx, gc, op.Selectors, mdColls, op.Options, op.Errors)
if err != nil {
return nil, errors.Wrap(err, "producing backup data collections")
}
@ -313,6 +313,7 @@ func produceBackupDataCollections(
sel selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Errors,
) ([]data.BackupCollection, map[string]struct{}, error) {
complete, closer := observe.MessageWithCompletion(ctx, observe.Safe("Discovering items to backup"))
defer func() {
@ -321,9 +322,7 @@ func produceBackupDataCollections(
closer()
}()
cols, excludes, errs := gc.DataCollections(ctx, sel, metadata, ctrlOpts)
return cols, excludes, errs
return gc.DataCollections(ctx, sel, metadata, ctrlOpts, errs)
}
// ---------------------------------------------------------------------------