larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove errors from connector status

Browse Source 
Now that fault is in place, we can remove the error
tracking functionality of graph status, and let that
focus purely on metrics.
This commit is contained in:
ryanfkeepers 2023-02-20 17:21:03 -07:00
parent 5593352226
commit 7b50d2401a
19 changed files with 192 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/internal/connector/data_collections_test.go
View File

@ -129,7 +129,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Successful)
assert.NotZero(t, status.Metrics.Successes)
t.Log(status.String())
})
}
@ -281,7 +281,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Successful)
assert.NotZero(t, status.Metrics.Successes)
t.Log(status.String())
})
}

1
src/internal/connector/exchange/data_collections_test.go
View File

@ -200,7 +200,6 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
func newStatusUpdater(t *testing.T, wg *sync.WaitGroup) func(status *support.ConnectorOperationStatus) {
updater := func(status *support.ConnectorOperationStatus) {
defer wg.Done()
assert.Zero(t, status.ErrorCount)
}
return updater

7
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -316,11 +316,10 @@ func (col *Collection) finishPopulation(
support.Backup,
1,
support.CollectionMetrics{
Objects: attempted,
Successes: success,
TotalBytes: totalBytes,
Objects: attempted,
Successes: success,
Bytes: totalBytes,
},
err,
col.fullPath.Folder(false))
logger.Ctx(ctx).Debugw("done streaming items", "status", status.String())

5
src/internal/connector/exchange/service_restore.go
View File

@ -343,7 +343,7 @@ func RestoreExchangeDataCollections(
temp, canceled := restoreCollection(ctx, gs, dc, containerID, policy, deets, errs)
metrics.Combine(temp)
metrics = support.CombineMetrics(metrics, temp)
if canceled {
break
@ -355,7 +355,6 @@ func RestoreExchangeDataCollections(
support.Restore,
len(dcs),
metrics,
el.Failure(),
dest.ContainerName)
return status, el.Failure()
@ -436,7 +435,7 @@ func restoreCollection(
continue
}
metrics.TotalBytes += int64(len(byteArray))
metrics.Bytes += int64(len(byteArray))
metrics.Successes++
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)

7
src/internal/connector/graph/metadata_collection.go
View File

@ -150,11 +150,10 @@ func (md MetadataCollection) Items(
support.Backup,
1,
support.CollectionMetrics{
Objects: len(md.items),
Successes: len(md.items),
TotalBytes: totalBytes,
Objects: len(md.items),
Successes: len(md.items),
Bytes: totalBytes,
},
nil,
md.fullPath.Folder(false),
)

4
src/internal/connector/graph/metadata_collection_test.go
View File

@ -84,8 +84,8 @@ func (suite *MetadataCollectionUnitSuite) TestItems() {
p,
items,
func(c *support.ConnectorOperationStatus) {
assert.Equal(t, len(itemNames), c.ObjectCount)
assert.Equal(t, len(itemNames), c.Successful)
assert.Equal(t, len(itemNames), c.Metrics.Objects)
assert.Equal(t, len(itemNames), c.Metrics.Successes)
},
)

21
src/internal/connector/graph_connector_disconnected_test.go
View File

@ -4,7 +4,6 @@ import (
"sync"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -92,17 +91,11 @@ func statusTestTask(gc *GraphConnector, objects, success, folder int) {
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
TotalBytes: 0,
Objects: objects,
Successes: success,
Bytes: 0,
},
support.WrapAndAppend(
"tres",
errors.New("three"),
support.WrapAndAppend("arc376", errors.New("one"), errors.New("two")),
),
"statusTestTask",
)
"statusTestTask")
gc.UpdateStatus(status)
}
@ -123,11 +116,11 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() {
assert.NotEmpty(t, gc.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, gc.Status().ObjectCount)
assert.Equal(t, 8, gc.Status().Metrics.Objects)
// Expect 2 success
assert.Equal(t, 2, gc.Status().Successful)
assert.Equal(t, 2, gc.Status().Metrics.Successes)
// Expect 2 folders
assert.Equal(t, 2, gc.Status().FolderCount)
assert.Equal(t, 2, gc.Status().Folders)
}
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {

36
src/internal/connector/graph_connector_test.go
View File

@ -251,9 +251,9 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
assert.NotNil(t, deets)
status := suite.connector.AwaitStatus()
assert.Equal(t, 0, status.ObjectCount)
assert.Equal(t, 0, status.FolderCount)
assert.Equal(t, 0, status.Successful)
assert.Equal(t, 0, status.Metrics.Objects)
assert.Equal(t, 0, status.Folders)
assert.Equal(t, 0, status.Metrics.Successes)
}
func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
@ -328,9 +328,9 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
assert.NotNil(t, deets)
stats := suite.connector.AwaitStatus()
assert.Zero(t, stats.ObjectCount)
assert.Zero(t, stats.FolderCount)
assert.Zero(t, stats.Successful)
assert.Zero(t, stats.Metrics.Objects)
assert.Zero(t, stats.Folders)
assert.Zero(t, stats.Metrics.Successes)
})
}
}
@ -432,10 +432,8 @@ func runRestore(
status := restoreGC.AwaitStatus()
runTime := time.Since(start)
assert.NoError(t, status.Err, "restored status.Err")
assert.Zero(t, status.ErrorCount, "restored status.ErrorCount")
assert.Equal(t, numRestoreItems, status.ObjectCount, "restored status.ObjectCount")
assert.Equal(t, numRestoreItems, status.Successful, "restored status.Successful")
assert.Equal(t, numRestoreItems, status.Metrics.Objects, "restored status.Metrics.Objects")
assert.Equal(t, numRestoreItems, status.Metrics.Successes, "restored status.Metrics.Successes")
assert.Len(
t,
deets.Entries,
@ -497,12 +495,10 @@ func runBackupAndCompare(
status := backupGC.AwaitStatus()
assert.NoError(t, status.Err, "backup status.Err")
assert.Zero(t, status.ErrorCount, "backup status.ErrorCount")
assert.Equalf(t, totalItems+skipped, status.ObjectCount,
"backup status.ObjectCount; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Successful,
"backup status.Successful; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Metrics.Objects,
"backup status.Metrics.Objects; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Metrics.Successes,
"backup status.Metrics.Successes; wanted %d items + %d skipped", totalItems, skipped)
}
func runRestoreBackupTest(
@ -966,8 +962,8 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
status := restoreGC.AwaitStatus()
// Always just 1 because it's just 1 collection.
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal(t, totalItems, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, totalItems, status.Metrics.Successes, "status.Metrics.Successes")
assert.Equal(
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
@ -1001,8 +997,8 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
skipped := checkCollections(t, ctx, allItems, allExpectedData, dcs, true)
status := backupGC.AwaitStatus()
assert.Equal(t, allItems+skipped, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, allItems+skipped, status.Successful, "status.Successful")
assert.Equal(t, allItems+skipped, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, allItems+skipped, status.Metrics.Successes, "status.Metrics.Successes")
})
}
}

18
src/internal/connector/onedrive/collection.go
View File

@ -233,7 +233,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
// `details.OneDriveInfo`
parentPathString, err := path.GetDriveFolderPath(oc.folderPath)
if err != nil {
oc.reportAsCompleted(ctx, 0, 0, 0, clues.Wrap(err, "getting drive path").WithClues(ctx))
oc.reportAsCompleted(ctx, 0, 0, 0)
return
}
@ -422,22 +422,22 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
wg.Wait()
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount, el.Failure())
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount)
}
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64, err error) {
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64) {
close(oc.data)
status := support.CreateStatus(ctx, support.Backup,
1, // num folders (always 1)
support.CollectionMetrics{
Objects: itemsFound, // items to read,
Successes: itemsRead, // items read successfully,
TotalBytes: byteCount, // Number of bytes read in the operation,
Objects: itemsFound,
Successes: itemsRead,
Bytes: byteCount,
},
err,
oc.folderPath.Folder(false), // Additional details
)
oc.folderPath.Folder(false))
logger.Ctx(ctx).Debugw("done streaming items", "status", status.String())
oc.statusUpdater(status)
}

18
src/internal/connector/onedrive/collection_test.go
View File

@ -52,7 +52,7 @@ func (suite *CollectionUnitTestSuite) testStatusUpdater(
statusToUpdate *support.ConnectorOperationStatus,
) support.StatusUpdater {
return func(s *support.ConnectorOperationStatus) {
suite.T().Logf("Update status %v, count %d, success %d", s, s.ObjectCount, s.Successful)
suite.T().Logf("Update status %v, count %d, success %d", s, s.Metrics.Objects, s.Metrics.Successes)
*statusToUpdate = *s
wg.Done()
@ -222,8 +222,8 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
}
// Expect only 1 item
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
// Validate item info and data
readItem := readItems[0]
@ -344,8 +344,8 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() {
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount, "only one object should be counted")
require.Equal(t, 1, collStatus.Successful, "TODO: should be 0, but allowing 1 to reduce async management")
require.Equal(t, 1, collStatus.Metrics.Objects, "only one object should be counted")
require.Equal(t, 1, collStatus.Metrics.Successes, "TODO: should be 0, but allowing 1 to reduce async management")
})
}
}
@ -424,8 +424,8 @@ func (suite *CollectionUnitTestSuite) TestCollectionDisablePermissionsBackup() {
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
for _, i := range readItems {
if strings.HasSuffix(i.UUID(), MetaFileSuffix) {
@ -514,8 +514,8 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
for _, i := range readItems {
if strings.HasSuffix(i.UUID(), MetaFileSuffix) {

7
src/internal/connector/onedrive/restore.go
View File

@ -169,7 +169,7 @@ func RestoreCollections(
parentPermissions[k] = v
}
restoreMetrics.Combine(metrics)
restoreMetrics = support.CombineMetrics(restoreMetrics, metrics)
if errors.Is(err, context.Canceled) {
break
@ -181,7 +181,6 @@ func RestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
el.Failure(),
dest.ContainerName)
return status, el.Failure()
@ -289,7 +288,7 @@ func RestoreCollection(
if strings.HasSuffix(name, DataFileSuffix) {
metrics.Objects++
metrics.TotalBytes += int64(len(copyBuffer))
metrics.Bytes += int64(len(copyBuffer))
trimmedName := strings.TrimSuffix(name, DataFileSuffix)
itemID, itemInfo, err = restoreData(
@ -378,7 +377,7 @@ func RestoreCollection(
}
} else {
metrics.Objects++
metrics.TotalBytes += int64(len(copyBuffer))
metrics.Bytes += int64(len(copyBuffer))
// No permissions stored at the moment for SharePoint
_, itemInfo, err = restoreData(

58
src/internal/connector/sharepoint/collection.go
View File

@ -43,12 +43,6 @@ var (
_ data.StreamModTime = &Item{}
)
type numMetrics struct {
attempts int
success int
totalBytes int64
}
// Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported
// by the oneDrive.Collection as the calls are identical for populating the Collection
type Collection struct {
@ -157,24 +151,17 @@ func (sd *Item) ModTime() time.Time {
func (sc *Collection) finishPopulation(
ctx context.Context,
attempts, success int,
totalBytes int64,
err error,
metrics support.CollectionMetrics,
) {
close(sc.data)
attempted := attempts
status := support.CreateStatus(
ctx,
support.Backup,
1, // 1 folder
support.CollectionMetrics{
Objects: attempted,
Successes: success,
TotalBytes: totalBytes,
},
err,
metrics,
sc.fullPath.Folder(false))
logger.Ctx(ctx).Debug(status.String())
if sc.statusUpdater != nil {
@ -184,15 +171,16 @@ func (sc *Collection) finishPopulation(
// populate utility function to retrieve data from back store for a given collection
func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) {
var (
metrics numMetrics
writer = kw.NewJsonSerializationWriter()
err error
)
metrics, _ := sc.runPopulate(ctx, errs)
sc.finishPopulation(ctx, metrics)
}
defer func() {
sc.finishPopulation(ctx, metrics.attempts, metrics.success, int64(metrics.totalBytes), err)
}()
func (sc *Collection) runPopulate(ctx context.Context, errs *fault.Bus) (support.CollectionMetrics, error) {
var (
err error
metrics support.CollectionMetrics
writer = kw.NewJsonSerializationWriter()
)
// TODO: Insert correct ID for CollectionProgress
colProgress, closer := observe.CollectionProgress(
@ -213,6 +201,8 @@ func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) {
case Pages:
metrics, err = sc.retrievePages(ctx, writer, colProgress, errs)
}
return metrics, err
}
// retrieveLists utility function for collection that downloads and serializes
@ -222,9 +212,9 @@ func (sc *Collection) retrieveLists(
wtr *kw.JsonSerializationWriter,
progress chan<- struct{},
errs *fault.Bus,
) (numMetrics, error) {
) (support.CollectionMetrics, error) {
var (
metrics numMetrics
metrics support.CollectionMetrics
el = errs.Local()
)
@ -233,7 +223,7 @@ func (sc *Collection) retrieveLists(
return metrics, err
}
metrics.attempts += len(lists)
metrics.Objects += len(lists)
// For each models.Listable, object is serialized and the metrics are collected.
// The progress is objected via the passed in channel.
for _, lst := range lists {
@ -255,9 +245,9 @@ func (sc *Collection) retrieveLists(
t = *t1
}
metrics.totalBytes += size
metrics.Bytes += size
metrics.success++
metrics.Successes++
sc.data <- &Item{
id: *lst.GetId(),
data: io.NopCloser(bytes.NewReader(byteArray)),
@ -277,9 +267,9 @@ func (sc *Collection) retrievePages(
wtr *kw.JsonSerializationWriter,
progress chan<- struct{},
errs *fault.Bus,
) (numMetrics, error) {
) (support.CollectionMetrics, error) {
var (
metrics numMetrics
metrics support.CollectionMetrics
el = errs.Local()
)
@ -293,7 +283,7 @@ func (sc *Collection) retrievePages(
return metrics, err
}
metrics.attempts = len(pages)
metrics.Objects = len(pages)
// For each models.Pageable, object is serialize and the metrics are collected and returned.
// Pageable objects are not supported in v1.0 of msgraph at this time.
// TODO: Verify Parsable interface supported with modified-Pageable
@ -311,8 +301,8 @@ func (sc *Collection) retrievePages(
size := int64(len(byteArray))
if size > 0 {
metrics.totalBytes += size
metrics.success++
metrics.Bytes += size
metrics.Successes++
sc.data <- &Item{
id: *pg.GetId(),
data: io.NopCloser(bytes.NewReader(byteArray)),

7
src/internal/connector/sharepoint/restore.go
View File

@ -98,7 +98,7 @@ func RestoreCollections(
return nil, clues.Wrap(clues.New(category.String()), "category not supported")
}
restoreMetrics.Combine(metrics)
restoreMetrics = support.CombineMetrics(restoreMetrics, metrics)
if err != nil {
break
@ -110,7 +110,6 @@ func RestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
err,
dest.ContainerName)
return status, err
@ -256,7 +255,7 @@ func RestoreListCollection(
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
metrics.Bytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
@ -339,7 +338,7 @@ func RestorePageCollection(
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
metrics.Bytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {

99
src/internal/connector/support/status.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"github.com/dustin/go-humanize"
multierror "github.com/hashicorp/go-multierror"
)
// ConnectorOperationStatus is a data type used to describe the state of
@ -15,27 +14,23 @@ import (
// @param incomplete: Bool representation of whether all intended items were download or uploaded.
// @param bytes: represents the total number of bytes that have been downloaded or uploaded.
type ConnectorOperationStatus struct {
lastOperation Operation
ObjectCount int
FolderCount int
Successful int
ErrorCount int
Err error
incomplete bool
incompleteReason string
additionalDetails string
bytes int64
Folders int
Metrics CollectionMetrics
details string
op Operation
}
type CollectionMetrics struct {
Objects, Successes int
TotalBytes int64
Bytes int64
}
func (cm *CollectionMetrics) Combine(additional CollectionMetrics) {
cm.Objects += additional.Objects
cm.Successes += additional.Successes
cm.TotalBytes += additional.TotalBytes
func CombineMetrics(a, b CollectionMetrics) CollectionMetrics {
return CollectionMetrics{
Objects: a.Objects + b.Objects,
Successes: a.Successes + b.Successes,
Bytes: a.Bytes + b.Bytes,
}
}
type Operation int
@ -53,30 +48,13 @@ func CreateStatus(
op Operation,
folders int,
cm CollectionMetrics,
err error,
details string,
) *ConnectorOperationStatus {
var reason string
if err != nil {
reason = err.Error()
}
hasErrors := err != nil
// TODO(keeprs): remove
numErr := GetNumberOfErrors(err)
status := ConnectorOperationStatus{
lastOperation: op,
ObjectCount: cm.Objects,
FolderCount: folders,
Successful: cm.Successes,
ErrorCount: numErr,
Err: err,
incomplete: hasErrors,
incompleteReason: reason,
bytes: cm.TotalBytes,
additionalDetails: details,
Folders: folders,
Metrics: cm,
details: details,
op: op,
}
return &status
@ -89,32 +67,19 @@ type StatusUpdater func(*ConnectorOperationStatus)
// MergeStatus combines ConnectorOperationsStatus value into a single status
func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
var hasErrors bool
if one.lastOperation == OpUnknown {
if one.op == OpUnknown {
return two
}
if two.lastOperation == OpUnknown {
if two.op == OpUnknown {
return one
}
if one.incomplete || two.incomplete {
hasErrors = true
}
status := ConnectorOperationStatus{
lastOperation: one.lastOperation,
ObjectCount: one.ObjectCount + two.ObjectCount,
FolderCount: one.FolderCount + two.FolderCount,
Successful: one.Successful + two.Successful,
// TODO: remove in favor of fault.Errors
ErrorCount: one.ErrorCount + two.ErrorCount,
Err: multierror.Append(one.Err, two.Err).ErrorOrNil(),
bytes: one.bytes + two.bytes,
incomplete: hasErrors,
incompleteReason: one.incompleteReason + ", " + two.incompleteReason,
additionalDetails: one.additionalDetails + ", " + two.additionalDetails,
Folders: one.Folders + two.Folders,
Metrics: CombineMetrics(one.Metrics, two.Metrics),
details: one.details + ", " + two.details,
op: one.op,
}
return status
@ -123,23 +88,19 @@ func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
func (cos *ConnectorOperationStatus) String() string {
var operationStatement string
switch cos.lastOperation {
switch cos.op {
case Backup:
operationStatement = "Downloaded from "
case Restore:
operationStatement = "Restored content to "
}
message := fmt.Sprintf("Action: %s performed on %d of %d objects (%s) within %d directories.",
cos.lastOperation.String(),
cos.Successful,
cos.ObjectCount,
humanize.Bytes(uint64(cos.bytes)),
cos.FolderCount)
if cos.incomplete {
message += " " + cos.incompleteReason
}
return message + " " + operationStatement + cos.additionalDetails
return fmt.Sprintf("Action: %s performed on %d of %d objects (%s) within %d directories. %s %s",
cos.op.String(),
cos.Metrics.Successes,
cos.Metrics.Objects,
humanize.Bytes(uint64(cos.Metrics.Bytes)),
cos.Folders,
operationStatement,
cos.details)
}

170
src/internal/connector/support/status_test.go
View File

@ -1,156 +1,122 @@
package support
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type GCStatusTestSuite struct {
suite.Suite
type StatusUnitSuite struct {
tester.Suite
}
func TestGraphConnectorStatus(t *testing.T) {
suite.Run(t, &GCStatusTestSuite{})
suite.Run(t, &StatusUnitSuite{tester.NewUnitSuite(t)})
}
// operationType, objects, success, folders, errCount int, errStatus string
type statusParams struct {
operationType Operation
objects int
success int
folders int
err error
func metricsMatch(t *testing.T, expect, result CollectionMetrics) {
assert.Equal(t, expect.Bytes, result.Bytes, "bytes")
assert.Equal(t, expect.Objects, result.Objects, "objects")
assert.Equal(t, expect.Successes, result.Successes, "successes")
}
func (suite *GCStatusTestSuite) TestCreateStatus() {
func (suite *StatusUnitSuite) TestCreateStatus() {
table := []struct {
name string
params statusParams
expect assert.BoolAssertionFunc
name string
op Operation
metrics CollectionMetrics
folders int
}{
{
name: "Test: Status Success",
params: statusParams{Backup, 12, 12, 3, nil},
expect: assert.False,
name: "Backup",
op: Backup,
metrics: CollectionMetrics{12, 12, 3},
folders: 1,
},
{
name: "Test: Status Failed",
params: statusParams{
Restore,
12, 9, 8,
WrapAndAppend("tres", errors.New("three"), WrapAndAppend("arc376", errors.New("one"), errors.New("two"))),
},
expect: assert.True,
name: "Backup",
op: Restore,
metrics: CollectionMetrics{12, 9, 8},
folders: 2,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext()
defer flush()
result := CreateStatus(
ctx,
test.params.operationType,
test.params.folders,
CollectionMetrics{test.params.objects, test.params.success, 0},
test.params.err,
"",
)
test.expect(t, result.incomplete, "status is incomplete")
test.op,
test.folders,
test.metrics,
"details")
assert.Equal(t, test.op, result.op, "operation")
assert.Equal(t, test.folders, result.Folders, "folders")
metricsMatch(t, test.metrics, result.Metrics)
})
}
}
func (suite *GCStatusTestSuite) TestCreateStatus_InvalidStatus() {
t := suite.T()
params := statusParams{Backup, 9, 3, 13, errors.New("invalidcl")}
require.NotPanics(t, func() {
ctx, flush := tester.NewContext()
defer flush()
CreateStatus(
ctx,
params.operationType,
params.folders,
CollectionMetrics{
params.objects,
params.success,
0,
},
params.err,
"",
)
})
}
func (suite *GCStatusTestSuite) TestMergeStatus() {
func (suite *StatusUnitSuite) TestMergeStatus() {
ctx, flush := tester.NewContext()
defer flush()
table := []struct {
name string
one ConnectorOperationStatus
two ConnectorOperationStatus
expected statusParams
isIncomplete assert.BoolAssertionFunc
name string
one ConnectorOperationStatus
two ConnectorOperationStatus
expectOp Operation
expectMetrics CollectionMetrics
expectFolders int
}{
{
name: "Test: Status + unknown",
one: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, nil, ""),
two: ConnectorOperationStatus{},
expected: statusParams{Backup, 1, 1, 1, nil},
isIncomplete: assert.False,
name: "Test: Status + unknown",
one: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, ""),
two: ConnectorOperationStatus{},
expectOp: Backup,
expectMetrics: CollectionMetrics{1, 1, 0},
expectFolders: 1,
},
{
name: "Test: unknown + Status",
one: ConnectorOperationStatus{},
two: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, nil, ""),
expected: statusParams{Backup, 1, 1, 1, nil},
isIncomplete: assert.False,
name: "Test: unknown + Status",
one: ConnectorOperationStatus{},
two: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, ""),
expectOp: Backup,
expectMetrics: CollectionMetrics{1, 1, 0},
expectFolders: 1,
},
{
name: "Test: Successful + Successful",
one: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, nil, ""),
two: *CreateStatus(ctx, Backup, 3, CollectionMetrics{3, 3, 0}, nil, ""),
expected: statusParams{Backup, 4, 4, 4, nil},
isIncomplete: assert.False,
name: "Test: Successful + Successful",
one: *CreateStatus(ctx, Backup, 1, CollectionMetrics{1, 1, 0}, ""),
two: *CreateStatus(ctx, Backup, 3, CollectionMetrics{3, 3, 0}, ""),
expectOp: Backup,
expectMetrics: CollectionMetrics{4, 4, 0},
expectFolders: 4,
},
{
name: "Test: Successful + Unsuccessful",
one: *CreateStatus(ctx, Backup, 13, CollectionMetrics{17, 17, 0}, nil, ""),
two: *CreateStatus(
ctx,
Backup,
8,
CollectionMetrics{
12,
9,
0,
},
WrapAndAppend("tres", errors.New("three"), WrapAndAppend("arc376", errors.New("one"), errors.New("two"))),
"",
),
expected: statusParams{Backup, 29, 26, 21, nil},
isIncomplete: assert.True,
name: "Test: Successful + Unsuccessful",
one: *CreateStatus(ctx, Backup, 13, CollectionMetrics{17, 17, 0}, ""),
two: *CreateStatus(ctx, Backup, 8, CollectionMetrics{12, 9, 0}, ""),
expectOp: Backup,
expectMetrics: CollectionMetrics{29, 26, 0},
expectFolders: 21,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
returned := MergeStatus(test.one, test.two)
suite.Equal(returned.FolderCount, test.expected.folders)
suite.Equal(returned.ObjectCount, test.expected.objects)
suite.Equal(returned.lastOperation, test.expected.operationType)
suite.Equal(returned.Successful, test.expected.success)
test.isIncomplete(t, returned.incomplete)
suite.Run(test.name, func() {
t := suite.T()
result := MergeStatus(test.one, test.two)
assert.Equal(t, test.expectFolders, result.Folders, "folders")
assert.Equal(t, test.expectOp, result.op, "operation")
metricsMatch(t, test.expectMetrics, result.Metrics)
})
}
}

8
src/internal/operations/backup.go
View File

@ -281,10 +281,6 @@ func (op *BackupOperation) do(
}
opStats.gc = gc.AwaitStatus()
// TODO(keepers): remove when fault.Errors handles all iterable error aggregation.
if opStats.gc.ErrorCount > 0 {
return nil, opStats.gc.Err
}
logger.Ctx(ctx).Debug(gc.PrintableStatus())
@ -648,11 +644,11 @@ func (op *BackupOperation) persistResults(
return errors.New("backup population never completed")
}
if opStats.gc.Successful == 0 {
if opStats.gc.Metrics.Successes == 0 {
op.Status = NoData
}
op.Results.ItemsRead = opStats.gc.Successful
op.Results.ItemsRead = opStats.gc.Metrics.Successes
return nil
}

4
src/internal/operations/backup_test.go
View File

@ -391,7 +391,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
TotalUploadedBytes: 1,
},
gc: &support.ConnectorOperationStatus{
Successful: 1,
Metrics: support.CollectionMetrics{Successes: 1},
},
},
},
@ -430,7 +430,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
test.expectErr(t, op.persistResults(now, &test.stats))
assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status")
assert.Equal(t, test.stats.gc.Successful, op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.gc.Metrics.Successes, op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.k.TotalFileCount, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.k.TotalHashedBytes, op.Results.BytesRead, "bytes read")
assert.Equal(t, test.stats.k.TotalUploadedBytes, op.Results.BytesUploaded, "bytes written")

8
src/internal/operations/restore.go
View File

@ -267,10 +267,6 @@ func (op *RestoreOperation) do(
restoreComplete <- struct{}{}
opStats.gc = gc.AwaitStatus()
// TODO(keepers): remove when fault.Errors handles all iterable error aggregation.
if opStats.gc.ErrorCount > 0 {
return nil, opStats.gc.Err
}
logger.Ctx(ctx).Debug(gc.PrintableStatus())
@ -309,11 +305,11 @@ func (op *RestoreOperation) persistResults(
return errors.New("restoration never completed")
}
if opStats.gc.Successful == 0 {
if opStats.gc.Metrics.Successes == 0 {
op.Status = NoData
}
op.Results.ItemsWritten = opStats.gc.Successful
op.Results.ItemsWritten = opStats.gc.Metrics.Successes
dur := op.Results.CompletedAt.Sub(op.Results.StartedAt)

8
src/internal/operations/restore_test.go
View File

@ -68,8 +68,10 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
},
},
gc: &support.ConnectorOperationStatus{
ObjectCount: 1,
Successful: 1,
Metrics: support.CollectionMetrics{
Objects: 1,
Successes: 1,
},
},
},
},
@ -109,7 +111,7 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() {
assert.Equal(t, test.expectStatus.String(), op.Status.String(), "status")
assert.Equal(t, len(test.stats.cs), op.Results.ItemsRead, "items read")
assert.Equal(t, test.stats.gc.Successful, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.gc.Metrics.Successes, op.Results.ItemsWritten, "items written")
assert.Equal(t, test.stats.bytesRead.NumBytes, op.Results.BytesRead, "resource owners")
assert.Equal(t, test.stats.resourceCount, op.Results.ResourceOwners, "resource owners")
assert.Equal(t, test.stats.readErr, op.Results.ReadErrors, "read errors")