store backup operation results in the backup manifest (#410)

* store backup operation results in the backup manifest

Adds backup operation metadata like the outcome
statistics and selector definitions to the backup manifest
entry.  These additional details will appear when users
call `corso backup list`.
This commit is contained in:
Keepers 2022-07-26 13:35:14 -06:00 committed by GitHub
parent 1929b2307f
commit 587c239dd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 501 additions and 328 deletions

View File

@ -6,6 +6,7 @@ import (
"os" "os"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/tidwall/pretty" "github.com/tidwall/pretty"
"github.com/tomlazar/table" "github.com/tomlazar/table"
@ -37,7 +38,7 @@ func Backups(bs []backup.Backup) {
} }
// Prints the entries to the terminal with stdout. // Prints the entries to the terminal with stdout.
func Entries(des []backup.DetailsEntry) { func Entries(des []details.DetailsEntry) {
ps := []Printable{} ps := []Printable{}
for _, de := range des { for _, de := range des {
ps = append(ps, de) ps = append(ps, de)

View File

@ -3,11 +3,11 @@ package exchange
import ( import (
"time" "time"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
) )
func MessageInfo(msg models.Messageable) *backup.ExchangeInfo { func MessageInfo(msg models.Messageable) *details.ExchangeInfo {
sender := "" sender := ""
subject := "" subject := ""
received := time.Time{} received := time.Time{}
@ -22,7 +22,7 @@ func MessageInfo(msg models.Messageable) *backup.ExchangeInfo {
if msg.GetReceivedDateTime() != nil { if msg.GetReceivedDateTime() != nil {
received = *msg.GetReceivedDateTime() received = *msg.GetReceivedDateTime()
} }
return &backup.ExchangeInfo{ return &details.ExchangeInfo{
Sender: sender, Sender: sender,
Subject: subject, Subject: subject,
Received: received, Received: received,

View File

@ -4,7 +4,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
) )
@ -20,17 +20,17 @@ func TestMessageSuite(t *testing.T) {
func (suite *MessageSuite) TestMessageInfo() { func (suite *MessageSuite) TestMessageInfo() {
tests := []struct { tests := []struct {
name string name string
msgAndRP func() (models.Messageable, *backup.ExchangeInfo) msgAndRP func() (models.Messageable, *details.ExchangeInfo)
}{ }{
{ {
name: "Empty message", name: "Empty message",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) { msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
return models.NewMessage(), &backup.ExchangeInfo{} return models.NewMessage(), &details.ExchangeInfo{}
}, },
}, },
{ {
name: "Just sender", name: "Just sender",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) { msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
sender := "foo@bar.com" sender := "foo@bar.com"
sr := models.NewRecipient() sr := models.NewRecipient()
sea := models.NewEmailAddress() sea := models.NewEmailAddress()
@ -38,30 +38,30 @@ func (suite *MessageSuite) TestMessageInfo() {
sea.SetAddress(&sender) sea.SetAddress(&sender)
sr.SetEmailAddress(sea) sr.SetEmailAddress(sea)
msg.SetSender(sr) msg.SetSender(sr)
return msg, &backup.ExchangeInfo{Sender: sender} return msg, &details.ExchangeInfo{Sender: sender}
}, },
}, },
{ {
name: "Just subject", name: "Just subject",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) { msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
subject := "Hello world" subject := "Hello world"
msg := models.NewMessage() msg := models.NewMessage()
msg.SetSubject(&subject) msg.SetSubject(&subject)
return msg, &backup.ExchangeInfo{Subject: subject} return msg, &details.ExchangeInfo{Subject: subject}
}, },
}, },
{ {
name: "Just receivedtime", name: "Just receivedtime",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) { msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
now := time.Now() now := time.Now()
msg := models.NewMessage() msg := models.NewMessage()
msg.SetReceivedDateTime(&now) msg.SetReceivedDateTime(&now)
return msg, &backup.ExchangeInfo{Received: now} return msg, &details.ExchangeInfo{Received: now}
}, },
}, },
{ {
name: "All fields", name: "All fields",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) { msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
sender := "foo@bar.com" sender := "foo@bar.com"
subject := "Hello world" subject := "Hello world"
now := time.Now() now := time.Now()
@ -73,7 +73,7 @@ func (suite *MessageSuite) TestMessageInfo() {
msg.SetSender(sr) msg.SetSender(sr)
msg.SetSubject(&subject) msg.SetSubject(&subject)
msg.SetReceivedDateTime(&now) msg.SetReceivedDateTime(&now)
return msg, &backup.ExchangeInfo{Sender: sender, Subject: subject, Received: now} return msg, &details.ExchangeInfo{Sender: sender, Subject: subject, Received: now}
}, },
}} }}
for _, tt := range tests { for _, tt := range tests {

View File

@ -5,7 +5,7 @@ import (
"io" "io"
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
) )
const ( const (
@ -41,7 +41,7 @@ type DataStream interface {
// DataStreamInfo is used to provide service specific // DataStreamInfo is used to provide service specific
// information about the DataStream // information about the DataStream
type DataStreamInfo interface { type DataStreamInfo interface {
Info() backup.ItemInfo Info() details.ItemInfo
} }
var _ DataCollection = &ExchangeDataCollection{} var _ DataCollection = &ExchangeDataCollection{}
@ -103,7 +103,7 @@ type ExchangeData struct {
// going forward. Using []byte for now but I assume we'll have // going forward. Using []byte for now but I assume we'll have
// some structured type in here (serialization to []byte can be done in `Read`) // some structured type in here (serialization to []byte can be done in `Read`)
message []byte message []byte
info *backup.ExchangeInfo info *details.ExchangeInfo
} }
func (ed *ExchangeData) UUID() string { func (ed *ExchangeData) UUID() string {
@ -114,6 +114,6 @@ func (ed *ExchangeData) ToReader() io.ReadCloser {
return io.NopCloser(bytes.NewReader(ed.message)) return io.NopCloser(bytes.NewReader(ed.message))
} }
func (ed *ExchangeData) Info() backup.ItemInfo { func (ed *ExchangeData) Info() details.ItemInfo {
return backup.ItemInfo{Exchange: ed.info} return details.ItemInfo{Exchange: ed.info}
} }

View File

@ -8,7 +8,7 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
"github.com/alcionai/corso/internal/connector" "github.com/alcionai/corso/internal/connector"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
) )
// MockExchangeDataCollection represents a mock exchange mailbox // MockExchangeDataCollection represents a mock exchange mailbox
@ -79,6 +79,6 @@ func (med *MockExchangeData) ToReader() io.ReadCloser {
return med.Reader return med.Reader
} }
func (med *MockExchangeData) Info() backup.ItemInfo { func (med *MockExchangeData) Info() details.ItemInfo {
return backup.ItemInfo{Exchange: &backup.ExchangeInfo{Sender: "foo@bar.com", Subject: "Hello world!", Received: time.Now()}} return details.ItemInfo{Exchange: &details.ExchangeInfo{Sender: "foo@bar.com", Subject: "Hello world!", Received: time.Now()}}
} }

View File

@ -11,7 +11,7 @@ type ConnectorOperationStatus struct {
lastOperation Operation lastOperation Operation
ObjectCount int ObjectCount int
folderCount int folderCount int
successful int Successful int
errorCount int errorCount int
incomplete bool incomplete bool
incompleteReason string incompleteReason string
@ -38,12 +38,12 @@ func CreateStatus(ctx context.Context, op Operation, objects, success, folders i
lastOperation: op, lastOperation: op,
ObjectCount: objects, ObjectCount: objects,
folderCount: folders, folderCount: folders,
successful: success, Successful: success,
errorCount: numErr, errorCount: numErr,
incomplete: hasErrors, incomplete: hasErrors,
incompleteReason: reason, incompleteReason: reason,
} }
if status.ObjectCount != status.errorCount+status.successful { if status.ObjectCount != status.errorCount+status.Successful {
logger.Ctx(ctx).DPanicw( logger.Ctx(ctx).DPanicw(
"status object count does not match errors + successes", "status object count does not match errors + successes",
"objects", objects, "objects", objects,
@ -55,7 +55,7 @@ func CreateStatus(ctx context.Context, op Operation, objects, success, folders i
func (cos *ConnectorOperationStatus) String() string { func (cos *ConnectorOperationStatus) String() string {
message := fmt.Sprintf("Action: %s performed on %d of %d objects within %d directories.", cos.lastOperation.String(), message := fmt.Sprintf("Action: %s performed on %d of %d objects within %d directories.", cos.lastOperation.String(),
cos.successful, cos.ObjectCount, cos.folderCount) cos.Successful, cos.ObjectCount, cos.folderCount)
if cos.incomplete { if cos.incomplete {
message += " " + cos.incompleteReason message += " " + cos.incompleteReason
} }

View File

@ -15,7 +15,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/internal/connector" "github.com/alcionai/corso/internal/connector"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/logger" "github.com/alcionai/corso/pkg/logger"
) )
@ -81,7 +81,7 @@ func (w *Wrapper) Close(ctx context.Context) error {
// DataCollection. // DataCollection.
func getStreamItemFunc( func getStreamItemFunc(
collection connector.DataCollection, collection connector.DataCollection,
details *backup.Details, details *details.Details,
) func(context.Context, func(context.Context, fs.Entry) error) error { ) func(context.Context, func(context.Context, fs.Entry) error) error {
return func(ctx context.Context, cb func(context.Context, fs.Entry) error) error { return func(ctx context.Context, cb func(context.Context, fs.Entry) error) error {
items := collection.Items() items := collection.Items()
@ -114,7 +114,7 @@ func getStreamItemFunc(
// buildKopiaDirs recursively builds a directory hierarchy from the roots up. // buildKopiaDirs recursively builds a directory hierarchy from the roots up.
// Returned directories are either virtualfs.StreamingDirectory or // Returned directories are either virtualfs.StreamingDirectory or
// virtualfs.staticDirectory. // virtualfs.staticDirectory.
func buildKopiaDirs(dirName string, dir *treeMap, details *backup.Details) (fs.Directory, error) { func buildKopiaDirs(dirName string, dir *treeMap, details *details.Details) (fs.Directory, error) {
// Don't support directories that have both a DataCollection and a set of // Don't support directories that have both a DataCollection and a set of
// static child directories. // static child directories.
if dir.collection != nil && len(dir.childDirs) > 0 { if dir.collection != nil && len(dir.childDirs) > 0 {
@ -156,7 +156,7 @@ func newTreeMap() *treeMap {
// ancestor of the streams and uses virtualfs.StaticDirectory for internal nodes // ancestor of the streams and uses virtualfs.StaticDirectory for internal nodes
// in the hierarchy. Leaf nodes are virtualfs.StreamingDirectory with the given // in the hierarchy. Leaf nodes are virtualfs.StreamingDirectory with the given
// DataCollections. // DataCollections.
func inflateDirTree(ctx context.Context, collections []connector.DataCollection, details *backup.Details) (fs.Directory, error) { func inflateDirTree(ctx context.Context, collections []connector.DataCollection, details *details.Details) (fs.Directory, error) {
roots := make(map[string]*treeMap) roots := make(map[string]*treeMap)
for _, s := range collections { for _, s := range collections {
@ -229,12 +229,12 @@ func inflateDirTree(ctx context.Context, collections []connector.DataCollection,
func (w Wrapper) BackupCollections( func (w Wrapper) BackupCollections(
ctx context.Context, ctx context.Context,
collections []connector.DataCollection, collections []connector.DataCollection,
) (*BackupStats, *backup.Details, error) { ) (*BackupStats, *details.Details, error) {
if w.c == nil { if w.c == nil {
return nil, nil, errNotConnected return nil, nil, errNotConnected
} }
details := &backup.Details{} details := &details.Details{}
dirTree, err := inflateDirTree(ctx, collections, details) dirTree, err := inflateDirTree(ctx, collections, details)
if err != nil { if err != nil {
@ -252,7 +252,7 @@ func (w Wrapper) BackupCollections(
func (w Wrapper) makeSnapshotWithRoot( func (w Wrapper) makeSnapshotWithRoot(
ctx context.Context, ctx context.Context,
root fs.Directory, root fs.Directory,
details *backup.Details, details *details.Details,
) (*BackupStats, error) { ) (*BackupStats, error) {
si := snapshot.SourceInfo{ si := snapshot.SourceInfo{
Host: corsoHost, Host: corsoHost,

View File

@ -20,7 +20,7 @@ import (
"github.com/alcionai/corso/internal/connector/mockconnector" "github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/kopia/mockkopia" "github.com/alcionai/corso/internal/kopia/mockkopia"
ctesting "github.com/alcionai/corso/internal/testing" ctesting "github.com/alcionai/corso/internal/testing"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
) )
const ( const (
@ -117,7 +117,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
user2: 42, user2: 42,
} }
details := &backup.Details{} details := &details.Details{}
collections := []connector.DataCollection{ collections := []connector.DataCollection{
mockconnector.NewMockExchangeDataCollection( mockconnector.NewMockExchangeDataCollection(
@ -180,7 +180,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_NoAncestorDirs() {
expectedFileCount := 42 expectedFileCount := 42
details := &backup.Details{} details := &details.Details{}
collections := []connector.DataCollection{ collections := []connector.DataCollection{
mockconnector.NewMockExchangeDataCollection( mockconnector.NewMockExchangeDataCollection(
[]string{emails}, []string{emails},
@ -259,7 +259,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
ctx := context.Background() ctx := context.Background()
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
details := &backup.Details{} details := &details.Details{}
_, err := inflateDirTree(ctx, test.layout, details) _, err := inflateDirTree(ctx, test.layout, details)
assert.Error(t, err) assert.Error(t, err)
}) })

View File

@ -10,8 +10,10 @@ import (
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/store" "github.com/alcionai/corso/pkg/store"
) )
@ -29,8 +31,8 @@ type BackupOperation struct {
// BackupResults aggregate the details of the result of the operation. // BackupResults aggregate the details of the result of the operation.
type BackupResults struct { type BackupResults struct {
summary stats.ReadWrites
metrics stats.StartAndEndTime
BackupID model.ID `json:"backupID"` BackupID model.ID `json:"backupID"`
} }
@ -71,12 +73,23 @@ type backupStats struct {
} }
// Run begins a synchronous backup operation. // Run begins a synchronous backup operation.
func (op *BackupOperation) Run(ctx context.Context) error { func (op *BackupOperation) Run(ctx context.Context) (err error) {
// TODO: persist initial state of backupOperation in modelstore // TODO: persist initial state of backupOperation in modelstore
// persist operation results to the model store on exit // persist operation results to the model store on exit
stats := backupStats{} var (
defer op.persistResults(time.Now(), &stats) stats backupStats
details *details.Details
)
defer func() {
op.persistResults(time.Now(), &stats)
err = op.createBackupModels(ctx, stats.k.SnapshotID, details)
if err != nil {
stats.writeErr = err
// todo: ^ we're not persisting this yet, except for the error shown to the user.
}
}()
// retrieve data from the producer // retrieve data from the producer
gc, err := connector.NewGraphConnector(op.account) gc, err := connector.NewGraphConnector(op.account)
@ -93,7 +106,6 @@ func (op *BackupOperation) Run(ctx context.Context) error {
} }
// hand the results to the consumer // hand the results to the consumer
var details *backup.Details
stats.k, details, err = op.kopia.BackupCollections(ctx, cs) stats.k, details, err = op.kopia.BackupCollections(ctx, cs)
if err != nil { if err != nil {
stats.writeErr = err stats.writeErr = err
@ -101,33 +113,11 @@ func (op *BackupOperation) Run(ctx context.Context) error {
} }
stats.gc = gc.AwaitStatus() stats.gc = gc.AwaitStatus()
err = op.createBackupModels(ctx, stats.k.SnapshotID, details)
if err != nil {
stats.writeErr = err
return err return err
}
return nil
} }
func (op *BackupOperation) createBackupModels(ctx context.Context, snapID string, details *backup.Details) error { // writes the results metrics to the operation results.
err := op.store.Put(ctx, model.BackupDetailsSchema, &details.DetailsModel) // later stored in the manifest using createBackupModels.
if err != nil {
return errors.Wrap(err, "creating backupdetails model")
}
bu := backup.New(snapID, string(details.ModelStoreID))
err = op.store.Put(ctx, model.BackupSchema, bu)
if err != nil {
return errors.Wrap(err, "creating backup model")
}
op.Results.BackupID = bu.StableID
return nil
}
// writes the backupOperation outcome to the modelStore.
func (op *BackupOperation) persistResults( func (op *BackupOperation) persistResults(
started time.Time, started time.Time,
stats *backupStats, stats *backupStats,
@ -141,7 +131,7 @@ func (op *BackupOperation) persistResults(
op.Results.WriteErrors = stats.writeErr op.Results.WriteErrors = stats.writeErr
if stats.gc != nil { if stats.gc != nil {
op.Results.ItemsRead = stats.gc.ObjectCount op.Results.ItemsRead = stats.gc.Successful
} }
if stats.k != nil { if stats.k != nil {
op.Results.ItemsWritten = stats.k.TotalFileCount op.Results.ItemsWritten = stats.k.TotalFileCount
@ -149,6 +139,26 @@ func (op *BackupOperation) persistResults(
op.Results.StartedAt = started op.Results.StartedAt = started
op.Results.CompletedAt = time.Now() op.Results.CompletedAt = time.Now()
}
// TODO: persist operation to modelstore
// stores the operation details, results, and selectors in the backup manifest.
func (op *BackupOperation) createBackupModels(ctx context.Context, snapID string, details *details.Details) error {
err := op.store.Put(ctx, model.BackupDetailsSchema, &details.DetailsModel)
if err != nil {
return errors.Wrap(err, "creating backupdetails model")
}
b := backup.New(
snapID, string(details.ModelStoreID), op.Status.String(),
op.Selectors,
op.Results.ReadWrites,
op.Results.StartAndEndTime,
)
err = op.store.Put(ctx, model.BackupSchema, b)
if err != nil {
return errors.Wrap(err, "creating backup model")
}
op.Results.BackupID = b.StableID
return nil
} }

View File

@ -48,7 +48,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
TotalFileCount: 1, TotalFileCount: 1,
}, },
gc: &support.ConnectorOperationStatus{ gc: &support.ConnectorOperationStatus{
ObjectCount: 1, Successful: 1,
}, },
} }
) )
@ -59,7 +59,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
op.persistResults(now, &stats) op.persistResults(now, &stats)
assert.Equal(t, op.Status, Failed) assert.Equal(t, op.Status, Failed)
assert.Equal(t, op.Results.ItemsRead, stats.gc.ObjectCount) assert.Equal(t, op.Results.ItemsRead, stats.gc.Successful)
assert.Equal(t, op.Results.ReadErrors, stats.readErr) assert.Equal(t, op.Results.ReadErrors, stats.readErr)
assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount) assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount)
assert.Equal(t, op.Results.WriteErrors, stats.writeErr) assert.Equal(t, op.Results.WriteErrors, stats.writeErr)

View File

@ -12,11 +12,12 @@ import (
type opStatus int type opStatus int
//go:generate stringer -type=opStatus -linecomment
const ( const (
Unknown opStatus = iota Unknown opStatus = iota // Status Unknown
InProgress InProgress // In Progress
Successful Successful // Successful
Failed Failed // Failed
) )
// -------------------------------------------------------------------------------- // --------------------------------------------------------------------------------
@ -72,22 +73,3 @@ func (op operation) validate() error {
} }
return nil return nil
} }
// --------------------------------------------------------------------------------
// Results
// --------------------------------------------------------------------------------
// Summary tracks the total files touched and errors produced
// during an operation.
type summary struct {
ItemsRead int `json:"itemsRead,omitempty"`
ItemsWritten int `json:"itemsWritten,omitempty"`
ReadErrors error `json:"readErrors,omitempty"`
WriteErrors error `json:"writeErrors,omitempty"`
}
// Metrics tracks performance details such as timing, throughput, etc.
type metrics struct {
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
}

View File

@ -0,0 +1,26 @@
// Code generated by "stringer -type=opStatus -linecomment"; DO NOT EDIT.
package operations
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[Unknown-0]
_ = x[InProgress-1]
_ = x[Successful-2]
_ = x[Failed-3]
}
const _opStatus_name = "Status UnknownIn ProgressSuccessfulFailed"
var _opStatus_index = [...]uint8{0, 14, 25, 35, 41}
func (i opStatus) String() string {
if i < 0 || i >= opStatus(len(_opStatus_index)-1) {
return "opStatus(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _opStatus_name[_opStatus_index[i]:_opStatus_index[i+1]]
}

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/kopia" "github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/store" "github.com/alcionai/corso/pkg/store"
@ -30,8 +31,8 @@ type RestoreOperation struct {
// RestoreResults aggregate the details of the results of the operation. // RestoreResults aggregate the details of the results of the operation.
type RestoreResults struct { type RestoreResults struct {
summary stats.ReadWrites
metrics stats.StartAndEndTime
} }
// NewRestoreOperation constructs and validates a restore operation. // NewRestoreOperation constructs and validates a restore operation.

View File

@ -0,0 +1,20 @@
package stats
import "time"
// ReadWrites tracks the total count of reads and writes, and of
// read and write errors. ItemsRead and ItemsWritten counts are
// assumed to be successful, so the total count of items involved
// would be ItemsRead+ReadErrors.
type ReadWrites struct {
ItemsRead int `json:"itemsRead,omitempty"`
ItemsWritten int `json:"itemsWritten,omitempty"`
ReadErrors error `json:"readErrors,omitempty"`
WriteErrors error `json:"writeErrors,omitempty"`
}
// StartAndEndTime tracks a paired starting time and ending time.
type StartAndEndTime struct {
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
}

View File

@ -1,10 +1,14 @@
package backup package backup
import ( import (
"sync" "strconv"
"time" "time"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/selectors"
) )
// Backup represents the result of a backup operation // Backup represents the result of a backup operation
@ -17,10 +21,17 @@ type Backup struct {
// Reference to `Details` // Reference to `Details`
// We store the ModelStoreID since Details is immutable // We store the ModelStoreID since Details is immutable
DetailsID string `json:"detailsId"` DetailsID string `json:"detailsID"`
// TODO: // Status of the operation
// - Backup "Specification" Status string `json:"status"`
// Selectors used in this operation
Selectors selectors.Selector `json:"selectors"`
// stats are embedded so that the values appear as top-level properties
stats.ReadWrites
stats.StartAndEndTime
} }
// Headers returns the human-readable names of properties in a Backup // Headers returns the human-readable names of properties in a Backup
@ -31,6 +42,14 @@ func (b Backup) Headers() []string {
"Stable ID", "Stable ID",
"Snapshot ID", "Snapshot ID",
"Details ID", "Details ID",
"Status",
"Selectors",
"Items Read",
"Items Written",
"Read Errors",
"Write Errors",
"Started At",
"Completed At",
} }
} }
@ -38,125 +57,34 @@ func (b Backup) Headers() []string {
// out to a terminal in a columnar display. // out to a terminal in a columnar display.
func (b Backup) Values() []string { func (b Backup) Values() []string {
return []string{ return []string{
b.CreationTime.Format(time.RFC3339Nano), common.FormatTime(b.CreationTime),
string(b.StableID), string(b.StableID),
b.SnapshotID, b.SnapshotID,
b.DetailsID, b.DetailsID,
b.Status,
b.Selectors.String(),
strconv.Itoa(b.ReadWrites.ItemsRead),
strconv.Itoa(b.ReadWrites.ItemsWritten),
strconv.Itoa(support.GetNumberOfErrors(b.ReadWrites.ReadErrors)),
strconv.Itoa(support.GetNumberOfErrors(b.ReadWrites.WriteErrors)),
common.FormatTime(b.StartAndEndTime.StartedAt),
common.FormatTime(b.StartAndEndTime.CompletedAt),
} }
} }
func New(snapshotID, detailsID string) *Backup { func New(
snapshotID, detailsID, status string,
selector selectors.Selector,
rw stats.ReadWrites,
se stats.StartAndEndTime,
) *Backup {
return &Backup{ return &Backup{
CreationTime: time.Now(), CreationTime: time.Now(),
SnapshotID: snapshotID, SnapshotID: snapshotID,
DetailsID: detailsID, DetailsID: detailsID,
Status: status,
Selectors: selector,
ReadWrites: rw,
StartAndEndTime: se,
} }
} }
// DetailsModel describes what was stored in a Backup
type DetailsModel struct {
model.BaseModel
Entries []DetailsEntry `json:"entries"`
}
// Details augments the core with a mutex for processing.
// Should be sliced back to d.DetailsModel for storage and
// printing.
type Details struct {
DetailsModel
// internal
mu sync.Mutex `json:"-"`
}
// DetailsEntry describes a single item stored in a Backup
type DetailsEntry struct {
// TODO: `RepoRef` is currently the full path to the item in Kopia
// This can be optimized.
RepoRef string `json:"repoRef"`
ItemInfo
}
// Paths returns the list of Paths extracted from the Entries slice.
func (dm DetailsModel) Paths() []string {
ents := dm.Entries
r := make([]string, len(ents))
for i := range ents {
r[i] = ents[i].RepoRef
}
return r
}
// Headers returns the human-readable names of properties in a DetailsEntry
// for printing out to a terminal in a columnar display.
func (de DetailsEntry) Headers() []string {
hs := []string{"Repo Ref"}
if de.ItemInfo.Exchange != nil {
hs = append(hs, de.ItemInfo.Exchange.Headers()...)
}
if de.ItemInfo.Sharepoint != nil {
hs = append(hs, de.ItemInfo.Sharepoint.Headers()...)
}
return hs
}
// Values returns the values matching the Headers list.
func (de DetailsEntry) Values() []string {
vs := []string{de.RepoRef}
if de.ItemInfo.Exchange != nil {
vs = append(vs, de.ItemInfo.Exchange.Values()...)
}
if de.ItemInfo.Sharepoint != nil {
vs = append(vs, de.ItemInfo.Sharepoint.Values()...)
}
return vs
}
// ItemInfo is a oneOf that contains service specific
// information about the item it tracks
type ItemInfo struct {
Exchange *ExchangeInfo `json:"exchange,omitempty"`
Sharepoint *SharepointInfo `json:"sharepoint,omitempty"`
}
// ExchangeInfo describes an exchange item
type ExchangeInfo struct {
Sender string `json:"sender"`
Subject string `json:"subject"`
Received time.Time `json:"received"`
}
// Headers returns the human-readable names of properties in an ExchangeInfo
// for printing out to a terminal in a columnar display.
func (e ExchangeInfo) Headers() []string {
return []string{"Sender", "Subject", "Received"}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (e ExchangeInfo) Values() []string {
return []string{e.Sender, e.Subject, e.Received.Format(time.RFC3339Nano)}
}
// SharepointInfo describes a sharepoint item
// TODO: Implement this. This is currently here
// just to illustrate usage
type SharepointInfo struct{}
func (d *Details) Add(repoRef string, info ItemInfo) {
d.mu.Lock()
defer d.mu.Unlock()
d.Entries = append(d.Entries, DetailsEntry{RepoRef: repoRef, ItemInfo: info})
}
// Headers returns the human-readable names of properties in a SharepointInfo
// for printing out to a terminal in a columnar display.
func (s SharepointInfo) Headers() []string {
return []string{}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (s SharepointInfo) Values() []string {
return []string{}
}

View File

@ -1,14 +1,19 @@
package backup_test package backup_test
import ( import (
"errors"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/zeebo/assert" "github.com/zeebo/assert"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors"
) )
type BackupSuite struct { type BackupSuite struct {
@ -30,6 +35,18 @@ func (suite *BackupSuite) TestBackup_HeadersValues() {
CreationTime: now, CreationTime: now,
SnapshotID: "snapshot", SnapshotID: "snapshot",
DetailsID: "details", DetailsID: "details",
Status: "status",
Selectors: selectors.Selector{},
ReadWrites: stats.ReadWrites{
ItemsRead: 1,
ItemsWritten: 1,
ReadErrors: errors.New("1"),
WriteErrors: errors.New("1"),
},
StartAndEndTime: stats.StartAndEndTime{
StartedAt: now,
CompletedAt: now,
},
} }
expectHs := []string{ expectHs := []string{
@ -37,15 +54,32 @@ func (suite *BackupSuite) TestBackup_HeadersValues() {
"Stable ID", "Stable ID",
"Snapshot ID", "Snapshot ID",
"Details ID", "Details ID",
"Status",
"Selectors",
"Items Read",
"Items Written",
"Read Errors",
"Write Errors",
"Started At",
"Completed At",
} }
hs := b.Headers() hs := b.Headers()
assert.DeepEqual(t, expectHs, hs) assert.DeepEqual(t, expectHs, hs)
nowFmt := common.FormatTime(now)
expectVs := []string{ expectVs := []string{
now.Format(time.RFC3339Nano), nowFmt,
"stable", "stable",
"snapshot", "snapshot",
"details", "details",
"status",
"{}",
"1",
"1",
"1",
"1",
nowFmt,
nowFmt,
} }
vs := b.Values() vs := b.Values()
assert.DeepEqual(t, expectVs, vs) assert.DeepEqual(t, expectVs, vs)
@ -57,13 +91,13 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
table := []struct { table := []struct {
name string name string
entry backup.DetailsEntry entry details.DetailsEntry
expectHs []string expectHs []string
expectVs []string expectVs []string
}{ }{
{ {
name: "no info", name: "no info",
entry: backup.DetailsEntry{ entry: details.DetailsEntry{
RepoRef: "reporef", RepoRef: "reporef",
}, },
expectHs: []string{"Repo Ref"}, expectHs: []string{"Repo Ref"},
@ -71,10 +105,10 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
}, },
{ {
name: "exhange info", name: "exhange info",
entry: backup.DetailsEntry{ entry: details.DetailsEntry{
RepoRef: "reporef", RepoRef: "reporef",
ItemInfo: backup.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &backup.ExchangeInfo{ Exchange: &details.ExchangeInfo{
Sender: "sender", Sender: "sender",
Subject: "subject", Subject: "subject",
Received: now, Received: now,
@ -86,10 +120,10 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
}, },
{ {
name: "sharepoint info", name: "sharepoint info",
entry: backup.DetailsEntry{ entry: details.DetailsEntry{
RepoRef: "reporef", RepoRef: "reporef",
ItemInfo: backup.ItemInfo{ ItemInfo: details.ItemInfo{
Sharepoint: &backup.SharepointInfo{}, Sharepoint: &details.SharepointInfo{},
}, },
}, },
expectHs: []string{"Repo Ref"}, expectHs: []string{"Repo Ref"},
@ -110,7 +144,7 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
func (suite *BackupSuite) TestDetailsModel_Path() { func (suite *BackupSuite) TestDetailsModel_Path() {
table := []struct { table := []struct {
name string name string
ents []backup.DetailsEntry ents []details.DetailsEntry
expect []string expect []string
}{ }{
{ {
@ -120,14 +154,14 @@ func (suite *BackupSuite) TestDetailsModel_Path() {
}, },
{ {
name: "single entry", name: "single entry",
ents: []backup.DetailsEntry{ ents: []details.DetailsEntry{
{RepoRef: "abcde"}, {RepoRef: "abcde"},
}, },
expect: []string{"abcde"}, expect: []string{"abcde"},
}, },
{ {
name: "multiple entries", name: "multiple entries",
ents: []backup.DetailsEntry{ ents: []details.DetailsEntry{
{RepoRef: "abcde"}, {RepoRef: "abcde"},
{RepoRef: "12345"}, {RepoRef: "12345"},
}, },
@ -136,8 +170,8 @@ func (suite *BackupSuite) TestDetailsModel_Path() {
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
d := backup.Details{ d := details.Details{
DetailsModel: backup.DetailsModel{ DetailsModel: details.DetailsModel{
Entries: test.ents, Entries: test.ents,
}, },
} }

View File

@ -0,0 +1,116 @@
package details
import (
"sync"
"time"
"github.com/alcionai/corso/internal/model"
)
// DetailsModel describes what was stored in a Backup
type DetailsModel struct {
model.BaseModel
Entries []DetailsEntry `json:"entries"`
}
// Details augments the core with a mutex for processing.
// Should be sliced back to d.DetailsModel for storage and
// printing.
type Details struct {
DetailsModel
// internal
mu sync.Mutex `json:"-"`
}
// DetailsEntry describes a single item stored in a Backup
type DetailsEntry struct {
// TODO: `RepoRef` is currently the full path to the item in Kopia
// This can be optimized.
RepoRef string `json:"repoRef"`
ItemInfo
}
// Paths returns the list of Paths extracted from the Entries slice.
func (dm DetailsModel) Paths() []string {
ents := dm.Entries
r := make([]string, len(ents))
for i := range ents {
r[i] = ents[i].RepoRef
}
return r
}
// Headers returns the human-readable names of properties in a DetailsEntry
// for printing out to a terminal in a columnar display.
func (de DetailsEntry) Headers() []string {
hs := []string{"Repo Ref"}
if de.ItemInfo.Exchange != nil {
hs = append(hs, de.ItemInfo.Exchange.Headers()...)
}
if de.ItemInfo.Sharepoint != nil {
hs = append(hs, de.ItemInfo.Sharepoint.Headers()...)
}
return hs
}
// Values returns the values matching the Headers list.
func (de DetailsEntry) Values() []string {
vs := []string{de.RepoRef}
if de.ItemInfo.Exchange != nil {
vs = append(vs, de.ItemInfo.Exchange.Values()...)
}
if de.ItemInfo.Sharepoint != nil {
vs = append(vs, de.ItemInfo.Sharepoint.Values()...)
}
return vs
}
// ItemInfo is a oneOf that contains service specific
// information about the item it tracks
type ItemInfo struct {
Exchange *ExchangeInfo `json:"exchange,omitempty"`
Sharepoint *SharepointInfo `json:"sharepoint,omitempty"`
}
// ExchangeInfo describes an exchange item
type ExchangeInfo struct {
Sender string `json:"sender"`
Subject string `json:"subject"`
Received time.Time `json:"received"`
}
// Headers returns the human-readable names of properties in an ExchangeInfo
// for printing out to a terminal in a columnar display.
func (e ExchangeInfo) Headers() []string {
return []string{"Sender", "Subject", "Received"}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (e ExchangeInfo) Values() []string {
return []string{e.Sender, e.Subject, e.Received.Format(time.RFC3339Nano)}
}
// SharepointInfo describes a sharepoint item
// TODO: Implement this. This is currently here
// just to illustrate usage
type SharepointInfo struct{}
func (d *Details) Add(repoRef string, info ItemInfo) {
d.mu.Lock()
defer d.mu.Unlock()
d.Entries = append(d.Entries, DetailsEntry{RepoRef: repoRef, ItemInfo: info})
}
// Headers returns the human-readable names of properties in a SharepointInfo
// for printing out to a terminal in a columnar display.
func (s SharepointInfo) Headers() []string {
return []string{}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (s SharepointInfo) Values() []string {
return []string{}
}

View File

@ -0,0 +1,116 @@
package details_test
import (
"context"
"testing"
"time"
"github.com/google/uuid"
"github.com/kopia/kopia/repo/manifest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/store"
storeMock "github.com/alcionai/corso/pkg/store/mock"
)
// ------------------------------------------------------------
// unit tests
// ------------------------------------------------------------
var (
detailsID = uuid.NewString()
bu = backup.Backup{
BaseModel: model.BaseModel{
StableID: model.ID(uuid.NewString()),
ModelStoreID: manifest.ID(uuid.NewString()),
},
CreationTime: time.Now(),
SnapshotID: uuid.NewString(),
DetailsID: detailsID,
}
deets = details.Details{
DetailsModel: details.DetailsModel{
BaseModel: model.BaseModel{
StableID: model.ID(detailsID),
ModelStoreID: manifest.ID(uuid.NewString()),
},
},
}
)
type StoreDetailsUnitSuite struct {
suite.Suite
}
func TestStoreDetailsUnitSuite(t *testing.T) {
suite.Run(t, new(StoreDetailsUnitSuite))
}
func (suite *StoreDetailsUnitSuite) TestGetDetails() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details",
mock: storeMock.NewMock(nil, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(nil, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{Storer: test.mock}
result, err := sm.GetDetails(ctx, manifest.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, result.StableID)
})
}
}
func (suite *StoreDetailsUnitSuite) TestGetDetailsFromBackupID() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details from backup id",
mock: storeMock.NewMock(&bu, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(&bu, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{Storer: test.mock}
dResult, bResult, err := store.GetDetailsFromBackupID(ctx, model.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, dResult.StableID)
assert.Equal(t, bu.StableID, bResult.StableID)
})
}
}

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/internal/operations" "github.com/alcionai/corso/internal/operations"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/storage" "github.com/alcionai/corso/pkg/storage"
"github.com/alcionai/corso/pkg/store" "github.com/alcionai/corso/pkg/store"
@ -162,7 +163,7 @@ func (r Repository) Backups(ctx context.Context) ([]backup.Backup, error) {
} }
// BackupDetails returns the specified backup details object // BackupDetails returns the specified backup details object
func (r Repository) BackupDetails(ctx context.Context, backupID string) (*backup.Details, *backup.Backup, error) { func (r Repository) BackupDetails(ctx context.Context, backupID string) (*details.Details, *backup.Backup, error) {
sw := store.NewKopiaStore(r.modelStore) sw := store.NewKopiaStore(r.modelStore)
return sw.GetDetailsFromBackupID(ctx, model.ID(backupID)) return sw.GetDetailsFromBackupID(ctx, model.ID(backupID))
} }

View File

@ -4,7 +4,7 @@ import (
"strings" "strings"
"github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -500,13 +500,13 @@ var categoryPathSet = map[exchangeCategory][]exchangeCategory{
} }
// matches returns true if either the path or the info matches the scope details. // matches returns true if either the path or the info matches the scope details.
func (s ExchangeScope) matches(cat exchangeCategory, path []string, info *backup.ExchangeInfo) bool { func (s ExchangeScope) matches(cat exchangeCategory, path []string, info *details.ExchangeInfo) bool {
return s.matchesPath(cat, path) || s.matchesInfo(cat, info) return s.matchesPath(cat, path) || s.matchesInfo(cat, info)
} }
// matchesInfo handles the standard behavior when comparing a scope and an exchangeInfo // matchesInfo handles the standard behavior when comparing a scope and an exchangeInfo
// returns true if the scope and info match for the provided category. // returns true if the scope and info match for the provided category.
func (s ExchangeScope) matchesInfo(cat exchangeCategory, info *backup.ExchangeInfo) bool { func (s ExchangeScope) matchesInfo(cat exchangeCategory, info *details.ExchangeInfo) bool {
// we need values to match against // we need values to match against
if info == nil { if info == nil {
return false return false
@ -634,7 +634,7 @@ func exchangeIDPath(cat exchangeCategory, path []string) map[exchangeCategory]st
// Reduce reduces the entries in a backupDetails struct to only // Reduce reduces the entries in a backupDetails struct to only
// those that match the inclusions, filters, and exclusions in the selector. // those that match the inclusions, filters, and exclusions in the selector.
func (s *ExchangeRestore) Reduce(deets *backup.Details) *backup.Details { func (s *ExchangeRestore) Reduce(deets *details.Details) *details.Details {
if deets == nil { if deets == nil {
return nil return nil
} }
@ -643,7 +643,7 @@ func (s *ExchangeRestore) Reduce(deets *backup.Details) *backup.Details {
entFilt := exchangeScopesByCategory(s.Filters) entFilt := exchangeScopesByCategory(s.Filters)
entIncs := exchangeScopesByCategory(s.Includes) entIncs := exchangeScopesByCategory(s.Includes)
ents := []backup.DetailsEntry{} ents := []details.DetailsEntry{}
for _, ent := range deets.Entries { for _, ent := range deets.Entries {
// todo: use Path pkg for this // todo: use Path pkg for this
@ -706,7 +706,7 @@ func exchangeScopesByCategory(scopes []map[string]string) map[string][]ExchangeS
func matchExchangeEntry( func matchExchangeEntry(
cat exchangeCategory, cat exchangeCategory,
path []string, path []string,
info *backup.ExchangeInfo, info *details.ExchangeInfo,
excs, filts, incs []ExchangeScope, excs, filts, incs []ExchangeScope,
) bool { ) bool {
// a passing match requires either a filter or an inclusion // a passing match requires either a filter or an inclusion

View File

@ -5,7 +5,7 @@ import (
"time" "time"
"github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup/details"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -498,7 +498,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_MatchesInfo() {
epoch = time.Time{} epoch = time.Time{}
now = time.Now() now = time.Now()
then = now.Add(1 * time.Minute) then = now.Add(1 * time.Minute)
info = &backup.ExchangeInfo{ info = &details.ExchangeInfo{
Sender: sender, Sender: sender,
Subject: subject, Subject: subject,
Received: now, Received: now,
@ -629,14 +629,14 @@ func (suite *ExchangeSourceSuite) TestIdPath() {
} }
func (suite *ExchangeSourceSuite) TestExchangeRestore_Reduce() { func (suite *ExchangeSourceSuite) TestExchangeRestore_Reduce() {
makeDeets := func(refs ...string) *backup.Details { makeDeets := func(refs ...string) *details.Details {
deets := &backup.Details{ deets := &details.Details{
DetailsModel: backup.DetailsModel{ DetailsModel: details.DetailsModel{
Entries: []backup.DetailsEntry{}, Entries: []details.DetailsEntry{},
}, },
} }
for _, r := range refs { for _, r := range refs {
deets.Entries = append(deets.Entries, backup.DetailsEntry{ deets.Entries = append(deets.Entries, details.DetailsEntry{
RepoRef: r, RepoRef: r,
}) })
} }
@ -652,7 +652,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_Reduce() {
} }
table := []struct { table := []struct {
name string name string
deets *backup.Details deets *details.Details
makeSelector func() *ExchangeRestore makeSelector func() *ExchangeRestore
expect []string expect []string
}{ }{
@ -825,7 +825,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScopesByCategory() {
} }
func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() { func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() {
var exchangeInfo *backup.ExchangeInfo var exchangeInfo *details.ExchangeInfo
const ( const (
mid = "mailID" mid = "mailID"
cat = ExchangeMail cat = ExchangeMail

View File

@ -1,6 +1,7 @@
package selectors package selectors
import ( import (
"encoding/json"
"strings" "strings"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -84,6 +85,14 @@ func None() []string {
return []string{NoneTgt} return []string{NoneTgt}
} }
func (s Selector) String() string {
bs, err := json.Marshal(s)
if err != nil {
return "error"
}
return string(bs)
}
type baseScope interface { type baseScope interface {
~map[string]string ~map[string]string
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
) )
// GetBackup gets a single backup by id. // GetBackup gets a single backup by id.
@ -39,8 +40,8 @@ func (w Wrapper) GetBackups(ctx context.Context) ([]backup.Backup, error) {
} }
// GetDetails gets the backup details by ID. // GetDetails gets the backup details by ID.
func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*backup.Details, error) { func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*details.Details, error) {
d := backup.Details{} d := details.Details{}
err := w.GetWithModelStoreID(ctx, model.BackupDetailsSchema, detailsID, &d) err := w.GetWithModelStoreID(ctx, model.BackupDetailsSchema, detailsID, &d)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "getting details") return nil, errors.Wrap(err, "getting details")
@ -49,7 +50,7 @@ func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*backup
} }
// GetDetailsFromBackupID retrieves the backup.Details within the specified backup. // GetDetailsFromBackupID retrieves the backup.Details within the specified backup.
func (w Wrapper) GetDetailsFromBackupID(ctx context.Context, backupID model.ID) (*backup.Details, *backup.Backup, error) { func (w Wrapper) GetDetailsFromBackupID(ctx context.Context, backupID model.ID) (*details.Details, *backup.Backup, error) {
b, err := w.GetBackup(ctx, backupID) b, err := w.GetBackup(ctx, backupID)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err

View File

@ -31,14 +31,6 @@ var (
SnapshotID: uuid.NewString(), SnapshotID: uuid.NewString(),
DetailsID: detailsID, DetailsID: detailsID,
} }
deets = backup.Details{
DetailsModel: backup.DetailsModel{
BaseModel: model.BaseModel{
StableID: model.ID(detailsID),
ModelStoreID: manifest.ID(uuid.NewString()),
},
},
}
) )
type StoreBackupUnitSuite struct { type StoreBackupUnitSuite struct {
@ -70,8 +62,8 @@ func (suite *StoreBackupUnitSuite) TestGetBackup() {
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{test.mock} sm := &store.Wrapper{Storer: test.mock}
result, err := store.GetBackup(ctx, model.ID(uuid.NewString())) result, err := sm.GetBackup(ctx, model.ID(uuid.NewString()))
test.expect(t, err) test.expect(t, err)
if err != nil { if err != nil {
return return
@ -102,7 +94,7 @@ func (suite *StoreBackupUnitSuite) TestGetBackups() {
} }
for _, test := range table { for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) { suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{test.mock} sm := &store.Wrapper{Storer: test.mock}
result, err := sm.GetBackups(ctx) result, err := sm.GetBackups(ctx)
test.expect(t, err) test.expect(t, err)
if err != nil { if err != nil {
@ -113,68 +105,3 @@ func (suite *StoreBackupUnitSuite) TestGetBackups() {
}) })
} }
} }
func (suite *StoreBackupUnitSuite) TestGetDetails() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details",
mock: storeMock.NewMock(nil, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(nil, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{test.mock}
result, err := sm.GetDetails(ctx, manifest.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, result.StableID)
})
}
}
func (suite *StoreBackupUnitSuite) TestGetDetailsFromBackupID() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details from backup id",
mock: storeMock.NewMock(&bu, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(&bu, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{test.mock}
dResult, bResult, err := store.GetDetailsFromBackupID(ctx, model.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, dResult.StableID)
assert.Equal(t, bu.StableID, bResult.StableID)
})
}
}

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/internal/model" "github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup" "github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
) )
// ------------------------------------------------------------ // ------------------------------------------------------------
@ -21,7 +22,7 @@ type MockModelStore struct {
err error err error
} }
func NewMock(b *backup.Backup, d *backup.Details, err error) *MockModelStore { func NewMock(b *backup.Backup, d *details.Details, err error) *MockModelStore {
return &MockModelStore{ return &MockModelStore{
backup: marshal(b), backup: marshal(b),
details: marshal(d), details: marshal(d),
@ -89,7 +90,7 @@ func (mms *MockModelStore) GetIDsForType(
unmarshal(mms.backup, &b) unmarshal(mms.backup, &b)
return []*model.BaseModel{&b.BaseModel}, nil return []*model.BaseModel{&b.BaseModel}, nil
case model.BackupDetailsSchema: case model.BackupDetailsSchema:
d := backup.Details{} d := details.Details{}
unmarshal(mms.backup, &d) unmarshal(mms.backup, &d)
return []*model.BaseModel{&d.BaseModel}, nil return []*model.BaseModel{&d.BaseModel}, nil
} }