larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

store backup operation results in the backup manifest (#410)

Browse Source 
* store backup operation results in the backup manifest

Adds backup operation metadata like the outcome
statistics and selector definitions to the backup manifest
entry.  These additional details will appear when users
call `corso backup list`.
This commit is contained in:
Keepers 2022-07-26 13:35:14 -06:00 committed by GitHub
parent 1929b2307f
commit 587c239dd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 501 additions and 328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/cli/print/print.go
View File

@ -6,6 +6,7 @@ import (
"os"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/spf13/cobra"
"github.com/tidwall/pretty"
"github.com/tomlazar/table"
@ -37,7 +38,7 @@ func Backups(bs []backup.Backup) {
}
// Prints the entries to the terminal with stdout.
func Entries(des []backup.DetailsEntry) {
func Entries(des []details.DetailsEntry) {
ps := []Printable{}
for _, de := range des {
ps = append(ps, de)

6
src/internal/connector/exchange/message.go
View File

@ -3,11 +3,11 @@ package exchange
import (
"time"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/microsoftgraph/msgraph-sdk-go/models"
)
func MessageInfo(msg models.Messageable) *backup.ExchangeInfo {
func MessageInfo(msg models.Messageable) *details.ExchangeInfo {
sender := ""
subject := ""
received := time.Time{}
@ -22,7 +22,7 @@ func MessageInfo(msg models.Messageable) *backup.ExchangeInfo {
if msg.GetReceivedDateTime() != nil {
received = *msg.GetReceivedDateTime()
}
return &backup.ExchangeInfo{
return &details.ExchangeInfo{
Sender: sender,
Subject: subject,
Received: received,

24
src/internal/connector/exchange/message_test.go
View File

@ -4,7 +4,7 @@ import (
"testing"
"time"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/suite"
)
@ -20,17 +20,17 @@ func TestMessageSuite(t *testing.T) {
func (suite *MessageSuite) TestMessageInfo() {
tests := []struct {
name string
msgAndRP func() (models.Messageable, *backup.ExchangeInfo)
msgAndRP func() (models.Messageable, *details.ExchangeInfo)
}{
{
name: "Empty message",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) {
return models.NewMessage(), &backup.ExchangeInfo{}
msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
return models.NewMessage(), &details.ExchangeInfo{}
},
},
{
name: "Just sender",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) {
msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
sender := "foo@bar.com"
sr := models.NewRecipient()
sea := models.NewEmailAddress()
@ -38,30 +38,30 @@ func (suite *MessageSuite) TestMessageInfo() {
sea.SetAddress(&sender)
sr.SetEmailAddress(sea)
msg.SetSender(sr)
return msg, &backup.ExchangeInfo{Sender: sender}
return msg, &details.ExchangeInfo{Sender: sender}
},
},
{
name: "Just subject",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) {
msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
subject := "Hello world"
msg := models.NewMessage()
msg.SetSubject(&subject)
return msg, &backup.ExchangeInfo{Subject: subject}
return msg, &details.ExchangeInfo{Subject: subject}
},
},
{
name: "Just receivedtime",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) {
msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
now := time.Now()
msg := models.NewMessage()
msg.SetReceivedDateTime(&now)
return msg, &backup.ExchangeInfo{Received: now}
return msg, &details.ExchangeInfo{Received: now}
},
},
{
name: "All fields",
msgAndRP: func() (models.Messageable, *backup.ExchangeInfo) {
msgAndRP: func() (models.Messageable, *details.ExchangeInfo) {
sender := "foo@bar.com"
subject := "Hello world"
now := time.Now()
@ -73,7 +73,7 @@ func (suite *MessageSuite) TestMessageInfo() {
msg.SetSender(sr)
msg.SetSubject(&subject)
msg.SetReceivedDateTime(&now)
return msg, &backup.ExchangeInfo{Sender: sender, Subject: subject, Received: now}
return msg, &details.ExchangeInfo{Sender: sender, Subject: subject, Received: now}
},
}}
for _, tt := range tests {

10
src/internal/connector/exchange_data_collection.go
View File

@ -5,7 +5,7 @@ import (
"io"
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
const (
@ -41,7 +41,7 @@ type DataStream interface {
// DataStreamInfo is used to provide service specific
// information about the DataStream
type DataStreamInfo interface {
Info() backup.ItemInfo
Info() details.ItemInfo
}
var _ DataCollection = &ExchangeDataCollection{}
@ -103,7 +103,7 @@ type ExchangeData struct {
// going forward. Using []byte for now but I assume we'll have
// some structured type in here (serialization to []byte can be done in `Read`)
message []byte
info *backup.ExchangeInfo
info *details.ExchangeInfo
}
func (ed *ExchangeData) UUID() string {
@ -114,6 +114,6 @@ func (ed *ExchangeData) ToReader() io.ReadCloser {
return io.NopCloser(bytes.NewReader(ed.message))
}
func (ed *ExchangeData) Info() backup.ItemInfo {
return backup.ItemInfo{Exchange: ed.info}
func (ed *ExchangeData) Info() details.ItemInfo {
return details.ItemInfo{Exchange: ed.info}
}

6
src/internal/connector/mockconnector/mock_data_collection.go
View File

@ -8,7 +8,7 @@ import (
"github.com/google/uuid"
"github.com/alcionai/corso/internal/connector"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
// MockExchangeDataCollection represents a mock exchange mailbox
@ -79,6 +79,6 @@ func (med *MockExchangeData) ToReader() io.ReadCloser {
return med.Reader
}
func (med *MockExchangeData) Info() backup.ItemInfo {
return backup.ItemInfo{Exchange: &backup.ExchangeInfo{Sender: "foo@bar.com", Subject: "Hello world!", Received: time.Now()}}
func (med *MockExchangeData) Info() details.ItemInfo {
return details.ItemInfo{Exchange: &details.ExchangeInfo{Sender: "foo@bar.com", Subject: "Hello world!", Received: time.Now()}}
}

8
src/internal/connector/support/status.go
View File

@ -11,7 +11,7 @@ type ConnectorOperationStatus struct {
lastOperation Operation
ObjectCount int
folderCount int
successful int
Successful int
errorCount int
incomplete bool
incompleteReason string
@ -38,12 +38,12 @@ func CreateStatus(ctx context.Context, op Operation, objects, success, folders i
lastOperation: op,
ObjectCount: objects,
folderCount: folders,
successful: success,
Successful: success,
errorCount: numErr,
incomplete: hasErrors,
incompleteReason: reason,
}
if status.ObjectCount != status.errorCount+status.successful {
if status.ObjectCount != status.errorCount+status.Successful {
logger.Ctx(ctx).DPanicw(
"status object count does not match errors + successes",
"objects", objects,
@ -55,7 +55,7 @@ func CreateStatus(ctx context.Context, op Operation, objects, success, folders i
func (cos *ConnectorOperationStatus) String() string {
message := fmt.Sprintf("Action: %s performed on %d of %d objects within %d directories.", cos.lastOperation.String(),
cos.successful, cos.ObjectCount, cos.folderCount)
cos.Successful, cos.ObjectCount, cos.folderCount)
if cos.incomplete {
message += " " + cos.incompleteReason
}

14
src/internal/kopia/wrapper.go
View File

@ -15,7 +15,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/internal/connector"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/logger"
)
@ -81,7 +81,7 @@ func (w *Wrapper) Close(ctx context.Context) error {
// DataCollection.
func getStreamItemFunc(
collection connector.DataCollection,
details *backup.Details,
details *details.Details,
) func(context.Context, func(context.Context, fs.Entry) error) error {
return func(ctx context.Context, cb func(context.Context, fs.Entry) error) error {
items := collection.Items()
@ -114,7 +114,7 @@ func getStreamItemFunc(
// buildKopiaDirs recursively builds a directory hierarchy from the roots up.
// Returned directories are either virtualfs.StreamingDirectory or
// virtualfs.staticDirectory.
func buildKopiaDirs(dirName string, dir *treeMap, details *backup.Details) (fs.Directory, error) {
func buildKopiaDirs(dirName string, dir *treeMap, details *details.Details) (fs.Directory, error) {
// Don't support directories that have both a DataCollection and a set of
// static child directories.
if dir.collection != nil && len(dir.childDirs) > 0 {
@ -156,7 +156,7 @@ func newTreeMap() *treeMap {
// ancestor of the streams and uses virtualfs.StaticDirectory for internal nodes
// in the hierarchy. Leaf nodes are virtualfs.StreamingDirectory with the given
// DataCollections.
func inflateDirTree(ctx context.Context, collections []connector.DataCollection, details *backup.Details) (fs.Directory, error) {
func inflateDirTree(ctx context.Context, collections []connector.DataCollection, details *details.Details) (fs.Directory, error) {
roots := make(map[string]*treeMap)
for _, s := range collections {
@ -229,12 +229,12 @@ func inflateDirTree(ctx context.Context, collections []connector.DataCollection,
func (w Wrapper) BackupCollections(
ctx context.Context,
collections []connector.DataCollection,
) (*BackupStats, *backup.Details, error) {
) (*BackupStats, *details.Details, error) {
if w.c == nil {
return nil, nil, errNotConnected
}
details := &backup.Details{}
details := &details.Details{}
dirTree, err := inflateDirTree(ctx, collections, details)
if err != nil {
@ -252,7 +252,7 @@ func (w Wrapper) BackupCollections(
func (w Wrapper) makeSnapshotWithRoot(
ctx context.Context,
root fs.Directory,
details *backup.Details,
details *details.Details,
) (*BackupStats, error) {
si := snapshot.SourceInfo{
Host: corsoHost,

8
src/internal/kopia/wrapper_test.go
View File

@ -20,7 +20,7 @@ import (
"github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/kopia/mockkopia"
ctesting "github.com/alcionai/corso/internal/testing"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
const (
@ -117,7 +117,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree() {
user2: 42,
}
details := &backup.Details{}
details := &details.Details{}
collections := []connector.DataCollection{
mockconnector.NewMockExchangeDataCollection(
@ -180,7 +180,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_NoAncestorDirs() {
expectedFileCount := 42
details := &backup.Details{}
details := &details.Details{}
collections := []connector.DataCollection{
mockconnector.NewMockExchangeDataCollection(
[]string{emails},
@ -259,7 +259,7 @@ func (suite *KopiaUnitSuite) TestBuildDirectoryTree_Fails() {
ctx := context.Background()
suite.T().Run(test.name, func(t *testing.T) {
details := &backup.Details{}
details := &details.Details{}
_, err := inflateDirTree(ctx, test.layout, details)
assert.Error(t, err)
})

76
src/internal/operations/backup.go
View File

@ -10,8 +10,10 @@ import (
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/store"
)
@ -29,8 +31,8 @@ type BackupOperation struct {
// BackupResults aggregate the details of the result of the operation.
type BackupResults struct {
summary
metrics
stats.ReadWrites
stats.StartAndEndTime
BackupID model.ID `json:"backupID"`
}
@ -71,12 +73,23 @@ type backupStats struct {
}
// Run begins a synchronous backup operation.
func (op *BackupOperation) Run(ctx context.Context) error {
func (op *BackupOperation) Run(ctx context.Context) (err error) {
// TODO: persist initial state of backupOperation in modelstore
// persist operation results to the model store on exit
stats := backupStats{}
defer op.persistResults(time.Now(), &stats)
var (
stats backupStats
details *details.Details
)
defer func() {
op.persistResults(time.Now(), &stats)
err = op.createBackupModels(ctx, stats.k.SnapshotID, details)
if err != nil {
stats.writeErr = err
// todo: ^ we're not persisting this yet, except for the error shown to the user.
}
}()
// retrieve data from the producer
gc, err := connector.NewGraphConnector(op.account)
@ -93,7 +106,6 @@ func (op *BackupOperation) Run(ctx context.Context) error {
}
// hand the results to the consumer
var details *backup.Details
stats.k, details, err = op.kopia.BackupCollections(ctx, cs)
if err != nil {
stats.writeErr = err
@ -101,33 +113,11 @@ func (op *BackupOperation) Run(ctx context.Context) error {
}
stats.gc = gc.AwaitStatus()
err = op.createBackupModels(ctx, stats.k.SnapshotID, details)
if err != nil {
stats.writeErr = err
return err
}
return nil
}
func (op *BackupOperation) createBackupModels(ctx context.Context, snapID string, details *backup.Details) error {
err := op.store.Put(ctx, model.BackupDetailsSchema, &details.DetailsModel)
if err != nil {
return errors.Wrap(err, "creating backupdetails model")
}
bu := backup.New(snapID, string(details.ModelStoreID))
err = op.store.Put(ctx, model.BackupSchema, bu)
if err != nil {
return errors.Wrap(err, "creating backup model")
}
op.Results.BackupID = bu.StableID
return nil
}
// writes the backupOperation outcome to the modelStore.
// writes the results metrics to the operation results.
// later stored in the manifest using createBackupModels.
func (op *BackupOperation) persistResults(
started time.Time,
stats *backupStats,
@ -141,7 +131,7 @@ func (op *BackupOperation) persistResults(
op.Results.WriteErrors = stats.writeErr
if stats.gc != nil {
op.Results.ItemsRead = stats.gc.ObjectCount
op.Results.ItemsRead = stats.gc.Successful
}
if stats.k != nil {
op.Results.ItemsWritten = stats.k.TotalFileCount
@ -149,6 +139,26 @@ func (op *BackupOperation) persistResults(
op.Results.StartedAt = started
op.Results.CompletedAt = time.Now()
// TODO: persist operation to modelstore
}
// stores the operation details, results, and selectors in the backup manifest.
func (op *BackupOperation) createBackupModels(ctx context.Context, snapID string, details *details.Details) error {
err := op.store.Put(ctx, model.BackupDetailsSchema, &details.DetailsModel)
if err != nil {
return errors.Wrap(err, "creating backupdetails model")
}
b := backup.New(
snapID, string(details.ModelStoreID), op.Status.String(),
op.Selectors,
op.Results.ReadWrites,
op.Results.StartAndEndTime,
)
err = op.store.Put(ctx, model.BackupSchema, b)
if err != nil {
return errors.Wrap(err, "creating backup model")
}
op.Results.BackupID = b.StableID
return nil
}

4
src/internal/operations/backup_test.go
View File

@ -48,7 +48,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
TotalFileCount: 1,
},
gc: &support.ConnectorOperationStatus{
ObjectCount: 1,
Successful: 1,
},
}
)
@ -59,7 +59,7 @@ func (suite *BackupOpSuite) TestBackupOperation_PersistResults() {
op.persistResults(now, &stats)
assert.Equal(t, op.Status, Failed)
assert.Equal(t, op.Results.ItemsRead, stats.gc.ObjectCount)
assert.Equal(t, op.Results.ItemsRead, stats.gc.Successful)
assert.Equal(t, op.Results.ReadErrors, stats.readErr)
assert.Equal(t, op.Results.ItemsWritten, stats.k.TotalFileCount)
assert.Equal(t, op.Results.WriteErrors, stats.writeErr)

28
src/internal/operations/operation.go
View File

@ -12,11 +12,12 @@ import (
type opStatus int
//go:generate stringer -type=opStatus -linecomment
const (
Unknown opStatus = iota
InProgress
Successful
Failed
Unknown opStatus = iota // Status Unknown
InProgress // In Progress
Successful // Successful
Failed // Failed
)
// --------------------------------------------------------------------------------
@ -72,22 +73,3 @@ func (op operation) validate() error {
}
return nil
}
// --------------------------------------------------------------------------------
// Results
// --------------------------------------------------------------------------------
// Summary tracks the total files touched and errors produced
// during an operation.
type summary struct {
ItemsRead int `json:"itemsRead,omitempty"`
ItemsWritten int `json:"itemsWritten,omitempty"`
ReadErrors error `json:"readErrors,omitempty"`
WriteErrors error `json:"writeErrors,omitempty"`
}
// Metrics tracks performance details such as timing, throughput, etc.
type metrics struct {
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
}

26
src/internal/operations/opstatus_string.go Normal file
View File

@ -0,0 +1,26 @@
// Code generated by "stringer -type=opStatus -linecomment"; DO NOT EDIT.
package operations
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[Unknown-0]
_ = x[InProgress-1]
_ = x[Successful-2]
_ = x[Failed-3]
}
const _opStatus_name = "Status UnknownIn ProgressSuccessfulFailed"
var _opStatus_index = [...]uint8{0, 14, 25, 35, 41}
func (i opStatus) String() string {
if i < 0 || i >= opStatus(len(_opStatus_index)-1) {
return "opStatus(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _opStatus_name[_opStatus_index[i]:_opStatus_index[i+1]]
}

5
src/internal/operations/restore.go
View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/kopia"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/store"
@ -30,8 +31,8 @@ type RestoreOperation struct {
// RestoreResults aggregate the details of the results of the operation.
type RestoreResults struct {
summary
metrics
stats.ReadWrites
stats.StartAndEndTime
}
// NewRestoreOperation constructs and validates a restore operation.

20
src/internal/stats/stats.go Normal file
View File

@ -0,0 +1,20 @@
package stats
import "time"
// ReadWrites tracks the total count of reads and writes, and of
// read and write errors. ItemsRead and ItemsWritten counts are
// assumed to be successful, so the total count of items involved
// would be ItemsRead+ReadErrors.
type ReadWrites struct {
ItemsRead int `json:"itemsRead,omitempty"`
ItemsWritten int `json:"itemsWritten,omitempty"`
ReadErrors error `json:"readErrors,omitempty"`
WriteErrors error `json:"writeErrors,omitempty"`
}
// StartAndEndTime tracks a paired starting time and ending time.
type StartAndEndTime struct {
StartedAt time.Time `json:"startedAt"`
CompletedAt time.Time `json:"completedAt"`
}

156
src/pkg/backup/backup.go
View File

@ -1,10 +1,14 @@
package backup
import (
"sync"
"strconv"
"time"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/selectors"
)
// Backup represents the result of a backup operation
@ -17,10 +21,17 @@ type Backup struct {
// Reference to `Details`
// We store the ModelStoreID since Details is immutable
DetailsID string `json:"detailsId"`
DetailsID string `json:"detailsID"`
// TODO:
// - Backup "Specification"
// Status of the operation
Status string `json:"status"`
// Selectors used in this operation
Selectors selectors.Selector `json:"selectors"`
// stats are embedded so that the values appear as top-level properties
stats.ReadWrites
stats.StartAndEndTime
}
// Headers returns the human-readable names of properties in a Backup
@ -31,6 +42,14 @@ func (b Backup) Headers() []string {
"Stable ID",
"Snapshot ID",
"Details ID",
"Status",
"Selectors",
"Items Read",
"Items Written",
"Read Errors",
"Write Errors",
"Started At",
"Completed At",
}
}
@ -38,125 +57,34 @@ func (b Backup) Headers() []string {
// out to a terminal in a columnar display.
func (b Backup) Values() []string {
return []string{
b.CreationTime.Format(time.RFC3339Nano),
common.FormatTime(b.CreationTime),
string(b.StableID),
b.SnapshotID,
b.DetailsID,
b.Status,
b.Selectors.String(),
strconv.Itoa(b.ReadWrites.ItemsRead),
strconv.Itoa(b.ReadWrites.ItemsWritten),
strconv.Itoa(support.GetNumberOfErrors(b.ReadWrites.ReadErrors)),
strconv.Itoa(support.GetNumberOfErrors(b.ReadWrites.WriteErrors)),
common.FormatTime(b.StartAndEndTime.StartedAt),
common.FormatTime(b.StartAndEndTime.CompletedAt),
}
}
func New(snapshotID, detailsID string) *Backup {
func New(
snapshotID, detailsID, status string,
selector selectors.Selector,
rw stats.ReadWrites,
se stats.StartAndEndTime,
) *Backup {
return &Backup{
CreationTime: time.Now(),
SnapshotID: snapshotID,
DetailsID: detailsID,
Status: status,
Selectors: selector,
ReadWrites: rw,
StartAndEndTime: se,
}
}
// DetailsModel describes what was stored in a Backup
type DetailsModel struct {
model.BaseModel
Entries []DetailsEntry `json:"entries"`
}
// Details augments the core with a mutex for processing.
// Should be sliced back to d.DetailsModel for storage and
// printing.
type Details struct {
DetailsModel
// internal
mu sync.Mutex `json:"-"`
}
// DetailsEntry describes a single item stored in a Backup
type DetailsEntry struct {
// TODO: `RepoRef` is currently the full path to the item in Kopia
// This can be optimized.
RepoRef string `json:"repoRef"`
ItemInfo
}
// Paths returns the list of Paths extracted from the Entries slice.
func (dm DetailsModel) Paths() []string {
ents := dm.Entries
r := make([]string, len(ents))
for i := range ents {
r[i] = ents[i].RepoRef
}
return r
}
// Headers returns the human-readable names of properties in a DetailsEntry
// for printing out to a terminal in a columnar display.
func (de DetailsEntry) Headers() []string {
hs := []string{"Repo Ref"}
if de.ItemInfo.Exchange != nil {
hs = append(hs, de.ItemInfo.Exchange.Headers()...)
}
if de.ItemInfo.Sharepoint != nil {
hs = append(hs, de.ItemInfo.Sharepoint.Headers()...)
}
return hs
}
// Values returns the values matching the Headers list.
func (de DetailsEntry) Values() []string {
vs := []string{de.RepoRef}
if de.ItemInfo.Exchange != nil {
vs = append(vs, de.ItemInfo.Exchange.Values()...)
}
if de.ItemInfo.Sharepoint != nil {
vs = append(vs, de.ItemInfo.Sharepoint.Values()...)
}
return vs
}
// ItemInfo is a oneOf that contains service specific
// information about the item it tracks
type ItemInfo struct {
Exchange *ExchangeInfo `json:"exchange,omitempty"`
Sharepoint *SharepointInfo `json:"sharepoint,omitempty"`
}
// ExchangeInfo describes an exchange item
type ExchangeInfo struct {
Sender string `json:"sender"`
Subject string `json:"subject"`
Received time.Time `json:"received"`
}
// Headers returns the human-readable names of properties in an ExchangeInfo
// for printing out to a terminal in a columnar display.
func (e ExchangeInfo) Headers() []string {
return []string{"Sender", "Subject", "Received"}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (e ExchangeInfo) Values() []string {
return []string{e.Sender, e.Subject, e.Received.Format(time.RFC3339Nano)}
}
// SharepointInfo describes a sharepoint item
// TODO: Implement this. This is currently here
// just to illustrate usage
type SharepointInfo struct{}
func (d *Details) Add(repoRef string, info ItemInfo) {
d.mu.Lock()
defer d.mu.Unlock()
d.Entries = append(d.Entries, DetailsEntry{RepoRef: repoRef, ItemInfo: info})
}
// Headers returns the human-readable names of properties in a SharepointInfo
// for printing out to a terminal in a columnar display.
func (s SharepointInfo) Headers() []string {
return []string{}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (s SharepointInfo) Values() []string {
return []string{}
}

62
src/pkg/backup/backup_test.go
View File

@ -1,14 +1,19 @@
package backup_test
import (
"errors"
"testing"
"time"
"github.com/stretchr/testify/suite"
"github.com/zeebo/assert"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/internal/stats"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors"
)
type BackupSuite struct {
@ -30,6 +35,18 @@ func (suite *BackupSuite) TestBackup_HeadersValues() {
CreationTime: now,
SnapshotID: "snapshot",
DetailsID: "details",
Status: "status",
Selectors: selectors.Selector{},
ReadWrites: stats.ReadWrites{
ItemsRead: 1,
ItemsWritten: 1,
ReadErrors: errors.New("1"),
WriteErrors: errors.New("1"),
},
StartAndEndTime: stats.StartAndEndTime{
StartedAt: now,
CompletedAt: now,
},
}
expectHs := []string{
@ -37,15 +54,32 @@ func (suite *BackupSuite) TestBackup_HeadersValues() {
"Stable ID",
"Snapshot ID",
"Details ID",
"Status",
"Selectors",
"Items Read",
"Items Written",
"Read Errors",
"Write Errors",
"Started At",
"Completed At",
}
hs := b.Headers()
assert.DeepEqual(t, expectHs, hs)
nowFmt := common.FormatTime(now)
expectVs := []string{
now.Format(time.RFC3339Nano),
nowFmt,
"stable",
"snapshot",
"details",
"status",
"{}",
"1",
"1",
"1",
"1",
nowFmt,
nowFmt,
}
vs := b.Values()
assert.DeepEqual(t, expectVs, vs)
@ -57,13 +91,13 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
table := []struct {
name string
entry backup.DetailsEntry
entry details.DetailsEntry
expectHs []string
expectVs []string
}{
{
name: "no info",
entry: backup.DetailsEntry{
entry: details.DetailsEntry{
RepoRef: "reporef",
},
expectHs: []string{"Repo Ref"},
@ -71,10 +105,10 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
},
{
name: "exhange info",
entry: backup.DetailsEntry{
entry: details.DetailsEntry{
RepoRef: "reporef",
ItemInfo: backup.ItemInfo{
Exchange: &backup.ExchangeInfo{
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
Sender: "sender",
Subject: "subject",
Received: now,
@ -86,10 +120,10 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
},
{
name: "sharepoint info",
entry: backup.DetailsEntry{
entry: details.DetailsEntry{
RepoRef: "reporef",
ItemInfo: backup.ItemInfo{
Sharepoint: &backup.SharepointInfo{},
ItemInfo: details.ItemInfo{
Sharepoint: &details.SharepointInfo{},
},
},
expectHs: []string{"Repo Ref"},
@ -110,7 +144,7 @@ func (suite *BackupSuite) TestDetailsEntry_HeadersValues() {
func (suite *BackupSuite) TestDetailsModel_Path() {
table := []struct {
name string
ents []backup.DetailsEntry
ents []details.DetailsEntry
expect []string
}{
{
@ -120,14 +154,14 @@ func (suite *BackupSuite) TestDetailsModel_Path() {
},
{
name: "single entry",
ents: []backup.DetailsEntry{
ents: []details.DetailsEntry{
{RepoRef: "abcde"},
},
expect: []string{"abcde"},
},
{
name: "multiple entries",
ents: []backup.DetailsEntry{
ents: []details.DetailsEntry{
{RepoRef: "abcde"},
{RepoRef: "12345"},
},
@ -136,8 +170,8 @@ func (suite *BackupSuite) TestDetailsModel_Path() {
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
d := backup.Details{
DetailsModel: backup.DetailsModel{
d := details.Details{
DetailsModel: details.DetailsModel{
Entries: test.ents,
},
}

116
src/pkg/backup/details/details.go Normal file
View File

@ -0,0 +1,116 @@
package details
import (
"sync"
"time"
"github.com/alcionai/corso/internal/model"
)
// DetailsModel describes what was stored in a Backup
type DetailsModel struct {
model.BaseModel
Entries []DetailsEntry `json:"entries"`
}
// Details augments the core with a mutex for processing.
// Should be sliced back to d.DetailsModel for storage and
// printing.
type Details struct {
DetailsModel
// internal
mu sync.Mutex `json:"-"`
}
// DetailsEntry describes a single item stored in a Backup
type DetailsEntry struct {
// TODO: `RepoRef` is currently the full path to the item in Kopia
// This can be optimized.
RepoRef string `json:"repoRef"`
ItemInfo
}
// Paths returns the list of Paths extracted from the Entries slice.
func (dm DetailsModel) Paths() []string {
ents := dm.Entries
r := make([]string, len(ents))
for i := range ents {
r[i] = ents[i].RepoRef
}
return r
}
// Headers returns the human-readable names of properties in a DetailsEntry
// for printing out to a terminal in a columnar display.
func (de DetailsEntry) Headers() []string {
hs := []string{"Repo Ref"}
if de.ItemInfo.Exchange != nil {
hs = append(hs, de.ItemInfo.Exchange.Headers()...)
}
if de.ItemInfo.Sharepoint != nil {
hs = append(hs, de.ItemInfo.Sharepoint.Headers()...)
}
return hs
}
// Values returns the values matching the Headers list.
func (de DetailsEntry) Values() []string {
vs := []string{de.RepoRef}
if de.ItemInfo.Exchange != nil {
vs = append(vs, de.ItemInfo.Exchange.Values()...)
}
if de.ItemInfo.Sharepoint != nil {
vs = append(vs, de.ItemInfo.Sharepoint.Values()...)
}
return vs
}
// ItemInfo is a oneOf that contains service specific
// information about the item it tracks
type ItemInfo struct {
Exchange *ExchangeInfo `json:"exchange,omitempty"`
Sharepoint *SharepointInfo `json:"sharepoint,omitempty"`
}
// ExchangeInfo describes an exchange item
type ExchangeInfo struct {
Sender string `json:"sender"`
Subject string `json:"subject"`
Received time.Time `json:"received"`
}
// Headers returns the human-readable names of properties in an ExchangeInfo
// for printing out to a terminal in a columnar display.
func (e ExchangeInfo) Headers() []string {
return []string{"Sender", "Subject", "Received"}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (e ExchangeInfo) Values() []string {
return []string{e.Sender, e.Subject, e.Received.Format(time.RFC3339Nano)}
}
// SharepointInfo describes a sharepoint item
// TODO: Implement this. This is currently here
// just to illustrate usage
type SharepointInfo struct{}
func (d *Details) Add(repoRef string, info ItemInfo) {
d.mu.Lock()
defer d.mu.Unlock()
d.Entries = append(d.Entries, DetailsEntry{RepoRef: repoRef, ItemInfo: info})
}
// Headers returns the human-readable names of properties in a SharepointInfo
// for printing out to a terminal in a columnar display.
func (s SharepointInfo) Headers() []string {
return []string{}
}
// Values returns the values matching the Headers list for printing
// out to a terminal in a columnar display.
func (s SharepointInfo) Values() []string {
return []string{}
}

116
src/pkg/backup/details/details_test.go Normal file
View File

@ -0,0 +1,116 @@
package details_test
import (
"context"
"testing"
"time"
"github.com/google/uuid"
"github.com/kopia/kopia/repo/manifest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/store"
storeMock "github.com/alcionai/corso/pkg/store/mock"
)
// ------------------------------------------------------------
// unit tests
// ------------------------------------------------------------
var (
detailsID = uuid.NewString()
bu = backup.Backup{
BaseModel: model.BaseModel{
StableID: model.ID(uuid.NewString()),
ModelStoreID: manifest.ID(uuid.NewString()),
},
CreationTime: time.Now(),
SnapshotID: uuid.NewString(),
DetailsID: detailsID,
}
deets = details.Details{
DetailsModel: details.DetailsModel{
BaseModel: model.BaseModel{
StableID: model.ID(detailsID),
ModelStoreID: manifest.ID(uuid.NewString()),
},
},
}
)
type StoreDetailsUnitSuite struct {
suite.Suite
}
func TestStoreDetailsUnitSuite(t *testing.T) {
suite.Run(t, new(StoreDetailsUnitSuite))
}
func (suite *StoreDetailsUnitSuite) TestGetDetails() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details",
mock: storeMock.NewMock(nil, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(nil, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{Storer: test.mock}
result, err := sm.GetDetails(ctx, manifest.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, result.StableID)
})
}
}
func (suite *StoreDetailsUnitSuite) TestGetDetailsFromBackupID() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details from backup id",
mock: storeMock.NewMock(&bu, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(&bu, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{Storer: test.mock}
dResult, bResult, err := store.GetDetailsFromBackupID(ctx, model.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, dResult.StableID)
assert.Equal(t, bu.StableID, bResult.StableID)
})
}
}

3
src/pkg/repository/repository.go
View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/internal/operations"
"github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/selectors"
"github.com/alcionai/corso/pkg/storage"
"github.com/alcionai/corso/pkg/store"
@ -162,7 +163,7 @@ func (r Repository) Backups(ctx context.Context) ([]backup.Backup, error) {
}
// BackupDetails returns the specified backup details object
func (r Repository) BackupDetails(ctx context.Context, backupID string) (*backup.Details, *backup.Backup, error) {
func (r Repository) BackupDetails(ctx context.Context, backupID string) (*details.Details, *backup.Backup, error) {
sw := store.NewKopiaStore(r.modelStore)
return sw.GetDetailsFromBackupID(ctx, model.ID(backupID))
}

12
src/pkg/selectors/exchange.go
View File

@ -4,7 +4,7 @@ import (
"strings"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
// ---------------------------------------------------------------------------
@ -500,13 +500,13 @@ var categoryPathSet = map[exchangeCategory][]exchangeCategory{
}
// matches returns true if either the path or the info matches the scope details.
func (s ExchangeScope) matches(cat exchangeCategory, path []string, info *backup.ExchangeInfo) bool {
func (s ExchangeScope) matches(cat exchangeCategory, path []string, info *details.ExchangeInfo) bool {
return s.matchesPath(cat, path) || s.matchesInfo(cat, info)
}
// matchesInfo handles the standard behavior when comparing a scope and an exchangeInfo
// returns true if the scope and info match for the provided category.
func (s ExchangeScope) matchesInfo(cat exchangeCategory, info *backup.ExchangeInfo) bool {
func (s ExchangeScope) matchesInfo(cat exchangeCategory, info *details.ExchangeInfo) bool {
// we need values to match against
if info == nil {
return false
@ -634,7 +634,7 @@ func exchangeIDPath(cat exchangeCategory, path []string) map[exchangeCategory]st
// Reduce reduces the entries in a backupDetails struct to only
// those that match the inclusions, filters, and exclusions in the selector.
func (s *ExchangeRestore) Reduce(deets *backup.Details) *backup.Details {
func (s *ExchangeRestore) Reduce(deets *details.Details) *details.Details {
if deets == nil {
return nil
}
@ -643,7 +643,7 @@ func (s *ExchangeRestore) Reduce(deets *backup.Details) *backup.Details {
entFilt := exchangeScopesByCategory(s.Filters)
entIncs := exchangeScopesByCategory(s.Includes)
ents := []backup.DetailsEntry{}
ents := []details.DetailsEntry{}
for _, ent := range deets.Entries {
// todo: use Path pkg for this
@ -706,7 +706,7 @@ func exchangeScopesByCategory(scopes []map[string]string) map[string][]ExchangeS
func matchExchangeEntry(
cat exchangeCategory,
path []string,
info *backup.ExchangeInfo,
info *details.ExchangeInfo,
excs, filts, incs []ExchangeScope,
) bool {
// a passing match requires either a filter or an inclusion

18
src/pkg/selectors/exchange_test.go
View File

@ -5,7 +5,7 @@ import (
"time"
"github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -498,7 +498,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScope_MatchesInfo() {
epoch = time.Time{}
now = time.Now()
then = now.Add(1 * time.Minute)
info = &backup.ExchangeInfo{
info = &details.ExchangeInfo{
Sender: sender,
Subject: subject,
Received: now,
@ -629,14 +629,14 @@ func (suite *ExchangeSourceSuite) TestIdPath() {
}
func (suite *ExchangeSourceSuite) TestExchangeRestore_Reduce() {
makeDeets := func(refs ...string) *backup.Details {
deets := &backup.Details{
DetailsModel: backup.DetailsModel{
Entries: []backup.DetailsEntry{},
makeDeets := func(refs ...string) *details.Details {
deets := &details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.DetailsEntry{},
},
}
for _, r := range refs {
deets.Entries = append(deets.Entries, backup.DetailsEntry{
deets.Entries = append(deets.Entries, details.DetailsEntry{
RepoRef: r,
})
}
@ -652,7 +652,7 @@ func (suite *ExchangeSourceSuite) TestExchangeRestore_Reduce() {
}
table := []struct {
name string
deets *backup.Details
deets *details.Details
makeSelector func() *ExchangeRestore
expect []string
}{
@ -825,7 +825,7 @@ func (suite *ExchangeSourceSuite) TestExchangeScopesByCategory() {
}
func (suite *ExchangeSourceSuite) TestMatchExchangeEntry() {
var exchangeInfo *backup.ExchangeInfo
var exchangeInfo *details.ExchangeInfo
const (
mid = "mailID"
cat = ExchangeMail

9
src/pkg/selectors/selectors.go
View File

@ -1,6 +1,7 @@
package selectors
import (
"encoding/json"
"strings"
"github.com/pkg/errors"
@ -84,6 +85,14 @@ func None() []string {
return []string{NoneTgt}
}
func (s Selector) String() string {
bs, err := json.Marshal(s)
if err != nil {
return "error"
}
return string(bs)
}
type baseScope interface {
~map[string]string
}

7
src/pkg/store/backup.go
View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
// GetBackup gets a single backup by id.
@ -39,8 +40,8 @@ func (w Wrapper) GetBackups(ctx context.Context) ([]backup.Backup, error) {
}
// GetDetails gets the backup details by ID.
func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*backup.Details, error) {
d := backup.Details{}
func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*details.Details, error) {
d := details.Details{}
err := w.GetWithModelStoreID(ctx, model.BackupDetailsSchema, detailsID, &d)
if err != nil {
return nil, errors.Wrap(err, "getting details")
@ -49,7 +50,7 @@ func (w Wrapper) GetDetails(ctx context.Context, detailsID manifest.ID) (*backup
}
// GetDetailsFromBackupID retrieves the backup.Details within the specified backup.
func (w Wrapper) GetDetailsFromBackupID(ctx context.Context, backupID model.ID) (*backup.Details, *backup.Backup, error) {
func (w Wrapper) GetDetailsFromBackupID(ctx context.Context, backupID model.ID) (*details.Details, *backup.Backup, error) {
b, err := w.GetBackup(ctx, backupID)
if err != nil {
return nil, nil, err

79
src/pkg/store/backup_test.go
View File

@ -31,14 +31,6 @@ var (
SnapshotID: uuid.NewString(),
DetailsID: detailsID,
}
deets = backup.Details{
DetailsModel: backup.DetailsModel{
BaseModel: model.BaseModel{
StableID: model.ID(detailsID),
ModelStoreID: manifest.ID(uuid.NewString()),
},
},
}
)
type StoreBackupUnitSuite struct {
@ -70,8 +62,8 @@ func (suite *StoreBackupUnitSuite) TestGetBackup() {
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{test.mock}
result, err := store.GetBackup(ctx, model.ID(uuid.NewString()))
sm := &store.Wrapper{Storer: test.mock}
result, err := sm.GetBackup(ctx, model.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
@ -102,7 +94,7 @@ func (suite *StoreBackupUnitSuite) TestGetBackups() {
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{test.mock}
sm := &store.Wrapper{Storer: test.mock}
result, err := sm.GetBackups(ctx)
test.expect(t, err)
if err != nil {
@ -113,68 +105,3 @@ func (suite *StoreBackupUnitSuite) TestGetBackups() {
})
}
}
func (suite *StoreBackupUnitSuite) TestGetDetails() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details",
mock: storeMock.NewMock(nil, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(nil, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
sm := &store.Wrapper{test.mock}
result, err := sm.GetDetails(ctx, manifest.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, result.StableID)
})
}
}
func (suite *StoreBackupUnitSuite) TestGetDetailsFromBackupID() {
ctx := context.Background()
table := []struct {
name string
mock *storeMock.MockModelStore
expect assert.ErrorAssertionFunc
}{
{
name: "gets details from backup id",
mock: storeMock.NewMock(&bu, &deets, nil),
expect: assert.NoError,
},
{
name: "errors",
mock: storeMock.NewMock(&bu, &deets, assert.AnError),
expect: assert.Error,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
store := &store.Wrapper{test.mock}
dResult, bResult, err := store.GetDetailsFromBackupID(ctx, model.ID(uuid.NewString()))
test.expect(t, err)
if err != nil {
return
}
assert.Equal(t, deets.StableID, dResult.StableID)
assert.Equal(t, bu.StableID, bResult.StableID)
})
}
}

5
src/pkg/store/mock/store_mock.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/internal/model"
"github.com/alcionai/corso/pkg/backup"
"github.com/alcionai/corso/pkg/backup/details"
)
// ------------------------------------------------------------
@ -21,7 +22,7 @@ type MockModelStore struct {
err error
}
func NewMock(b *backup.Backup, d *backup.Details, err error) *MockModelStore {
func NewMock(b *backup.Backup, d *details.Details, err error) *MockModelStore {
return &MockModelStore{
backup: marshal(b),
details: marshal(d),
@ -89,7 +90,7 @@ func (mms *MockModelStore) GetIDsForType(
unmarshal(mms.backup, &b)
return []*model.BaseModel{&b.BaseModel}, nil
case model.BackupDetailsSchema:
d := backup.Details{}
d := details.Details{}
unmarshal(mms.backup, &d)
return []*model.BaseModel{&d.BaseModel}, nil
}