1867a13f4d
Needed to allow backup filtering. Would be better to have a function that returns the values based on a Reason or something.
492 lines
12 KiB
Go
492 lines
12 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/alcionai/clues"
|
|
"github.com/dustin/go-humanize"
|
|
"golang.org/x/exp/maps"
|
|
|
|
"github.com/alcionai/corso/src/cli/print"
|
|
"github.com/alcionai/corso/src/internal/common/dttm"
|
|
"github.com/alcionai/corso/src/internal/common/errs"
|
|
"github.com/alcionai/corso/src/internal/common/str"
|
|
"github.com/alcionai/corso/src/internal/model"
|
|
"github.com/alcionai/corso/src/internal/stats"
|
|
"github.com/alcionai/corso/src/pkg/backup/identity"
|
|
"github.com/alcionai/corso/src/pkg/fault"
|
|
"github.com/alcionai/corso/src/pkg/selectors"
|
|
)
|
|
|
|
// Backup represents the result of a backup operation
|
|
type Backup struct {
|
|
model.BaseModel
|
|
CreationTime time.Time `json:"creationTime"`
|
|
|
|
// SnapshotID is the kopia snapshot ID
|
|
SnapshotID string `json:"snapshotID"`
|
|
|
|
// Reference to the details and fault errors storage location.
|
|
// Used to read backup.Details and fault.Errors from the streamstore.
|
|
StreamStoreID string `json:"streamStoreID"`
|
|
|
|
// Status of the operation, eg: completed, failed, etc
|
|
Status string `json:"status"`
|
|
|
|
// Selector used in this operation
|
|
Selector selectors.Selector `json:"selectors"`
|
|
|
|
// ** DO NOT CHANGE JSON TAG NAMES **
|
|
// These are in-memory only variable renames of previously persisted fields.
|
|
// ** CHANGING THE JSON TAGS WILL BREAK THINGS BECAUSE THE MODEL WON'T **
|
|
// ** DESERIALIZE PROPERLY **
|
|
ProtectedResourceID string `json:"resourceOwnerID,omitempty"`
|
|
ProtectedResourceName string `json:"resourceOwnerName,omitempty"`
|
|
|
|
// Version represents the version of the backup format
|
|
Version int `json:"version"`
|
|
|
|
FailFast bool `json:"failFast"`
|
|
|
|
// the quantity of errors, both hard failure and recoverable.
|
|
ErrorCount int `json:"errorCount"`
|
|
|
|
// the non-recoverable failure message, only populated if one occurred.
|
|
Failure string `json:"failure"`
|
|
|
|
// stats are embedded so that the values appear as top-level properties
|
|
stats.ReadWrites
|
|
stats.StartAndEndTime
|
|
stats.SkippedCounts
|
|
|
|
// MergeBases records the set of merge bases used for this backup and the
|
|
// Reason(s) each merge base was selected. Reasons are serialized the same
|
|
// way that Reason tags are serialized.
|
|
MergeBases map[model.StableID][]string `json:"mergeBases,omitempty"`
|
|
// AssistBases records the set of assist bases used for this backup and the
|
|
// Reason(s) each assist base was selected. Reasons are serialized the same
|
|
// way that Reason tags are serialized.
|
|
AssistBases map[model.StableID][]string `json:"assistBases,omitempty"`
|
|
|
|
// **Deprecated**
|
|
// Reference to the backup details storage location.
|
|
// Used to read backup.Details from the streamstore.
|
|
DetailsID string `json:"detailsID"`
|
|
}
|
|
|
|
// interface compliance checks
|
|
var _ print.Printable = &Backup{}
|
|
|
|
func New(
|
|
snapshotID, streamStoreID, status string,
|
|
version int,
|
|
id model.StableID,
|
|
selector selectors.Selector,
|
|
ownerID, ownerName string,
|
|
rw stats.ReadWrites,
|
|
se stats.StartAndEndTime,
|
|
reasons []identity.Reasoner,
|
|
bases BackupBases,
|
|
fe *fault.Errors,
|
|
tags map[string]string,
|
|
) *Backup {
|
|
if fe == nil {
|
|
fe = &fault.Errors{}
|
|
}
|
|
|
|
var (
|
|
errCount = len(fe.Items)
|
|
skipCount = len(fe.Skipped)
|
|
failMsg string
|
|
|
|
malware, invalidONFile, otherSkips int
|
|
)
|
|
|
|
if fe.Failure != nil {
|
|
failMsg = fe.Failure.Msg
|
|
errCount++
|
|
}
|
|
|
|
for _, s := range fe.Skipped {
|
|
switch true {
|
|
case s.HasCause(fault.SkipMalware):
|
|
malware++
|
|
case s.HasCause(fault.SkipBigOneNote):
|
|
invalidONFile++
|
|
default:
|
|
otherSkips++
|
|
}
|
|
}
|
|
|
|
// maps.Clone throws an NPE if passed nil on Mac for some reason.
|
|
if tags == nil {
|
|
tags = map[string]string{}
|
|
}
|
|
|
|
b := &Backup{
|
|
BaseModel: model.BaseModel{
|
|
ID: id,
|
|
Tags: maps.Clone(tags),
|
|
},
|
|
|
|
ProtectedResourceID: ownerID,
|
|
ProtectedResourceName: ownerName,
|
|
|
|
Version: version,
|
|
SnapshotID: snapshotID,
|
|
StreamStoreID: streamStoreID,
|
|
|
|
CreationTime: time.Now(),
|
|
Status: status,
|
|
|
|
Selector: selector,
|
|
FailFast: fe.FailFast,
|
|
|
|
ErrorCount: errCount,
|
|
Failure: failMsg,
|
|
|
|
ReadWrites: rw,
|
|
StartAndEndTime: se,
|
|
SkippedCounts: stats.SkippedCounts{
|
|
TotalSkippedItems: skipCount,
|
|
SkippedMalware: malware,
|
|
SkippedInvalidOneNoteFile: invalidONFile,
|
|
},
|
|
}
|
|
|
|
if bases != nil {
|
|
mergeBases := map[model.StableID][]string{}
|
|
assistBases := map[model.StableID][]string{}
|
|
|
|
for _, backup := range bases.Backups() {
|
|
for _, reason := range backup.Reasons {
|
|
mergeBases[backup.ID] = append(
|
|
mergeBases[backup.ID],
|
|
ServiceCatString(reason.Service(), reason.Category()))
|
|
}
|
|
}
|
|
|
|
for _, backup := range bases.UniqueAssistBackups() {
|
|
for _, reason := range backup.Reasons {
|
|
assistBases[backup.ID] = append(
|
|
assistBases[backup.ID],
|
|
ServiceCatString(reason.Service(), reason.Category()))
|
|
}
|
|
}
|
|
|
|
if len(mergeBases) > 0 {
|
|
b.MergeBases = mergeBases
|
|
}
|
|
|
|
if len(assistBases) > 0 {
|
|
b.AssistBases = assistBases
|
|
}
|
|
}
|
|
|
|
for _, reason := range reasons {
|
|
for k, v := range reasonTags(reason) {
|
|
b.Tags[k] = v
|
|
}
|
|
}
|
|
|
|
return b
|
|
}
|
|
|
|
// PersistedBaseSet contains information extracted from the backup model
|
|
// relating to its lineage. It only contains the backup ID and Reasons each
|
|
// base was selected instead of the full set of information contained in other
|
|
// structs like BackupBases.
|
|
type PersistedBaseSet struct {
|
|
Merge map[model.StableID][]identity.Reasoner
|
|
Assist map[model.StableID][]identity.Reasoner
|
|
}
|
|
|
|
func (b Backup) Bases() (PersistedBaseSet, error) {
|
|
res := PersistedBaseSet{
|
|
Merge: map[model.StableID][]identity.Reasoner{},
|
|
Assist: map[model.StableID][]identity.Reasoner{},
|
|
}
|
|
|
|
for id, reasons := range b.MergeBases {
|
|
for _, reason := range reasons {
|
|
service, cat, err := serviceCatStringToTypes(reason)
|
|
if err != nil {
|
|
return res, clues.Wrap(err, "getting Reason info").With(
|
|
"base_type", "merge",
|
|
"base_backup_id", id,
|
|
"input_string", reason)
|
|
}
|
|
|
|
res.Merge[id] = append(res.Merge[id], identity.NewReason(
|
|
// Tenant ID not currently stored in backup model.
|
|
"",
|
|
str.First(
|
|
b.ProtectedResourceID,
|
|
b.Selector.DiscreteOwner),
|
|
service,
|
|
cat))
|
|
}
|
|
}
|
|
|
|
for id, reasons := range b.AssistBases {
|
|
for _, reason := range reasons {
|
|
service, cat, err := serviceCatStringToTypes(reason)
|
|
if err != nil {
|
|
return res, clues.Wrap(err, "getting Reason info").With(
|
|
"base_type", "assist",
|
|
"base_backup_id", id,
|
|
"input_string", reason)
|
|
}
|
|
|
|
res.Assist[id] = append(res.Assist[id], identity.NewReason(
|
|
// Tenant ID not currently stored in backup model.
|
|
"",
|
|
str.First(
|
|
b.ProtectedResourceID,
|
|
b.Selector.DiscreteOwner),
|
|
service,
|
|
cat))
|
|
}
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
func (b Backup) Tenant() (string, error) {
|
|
t := b.Tags[TenantIDKey]
|
|
if len(t) == 0 {
|
|
return "", clues.Wrap(errs.NotFound, "getting tenant")
|
|
}
|
|
|
|
return t, nil
|
|
}
|
|
|
|
// Reasons returns the set of services and categories this backup encompassed
|
|
// for the tenant and protected resource.
|
|
func (b Backup) Reasons() ([]identity.Reasoner, error) {
|
|
tenant, err := b.Tenant()
|
|
if err != nil {
|
|
return nil, clues.Stack(err)
|
|
}
|
|
|
|
var res []identity.Reasoner
|
|
|
|
for tag := range b.Tags {
|
|
service, cat, err := serviceCatStringToTypes(tag)
|
|
if err != nil {
|
|
// Assume it's just not one of the Reason tags.
|
|
if errors.Is(err, errMissingPrefix) {
|
|
continue
|
|
}
|
|
|
|
return nil, clues.Wrap(err, "parsing reasons")
|
|
}
|
|
|
|
res = append(
|
|
res,
|
|
identity.NewReason(
|
|
tenant,
|
|
str.First(b.ProtectedResourceID, b.Selector.DiscreteOwner),
|
|
service,
|
|
cat))
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// --------------------------------------------------------------------------------
|
|
// CLI Output
|
|
// --------------------------------------------------------------------------------
|
|
|
|
// ----- print backups
|
|
|
|
// Print writes the Backup to StdOut, in the format requested by the caller.
|
|
func (b Backup) Print(ctx context.Context) {
|
|
print.Item(ctx, b)
|
|
}
|
|
|
|
// PrintAll writes the slice of Backups to StdOut, in the format requested by the caller.
|
|
func PrintAll(ctx context.Context, bs []*Backup) {
|
|
if len(bs) == 0 {
|
|
print.Info(ctx, "No backups available")
|
|
return
|
|
}
|
|
|
|
ps := []print.Printable{}
|
|
for _, b := range bs {
|
|
ps = append(ps, print.Printable(b))
|
|
}
|
|
|
|
print.All(ctx, ps...)
|
|
}
|
|
|
|
type Printable struct {
|
|
ID model.StableID `json:"id"`
|
|
Status string `json:"status"`
|
|
Version string `json:"version"`
|
|
ProtectedResourceID string `json:"protectedResourceID,omitempty"`
|
|
ProtectedResourceName string `json:"protectedResourceName,omitempty"`
|
|
Owner string `json:"owner,omitempty"`
|
|
Stats backupStats `json:"stats"`
|
|
}
|
|
|
|
// ToPrintable reduces the Backup to its minimally printable details.
|
|
func (b Backup) ToPrintable() Printable {
|
|
return Printable{
|
|
ID: b.ID,
|
|
Status: b.Status,
|
|
Version: "0",
|
|
ProtectedResourceID: b.Selector.DiscreteOwner,
|
|
ProtectedResourceName: b.Selector.DiscreteOwnerName,
|
|
Owner: b.Selector.DiscreteOwner,
|
|
Stats: b.toStats(),
|
|
}
|
|
}
|
|
|
|
// MinimumPrintable reduces the Backup to its minimally printable details.
|
|
func (b Backup) MinimumPrintable() any {
|
|
return b.ToPrintable()
|
|
}
|
|
|
|
// Headers returns the human-readable names of properties in a Backup
|
|
// for printing out to a terminal in a columnar display.
|
|
func (b Backup) Headers() []string {
|
|
return []string{
|
|
"ID",
|
|
"Started At",
|
|
"Duration",
|
|
"Status",
|
|
"Resource Owner",
|
|
}
|
|
}
|
|
|
|
// Values returns the values matching the Headers list for printing
|
|
// out to a terminal in a columnar display.
|
|
func (b Backup) Values() []string {
|
|
var (
|
|
status = b.Status
|
|
errCount = b.ErrorCount
|
|
)
|
|
|
|
if errCount+b.TotalSkippedItems > 0 {
|
|
status += (" (")
|
|
}
|
|
|
|
if errCount > 0 {
|
|
status += fmt.Sprintf("%d errors", errCount)
|
|
}
|
|
|
|
if errCount > 0 && b.TotalSkippedItems > 0 {
|
|
status += ", "
|
|
}
|
|
|
|
if b.TotalSkippedItems > 0 {
|
|
status += fmt.Sprintf("%d skipped", b.TotalSkippedItems)
|
|
|
|
if b.SkippedMalware+b.SkippedInvalidOneNoteFile > 0 {
|
|
status += ": "
|
|
}
|
|
}
|
|
|
|
skipped := []string{}
|
|
|
|
if b.SkippedMalware > 0 {
|
|
skipped = append(skipped, fmt.Sprintf("%d malware", b.SkippedMalware))
|
|
}
|
|
|
|
if b.SkippedInvalidOneNoteFile > 0 {
|
|
skipped = append(skipped, fmt.Sprintf("%d invalid OneNote file", b.SkippedInvalidOneNoteFile))
|
|
}
|
|
|
|
status += strings.Join(skipped, ", ")
|
|
|
|
if errCount+b.TotalSkippedItems > 0 {
|
|
status += (")")
|
|
}
|
|
|
|
name := str.First(
|
|
b.ProtectedResourceName,
|
|
b.ProtectedResourceID,
|
|
b.Selector.Name())
|
|
|
|
bs := b.toStats()
|
|
|
|
return []string{
|
|
string(b.ID),
|
|
dttm.FormatToTabularDisplay(b.StartedAt),
|
|
bs.EndedAt.Sub(bs.StartedAt).String(),
|
|
status,
|
|
name,
|
|
}
|
|
}
|
|
|
|
// ----- print backup stats
|
|
|
|
func (b Backup) toStats() backupStats {
|
|
return backupStats{
|
|
ID: string(b.ID),
|
|
BytesRead: b.BytesRead,
|
|
BytesUploaded: b.NonMetaBytesUploaded,
|
|
EndedAt: b.CompletedAt,
|
|
ErrorCount: b.ErrorCount,
|
|
ItemsRead: b.ItemsRead,
|
|
ItemsSkipped: b.TotalSkippedItems,
|
|
ItemsWritten: b.NonMetaItemsWritten,
|
|
StartedAt: b.StartedAt,
|
|
}
|
|
}
|
|
|
|
// interface compliance checks
|
|
var _ print.Printable = &backupStats{}
|
|
|
|
type backupStats struct {
|
|
ID string `json:"id"`
|
|
BytesRead int64 `json:"bytesRead"`
|
|
BytesUploaded int64 `json:"bytesUploaded"`
|
|
EndedAt time.Time `json:"endedAt"`
|
|
ErrorCount int `json:"errorCount"`
|
|
ItemsRead int `json:"itemsRead"`
|
|
ItemsSkipped int `json:"itemsSkipped"`
|
|
ItemsWritten int `json:"itemsWritten"`
|
|
StartedAt time.Time `json:"startedAt"`
|
|
}
|
|
|
|
// Print writes the Backup to StdOut, in the format requested by the caller.
|
|
func (bs backupStats) Print(ctx context.Context) {
|
|
print.Item(ctx, bs)
|
|
}
|
|
|
|
// MinimumPrintable reduces the Backup to its minimally printable details.
|
|
func (bs backupStats) MinimumPrintable() any {
|
|
return bs
|
|
}
|
|
|
|
// Headers returns the human-readable names of properties in a Backup
|
|
// for printing out to a terminal in a columnar display.
|
|
func (bs backupStats) Headers() []string {
|
|
return []string{
|
|
"ID",
|
|
"Bytes Uploaded",
|
|
"Items Uploaded",
|
|
"Items Skipped",
|
|
"Errors",
|
|
}
|
|
}
|
|
|
|
// Values returns the values matching the Headers list for printing
|
|
// out to a terminal in a columnar display.
|
|
func (bs backupStats) Values() []string {
|
|
return []string{
|
|
bs.ID,
|
|
humanize.Bytes(uint64(bs.BytesUploaded)),
|
|
strconv.Itoa(bs.ItemsWritten),
|
|
strconv.Itoa(bs.ItemsSkipped),
|
|
strconv.Itoa(bs.ErrorCount),
|
|
}
|
|
}
|