Compare commits
30 Commits
main
...
lineage-co
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
addb979541 | ||
|
|
e513c0b144 | ||
|
|
d648f68f3a | ||
|
|
6be14da682 | ||
|
|
1867a13f4d | ||
|
|
dab4d7f0c7 | ||
|
|
2e69a97e86 | ||
|
|
5c6d46a581 | ||
|
|
9b8ee4f3f6 | ||
|
|
0ebd6c8eec | ||
|
|
0e5f5bde49 | ||
|
|
1ad135bd93 | ||
|
|
0ce35f4f62 | ||
|
|
d2253145bf | ||
|
|
bb5700917b | ||
|
|
8131b6d388 | ||
|
|
59b0fb3ebc | ||
|
|
08a7d435e3 | ||
|
|
2295c5cc20 | ||
|
|
d1fba10aeb | ||
|
|
e38ecc38fe | ||
|
|
405ce1548d | ||
|
|
72269441ea | ||
|
|
ffcc4e1bb8 | ||
|
|
ad5d77f6f8 | ||
|
|
44c2b7fb68 | ||
|
|
cd4905c0ad | ||
|
|
7f438533f8 | ||
|
|
65af82f0f5 | ||
|
|
216f29888f |
@ -12,8 +12,8 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
@ -174,7 +174,7 @@ func genericCreateCommand(
|
||||
) error {
|
||||
var (
|
||||
bIDs []string
|
||||
errs = []error{}
|
||||
bus = []error{}
|
||||
)
|
||||
|
||||
for _, discSel := range selectorSet {
|
||||
@ -187,7 +187,7 @@ func genericCreateCommand(
|
||||
|
||||
bo, err := r.NewBackupWithLookup(ictx, discSel, ins)
|
||||
if err != nil {
|
||||
errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
|
||||
bus = append(bus, clues.Wrap(err, owner).WithClues(ictx))
|
||||
Errf(ictx, "%v\n", err)
|
||||
|
||||
continue
|
||||
@ -208,7 +208,7 @@ func genericCreateCommand(
|
||||
continue
|
||||
}
|
||||
|
||||
errs = append(errs, clues.Wrap(err, owner).WithClues(ictx))
|
||||
bus = append(bus, clues.Wrap(err, owner).WithClues(ictx))
|
||||
Errf(ictx, "%v\n", err)
|
||||
|
||||
continue
|
||||
@ -235,10 +235,10 @@ func genericCreateCommand(
|
||||
|
||||
backup.PrintAll(ctx, bups)
|
||||
|
||||
if len(errs) > 0 {
|
||||
sb := fmt.Sprintf("%d of %d backups failed:\n", len(errs), len(selectorSet))
|
||||
if len(bus) > 0 {
|
||||
sb := fmt.Sprintf("%d of %d backups failed:\n", len(bus), len(selectorSet))
|
||||
|
||||
for i, e := range errs {
|
||||
for i, e := range bus {
|
||||
logger.CtxErr(ctx, e).Errorf("Backup %d of %d failed", i+1, len(selectorSet))
|
||||
sb += "∙ " + e.Error() + "\n"
|
||||
}
|
||||
@ -305,13 +305,13 @@ func genericListCommand(
|
||||
defer utils.CloseRepo(ctx, r)
|
||||
|
||||
if len(bID) > 0 {
|
||||
fe, b, errs := r.GetBackupErrors(ctx, bID)
|
||||
if errs.Failure() != nil {
|
||||
if errors.Is(errs.Failure(), data.ErrNotFound) {
|
||||
fe, b, bus := r.GetBackupErrors(ctx, bID)
|
||||
if bus.Failure() != nil {
|
||||
if errors.Is(bus.Failure(), errs.NotFound) {
|
||||
return Only(ctx, clues.New("No backup exists with the id "+bID))
|
||||
}
|
||||
|
||||
return Only(ctx, clues.Wrap(errs.Failure(), "Failed to list backup id "+bID))
|
||||
return Only(ctx, clues.Wrap(bus.Failure(), "Failed to list backup id "+bID))
|
||||
}
|
||||
|
||||
b.Print(ctx)
|
||||
@ -367,21 +367,21 @@ func genericDetailsCore(
|
||||
|
||||
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
|
||||
|
||||
d, _, errs := bg.GetBackupDetails(ctx, backupID)
|
||||
d, _, bus := bg.GetBackupDetails(ctx, backupID)
|
||||
// TODO: log/track recoverable errors
|
||||
if errs.Failure() != nil {
|
||||
if errors.Is(errs.Failure(), data.ErrNotFound) {
|
||||
if bus.Failure() != nil {
|
||||
if errors.Is(bus.Failure(), errs.NotFound) {
|
||||
return nil, clues.New("no backup exists with the id " + backupID)
|
||||
}
|
||||
|
||||
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
|
||||
return nil, clues.Wrap(bus.Failure(), "Failed to get backup details in the repository")
|
||||
}
|
||||
|
||||
if opts.SkipReduce {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
d, err := sel.Reduce(ctx, d, errs)
|
||||
d, err := sel.Reduce(ctx, d, bus)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "filtering backup details to selection")
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/export"
|
||||
@ -93,7 +93,7 @@ func runExport(
|
||||
|
||||
expColl, err := eo.Run(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
if errors.Is(err, errs.NotFound) {
|
||||
return Only(ctx, clues.New("Backup or backup details missing for id "+backupID))
|
||||
}
|
||||
|
||||
|
||||
@ -1,9 +1,14 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/goccy/go-graphviz"
|
||||
"github.com/goccy/go-graphviz/cgraph"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
@ -11,13 +16,16 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/pkg/control/repository"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
repo "github.com/alcionai/corso/src/pkg/repository"
|
||||
)
|
||||
|
||||
const (
|
||||
initCommand = "init"
|
||||
connectCommand = "connect"
|
||||
maintenanceCommand = "maintenance"
|
||||
lineageCommand = "lineage"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -39,12 +47,14 @@ func AddCommands(cmd *cobra.Command) {
|
||||
initCmd = initCmd()
|
||||
connectCmd = connectCmd()
|
||||
maintenanceCmd = maintenanceCmd()
|
||||
lineageCmd = lineageCmd()
|
||||
)
|
||||
|
||||
cmd.AddCommand(repoCmd)
|
||||
repoCmd.AddCommand(initCmd)
|
||||
repoCmd.AddCommand(connectCmd)
|
||||
repoCmd.AddCommand(maintenanceCmd)
|
||||
repoCmd.AddCommand(lineageCmd)
|
||||
|
||||
flags.AddMaintenanceModeFlag(maintenanceCmd)
|
||||
flags.AddForceMaintenanceFlag(maintenanceCmd)
|
||||
@ -175,3 +185,242 @@ func getMaintenanceType(t string) (repository.MaintenanceType, error) {
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func lineageCmd() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: lineageCommand,
|
||||
Short: "Run maintenance on an existing repository",
|
||||
Long: `Run maintenance on an existing repository to optimize performance and storage use`,
|
||||
RunE: handleLineageCmd,
|
||||
Args: cobra.NoArgs,
|
||||
}
|
||||
}
|
||||
|
||||
func handleLineageCmd(cmd *cobra.Command, args []string) error {
|
||||
ctx := cmd.Context()
|
||||
|
||||
r, acct, err := utils.GetAccountAndConnect(ctx, cmd, path.UnknownService)
|
||||
if err != nil {
|
||||
return print.Only(ctx, err)
|
||||
}
|
||||
|
||||
roots, err := r.BackupLineage(
|
||||
ctx,
|
||||
acct.Repo.Account.ID(),
|
||||
"c8006e42-5b84-4d37-a027-14f3d09fe6c7",
|
||||
path.UnknownService,
|
||||
path.UnknownCategory,
|
||||
)
|
||||
if err != nil {
|
||||
return print.Only(ctx, err)
|
||||
}
|
||||
|
||||
for _, root := range roots {
|
||||
fmt.Printf("Tree rooted with %v\n", root.Label)
|
||||
printTree(root, 0)
|
||||
}
|
||||
|
||||
if err := drawTree(ctx, roots); err != nil {
|
||||
return print.Only(ctx, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func printTree(root *repo.BackupNode, ident int) {
|
||||
if root == nil {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf(strings.Repeat("\t", ident)+"%+v\n", root)
|
||||
|
||||
for _, child := range root.Children {
|
||||
printTree(child.BackupNode, ident+1)
|
||||
}
|
||||
}
|
||||
|
||||
func drawTree(ctx context.Context, roots []*repo.BackupNode) error {
|
||||
const port = ":6060"
|
||||
|
||||
g := graphviz.New()
|
||||
|
||||
graph, err := g.Graph()
|
||||
if err != nil {
|
||||
return clues.Wrap(err, "getting graph")
|
||||
}
|
||||
|
||||
defer func() {
|
||||
graph.Close()
|
||||
g.Close()
|
||||
}()
|
||||
|
||||
graph.SetRankDir(cgraph.LRRank)
|
||||
|
||||
for _, root := range roots {
|
||||
if err := buildGraph(ctx, graph, root); err != nil {
|
||||
return clues.Wrap(err, "building graph")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("starting http server on port %s", port)
|
||||
|
||||
// Start an http server that has the redered image.
|
||||
http.HandleFunc(
|
||||
"/",
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/svg+xml")
|
||||
|
||||
if err := g.Render(graph, graphviz.SVG, w); err != nil {
|
||||
logger.CtxErr(ctx, err).Info("sending svg to server")
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
return clues.Stack(http.ListenAndServe(port, nil)).OrNil()
|
||||
}
|
||||
|
||||
func buildGraph(
|
||||
ctx context.Context,
|
||||
graph *cgraph.Graph,
|
||||
root *repo.BackupNode,
|
||||
) error {
|
||||
// Add all nodes to the map and track them by ID. The root is just mapped as
|
||||
// "root" and has the resource ID.
|
||||
allNodes := map[string]*cgraph.Node{}
|
||||
|
||||
if err := addNodes(graph, root, allNodes); err != nil {
|
||||
return clues.Stack(err)
|
||||
}
|
||||
|
||||
// To keep from adding edges multiple times, track which nodes we've already
|
||||
// processed. This is required because there can be multiple paths to a node.
|
||||
visitedNodes := map[string]struct{}{}
|
||||
|
||||
if err := addEdges(graph, root, allNodes, visitedNodes); err != nil {
|
||||
return clues.Stack(err)
|
||||
}
|
||||
|
||||
// Go through and add edges between all nodes. The edge info will be based
|
||||
// on the Reason contained in the edge struct.
|
||||
return nil
|
||||
}
|
||||
|
||||
func addNodes(
|
||||
graph *cgraph.Graph,
|
||||
node *repo.BackupNode,
|
||||
allNodes map[string]*cgraph.Node,
|
||||
) error {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, ok := allNodes[node.Label]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Need unique keys for nodes so use the backupID.
|
||||
n, err := graph.CreateNode(node.Label)
|
||||
if err != nil {
|
||||
return clues.Wrap(err, "creating node").With("backup_id", node.Label)
|
||||
}
|
||||
|
||||
// Set tooltip info to have Reasons for backup and backup type.
|
||||
var toolTip string
|
||||
|
||||
if node.Deleted {
|
||||
toolTip += "This backup was deleted, Reasons are a best guess!\n"
|
||||
}
|
||||
|
||||
toolTip += "BackupID: " + node.Label + "\n"
|
||||
|
||||
switch node.Type {
|
||||
case repo.MergeNode:
|
||||
toolTip += "Base Type: merge\n"
|
||||
|
||||
case repo.AssistNode:
|
||||
toolTip += "Base Type: assist\n"
|
||||
}
|
||||
|
||||
toolTip += fmt.Sprintf("Created At: %v\n", node.Created)
|
||||
|
||||
var reasonStrings []string
|
||||
|
||||
for _, reason := range node.Reasons {
|
||||
reasonStrings = append(
|
||||
reasonStrings,
|
||||
fmt.Sprintf("%s/%s", reason.Service(), reason.Category()),
|
||||
)
|
||||
}
|
||||
|
||||
n.
|
||||
SetLabel(strings.Join(reasonStrings, "\n")).
|
||||
SetTooltip(toolTip).
|
||||
SetStyle(cgraph.FilledNodeStyle).
|
||||
SetFillColor("white")
|
||||
|
||||
if node.Deleted {
|
||||
n.SetFillColor("indianred")
|
||||
}
|
||||
|
||||
if node.Type == repo.AssistNode {
|
||||
n.SetFillColor("grey")
|
||||
}
|
||||
|
||||
allNodes[node.Label] = n
|
||||
|
||||
for _, child := range node.Children {
|
||||
if err := addNodes(graph, child.BackupNode, allNodes); err != nil {
|
||||
return clues.Stack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func addEdges(
|
||||
graph *cgraph.Graph,
|
||||
node *repo.BackupNode,
|
||||
allNodes map[string]*cgraph.Node,
|
||||
visitedNodes map[string]struct{},
|
||||
) error {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, ok := visitedNodes[node.Label]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
visitedNodes[node.Label] = struct{}{}
|
||||
|
||||
n := allNodes[node.Label]
|
||||
|
||||
for _, child := range node.Children {
|
||||
var edgeReasons []string
|
||||
|
||||
for _, reason := range child.Reasons {
|
||||
edgeReasons = append(
|
||||
edgeReasons,
|
||||
fmt.Sprintf("%s/%s", reason.Service(), reason.Category()),
|
||||
)
|
||||
}
|
||||
|
||||
edgeLabel := strings.Join(edgeReasons, ",\n")
|
||||
|
||||
e, err := graph.CreateEdge(edgeLabel, n, allNodes[child.Label])
|
||||
if err != nil {
|
||||
return clues.Wrap(err, "adding edge").With(
|
||||
"parent", node.Label,
|
||||
"child", child.Label,
|
||||
)
|
||||
}
|
||||
|
||||
e.SetDir(cgraph.ForwardDir).SetLabel(edgeLabel).SetTooltip(" ")
|
||||
|
||||
if err := addEdges(graph, child.BackupNode, allNodes, visitedNodes); err != nil {
|
||||
return clues.Stack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
"github.com/alcionai/corso/src/cli/flags"
|
||||
. "github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/cli/utils"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -114,7 +114,7 @@ func runRestore(
|
||||
|
||||
ds, err := ro.Run(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
if errors.Is(err, errs.NotFound) {
|
||||
return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV))
|
||||
}
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ require (
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-xray-sdk-go v1.8.2
|
||||
github.com/cenkalti/backoff/v4 v4.2.1
|
||||
github.com/goccy/go-graphviz v0.1.1
|
||||
github.com/google/uuid v1.3.1
|
||||
github.com/h2non/gock v1.2.0
|
||||
github.com/kopia/kopia v0.13.0
|
||||
@ -44,9 +45,11 @@ require (
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/aws/aws-sdk-go v1.45.0 // indirect
|
||||
github.com/fogleman/gg v1.3.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/cronexpr v1.1.2 // indirect
|
||||
@ -62,6 +65,7 @@ require (
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.48.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.18.0 // indirect
|
||||
golang.org/x/image v0.6.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
|
||||
)
|
||||
|
||||
|
||||
19
src/go.sum
19
src/go.sum
@ -98,6 +98,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/corona10/goimagehash v1.0.2 h1:pUfB0LnsJASMPGEZLj7tGY251vF+qLGqOgEP4rUs6kA=
|
||||
github.com/corona10/goimagehash v1.0.2/go.mod h1:/l9umBhvcHQXVtQO1V6Gp1yD20STawkhRnnX0D1bvVI=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=
|
||||
github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec=
|
||||
@ -116,6 +118,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
||||
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
|
||||
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
|
||||
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
|
||||
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
@ -133,6 +137,8 @@ github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/goccy/go-graphviz v0.1.1 h1:MGrsnzBxTyt7KG8FhHsFPDTGvF7UaQMmSa6A610DqPg=
|
||||
github.com/goccy/go-graphviz v0.1.1/go.mod h1:lpnwvVDjskayq84ZxG8tGCPeZX/WxP88W+OJajh+gFk=
|
||||
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
|
||||
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
|
||||
@ -142,6 +148,8 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
@ -325,6 +333,8 @@ github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0
|
||||
github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
|
||||
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
|
||||
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5 h1:BvoENQQU+fZ9uukda/RzCAL/191HHwJA5b13R6diVlY=
|
||||
github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
|
||||
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
|
||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
|
||||
@ -486,6 +496,8 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjs
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.6.0 h1:bR8b5okrPI3g/gyZakLZHeWxAR8Dn5CyxXv1hLH5g/4=
|
||||
golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@ -508,6 +520,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -546,6 +559,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@ -568,6 +582,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -624,6 +639,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -633,6 +649,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@ -688,6 +706,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
5
src/internal/common/errs/errs.go
Normal file
5
src/internal/common/errs/errs.go
Normal file
@ -0,0 +1,5 @@
|
||||
package errs
|
||||
|
||||
import "github.com/alcionai/clues"
|
||||
|
||||
var NotFound = clues.New("not found")
|
||||
@ -3,15 +3,12 @@ package data
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
var ErrNotFound = clues.New("not found")
|
||||
|
||||
type CollectionState int
|
||||
|
||||
const (
|
||||
@ -27,13 +24,13 @@ type FetchRestoreCollection struct {
|
||||
}
|
||||
|
||||
// NoFetchRestoreCollection is a wrapper for a Collection that returns
|
||||
// ErrNotFound for all Fetch calls.
|
||||
// errs.NotFound for all Fetch calls.
|
||||
type NoFetchRestoreCollection struct {
|
||||
Collection
|
||||
}
|
||||
|
||||
func (c NoFetchRestoreCollection) FetchItemByName(context.Context, string) (Item, error) {
|
||||
return nil, ErrNotFound
|
||||
return nil, errs.NotFound
|
||||
}
|
||||
|
||||
// StateOf lets us figure out the state of the collection from the
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/spatialcurrent/go-lazy/pkg/lazy"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -136,14 +137,14 @@ func NewUnindexedLazyItem(
|
||||
itemGetter ItemDataGetter,
|
||||
itemID string,
|
||||
modTime time.Time,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) *unindexedLazyItem {
|
||||
return &unindexedLazyItem{
|
||||
ctx: ctx,
|
||||
id: itemID,
|
||||
itemGetter: itemGetter,
|
||||
modTime: modTime,
|
||||
errs: errs,
|
||||
bus: bus,
|
||||
}
|
||||
}
|
||||
|
||||
@ -157,7 +158,7 @@ type unindexedLazyItem struct {
|
||||
ctx context.Context
|
||||
mu sync.Mutex
|
||||
id string
|
||||
errs *fault.Bus
|
||||
bus *fault.Bus
|
||||
itemGetter ItemDataGetter
|
||||
|
||||
modTime time.Time
|
||||
@ -185,7 +186,7 @@ func (i *unindexedLazyItem) ToReader() io.ReadCloser {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
reader, info, delInFlight, err := i.itemGetter.GetData(i.ctx, i.errs)
|
||||
reader, info, delInFlight, err := i.itemGetter.GetData(i.ctx, i.bus)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
}
|
||||
@ -232,7 +233,7 @@ func NewLazyItem(
|
||||
itemGetter ItemDataGetter,
|
||||
itemID string,
|
||||
modTime time.Time,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) *lazyItem {
|
||||
return &lazyItem{
|
||||
unindexedLazyItem: NewUnindexedLazyItem(
|
||||
@ -240,7 +241,7 @@ func NewLazyItem(
|
||||
itemGetter,
|
||||
itemID,
|
||||
modTime,
|
||||
errs),
|
||||
bus),
|
||||
}
|
||||
}
|
||||
|
||||
@ -259,7 +260,7 @@ func (i *lazyItem) Info() (details.ItemInfo, error) {
|
||||
defer i.mu.Unlock()
|
||||
|
||||
if i.delInFlight {
|
||||
return details.ItemInfo{}, clues.Stack(ErrNotFound).WithClues(i.ctx)
|
||||
return details.ItemInfo{}, clues.Stack(errs.NotFound).WithClues(i.ctx)
|
||||
} else if i.info == nil {
|
||||
return details.ItemInfo{}, clues.New("requesting ItemInfo before data retrieval").
|
||||
WithClues(i.ctx)
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
@ -191,12 +192,12 @@ func (mid *mockItemDataGetter) check(t *testing.T, expectCalled bool) {
|
||||
|
||||
func (mid *mockItemDataGetter) GetData(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) (io.ReadCloser, *details.ItemInfo, bool, error) {
|
||||
mid.getCalled = true
|
||||
|
||||
if mid.err != nil {
|
||||
errs.AddRecoverable(ctx, mid.err)
|
||||
bus.AddRecoverable(ctx, mid.err)
|
||||
}
|
||||
|
||||
return mid.reader, mid.info, mid.delInFlight, mid.err
|
||||
@ -287,7 +288,7 @@ func (suite *ItemUnitSuite) TestLazyItem() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
errs := fault.New(true)
|
||||
bus := fault.New(true)
|
||||
|
||||
defer test.mid.check(t, true)
|
||||
|
||||
@ -296,7 +297,7 @@ func (suite *ItemUnitSuite) TestLazyItem() {
|
||||
test.mid,
|
||||
id,
|
||||
now,
|
||||
errs)
|
||||
bus)
|
||||
|
||||
assert.Equal(t, id, item.ID(), "ID")
|
||||
assert.False(t, item.Deleted(), "deleted")
|
||||
@ -324,7 +325,7 @@ func (suite *ItemUnitSuite) TestLazyItem() {
|
||||
_, err = item.Info()
|
||||
test.infoErr(t, err, "Info(): %v", clues.ToCore(err))
|
||||
|
||||
e := errs.Errors()
|
||||
e := bus.Errors()
|
||||
|
||||
if !test.expectBusErr {
|
||||
assert.Nil(t, e.Failure, "hard failure")
|
||||
@ -349,12 +350,12 @@ func (suite *ItemUnitSuite) TestLazyItem_DeletedInFlight() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
errs := fault.New(true)
|
||||
bus := fault.New(true)
|
||||
|
||||
mid := &mockItemDataGetter{delInFlight: true}
|
||||
defer mid.check(t, true)
|
||||
|
||||
item := data.NewLazyItem(ctx, mid, id, now, errs)
|
||||
item := data.NewLazyItem(ctx, mid, id, now, bus)
|
||||
|
||||
assert.Equal(t, id, item.ID(), "ID")
|
||||
assert.False(t, item.Deleted(), "deleted")
|
||||
@ -376,9 +377,9 @@ func (suite *ItemUnitSuite) TestLazyItem_DeletedInFlight() {
|
||||
assert.Empty(t, readData, "read data")
|
||||
|
||||
_, err = item.Info()
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, "Info() error")
|
||||
assert.ErrorIs(t, err, errs.NotFound, "Info() error")
|
||||
|
||||
e := errs.Errors()
|
||||
e := bus.Errors()
|
||||
|
||||
assert.Nil(t, e.Failure, "hard failure")
|
||||
assert.Empty(t, e.Recovered, "recovered")
|
||||
@ -395,12 +396,12 @@ func (suite *ItemUnitSuite) TestLazyItem_InfoBeforeReadErrors() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
errs := fault.New(true)
|
||||
bus := fault.New(true)
|
||||
|
||||
mid := &mockItemDataGetter{}
|
||||
defer mid.check(t, false)
|
||||
|
||||
item := data.NewLazyItem(ctx, mid, id, now, errs)
|
||||
item := data.NewLazyItem(ctx, mid, id, now, bus)
|
||||
|
||||
assert.Equal(t, id, item.ID(), "ID")
|
||||
assert.False(t, item.Deleted(), "deleted")
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
@ -92,13 +93,13 @@ type Collection struct {
|
||||
AuxItems map[string]data.Item
|
||||
}
|
||||
|
||||
func (c Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item {
|
||||
func (c Collection) Items(ctx context.Context, bus *fault.Bus) <-chan data.Item {
|
||||
ch := make(chan data.Item)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
el := errs.Local()
|
||||
el := bus.Local()
|
||||
|
||||
for _, item := range c.ItemData {
|
||||
it, ok := item.(*Item)
|
||||
@ -112,7 +113,7 @@ func (c Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item
|
||||
}()
|
||||
|
||||
for _, err := range c.ItemsRecoverableErrs {
|
||||
errs.AddRecoverable(ctx, err)
|
||||
bus.AddRecoverable(ctx, err)
|
||||
}
|
||||
|
||||
return ch
|
||||
@ -144,7 +145,7 @@ func (c Collection) FetchItemByName(
|
||||
) (data.Item, error) {
|
||||
res := c.AuxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
return nil, errs.NotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
@ -163,7 +164,7 @@ func (rc RestoreCollection) FetchItemByName(
|
||||
) (data.Item, error) {
|
||||
res := rc.AuxItems[name]
|
||||
if res == nil {
|
||||
return nil, data.ErrNotFound
|
||||
return nil, errs.NotFound
|
||||
}
|
||||
|
||||
return res, nil
|
||||
@ -206,13 +207,13 @@ type unversionedRestoreCollection struct {
|
||||
|
||||
func (c *unversionedRestoreCollection) Items(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) <-chan data.Item {
|
||||
res := make(chan data.Item)
|
||||
go func() {
|
||||
defer close(res)
|
||||
|
||||
for item := range c.RestoreCollection.Items(ctx, errs) {
|
||||
for item := range c.RestoreCollection.Items(ctx, bus) {
|
||||
r, err := readers.NewVersionedRestoreReader(item.ToReader())
|
||||
require.NoError(c.t, err, clues.ToCore(err))
|
||||
|
||||
@ -248,13 +249,13 @@ type versionedBackupCollection struct {
|
||||
|
||||
func (c *versionedBackupCollection) Items(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) <-chan data.Item {
|
||||
res := make(chan data.Item)
|
||||
go func() {
|
||||
defer close(res)
|
||||
|
||||
for item := range c.BackupCollection.Items(ctx, errs) {
|
||||
for item := range c.BackupCollection.Items(ctx, bus) {
|
||||
r, err := readers.NewVersionedBackupReader(
|
||||
readers.SerializationFormat{
|
||||
Version: readers.DefaultSerializationVersion,
|
||||
|
||||
@ -8,49 +8,27 @@ import (
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
// TODO(ashmrtn): Move this into some inject package. Here to avoid import
|
||||
// cycles.
|
||||
type BackupBases interface {
|
||||
// ConvertToAssistBase converts the base with the given item data snapshot ID
|
||||
// from a merge base to an assist base.
|
||||
ConvertToAssistBase(manifestID manifest.ID)
|
||||
Backups() []BackupEntry
|
||||
UniqueAssistBackups() []BackupEntry
|
||||
MinBackupVersion() int
|
||||
MergeBases() []ManifestEntry
|
||||
DisableMergeBases()
|
||||
UniqueAssistBases() []ManifestEntry
|
||||
DisableAssistBases()
|
||||
MergeBackupBases(
|
||||
ctx context.Context,
|
||||
other BackupBases,
|
||||
reasonToKey func(identity.Reasoner) string,
|
||||
) BackupBases
|
||||
// SnapshotAssistBases returns the set of bases to use for kopia assisted
|
||||
// incremental snapshot operations. It consists of the union of merge bases
|
||||
// and assist bases. If DisableAssistBases has been called then it returns
|
||||
// nil.
|
||||
SnapshotAssistBases() []ManifestEntry
|
||||
}
|
||||
var _ backup.BackupBases = &backupBases{}
|
||||
|
||||
type backupBases struct {
|
||||
// backups and mergeBases should be modified together as they relate similar
|
||||
// data.
|
||||
backups []BackupEntry
|
||||
mergeBases []ManifestEntry
|
||||
assistBackups []BackupEntry
|
||||
assistBases []ManifestEntry
|
||||
backups []backup.BackupEntry
|
||||
mergeBases []backup.ManifestEntry
|
||||
assistBackups []backup.BackupEntry
|
||||
assistBases []backup.ManifestEntry
|
||||
|
||||
// disableAssistBases denote whether any assist bases should be returned to
|
||||
// kopia during snapshot operation.
|
||||
disableAssistBases bool
|
||||
}
|
||||
|
||||
func (bb *backupBases) SnapshotAssistBases() []ManifestEntry {
|
||||
func (bb *backupBases) SnapshotAssistBases() []backup.ManifestEntry {
|
||||
if bb.disableAssistBases {
|
||||
return nil
|
||||
}
|
||||
@ -62,14 +40,14 @@ func (bb *backupBases) SnapshotAssistBases() []ManifestEntry {
|
||||
|
||||
func (bb *backupBases) ConvertToAssistBase(manifestID manifest.ID) {
|
||||
var (
|
||||
snapshotMan ManifestEntry
|
||||
base BackupEntry
|
||||
snapshotMan backup.ManifestEntry
|
||||
base backup.BackupEntry
|
||||
snapFound bool
|
||||
)
|
||||
|
||||
idx := slices.IndexFunc(
|
||||
bb.mergeBases,
|
||||
func(man ManifestEntry) bool {
|
||||
func(man backup.ManifestEntry) bool {
|
||||
return man.ID == manifestID
|
||||
})
|
||||
if idx >= 0 {
|
||||
@ -80,7 +58,7 @@ func (bb *backupBases) ConvertToAssistBase(manifestID manifest.ID) {
|
||||
|
||||
idx = slices.IndexFunc(
|
||||
bb.backups,
|
||||
func(bup BackupEntry) bool {
|
||||
func(bup backup.BackupEntry) bool {
|
||||
return bup.SnapshotID == string(manifestID)
|
||||
})
|
||||
if idx >= 0 {
|
||||
@ -95,11 +73,11 @@ func (bb *backupBases) ConvertToAssistBase(manifestID manifest.ID) {
|
||||
}
|
||||
}
|
||||
|
||||
func (bb backupBases) Backups() []BackupEntry {
|
||||
func (bb backupBases) Backups() []backup.BackupEntry {
|
||||
return slices.Clone(bb.backups)
|
||||
}
|
||||
|
||||
func (bb backupBases) UniqueAssistBackups() []BackupEntry {
|
||||
func (bb backupBases) UniqueAssistBackups() []backup.BackupEntry {
|
||||
if bb.disableAssistBases {
|
||||
return nil
|
||||
}
|
||||
@ -123,7 +101,7 @@ func (bb *backupBases) MinBackupVersion() int {
|
||||
return min
|
||||
}
|
||||
|
||||
func (bb backupBases) MergeBases() []ManifestEntry {
|
||||
func (bb backupBases) MergeBases() []backup.ManifestEntry {
|
||||
return slices.Clone(bb.mergeBases)
|
||||
}
|
||||
|
||||
@ -140,7 +118,7 @@ func (bb *backupBases) DisableMergeBases() {
|
||||
bb.backups = nil
|
||||
}
|
||||
|
||||
func (bb backupBases) UniqueAssistBases() []ManifestEntry {
|
||||
func (bb backupBases) UniqueAssistBases() []backup.ManifestEntry {
|
||||
if bb.disableAssistBases {
|
||||
return nil
|
||||
}
|
||||
@ -175,9 +153,9 @@ func (bb *backupBases) DisableAssistBases() {
|
||||
// MergeBase in the other BackupBases.
|
||||
func (bb *backupBases) MergeBackupBases(
|
||||
ctx context.Context,
|
||||
other BackupBases,
|
||||
other backup.BackupBases,
|
||||
reasonToKey func(reason identity.Reasoner) string,
|
||||
) BackupBases {
|
||||
) backup.BackupBases {
|
||||
if other == nil || (len(other.MergeBases()) == 0 && len(other.UniqueAssistBases()) == 0) {
|
||||
return bb
|
||||
}
|
||||
@ -206,7 +184,7 @@ func (bb *backupBases) MergeBackupBases(
|
||||
}
|
||||
}
|
||||
|
||||
var toAdd []ManifestEntry
|
||||
var toAdd []backup.ManifestEntry
|
||||
|
||||
// Calculate the set of mergeBases to pull from other into this one.
|
||||
for _, m := range other.MergeBases() {
|
||||
@ -267,10 +245,10 @@ func (bb *backupBases) MergeBackupBases(
|
||||
|
||||
func findNonUniqueManifests(
|
||||
ctx context.Context,
|
||||
manifests []ManifestEntry,
|
||||
manifests []backup.ManifestEntry,
|
||||
) map[manifest.ID]struct{} {
|
||||
// ReasonKey -> manifests with that reason.
|
||||
reasons := map[string][]ManifestEntry{}
|
||||
reasons := map[string][]backup.ManifestEntry{}
|
||||
toDrop := map[manifest.ID]struct{}{}
|
||||
|
||||
for _, man := range manifests {
|
||||
@ -323,17 +301,20 @@ func findNonUniqueManifests(
|
||||
return toDrop
|
||||
}
|
||||
|
||||
func getBackupByID(backups []BackupEntry, bID string) (BackupEntry, bool) {
|
||||
func getBackupByID(
|
||||
backups []backup.BackupEntry,
|
||||
bID string,
|
||||
) (backup.BackupEntry, bool) {
|
||||
if len(bID) == 0 {
|
||||
return BackupEntry{}, false
|
||||
return backup.BackupEntry{}, false
|
||||
}
|
||||
|
||||
idx := slices.IndexFunc(backups, func(b BackupEntry) bool {
|
||||
idx := slices.IndexFunc(backups, func(b backup.BackupEntry) bool {
|
||||
return string(b.ID) == bID
|
||||
})
|
||||
|
||||
if idx < 0 || idx >= len(backups) {
|
||||
return BackupEntry{}, false
|
||||
return backup.BackupEntry{}, false
|
||||
}
|
||||
|
||||
return backups[idx], true
|
||||
@ -356,10 +337,10 @@ func (bb *backupBases) fixupAndVerify(ctx context.Context) {
|
||||
toDrop := findNonUniqueManifests(ctx, bb.mergeBases)
|
||||
|
||||
var (
|
||||
backupsToKeep []BackupEntry
|
||||
assistBackupsToKeep []BackupEntry
|
||||
mergeToKeep []ManifestEntry
|
||||
assistToKeep []ManifestEntry
|
||||
backupsToKeep []backup.BackupEntry
|
||||
assistBackupsToKeep []backup.BackupEntry
|
||||
mergeToKeep []backup.ManifestEntry
|
||||
assistToKeep []backup.ManifestEntry
|
||||
)
|
||||
|
||||
for _, man := range bb.mergeBases {
|
||||
|
||||
@ -18,10 +18,15 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
func makeManifest(id, incmpl, bID string, reasons ...identity.Reasoner) ManifestEntry {
|
||||
bIDKey, _ := makeTagKV(TagBackupID)
|
||||
func makeManifest(
|
||||
id,
|
||||
incmpl,
|
||||
bID string,
|
||||
reasons ...identity.Reasoner,
|
||||
) backup.ManifestEntry {
|
||||
bIDKey, _ := backup.MakeTagKV(TagBackupID)
|
||||
|
||||
return ManifestEntry{
|
||||
return backup.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
IncompleteReason: incmpl,
|
||||
@ -57,7 +62,7 @@ func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
|
||||
{
|
||||
name: "Unsorted Backups",
|
||||
bb: &backupBases{
|
||||
backups: []BackupEntry{
|
||||
backups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
Version: 4,
|
||||
@ -86,13 +91,13 @@ func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestConvertToAssistBase() {
|
||||
backups := []BackupEntry{
|
||||
backups := []backup.BackupEntry{
|
||||
{Backup: &backup.Backup{SnapshotID: "1"}},
|
||||
{Backup: &backup.Backup{SnapshotID: "2"}},
|
||||
{Backup: &backup.Backup{SnapshotID: "3"}},
|
||||
}
|
||||
|
||||
merges := []ManifestEntry{
|
||||
merges := []backup.ManifestEntry{
|
||||
makeManifest("1", "", ""),
|
||||
makeManifest("2", "", ""),
|
||||
makeManifest("3", "", ""),
|
||||
@ -185,8 +190,8 @@ func (suite *BackupBasesUnitSuite) TestConvertToAssistBase() {
|
||||
}
|
||||
|
||||
expected := &backupBases{
|
||||
backups: []BackupEntry{backups[0], backups[1]},
|
||||
mergeBases: []ManifestEntry{merges[0], merges[1]},
|
||||
backups: []backup.BackupEntry{backups[0], backups[1]},
|
||||
mergeBases: []backup.ManifestEntry{merges[0], merges[1]},
|
||||
}
|
||||
|
||||
for _, i := range test.expectAssist {
|
||||
@ -203,20 +208,20 @@ func (suite *BackupBasesUnitSuite) TestConvertToAssistBase() {
|
||||
func (suite *BackupBasesUnitSuite) TestDisableMergeBases() {
|
||||
t := suite.T()
|
||||
|
||||
merge := []BackupEntry{
|
||||
merge := []backup.BackupEntry{
|
||||
{Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "m1"}}},
|
||||
{Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "m2"}}},
|
||||
}
|
||||
assist := []BackupEntry{
|
||||
assist := []backup.BackupEntry{
|
||||
{Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "a1"}}},
|
||||
{Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "a2"}}},
|
||||
}
|
||||
|
||||
bb := &backupBases{
|
||||
backups: slices.Clone(merge),
|
||||
mergeBases: make([]ManifestEntry, 2),
|
||||
mergeBases: make([]backup.ManifestEntry, 2),
|
||||
assistBackups: slices.Clone(assist),
|
||||
assistBases: make([]ManifestEntry, 2),
|
||||
assistBases: make([]backup.ManifestEntry, 2),
|
||||
}
|
||||
|
||||
bb.DisableMergeBases()
|
||||
@ -237,10 +242,10 @@ func (suite *BackupBasesUnitSuite) TestDisableMergeBases() {
|
||||
func (suite *BackupBasesUnitSuite) TestDisableAssistBases() {
|
||||
t := suite.T()
|
||||
bb := &backupBases{
|
||||
backups: make([]BackupEntry, 2),
|
||||
mergeBases: make([]ManifestEntry, 2),
|
||||
assistBases: make([]ManifestEntry, 2),
|
||||
assistBackups: make([]BackupEntry, 2),
|
||||
backups: make([]backup.BackupEntry, 2),
|
||||
mergeBases: make([]backup.ManifestEntry, 2),
|
||||
assistBases: make([]backup.ManifestEntry, 2),
|
||||
assistBackups: make([]backup.BackupEntry, 2),
|
||||
}
|
||||
|
||||
bb.DisableAssistBases()
|
||||
@ -270,12 +275,12 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
|
||||
reasons := make([]identity.Reasoner, 0, len(i.cat))
|
||||
|
||||
for _, c := range i.cat {
|
||||
reasons = append(reasons, NewReason("", ro, path.ExchangeService, c))
|
||||
reasons = append(reasons, identity.NewReason("", ro, path.ExchangeService, c))
|
||||
}
|
||||
|
||||
m := makeManifest(baseID, "", "b"+baseID, reasons...)
|
||||
|
||||
b := BackupEntry{
|
||||
b := backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)},
|
||||
SnapshotID: baseID,
|
||||
@ -294,12 +299,12 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
|
||||
reasons := make([]identity.Reasoner, 0, len(i.cat))
|
||||
|
||||
for _, c := range i.cat {
|
||||
reasons = append(reasons, NewReason("", ro, path.ExchangeService, c))
|
||||
reasons = append(reasons, identity.NewReason("", ro, path.ExchangeService, c))
|
||||
}
|
||||
|
||||
m := makeManifest(baseID, "", "a"+baseID, reasons...)
|
||||
|
||||
b := BackupEntry{
|
||||
b := backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: model.StableID("a" + baseID),
|
||||
@ -528,15 +533,19 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
|
||||
func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
ro := "resource_owner"
|
||||
|
||||
makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry {
|
||||
r := NewReason("", ro, path.ExchangeService, pct)
|
||||
makeMan := func(
|
||||
pct path.CategoryType,
|
||||
id, incmpl,
|
||||
bID string,
|
||||
) backup.ManifestEntry {
|
||||
r := identity.NewReason("", ro, path.ExchangeService, pct)
|
||||
return makeManifest(id, incmpl, bID, r)
|
||||
}
|
||||
|
||||
// Make a function so tests can modify things without messing with each other.
|
||||
validMail1 := func() *backupBases {
|
||||
return &backupBases{
|
||||
backups: []BackupEntry{
|
||||
backups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
@ -547,10 +556,10 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
},
|
||||
},
|
||||
},
|
||||
mergeBases: []ManifestEntry{
|
||||
mergeBases: []backup.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "id1", "", "bid1"),
|
||||
},
|
||||
assistBackups: []BackupEntry{
|
||||
assistBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
@ -562,7 +571,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
},
|
||||
},
|
||||
},
|
||||
assistBases: []ManifestEntry{
|
||||
assistBases: []backup.ManifestEntry{
|
||||
makeMan(path.EmailCategory, "id2", "", "bid2"),
|
||||
},
|
||||
}
|
||||
@ -571,7 +580,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
table := []struct {
|
||||
name string
|
||||
bb *backupBases
|
||||
expect BackupBases
|
||||
expect backup.BackupBases
|
||||
}{
|
||||
{
|
||||
name: "empty BaseBackups",
|
||||
@ -727,11 +736,11 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].Reasons = append(
|
||||
res.mergeBases[0].Reasons,
|
||||
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
|
||||
res.assistBases[0].Reasons = append(
|
||||
res.assistBases[0].Reasons,
|
||||
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
|
||||
return res
|
||||
}(),
|
||||
@ -739,11 +748,11 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].Reasons = append(
|
||||
res.mergeBases[0].Reasons,
|
||||
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
|
||||
res.assistBases[0].Reasons = append(
|
||||
res.assistBases[0].Reasons,
|
||||
NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory))
|
||||
|
||||
return res
|
||||
}(),
|
||||
@ -769,14 +778,14 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.backups = append(
|
||||
res.backups,
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid3",
|
||||
},
|
||||
},
|
||||
},
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid4",
|
||||
@ -796,7 +805,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.backups = append(
|
||||
res.backups,
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid4",
|
||||
@ -818,7 +827,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.assistBackups = append(
|
||||
res.assistBackups,
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid3",
|
||||
@ -826,7 +835,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
},
|
||||
},
|
||||
},
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid4",
|
||||
@ -847,7 +856,7 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
res := validMail1()
|
||||
res.assistBackups = append(
|
||||
res.assistBackups,
|
||||
BackupEntry{
|
||||
backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid4",
|
||||
|
||||
@ -17,68 +17,6 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// Kopia does not do comparisons properly for empty tags right now so add some
|
||||
// placeholder value to them.
|
||||
defaultTagValue = "0"
|
||||
|
||||
// Kopia CLI prefixes all user tags with "tag:"[1]. Maintaining this will
|
||||
// ensure we don't accidentally take reserved tags and that tags can be
|
||||
// displayed with kopia CLI.
|
||||
// (permalinks)
|
||||
// [1] https://github.com/kopia/kopia/blob/05e729a7858a6e86cb48ba29fb53cb6045efce2b/cli/command_snapshot_create.go#L169
|
||||
userTagPrefix = "tag:"
|
||||
)
|
||||
|
||||
func NewReason(
|
||||
tenant, resource string,
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
) identity.Reasoner {
|
||||
return reason{
|
||||
tenant: tenant,
|
||||
resource: resource,
|
||||
service: service,
|
||||
category: category,
|
||||
}
|
||||
}
|
||||
|
||||
type reason struct {
|
||||
// tenant appears here so that when this is moved to an inject package nothing
|
||||
// needs changed. However, kopia itself is blind to the fields in the reason
|
||||
// struct and relies on helper functions to get the information it needs.
|
||||
tenant string
|
||||
resource string
|
||||
service path.ServiceType
|
||||
category path.CategoryType
|
||||
}
|
||||
|
||||
func (r reason) Tenant() string {
|
||||
return r.tenant
|
||||
}
|
||||
|
||||
func (r reason) ProtectedResource() string {
|
||||
return r.resource
|
||||
}
|
||||
|
||||
func (r reason) Service() path.ServiceType {
|
||||
return r.service
|
||||
}
|
||||
|
||||
func (r reason) Category() path.CategoryType {
|
||||
return r.category
|
||||
}
|
||||
|
||||
func (r reason) SubtreePath() (path.Path, error) {
|
||||
p, err := path.BuildPrefix(
|
||||
r.Tenant(),
|
||||
r.ProtectedResource(),
|
||||
r.Service(),
|
||||
r.Category())
|
||||
|
||||
return p, clues.Wrap(err, "building path").OrNil()
|
||||
}
|
||||
|
||||
func tagKeys(r identity.Reasoner) []string {
|
||||
return []string{
|
||||
r.ProtectedResource(),
|
||||
@ -91,49 +29,15 @@ func reasonKey(r identity.Reasoner) string {
|
||||
return r.ProtectedResource() + r.Service().String() + r.Category().String()
|
||||
}
|
||||
|
||||
type BackupEntry struct {
|
||||
*backup.Backup
|
||||
Reasons []identity.Reasoner
|
||||
}
|
||||
|
||||
type ManifestEntry struct {
|
||||
*snapshot.Manifest
|
||||
// Reasons contains the ResourceOwners and Service/Categories that caused this
|
||||
// snapshot to be selected as a base. We can't reuse OwnersCats here because
|
||||
// it's possible some ResourceOwners will have a subset of the Categories as
|
||||
// the reason for selecting a snapshot. For example:
|
||||
// 1. backup user1 email,contacts -> B1
|
||||
// 2. backup user1 contacts -> B2 (uses B1 as base)
|
||||
// 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts)
|
||||
Reasons []identity.Reasoner
|
||||
}
|
||||
|
||||
func (me ManifestEntry) GetTag(key string) (string, bool) {
|
||||
k, _ := makeTagKV(key)
|
||||
v, ok := me.Tags[k]
|
||||
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func serviceCatString(s path.ServiceType, c path.CategoryType) string {
|
||||
return s.String() + c.String()
|
||||
}
|
||||
|
||||
// MakeTagKV normalizes the provided key to protect it from clobbering
|
||||
// similarly named tags from non-user input (user inputs are still open
|
||||
// to collisions amongst eachother).
|
||||
// Returns the normalized Key plus a default value. If you're embedding a
|
||||
// key-only tag, the returned default value msut be used instead of an
|
||||
// empty string.
|
||||
func makeTagKV(k string) (string, string) {
|
||||
return userTagPrefix + k, defaultTagValue
|
||||
}
|
||||
|
||||
func normalizeTagKVs(tags map[string]string) map[string]string {
|
||||
t2 := make(map[string]string, len(tags))
|
||||
|
||||
for k, v := range tags {
|
||||
mk, mv := makeTagKV(k)
|
||||
mk, mv := backup.MakeTagKV(k)
|
||||
|
||||
if len(v) == 0 {
|
||||
v = mv
|
||||
@ -172,7 +76,7 @@ func (b *baseFinder) getBackupModel(
|
||||
ctx context.Context,
|
||||
man *snapshot.Manifest,
|
||||
) (*backup.Backup, error) {
|
||||
k, _ := makeTagKV(TagBackupID)
|
||||
k, _ := backup.MakeTagKV(TagBackupID)
|
||||
bID := man.Tags[k]
|
||||
|
||||
ctx = clues.Add(ctx, "search_backup_id", bID)
|
||||
@ -380,7 +284,7 @@ func (b *baseFinder) FindBases(
|
||||
ctx context.Context,
|
||||
reasons []identity.Reasoner,
|
||||
tags map[string]string,
|
||||
) BackupBases {
|
||||
) backup.BackupBases {
|
||||
var (
|
||||
// Backup models and item data snapshot manifests are 1:1 for bases so just
|
||||
// track things by the backup ID. We need to track by ID so we can coalesce
|
||||
@ -431,16 +335,16 @@ func (b *baseFinder) FindBases(
|
||||
// Convert what we got to the format that backupBases takes right now.
|
||||
// TODO(ashmrtn): Remove when backupBases has consolidated fields.
|
||||
res := &backupBases{}
|
||||
bups := make([]BackupEntry, 0, len(mergeBases))
|
||||
snaps := make([]ManifestEntry, 0, len(mergeBases))
|
||||
bups := make([]backup.BackupEntry, 0, len(mergeBases))
|
||||
snaps := make([]backup.ManifestEntry, 0, len(mergeBases))
|
||||
|
||||
for _, base := range mergeBases {
|
||||
bups = append(bups, BackupEntry{
|
||||
bups = append(bups, backup.BackupEntry{
|
||||
Backup: base.Backup,
|
||||
Reasons: base.Reasons,
|
||||
})
|
||||
|
||||
snaps = append(snaps, ManifestEntry{
|
||||
snaps = append(snaps, backup.ManifestEntry{
|
||||
Manifest: base.ItemDataSnapshot,
|
||||
Reasons: base.Reasons,
|
||||
})
|
||||
@ -449,16 +353,16 @@ func (b *baseFinder) FindBases(
|
||||
res.backups = bups
|
||||
res.mergeBases = snaps
|
||||
|
||||
bups = make([]BackupEntry, 0, len(assistBases))
|
||||
snaps = make([]ManifestEntry, 0, len(assistBases))
|
||||
bups = make([]backup.BackupEntry, 0, len(assistBases))
|
||||
snaps = make([]backup.ManifestEntry, 0, len(assistBases))
|
||||
|
||||
for _, base := range assistBases {
|
||||
bups = append(bups, BackupEntry{
|
||||
bups = append(bups, backup.BackupEntry{
|
||||
Backup: base.Backup,
|
||||
Reasons: base.Reasons,
|
||||
})
|
||||
|
||||
snaps = append(snaps, ManifestEntry{
|
||||
snaps = append(snaps, backup.ManifestEntry{
|
||||
Manifest: base.ItemDataSnapshot,
|
||||
Reasons: base.Reasons,
|
||||
})
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
@ -47,22 +47,22 @@ var (
|
||||
|
||||
testAllUsersAllCats = []identity.Reasoner{
|
||||
// User1 email and events.
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
// User2 email and events.
|
||||
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
// User3 email and events.
|
||||
NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
|
||||
}
|
||||
testAllUsersMail = []identity.Reasoner{
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
}
|
||||
testUser1Mail = []identity.Reasoner{
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
}
|
||||
)
|
||||
|
||||
@ -91,7 +91,7 @@ func (mg mockEmptyModelGetter) GetBackup(
|
||||
context.Context,
|
||||
model.StableID,
|
||||
) (*backup.Backup, error) {
|
||||
return nil, data.ErrNotFound
|
||||
return nil, errs.NotFound
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
@ -121,7 +121,7 @@ func newManifestInfo(
|
||||
structTags := make(map[string]string, len(tags))
|
||||
|
||||
for _, t := range tags {
|
||||
tk, _ := makeTagKV(t)
|
||||
tk, _ := backup.MakeTagKV(t)
|
||||
structTags[tk] = ""
|
||||
}
|
||||
|
||||
@ -141,7 +141,7 @@ func newManifestInfo(
|
||||
}
|
||||
|
||||
if len(backupID) > 0 {
|
||||
k, _ := makeTagKV(TagBackupID)
|
||||
k, _ := backup.MakeTagKV(TagBackupID)
|
||||
res.metadata.Labels[k] = backupID
|
||||
res.man.Tags[k] = backupID
|
||||
}
|
||||
@ -269,7 +269,7 @@ func (mg mockModelGetter) GetBackup(
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
return nil, data.ErrNotFound
|
||||
return nil, errs.NotFound
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
@ -294,7 +294,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() {
|
||||
bg: mockEmptyModelGetter{},
|
||||
}
|
||||
reasons := []identity.Reasoner{
|
||||
NewReason("", "a-user", path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", "a-user", path.ExchangeService, path.EmailCategory),
|
||||
}
|
||||
|
||||
bb := bf.FindBases(ctx, reasons, nil)
|
||||
@ -314,7 +314,7 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() {
|
||||
bg: mockEmptyModelGetter{},
|
||||
}
|
||||
reasons := []identity.Reasoner{
|
||||
NewReason("", "a-user", path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", "a-user", path.ExchangeService, path.EmailCategory),
|
||||
}
|
||||
|
||||
bb := bf.FindBases(ctx, reasons, nil)
|
||||
@ -561,14 +561,14 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
|
||||
},
|
||||
expectedBaseReasons: map[int][]identity.Reasoner{
|
||||
0: {
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser3, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
1: {
|
||||
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser3, path.ExchangeService, path.EventsCategory),
|
||||
},
|
||||
},
|
||||
backupData: []backupInfo{
|
||||
@ -611,20 +611,20 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
|
||||
},
|
||||
expectedBaseReasons: map[int][]identity.Reasoner{
|
||||
2: {
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
},
|
||||
},
|
||||
expectedAssistReasons: map[int][]identity.Reasoner{
|
||||
0: {
|
||||
NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EventsCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EventsCategory),
|
||||
},
|
||||
1: {
|
||||
NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser1, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", testUser2, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
},
|
||||
backupData: []backupInfo{
|
||||
@ -1078,7 +1078,7 @@ func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() {
|
||||
|
||||
func checkManifestEntriesMatch(
|
||||
t *testing.T,
|
||||
retSnaps []ManifestEntry,
|
||||
retSnaps []backup.ManifestEntry,
|
||||
allExpected []manifestInfo,
|
||||
expectedIdxsAndReasons map[int][]identity.Reasoner,
|
||||
) {
|
||||
@ -1119,7 +1119,7 @@ func checkManifestEntriesMatch(
|
||||
|
||||
func checkBackupEntriesMatch(
|
||||
t *testing.T,
|
||||
retBups []BackupEntry,
|
||||
retBups []backup.BackupEntry,
|
||||
allExpected []backupInfo,
|
||||
expectedIdxsAndReasons map[int][]identity.Reasoner,
|
||||
) {
|
||||
|
||||
@ -12,7 +12,7 @@ import (
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
@ -99,7 +99,7 @@ func cleanupOrphanedData(
|
||||
|
||||
toDelete[snap.ID] = struct{}{}
|
||||
|
||||
k, _ := makeTagKV(TagBackupCategory)
|
||||
k, _ := backup.MakeTagKV(TagBackupCategory)
|
||||
if _, ok := snap.Labels[k]; ok {
|
||||
dataSnaps[snap.ID] = snap
|
||||
continue
|
||||
@ -160,7 +160,7 @@ func cleanupOrphanedData(
|
||||
model.BackupSchema,
|
||||
bup.ModelStoreID,
|
||||
&bm); err != nil {
|
||||
if !errors.Is(err, data.ErrNotFound) {
|
||||
if !errors.Is(err, errs.NotFound) {
|
||||
return clues.Wrap(err, "getting backup model").
|
||||
With("search_backup_id", bup.ID)
|
||||
}
|
||||
@ -315,7 +315,7 @@ func transferTags(snap *manifest.EntryMetadata, bup *backup.Backup) error {
|
||||
skipTags := map[string]struct{}{}
|
||||
|
||||
for _, k := range skipKeys {
|
||||
key, _ := makeTagKV(k)
|
||||
key, _ := backup.MakeTagKV(k)
|
||||
skipTags[key] = struct{}{}
|
||||
}
|
||||
|
||||
@ -324,7 +324,7 @@ func transferTags(snap *manifest.EntryMetadata, bup *backup.Backup) error {
|
||||
// backups.
|
||||
roid := bup.ProtectedResourceID
|
||||
|
||||
roidK, _ := makeTagKV(roid)
|
||||
roidK, _ := backup.MakeTagKV(roid)
|
||||
skipTags[roidK] = struct{}{}
|
||||
|
||||
// This is hacky, but right now we don't have a good way to get only the
|
||||
@ -336,11 +336,11 @@ func transferTags(snap *manifest.EntryMetadata, bup *backup.Backup) error {
|
||||
// Convert them to the newer format that we'd like to have where the
|
||||
// service/category tags have the form "sc-<service><category>".
|
||||
for tag := range snap.Labels {
|
||||
if _, ok := skipTags[tag]; ok || !strings.HasPrefix(tag, userTagPrefix) {
|
||||
if _, ok := skipTags[tag]; ok || !strings.HasPrefix(tag, backup.LegacyUserTagPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
bup.Tags[strings.Replace(tag, userTagPrefix, serviceCatTagPrefix, 1)] = "0"
|
||||
bup.Tags[strings.Replace(tag, backup.LegacyUserTagPrefix, serviceCatTagPrefix, 1)] = "0"
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
@ -116,7 +116,7 @@ func (ms mockStorer) GetWithModelStoreID(
|
||||
}
|
||||
}
|
||||
|
||||
return clues.Stack(data.ErrNotFound)
|
||||
return clues.Stack(errs.NotFound)
|
||||
}
|
||||
|
||||
func (ms mockStorer) DeleteWithModelStoreIDs(
|
||||
@ -136,7 +136,7 @@ type backupRes struct {
|
||||
}
|
||||
|
||||
func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
backupTag, _ := makeTagKV(TagBackupCategory)
|
||||
backupTag, _ := backup.MakeTagKV(TagBackupCategory)
|
||||
|
||||
// Current backup and snapshots.
|
||||
bupCurrent := func() *backup.Backup {
|
||||
@ -331,13 +331,13 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
// Add the given reasons.
|
||||
for _, r := range reasons {
|
||||
for _, k := range tagKeys(r) {
|
||||
key, _ := makeTagKV(k)
|
||||
key, _ := backup.MakeTagKV(k)
|
||||
res.Labels[key] = "0"
|
||||
}
|
||||
}
|
||||
|
||||
// Also add other common reasons on item data snapshots.
|
||||
k, _ := makeTagKV(TagBackupCategory)
|
||||
k, _ := backup.MakeTagKV(TagBackupCategory)
|
||||
res.Labels[k] = "0"
|
||||
|
||||
return &res
|
||||
@ -369,13 +369,6 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
return &res
|
||||
}
|
||||
|
||||
backupWithLegacyResource := func(protectedResource string, b *backup.Backup) *backup.Backup {
|
||||
res := *b
|
||||
res.ResourceOwnerID = protectedResource
|
||||
|
||||
return &res
|
||||
}
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
snapshots []*manifest.EntryMetadata
|
||||
@ -525,11 +518,11 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
{bup: bupCurrent()},
|
||||
{
|
||||
bup: bupLegacy(),
|
||||
err: data.ErrNotFound,
|
||||
err: errs.NotFound,
|
||||
},
|
||||
{
|
||||
bup: bupNoDetails(),
|
||||
err: data.ErrNotFound,
|
||||
err: errs.NotFound,
|
||||
},
|
||||
},
|
||||
// Backup IDs are still included in here because they're added to the
|
||||
@ -630,7 +623,7 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
backups: []backupRes{
|
||||
{
|
||||
bup: backupWithTime(baseTime, bupCurrent()),
|
||||
err: data.ErrNotFound,
|
||||
err: errs.NotFound,
|
||||
},
|
||||
},
|
||||
time: baseTime,
|
||||
@ -648,13 +641,13 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -675,19 +668,19 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Minute), snapCurrent3()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Minute), deetsCurrent3()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -711,6 +704,9 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
// not ideal, but some older versions of corso didn't even populate the
|
||||
// resource owner ID.
|
||||
//
|
||||
// The old version of corso does not populated the ProtectedResourceID
|
||||
// field in the backup model.
|
||||
//
|
||||
// Worst case, the assist base will be cleaned up when the user upgrades
|
||||
// corso and generates either a new assist base or merge base with the
|
||||
// same reason.
|
||||
@ -719,18 +715,18 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
{bup: backupWithResource("ro", true, backupWithTime(baseTime, bupCurrent()))},
|
||||
{bup: backupWithLegacyResource("ro", backupWithTime(baseTime.Add(time.Second), bupCurrent2()))},
|
||||
{bup: backupWithTime(baseTime.Add(time.Second), bupCurrent2())},
|
||||
},
|
||||
time: baseTime.Add(48 * time.Hour),
|
||||
buffer: 24 * time.Hour,
|
||||
@ -742,6 +738,9 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
// reason and an even newer merge base from a current version of corso
|
||||
// causes the assist base to be garbage collected.
|
||||
//
|
||||
// The old version of corso does not populated the ProtectedResourceID
|
||||
// field in the backup model.
|
||||
//
|
||||
// This also tests that bases without a merge or assist tag are not
|
||||
// garbage collected as an assist base.
|
||||
name: "AssistAndLegacyAndCurrentMergeBases NotYoungest CausesCleanup",
|
||||
@ -749,24 +748,24 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Minute), snapCurrent3()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Minute), deetsCurrent3()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
{bup: backupWithResource("ro", true, backupWithTime(baseTime, bupCurrent()))},
|
||||
{bup: backupWithLegacyResource("ro", backupWithTime(baseTime.Add(time.Second), bupCurrent2()))},
|
||||
{bup: backupWithTime(baseTime.Add(time.Second), bupCurrent2())},
|
||||
{bup: backupWithResource("ro", false, backupWithTime(baseTime.Add(time.Minute), bupCurrent3()))},
|
||||
},
|
||||
time: baseTime.Add(48 * time.Hour),
|
||||
@ -786,19 +785,19 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Minute), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Minute), deetsCurrent2()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent3()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent3()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -823,14 +822,14 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory),
|
||||
NewReason("", "ro", path.ExchangeService, path.ContactsCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.ContactsCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -851,13 +850,13 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro1", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro1", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro2", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro2", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -878,13 +877,13 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant2",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
@ -905,19 +904,19 @@ func (suite *BackupCleanupUnitSuite) TestCleanupOrphanedData() {
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime, snapCurrent()),
|
||||
"",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime, deetsCurrent()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Second), snapCurrent2()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Second), deetsCurrent2()),
|
||||
|
||||
manifestWithReasons(
|
||||
manifestWithTime(baseTime.Add(time.Minute), snapCurrent3()),
|
||||
"tenant1",
|
||||
NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
identity.NewReason("", "ro", path.ExchangeService, path.EmailCategory)),
|
||||
manifestWithTime(baseTime.Add(time.Minute), deetsCurrent3()),
|
||||
},
|
||||
backups: []backupRes{
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/kopia/kopia/fs"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -30,11 +31,11 @@ type kopiaDataCollection struct {
|
||||
|
||||
func (kdc *kopiaDataCollection) Items(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) <-chan data.Item {
|
||||
var (
|
||||
res = make(chan data.Item)
|
||||
el = errs.Local()
|
||||
el = bus.Local()
|
||||
loadCount = 0
|
||||
)
|
||||
|
||||
@ -74,7 +75,7 @@ func (kdc kopiaDataCollection) FullPath() path.Path {
|
||||
}
|
||||
|
||||
// Fetch returns the file with the given name from the collection as a
|
||||
// data.Item. Returns a data.ErrNotFound error if the file isn't in the
|
||||
// data.Item. Returns a errs.NotFound error if the file isn't in the
|
||||
// collection.
|
||||
func (kdc kopiaDataCollection) FetchItemByName(
|
||||
ctx context.Context,
|
||||
@ -93,7 +94,7 @@ func (kdc kopiaDataCollection) FetchItemByName(
|
||||
e, err := kdc.dir.Child(ctx, encodeAsPath(name))
|
||||
if err != nil {
|
||||
if isErrEntryNotFound(err) {
|
||||
err = clues.Stack(data.ErrNotFound, err)
|
||||
err = clues.Stack(errs.NotFound, err)
|
||||
}
|
||||
|
||||
return nil, clues.Wrap(err, "getting item").WithClues(ctx)
|
||||
|
||||
@ -13,6 +13,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
dataMock "github.com/alcionai/corso/src/internal/data/mock"
|
||||
@ -410,7 +411,7 @@ func (suite *KopiaDataCollectionUnitSuite) TestFetchItemByName() {
|
||||
|
||||
if err != nil {
|
||||
if test.notFoundErr {
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -17,7 +18,7 @@ type (
|
||||
ConsumeBackupCollections(
|
||||
ctx context.Context,
|
||||
backupReasons []identity.Reasoner,
|
||||
bases kopia.BackupBases,
|
||||
bases backup.BackupBases,
|
||||
cs []data.BackupCollection,
|
||||
pmr prefixmatcher.StringSetReader,
|
||||
tags map[string]string,
|
||||
@ -41,6 +42,6 @@ type (
|
||||
ctx context.Context,
|
||||
reasons []identity.Reasoner,
|
||||
tags map[string]string,
|
||||
) kopia.BackupBases
|
||||
) backup.BackupBases
|
||||
}
|
||||
)
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
@ -63,7 +64,7 @@ func (mc mergeCollection) FullPath() path.Path {
|
||||
|
||||
func (mc *mergeCollection) Items(
|
||||
ctx context.Context,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) <-chan data.Item {
|
||||
res := make(chan data.Item)
|
||||
|
||||
@ -82,7 +83,7 @@ func (mc *mergeCollection) Items(
|
||||
"merged_collection_storage_path", path.LoggableDir(c.storagePath))
|
||||
logger.Ctx(ictx).Debug("sending items from merged collection")
|
||||
|
||||
for item := range c.Items(ictx, errs) {
|
||||
for item := range c.Items(ictx, bus) {
|
||||
res <- item
|
||||
}
|
||||
}
|
||||
@ -92,7 +93,7 @@ func (mc *mergeCollection) Items(
|
||||
}
|
||||
|
||||
// Fetch goes through all the collections in this one and returns the first
|
||||
// match found or the first error that is not data.ErrNotFound. If multiple
|
||||
// match found or the first error that is not errs.NotFound. If multiple
|
||||
// collections have the requested item, the instance in the collection with the
|
||||
// lexicographically smallest storage path is returned.
|
||||
func (mc *mergeCollection) FetchItemByName(
|
||||
@ -113,11 +114,11 @@ func (mc *mergeCollection) FetchItemByName(
|
||||
s, err := c.FetchItemByName(ictx, name)
|
||||
if err == nil {
|
||||
return s, nil
|
||||
} else if err != nil && !errors.Is(err, data.ErrNotFound) {
|
||||
} else if err != nil && !errors.Is(err, errs.NotFound) {
|
||||
return nil, clues.Wrap(err, "fetching from merged collection").
|
||||
WithClues(ictx)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, clues.Wrap(data.ErrNotFound, "merged collection fetch")
|
||||
return nil, clues.Wrap(errs.NotFound, "merged collection fetch")
|
||||
}
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
|
||||
@ -292,7 +293,7 @@ func (suite *MergeCollectionUnitSuite) TestFetchItemByName() {
|
||||
|
||||
if err != nil {
|
||||
if test.notFoundErr {
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@ -4,9 +4,13 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
)
|
||||
|
||||
func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) {
|
||||
var _ backup.BackupBases = &MockBackupBases{}
|
||||
|
||||
func AssertBackupBasesEqual(t *testing.T, expect, got backup.BackupBases) {
|
||||
if expect == nil && got == nil {
|
||||
return
|
||||
}
|
||||
@ -48,22 +52,22 @@ type MockBackupBases struct {
|
||||
*backupBases
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithBackups(b ...BackupEntry) *MockBackupBases {
|
||||
func (bb *MockBackupBases) WithBackups(b ...backup.BackupEntry) *MockBackupBases {
|
||||
bb.backupBases.backups = append(bb.Backups(), b...)
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithMergeBases(m ...ManifestEntry) *MockBackupBases {
|
||||
func (bb *MockBackupBases) WithMergeBases(m ...backup.ManifestEntry) *MockBackupBases {
|
||||
bb.backupBases.mergeBases = append(bb.MergeBases(), m...)
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithAssistBackups(b ...BackupEntry) *MockBackupBases {
|
||||
func (bb *MockBackupBases) WithAssistBackups(b ...backup.BackupEntry) *MockBackupBases {
|
||||
bb.backupBases.assistBackups = append(bb.UniqueAssistBackups(), b...)
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithAssistBases(m ...ManifestEntry) *MockBackupBases {
|
||||
func (bb *MockBackupBases) WithAssistBases(m ...backup.ManifestEntry) *MockBackupBases {
|
||||
bb.backupBases.assistBases = append(bb.UniqueAssistBases(), m...)
|
||||
return bb
|
||||
}
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/pkg/store"
|
||||
)
|
||||
@ -292,7 +292,7 @@ func (ms *ModelStore) getModelStoreID(
|
||||
}
|
||||
|
||||
if len(metadata) == 0 {
|
||||
return "", clues.Wrap(data.ErrNotFound, "getting ModelStoreID").WithClues(ctx)
|
||||
return "", clues.Wrap(errs.NotFound, "getting ModelStoreID").WithClues(ctx)
|
||||
}
|
||||
|
||||
if len(metadata) != 1 {
|
||||
@ -347,7 +347,7 @@ func (ms *ModelStore) GetWithModelStoreID(
|
||||
metadata, err := ms.c.GetManifest(ctx, id, m)
|
||||
if err != nil {
|
||||
if errors.Is(err, manifest.ErrNotFound) {
|
||||
err = data.ErrNotFound
|
||||
err = errs.NotFound
|
||||
}
|
||||
|
||||
return clues.Wrap(err, "getting model data").WithClues(ctx)
|
||||
@ -490,7 +490,7 @@ func (ms *ModelStore) Delete(ctx context.Context, s model.Schema, id model.Stabl
|
||||
|
||||
latest, err := ms.getModelStoreID(ctx, s, id)
|
||||
if err != nil {
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
if errors.Is(err, errs.NotFound) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
@ -381,10 +381,10 @@ func (suite *ModelStoreIntegrationSuite) TestGet_NotFoundErrors() {
|
||||
t := suite.T()
|
||||
|
||||
err := suite.m.Get(suite.ctx, model.BackupOpSchema, "baz", nil)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
|
||||
err = suite.m.GetWithModelStoreID(suite.ctx, model.BackupOpSchema, "baz", nil)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *ModelStoreIntegrationSuite) TestPutGetOfTypeBadVersion() {
|
||||
@ -670,7 +670,7 @@ func (suite *ModelStoreIntegrationSuite) TestPutUpdate() {
|
||||
}
|
||||
|
||||
err = m.GetWithModelStoreID(ctx, theModelType, oldModelID, nil)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -737,7 +737,7 @@ func (suite *ModelStoreIntegrationSuite) TestPutDelete() {
|
||||
|
||||
returned := &fooModel{}
|
||||
err = suite.m.GetWithModelStoreID(suite.ctx, theModelType, foo.ModelStoreID, returned)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
}
|
||||
|
||||
func (suite *ModelStoreIntegrationSuite) TestPutDeleteBatch() {
|
||||
@ -760,7 +760,7 @@ func (suite *ModelStoreIntegrationSuite) TestPutDeleteBatch() {
|
||||
for _, id := range ids {
|
||||
returned := &fooModel{}
|
||||
err := suite.m.GetWithModelStoreID(suite.ctx, theModelType, id, returned)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
}
|
||||
}
|
||||
|
||||
@ -843,7 +843,7 @@ func (suite *ModelStoreRegressionSuite) TestFailDuringWriteSessionHasNoVisibleEf
|
||||
assert.ErrorIs(t, err, assert.AnError, clues.ToCore(err))
|
||||
|
||||
err = m.GetWithModelStoreID(ctx, theModelType, newID, nil)
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, clues.ToCore(err))
|
||||
assert.ErrorIs(t, err, errs.NotFound, clues.ToCore(err))
|
||||
|
||||
returned := &fooModel{}
|
||||
|
||||
|
||||
@ -17,12 +17,14 @@ import (
|
||||
"github.com/kopia/kopia/snapshot/snapshotfs"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/diagnostics"
|
||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||
"github.com/alcionai/corso/src/internal/m365/graph/metadata"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -58,7 +60,7 @@ type corsoProgress struct {
|
||||
toMerge *mergeDetails
|
||||
mu sync.RWMutex
|
||||
totalBytes int64
|
||||
errs *fault.Bus
|
||||
bus *fault.Bus
|
||||
// expectedIgnoredErrors is a count of error cases caught in the Error wrapper
|
||||
// which are well known and actually ignorable. At the end of a run, if the
|
||||
// manifest ignored error count is equal to this count, then everything is good.
|
||||
@ -107,7 +109,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
|
||||
// never had to materialize their details in-memory.
|
||||
if d.infoer == nil || d.cached {
|
||||
if d.prevPath == nil {
|
||||
cp.errs.AddRecoverable(ctx, clues.New("finished file sourced from previous backup with no previous path").
|
||||
cp.bus.AddRecoverable(ctx, clues.New("finished file sourced from previous backup with no previous path").
|
||||
WithClues(ctx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
|
||||
@ -123,7 +125,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
|
||||
d.repoPath,
|
||||
d.locationPath)
|
||||
if err != nil {
|
||||
cp.errs.AddRecoverable(ctx, clues.Wrap(err, "adding finished file to merge list").
|
||||
cp.bus.AddRecoverable(ctx, clues.Wrap(err, "adding finished file to merge list").
|
||||
WithClues(ctx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
}
|
||||
@ -132,18 +134,18 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
|
||||
}
|
||||
|
||||
info, err := d.infoer.Info()
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
if errors.Is(err, errs.NotFound) {
|
||||
// The item was deleted between enumeration and trying to get data. Skip
|
||||
// adding it to details since there's no data for it.
|
||||
return
|
||||
} else if err != nil {
|
||||
cp.errs.AddRecoverable(ctx, clues.Wrap(err, "getting ItemInfo").
|
||||
cp.bus.AddRecoverable(ctx, clues.Wrap(err, "getting ItemInfo").
|
||||
WithClues(ctx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
|
||||
return
|
||||
} else if !ptr.Val(d.modTime).Equal(info.Modified()) {
|
||||
cp.errs.AddRecoverable(ctx, clues.New("item modTime mismatch").
|
||||
cp.bus.AddRecoverable(ctx, clues.New("item modTime mismatch").
|
||||
WithClues(ctx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
|
||||
@ -152,7 +154,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
|
||||
|
||||
err = cp.deets.Add(d.repoPath, d.locationPath, info)
|
||||
if err != nil {
|
||||
cp.errs.AddRecoverable(ctx, clues.Wrap(err, "adding finished file to details").
|
||||
cp.bus.AddRecoverable(ctx, clues.Wrap(err, "adding finished file to details").
|
||||
WithClues(ctx).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
|
||||
@ -216,7 +218,7 @@ func (cp *corsoProgress) Error(relpath string, err error, isIgnored bool) {
|
||||
|
||||
defer cp.UploadProgress.Error(relpath, err, isIgnored)
|
||||
|
||||
cp.errs.AddRecoverable(cp.ctx, clues.Wrap(err, "kopia reported error").
|
||||
cp.bus.AddRecoverable(cp.ctx, clues.Wrap(err, "kopia reported error").
|
||||
With("is_ignored", isIgnored, "relative_path", relpath).
|
||||
Label(fault.LabelForceNoBackupCreation))
|
||||
}
|
||||
@ -250,7 +252,7 @@ func collectionEntries(
|
||||
// Track which items have already been seen so we can skip them if we see
|
||||
// them again in the data from the base snapshot.
|
||||
seen = map[string]struct{}{}
|
||||
items = streamedEnts.Items(ctx, progress.errs)
|
||||
items = streamedEnts.Items(ctx, progress.bus)
|
||||
)
|
||||
|
||||
if lp, ok := streamedEnts.(data.LocationPather); ok {
|
||||
@ -288,7 +290,7 @@ func collectionEntries(
|
||||
itemPath, err := streamedEnts.FullPath().AppendItem(e.ID())
|
||||
if err != nil {
|
||||
err = clues.Wrap(err, "getting full item path")
|
||||
progress.errs.AddRecoverable(ctx, err)
|
||||
progress.bus.AddRecoverable(ctx, err)
|
||||
|
||||
logger.CtxErr(ctx, err).Error("getting full item path")
|
||||
|
||||
@ -1039,7 +1041,7 @@ func traverseBaseDir(
|
||||
return nil
|
||||
}
|
||||
|
||||
func logBaseInfo(ctx context.Context, m ManifestEntry) {
|
||||
func logBaseInfo(ctx context.Context, m backup.ManifestEntry) {
|
||||
svcs := map[string]struct{}{}
|
||||
cats := map[string]struct{}{}
|
||||
|
||||
@ -1085,7 +1087,7 @@ const (
|
||||
func inflateBaseTree(
|
||||
ctx context.Context,
|
||||
loader snapshotLoader,
|
||||
snap ManifestEntry,
|
||||
snap backup.ManifestEntry,
|
||||
updatedPaths map[string]path.Path,
|
||||
roots map[string]*treeMap,
|
||||
) error {
|
||||
@ -1196,7 +1198,7 @@ func inflateBaseTree(
|
||||
func inflateDirTree(
|
||||
ctx context.Context,
|
||||
loader snapshotLoader,
|
||||
baseSnaps []ManifestEntry,
|
||||
baseSnaps []backup.ManifestEntry,
|
||||
collections []data.BackupCollection,
|
||||
globalExcludeSet prefixmatcher.StringSetReader,
|
||||
progress *corsoProgress,
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
@ -377,7 +378,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFile() {
|
||||
deets: bd,
|
||||
toMerge: newMergeDetails(),
|
||||
pending: map[string]*itemDetails{},
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
ci := test.cachedItems(suite.targetFileName, suite.targetFilePath)
|
||||
@ -475,7 +476,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() {
|
||||
UploadProgress: &snapshotfs.NullUploadProgress{},
|
||||
deets: bd,
|
||||
pending: map[string]*itemDetails{},
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
for k, v := range cachedItems {
|
||||
@ -491,7 +492,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileCachedNoPrevPathErrors() {
|
||||
|
||||
assert.Empty(t, cp.pending)
|
||||
assert.Empty(t, bd.Details().Entries)
|
||||
assert.Error(t, cp.errs.Failure(), clues.ToCore(cp.errs.Failure()))
|
||||
assert.Error(t, cp.bus.Failure(), clues.ToCore(cp.bus.Failure()))
|
||||
}
|
||||
|
||||
func (suite *CorsoProgressUnitSuite) TestFinishedFileBaseItemDoesntBuildHierarchy() {
|
||||
@ -526,7 +527,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedFileBaseItemDoesntBuildHierarch
|
||||
deets: db,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
deets := &itemDetails{
|
||||
@ -568,7 +569,7 @@ func (suite *CorsoProgressUnitSuite) TestFinishedHashingFile() {
|
||||
UploadProgress: &snapshotfs.NullUploadProgress{},
|
||||
deets: bd,
|
||||
pending: map[string]*itemDetails{},
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
ci := test.cachedItems(suite.targetFileName, suite.targetFilePath)
|
||||
@ -631,7 +632,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree() {
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
collections := []data.BackupCollection{
|
||||
@ -751,7 +752,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_MixedDirectory()
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
dirTree, err := inflateDirTree(ctx, nil, nil, test.layout, pmMock.NewPrefixMap(nil), progress)
|
||||
@ -858,7 +859,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_Fails() {
|
||||
progress := &corsoProgress{
|
||||
ctx: ctx,
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
_, err := inflateDirTree(ctx, nil, nil, test.layout, pmMock.NewPrefixMap(nil), progress)
|
||||
@ -879,14 +880,14 @@ func makeManifestEntry(
|
||||
id, tenant, resourceOwner string,
|
||||
service path.ServiceType,
|
||||
categories ...path.CategoryType,
|
||||
) ManifestEntry {
|
||||
) backup.ManifestEntry {
|
||||
var reasons []identity.Reasoner
|
||||
|
||||
for _, c := range categories {
|
||||
reasons = append(reasons, NewReason(tenant, resourceOwner, service, c))
|
||||
reasons = append(reasons, identity.NewReason(tenant, resourceOwner, service, c))
|
||||
}
|
||||
|
||||
return ManifestEntry{
|
||||
return backup.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
},
|
||||
@ -957,7 +958,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeErrors() {
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
cols := []data.BackupCollection{}
|
||||
@ -1189,7 +1190,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() {
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
msw := &mockSnapshotWalker{
|
||||
snapshotRoot: getBaseSnapshot(),
|
||||
@ -1198,7 +1199,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSingleSubtree() {
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
test.inputCollections(),
|
||||
@ -1899,7 +1900,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
msw := &mockSnapshotWalker{
|
||||
snapshotRoot: getBaseSnapshot(),
|
||||
@ -1913,7 +1914,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeMultipleSubdirecto
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
test.inputCollections(t),
|
||||
@ -2033,7 +2034,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
mc := exchMock.NewCollection(suite.testStoragePath, suite.testStoragePath, 1)
|
||||
mc.PrevPath = mc.FullPath()
|
||||
@ -2057,7 +2058,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSkipsDeletedSubtre
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
collections,
|
||||
@ -2130,7 +2131,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_HandleEmptyBase()
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
mc := exchMock.NewCollection(archiveStorePath, archiveLocPath, 1)
|
||||
mc.ColState = data.NewState
|
||||
@ -2157,7 +2158,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_HandleEmptyBase()
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("", testTenant, testUser, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
collections,
|
||||
@ -2352,7 +2353,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
mc := exchMock.NewCollection(inboxPath, inboxPath, 1)
|
||||
@ -2373,7 +2374,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsCorrectSubt
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.ContactsCategory),
|
||||
makeManifestEntry("id2", testTenant, testUser, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
@ -2508,7 +2509,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
|
||||
mce := exchMock.NewCollection(newPrefixPathEmail, nil, 0)
|
||||
@ -2526,7 +2527,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("id1", testTenant, testUser, path.ExchangeService, path.EmailCategory, path.ContactsCategory),
|
||||
},
|
||||
[]data.BackupCollection{mce, mcc},
|
||||
@ -3436,7 +3437,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_SelectiveSubtreeP
|
||||
ctx: ctx,
|
||||
pending: map[string]*itemDetails{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: fault.New(true),
|
||||
bus: fault.New(true),
|
||||
}
|
||||
snapshotRoot, counters := getBaseSnapshot()
|
||||
msw := &mockSnapshotWalker{
|
||||
@ -3451,7 +3452,7 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTree_SelectiveSubtreeP
|
||||
dirTree, err := inflateDirTree(
|
||||
ctx,
|
||||
msw,
|
||||
[]ManifestEntry{
|
||||
[]backup.ManifestEntry{
|
||||
makeManifestEntry("", tenant, user, path.OneDriveService, path.FilesCategory),
|
||||
},
|
||||
test.inputCollections(t),
|
||||
|
||||
@ -17,12 +17,14 @@ import (
|
||||
"github.com/kopia/kopia/snapshot/snapshotmaintenance"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/diagnostics"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/internal/stats"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/control/repository"
|
||||
@ -139,12 +141,12 @@ func (w *Wrapper) Close(ctx context.Context) error {
|
||||
func (w Wrapper) ConsumeBackupCollections(
|
||||
ctx context.Context,
|
||||
backupReasons []identity.Reasoner,
|
||||
bases BackupBases,
|
||||
bases backup.BackupBases,
|
||||
collections []data.BackupCollection,
|
||||
globalExcludeSet prefixmatcher.StringSetReader,
|
||||
additionalTags map[string]string,
|
||||
buildTreeWithBase bool,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) (*BackupStats, *details.Builder, DetailsMergeInfoer, error) {
|
||||
if w.c == nil {
|
||||
return nil, nil, nil, clues.Stack(errNotConnected).WithClues(ctx)
|
||||
@ -162,15 +164,15 @@ func (w Wrapper) ConsumeBackupCollections(
|
||||
pending: map[string]*itemDetails{},
|
||||
deets: &details.Builder{},
|
||||
toMerge: newMergeDetails(),
|
||||
errs: errs,
|
||||
bus: bus,
|
||||
}
|
||||
|
||||
// When running an incremental backup, we need to pass the prior
|
||||
// snapshot bases into inflateDirTree so that the new snapshot
|
||||
// includes historical data.
|
||||
var (
|
||||
mergeBase []ManifestEntry
|
||||
assistBase []ManifestEntry
|
||||
mergeBase []backup.ManifestEntry
|
||||
assistBase []backup.ManifestEntry
|
||||
)
|
||||
|
||||
if bases != nil {
|
||||
@ -215,12 +217,12 @@ func (w Wrapper) ConsumeBackupCollections(
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
return s, progress.deets, progress.toMerge, progress.errs.Failure()
|
||||
return s, progress.deets, progress.toMerge, progress.bus.Failure()
|
||||
}
|
||||
|
||||
func (w Wrapper) makeSnapshotWithRoot(
|
||||
ctx context.Context,
|
||||
prevSnapEntries []ManifestEntry,
|
||||
prevSnapEntries []backup.ManifestEntry,
|
||||
root fs.Directory,
|
||||
addlTags map[string]string,
|
||||
progress *corsoProgress,
|
||||
@ -253,7 +255,7 @@ func (w Wrapper) makeSnapshotWithRoot(
|
||||
tags := map[string]string{}
|
||||
|
||||
for k, v := range addlTags {
|
||||
mk, mv := makeTagKV(k)
|
||||
mk, mv := backup.MakeTagKV(k)
|
||||
|
||||
if len(v) == 0 {
|
||||
v = mv
|
||||
@ -369,7 +371,7 @@ func getDir(
|
||||
encodeElements(dirPath.PopFront().Elements()...))
|
||||
if err != nil {
|
||||
if isErrEntryNotFound(err) {
|
||||
err = clues.Stack(data.ErrNotFound, err).WithClues(ctx)
|
||||
err = clues.Stack(errs.NotFound, err).WithClues(ctx)
|
||||
}
|
||||
|
||||
return nil, clues.Wrap(err, "getting nested object handle").WithClues(ctx)
|
||||
@ -474,7 +476,7 @@ func (w Wrapper) ProduceRestoreCollections(
|
||||
snapshotID string,
|
||||
paths []path.RestorePaths,
|
||||
bcounter ByteCounter,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) ([]data.RestoreCollection, error) {
|
||||
ctx, end := diagnostics.Span(ctx, "kopia:produceRestoreCollections")
|
||||
defer end()
|
||||
@ -495,7 +497,7 @@ func (w Wrapper) ProduceRestoreCollections(
|
||||
// RestorePath -> []StoragePath directory -> set of items to load from the
|
||||
// directory.
|
||||
dirsToItems = map[string]*restoreCollection{}
|
||||
el = errs.Local()
|
||||
el = bus.Local()
|
||||
)
|
||||
|
||||
for _, itemPaths := range paths {
|
||||
@ -550,7 +552,7 @@ func (w Wrapper) ProduceRestoreCollections(
|
||||
|
||||
// Now that we've grouped everything, go through and load each directory and
|
||||
// then load the items from the directory.
|
||||
res, err := loadDirsAndItems(ctx, snapshotRoot, bcounter, dirsToItems, errs)
|
||||
res, err := loadDirsAndItems(ctx, snapshotRoot, bcounter, dirsToItems, bus)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "loading items").WithClues(ctx)
|
||||
}
|
||||
|
||||
@ -21,6 +21,7 @@ import (
|
||||
"github.com/stretchr/testify/suite"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
pmMock "github.com/alcionai/corso/src/internal/common/prefixmatcher/mock"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
@ -28,6 +29,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
|
||||
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/control/repository"
|
||||
@ -811,12 +813,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
}
|
||||
|
||||
reasons := []identity.Reasoner{
|
||||
NewReason(
|
||||
identity.NewReason(
|
||||
testTenant,
|
||||
suite.storePath1.ProtectedResource(),
|
||||
suite.storePath1.Service(),
|
||||
suite.storePath1.Category()),
|
||||
NewReason(
|
||||
identity.NewReason(
|
||||
testTenant,
|
||||
suite.storePath2.ProtectedResource(),
|
||||
suite.storePath2.Service(),
|
||||
@ -837,7 +839,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
|
||||
type testCase struct {
|
||||
name string
|
||||
baseBackups func(base ManifestEntry) BackupBases
|
||||
baseBackups func(base backup.ManifestEntry) backup.BackupBases
|
||||
collections []data.BackupCollection
|
||||
expectedUploadedFiles int
|
||||
expectedCachedFiles int
|
||||
@ -862,7 +864,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
// Initial backup. All files should be considered new by kopia.
|
||||
baseBackupCase := testCase{
|
||||
name: "Uncached",
|
||||
baseBackups: func(ManifestEntry) BackupBases {
|
||||
baseBackups: func(backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases()
|
||||
},
|
||||
collections: collections,
|
||||
@ -873,8 +875,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
uploadedBytes: []int64{8000, 10000},
|
||||
}
|
||||
|
||||
runAndTestBackup := func(test testCase, base ManifestEntry) ManifestEntry {
|
||||
var res ManifestEntry
|
||||
runAndTestBackup := func(test testCase, base backup.ManifestEntry) backup.ManifestEntry {
|
||||
var res backup.ManifestEntry
|
||||
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
@ -945,7 +947,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
manifest.ID(stats.SnapshotID))
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
res = ManifestEntry{
|
||||
res = backup.ManifestEntry{
|
||||
Manifest: snap,
|
||||
Reasons: reasons,
|
||||
}
|
||||
@ -954,12 +956,12 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
return res
|
||||
}
|
||||
|
||||
base := runAndTestBackup(baseBackupCase, ManifestEntry{})
|
||||
base := runAndTestBackup(baseBackupCase, backup.ManifestEntry{})
|
||||
|
||||
table := []testCase{
|
||||
{
|
||||
name: "Kopia Assist And Merge All Files Changed",
|
||||
baseBackups: func(base ManifestEntry) BackupBases {
|
||||
baseBackups: func(base backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases().WithMergeBases(base)
|
||||
},
|
||||
collections: collections,
|
||||
@ -973,7 +975,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
},
|
||||
{
|
||||
name: "Kopia Assist And Merge No Files Changed",
|
||||
baseBackups: func(base ManifestEntry) BackupBases {
|
||||
baseBackups: func(base backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases().WithMergeBases(base)
|
||||
},
|
||||
// Pass in empty collections to force a backup. Otherwise we'll skip
|
||||
@ -995,7 +997,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
},
|
||||
{
|
||||
name: "Kopia Assist Only",
|
||||
baseBackups: func(base ManifestEntry) BackupBases {
|
||||
baseBackups: func(base backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases().WithAssistBases(base)
|
||||
},
|
||||
collections: collections,
|
||||
@ -1008,7 +1010,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
},
|
||||
{
|
||||
name: "Merge Only",
|
||||
baseBackups: func(base ManifestEntry) BackupBases {
|
||||
baseBackups: func(base backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases().WithMergeBases(base).MockDisableAssistBases()
|
||||
},
|
||||
// Pass in empty collections to force a backup. Otherwise we'll skip
|
||||
@ -1028,7 +1030,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
|
||||
},
|
||||
{
|
||||
name: "Content Hash Only",
|
||||
baseBackups: func(base ManifestEntry) BackupBases {
|
||||
baseBackups: func(base backup.ManifestEntry) backup.BackupBases {
|
||||
return NewMockBackupBases()
|
||||
},
|
||||
collections: collections,
|
||||
@ -1077,7 +1079,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
|
||||
}
|
||||
|
||||
reasons := []identity.Reasoner{
|
||||
NewReason(
|
||||
identity.NewReason(
|
||||
testTenant,
|
||||
storePath.ProtectedResource(),
|
||||
storePath.Service(),
|
||||
@ -1231,7 +1233,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
prevSnaps.WithMergeBases(
|
||||
ManifestEntry{
|
||||
backup.ManifestEntry{
|
||||
Manifest: snap,
|
||||
Reasons: reasons,
|
||||
})
|
||||
@ -1253,7 +1255,7 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
|
||||
|
||||
w := &Wrapper{k}
|
||||
|
||||
r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
r := identity.NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
|
||||
dc1 := exchMock.NewCollection(suite.storePath1, suite.locPath1, 1)
|
||||
dc2 := exchMock.NewCollection(suite.storePath2, suite.locPath2, 1)
|
||||
@ -1303,7 +1305,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
||||
|
||||
loc1 := path.Builder{}.Append(suite.storePath1.Folders()...)
|
||||
loc2 := path.Builder{}.Append(suite.storePath2.Folders()...)
|
||||
r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
r := identity.NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
|
||||
collections := []data.BackupCollection{
|
||||
&dataMock.Collection{
|
||||
@ -1350,7 +1352,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
||||
},
|
||||
}
|
||||
|
||||
errs := fault.New(true)
|
||||
bus := fault.New(true)
|
||||
|
||||
stats, deets, _, err := suite.w.ConsumeBackupCollections(
|
||||
suite.ctx,
|
||||
@ -1360,13 +1362,13 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
||||
nil,
|
||||
nil,
|
||||
true,
|
||||
errs)
|
||||
bus)
|
||||
require.Error(t, err, clues.ToCore(err))
|
||||
assert.Equal(t, 0, stats.ErrorCount, "error count")
|
||||
assert.Equal(t, 5, stats.TotalFileCount, "total files")
|
||||
assert.Equal(t, 6, stats.TotalDirectoryCount, "total directories")
|
||||
assert.Equal(t, 0, stats.IgnoredErrorCount, "ignored errors")
|
||||
assert.Equal(t, 1, len(errs.Errors().Recovered), "recovered errors")
|
||||
assert.Equal(t, 1, len(bus.Errors().Recovered), "recovered errors")
|
||||
assert.False(t, stats.Incomplete, "incomplete")
|
||||
// 5 file and 2 folder entries.
|
||||
assert.Len(t, deets.Details().Entries, 5+2)
|
||||
@ -1386,8 +1388,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
||||
|
||||
require.Len(t, dcs, 1, "number of restore collections")
|
||||
|
||||
errs = fault.New(true)
|
||||
items := dcs[0].Items(suite.ctx, errs)
|
||||
bus = fault.New(true)
|
||||
items := dcs[0].Items(suite.ctx, bus)
|
||||
|
||||
// Get all the items from channel
|
||||
//nolint:revive
|
||||
@ -1397,7 +1399,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
|
||||
// Files that had an error shouldn't make a dir entry in kopia. If they do we
|
||||
// may run into kopia-assisted incrementals issues because only mod time and
|
||||
// not file size is checked for StreamingFiles.
|
||||
assert.ErrorIs(t, errs.Failure(), data.ErrNotFound, "errored file is restorable", clues.ToCore(err))
|
||||
assert.ErrorIs(t, bus.Failure(), errs.NotFound, "errored file is restorable", clues.ToCore(err))
|
||||
}
|
||||
|
||||
type backedupFile struct {
|
||||
@ -1585,7 +1587,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
|
||||
dataMock.NewVersionedBackupCollection(t, collection))
|
||||
}
|
||||
|
||||
r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
r := identity.NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
|
||||
// Other tests check basic things about deets so not doing that again here.
|
||||
stats, _, _, err := suite.w.ConsumeBackupCollections(
|
||||
@ -1622,7 +1624,7 @@ func (c *i64counter) Count(i int64) {
|
||||
}
|
||||
|
||||
func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
|
||||
r := NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
r := identity.NewReason(testTenant, testUser, path.ExchangeService, path.EmailCategory)
|
||||
|
||||
man, err := suite.w.c.LoadSnapshot(suite.ctx, suite.snapshotID)
|
||||
require.NoError(suite.T(), err, "getting base snapshot: %v", clues.ToCore(err))
|
||||
@ -1719,7 +1721,7 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
|
||||
suite.ctx,
|
||||
[]identity.Reasoner{r},
|
||||
NewMockBackupBases().WithMergeBases(
|
||||
ManifestEntry{
|
||||
backup.ManifestEntry{
|
||||
Manifest: man,
|
||||
Reasons: []identity.Reasoner{r},
|
||||
}),
|
||||
@ -1750,15 +1752,15 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
|
||||
assert.NoError(t, err, "errors producing collection", clues.ToCore(err))
|
||||
require.Len(t, dcs, 1, "unexpected number of restore collections")
|
||||
|
||||
errs := fault.New(true)
|
||||
items := dcs[0].Items(suite.ctx, errs)
|
||||
bus := fault.New(true)
|
||||
items := dcs[0].Items(suite.ctx, bus)
|
||||
|
||||
// Get all the items from channel
|
||||
//nolint:revive
|
||||
for range items {
|
||||
}
|
||||
|
||||
test.restoreCheck(t, errs.Failure(), errs)
|
||||
test.restoreCheck(t, bus.Failure(), bus)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1871,19 +1873,19 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() {
|
||||
return
|
||||
}
|
||||
|
||||
errs := fault.New(true)
|
||||
bus := fault.New(true)
|
||||
|
||||
for _, dc := range result {
|
||||
// Get all the items from channel
|
||||
items := dc.Items(suite.ctx, errs)
|
||||
items := dc.Items(suite.ctx, bus)
|
||||
//nolint:revive
|
||||
for range items {
|
||||
}
|
||||
}
|
||||
|
||||
test.expectedErr(t, errs.Failure(), errs.Failure(), "getting items")
|
||||
test.expectedErr(t, bus.Failure(), bus.Failure(), "getting items")
|
||||
|
||||
if errs.Failure() != nil {
|
||||
if bus.Failure() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@ -8,7 +8,6 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/diagnostics"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
||||
"github.com/alcionai/corso/src/internal/m365/graph"
|
||||
"github.com/alcionai/corso/src/internal/m365/service/exchange"
|
||||
@ -16,6 +15,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/m365/service/onedrive"
|
||||
"github.com/alcionai/corso/src/internal/m365/service/sharepoint"
|
||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
bupMD "github.com/alcionai/corso/src/pkg/backup/metadata"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/filters"
|
||||
@ -176,7 +176,7 @@ func verifyBackupInputs(sels selectors.Selector, cachedIDs []string) error {
|
||||
func (ctrl *Controller) GetMetadataPaths(
|
||||
ctx context.Context,
|
||||
r kinject.RestoreProducer,
|
||||
man kopia.ManifestEntry,
|
||||
man backup.ManifestEntry,
|
||||
errs *fault.Bus,
|
||||
) ([]path.RestorePaths, error) {
|
||||
var (
|
||||
|
||||
@ -16,6 +16,7 @@ import (
|
||||
"golang.org/x/exp/maps"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||
"github.com/alcionai/corso/src/internal/common/readers"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
@ -276,7 +277,7 @@ func (suite *CollectionUnitSuite) TestPrefetchCollection_Items() {
|
||||
suite.Run(test.name, func() {
|
||||
var (
|
||||
t = suite.T()
|
||||
errs = fault.New(true)
|
||||
bus = fault.New(true)
|
||||
itemCount int
|
||||
)
|
||||
|
||||
@ -297,7 +298,7 @@ func (suite *CollectionUnitSuite) TestPrefetchCollection_Items() {
|
||||
false,
|
||||
statusUpdater)
|
||||
|
||||
for item := range col.Items(ctx, errs) {
|
||||
for item := range col.Items(ctx, bus) {
|
||||
itemCount++
|
||||
|
||||
_, rok := test.removed[item.ID()]
|
||||
@ -316,7 +317,7 @@ func (suite *CollectionUnitSuite) TestPrefetchCollection_Items() {
|
||||
assert.True(t, aok || rok, "item must be either added or removed: %q", item.ID())
|
||||
}
|
||||
|
||||
assert.NoError(t, errs.Failure())
|
||||
assert.NoError(t, bus.Failure())
|
||||
assert.Equal(
|
||||
t,
|
||||
test.expectItemCount,
|
||||
@ -336,10 +337,10 @@ func (mlg *mockLazyItemGetterSerializer) GetItem(
|
||||
user string,
|
||||
itemID string,
|
||||
immutableIDs bool,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) (serialization.Parsable, *details.ExchangeInfo, error) {
|
||||
mlg.callIDs = append(mlg.callIDs, itemID)
|
||||
return mlg.ItemGetSerialize.GetItem(ctx, user, itemID, immutableIDs, errs)
|
||||
return mlg.ItemGetSerialize.GetItem(ctx, user, itemID, immutableIDs, bus)
|
||||
}
|
||||
|
||||
func (mlg *mockLazyItemGetterSerializer) check(t *testing.T, expectIDs []string) {
|
||||
@ -409,7 +410,7 @@ func (suite *CollectionUnitSuite) TestLazyFetchCollection_Items_LazyFetch() {
|
||||
suite.Run(test.name, func() {
|
||||
var (
|
||||
t = suite.T()
|
||||
errs = fault.New(true)
|
||||
bus = fault.New(true)
|
||||
itemCount int
|
||||
)
|
||||
|
||||
@ -435,7 +436,7 @@ func (suite *CollectionUnitSuite) TestLazyFetchCollection_Items_LazyFetch() {
|
||||
true,
|
||||
statusUpdater)
|
||||
|
||||
for item := range col.Items(ctx, errs) {
|
||||
for item := range col.Items(ctx, bus) {
|
||||
itemCount++
|
||||
|
||||
_, rok := test.removed[item.ID()]
|
||||
@ -478,7 +479,7 @@ func (suite *CollectionUnitSuite) TestLazyFetchCollection_Items_LazyFetch() {
|
||||
assert.True(t, aok || rok, "item must be either added or removed: %q", item.ID())
|
||||
}
|
||||
|
||||
assert.NoError(t, errs.Failure())
|
||||
assert.NoError(t, bus.Failure())
|
||||
assert.Equal(
|
||||
t,
|
||||
test.expectItemCount,
|
||||
@ -625,7 +626,7 @@ func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlig
|
||||
assert.Empty(t, readData, "read item data")
|
||||
|
||||
_, err = li.Info()
|
||||
assert.ErrorIs(t, err, data.ErrNotFound, "Info() error")
|
||||
assert.ErrorIs(t, err, errs.NotFound, "Info() error")
|
||||
}
|
||||
|
||||
func (suite *CollectionUnitSuite) TestLazyItem() {
|
||||
|
||||
@ -8,9 +8,9 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/count"
|
||||
@ -53,7 +53,7 @@ func (ctrl Controller) ProduceBackupCollections(
|
||||
func (ctrl *Controller) GetMetadataPaths(
|
||||
ctx context.Context,
|
||||
r kinject.RestoreProducer,
|
||||
man kopia.ManifestEntry,
|
||||
man backup.ManifestEntry,
|
||||
errs *fault.Bus,
|
||||
) ([]path.RestorePaths, error) {
|
||||
return nil, clues.New("not implemented")
|
||||
|
||||
@ -85,14 +85,14 @@ func (suite *GroupsBackupUnitSuite) TestMetadataFiles() {
|
||||
}{
|
||||
{
|
||||
name: "error",
|
||||
reason: kopia.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
reason: identity.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
manID: "manifestID",
|
||||
r: mockRestoreProducer{err: assert.AnError},
|
||||
expectErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "single site",
|
||||
reason: kopia.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
reason: identity.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
manID: "manifestID",
|
||||
r: mockRestoreProducer{
|
||||
rc: []data.RestoreCollection{
|
||||
@ -108,7 +108,7 @@ func (suite *GroupsBackupUnitSuite) TestMetadataFiles() {
|
||||
},
|
||||
{
|
||||
name: "multiple sites",
|
||||
reason: kopia.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
reason: identity.NewReason("tenant", "user", path.GroupsService, path.LibrariesCategory),
|
||||
manID: "manifestID",
|
||||
r: mockRestoreProducer{
|
||||
rc: []data.RestoreCollection{
|
||||
|
||||
@ -278,7 +278,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
|
||||
|
||||
observe.Message(ctx, "Backing Up", observe.Bullet, clues.Hide(op.ResourceOwner.Name()))
|
||||
|
||||
deets, err := op.do(
|
||||
reasons, bases, deets, err := op.do(
|
||||
ctx,
|
||||
&opStats,
|
||||
sstore,
|
||||
@ -304,6 +304,8 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
|
||||
err = op.createBackupModels(
|
||||
ctx,
|
||||
sstore,
|
||||
reasons,
|
||||
bases,
|
||||
opStats,
|
||||
op.Results.BackupID,
|
||||
op.BackupVersion,
|
||||
@ -329,17 +331,17 @@ func (op *BackupOperation) do(
|
||||
opStats *backupStats,
|
||||
detailsStore streamstore.Streamer,
|
||||
backupID model.StableID,
|
||||
) (*details.Builder, error) {
|
||||
) ([]identity.Reasoner, backup.BackupBases, *details.Builder, error) {
|
||||
lastBackupVersion := version.NoBackup
|
||||
|
||||
reasons, err := op.Selectors.Reasons(op.account.ID(), false)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "getting reasons")
|
||||
return nil, nil, nil, clues.Wrap(err, "getting reasons")
|
||||
}
|
||||
|
||||
fallbackReasons, err := makeFallbackReasons(op.account.ID(), op.Selectors)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "getting fallback reasons")
|
||||
return nil, nil, nil, clues.Wrap(err, "getting fallback reasons")
|
||||
}
|
||||
|
||||
logger.Ctx(ctx).With(
|
||||
@ -352,10 +354,10 @@ func (op *BackupOperation) do(
|
||||
|
||||
kbf, err := op.kopia.NewBaseFinder(op.store)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
return nil, nil, nil, clues.Stack(err)
|
||||
}
|
||||
|
||||
mans, mdColls, canUseMetadata, err := produceManifestsAndMetadata(
|
||||
bases, mdColls, canUseMetadata, err := produceManifestsAndMetadata(
|
||||
ctx,
|
||||
kbf,
|
||||
op.bp,
|
||||
@ -365,17 +367,17 @@ func (op *BackupOperation) do(
|
||||
op.incremental,
|
||||
op.disableAssistBackup)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "producing manifests and metadata")
|
||||
return nil, nil, nil, clues.Wrap(err, "producing manifests and metadata")
|
||||
}
|
||||
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"can_use_metadata", canUseMetadata,
|
||||
"assist_bases", len(mans.UniqueAssistBases()),
|
||||
"merge_bases", len(mans.MergeBases()))
|
||||
"assist_bases", len(bases.UniqueAssistBases()),
|
||||
"merge_bases", len(bases.MergeBases()))
|
||||
|
||||
if canUseMetadata {
|
||||
lastBackupVersion = mans.MinBackupVersion()
|
||||
lastBackupVersion = bases.MinBackupVersion()
|
||||
}
|
||||
|
||||
// TODO(ashmrtn): This should probably just return a collection that deletes
|
||||
@ -392,7 +394,7 @@ func (op *BackupOperation) do(
|
||||
op.Options,
|
||||
op.Errors)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "producing backup data collections")
|
||||
return nil, nil, nil, clues.Wrap(err, "producing backup data collections")
|
||||
}
|
||||
|
||||
ctx = clues.Add(
|
||||
@ -405,14 +407,14 @@ func (op *BackupOperation) do(
|
||||
op.kopia,
|
||||
op.account.ID(),
|
||||
reasons,
|
||||
mans,
|
||||
bases,
|
||||
cs,
|
||||
ssmb,
|
||||
backupID,
|
||||
op.incremental && canUseMetadata && canUsePreviousBackup,
|
||||
op.Errors)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "persisting collection backups")
|
||||
return nil, nil, nil, clues.Wrap(err, "persisting collection backups")
|
||||
}
|
||||
|
||||
opStats.hasNewDetailEntries = (deets != nil && !deets.Empty()) ||
|
||||
@ -422,21 +424,21 @@ func (op *BackupOperation) do(
|
||||
err = mergeDetails(
|
||||
ctx,
|
||||
detailsStore,
|
||||
mans,
|
||||
bases,
|
||||
toMerge,
|
||||
deets,
|
||||
writeStats,
|
||||
op.Selectors.PathService(),
|
||||
op.Errors)
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "merging details")
|
||||
return nil, nil, nil, clues.Wrap(err, "merging details")
|
||||
}
|
||||
|
||||
opStats.ctrl = op.bp.Wait()
|
||||
|
||||
logger.Ctx(ctx).Debug(opStats.ctrl)
|
||||
|
||||
return deets, nil
|
||||
return reasons, bases, deets, nil
|
||||
}
|
||||
|
||||
func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) {
|
||||
@ -494,7 +496,7 @@ func consumeBackupCollections(
|
||||
bc kinject.BackupConsumer,
|
||||
tenantID string,
|
||||
reasons []identity.Reasoner,
|
||||
bbs kopia.BackupBases,
|
||||
bbs backup.BackupBases,
|
||||
cs []data.BackupCollection,
|
||||
pmr prefixmatcher.StringSetReader,
|
||||
backupID model.StableID,
|
||||
@ -598,7 +600,7 @@ func getNewPathRefs(
|
||||
func mergeItemsFromBase(
|
||||
ctx context.Context,
|
||||
checkReason bool,
|
||||
baseBackup kopia.BackupEntry,
|
||||
baseBackup backup.BackupEntry,
|
||||
detailsStore streamstore.Streamer,
|
||||
dataFromBackup kopia.DetailsMergeInfoer,
|
||||
deets *details.Builder,
|
||||
@ -702,7 +704,7 @@ func mergeItemsFromBase(
|
||||
func mergeDetails(
|
||||
ctx context.Context,
|
||||
detailsStore streamstore.Streamer,
|
||||
bases kopia.BackupBases,
|
||||
bases backup.BackupBases,
|
||||
dataFromBackup kopia.DetailsMergeInfoer,
|
||||
deets *details.Builder,
|
||||
writeStats *kopia.BackupStats,
|
||||
@ -843,6 +845,8 @@ func (op *BackupOperation) persistResults(
|
||||
func (op *BackupOperation) createBackupModels(
|
||||
ctx context.Context,
|
||||
sscw streamstore.CollectorWriter,
|
||||
reasons []identity.Reasoner,
|
||||
bases backup.BackupBases,
|
||||
opStats backupStats,
|
||||
backupID model.StableID,
|
||||
backupVersion int,
|
||||
@ -928,6 +932,8 @@ func (op *BackupOperation) createBackupModels(
|
||||
op.ResourceOwner.Name(),
|
||||
op.Results.ReadWrites,
|
||||
op.Results.StartAndEndTime,
|
||||
reasons,
|
||||
bases,
|
||||
op.Errors.Errors(),
|
||||
tags)
|
||||
|
||||
|
||||
@ -121,7 +121,7 @@ func checkPaths(t *testing.T, expected, got []path.Path) {
|
||||
type mockBackupConsumer struct {
|
||||
checkFunc func(
|
||||
backupReasons []identity.Reasoner,
|
||||
bases kopia.BackupBases,
|
||||
bases backup.BackupBases,
|
||||
cs []data.BackupCollection,
|
||||
tags map[string]string,
|
||||
buildTreeWithBase bool)
|
||||
@ -130,7 +130,7 @@ type mockBackupConsumer struct {
|
||||
func (mbu mockBackupConsumer) ConsumeBackupCollections(
|
||||
ctx context.Context,
|
||||
backupReasons []identity.Reasoner,
|
||||
bases kopia.BackupBases,
|
||||
bases backup.BackupBases,
|
||||
cs []data.BackupCollection,
|
||||
excluded prefixmatcher.StringSetReader,
|
||||
tags map[string]string,
|
||||
@ -453,12 +453,12 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
|
||||
tenant = "a-tenant"
|
||||
resourceOwner = "a-user"
|
||||
|
||||
emailReason = kopia.NewReason(
|
||||
emailReason = identity.NewReason(
|
||||
tenant,
|
||||
resourceOwner,
|
||||
path.ExchangeService,
|
||||
path.EmailCategory)
|
||||
contactsReason = kopia.NewReason(
|
||||
contactsReason = identity.NewReason(
|
||||
tenant,
|
||||
resourceOwner,
|
||||
path.ExchangeService,
|
||||
@ -477,13 +477,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
|
||||
}
|
||||
|
||||
bases = kopia.NewMockBackupBases().WithMergeBases(
|
||||
kopia.ManifestEntry{
|
||||
backup.ManifestEntry{
|
||||
Manifest: manifest1,
|
||||
Reasons: []identity.Reasoner{
|
||||
emailReason,
|
||||
},
|
||||
}).WithAssistBases(
|
||||
kopia.ManifestEntry{
|
||||
backup.ManifestEntry{
|
||||
Manifest: manifest2,
|
||||
Reasons: []identity.Reasoner{
|
||||
contactsReason,
|
||||
@ -500,7 +500,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
|
||||
mbu := &mockBackupConsumer{
|
||||
checkFunc: func(
|
||||
backupReasons []identity.Reasoner,
|
||||
gotBases kopia.BackupBases,
|
||||
gotBases backup.BackupBases,
|
||||
cs []data.BackupCollection,
|
||||
gotTags map[string]string,
|
||||
buildTreeWithBase bool,
|
||||
@ -590,12 +590,12 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
DetailsID: "did2",
|
||||
}
|
||||
|
||||
pathReason1 = kopia.NewReason(
|
||||
pathReason1 = identity.NewReason(
|
||||
"",
|
||||
itemPath1.ProtectedResource(),
|
||||
itemPath1.Service(),
|
||||
itemPath1.Category())
|
||||
pathReason3 = kopia.NewReason(
|
||||
pathReason3 = identity.NewReason(
|
||||
"",
|
||||
itemPath3.ProtectedResource(),
|
||||
itemPath3.Service(),
|
||||
@ -616,7 +616,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
},
|
||||
true)
|
||||
exchangeLocationPath1 = path.Builder{}.Append("work-display-name")
|
||||
exchangePathReason1 = kopia.NewReason(
|
||||
exchangePathReason1 = identity.NewReason(
|
||||
"",
|
||||
exchangeItemPath1.ProtectedResource(),
|
||||
exchangeItemPath1.Service(),
|
||||
@ -631,8 +631,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
table := []struct {
|
||||
name string
|
||||
populatedDetails map[string]*details.Details
|
||||
inputBackups []kopia.BackupEntry
|
||||
inputAssistBackups []kopia.BackupEntry
|
||||
inputBackups []backup.BackupEntry
|
||||
inputAssistBackups []backup.BackupEntry
|
||||
mdm *mockDetailsMergeInfoer
|
||||
|
||||
errCheck assert.ErrorAssertionFunc
|
||||
@ -659,7 +659,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
@ -683,7 +683,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -710,7 +710,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -766,7 +766,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -793,7 +793,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -823,7 +823,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -853,7 +853,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -883,7 +883,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -914,7 +914,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -945,7 +945,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -993,7 +993,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -1002,7 +1002,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
},
|
||||
},
|
||||
},
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup2},
|
||||
},
|
||||
populatedDetails: map[string]*details.Details{
|
||||
@ -1037,7 +1037,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputBackups: []kopia.BackupEntry{
|
||||
inputBackups: []backup.BackupEntry{
|
||||
{
|
||||
Backup: &backup1,
|
||||
Reasons: []identity.Reasoner{
|
||||
@ -1045,7 +1045,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
},
|
||||
},
|
||||
},
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup2},
|
||||
},
|
||||
populatedDetails: map[string]*details.Details{
|
||||
@ -1077,7 +1077,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup1},
|
||||
{Backup: &backup2},
|
||||
},
|
||||
@ -1110,7 +1110,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup1},
|
||||
{Backup: &backup2},
|
||||
},
|
||||
@ -1143,7 +1143,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
|
||||
return res
|
||||
}(),
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup1},
|
||||
{Backup: &backup2},
|
||||
},
|
||||
@ -1173,7 +1173,7 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
|
||||
mdm: func() *mockDetailsMergeInfoer {
|
||||
return newMockDetailsMergeInfoer()
|
||||
}(),
|
||||
inputAssistBackups: []kopia.BackupEntry{
|
||||
inputAssistBackups: []backup.BackupEntry{
|
||||
{Backup: &backup1},
|
||||
},
|
||||
populatedDetails: map[string]*details.Details{
|
||||
@ -1267,13 +1267,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde
|
||||
|
||||
locPath1 = path.Builder{}.Append(itemPath1.Folders()...)
|
||||
|
||||
pathReason1 = kopia.NewReason(
|
||||
pathReason1 = identity.NewReason(
|
||||
"",
|
||||
itemPath1.ProtectedResource(),
|
||||
itemPath1.Service(),
|
||||
itemPath1.Category())
|
||||
|
||||
backup1 = kopia.BackupEntry{
|
||||
backup1 = backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid1",
|
||||
|
||||
@ -6,8 +6,8 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/kopia/inject"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/control/repository"
|
||||
@ -37,7 +37,7 @@ type (
|
||||
GetMetadataPaths(
|
||||
ctx context.Context,
|
||||
r inject.RestoreProducer,
|
||||
man kopia.ManifestEntry,
|
||||
man backup.ManifestEntry,
|
||||
errs *fault.Bus,
|
||||
) ([]path.RestorePaths, error)
|
||||
|
||||
|
||||
@ -7,10 +7,10 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/prefixmatcher"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
kinject "github.com/alcionai/corso/src/internal/kopia/inject"
|
||||
"github.com/alcionai/corso/src/internal/m365"
|
||||
"github.com/alcionai/corso/src/internal/operations/inject"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
@ -62,7 +62,7 @@ func (mbp *mockBackupProducer) Wait() *data.CollectionStats {
|
||||
func (mbp mockBackupProducer) GetMetadataPaths(
|
||||
ctx context.Context,
|
||||
r kinject.RestoreProducer,
|
||||
man kopia.ManifestEntry,
|
||||
man backup.ManifestEntry,
|
||||
errs *fault.Bus,
|
||||
) ([]path.RestorePaths, error) {
|
||||
ctrl := m365.Controller{}
|
||||
|
||||
@ -6,10 +6,12 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/kopia/inject"
|
||||
oinject "github.com/alcionai/corso/src/internal/operations/inject"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
@ -23,7 +25,7 @@ func produceManifestsAndMetadata(
|
||||
reasons, fallbackReasons []identity.Reasoner,
|
||||
tenantID string,
|
||||
getMetadata, dropAssistBases bool,
|
||||
) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
|
||||
) (backup.BackupBases, []data.RestoreCollection, bool, error) {
|
||||
bb, meta, useMergeBases, err := getManifestsAndMetadata(
|
||||
ctx,
|
||||
bf,
|
||||
@ -62,7 +64,7 @@ func getManifestsAndMetadata(
|
||||
reasons, fallbackReasons []identity.Reasoner,
|
||||
tenantID string,
|
||||
getMetadata bool,
|
||||
) (kopia.BackupBases, []data.RestoreCollection, bool, error) {
|
||||
) (backup.BackupBases, []data.RestoreCollection, bool, error) {
|
||||
var (
|
||||
tags = map[string]string{kopia.TagBackupCategory: ""}
|
||||
collections []data.RestoreCollection
|
||||
@ -127,7 +129,7 @@ func getManifestsAndMetadata(
|
||||
// should be safe to leave this manifest in the AssistBases set, though we
|
||||
// could remove it there too if we want to be conservative. That can be done
|
||||
// by finding the manifest ID.
|
||||
if err != nil && !errors.Is(err, data.ErrNotFound) {
|
||||
if err != nil && !errors.Is(err, errs.NotFound) {
|
||||
// prior metadata isn't guaranteed to exist.
|
||||
// if it doesn't, we'll just have to do a
|
||||
// full backup for that data.
|
||||
|
||||
@ -50,14 +50,14 @@ type mockBackupFinder struct {
|
||||
// beyond that and results are returned for the union of the reasons anyway.
|
||||
// This does assume that the return data is properly constructed to return a
|
||||
// union of the reasons etc.
|
||||
data map[string]kopia.BackupBases
|
||||
data map[string]backup.BackupBases
|
||||
}
|
||||
|
||||
func (bf *mockBackupFinder) FindBases(
|
||||
_ context.Context,
|
||||
reasons []identity.Reasoner,
|
||||
_ map[string]string,
|
||||
) kopia.BackupBases {
|
||||
) backup.BackupBases {
|
||||
if len(reasons) == 0 {
|
||||
return kopia.NewMockBackupBases()
|
||||
}
|
||||
@ -163,7 +163,7 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
name: "single reason",
|
||||
manID: "single",
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason(tid, ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
preFetchPaths: []string{},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
@ -183,8 +183,8 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
name: "multiple reasons",
|
||||
manID: "multi",
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason(tid, ro, path.ExchangeService, path.EmailCategory),
|
||||
kopia.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason(tid, ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason(tid, ro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
preFetchPaths: []string{},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
@ -209,7 +209,7 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
name: "single reason sp libraries",
|
||||
manID: "single-sp-libraries",
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason(tid, ro, path.SharePointService, path.LibrariesCategory),
|
||||
identity.NewReason(tid, ro, path.SharePointService, path.LibrariesCategory),
|
||||
},
|
||||
preFetchPaths: []string{},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
@ -229,7 +229,7 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
name: "single reason groups messages",
|
||||
manID: "single-groups-messages",
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason(tid, ro, path.GroupsService, path.ChannelMessagesCategory),
|
||||
identity.NewReason(tid, ro, path.GroupsService, path.ChannelMessagesCategory),
|
||||
},
|
||||
preFetchPaths: []string{},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
@ -249,7 +249,7 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
name: "single reason groups libraries",
|
||||
manID: "single-groups-libraries",
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason(tid, ro, path.GroupsService, path.LibrariesCategory),
|
||||
identity.NewReason(tid, ro, path.GroupsService, path.LibrariesCategory),
|
||||
},
|
||||
preFetchPaths: []string{"previouspath"},
|
||||
expectPaths: func(t *testing.T, files []string) []path.Path {
|
||||
@ -290,7 +290,7 @@ func (suite *OperationsManifestsUnitSuite) TestGetMetadataPaths() {
|
||||
mr := mockRestoreProducer{err: test.expectErr, colls: test.preFetchCollection}
|
||||
mr.buildRestoreFunc(t, test.manID, paths)
|
||||
|
||||
man := kopia.ManifestEntry{
|
||||
man := backup.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{ID: manifest.ID(test.manID)},
|
||||
Reasons: test.reasons,
|
||||
}
|
||||
@ -314,7 +314,7 @@ func buildReasons(
|
||||
for _, cat := range cats {
|
||||
reasons = append(
|
||||
reasons,
|
||||
kopia.NewReason(tenant, ro, service, cat))
|
||||
identity.NewReason(tenant, ro, service, cat))
|
||||
}
|
||||
|
||||
return reasons
|
||||
@ -327,8 +327,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
did = "detailsid"
|
||||
)
|
||||
|
||||
makeMan := func(id, incmpl string, cats ...path.CategoryType) kopia.ManifestEntry {
|
||||
return kopia.ManifestEntry{
|
||||
makeMan := func(id, incmpl string, cats ...path.CategoryType) backup.ManifestEntry {
|
||||
return backup.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
IncompleteReason: incmpl,
|
||||
@ -337,8 +337,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
}
|
||||
}
|
||||
|
||||
makeBackup := func(snapID string, cats ...path.CategoryType) kopia.BackupEntry {
|
||||
return kopia.BackupEntry{
|
||||
makeBackup := func(snapID string, cats ...path.CategoryType) backup.BackupEntry {
|
||||
return backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: model.StableID(snapID + "bup"),
|
||||
@ -361,7 +361,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
assertB assert.BoolAssertionFunc
|
||||
expectDCS []mockColl
|
||||
expectPaths func(t *testing.T, gotPaths []path.Path)
|
||||
expectMans kopia.BackupBases
|
||||
expectMans backup.BackupBases
|
||||
}{
|
||||
{
|
||||
name: "don't get metadata, no mans",
|
||||
@ -376,7 +376,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "don't get metadata",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
|
||||
WithBackups(makeBackup("id1", path.EmailCategory)),
|
||||
@ -384,7 +384,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
},
|
||||
rp: mockRestoreProducer{},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: false,
|
||||
assertErr: assert.NoError,
|
||||
@ -398,14 +398,14 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "don't get metadata, incomplete manifest",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
makeMan("id1", "checkpoint", path.EmailCategory)),
|
||||
},
|
||||
},
|
||||
rp: mockRestoreProducer{},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -420,7 +420,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "one valid man, multiple reasons",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan("id1", "", path.EmailCategory, path.ContactsCategory)),
|
||||
},
|
||||
@ -431,8 +431,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
},
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -462,7 +462,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "one valid man, extra incomplete man",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
|
||||
WithAssistBases(makeMan("id2", "checkpoint", path.EmailCategory)),
|
||||
@ -475,7 +475,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
},
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -488,7 +488,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "one valid man, extra incomplete man, drop assist bases",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan("id1", "", path.EmailCategory)).
|
||||
WithAssistBases(makeMan("id2", "checkpoint", path.EmailCategory)),
|
||||
@ -501,7 +501,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
},
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
dropAssist: true,
|
||||
@ -515,7 +515,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "multiple valid mans",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan("id1", "", path.EmailCategory),
|
||||
makeMan("id2", "", path.EmailCategory)),
|
||||
@ -528,7 +528,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
},
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -541,14 +541,14 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
|
||||
{
|
||||
name: "error collecting metadata",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan("id1", "", path.EmailCategory)),
|
||||
},
|
||||
},
|
||||
rp: mockRestoreProducer{err: assert.AnError},
|
||||
reasons: []identity.Reasoner{
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.EmailCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.Error,
|
||||
@ -628,8 +628,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
did = "detailsid"
|
||||
)
|
||||
|
||||
makeMan := func(ro, id, incmpl string, cats ...path.CategoryType) kopia.ManifestEntry {
|
||||
return kopia.ManifestEntry{
|
||||
makeMan := func(ro, id, incmpl string, cats ...path.CategoryType) backup.ManifestEntry {
|
||||
return backup.ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
IncompleteReason: incmpl,
|
||||
@ -639,8 +639,8 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
}
|
||||
}
|
||||
|
||||
makeBackup := func(ro, snapID string, cats ...path.CategoryType) kopia.BackupEntry {
|
||||
return kopia.BackupEntry{
|
||||
makeBackup := func(ro, snapID string, cats ...path.CategoryType) backup.BackupEntry {
|
||||
return backup.BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: model.StableID(snapID + "bup"),
|
||||
@ -652,13 +652,13 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
}
|
||||
}
|
||||
|
||||
emailReason := kopia.NewReason(
|
||||
emailReason := identity.NewReason(
|
||||
"",
|
||||
ro,
|
||||
path.ExchangeService,
|
||||
path.EmailCategory)
|
||||
|
||||
fbEmailReason := kopia.NewReason(
|
||||
fbEmailReason := identity.NewReason(
|
||||
"",
|
||||
fbro,
|
||||
path.ExchangeService,
|
||||
@ -675,12 +675,12 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
assertErr assert.ErrorAssertionFunc
|
||||
assertB assert.BoolAssertionFunc
|
||||
expectDCS []mockColl
|
||||
expectMans kopia.BackupBases
|
||||
expectMans backup.BackupBases
|
||||
}{
|
||||
{
|
||||
name: "don't get metadata, only fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
|
||||
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
|
||||
@ -700,7 +700,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "only fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
fbro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan(fbro, "fb_id1", "", path.EmailCategory)).WithBackups(
|
||||
makeBackup(fbro, "fb_id1", path.EmailCategory)),
|
||||
@ -723,7 +723,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "only fallbacks, drop assist",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan(fbro, "fb_id1", "", path.EmailCategory)).
|
||||
WithBackups(makeBackup(fbro, "fb_id1", path.EmailCategory)),
|
||||
@ -748,7 +748,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete mans and fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan(ro, "id1", "", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -774,7 +774,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "incomplete mans and fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
@ -799,7 +799,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete and incomplete mans and fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().
|
||||
WithMergeBases(makeMan(ro, "id1", "", path.EmailCategory)).
|
||||
WithAssistBases(makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
|
||||
@ -830,7 +830,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "incomplete mans and complete fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -858,7 +858,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "incomplete mans and complete fallbacks, no assist bases",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
makeMan(ro, "id2", "checkpoint", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -887,7 +887,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete mans and incomplete fallbacks",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan(ro, "id1", "", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().WithAssistBases(
|
||||
@ -912,7 +912,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete mans and complete fallbacks, multiple reasons",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan(ro, "id1", "", path.EmailCategory, path.ContactsCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -928,11 +928,11 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
emailReason,
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
fallbackReasons: []identity.Reasoner{
|
||||
fbEmailReason,
|
||||
kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -944,7 +944,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete mans and complete fallbacks, distinct reasons",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan(ro, "id1", "", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -960,7 +960,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
},
|
||||
reasons: []identity.Reasoner{emailReason},
|
||||
fallbackReasons: []identity.Reasoner{
|
||||
kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
@ -975,7 +975,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
{
|
||||
name: "complete mans and complete fallbacks, fallback has superset of reasons",
|
||||
bf: &mockBackupFinder{
|
||||
data: map[string]kopia.BackupBases{
|
||||
data: map[string]backup.BackupBases{
|
||||
ro: kopia.NewMockBackupBases().WithMergeBases(
|
||||
makeMan(ro, "id1", "", path.EmailCategory)),
|
||||
fbro: kopia.NewMockBackupBases().
|
||||
@ -993,11 +993,11 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata_Fallb
|
||||
},
|
||||
reasons: []identity.Reasoner{
|
||||
emailReason,
|
||||
kopia.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", ro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
fallbackReasons: []identity.Reasoner{
|
||||
fbEmailReason,
|
||||
kopia.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
identity.NewReason("", fbro, path.ExchangeService, path.ContactsCategory),
|
||||
},
|
||||
getMeta: true,
|
||||
assertErr: assert.NoError,
|
||||
|
||||
@ -76,28 +76,28 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
|
||||
category path.CategoryType
|
||||
metadataFiles [][]string
|
||||
}{
|
||||
// {
|
||||
// name: "Mail",
|
||||
// selector: func() *selectors.ExchangeBackup {
|
||||
// sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
||||
// sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
||||
// sel.DiscreteOwner = suite.its.user.ID
|
||||
{
|
||||
name: "Mail",
|
||||
selector: func() *selectors.ExchangeBackup {
|
||||
sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
||||
sel.Include(sel.MailFolders([]string{api.MailInbox}, selectors.PrefixMatch()))
|
||||
sel.DiscreteOwner = suite.its.user.ID
|
||||
|
||||
// return sel
|
||||
// },
|
||||
// category: path.EmailCategory,
|
||||
// metadataFiles: exchange.MetadataFileNames(path.EmailCategory),
|
||||
// },
|
||||
// {
|
||||
// name: "Contacts",
|
||||
// selector: func() *selectors.ExchangeBackup {
|
||||
// sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
||||
// sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()))
|
||||
// return sel
|
||||
// },
|
||||
// category: path.ContactsCategory,
|
||||
// metadataFiles: exchange.MetadataFileNames(path.ContactsCategory),
|
||||
// },
|
||||
return sel
|
||||
},
|
||||
category: path.EmailCategory,
|
||||
metadataFiles: MetadataFileNames(path.EmailCategory),
|
||||
},
|
||||
{
|
||||
name: "Contacts",
|
||||
selector: func() *selectors.ExchangeBackup {
|
||||
sel := selectors.NewExchangeBackup([]string{suite.its.user.ID})
|
||||
sel.Include(sel.ContactFolders([]string{api.DefaultContacts}, selectors.PrefixMatch()))
|
||||
return sel
|
||||
},
|
||||
category: path.ContactsCategory,
|
||||
metadataFiles: MetadataFileNames(path.ContactsCategory),
|
||||
},
|
||||
{
|
||||
name: "Calendar Events",
|
||||
selector: func() *selectors.ExchangeBackup {
|
||||
@ -175,6 +175,20 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
|
||||
expectDeets,
|
||||
false)
|
||||
|
||||
// Lineage for the initial backup should be empty.
|
||||
bup, err := bod.sw.GetBackup(ctx, bo.Results.BackupID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
assert.Empty(t, bup.MergeBases, "base backup merge base lineage")
|
||||
assert.Empty(t, bup.AssistBases, "base backup assist base lineage")
|
||||
|
||||
// Reason for backup should have Exchange and the category.
|
||||
reasons, err := bup.Reasons()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
require.Len(t, reasons, 1, "number of reasons for backup")
|
||||
assert.Equal(t, path.ExchangeService, reasons[0].Service())
|
||||
assert.Equal(t, test.category, reasons[0].Category())
|
||||
|
||||
// Basic, happy path incremental test. No changes are dictated or expected.
|
||||
// This only tests that an incremental backup is runnable at all, and that it
|
||||
// produces fewer results than the last backup.
|
||||
@ -218,6 +232,36 @@ func (suite *ExchangeBackupIntgSuite) TestBackup_Run_exchange() {
|
||||
expectDeets,
|
||||
false)
|
||||
|
||||
// Incremental backup should have the initial backup as a merge base in
|
||||
// the lineage information.
|
||||
bup, err = bod.sw.GetBackup(ctx, incBO.Results.BackupID)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
lineage, err := bup.Bases()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
|
||||
// No assist bases.
|
||||
assert.Empty(t, lineage.Assist)
|
||||
// Expect one merge base with Reason we're testing. Right now tenant isn't
|
||||
// populated and protected resource ID may be incorrect due to inputs to
|
||||
// the test. Just compare service/category.
|
||||
require.Len(t, lineage.Merge[bo.Results.BackupID], 1)
|
||||
assert.Equal(
|
||||
t,
|
||||
path.ExchangeService,
|
||||
lineage.Merge[bo.Results.BackupID][0].Service())
|
||||
assert.Equal(
|
||||
t,
|
||||
test.category,
|
||||
lineage.Merge[bo.Results.BackupID][0].Category())
|
||||
|
||||
// Reason for backup should have Exchange and the category.
|
||||
reasons, err = bup.Reasons()
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
require.Len(t, reasons, 1, "number of reasons for backup")
|
||||
assert.Equal(t, path.ExchangeService, reasons[0].Service())
|
||||
assert.Equal(t, test.category, reasons[0].Category())
|
||||
|
||||
// do some additional checks to ensure the incremental dealt with fewer items.
|
||||
assert.Greater(t, bo.Results.ItemsWritten, incBO.Results.ItemsWritten, "incremental items written")
|
||||
assert.Greater(t, bo.Results.ItemsRead, incBO.Results.ItemsRead, "incremental items read")
|
||||
|
||||
@ -244,7 +244,7 @@ func checkBackupIsInManifests(
|
||||
for _, category := range categories {
|
||||
t.Run(category.String(), func(t *testing.T) {
|
||||
var (
|
||||
r = kopia.NewReason("", resourceOwner, sel.PathService(), category)
|
||||
r = identity.NewReason("", resourceOwner, sel.PathService(), category)
|
||||
tags = map[string]string{kopia.TagBackupCategory: ""}
|
||||
found bool
|
||||
)
|
||||
|
||||
@ -9,7 +9,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
@ -76,13 +76,13 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
table := []struct {
|
||||
name string
|
||||
deets func(*testing.T) *details.Details
|
||||
errs func(context.Context) *fault.Errors
|
||||
bus func(context.Context) *fault.Errors
|
||||
hasSnapID assert.ValueAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "none",
|
||||
deets: func(*testing.T) *details.Details { return nil },
|
||||
errs: func(context.Context) *fault.Errors { return nil },
|
||||
bus: func(context.Context) *fault.Errors { return nil },
|
||||
hasSnapID: assert.Empty,
|
||||
},
|
||||
{
|
||||
@ -100,13 +100,13 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
}))
|
||||
return deetsBuilder.Details()
|
||||
},
|
||||
errs: func(context.Context) *fault.Errors { return nil },
|
||||
bus: func(context.Context) *fault.Errors { return nil },
|
||||
hasSnapID: assert.NotEmpty,
|
||||
},
|
||||
{
|
||||
name: "errors",
|
||||
deets: func(*testing.T) *details.Details { return nil },
|
||||
errs: func(ctx context.Context) *fault.Errors {
|
||||
bus: func(ctx context.Context) *fault.Errors {
|
||||
bus := fault.New(false)
|
||||
bus.Fail(clues.New("foo"))
|
||||
bus.AddRecoverable(ctx, clues.New("bar"))
|
||||
@ -136,7 +136,7 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
|
||||
return deetsBuilder.Details()
|
||||
},
|
||||
errs: func(ctx context.Context) *fault.Errors {
|
||||
bus: func(ctx context.Context) *fault.Errors {
|
||||
bus := fault.New(false)
|
||||
bus.Fail(clues.New("foo"))
|
||||
bus.AddRecoverable(ctx, clues.New("bar"))
|
||||
@ -169,9 +169,9 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
errs := test.errs(ctx)
|
||||
if errs != nil {
|
||||
err = ss.Collect(ctx, FaultErrorsCollector(errs))
|
||||
bus := test.bus(ctx)
|
||||
if bus != nil {
|
||||
err = ss.Collect(ctx, FaultErrorsCollector(bus))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
@ -207,12 +207,12 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
snapid,
|
||||
DetailsReader(details.UnmarshalTo(&readDeets)),
|
||||
fault.New(true))
|
||||
assert.ErrorIs(t, err, data.ErrNotFound)
|
||||
assert.ErrorIs(t, err, errs.NotFound)
|
||||
assert.Empty(t, readDeets)
|
||||
}
|
||||
|
||||
var readErrs fault.Errors
|
||||
if errs != nil {
|
||||
if bus != nil {
|
||||
err = ss.Read(
|
||||
ctx,
|
||||
snapid,
|
||||
@ -221,15 +221,15 @@ func (suite *StreamStoreIntgSuite) TestStreamer() {
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, readErrs)
|
||||
|
||||
assert.ElementsMatch(t, errs.Skipped, readErrs.Skipped)
|
||||
assert.ElementsMatch(t, errs.Recovered, readErrs.Recovered)
|
||||
assert.ElementsMatch(t, bus.Skipped, readErrs.Skipped)
|
||||
assert.ElementsMatch(t, bus.Recovered, readErrs.Recovered)
|
||||
} else {
|
||||
err := ss.Read(
|
||||
ctx,
|
||||
snapid,
|
||||
FaultErrorsReader(fault.UnmarshalErrorsTo(&readErrs)),
|
||||
fault.New(true))
|
||||
assert.ErrorIs(t, err, data.ErrNotFound)
|
||||
assert.ErrorIs(t, err, errs.NotFound)
|
||||
assert.Empty(t, readErrs)
|
||||
}
|
||||
})
|
||||
|
||||
@ -2,18 +2,23 @@ package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/dustin/go-humanize"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/cli/print"
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/str"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/stats"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
@ -36,10 +41,12 @@ type Backup struct {
|
||||
// Selector used in this operation
|
||||
Selector selectors.Selector `json:"selectors"`
|
||||
|
||||
// TODO: in process of gaining support, most cases will still use
|
||||
// ResourceOwner and ResourceOwnerName.
|
||||
ProtectedResourceID string `json:"protectedResourceID,omitempty"`
|
||||
ProtectedResourceName string `json:"protectedResourceName,omitempty"`
|
||||
// ** DO NOT CHANGE JSON TAG NAMES **
|
||||
// These are in-memory only variable renames of previously persisted fields.
|
||||
// ** CHANGING THE JSON TAGS WILL BREAK THINGS BECAUSE THE MODEL WON'T **
|
||||
// ** DESERIALIZE PROPERLY **
|
||||
ProtectedResourceID string `json:"resourceOwnerID,omitempty"`
|
||||
ProtectedResourceName string `json:"resourceOwnerName,omitempty"`
|
||||
|
||||
// Version represents the version of the backup format
|
||||
Version int `json:"version"`
|
||||
@ -57,14 +64,19 @@ type Backup struct {
|
||||
stats.StartAndEndTime
|
||||
stats.SkippedCounts
|
||||
|
||||
// MergeBases records the set of merge bases used for this backup and the
|
||||
// Reason(s) each merge base was selected. Reasons are serialized the same
|
||||
// way that Reason tags are serialized.
|
||||
MergeBases map[model.StableID][]string `json:"mergeBases,omitempty"`
|
||||
// AssistBases records the set of assist bases used for this backup and the
|
||||
// Reason(s) each assist base was selected. Reasons are serialized the same
|
||||
// way that Reason tags are serialized.
|
||||
AssistBases map[model.StableID][]string `json:"assistBases,omitempty"`
|
||||
|
||||
// **Deprecated**
|
||||
// Reference to the backup details storage location.
|
||||
// Used to read backup.Details from the streamstore.
|
||||
DetailsID string `json:"detailsID"`
|
||||
|
||||
// prefer protectedResource
|
||||
ResourceOwnerID string `json:"resourceOwnerID,omitempty"`
|
||||
ResourceOwnerName string `json:"resourceOwnerName,omitempty"`
|
||||
}
|
||||
|
||||
// interface compliance checks
|
||||
@ -78,6 +90,8 @@ func New(
|
||||
ownerID, ownerName string,
|
||||
rw stats.ReadWrites,
|
||||
se stats.StartAndEndTime,
|
||||
reasons []identity.Reasoner,
|
||||
bases BackupBases,
|
||||
fe *fault.Errors,
|
||||
tags map[string]string,
|
||||
) *Backup {
|
||||
@ -109,14 +123,19 @@ func New(
|
||||
}
|
||||
}
|
||||
|
||||
return &Backup{
|
||||
// maps.Clone throws an NPE if passed nil on Mac for some reason.
|
||||
if tags == nil {
|
||||
tags = map[string]string{}
|
||||
}
|
||||
|
||||
b := &Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: id,
|
||||
Tags: tags,
|
||||
Tags: maps.Clone(tags),
|
||||
},
|
||||
|
||||
ResourceOwnerID: ownerID,
|
||||
ResourceOwnerName: ownerName,
|
||||
ProtectedResourceID: ownerID,
|
||||
ProtectedResourceName: ownerName,
|
||||
|
||||
Version: version,
|
||||
SnapshotID: snapshotID,
|
||||
@ -139,6 +158,145 @@ func New(
|
||||
SkippedInvalidOneNoteFile: invalidONFile,
|
||||
},
|
||||
}
|
||||
|
||||
if bases != nil {
|
||||
mergeBases := map[model.StableID][]string{}
|
||||
assistBases := map[model.StableID][]string{}
|
||||
|
||||
for _, backup := range bases.Backups() {
|
||||
for _, reason := range backup.Reasons {
|
||||
mergeBases[backup.ID] = append(
|
||||
mergeBases[backup.ID],
|
||||
ServiceCatString(reason.Service(), reason.Category()))
|
||||
}
|
||||
}
|
||||
|
||||
for _, backup := range bases.UniqueAssistBackups() {
|
||||
for _, reason := range backup.Reasons {
|
||||
assistBases[backup.ID] = append(
|
||||
assistBases[backup.ID],
|
||||
ServiceCatString(reason.Service(), reason.Category()))
|
||||
}
|
||||
}
|
||||
|
||||
if len(mergeBases) > 0 {
|
||||
b.MergeBases = mergeBases
|
||||
}
|
||||
|
||||
if len(assistBases) > 0 {
|
||||
b.AssistBases = assistBases
|
||||
}
|
||||
}
|
||||
|
||||
for _, reason := range reasons {
|
||||
for k, v := range reasonTags(reason) {
|
||||
b.Tags[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
// PersistedBaseSet contains information extracted from the backup model
|
||||
// relating to its lineage. It only contains the backup ID and Reasons each
|
||||
// base was selected instead of the full set of information contained in other
|
||||
// structs like BackupBases.
|
||||
type PersistedBaseSet struct {
|
||||
Merge map[model.StableID][]identity.Reasoner
|
||||
Assist map[model.StableID][]identity.Reasoner
|
||||
}
|
||||
|
||||
func (b Backup) Bases() (PersistedBaseSet, error) {
|
||||
res := PersistedBaseSet{
|
||||
Merge: map[model.StableID][]identity.Reasoner{},
|
||||
Assist: map[model.StableID][]identity.Reasoner{},
|
||||
}
|
||||
|
||||
for id, reasons := range b.MergeBases {
|
||||
for _, reason := range reasons {
|
||||
service, cat, err := serviceCatStringToTypes(reason)
|
||||
if err != nil {
|
||||
return res, clues.Wrap(err, "getting Reason info").With(
|
||||
"base_type", "merge",
|
||||
"base_backup_id", id,
|
||||
"input_string", reason)
|
||||
}
|
||||
|
||||
res.Merge[id] = append(res.Merge[id], identity.NewReason(
|
||||
// Tenant ID not currently stored in backup model.
|
||||
"",
|
||||
str.First(
|
||||
b.ProtectedResourceID,
|
||||
b.Selector.DiscreteOwner),
|
||||
service,
|
||||
cat))
|
||||
}
|
||||
}
|
||||
|
||||
for id, reasons := range b.AssistBases {
|
||||
for _, reason := range reasons {
|
||||
service, cat, err := serviceCatStringToTypes(reason)
|
||||
if err != nil {
|
||||
return res, clues.Wrap(err, "getting Reason info").With(
|
||||
"base_type", "assist",
|
||||
"base_backup_id", id,
|
||||
"input_string", reason)
|
||||
}
|
||||
|
||||
res.Assist[id] = append(res.Assist[id], identity.NewReason(
|
||||
// Tenant ID not currently stored in backup model.
|
||||
"",
|
||||
str.First(
|
||||
b.ProtectedResourceID,
|
||||
b.Selector.DiscreteOwner),
|
||||
service,
|
||||
cat))
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (b Backup) Tenant() (string, error) {
|
||||
t := b.Tags[TenantIDKey]
|
||||
if len(t) == 0 {
|
||||
return "", clues.Wrap(errs.NotFound, "getting tenant")
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Reasons returns the set of services and categories this backup encompassed
|
||||
// for the tenant and protected resource.
|
||||
func (b Backup) Reasons() ([]identity.Reasoner, error) {
|
||||
tenant, err := b.Tenant()
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
}
|
||||
|
||||
var res []identity.Reasoner
|
||||
|
||||
for tag := range b.Tags {
|
||||
service, cat, err := serviceCatStringToTypes(tag)
|
||||
if err != nil {
|
||||
// Assume it's just not one of the Reason tags.
|
||||
if errors.Is(err, errMissingPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
return nil, clues.Wrap(err, "parsing reasons")
|
||||
}
|
||||
|
||||
res = append(
|
||||
res,
|
||||
identity.NewReason(
|
||||
tenant,
|
||||
str.First(b.ProtectedResourceID, b.Selector.DiscreteOwner),
|
||||
service,
|
||||
cat))
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
@ -253,9 +411,7 @@ func (b Backup) Values() []string {
|
||||
|
||||
name := str.First(
|
||||
b.ProtectedResourceName,
|
||||
b.ResourceOwnerName,
|
||||
b.ProtectedResourceID,
|
||||
b.ResourceOwnerID,
|
||||
b.Selector.Name())
|
||||
|
||||
bs := b.toStats()
|
||||
|
||||
152
src/pkg/backup/backup_bases.go
Normal file
152
src/pkg/backup/backup_bases.go
Normal file
@ -0,0 +1,152 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
const (
|
||||
// Kopia does not do comparisons properly for empty tags right now so add some
|
||||
// placeholder value to them.
|
||||
legacyDefaultTagValue = "0"
|
||||
|
||||
// Kopia CLI prefixes all user tags with "tag:"[1]. Maintaining this will
|
||||
// ensure we don't accidentally take reserved tags and that tags can be
|
||||
// displayed with kopia CLI.
|
||||
// (permalinks)
|
||||
// [1] https://github.com/kopia/kopia/blob/05e729a7858a6e86cb48ba29fb53cb6045efce2b/cli/command_snapshot_create.go#L169
|
||||
LegacyUserTagPrefix = "tag:"
|
||||
|
||||
TenantIDKey = "tenant"
|
||||
ResourceIDKey = "protectedResource"
|
||||
serviceCatPrefix = "sc-"
|
||||
separator = "_"
|
||||
|
||||
// Sentinel value for tags. Could technically be empty but we'll store
|
||||
// something for now.
|
||||
//nolint
|
||||
DefaultTagValue = "1"
|
||||
)
|
||||
|
||||
var errMissingPrefix = clues.New("missing tag prefix")
|
||||
|
||||
func ServiceCatString(
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
) string {
|
||||
return serviceCatPrefix + service.String() + separator + category.String()
|
||||
}
|
||||
|
||||
func serviceCatStringToTypes(
|
||||
input string,
|
||||
) (path.ServiceType, path.CategoryType, error) {
|
||||
trimmed := strings.TrimPrefix(input, serviceCatPrefix)
|
||||
// No prefix found -> unexpected format.
|
||||
if trimmed == input {
|
||||
return path.UnknownService,
|
||||
path.UnknownCategory,
|
||||
clues.Stack(errMissingPrefix).With(
|
||||
"expected_prefix", serviceCatPrefix,
|
||||
"input", input)
|
||||
}
|
||||
|
||||
parts := strings.Split(trimmed, separator)
|
||||
if len(parts) != 2 {
|
||||
return path.UnknownService,
|
||||
path.UnknownCategory,
|
||||
clues.New("missing tag separator")
|
||||
}
|
||||
|
||||
cat := path.ToCategoryType(parts[1])
|
||||
if cat == path.UnknownCategory {
|
||||
return path.UnknownService,
|
||||
path.UnknownCategory,
|
||||
clues.New("parsing category").With("input_category", parts[1])
|
||||
}
|
||||
|
||||
service := path.ToServiceType(parts[0])
|
||||
if service == path.UnknownService {
|
||||
return path.UnknownService,
|
||||
path.UnknownCategory,
|
||||
clues.New("parsing service").With("input_service", parts[0])
|
||||
}
|
||||
|
||||
return service, cat, nil
|
||||
}
|
||||
|
||||
// reasonTags returns the set of key-value pairs that can be used as tags to
|
||||
// represent this Reason.
|
||||
// nolint
|
||||
func reasonTags(r identity.Reasoner) map[string]string {
|
||||
return map[string]string{
|
||||
TenantIDKey: r.Tenant(),
|
||||
ResourceIDKey: r.ProtectedResource(),
|
||||
ServiceCatString(r.Service(), r.Category()): DefaultTagValue,
|
||||
}
|
||||
}
|
||||
|
||||
// nolint
|
||||
type BackupEntry struct {
|
||||
*Backup
|
||||
Reasons []identity.Reasoner
|
||||
}
|
||||
|
||||
type ManifestEntry struct {
|
||||
*snapshot.Manifest
|
||||
// Reasons contains the ResourceOwners and Service/Categories that caused this
|
||||
// snapshot to be selected as a base. We can't reuse OwnersCats here because
|
||||
// it's possible some ResourceOwners will have a subset of the Categories as
|
||||
// the reason for selecting a snapshot. For example:
|
||||
// 1. backup user1 email,contacts -> B1
|
||||
// 2. backup user1 contacts -> B2 (uses B1 as base)
|
||||
// 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts)
|
||||
Reasons []identity.Reasoner
|
||||
}
|
||||
|
||||
// MakeTagKV normalizes the provided key to protect it from clobbering
|
||||
// similarly named tags from non-user input (user inputs are still open
|
||||
// to collisions amongst eachother).
|
||||
// Returns the normalized Key plus a default value. If you're embedding a
|
||||
// key-only tag, the returned default value msut be used instead of an
|
||||
// empty string.
|
||||
func MakeTagKV(k string) (string, string) {
|
||||
return LegacyUserTagPrefix + k, legacyDefaultTagValue
|
||||
}
|
||||
|
||||
func (me ManifestEntry) GetTag(key string) (string, bool) {
|
||||
k, _ := MakeTagKV(key)
|
||||
v, ok := me.Tags[k]
|
||||
|
||||
return v, ok
|
||||
}
|
||||
|
||||
// nolint
|
||||
type BackupBases interface {
|
||||
// ConvertToAssistBase converts the base with the given item data snapshot ID
|
||||
// from a merge base to an assist base.
|
||||
ConvertToAssistBase(manifestID manifest.ID)
|
||||
Backups() []BackupEntry
|
||||
UniqueAssistBackups() []BackupEntry
|
||||
MinBackupVersion() int
|
||||
MergeBases() []ManifestEntry
|
||||
DisableMergeBases()
|
||||
UniqueAssistBases() []ManifestEntry
|
||||
DisableAssistBases()
|
||||
MergeBackupBases(
|
||||
ctx context.Context,
|
||||
other BackupBases,
|
||||
reasonToKey func(identity.Reasoner) string,
|
||||
) BackupBases
|
||||
// SnapshotAssistBases returns the set of bases to use for kopia assisted
|
||||
// incremental snapshot operations. It consists of the union of merge bases
|
||||
// and assist bases. If DisableAssistBases has been called then it returns
|
||||
// nil.
|
||||
SnapshotAssistBases() []ManifestEntry
|
||||
}
|
||||
70
src/pkg/backup/backup_bases_test.go
Normal file
70
src/pkg/backup/backup_bases_test.go
Normal file
@ -0,0 +1,70 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
type BackupBasesUnitSuite struct {
|
||||
tester.Suite
|
||||
}
|
||||
|
||||
func TestBackupBasesUnitSuite(t *testing.T) {
|
||||
suite.Run(t, &BackupBasesUnitSuite{Suite: tester.NewUnitSuite(t)})
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestServiceCategorySerialization() {
|
||||
table := []struct {
|
||||
name string
|
||||
input string
|
||||
expectErr assert.ErrorAssertionFunc
|
||||
expectService path.ServiceType
|
||||
expectCategory path.CategoryType
|
||||
}{
|
||||
{
|
||||
name: "ProperFormat",
|
||||
input: ServiceCatString(path.ExchangeService, path.EmailCategory),
|
||||
expectErr: assert.NoError,
|
||||
expectService: path.ExchangeService,
|
||||
expectCategory: path.EmailCategory,
|
||||
},
|
||||
{
|
||||
name: "MissingPrefix",
|
||||
input: strings.TrimPrefix(
|
||||
ServiceCatString(path.ExchangeService, path.EmailCategory),
|
||||
serviceCatPrefix),
|
||||
expectErr: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "MissingSeparator",
|
||||
input: strings.ReplaceAll(
|
||||
ServiceCatString(path.ExchangeService, path.EmailCategory),
|
||||
separator,
|
||||
""),
|
||||
expectErr: assert.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
service, cat, err := serviceCatStringToTypes(test.input)
|
||||
test.expectErr(t, err, clues.ToCore(err))
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, test.expectService, service)
|
||||
assert.Equal(t, test.expectCategory, cat)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,20 +1,24 @@
|
||||
package backup_test
|
||||
package backup
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
"golang.org/x/exp/maps"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/dttm"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/stats"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
@ -26,11 +30,11 @@ func TestBackupUnitSuite(t *testing.T) {
|
||||
suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)})
|
||||
}
|
||||
|
||||
func stubBackup(t time.Time, ownerID, ownerName string) backup.Backup {
|
||||
func stubBackup(t time.Time, ownerID, ownerName string) Backup {
|
||||
sel := selectors.NewExchangeBackup([]string{"test"})
|
||||
sel.Include(sel.AllData())
|
||||
|
||||
return backup.Backup{
|
||||
return Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: model.StableID("id"),
|
||||
Tags: map[string]string{
|
||||
@ -40,10 +44,8 @@ func stubBackup(t time.Time, ownerID, ownerName string) backup.Backup {
|
||||
CreationTime: t,
|
||||
SnapshotID: "snapshot",
|
||||
DetailsID: "details",
|
||||
ProtectedResourceID: ownerID + "-pr",
|
||||
ProtectedResourceName: ownerName + "-pr",
|
||||
ResourceOwnerID: ownerID + "-ro",
|
||||
ResourceOwnerName: ownerName + "-ro",
|
||||
ProtectedResourceID: ownerID + "-ro",
|
||||
ProtectedResourceName: ownerName + "-ro",
|
||||
Status: "status",
|
||||
Selector: sel.Selector,
|
||||
ErrorCount: 2,
|
||||
@ -67,6 +69,371 @@ func stubBackup(t time.Time, ownerID, ownerName string) backup.Backup {
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupUnitSuite) TestBackup_Bases() {
|
||||
const (
|
||||
mergeID model.StableID = "merge-backup-id"
|
||||
assistID model.StableID = "assist-backup-id"
|
||||
userID = "user-id"
|
||||
)
|
||||
|
||||
stub := stubBackup(time.Now(), userID, "user-name")
|
||||
|
||||
defaultEmailReason := identity.NewReason(
|
||||
"",
|
||||
stub.ProtectedResourceID,
|
||||
path.ExchangeService,
|
||||
path.EmailCategory)
|
||||
defaultContactsReason := identity.NewReason(
|
||||
"",
|
||||
stub.ProtectedResourceID,
|
||||
path.ExchangeService,
|
||||
path.ContactsCategory)
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
getBackup func() *Backup
|
||||
expectErr assert.ErrorAssertionFunc
|
||||
expectMerge map[model.StableID][]identity.Reasoner
|
||||
expectAssist map[model.StableID][]identity.Reasoner
|
||||
}{
|
||||
{
|
||||
name: "MergeAndAssist SameReasonEach",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.MergeBases = map[model.StableID][]string{}
|
||||
res.AssistBases = map[model.StableID][]string{}
|
||||
|
||||
res.MergeBases[mergeID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
}
|
||||
|
||||
res.AssistBases[assistID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expectMerge: map[model.StableID][]identity.Reasoner{
|
||||
mergeID: {defaultEmailReason},
|
||||
},
|
||||
expectAssist: map[model.StableID][]identity.Reasoner{
|
||||
assistID: {defaultEmailReason},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "MergeAndAssist DifferentReasonEach",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.MergeBases = map[model.StableID][]string{}
|
||||
res.AssistBases = map[model.StableID][]string{}
|
||||
|
||||
res.MergeBases[mergeID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
}
|
||||
|
||||
res.AssistBases[assistID] = []string{
|
||||
ServiceCatString(
|
||||
defaultContactsReason.Service(),
|
||||
defaultContactsReason.Category()),
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expectMerge: map[model.StableID][]identity.Reasoner{
|
||||
mergeID: {defaultEmailReason},
|
||||
},
|
||||
expectAssist: map[model.StableID][]identity.Reasoner{
|
||||
assistID: {defaultContactsReason},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "MergeAndAssist MultipleReasonsEach",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.MergeBases = map[model.StableID][]string{}
|
||||
res.AssistBases = map[model.StableID][]string{}
|
||||
|
||||
res.MergeBases[mergeID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
ServiceCatString(
|
||||
defaultContactsReason.Service(),
|
||||
defaultContactsReason.Category()),
|
||||
}
|
||||
|
||||
res.AssistBases[assistID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
ServiceCatString(
|
||||
defaultContactsReason.Service(),
|
||||
defaultContactsReason.Category()),
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expectMerge: map[model.StableID][]identity.Reasoner{
|
||||
mergeID: {
|
||||
defaultEmailReason,
|
||||
defaultContactsReason,
|
||||
},
|
||||
},
|
||||
expectAssist: map[model.StableID][]identity.Reasoner{
|
||||
assistID: {
|
||||
defaultEmailReason,
|
||||
defaultContactsReason,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OnlyMerge SingleReason",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.MergeBases = map[model.StableID][]string{}
|
||||
|
||||
res.MergeBases[mergeID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expectMerge: map[model.StableID][]identity.Reasoner{
|
||||
mergeID: {defaultEmailReason},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OnlyAssist SingleReason",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.AssistBases = map[model.StableID][]string{}
|
||||
|
||||
res.AssistBases[mergeID] = []string{
|
||||
ServiceCatString(
|
||||
defaultEmailReason.Service(),
|
||||
defaultEmailReason.Category()),
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expectAssist: map[model.StableID][]identity.Reasoner{
|
||||
mergeID: {defaultEmailReason},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "BadReasonFormat",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.AssistBases = map[model.StableID][]string{}
|
||||
|
||||
res.AssistBases[mergeID] = []string{"foo"}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
bup := test.getBackup()
|
||||
|
||||
got, err := bup.Bases()
|
||||
test.expectErr(t, err, clues.ToCore(err))
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Since the result contains a slice of Reasons directly calling Equals
|
||||
// will fail because we want ElementsMatch on the internal slices.
|
||||
assert.ElementsMatch(
|
||||
t,
|
||||
maps.Keys(test.expectMerge),
|
||||
maps.Keys(got.Merge),
|
||||
"merge base keys")
|
||||
assert.ElementsMatch(
|
||||
t,
|
||||
maps.Keys(test.expectAssist),
|
||||
maps.Keys(got.Assist),
|
||||
"assist base keys")
|
||||
|
||||
for id, e := range test.expectMerge {
|
||||
assert.ElementsMatch(t, e, got.Merge[id], "merge bases")
|
||||
}
|
||||
|
||||
for id, e := range test.expectAssist {
|
||||
assert.ElementsMatch(t, e, got.Assist[id], "assist bases")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupUnitSuite) TestBackup_Tenant() {
|
||||
const tenant = "tenant-id"
|
||||
|
||||
stub := stubBackup(time.Now(), "user-id", "user-name")
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
inputKey string
|
||||
inputValue string
|
||||
expectErr assert.ErrorAssertionFunc
|
||||
expect string
|
||||
}{
|
||||
{
|
||||
name: "ProperlyFormatted",
|
||||
inputKey: TenantIDKey,
|
||||
inputValue: tenant,
|
||||
expectErr: assert.NoError,
|
||||
expect: tenant,
|
||||
},
|
||||
{
|
||||
name: "WrongKey",
|
||||
inputKey: "foo",
|
||||
inputValue: tenant,
|
||||
expectErr: assert.Error,
|
||||
},
|
||||
{
|
||||
name: "EmptyValue",
|
||||
inputKey: TenantIDKey,
|
||||
inputValue: "",
|
||||
expectErr: assert.Error,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
b := stub
|
||||
b.Tags = map[string]string{test.inputKey: test.inputValue}
|
||||
|
||||
gotTenant, err := b.Tenant()
|
||||
test.expectErr(t, err, clues.ToCore(err))
|
||||
|
||||
if err != nil {
|
||||
assert.ErrorIs(t, err, errs.NotFound)
|
||||
return
|
||||
}
|
||||
|
||||
assert.Equal(t, test.expect, gotTenant)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupUnitSuite) TestBackup_Reasons() {
|
||||
const (
|
||||
tenantID = "tenant-id"
|
||||
userID = "user-id"
|
||||
)
|
||||
|
||||
stub := stubBackup(time.Now(), userID, "user-name")
|
||||
|
||||
defaultEmailReason := identity.NewReason(
|
||||
tenantID,
|
||||
stub.ProtectedResourceID,
|
||||
path.ExchangeService,
|
||||
path.EmailCategory)
|
||||
defaultContactsReason := identity.NewReason(
|
||||
tenantID,
|
||||
stub.ProtectedResourceID,
|
||||
path.ExchangeService,
|
||||
path.ContactsCategory)
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
getBackup func() *Backup
|
||||
expectErr assert.ErrorAssertionFunc
|
||||
expect []identity.Reasoner
|
||||
}{
|
||||
{
|
||||
name: "SingleReason",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.Tags = map[string]string{}
|
||||
|
||||
for k, v := range reasonTags(defaultEmailReason) {
|
||||
res.Tags[k] = v
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expect: []identity.Reasoner{defaultEmailReason},
|
||||
},
|
||||
{
|
||||
name: "MultipleReasons",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.Tags = map[string]string{}
|
||||
|
||||
for _, reason := range []identity.Reasoner{defaultEmailReason, defaultContactsReason} {
|
||||
for k, v := range reasonTags(reason) {
|
||||
res.Tags[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expect: []identity.Reasoner{
|
||||
defaultEmailReason,
|
||||
defaultContactsReason,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SingleReason OtherTags",
|
||||
getBackup: func() *Backup {
|
||||
res := stub
|
||||
res.Tags = map[string]string{}
|
||||
|
||||
for k, v := range reasonTags(defaultEmailReason) {
|
||||
res.Tags[k] = v
|
||||
}
|
||||
|
||||
res.Tags["foo"] = "bar"
|
||||
|
||||
return &res
|
||||
},
|
||||
expectErr: assert.NoError,
|
||||
expect: []identity.Reasoner{defaultEmailReason},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
bup := test.getBackup()
|
||||
|
||||
got, err := bup.Reasons()
|
||||
test.expectErr(t, err, clues.ToCore(err))
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, test.expect, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupUnitSuite) TestBackup_HeadersValues() {
|
||||
var (
|
||||
t = suite.T()
|
||||
@ -86,7 +453,7 @@ func (suite *BackupUnitSuite) TestBackup_HeadersValues() {
|
||||
nowFmt,
|
||||
"1m0s",
|
||||
"status (2 errors, 1 skipped: 1 malware)",
|
||||
"name-pr",
|
||||
"name-ro",
|
||||
}
|
||||
)
|
||||
|
||||
@ -123,9 +490,6 @@ func (suite *BackupUnitSuite) TestBackup_HeadersValues_onlyResourceOwners() {
|
||||
}
|
||||
)
|
||||
|
||||
b.ProtectedResourceID = ""
|
||||
b.ProtectedResourceName = ""
|
||||
|
||||
b.StartAndEndTime.CompletedAt = later
|
||||
|
||||
// single skipped malware
|
||||
@ -139,17 +503,17 @@ func (suite *BackupUnitSuite) TestBackup_HeadersValues_onlyResourceOwners() {
|
||||
func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
table := []struct {
|
||||
name string
|
||||
bup backup.Backup
|
||||
bup Backup
|
||||
expect string
|
||||
}{
|
||||
{
|
||||
name: "no extras",
|
||||
bup: backup.Backup{Status: "test"},
|
||||
bup: Backup{Status: "test"},
|
||||
expect: "test",
|
||||
},
|
||||
{
|
||||
name: "errors",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
ErrorCount: 42,
|
||||
},
|
||||
@ -157,7 +521,7 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
},
|
||||
{
|
||||
name: "malware",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
SkippedCounts: stats.SkippedCounts{
|
||||
TotalSkippedItems: 2,
|
||||
@ -168,7 +532,7 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
},
|
||||
{
|
||||
name: "errors and malware",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
ErrorCount: 42,
|
||||
SkippedCounts: stats.SkippedCounts{
|
||||
@ -180,7 +544,7 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
},
|
||||
{
|
||||
name: "errors and skipped",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
ErrorCount: 42,
|
||||
SkippedCounts: stats.SkippedCounts{
|
||||
@ -191,7 +555,7 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
},
|
||||
{
|
||||
name: "errors and invalid OneNote",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
ErrorCount: 42,
|
||||
SkippedCounts: stats.SkippedCounts{
|
||||
@ -203,7 +567,7 @@ func (suite *BackupUnitSuite) TestBackup_Values_statusVariations() {
|
||||
},
|
||||
{
|
||||
name: "errors, malware, notFound, invalid OneNote",
|
||||
bup: backup.Backup{
|
||||
bup: Backup{
|
||||
Status: "test",
|
||||
ErrorCount: 42,
|
||||
SkippedCounts: stats.SkippedCounts{
|
||||
@ -229,7 +593,7 @@ func (suite *BackupUnitSuite) TestBackup_MinimumPrintable() {
|
||||
b := stubBackup(now, "id", "name")
|
||||
|
||||
resultIface := b.MinimumPrintable()
|
||||
result, ok := resultIface.(backup.Printable)
|
||||
result, ok := resultIface.(Printable)
|
||||
require.True(t, ok)
|
||||
|
||||
assert.Equal(t, b.ID, result.ID, "id")
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
package identity
|
||||
|
||||
import "github.com/alcionai/corso/src/pkg/path"
|
||||
import (
|
||||
"github.com/alcionai/clues"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
// Reasoner describes the parts of the backup that make up its
|
||||
// data identity: the tenant, protected resources, services, and
|
||||
@ -14,3 +18,52 @@ type Reasoner interface {
|
||||
// parameters (tenant, protected resourced, etc) that match this Reasoner.
|
||||
SubtreePath() (path.Path, error)
|
||||
}
|
||||
|
||||
func NewReason(
|
||||
tenant, resource string,
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
) Reasoner {
|
||||
return reason{
|
||||
tenant: tenant,
|
||||
resource: resource,
|
||||
service: service,
|
||||
category: category,
|
||||
}
|
||||
}
|
||||
|
||||
type reason struct {
|
||||
// tenant appears here so that when this is moved to an inject package nothing
|
||||
// needs changed. However, kopia itself is blind to the fields in the reason
|
||||
// struct and relies on helper functions to get the information it needs.
|
||||
tenant string
|
||||
resource string
|
||||
service path.ServiceType
|
||||
category path.CategoryType
|
||||
}
|
||||
|
||||
func (r reason) Tenant() string {
|
||||
return r.tenant
|
||||
}
|
||||
|
||||
func (r reason) ProtectedResource() string {
|
||||
return r.resource
|
||||
}
|
||||
|
||||
func (r reason) Service() path.ServiceType {
|
||||
return r.service
|
||||
}
|
||||
|
||||
func (r reason) Category() path.CategoryType {
|
||||
return r.category
|
||||
}
|
||||
|
||||
func (r reason) SubtreePath() (path.Path, error) {
|
||||
p, err := path.BuildPrefix(
|
||||
r.Tenant(),
|
||||
r.ProtectedResource(),
|
||||
r.Service(),
|
||||
r.Category())
|
||||
|
||||
return p, clues.Wrap(err, "building path").OrNil()
|
||||
}
|
||||
|
||||
@ -6,9 +6,10 @@ import (
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/common/idname"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
@ -17,7 +18,9 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/store"
|
||||
)
|
||||
@ -120,7 +123,7 @@ func getBackup(
|
||||
func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) {
|
||||
var (
|
||||
bups []*backup.Backup
|
||||
errs = fault.New(false)
|
||||
bus = fault.New(false)
|
||||
sw = store.NewWrapper(r.modelStore)
|
||||
)
|
||||
|
||||
@ -129,13 +132,260 @@ func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup
|
||||
|
||||
b, err := sw.GetBackup(ictx, model.StableID(id))
|
||||
if err != nil {
|
||||
errs.AddRecoverable(ctx, errWrapper(err))
|
||||
bus.AddRecoverable(ctx, errWrapper(err))
|
||||
}
|
||||
|
||||
bups = append(bups, b)
|
||||
}
|
||||
|
||||
return bups, errs
|
||||
return bups, bus
|
||||
}
|
||||
|
||||
func addBackup(
|
||||
bup *lineageNode,
|
||||
seen map[model.StableID]*lineageNode,
|
||||
allNodes map[model.StableID]*lineageNode,
|
||||
) {
|
||||
if bup == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := seen[bup.ID]; ok {
|
||||
// We've already traversed this node.
|
||||
return
|
||||
}
|
||||
|
||||
for baseID := range bup.MergeBases {
|
||||
addBackup(allNodes[baseID], seen, allNodes)
|
||||
}
|
||||
|
||||
for baseID := range bup.AssistBases {
|
||||
addBackup(allNodes[baseID], seen, allNodes)
|
||||
}
|
||||
|
||||
seen[bup.ID] = bup
|
||||
|
||||
for _, descendent := range bup.children {
|
||||
addBackup(allNodes[descendent.ID], seen, allNodes)
|
||||
}
|
||||
}
|
||||
|
||||
func filterLineages(
|
||||
bups map[model.StableID]*lineageNode,
|
||||
backupIDs ...string,
|
||||
) map[model.StableID]*lineageNode {
|
||||
if len(backupIDs) == 0 {
|
||||
return bups
|
||||
}
|
||||
|
||||
res := map[model.StableID]*lineageNode{}
|
||||
|
||||
// For each backup we're interested in, traverse up and down the hierarchy.
|
||||
// Going down the hierarchy is more difficult because backups only have
|
||||
// backpointers to their ancestors.
|
||||
for _, id := range backupIDs {
|
||||
addBackup(bups[model.StableID(id)], res, bups)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func addBase(
|
||||
baseID model.StableID,
|
||||
baseReasons []identity.Reasoner,
|
||||
current *BackupNode,
|
||||
allNodes map[model.StableID]*BackupNode,
|
||||
bups map[model.StableID]*lineageNode,
|
||||
) {
|
||||
parent, parentOK := allNodes[baseID]
|
||||
if !parentOK {
|
||||
parent = &BackupNode{}
|
||||
allNodes[baseID] = parent
|
||||
}
|
||||
|
||||
parent.Label = string(baseID)
|
||||
|
||||
// If the parent isn't in the set of backups passed in it must have been
|
||||
// deleted.
|
||||
if p, ok := bups[baseID]; !ok || p.deleted {
|
||||
parent.Deleted = true
|
||||
|
||||
// If the backup was deleted we should also attempt to recreate the
|
||||
// set of Reasons which it encompassed. We can get partial info on this
|
||||
// by collecting all the Reasons it was a base.
|
||||
for _, reason := range baseReasons {
|
||||
if !slices.ContainsFunc(
|
||||
parent.Reasons,
|
||||
func(other identity.Reasoner) bool {
|
||||
return other.Service() == reason.Service() &&
|
||||
other.Category() == reason.Category()
|
||||
}) {
|
||||
parent.Reasons = append(parent.Reasons, reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parent.Children = append(
|
||||
parent.Children,
|
||||
&BackupEdge{
|
||||
BackupNode: current,
|
||||
Reasons: baseReasons,
|
||||
})
|
||||
}
|
||||
|
||||
func buildOutput(bups map[model.StableID]*lineageNode) ([]*BackupNode, error) {
|
||||
var roots []*BackupNode
|
||||
|
||||
allNodes := map[model.StableID]*BackupNode{}
|
||||
|
||||
for _, bup := range bups {
|
||||
node := allNodes[bup.ID]
|
||||
if node == nil {
|
||||
node = &BackupNode{}
|
||||
allNodes[bup.ID] = node
|
||||
}
|
||||
|
||||
node.Label = string(bup.ID)
|
||||
node.Type = MergeNode
|
||||
node.Created = bup.CreationTime
|
||||
|
||||
if bup.Tags[model.BackupTypeTag] == model.AssistBackup {
|
||||
node.Type = AssistNode
|
||||
}
|
||||
|
||||
topLevel := true
|
||||
|
||||
if !bup.deleted {
|
||||
reasons, err := bup.Reasons()
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "getting reasons").With("backup_id", bup.ID)
|
||||
}
|
||||
|
||||
node.Reasons = reasons
|
||||
|
||||
bases, err := bup.Bases()
|
||||
if err != nil {
|
||||
return nil, clues.Wrap(err, "getting bases").With("backup_id", bup.ID)
|
||||
}
|
||||
|
||||
for baseID, baseReasons := range bases.Merge {
|
||||
topLevel = false
|
||||
|
||||
addBase(baseID, baseReasons, node, allNodes, bups)
|
||||
}
|
||||
|
||||
for baseID, baseReasons := range bases.Assist {
|
||||
topLevel = false
|
||||
|
||||
addBase(baseID, baseReasons, node, allNodes, bups)
|
||||
}
|
||||
}
|
||||
|
||||
// If this node has no ancestors then add it directly to the root.
|
||||
if bup.deleted || topLevel {
|
||||
roots = append(roots, node)
|
||||
}
|
||||
}
|
||||
|
||||
return roots, nil
|
||||
}
|
||||
|
||||
// lineageNode is a small in-memory wrapper around *backup.Backup that provides
|
||||
// information about children. This just makes it easier to traverse lineages
|
||||
// during filtering.
|
||||
type lineageNode struct {
|
||||
*backup.Backup
|
||||
children []*lineageNode
|
||||
deleted bool
|
||||
}
|
||||
|
||||
func (r repository) BackupLineage(
|
||||
ctx context.Context,
|
||||
tenantID string,
|
||||
protectedResourceID string,
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
backupIDs ...string,
|
||||
) ([]*BackupNode, error) {
|
||||
sw := store.NewWrapper(r.modelStore)
|
||||
|
||||
fs := []store.FilterOption{
|
||||
store.Tenant(tenantID),
|
||||
//store.ProtectedResource(protectedResourceID),
|
||||
//store.Reason(service, category),
|
||||
}
|
||||
|
||||
bs, err := sw.GetBackups(ctx, fs...)
|
||||
if err != nil {
|
||||
return nil, clues.Stack(err)
|
||||
}
|
||||
|
||||
if len(bs) == 0 {
|
||||
return nil, clues.Stack(errs.NotFound)
|
||||
}
|
||||
|
||||
// Put all the backups in a map so we can access them easier when building the
|
||||
// graph.
|
||||
bups := make(map[model.StableID]*lineageNode, len(bs))
|
||||
|
||||
for _, b := range bs {
|
||||
current := bups[b.ID]
|
||||
|
||||
if current == nil {
|
||||
current = &lineageNode{}
|
||||
}
|
||||
|
||||
current.Backup = b
|
||||
current.deleted = false
|
||||
bups[b.ID] = current
|
||||
|
||||
for id := range b.MergeBases {
|
||||
parent := bups[id]
|
||||
if parent == nil {
|
||||
// Populate the ID so we don't NPE on it when building the tree if
|
||||
// something was deleted.
|
||||
parent = &lineageNode{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: id,
|
||||
},
|
||||
},
|
||||
deleted: true,
|
||||
}
|
||||
}
|
||||
|
||||
parent.children = append(parent.children, current)
|
||||
bups[id] = parent
|
||||
}
|
||||
|
||||
for id := range b.AssistBases {
|
||||
parent := bups[id]
|
||||
if parent == nil {
|
||||
// Populate the ID so we don't NPE on it when building the tree if
|
||||
// something was deleted.
|
||||
parent = &lineageNode{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: id,
|
||||
},
|
||||
},
|
||||
deleted: true,
|
||||
}
|
||||
}
|
||||
|
||||
parent.children = append(parent.children, current)
|
||||
bups[id] = parent
|
||||
}
|
||||
}
|
||||
|
||||
// Filter the map of backups to just those in the lineages we're interested
|
||||
// about.
|
||||
filtered := filterLineages(bups, backupIDs...)
|
||||
|
||||
// Build the output graph.
|
||||
res, err := buildOutput(filtered)
|
||||
|
||||
return res, clues.Stack(err).OrNil()
|
||||
}
|
||||
|
||||
// BackupsByTag lists all backups in a repository that contain all the tags
|
||||
@ -177,7 +427,7 @@ func (r repository) GetBackupDetails(
|
||||
ctx context.Context,
|
||||
backupID string,
|
||||
) (*details.Details, *backup.Backup, *fault.Bus) {
|
||||
errs := fault.New(false)
|
||||
bus := fault.New(false)
|
||||
|
||||
deets, bup, err := getBackupDetails(
|
||||
ctx,
|
||||
@ -185,9 +435,9 @@ func (r repository) GetBackupDetails(
|
||||
r.Account.ID(),
|
||||
r.dataLayer,
|
||||
store.NewWrapper(r.modelStore),
|
||||
errs)
|
||||
bus)
|
||||
|
||||
return deets, bup, errs.Fail(err)
|
||||
return deets, bup, bus.Fail(err)
|
||||
}
|
||||
|
||||
// getBackupDetails handles the processing for GetBackupDetails.
|
||||
@ -196,7 +446,7 @@ func getBackupDetails(
|
||||
backupID, tenantID string,
|
||||
kw *kopia.Wrapper,
|
||||
sw store.BackupGetter,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) (*details.Details, *backup.Backup, error) {
|
||||
b, err := sw.GetBackup(ctx, model.StableID(backupID))
|
||||
if err != nil {
|
||||
@ -221,7 +471,7 @@ func getBackupDetails(
|
||||
ctx,
|
||||
ssid,
|
||||
streamstore.DetailsReader(details.UnmarshalTo(&deets)),
|
||||
errs)
|
||||
bus)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -247,7 +497,7 @@ func (r repository) GetBackupErrors(
|
||||
ctx context.Context,
|
||||
backupID string,
|
||||
) (*fault.Errors, *backup.Backup, *fault.Bus) {
|
||||
errs := fault.New(false)
|
||||
bus := fault.New(false)
|
||||
|
||||
fe, bup, err := getBackupErrors(
|
||||
ctx,
|
||||
@ -255,9 +505,9 @@ func (r repository) GetBackupErrors(
|
||||
r.Account.ID(),
|
||||
r.dataLayer,
|
||||
store.NewWrapper(r.modelStore),
|
||||
errs)
|
||||
bus)
|
||||
|
||||
return fe, bup, errs.Fail(err)
|
||||
return fe, bup, bus.Fail(err)
|
||||
}
|
||||
|
||||
// getBackupErrors handles the processing for GetBackupErrors.
|
||||
@ -266,7 +516,7 @@ func getBackupErrors(
|
||||
backupID, tenantID string,
|
||||
kw *kopia.Wrapper,
|
||||
sw store.BackupGetter,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) (*fault.Errors, *backup.Backup, error) {
|
||||
b, err := sw.GetBackup(ctx, model.StableID(backupID))
|
||||
if err != nil {
|
||||
@ -287,7 +537,7 @@ func getBackupErrors(
|
||||
ctx,
|
||||
ssid,
|
||||
streamstore.FaultErrorsReader(fault.UnmarshalErrorsTo(&fe)),
|
||||
errs)
|
||||
bus)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -330,7 +580,7 @@ func deleteBackups(
|
||||
for _, id := range ids {
|
||||
b, err := sw.GetBackup(ctx, model.StableID(id))
|
||||
if err != nil {
|
||||
if !failOnMissing && errors.Is(err, data.ErrNotFound) {
|
||||
if !failOnMissing && errors.Is(err, errs.NotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@ -9,13 +9,14 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common/crash"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/events"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
"github.com/alcionai/corso/src/pkg/account"
|
||||
"github.com/alcionai/corso/src/pkg/backup/identity"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
@ -31,6 +32,29 @@ var (
|
||||
ErrorBackupNotFound = clues.New("no backup exists with that id")
|
||||
)
|
||||
|
||||
type NodeType int
|
||||
|
||||
const (
|
||||
UnknownNode NodeType = iota
|
||||
RootNode
|
||||
MergeNode
|
||||
AssistNode
|
||||
)
|
||||
|
||||
type BackupEdge struct {
|
||||
Reasons []identity.Reasoner
|
||||
*BackupNode
|
||||
}
|
||||
|
||||
type BackupNode struct {
|
||||
Type NodeType
|
||||
Deleted bool
|
||||
Reasons []identity.Reasoner
|
||||
Created time.Time
|
||||
Label string
|
||||
Children []*BackupEdge
|
||||
}
|
||||
|
||||
type Repositoryer interface {
|
||||
Backuper
|
||||
BackupGetter
|
||||
@ -57,6 +81,23 @@ type Repositoryer interface {
|
||||
ctx context.Context,
|
||||
rcOpts ctrlRepo.Retention,
|
||||
) (operations.RetentionConfigOperation, error)
|
||||
|
||||
// BackupLineage returns all backups for the given tenant and
|
||||
// protectedResourceID. If one or more backupIDs are given then filters the
|
||||
// lineage down to include only those backups that either match the given IDs
|
||||
// or are reachable from the backups with the given IDs.
|
||||
//
|
||||
// The root node representing the tenant/protectedResourceID is returned. All
|
||||
// discovered backups in the lineage are reachable by traversing the tree
|
||||
// starting with the returned node.
|
||||
BackupLineage(
|
||||
ctx context.Context,
|
||||
teantID string,
|
||||
protectedResourceID string,
|
||||
service path.ServiceType,
|
||||
category path.CategoryType,
|
||||
backupIDs ...string,
|
||||
) ([]*BackupNode, error)
|
||||
}
|
||||
|
||||
// Repository contains storage provider information.
|
||||
@ -385,7 +426,7 @@ func newRepoID(s storage.Storage) string {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func errWrapper(err error) error {
|
||||
if errors.Is(err, data.ErrNotFound) {
|
||||
if errors.Is(err, errs.NotFound) {
|
||||
return clues.Stack(ErrorBackupNotFound, err)
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/common/errs"
|
||||
"github.com/alcionai/corso/src/internal/kopia"
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/operations"
|
||||
@ -107,11 +107,11 @@ func (suite *RepositoryBackupsUnitSuite) TestGetBackup() {
|
||||
name: "get error",
|
||||
sw: mock.BackupWrapper{
|
||||
Backup: bup,
|
||||
GetErr: data.ErrNotFound,
|
||||
GetErr: errs.NotFound,
|
||||
DeleteErr: nil,
|
||||
},
|
||||
expectErr: func(t *testing.T, result error) {
|
||||
assert.ErrorIs(t, result, data.ErrNotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, errs.NotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, ErrorBackupNotFound, clues.ToCore(result))
|
||||
},
|
||||
expectID: bup.ID,
|
||||
@ -446,14 +446,14 @@ func (suite *RepositoryBackupsUnitSuite) TestDeleteBackups() {
|
||||
bup.ID,
|
||||
},
|
||||
gets: []getRes{
|
||||
{err: data.ErrNotFound},
|
||||
{err: errs.NotFound},
|
||||
},
|
||||
expectGets: []model.StableID{
|
||||
bup.ID,
|
||||
},
|
||||
failOnMissing: true,
|
||||
expectErr: func(t *testing.T, result error) {
|
||||
assert.ErrorIs(t, result, data.ErrNotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, errs.NotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, ErrorBackupNotFound, clues.ToCore(result))
|
||||
},
|
||||
},
|
||||
@ -598,7 +598,7 @@ func (suite *RepositoryBackupsUnitSuite) TestDeleteBackups() {
|
||||
{bup: bup},
|
||||
{bup: bupLegacy},
|
||||
{bup: bupNoSnapshot},
|
||||
{err: data.ErrNotFound},
|
||||
{err: errs.NotFound},
|
||||
},
|
||||
expectGets: []model.StableID{
|
||||
bup.ID,
|
||||
@ -608,7 +608,7 @@ func (suite *RepositoryBackupsUnitSuite) TestDeleteBackups() {
|
||||
},
|
||||
failOnMissing: true,
|
||||
expectErr: func(t *testing.T, result error) {
|
||||
assert.ErrorIs(t, result, data.ErrNotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, errs.NotFound, clues.ToCore(result))
|
||||
assert.ErrorIs(t, result, ErrorBackupNotFound, clues.ToCore(result))
|
||||
},
|
||||
},
|
||||
@ -622,7 +622,7 @@ func (suite *RepositoryBackupsUnitSuite) TestDeleteBackups() {
|
||||
},
|
||||
gets: []getRes{
|
||||
{bup: bup},
|
||||
{err: data.ErrNotFound},
|
||||
{err: errs.NotFound},
|
||||
{bup: bupNoSnapshot},
|
||||
{bup: bupNoDetails},
|
||||
},
|
||||
@ -793,7 +793,7 @@ func writeBackup(
|
||||
ownerID, ownerName string,
|
||||
deets *details.Details,
|
||||
fe *fault.Errors,
|
||||
errs *fault.Bus,
|
||||
bus *fault.Bus,
|
||||
) *backup.Backup {
|
||||
var (
|
||||
serv = sel.PathService()
|
||||
@ -806,7 +806,7 @@ func writeBackup(
|
||||
err = sstore.Collect(ctx, streamstore.FaultErrorsCollector(fe))
|
||||
require.NoError(t, err, "collecting errors in streamstore")
|
||||
|
||||
ssid, err := sstore.Write(ctx, errs)
|
||||
ssid, err := sstore.Write(ctx, bus)
|
||||
require.NoError(t, err, "writing to streamstore")
|
||||
|
||||
tags := map[string]string{
|
||||
@ -822,6 +822,8 @@ func writeBackup(
|
||||
ownerID, ownerName,
|
||||
stats.ReadWrites{},
|
||||
stats.StartAndEndTime{},
|
||||
nil,
|
||||
nil,
|
||||
fe,
|
||||
tags)
|
||||
|
||||
|
||||
@ -37,6 +37,24 @@ func Service(pst path.ServiceType) FilterOption {
|
||||
}
|
||||
}
|
||||
|
||||
func Reason(service path.ServiceType, category path.CategoryType) FilterOption {
|
||||
return func(qf *queryFilters) {
|
||||
qf.tags[backup.ServiceCatString(service, category)] = backup.DefaultTagValue
|
||||
}
|
||||
}
|
||||
|
||||
func Tenant(tenant string) FilterOption {
|
||||
return func(qf *queryFilters) {
|
||||
qf.tags[backup.TenantIDKey] = tenant
|
||||
}
|
||||
}
|
||||
|
||||
func ProtectedResource(resourceID string) FilterOption {
|
||||
return func(qf *queryFilters) {
|
||||
qf.tags[backup.ResourceIDKey] = resourceID
|
||||
}
|
||||
}
|
||||
|
||||
type (
|
||||
BackupWrapper interface {
|
||||
BackupGetterDeleter
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user