Add struct functions for backup bases (#3595)
Move most of the stuff that was acting on backup bases to be functions that are defined for backup bases. Other code can be removed at a later point New functions aren't called yet in other code --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [x] 🧹 Tech Debt/Cleanup #### Issue(s) * #3525 #### Test Plan - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
ashmrtn
GitHub
parent
960e8b79a0
commit
7120164db6
387
src/internal/kopia/backup_bases.go
Normal file
387
src/internal/kopia/backup_bases.go
Normal file
@ -0,0 +1,387 @@
|
||||
package kopia
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
// TODO(ashmrtn): Move this into some inject package. Here to avoid import
|
||||
// cycles.
|
||||
type BackupBases interface {
|
||||
RemoveMergeBaseByManifestID(manifestID manifest.ID)
|
||||
Backups() []BackupEntry
|
||||
MinBackupVersion() int
|
||||
MergeBases() []ManifestEntry
|
||||
ClearMergeBases()
|
||||
AssistBases() []ManifestEntry
|
||||
ClearAssistBases()
|
||||
MergeBackupBases(
|
||||
ctx context.Context,
|
||||
other BackupBases,
|
||||
reasonToKey func(Reason) string,
|
||||
) BackupBases
|
||||
}
|
||||
|
||||
type backupBases struct {
|
||||
// backups and mergeBases should be modified together as they relate similar
|
||||
// data.
|
||||
backups []BackupEntry
|
||||
mergeBases []ManifestEntry
|
||||
assistBases []ManifestEntry
|
||||
}
|
||||
|
||||
func (bb *backupBases) RemoveMergeBaseByManifestID(manifestID manifest.ID) {
|
||||
idx := slices.IndexFunc(
|
||||
bb.mergeBases,
|
||||
func(man ManifestEntry) bool {
|
||||
return man.ID == manifestID
|
||||
})
|
||||
if idx >= 0 {
|
||||
bb.mergeBases = slices.Delete(bb.mergeBases, idx, idx+1)
|
||||
}
|
||||
|
||||
// TODO(ashmrtn): This may not be strictly necessary but is at least easier to
|
||||
// reason about.
|
||||
idx = slices.IndexFunc(
|
||||
bb.assistBases,
|
||||
func(man ManifestEntry) bool {
|
||||
return man.ID == manifestID
|
||||
})
|
||||
if idx >= 0 {
|
||||
bb.assistBases = slices.Delete(bb.assistBases, idx, idx+1)
|
||||
}
|
||||
|
||||
idx = slices.IndexFunc(
|
||||
bb.backups,
|
||||
func(bup BackupEntry) bool {
|
||||
return bup.SnapshotID == string(manifestID)
|
||||
})
|
||||
if idx >= 0 {
|
||||
bb.backups = slices.Delete(bb.backups, idx, idx+1)
|
||||
}
|
||||
}
|
||||
|
||||
func (bb backupBases) Backups() []BackupEntry {
|
||||
return slices.Clone(bb.backups)
|
||||
}
|
||||
|
||||
func (bb *backupBases) MinBackupVersion() int {
|
||||
min := version.NoBackup
|
||||
|
||||
if bb == nil {
|
||||
return min
|
||||
}
|
||||
|
||||
for _, bup := range bb.backups {
|
||||
if min == version.NoBackup || bup.Version < min {
|
||||
min = bup.Version
|
||||
}
|
||||
}
|
||||
|
||||
return min
|
||||
}
|
||||
|
||||
func (bb backupBases) MergeBases() []ManifestEntry {
|
||||
return slices.Clone(bb.mergeBases)
|
||||
}
|
||||
|
||||
func (bb *backupBases) ClearMergeBases() {
|
||||
bb.mergeBases = nil
|
||||
bb.backups = nil
|
||||
}
|
||||
|
||||
func (bb backupBases) AssistBases() []ManifestEntry {
|
||||
return slices.Clone(bb.assistBases)
|
||||
}
|
||||
|
||||
func (bb *backupBases) ClearAssistBases() {
|
||||
bb.assistBases = nil
|
||||
}
|
||||
|
||||
// MergeBackupBases reduces the two BackupBases into a single BackupBase.
|
||||
// Assumes the passed in BackupBases represents a prior backup version (across
|
||||
// some migration that disrupts lookup), and that the BackupBases used to call
|
||||
// this function contains the current version.
|
||||
//
|
||||
// reasonToKey should be a function that, given a Reason, will produce some
|
||||
// string that represents Reason in the context of the merge operation. For
|
||||
// example, to merge BackupBases across a ResourceOwner migration, the Reason's
|
||||
// service and category can be used as the key.
|
||||
//
|
||||
// Selection priority, for each reason key generated by reasonsToKey, follows
|
||||
// these rules:
|
||||
// 1. If the called BackupBases has an entry for a given resaon, ignore the
|
||||
// other BackupBases matching that reason.
|
||||
// 2. If the the receiver BackupBases has only AssistBases, look for a matching
|
||||
// MergeBase manifest in the passed in BackupBases.
|
||||
// 3. If the called BackupBases has no entry for a reason, look for both
|
||||
// AssistBases and MergeBases in the passed in BackupBases.
|
||||
func (bb *backupBases) MergeBackupBases(
|
||||
ctx context.Context,
|
||||
other BackupBases,
|
||||
reasonToKey func(reason Reason) string,
|
||||
) BackupBases {
|
||||
if other == nil || (len(other.MergeBases()) == 0 && len(other.AssistBases()) == 0) {
|
||||
return bb
|
||||
}
|
||||
|
||||
if bb == nil || (len(bb.MergeBases()) == 0 && len(bb.AssistBases()) == 0) {
|
||||
return other
|
||||
}
|
||||
|
||||
toMerge := map[string]struct{}{}
|
||||
assist := map[string]struct{}{}
|
||||
|
||||
// Track the bases in bb.
|
||||
for _, m := range bb.mergeBases {
|
||||
for _, r := range m.Reasons {
|
||||
k := reasonToKey(r)
|
||||
|
||||
toMerge[k] = struct{}{}
|
||||
assist[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range bb.assistBases {
|
||||
for _, r := range m.Reasons {
|
||||
k := reasonToKey(r)
|
||||
assist[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
var toAdd []ManifestEntry
|
||||
|
||||
// Calculate the set of mergeBases to pull from other into this one.
|
||||
for _, m := range other.MergeBases() {
|
||||
useReasons := []Reason{}
|
||||
|
||||
for _, r := range m.Reasons {
|
||||
k := reasonToKey(r)
|
||||
if _, ok := toMerge[k]; ok {
|
||||
// Assume other contains prior manifest versions.
|
||||
// We don't want to stack a prior version incomplete onto
|
||||
// a current version's complete snapshot.
|
||||
continue
|
||||
}
|
||||
|
||||
useReasons = append(useReasons, r)
|
||||
}
|
||||
|
||||
if len(useReasons) > 0 {
|
||||
m.Reasons = useReasons
|
||||
toAdd = append(toAdd, m)
|
||||
}
|
||||
}
|
||||
|
||||
res := &backupBases{
|
||||
backups: bb.Backups(),
|
||||
mergeBases: bb.MergeBases(),
|
||||
assistBases: bb.AssistBases(),
|
||||
}
|
||||
|
||||
// Add new mergeBases and backups.
|
||||
for _, man := range toAdd {
|
||||
// Will get empty string if not found which is fine, it'll fail one of the
|
||||
// other checks.
|
||||
bID, _ := man.GetTag(TagBackupID)
|
||||
|
||||
bup, ok := getBackupByID(other.Backups(), bID)
|
||||
if !ok {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"not unioning snapshot missing backup",
|
||||
"other_manifest_id", man.ID,
|
||||
"other_backup_id", bID)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
bup.Reasons = man.Reasons
|
||||
|
||||
res.backups = append(res.backups, bup)
|
||||
res.mergeBases = append(res.mergeBases, man)
|
||||
res.assistBases = append(res.assistBases, man)
|
||||
}
|
||||
|
||||
// Add assistBases from other to this one as needed.
|
||||
for _, m := range other.AssistBases() {
|
||||
useReasons := []Reason{}
|
||||
|
||||
// Assume that all complete manifests in assist overlap with MergeBases.
|
||||
if len(m.IncompleteReason) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, r := range m.Reasons {
|
||||
k := reasonToKey(r)
|
||||
if _, ok := assist[k]; ok {
|
||||
// This reason is already covered by either:
|
||||
// * complete manifest in bb
|
||||
// * incomplete manifest in bb
|
||||
//
|
||||
// If it was already in the assist set then it must be the case that
|
||||
// it's newer than any complete manifests in other for the same reason.
|
||||
continue
|
||||
}
|
||||
|
||||
useReasons = append(useReasons, r)
|
||||
}
|
||||
|
||||
if len(useReasons) > 0 {
|
||||
m.Reasons = useReasons
|
||||
res.assistBases = append(res.assistBases, m)
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
func findNonUniqueManifests(
|
||||
ctx context.Context,
|
||||
manifests []ManifestEntry,
|
||||
) map[manifest.ID]struct{} {
|
||||
// ReasonKey -> manifests with that reason.
|
||||
reasons := map[string][]ManifestEntry{}
|
||||
toDrop := map[manifest.ID]struct{}{}
|
||||
|
||||
for _, man := range manifests {
|
||||
// Incomplete snapshots are used only for kopia-assisted incrementals. The
|
||||
// fact that we need this check here makes it seem like this should live in
|
||||
// the kopia code. However, keeping it here allows for better debugging as
|
||||
// the kopia code only has access to a path builder which means it cannot
|
||||
// remove the resource owner from the error/log output. That is also below
|
||||
// the point where we decide if we should do a full backup or an incremental.
|
||||
if len(man.IncompleteReason) > 0 {
|
||||
logger.Ctx(ctx).Infow(
|
||||
"dropping incomplete manifest",
|
||||
"manifest_id", man.ID)
|
||||
|
||||
toDrop[man.ID] = struct{}{}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
for _, reason := range man.Reasons {
|
||||
reasonKey := reason.ResourceOwner + reason.Service.String() + reason.Category.String()
|
||||
reasons[reasonKey] = append(reasons[reasonKey], man)
|
||||
}
|
||||
}
|
||||
|
||||
for reason, mans := range reasons {
|
||||
ictx := clues.Add(ctx, "reason", reason)
|
||||
|
||||
if len(mans) == 0 {
|
||||
// Not sure how this would happen but just in case...
|
||||
continue
|
||||
} else if len(mans) > 1 {
|
||||
mIDs := make([]manifest.ID, 0, len(mans))
|
||||
for _, m := range mans {
|
||||
toDrop[m.ID] = struct{}{}
|
||||
mIDs = append(mIDs, m.ID)
|
||||
}
|
||||
|
||||
// TODO(ashmrtn): We should actually just remove this reason from the
|
||||
// manifests and then if they have no reasons remaining drop them from the
|
||||
// set.
|
||||
logger.Ctx(ictx).Infow(
|
||||
"dropping manifests with duplicate reason",
|
||||
"manifest_ids", mIDs)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return toDrop
|
||||
}
|
||||
|
||||
func getBackupByID(backups []BackupEntry, bID string) (BackupEntry, bool) {
|
||||
if len(bID) == 0 {
|
||||
return BackupEntry{}, false
|
||||
}
|
||||
|
||||
idx := slices.IndexFunc(backups, func(b BackupEntry) bool {
|
||||
return string(b.ID) == bID
|
||||
})
|
||||
|
||||
if idx < 0 || idx >= len(backups) {
|
||||
return BackupEntry{}, false
|
||||
}
|
||||
|
||||
return backups[idx], true
|
||||
}
|
||||
|
||||
// fixupAndVerify goes through the set of backups and snapshots used for merging
|
||||
// and ensures:
|
||||
// - the reasons for selecting merge snapshots are distinct
|
||||
// - all bases used for merging have a backup model with item and details
|
||||
// snapshot ID
|
||||
//
|
||||
// Backups that have overlapping reasons or that are not complete are removed
|
||||
// from the set. Dropping these is safe because it only affects how much data we
|
||||
// pull. On the other hand, *not* dropping them is unsafe as it will muck up
|
||||
// merging when we add stuff to kopia (possibly multiple entries for the same
|
||||
// item etc).
|
||||
func (bb *backupBases) fixupAndVerify(ctx context.Context) {
|
||||
toDrop := findNonUniqueManifests(ctx, bb.mergeBases)
|
||||
|
||||
var (
|
||||
backupsToKeep []BackupEntry
|
||||
mergeToKeep []ManifestEntry
|
||||
)
|
||||
|
||||
for _, man := range bb.mergeBases {
|
||||
if _, ok := toDrop[man.ID]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
bID, _ := man.GetTag(TagBackupID)
|
||||
|
||||
bup, ok := getBackupByID(bb.backups, bID)
|
||||
if !ok {
|
||||
toDrop[man.ID] = struct{}{}
|
||||
|
||||
logger.Ctx(ctx).Info(
|
||||
"dropping manifest due to missing backup",
|
||||
"manifest_id", man.ID)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
deetsID := bup.StreamStoreID
|
||||
if len(deetsID) == 0 {
|
||||
deetsID = bup.DetailsID
|
||||
}
|
||||
|
||||
if len(bup.SnapshotID) == 0 || len(deetsID) == 0 {
|
||||
toDrop[man.ID] = struct{}{}
|
||||
|
||||
logger.Ctx(ctx).Info(
|
||||
"dropping manifest due to invalid backup",
|
||||
"manifest_id", man.ID)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
backupsToKeep = append(backupsToKeep, bup)
|
||||
mergeToKeep = append(mergeToKeep, man)
|
||||
}
|
||||
|
||||
var assistToKeep []ManifestEntry
|
||||
|
||||
for _, man := range bb.assistBases {
|
||||
if _, ok := toDrop[man.ID]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
assistToKeep = append(assistToKeep, man)
|
||||
}
|
||||
|
||||
bb.backups = backupsToKeep
|
||||
bb.mergeBases = mergeToKeep
|
||||
bb.assistBases = assistToKeep
|
||||
}
|
||||
705
src/internal/kopia/backup_bases_test.go
Normal file
705
src/internal/kopia/backup_bases_test.go
Normal file
@ -0,0 +1,705 @@
|
||||
package kopia
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/kopia/kopia/repo/manifest"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/model"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/internal/version"
|
||||
"github.com/alcionai/corso/src/pkg/backup"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
func makeManifest(id, incmpl, bID string, reasons ...Reason) ManifestEntry {
|
||||
bIDKey, _ := makeTagKV(TagBackupID)
|
||||
|
||||
return ManifestEntry{
|
||||
Manifest: &snapshot.Manifest{
|
||||
ID: manifest.ID(id),
|
||||
IncompleteReason: incmpl,
|
||||
Tags: map[string]string{bIDKey: bID},
|
||||
},
|
||||
Reasons: reasons,
|
||||
}
|
||||
}
|
||||
|
||||
type BackupBasesUnitSuite struct {
|
||||
tester.Suite
|
||||
}
|
||||
|
||||
func TestBackupBasesUnitSuite(t *testing.T) {
|
||||
suite.Run(t, &BackupBasesUnitSuite{Suite: tester.NewUnitSuite(t)})
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestMinBackupVersion() {
|
||||
table := []struct {
|
||||
name string
|
||||
bb *backupBases
|
||||
expectedVersion int
|
||||
}{
|
||||
{
|
||||
name: "Nil BackupBase",
|
||||
expectedVersion: version.NoBackup,
|
||||
},
|
||||
{
|
||||
name: "No Backups",
|
||||
bb: &backupBases{},
|
||||
expectedVersion: version.NoBackup,
|
||||
},
|
||||
{
|
||||
name: "Unsorted Backups",
|
||||
bb: &backupBases{
|
||||
backups: []BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
Version: 4,
|
||||
},
|
||||
},
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
Version: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
Version: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedVersion: 0,
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
assert.Equal(suite.T(), test.expectedVersion, test.bb.MinBackupVersion())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestRemoveMergeBaseByManifestID() {
|
||||
backups := []BackupEntry{
|
||||
{Backup: &backup.Backup{SnapshotID: "1"}},
|
||||
{Backup: &backup.Backup{SnapshotID: "2"}},
|
||||
{Backup: &backup.Backup{SnapshotID: "3"}},
|
||||
}
|
||||
|
||||
merges := []ManifestEntry{
|
||||
makeManifest("1", "", ""),
|
||||
makeManifest("2", "", ""),
|
||||
makeManifest("3", "", ""),
|
||||
}
|
||||
|
||||
expected := &backupBases{
|
||||
backups: []BackupEntry{backups[0], backups[1]},
|
||||
mergeBases: []ManifestEntry{merges[0], merges[1]},
|
||||
assistBases: []ManifestEntry{merges[0], merges[1]},
|
||||
}
|
||||
|
||||
delID := manifest.ID("3")
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
// Below indices specify which items to add from the defined sets above.
|
||||
backup []int
|
||||
merge []int
|
||||
assist []int
|
||||
}{
|
||||
{
|
||||
name: "Not In Bases",
|
||||
backup: []int{0, 1},
|
||||
merge: []int{0, 1},
|
||||
assist: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "Different Indexes",
|
||||
backup: []int{2, 0, 1},
|
||||
merge: []int{0, 2, 1},
|
||||
assist: []int{0, 1, 2},
|
||||
},
|
||||
{
|
||||
name: "First Item",
|
||||
backup: []int{2, 0, 1},
|
||||
merge: []int{2, 0, 1},
|
||||
assist: []int{2, 0, 1},
|
||||
},
|
||||
{
|
||||
name: "Middle Item",
|
||||
backup: []int{0, 2, 1},
|
||||
merge: []int{0, 2, 1},
|
||||
assist: []int{0, 2, 1},
|
||||
},
|
||||
{
|
||||
name: "Final Item",
|
||||
backup: []int{0, 1, 2},
|
||||
merge: []int{0, 1, 2},
|
||||
assist: []int{0, 1, 2},
|
||||
},
|
||||
{
|
||||
name: "Only In Backups",
|
||||
backup: []int{0, 1, 2},
|
||||
merge: []int{0, 1},
|
||||
assist: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "Only In Merges",
|
||||
backup: []int{0, 1},
|
||||
merge: []int{0, 1, 2},
|
||||
assist: []int{0, 1},
|
||||
},
|
||||
{
|
||||
name: "Only In Assists",
|
||||
backup: []int{0, 1},
|
||||
merge: []int{0, 1},
|
||||
assist: []int{0, 1, 2},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
bb := &backupBases{}
|
||||
|
||||
for _, i := range test.backup {
|
||||
bb.backups = append(bb.backups, backups[i])
|
||||
}
|
||||
|
||||
for _, i := range test.merge {
|
||||
bb.mergeBases = append(bb.mergeBases, merges[i])
|
||||
}
|
||||
|
||||
for _, i := range test.assist {
|
||||
bb.assistBases = append(bb.assistBases, merges[i])
|
||||
}
|
||||
|
||||
bb.RemoveMergeBaseByManifestID(delID)
|
||||
AssertBackupBasesEqual(t, expected, bb)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestClearMergeBases() {
|
||||
bb := &backupBases{
|
||||
backups: make([]BackupEntry, 2),
|
||||
mergeBases: make([]ManifestEntry, 2),
|
||||
}
|
||||
|
||||
bb.ClearMergeBases()
|
||||
assert.Empty(suite.T(), bb.Backups())
|
||||
assert.Empty(suite.T(), bb.MergeBases())
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestClearAssistBases() {
|
||||
bb := &backupBases{assistBases: make([]ManifestEntry, 2)}
|
||||
|
||||
bb.ClearAssistBases()
|
||||
assert.Empty(suite.T(), bb.AssistBases())
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
|
||||
ro := "resource_owner"
|
||||
|
||||
type testInput struct {
|
||||
id int
|
||||
incomplete bool
|
||||
cat []path.CategoryType
|
||||
}
|
||||
|
||||
// Make a function so tests can modify things without messing with each other.
|
||||
makeBackupBases := func(ti []testInput) *backupBases {
|
||||
res := &backupBases{}
|
||||
|
||||
for _, i := range ti {
|
||||
baseID := fmt.Sprintf("id%d", i.id)
|
||||
ir := ""
|
||||
|
||||
if i.incomplete {
|
||||
ir = "checkpoint"
|
||||
}
|
||||
|
||||
reasons := make([]Reason, 0, len(i.cat))
|
||||
|
||||
for _, c := range i.cat {
|
||||
reasons = append(reasons, Reason{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: c,
|
||||
})
|
||||
}
|
||||
|
||||
m := makeManifest(baseID, ir, "b"+baseID, reasons...)
|
||||
res.assistBases = append(res.assistBases, m)
|
||||
|
||||
if i.incomplete {
|
||||
continue
|
||||
}
|
||||
|
||||
b := BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)},
|
||||
SnapshotID: baseID,
|
||||
StreamStoreID: "ss" + baseID,
|
||||
},
|
||||
Reasons: reasons,
|
||||
}
|
||||
|
||||
res.backups = append(res.backups, b)
|
||||
res.mergeBases = append(res.mergeBases, m)
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
bb []testInput
|
||||
other []testInput
|
||||
expect []testInput
|
||||
}{
|
||||
{
|
||||
name: "Other Empty",
|
||||
bb: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "BB Empty",
|
||||
other: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Other overlaps Complete And Incomplete",
|
||||
bb: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
other: []testInput{
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Other Overlaps Complete",
|
||||
bb: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
other: []testInput{
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Other Overlaps Incomplete",
|
||||
bb: []testInput{
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
other: []testInput{
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
expect: []testInput{
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Other Disjoint",
|
||||
bb: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
other: []testInput{
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Other Reduced Reasons",
|
||||
bb: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
other: []testInput{
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{
|
||||
path.EmailCategory,
|
||||
path.ContactsCategory,
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{
|
||||
path.EmailCategory,
|
||||
path.ContactsCategory,
|
||||
},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
expect: []testInput{
|
||||
{cat: []path.CategoryType{path.EmailCategory}},
|
||||
{
|
||||
id: 1,
|
||||
cat: []path.CategoryType{path.EmailCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
cat: []path.CategoryType{path.ContactsCategory},
|
||||
incomplete: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
t := suite.T()
|
||||
|
||||
bb := makeBackupBases(test.bb)
|
||||
other := makeBackupBases(test.other)
|
||||
expect := makeBackupBases(test.expect)
|
||||
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
got := bb.MergeBackupBases(
|
||||
ctx,
|
||||
other,
|
||||
func(reason Reason) string {
|
||||
return reason.Service.String() + reason.Category.String()
|
||||
})
|
||||
AssertBackupBasesEqual(t, expect, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
|
||||
ro := "resource_owner"
|
||||
|
||||
makeMan := func(pct path.CategoryType, id, incmpl, bID string) ManifestEntry {
|
||||
reason := Reason{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: pct,
|
||||
}
|
||||
|
||||
return makeManifest(id, incmpl, bID, reason)
|
||||
}
|
||||
|
||||
// Make a function so tests can modify things without messing with each other.
|
||||
validMail1 := func() *backupBases {
|
||||
return &backupBases{
|
||||
backups: []BackupEntry{
|
||||
{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid1",
|
||||
},
|
||||
SnapshotID: "id1",
|
||||
StreamStoreID: "ssid1",
|
||||
},
|
||||
},
|
||||
},
|
||||
mergeBases: []ManifestEntry{
|
||||
makeMan(path.EmailCategory, "id1", "", "bid1"),
|
||||
},
|
||||
assistBases: []ManifestEntry{
|
||||
makeMan(path.EmailCategory, "id1", "", "bid1"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
table := []struct {
|
||||
name string
|
||||
bb *backupBases
|
||||
expect BackupBases
|
||||
}{
|
||||
{
|
||||
name: "empty BaseBackups",
|
||||
bb: &backupBases{},
|
||||
},
|
||||
{
|
||||
name: "Merge Base Without Backup",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups = nil
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Backup Missing Snapshot ID",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups[0].SnapshotID = ""
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Backup Missing Deets ID",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups[0].StreamStoreID = ""
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Incomplete Snapshot",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].IncompleteReason = "ir"
|
||||
res.assistBases[0].IncompleteReason = "ir"
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Duplicate Reason",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].Reasons = append(
|
||||
res.mergeBases[0].Reasons,
|
||||
res.mergeBases[0].Reasons[0])
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Single Valid Entry",
|
||||
bb: validMail1(),
|
||||
expect: validMail1(),
|
||||
},
|
||||
{
|
||||
name: "Single Valid Entry With Incomplete Assist With Same Reason",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.assistBases = append(
|
||||
res.assistBases,
|
||||
makeMan(path.EmailCategory, "id2", "checkpoint", "bid2"))
|
||||
|
||||
return res
|
||||
}(),
|
||||
expect: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.assistBases = append(
|
||||
res.assistBases,
|
||||
makeMan(path.EmailCategory, "id2", "checkpoint", "bid2"))
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Single Valid Entry With Backup With Old Deets ID",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups[0].DetailsID = res.backups[0].StreamStoreID
|
||||
res.backups[0].StreamStoreID = ""
|
||||
|
||||
return res
|
||||
}(),
|
||||
expect: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups[0].DetailsID = res.backups[0].StreamStoreID
|
||||
res.backups[0].StreamStoreID = ""
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Single Valid Entry With Multiple Reasons",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].Reasons = append(
|
||||
res.mergeBases[0].Reasons,
|
||||
Reason{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
})
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
expect: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.mergeBases[0].Reasons = append(
|
||||
res.mergeBases[0].Reasons,
|
||||
Reason{
|
||||
ResourceOwner: ro,
|
||||
Service: path.ExchangeService,
|
||||
Category: path.ContactsCategory,
|
||||
})
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Two Entries Overlapping Reasons",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.mergeBases = append(
|
||||
res.mergeBases,
|
||||
makeMan(path.EmailCategory, "id2", "", "bid2"))
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "Three Entries One Invalid",
|
||||
bb: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups = append(
|
||||
res.backups,
|
||||
BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid2",
|
||||
},
|
||||
},
|
||||
},
|
||||
BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid3",
|
||||
},
|
||||
SnapshotID: "id3",
|
||||
StreamStoreID: "ssid3",
|
||||
},
|
||||
})
|
||||
res.mergeBases = append(
|
||||
res.mergeBases,
|
||||
makeMan(path.ContactsCategory, "id2", "checkpoint", "bid2"),
|
||||
makeMan(path.EventsCategory, "id3", "", "bid3"))
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
expect: func() *backupBases {
|
||||
res := validMail1()
|
||||
res.backups = append(
|
||||
res.backups,
|
||||
BackupEntry{
|
||||
Backup: &backup.Backup{
|
||||
BaseModel: model.BaseModel{
|
||||
ID: "bid3",
|
||||
},
|
||||
SnapshotID: "id3",
|
||||
StreamStoreID: "ssid3",
|
||||
},
|
||||
})
|
||||
res.mergeBases = append(
|
||||
res.mergeBases,
|
||||
makeMan(path.EventsCategory, "id3", "", "bid3"))
|
||||
res.assistBases = res.mergeBases
|
||||
|
||||
return res
|
||||
}(),
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
ctx, flush := tester.NewContext(suite.T())
|
||||
defer flush()
|
||||
|
||||
test.bb.fixupAndVerify(ctx)
|
||||
AssertBackupBasesEqual(suite.T(), test.expect, test.bb)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -47,12 +47,6 @@ func (r Reason) Key() string {
|
||||
return r.ResourceOwner + r.Service.String() + r.Category.String()
|
||||
}
|
||||
|
||||
type backupBases struct {
|
||||
backups []BackupEntry
|
||||
mergeBases []ManifestEntry
|
||||
assistBases []ManifestEntry
|
||||
}
|
||||
|
||||
type BackupEntry struct {
|
||||
*backup.Backup
|
||||
Reasons []Reason
|
||||
|
||||
@ -342,8 +342,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_NoBackupsOrSnapshots() {
|
||||
|
||||
bb, err := bf.findBases(ctx, reasons, nil)
|
||||
assert.NoError(t, err, "getting bases: %v", clues.ToCore(err))
|
||||
assert.Empty(t, bb.mergeBases)
|
||||
assert.Empty(t, bb.assistBases)
|
||||
assert.Empty(t, bb.MergeBases())
|
||||
assert.Empty(t, bb.AssistBases())
|
||||
}
|
||||
|
||||
func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() {
|
||||
@ -366,8 +366,8 @@ func (suite *BaseFinderUnitSuite) TestNoResult_ErrorListingSnapshots() {
|
||||
|
||||
bb, err := bf.findBases(ctx, reasons, nil)
|
||||
assert.NoError(t, err, "getting bases: %v", clues.ToCore(err))
|
||||
assert.Empty(t, bb.mergeBases)
|
||||
assert.Empty(t, bb.assistBases)
|
||||
assert.Empty(t, bb.MergeBases())
|
||||
assert.Empty(t, bb.AssistBases())
|
||||
}
|
||||
|
||||
func (suite *BaseFinderUnitSuite) TestGetBases() {
|
||||
@ -831,24 +831,24 @@ func (suite *BaseFinderUnitSuite) TestGetBases() {
|
||||
|
||||
checkBackupEntriesMatch(
|
||||
t,
|
||||
bb.backups,
|
||||
bb.Backups(),
|
||||
test.backupData,
|
||||
test.expectedBaseReasons)
|
||||
checkManifestEntriesMatch(
|
||||
t,
|
||||
bb.mergeBases,
|
||||
bb.MergeBases(),
|
||||
test.manifestData,
|
||||
test.expectedBaseReasons)
|
||||
checkManifestEntriesMatch(
|
||||
t,
|
||||
bb.assistBases,
|
||||
bb.AssistBases(),
|
||||
test.manifestData,
|
||||
test.expectedAssistManifestReasons)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() {
|
||||
func (suite *BaseFinderUnitSuite) TestFindBases_CustomTags() {
|
||||
manifestData := []manifestInfo{
|
||||
newManifestInfo2(
|
||||
testID1,
|
||||
@ -926,7 +926,7 @@ func (suite *BaseFinderUnitSuite) TestFetchPrevSnapshots_CustomTags() {
|
||||
|
||||
checkManifestEntriesMatch(
|
||||
t,
|
||||
bb.mergeBases,
|
||||
bb.MergeBases(),
|
||||
manifestData,
|
||||
test.expectedIdxs)
|
||||
})
|
||||
|
||||
63
src/internal/kopia/mock_backup_base.go
Normal file
63
src/internal/kopia/mock_backup_base.go
Normal file
@ -0,0 +1,63 @@
|
||||
package kopia
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func AssertBackupBasesEqual(t *testing.T, expect, got BackupBases) {
|
||||
if expect == nil && got == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if expect == nil {
|
||||
assert.Empty(t, got.Backups(), "backups")
|
||||
assert.Empty(t, got.MergeBases(), "merge bases")
|
||||
assert.Empty(t, got.AssistBases(), "assist bases")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if got == nil {
|
||||
if len(expect.Backups()) > 0 && len(expect.MergeBases()) > 0 && len(expect.AssistBases()) > 0 {
|
||||
assert.Fail(t, "got was nil but expected non-nil result %v", expect)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, expect.Backups(), got.Backups(), "backups")
|
||||
assert.ElementsMatch(t, expect.MergeBases(), got.MergeBases(), "merge bases")
|
||||
assert.ElementsMatch(t, expect.AssistBases(), got.AssistBases(), "assist bases")
|
||||
}
|
||||
|
||||
func NewMockBackupBases() *MockBackupBases {
|
||||
return &MockBackupBases{backupBases: &backupBases{}}
|
||||
}
|
||||
|
||||
type MockBackupBases struct {
|
||||
*backupBases
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithBackups(b ...BackupEntry) *MockBackupBases {
|
||||
bb.backupBases.backups = append(bb.Backups(), b...)
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithMergeBases(m ...ManifestEntry) *MockBackupBases {
|
||||
bb.backupBases.mergeBases = append(bb.MergeBases(), m...)
|
||||
bb.backupBases.assistBases = append(bb.AssistBases(), m...)
|
||||
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) WithAssistBases(m ...ManifestEntry) *MockBackupBases {
|
||||
bb.backupBases.assistBases = append(bb.AssistBases(), m...)
|
||||
return bb
|
||||
}
|
||||
|
||||
func (bb *MockBackupBases) ClearMockAssistBases() *MockBackupBases {
|
||||
bb.backupBases.ClearAssistBases()
|
||||
return bb
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user