corso/src/internal/kopia/backup_bases_test.go
ashmrtn f61448d650
Groups version bump (#4561)
Bump the backup version and force a full backup if
there's a backup for teams/groups that has base(s)
from an older version of corso

This will avoid propagating older details formats
forward. Those formats don't have all the data
newer formats do

This is mostly a stop-gap, a more robust solution
can be added later

Manually tested that it forces a full backup

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #4569

#### Test Plan

- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
2023-10-27 19:37:39 +00:00

1046 lines
23 KiB
Go

package kopia
import (
"fmt"
"testing"
"time"
"github.com/alcionai/clues"
"github.com/kopia/kopia/repo/manifest"
"github.com/kopia/kopia/snapshot"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/path"
)
func makeManifest(id, incmpl, bID string) *snapshot.Manifest {
bIDKey, _ := makeTagKV(TagBackupID)
return &snapshot.Manifest{
ID: manifest.ID(id),
IncompleteReason: incmpl,
Tags: map[string]string{bIDKey: bID},
}
}
type testInput struct {
tenant string
protectedResource string
id int
cat []path.CategoryType
isAssist bool
incompleteReason string
}
func makeBase(ti testInput) BackupBase {
baseID := fmt.Sprintf("id%d", ti.id)
reasons := make([]identity.Reasoner, 0, len(ti.cat))
for _, c := range ti.cat {
reasons = append(
reasons,
identity.NewReason(
ti.tenant,
ti.protectedResource,
path.ExchangeService,
c))
}
return BackupBase{
Backup: &backup.Backup{
BaseModel: model.BaseModel{ID: model.StableID("b" + baseID)},
SnapshotID: baseID,
StreamStoreID: "ss" + baseID,
},
ItemDataSnapshot: makeManifest(baseID, ti.incompleteReason, "b"+baseID),
Reasons: reasons,
}
}
// Make a function so tests can modify things without messing with each other.
func makeBackupBases(
mergeInputs []testInput,
assistInputs []testInput,
) *backupBases {
res := &backupBases{}
for _, i := range mergeInputs {
res.mergeBases = append(res.mergeBases, makeBase(i))
}
for _, i := range assistInputs {
i.isAssist = true
res.assistBases = append(res.assistBases, makeBase(i))
}
return res
}
type BackupBasesUnitSuite struct {
tester.Suite
}
func TestBackupBasesUnitSuite(t *testing.T) {
suite.Run(t, &BackupBasesUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *BackupBasesUnitSuite) TestBackupBases_minVersions() {
table := []struct {
name string
bb *backupBases
expectedBackupVersion int
expectedAssistVersion int
}{
{
name: "Nil BackupBase",
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: version.NoBackup,
},
{
name: "No Backups",
bb: &backupBases{},
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: version.NoBackup,
},
{
name: "Unsorted Backups",
bb: &backupBases{
mergeBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 0,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
},
},
expectedBackupVersion: 0,
expectedAssistVersion: version.NoBackup,
},
{
name: "Only Assist Bases",
bb: &backupBases{
assistBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 0,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
},
},
expectedBackupVersion: version.NoBackup,
expectedAssistVersion: 0,
},
{
name: "Assist and Merge Bases, min merge",
bb: &backupBases{
mergeBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 1,
},
},
{
Backup: &backup.Backup{
Version: 5,
},
},
{
Backup: &backup.Backup{
Version: 3,
},
},
},
assistBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
{
Backup: &backup.Backup{
Version: 6,
},
},
},
},
expectedBackupVersion: 1,
expectedAssistVersion: 2,
},
{
name: "Assist and Merge Bases, min assist",
bb: &backupBases{
mergeBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 7,
},
},
{
Backup: &backup.Backup{
Version: 5,
},
},
{
Backup: &backup.Backup{
Version: 3,
},
},
},
assistBases: []BackupBase{
{
Backup: &backup.Backup{
Version: 4,
},
},
{
Backup: &backup.Backup{
Version: 2,
},
},
{
Backup: &backup.Backup{
Version: 6,
},
},
},
},
expectedBackupVersion: 3,
expectedAssistVersion: 2,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expectedBackupVersion, test.bb.MinBackupVersion(), "backup")
assert.Equal(t, test.expectedAssistVersion, test.bb.MinAssistVersion(), "assist")
})
}
}
func (suite *BackupBasesUnitSuite) TestConvertToAssistBase() {
bases := []BackupBase{
{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "1",
},
SnapshotID: "its1",
},
ItemDataSnapshot: makeManifest("its1", "", ""),
},
{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "2",
},
SnapshotID: "its2",
},
ItemDataSnapshot: makeManifest("its2", "", ""),
},
{
Backup: &backup.Backup{
BaseModel: model.BaseModel{
ID: "3",
},
SnapshotID: "its3",
},
ItemDataSnapshot: makeManifest("its3", "", ""),
},
}
delID := model.StableID("3")
table := []struct {
name string
// Below indices specify which items to add from the defined sets above.
merge []int
assist []int
expectMerge []int
expectAssist []int
}{
{
name: "Not In Bases",
merge: []int{0, 1},
assist: []int{0, 1},
expectMerge: []int{0, 1},
expectAssist: []int{0, 1},
},
{
name: "First Item",
merge: []int{2, 0, 1},
assist: []int{0, 1},
expectMerge: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Middle Item",
merge: []int{0, 2, 1},
assist: []int{0, 1},
expectMerge: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Final Item",
merge: []int{0, 1, 2},
assist: []int{0, 1},
expectMerge: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
{
name: "Only In Assists Noops",
merge: []int{0, 1},
assist: []int{0, 1, 2},
expectMerge: []int{0, 1},
expectAssist: []int{0, 1, 2},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
bb := &backupBases{}
for _, i := range test.merge {
bb.mergeBases = append(bb.mergeBases, bases[i])
}
for _, i := range test.assist {
bb.assistBases = append(bb.assistBases, bases[i])
}
expected := &backupBases{}
for _, i := range test.expectMerge {
expected.mergeBases = append(expected.mergeBases, bases[i])
}
for _, i := range test.expectAssist {
expected.assistBases = append(expected.assistBases, bases[i])
}
bb.ConvertToAssistBase(delID)
AssertBackupBasesEqual(t, expected, bb)
})
}
}
func (suite *BackupBasesUnitSuite) TestDisableMergeBases() {
t := suite.T()
merge := []BackupBase{
{
Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "m1"}},
ItemDataSnapshot: &snapshot.Manifest{ID: "ms1"},
},
{
Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "m2"}},
ItemDataSnapshot: &snapshot.Manifest{ID: "ms2"},
},
}
assist := []BackupBase{
{
Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "a1"}},
ItemDataSnapshot: &snapshot.Manifest{ID: "as1"},
},
{
Backup: &backup.Backup{BaseModel: model.BaseModel{ID: "a2"}},
ItemDataSnapshot: &snapshot.Manifest{ID: "as2"},
},
}
bb := &backupBases{
mergeBases: merge,
assistBases: assist,
}
bb.DisableMergeBases()
assert.Empty(t, bb.MergeBases())
// Merge bases should still appear in the assist base set passed in for kopia
// snapshots and details merging.
assert.ElementsMatch(
t,
append(slices.Clone(merge), assist...),
bb.SnapshotAssistBases())
assert.ElementsMatch(
t,
append(slices.Clone(merge), assist...),
bb.UniqueAssistBases())
}
func (suite *BackupBasesUnitSuite) TestDisableAssistBases() {
t := suite.T()
bb := &backupBases{
mergeBases: make([]BackupBase, 2),
assistBases: make([]BackupBase, 2),
}
bb.DisableAssistBases()
assert.Empty(t, bb.UniqueAssistBases())
assert.Empty(t, bb.SnapshotAssistBases())
// Merge base should be unchanged.
assert.Len(t, bb.MergeBases(), 2)
}
func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
table := []struct {
name string
merge []testInput
assist []testInput
otherMerge []testInput
otherAssist []testInput
expect func() *backupBases
}{
{
name: "Other Empty",
merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
},
assist: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}},
}, []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
})
return bs
},
},
{
name: "current Empty",
otherMerge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
},
otherAssist: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}},
}, []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
})
return bs
},
},
{
name: "Other overlaps merge and assist",
merge: []testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
},
assist: []testInput{
{
id: 4,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherMerge: []testInput{
{
id: 2,
cat: []path.CategoryType{path.EmailCategory},
},
{
id: 3,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherAssist: []testInput{
{
id: 5,
cat: []path.CategoryType{path.EmailCategory},
},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
}, []testInput{
{
id: 4,
cat: []path.CategoryType{path.EmailCategory},
},
})
return bs
},
},
{
name: "Other overlaps merge",
merge: []testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherMerge: []testInput{
{
id: 2,
cat: []path.CategoryType{path.EmailCategory},
},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
}, nil)
return bs
},
},
{
name: "Current assist overlaps with Other merge",
assist: []testInput{
{
id: 3,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherMerge: []testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherAssist: []testInput{
{
id: 2,
cat: []path.CategoryType{path.EmailCategory},
},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
}, []testInput{
{
id: 3,
cat: []path.CategoryType{path.EmailCategory},
},
})
return bs
},
},
{
name: "Other Disjoint",
merge: []testInput{
{cat: []path.CategoryType{path.EmailCategory}},
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
},
otherMerge: []testInput{
{
id: 2,
cat: []path.CategoryType{path.ContactsCategory},
},
},
expect: func() *backupBases {
bs := makeBackupBases([]testInput{
{cat: []path.CategoryType{path.EmailCategory}},
{
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
{
id: 2,
cat: []path.CategoryType{path.ContactsCategory},
},
}, nil)
return bs
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
bb := makeBackupBases(test.merge, test.assist)
other := makeBackupBases(test.otherMerge, test.otherAssist)
expected := test.expect()
ctx, flush := tester.NewContext(t)
defer flush()
got := bb.MergeBackupBases(
ctx,
other,
func(r identity.Reasoner) string {
return r.Service().String() + r.Category().String()
})
AssertBackupBasesEqual(t, expected, got)
})
}
}
func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
ro := "resource_owner"
// Make a function so tests can modify things without messing with each other.
validMail1 := func() *backupBases {
return makeBackupBases(
[]testInput{
{
tenant: "t",
protectedResource: ro,
id: 1,
cat: []path.CategoryType{path.EmailCategory},
},
},
[]testInput{
{
tenant: "t",
protectedResource: ro,
id: 2,
cat: []path.CategoryType{path.EmailCategory},
},
})
}
table := []struct {
name string
bb *backupBases
expect BackupBases
}{
{
name: "EmptyBaseBackups",
bb: &backupBases{},
},
{
name: "MergeBase MissingBackup",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Backup = nil
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
return res
}(),
},
{
name: "MergeBase MissingSnapshot",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].ItemDataSnapshot = nil
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
return res
}(),
},
{
name: "AssistBase MissingBackup",
bb: func() *backupBases {
res := validMail1()
res.assistBases[0].Backup = nil
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = nil
return res
}(),
},
{
name: "AssistBase MissingSnapshot",
bb: func() *backupBases {
res := validMail1()
res.assistBases[0].ItemDataSnapshot = nil
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = nil
return res
}(),
},
{
name: "MergeBase MissingDeetsID",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Backup.StreamStoreID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = nil
return res
}(),
},
{
name: "AssistBase MissingDeetsID",
bb: func() *backupBases {
res := validMail1()
res.assistBases[0].Backup.StreamStoreID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = nil
return res
}(),
},
{
name: "MergeAndAssistBase IncompleteSnapshot",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].ItemDataSnapshot.IncompleteReason = "ir"
res.assistBases[0].ItemDataSnapshot.IncompleteReason = "ir"
return res
}(),
},
{
name: "MergeAndAssistBase DuplicateReasonInBase",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons,
res.mergeBases[0].Reasons[0])
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
res.assistBases[0].Reasons[0])
return res
}(),
expect: validMail1(),
},
{
name: "MergeAndAssistBase MissingReason",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Reasons = nil
res.assistBases[0].Reasons = nil
return res
}(),
},
{
name: "SingleValidEntry",
bb: validMail1(),
expect: validMail1(),
},
{
name: "SingleValidEntry IncompleteAssistWithSameReason",
bb: func() *backupBases {
res := validMail1()
res.assistBases = append(
res.assistBases,
makeBase(testInput{
tenant: "t",
protectedResource: "ro",
id: 3,
cat: []path.CategoryType{path.EmailCategory},
isAssist: true,
incompleteReason: "checkpoint",
}))
return res
}(),
expect: validMail1(),
},
{
name: "SingleValidEntry BackupWithOldDeetsID",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Backup.DetailsID = res.mergeBases[0].Backup.StreamStoreID
res.mergeBases[0].Backup.StreamStoreID = ""
res.assistBases[0].Backup.DetailsID = res.assistBases[0].Backup.StreamStoreID
res.assistBases[0].Backup.StreamStoreID = ""
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases[0].Backup.DetailsID = res.mergeBases[0].Backup.StreamStoreID
res.mergeBases[0].Backup.StreamStoreID = ""
res.assistBases[0].Backup.DetailsID = res.assistBases[0].Backup.StreamStoreID
res.assistBases[0].Backup.StreamStoreID = ""
return res
}(),
},
{
name: "SingleValidEntry MultipleReasons",
bb: func() *backupBases {
res := validMail1()
res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons,
identity.NewReason("t", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
identity.NewReason("t", ro, path.ExchangeService, path.ContactsCategory))
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases[0].Reasons = append(
res.mergeBases[0].Reasons,
identity.NewReason("t", ro, path.ExchangeService, path.ContactsCategory))
res.assistBases[0].Reasons = append(
res.assistBases[0].Reasons,
identity.NewReason("t", ro, path.ExchangeService, path.ContactsCategory))
return res
}(),
},
{
name: "TwoEntries OverlappingReasons",
bb: func() *backupBases {
t1, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
require.NoError(suite.T(), err, clues.ToCore(err))
res := validMail1()
res.mergeBases[0].Backup.CreationTime = t1
res.assistBases[0].Backup.CreationTime = t1.Add(2 * time.Hour)
other := makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 3,
cat: []path.CategoryType{path.EmailCategory},
})
other.Backup.CreationTime = t1.Add(-time.Minute)
res.mergeBases = append(res.mergeBases, other)
other = makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EmailCategory},
isAssist: true,
})
other.Backup.CreationTime = t1.Add(time.Hour)
res.assistBases = append(res.assistBases, other)
return res
}(),
expect: func() *backupBases {
t1, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
require.NoError(suite.T(), err, clues.ToCore(err))
res := validMail1()
res.mergeBases[0].Backup.CreationTime = t1
res.assistBases[0].Backup.CreationTime = t1.Add(2 * time.Hour)
return res
}(),
},
{
name: "TwoEntries OverlappingReasons OneInvalid",
bb: func() *backupBases {
t1, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
require.NoError(suite.T(), err, clues.ToCore(err))
res := validMail1()
res.mergeBases[0].Backup.CreationTime = t1
res.assistBases[0].Backup.CreationTime = t1.Add(2 * time.Hour)
other := makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 3,
cat: []path.CategoryType{path.EmailCategory},
})
other.Backup.CreationTime = t1.Add(time.Minute)
other.Backup.StreamStoreID = ""
res.mergeBases = append(res.mergeBases, other)
other = makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EmailCategory},
isAssist: true,
})
other.Backup.CreationTime = t1.Add(3 * time.Hour)
other.Backup.StreamStoreID = ""
res.assistBases = append(res.assistBases, other)
return res
}(),
expect: func() *backupBases {
t1, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
require.NoError(suite.T(), err, clues.ToCore(err))
res := validMail1()
res.mergeBases[0].Backup.CreationTime = t1
res.assistBases[0].Backup.CreationTime = t1.Add(2 * time.Hour)
return res
}(),
},
{
name: "MergeBase ThreeEntriesOneInvalid",
bb: func() *backupBases {
invalid := makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 3,
cat: []path.CategoryType{path.ContactsCategory},
incompleteReason: "checkpoint",
})
invalid.Backup.StreamStoreID = ""
invalid.Backup.SnapshotID = ""
res := validMail1()
res.mergeBases = append(
res.mergeBases,
invalid,
makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EventsCategory},
}))
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.mergeBases = append(
res.mergeBases,
makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EventsCategory},
}))
return res
}(),
},
{
name: "AssistBase ThreeEntriesOneInvalid",
bb: func() *backupBases {
invalid := makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 3,
cat: []path.CategoryType{path.ContactsCategory},
isAssist: true,
incompleteReason: "checkpoint",
})
invalid.Backup.StreamStoreID = ""
invalid.Backup.SnapshotID = ""
res := validMail1()
res.assistBases = append(
res.assistBases,
invalid,
makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EventsCategory},
isAssist: true,
}))
return res
}(),
expect: func() *backupBases {
res := validMail1()
res.assistBases = append(
res.assistBases,
makeBase(testInput{
tenant: "t",
protectedResource: ro,
id: 4,
cat: []path.CategoryType{path.EventsCategory},
isAssist: true,
}))
return res
}(),
},
}
for _, test := range table {
suite.Run(test.name, func() {
ctx, flush := tester.NewContext(suite.T())
defer flush()
test.bb.fixupAndVerify(ctx)
AssertBackupBasesEqual(suite.T(), test.expect, test.bb)
})
}
}