larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Say reasons why a base snapshot was selected (#1811)

Browse Source 
## Description

For each snapshot manifest returned when looking for previous snapshots for a given set of owners cats, return the reason the snapshot was selected. This will allow other code to select the correct metadata and base snapshot subtree(s) when making incremental backups.

An example of when all metadata and directories in a base snapshot may not be needed is
```text
backup create exchange --data email,contacts --users user1 -> B1

// uses B1 as the base
backup create exchange --data email --users user1 -> B2

// uses B1 as the base for contacts and B2 as the base for email
backup create exchange --data email,contacts --users user1 -> B3
```

## Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No 

## Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* closes #1779

## Test Plan

- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2022-12-14 14:09:17 -08:00 committed by GitHub
parent a10c1c6bbd
commit 01b8111404
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 462 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/internal/kopia/snapshot_manager.go
View File

@ -25,6 +25,24 @@ const (
userTagPrefix = "tag:"
)
type Reason struct {
ResourceOwner string
Service path.ServiceType
Category path.CategoryType
}
type ManifestEntry struct {
*snapshot.Manifest
// Reason contains the ResourceOwners and Service/Categories that caused this
// snapshot to be selected as a base. We can't reuse OwnersCats here because
// it's possible some ResourceOwners will have a subset of the Categories as
// the reason for selecting a snapshot. For example:
// 1. backup user1 email,contacts -> B1
// 2. backup user1 contacts -> B2 (uses B1 as base)
// 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts)
Reasons []Reason
}
type snapshotManager interface {
FindManifests(
ctx context.Context,
@ -91,12 +109,12 @@ func tagsFromStrings(oc *OwnersCats) map[string]string {
}
// getLastIdx searches for manifests contained in both foundMans and metas
// and returns the most recent complete manifest index. If no complete manifest
// is in both lists returns -1.
// and returns the most recent complete manifest index and the manifest it
// corresponds to. If no complete manifest is in both lists returns nil, -1.
func getLastIdx(
foundMans map[manifest.ID]*snapshot.Manifest,
foundMans map[manifest.ID]*ManifestEntry,
metas []*manifest.EntryMetadata,
) int {
) (*ManifestEntry, int) {
// Minor optimization: the current code seems to return the entries from
// earliest timestamp to latest (this is undocumented). Sort in the same
// fashion so that we don't incur a bunch of swaps.
@ -111,24 +129,25 @@ func getLastIdx(
continue
}
return i
return m, i
}
return -1
return nil, -1
}
// manifestsSinceLastComplete searches through mans and returns the most recent
// complete manifest (if one exists) and maybe the most recent incomplete
// manifest. If the newest incomplete manifest is more recent than the newest
// complete manifest then adds it to the returned list. Otherwise no incomplete
// manifest is returned. Returns nil if there are no complete or incomplete
// manifests in mans.
// complete manifest (if one exists), maybe the most recent incomplete
// manifest, and a bool denoting if a complete manifest was found. If the newest
// incomplete manifest is more recent than the newest complete manifest then
// adds it to the returned list. Otherwise no incomplete manifest is returned.
// Returns nil if there are no complete or incomplete manifests in mans.
func manifestsSinceLastComplete(
mans []*snapshot.Manifest,
) []*snapshot.Manifest {
) ([]*snapshot.Manifest, bool) {
var (
res []*snapshot.Manifest
foundIncomplete = false
foundIncomplete bool
foundComplete bool
)
// Manifests should maintain the sort order of the original IDs that were used
@ -151,11 +170,12 @@ func manifestsSinceLastComplete(
// Once we find a complete snapshot we're done, even if we haven't
// found an incomplete one yet.
res = append(res, m)
foundComplete = true
break
}
return res
return res, foundComplete
}
// fetchPrevManifests returns the most recent, as-of-yet unfound complete and
@ -166,10 +186,29 @@ func manifestsSinceLastComplete(
func fetchPrevManifests(
ctx context.Context,
sm snapshotManager,
foundMans map[manifest.ID]*snapshot.Manifest,
foundMans map[manifest.ID]*ManifestEntry,
serviceCat ServiceCat,
resourceOwner string,
tags map[string]string,
) ([]*snapshot.Manifest, error) {
metas, err := sm.FindManifests(ctx, tags)
) ([]*ManifestEntry, error) {
tags = normalizeTagKVs(tags)
serviceCatKey, _ := MakeServiceCat(serviceCat.Service, serviceCat.Category)
allTags := normalizeTagKVs(map[string]string{
serviceCatKey: "",
resourceOwner: "",
})
for k, v := range tags {
allTags[k] = v
}
reason := Reason{
ResourceOwner: resourceOwner,
Service: serviceCat.Service,
Category: serviceCat.Category,
}
metas, err := sm.FindManifests(ctx, allTags)
if err != nil {
return nil, errors.Wrap(err, "fetching manifest metas by tag")
}
@ -178,11 +217,12 @@ func fetchPrevManifests(
return nil, nil
}
lastCompleteIdx := getLastIdx(foundMans, metas)
man, lastCompleteIdx := getLastIdx(foundMans, metas)
// We have a complete cached snapshot and it's the most recent. No need
// to do anything else.
if lastCompleteIdx == len(metas)-1 {
man.Reasons = append(man.Reasons, reason)
return nil, nil
}
@ -203,7 +243,24 @@ func fetchPrevManifests(
return nil, errors.Wrap(err, "fetching previous manifests")
}
return manifestsSinceLastComplete(mans), nil
found, hasCompleted := manifestsSinceLastComplete(mans)
res := make([]*ManifestEntry, 0, len(found))
for _, m := range found {
res = append(res, &ManifestEntry{
Manifest: m,
Reasons: []Reason{reason},
})
}
// If we didn't find another complete manifest then we need to mark the
// previous complete manifest as having this ResourceOwner, Service, Category
// as the reason as well.
if !hasCompleted && man != nil {
man.Reasons = append(man.Reasons, reason)
}
return res, nil
}
// fetchPrevSnapshotManifests returns a set of manifests for complete and maybe
@ -221,30 +278,27 @@ func fetchPrevSnapshotManifests(
sm snapshotManager,
oc *OwnersCats,
tags map[string]string,
) []*snapshot.Manifest {
) []*ManifestEntry {
if oc == nil {
return nil
}
mans := map[manifest.ID]*snapshot.Manifest{}
tags = normalizeTagKVs(tags)
mans := map[manifest.ID]*ManifestEntry{}
// For each serviceCat/resource owner pair that we will be backing up, see if
// there's a previous incomplete snapshot and/or a previous complete snapshot
// we can pass in. Can be expanded to return more than the most recent
// snapshots, but may require more memory at runtime.
for serviceCat := range oc.ServiceCats {
for _, serviceCat := range oc.ServiceCats {
for resourceOwner := range oc.ResourceOwners {
allTags := normalizeTagKVs(map[string]string{
serviceCat: "",
resourceOwner: "",
})
for k, v := range tags {
allTags[k] = v
}
found, err := fetchPrevManifests(ctx, sm, mans, allTags)
found, err := fetchPrevManifests(
ctx,
sm,
mans,
serviceCat,
resourceOwner,
tags,
)
if err != nil {
logger.Ctx(ctx).Warnw(
"fetching previous snapshot manifests for service/category/resource owner",
@ -260,12 +314,25 @@ func fetchPrevSnapshotManifests(
// If we found more recent snapshots then add them.
for _, m := range found {
mans[m.ID] = m
found := mans[m.ID]
if found == nil {
mans[m.ID] = m
continue
}
// If the manifest already exists and it's incomplete then we should
// merge the reasons for consistency. This will become easier to handle
// once we update how checkpoint manifests are tagged.
if len(found.IncompleteReason) == 0 {
continue
}
found.Reasons = append(found.Reasons, m.Reasons...)
}
}
}
res := make([]*snapshot.Manifest, 0, len(mans))
res := make([]*ManifestEntry, 0, len(mans))
for _, m := range mans {
res = append(res, m)
}

358
src/internal/kopia/snapshot_manager_test.go
View File

@ -29,11 +29,19 @@ var (
testID2 = manifest.ID("snap2")
testID3 = manifest.ID("snap3")
testMail = path.ExchangeService.String() + path.EmailCategory.String()
testEvents = path.ExchangeService.String() + path.EventsCategory.String()
testUser1 = "user1"
testUser2 = "user2"
testUser3 = "user3"
testMail = path.ExchangeService.String() + path.EmailCategory.String()
testMailServiceCat = ServiceCat{
Service: path.ExchangeService,
Category: path.EmailCategory,
}
testEvents = path.ExchangeService.String() + path.EventsCategory.String()
testEventsServiceCat = ServiceCat{
Service: path.ExchangeService,
Category: path.EventsCategory,
}
testUser1 = "user1"
testUser2 = "user2"
testUser3 = "user3"
testAllUsersAllCats = &OwnersCats{
ResourceOwners: map[string]struct{}{
@ -42,8 +50,8 @@ var (
testUser3: {},
},
ServiceCats: map[string]ServiceCat{
testMail: {},
testEvents: {},
testMail: testMailServiceCat,
testEvents: testEventsServiceCat,
},
}
testAllUsersMail = &OwnersCats{
@ -53,7 +61,7 @@ var (
testUser3: {},
},
ServiceCats: map[string]ServiceCat{
testMail: {},
testMail: testMailServiceCat,
},
}
)
@ -174,6 +182,9 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
// defining data in a table while not repeating things between data and
// expected.
expectedIdxs []int
// Use this to denote the Reasons a manifest is selected. The int maps to
// the index of the manifest in data.
expectedReasons map[int][]Reason
// Expected number of times a manifest should try to be loaded from kopia.
// Used to check that caching is functioning properly.
expectedLoadCounts map[manifest.ID]int
@ -194,6 +205,40 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
},
@ -222,6 +267,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0, 1},
expectedReasons: map[int][]Reason{
0: {
{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
1: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
testID2: 1,
@ -251,6 +332,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0, 1},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
1: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
testID2: 3,
@ -280,6 +397,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{1},
expectedReasons: map[int][]Reason{
1: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
testID2: 1,
@ -300,6 +436,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 3,
},
@ -328,6 +483,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{1},
expectedReasons: map[int][]Reason{
1: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
testID2: 1,
@ -357,6 +531,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{1},
expectedReasons: map[int][]Reason{
1: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 3,
testID2: 3,
@ -384,6 +577,91 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0, 1},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
1: {
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 2,
testID2: 1,
},
},
{
name: "SomeCachedSomeNewerDifferentCategories",
input: testAllUsersAllCats,
data: []manifestInfo{
newManifestInfo(
testID1,
testT1,
testCompleteMan,
testMail,
testEvents,
testUser1,
testUser2,
testUser3,
),
newManifestInfo(
testID2,
testT2,
testCompleteMan,
testMail,
testUser3,
),
},
expectedIdxs: []int{0, 1},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EventsCategory,
},
},
1: {
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 2,
testID2: 1,
@ -411,6 +689,32 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
),
},
expectedIdxs: []int{0, 1},
expectedReasons: map[int][]Reason{
0: {
Reason{
ResourceOwner: testUser1,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser2,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
1: {
Reason{
ResourceOwner: testUser3,
Service: path.ExchangeService,
Category: path.EmailCategory,
},
},
},
expectedLoadCounts: map[manifest.ID]int{
testID1: 1,
testID2: 1,
@ -444,13 +748,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
snaps := fetchPrevSnapshotManifests(ctx, msm, test.input, nil)
// Check the proper snapshot manifests were returned.
expected := make([]*snapshot.Manifest, 0, len(test.expectedIdxs))
for _, i := range test.expectedIdxs {
expected = append(expected, test.data[i].man)
}
assert.ElementsMatch(t, expected, snaps)
got := make([]*snapshot.Manifest, 0, len(snaps))
for _, s := range snaps {
got = append(got, s.Manifest)
}
assert.ElementsMatch(t, expected, got)
// Check the resons for selecting each manifest are correct.
expectedReasons := make(map[manifest.ID][]Reason, len(test.expectedReasons))
for idx, reason := range test.expectedReasons {
expectedReasons[test.data[idx].man.ID] = reason
}
for _, found := range snaps {
reason, ok := expectedReasons[found.ID]
if !ok {
// Missing or extra snapshots will be reported by earlier checks.
continue
}
assert.ElementsMatch(
t,
reason,
found.Reasons,
"incorrect reasons for snapshot with ID %s",
found.ID,
)
}
// Check number of loads to make sure caching is working properly.
// Need to manually check because we don't know the order the
// user/service/category labels will be iterated over. For some tests this
// could cause more loads than the ideal case.
@ -542,7 +875,12 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots_customTags() {
expected = append(expected, data[i].man)
}
assert.ElementsMatch(t, expected, snaps)
got := make([]*snapshot.Manifest, 0, len(snaps))
for _, s := range snaps {
got = append(got, s.Manifest)
}
assert.ElementsMatch(t, expected, got)
// Need to manually check because we don't know the order the
// user/service/category labels will be iterated over. For some tests this

15
src/internal/kopia/wrapper.go
View File

@ -113,7 +113,7 @@ func (w *Wrapper) Close(ctx context.Context) error {
// TODO(ashmrtn): Use previousSnapshots parameter.
func (w Wrapper) BackupCollections(
ctx context.Context,
previousSnapshots []*snapshot.Manifest,
previousSnapshots []*ManifestEntry,
collections []data.Collection,
service path.ServiceType,
oc *OwnersCats,
@ -156,11 +156,16 @@ func (w Wrapper) makeSnapshotWithRoot(
addlTags map[string]string,
) (*BackupStats, error) {
var (
man *snapshot.Manifest
prevSnaps = fetchPrevSnapshotManifests(ctx, w.c, oc, nil)
bc = &stats.ByteCounter{}
man *snapshot.Manifest
prevSnapEntries = fetchPrevSnapshotManifests(ctx, w.c, oc, nil)
bc = &stats.ByteCounter{}
)
prevSnaps := make([]*snapshot.Manifest, 0, len(prevSnapEntries))
for _, ent := range prevSnapEntries {
prevSnaps = append(prevSnaps, ent.Manifest)
}
err := repo.WriteSession(
ctx,
w.c,
@ -414,7 +419,7 @@ func (w Wrapper) FetchPrevSnapshotManifests(
ctx context.Context,
oc *OwnersCats,
tags map[string]string,
) ([]*snapshot.Manifest, error) {
) ([]*ManifestEntry, error) {
if w.c == nil {
return nil, errors.WithStack(errNotConnected)
}

5
src/internal/operations/backup.go
View File

@ -6,7 +6,6 @@ import (
"github.com/google/uuid"
multierror "github.com/hashicorp/go-multierror"
"github.com/kopia/kopia/snapshot"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
@ -181,7 +180,7 @@ func produceManifestsAndMetadata(
sw *store.Wrapper,
oc *kopia.OwnersCats,
acct account.Account,
) ([]*snapshot.Manifest, []data.Collection, error) {
) ([]*kopia.ManifestEntry, []data.Collection, error) {
complete, closer := observe.MessageWithCompletion("Fetching backup heuristics:")
defer func() {
complete <- struct{}{}
@ -337,7 +336,7 @@ func consumeBackupDataCollections(
kw *kopia.Wrapper,
sel selectors.Selector,
oc *kopia.OwnersCats,
mans []*snapshot.Manifest,
mans []*kopia.ManifestEntry,
cs []data.Collection,
backupID model.StableID,
) (*kopia.BackupStats, *details.Details, error) {