Say reasons why a base snapshot was selected (#1811)
## Description For each snapshot manifest returned when looking for previous snapshots for a given set of owners cats, return the reason the snapshot was selected. This will allow other code to select the correct metadata and base snapshot subtree(s) when making incremental backups. An example of when all metadata and directories in a base snapshot may not be needed is ```text backup create exchange --data email,contacts --users user1 -> B1 // uses B1 as the base backup create exchange --data email --users user1 -> B2 // uses B1 as the base for contacts and B2 as the base for email backup create exchange --data email,contacts --users user1 -> B3 ``` ## Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No ## Type of change - [x] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🐹 Trivial/Minor ## Issue(s) * closes #1779 ## Test Plan - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
a10c1c6bbd
commit
01b8111404
@ -25,6 +25,24 @@ const (
|
|||||||
userTagPrefix = "tag:"
|
userTagPrefix = "tag:"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type Reason struct {
|
||||||
|
ResourceOwner string
|
||||||
|
Service path.ServiceType
|
||||||
|
Category path.CategoryType
|
||||||
|
}
|
||||||
|
|
||||||
|
type ManifestEntry struct {
|
||||||
|
*snapshot.Manifest
|
||||||
|
// Reason contains the ResourceOwners and Service/Categories that caused this
|
||||||
|
// snapshot to be selected as a base. We can't reuse OwnersCats here because
|
||||||
|
// it's possible some ResourceOwners will have a subset of the Categories as
|
||||||
|
// the reason for selecting a snapshot. For example:
|
||||||
|
// 1. backup user1 email,contacts -> B1
|
||||||
|
// 2. backup user1 contacts -> B2 (uses B1 as base)
|
||||||
|
// 3. backup user1 email,contacts,events (uses B1 for email, B2 for contacts)
|
||||||
|
Reasons []Reason
|
||||||
|
}
|
||||||
|
|
||||||
type snapshotManager interface {
|
type snapshotManager interface {
|
||||||
FindManifests(
|
FindManifests(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@ -91,12 +109,12 @@ func tagsFromStrings(oc *OwnersCats) map[string]string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// getLastIdx searches for manifests contained in both foundMans and metas
|
// getLastIdx searches for manifests contained in both foundMans and metas
|
||||||
// and returns the most recent complete manifest index. If no complete manifest
|
// and returns the most recent complete manifest index and the manifest it
|
||||||
// is in both lists returns -1.
|
// corresponds to. If no complete manifest is in both lists returns nil, -1.
|
||||||
func getLastIdx(
|
func getLastIdx(
|
||||||
foundMans map[manifest.ID]*snapshot.Manifest,
|
foundMans map[manifest.ID]*ManifestEntry,
|
||||||
metas []*manifest.EntryMetadata,
|
metas []*manifest.EntryMetadata,
|
||||||
) int {
|
) (*ManifestEntry, int) {
|
||||||
// Minor optimization: the current code seems to return the entries from
|
// Minor optimization: the current code seems to return the entries from
|
||||||
// earliest timestamp to latest (this is undocumented). Sort in the same
|
// earliest timestamp to latest (this is undocumented). Sort in the same
|
||||||
// fashion so that we don't incur a bunch of swaps.
|
// fashion so that we don't incur a bunch of swaps.
|
||||||
@ -111,24 +129,25 @@ func getLastIdx(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
return i
|
return m, i
|
||||||
}
|
}
|
||||||
|
|
||||||
return -1
|
return nil, -1
|
||||||
}
|
}
|
||||||
|
|
||||||
// manifestsSinceLastComplete searches through mans and returns the most recent
|
// manifestsSinceLastComplete searches through mans and returns the most recent
|
||||||
// complete manifest (if one exists) and maybe the most recent incomplete
|
// complete manifest (if one exists), maybe the most recent incomplete
|
||||||
// manifest. If the newest incomplete manifest is more recent than the newest
|
// manifest, and a bool denoting if a complete manifest was found. If the newest
|
||||||
// complete manifest then adds it to the returned list. Otherwise no incomplete
|
// incomplete manifest is more recent than the newest complete manifest then
|
||||||
// manifest is returned. Returns nil if there are no complete or incomplete
|
// adds it to the returned list. Otherwise no incomplete manifest is returned.
|
||||||
// manifests in mans.
|
// Returns nil if there are no complete or incomplete manifests in mans.
|
||||||
func manifestsSinceLastComplete(
|
func manifestsSinceLastComplete(
|
||||||
mans []*snapshot.Manifest,
|
mans []*snapshot.Manifest,
|
||||||
) []*snapshot.Manifest {
|
) ([]*snapshot.Manifest, bool) {
|
||||||
var (
|
var (
|
||||||
res []*snapshot.Manifest
|
res []*snapshot.Manifest
|
||||||
foundIncomplete = false
|
foundIncomplete bool
|
||||||
|
foundComplete bool
|
||||||
)
|
)
|
||||||
|
|
||||||
// Manifests should maintain the sort order of the original IDs that were used
|
// Manifests should maintain the sort order of the original IDs that were used
|
||||||
@ -151,11 +170,12 @@ func manifestsSinceLastComplete(
|
|||||||
// Once we find a complete snapshot we're done, even if we haven't
|
// Once we find a complete snapshot we're done, even if we haven't
|
||||||
// found an incomplete one yet.
|
// found an incomplete one yet.
|
||||||
res = append(res, m)
|
res = append(res, m)
|
||||||
|
foundComplete = true
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
return res
|
return res, foundComplete
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchPrevManifests returns the most recent, as-of-yet unfound complete and
|
// fetchPrevManifests returns the most recent, as-of-yet unfound complete and
|
||||||
@ -166,10 +186,29 @@ func manifestsSinceLastComplete(
|
|||||||
func fetchPrevManifests(
|
func fetchPrevManifests(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
sm snapshotManager,
|
sm snapshotManager,
|
||||||
foundMans map[manifest.ID]*snapshot.Manifest,
|
foundMans map[manifest.ID]*ManifestEntry,
|
||||||
|
serviceCat ServiceCat,
|
||||||
|
resourceOwner string,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
) ([]*snapshot.Manifest, error) {
|
) ([]*ManifestEntry, error) {
|
||||||
metas, err := sm.FindManifests(ctx, tags)
|
tags = normalizeTagKVs(tags)
|
||||||
|
serviceCatKey, _ := MakeServiceCat(serviceCat.Service, serviceCat.Category)
|
||||||
|
allTags := normalizeTagKVs(map[string]string{
|
||||||
|
serviceCatKey: "",
|
||||||
|
resourceOwner: "",
|
||||||
|
})
|
||||||
|
|
||||||
|
for k, v := range tags {
|
||||||
|
allTags[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
reason := Reason{
|
||||||
|
ResourceOwner: resourceOwner,
|
||||||
|
Service: serviceCat.Service,
|
||||||
|
Category: serviceCat.Category,
|
||||||
|
}
|
||||||
|
|
||||||
|
metas, err := sm.FindManifests(ctx, allTags)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "fetching manifest metas by tag")
|
return nil, errors.Wrap(err, "fetching manifest metas by tag")
|
||||||
}
|
}
|
||||||
@ -178,11 +217,12 @@ func fetchPrevManifests(
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
lastCompleteIdx := getLastIdx(foundMans, metas)
|
man, lastCompleteIdx := getLastIdx(foundMans, metas)
|
||||||
|
|
||||||
// We have a complete cached snapshot and it's the most recent. No need
|
// We have a complete cached snapshot and it's the most recent. No need
|
||||||
// to do anything else.
|
// to do anything else.
|
||||||
if lastCompleteIdx == len(metas)-1 {
|
if lastCompleteIdx == len(metas)-1 {
|
||||||
|
man.Reasons = append(man.Reasons, reason)
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -203,7 +243,24 @@ func fetchPrevManifests(
|
|||||||
return nil, errors.Wrap(err, "fetching previous manifests")
|
return nil, errors.Wrap(err, "fetching previous manifests")
|
||||||
}
|
}
|
||||||
|
|
||||||
return manifestsSinceLastComplete(mans), nil
|
found, hasCompleted := manifestsSinceLastComplete(mans)
|
||||||
|
res := make([]*ManifestEntry, 0, len(found))
|
||||||
|
|
||||||
|
for _, m := range found {
|
||||||
|
res = append(res, &ManifestEntry{
|
||||||
|
Manifest: m,
|
||||||
|
Reasons: []Reason{reason},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we didn't find another complete manifest then we need to mark the
|
||||||
|
// previous complete manifest as having this ResourceOwner, Service, Category
|
||||||
|
// as the reason as well.
|
||||||
|
if !hasCompleted && man != nil {
|
||||||
|
man.Reasons = append(man.Reasons, reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
return res, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchPrevSnapshotManifests returns a set of manifests for complete and maybe
|
// fetchPrevSnapshotManifests returns a set of manifests for complete and maybe
|
||||||
@ -221,30 +278,27 @@ func fetchPrevSnapshotManifests(
|
|||||||
sm snapshotManager,
|
sm snapshotManager,
|
||||||
oc *OwnersCats,
|
oc *OwnersCats,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
) []*snapshot.Manifest {
|
) []*ManifestEntry {
|
||||||
if oc == nil {
|
if oc == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
mans := map[manifest.ID]*snapshot.Manifest{}
|
mans := map[manifest.ID]*ManifestEntry{}
|
||||||
tags = normalizeTagKVs(tags)
|
|
||||||
|
|
||||||
// For each serviceCat/resource owner pair that we will be backing up, see if
|
// For each serviceCat/resource owner pair that we will be backing up, see if
|
||||||
// there's a previous incomplete snapshot and/or a previous complete snapshot
|
// there's a previous incomplete snapshot and/or a previous complete snapshot
|
||||||
// we can pass in. Can be expanded to return more than the most recent
|
// we can pass in. Can be expanded to return more than the most recent
|
||||||
// snapshots, but may require more memory at runtime.
|
// snapshots, but may require more memory at runtime.
|
||||||
for serviceCat := range oc.ServiceCats {
|
for _, serviceCat := range oc.ServiceCats {
|
||||||
for resourceOwner := range oc.ResourceOwners {
|
for resourceOwner := range oc.ResourceOwners {
|
||||||
allTags := normalizeTagKVs(map[string]string{
|
found, err := fetchPrevManifests(
|
||||||
serviceCat: "",
|
ctx,
|
||||||
resourceOwner: "",
|
sm,
|
||||||
})
|
mans,
|
||||||
|
serviceCat,
|
||||||
for k, v := range tags {
|
resourceOwner,
|
||||||
allTags[k] = v
|
tags,
|
||||||
}
|
)
|
||||||
|
|
||||||
found, err := fetchPrevManifests(ctx, sm, mans, allTags)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Ctx(ctx).Warnw(
|
logger.Ctx(ctx).Warnw(
|
||||||
"fetching previous snapshot manifests for service/category/resource owner",
|
"fetching previous snapshot manifests for service/category/resource owner",
|
||||||
@ -260,12 +314,25 @@ func fetchPrevSnapshotManifests(
|
|||||||
|
|
||||||
// If we found more recent snapshots then add them.
|
// If we found more recent snapshots then add them.
|
||||||
for _, m := range found {
|
for _, m := range found {
|
||||||
mans[m.ID] = m
|
found := mans[m.ID]
|
||||||
|
if found == nil {
|
||||||
|
mans[m.ID] = m
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the manifest already exists and it's incomplete then we should
|
||||||
|
// merge the reasons for consistency. This will become easier to handle
|
||||||
|
// once we update how checkpoint manifests are tagged.
|
||||||
|
if len(found.IncompleteReason) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
found.Reasons = append(found.Reasons, m.Reasons...)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res := make([]*snapshot.Manifest, 0, len(mans))
|
res := make([]*ManifestEntry, 0, len(mans))
|
||||||
for _, m := range mans {
|
for _, m := range mans {
|
||||||
res = append(res, m)
|
res = append(res, m)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -29,11 +29,19 @@ var (
|
|||||||
testID2 = manifest.ID("snap2")
|
testID2 = manifest.ID("snap2")
|
||||||
testID3 = manifest.ID("snap3")
|
testID3 = manifest.ID("snap3")
|
||||||
|
|
||||||
testMail = path.ExchangeService.String() + path.EmailCategory.String()
|
testMail = path.ExchangeService.String() + path.EmailCategory.String()
|
||||||
testEvents = path.ExchangeService.String() + path.EventsCategory.String()
|
testMailServiceCat = ServiceCat{
|
||||||
testUser1 = "user1"
|
Service: path.ExchangeService,
|
||||||
testUser2 = "user2"
|
Category: path.EmailCategory,
|
||||||
testUser3 = "user3"
|
}
|
||||||
|
testEvents = path.ExchangeService.String() + path.EventsCategory.String()
|
||||||
|
testEventsServiceCat = ServiceCat{
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
}
|
||||||
|
testUser1 = "user1"
|
||||||
|
testUser2 = "user2"
|
||||||
|
testUser3 = "user3"
|
||||||
|
|
||||||
testAllUsersAllCats = &OwnersCats{
|
testAllUsersAllCats = &OwnersCats{
|
||||||
ResourceOwners: map[string]struct{}{
|
ResourceOwners: map[string]struct{}{
|
||||||
@ -42,8 +50,8 @@ var (
|
|||||||
testUser3: {},
|
testUser3: {},
|
||||||
},
|
},
|
||||||
ServiceCats: map[string]ServiceCat{
|
ServiceCats: map[string]ServiceCat{
|
||||||
testMail: {},
|
testMail: testMailServiceCat,
|
||||||
testEvents: {},
|
testEvents: testEventsServiceCat,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
testAllUsersMail = &OwnersCats{
|
testAllUsersMail = &OwnersCats{
|
||||||
@ -53,7 +61,7 @@ var (
|
|||||||
testUser3: {},
|
testUser3: {},
|
||||||
},
|
},
|
||||||
ServiceCats: map[string]ServiceCat{
|
ServiceCats: map[string]ServiceCat{
|
||||||
testMail: {},
|
testMail: testMailServiceCat,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@ -174,6 +182,9 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
// defining data in a table while not repeating things between data and
|
// defining data in a table while not repeating things between data and
|
||||||
// expected.
|
// expected.
|
||||||
expectedIdxs []int
|
expectedIdxs []int
|
||||||
|
// Use this to denote the Reasons a manifest is selected. The int maps to
|
||||||
|
// the index of the manifest in data.
|
||||||
|
expectedReasons map[int][]Reason
|
||||||
// Expected number of times a manifest should try to be loaded from kopia.
|
// Expected number of times a manifest should try to be loaded from kopia.
|
||||||
// Used to check that caching is functioning properly.
|
// Used to check that caching is functioning properly.
|
||||||
expectedLoadCounts map[manifest.ID]int
|
expectedLoadCounts map[manifest.ID]int
|
||||||
@ -194,6 +205,40 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0},
|
expectedIdxs: []int{0},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
},
|
},
|
||||||
@ -222,6 +267,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0, 1},
|
expectedIdxs: []int{0, 1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
testID2: 1,
|
testID2: 1,
|
||||||
@ -251,6 +332,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0, 1},
|
expectedIdxs: []int{0, 1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
testID2: 3,
|
testID2: 3,
|
||||||
@ -280,6 +397,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{1},
|
expectedIdxs: []int{1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
testID2: 1,
|
testID2: 1,
|
||||||
@ -300,6 +436,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0},
|
expectedIdxs: []int{0},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 3,
|
testID1: 3,
|
||||||
},
|
},
|
||||||
@ -328,6 +483,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{1},
|
expectedIdxs: []int{1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
testID2: 1,
|
testID2: 1,
|
||||||
@ -357,6 +531,25 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{1},
|
expectedIdxs: []int{1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 3,
|
testID1: 3,
|
||||||
testID2: 3,
|
testID2: 3,
|
||||||
@ -384,6 +577,91 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0, 1},
|
expectedIdxs: []int{0, 1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
|
testID1: 2,
|
||||||
|
testID2: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "SomeCachedSomeNewerDifferentCategories",
|
||||||
|
input: testAllUsersAllCats,
|
||||||
|
data: []manifestInfo{
|
||||||
|
newManifestInfo(
|
||||||
|
testID1,
|
||||||
|
testT1,
|
||||||
|
testCompleteMan,
|
||||||
|
testMail,
|
||||||
|
testEvents,
|
||||||
|
testUser1,
|
||||||
|
testUser2,
|
||||||
|
testUser3,
|
||||||
|
),
|
||||||
|
newManifestInfo(
|
||||||
|
testID2,
|
||||||
|
testT2,
|
||||||
|
testCompleteMan,
|
||||||
|
testMail,
|
||||||
|
testUser3,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
expectedIdxs: []int{0, 1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EventsCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 2,
|
testID1: 2,
|
||||||
testID2: 1,
|
testID2: 1,
|
||||||
@ -411,6 +689,32 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
),
|
),
|
||||||
},
|
},
|
||||||
expectedIdxs: []int{0, 1},
|
expectedIdxs: []int{0, 1},
|
||||||
|
expectedReasons: map[int][]Reason{
|
||||||
|
0: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser1,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser2,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
1: {
|
||||||
|
Reason{
|
||||||
|
ResourceOwner: testUser3,
|
||||||
|
Service: path.ExchangeService,
|
||||||
|
Category: path.EmailCategory,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
expectedLoadCounts: map[manifest.ID]int{
|
expectedLoadCounts: map[manifest.ID]int{
|
||||||
testID1: 1,
|
testID1: 1,
|
||||||
testID2: 1,
|
testID2: 1,
|
||||||
@ -444,13 +748,42 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots() {
|
|||||||
|
|
||||||
snaps := fetchPrevSnapshotManifests(ctx, msm, test.input, nil)
|
snaps := fetchPrevSnapshotManifests(ctx, msm, test.input, nil)
|
||||||
|
|
||||||
|
// Check the proper snapshot manifests were returned.
|
||||||
expected := make([]*snapshot.Manifest, 0, len(test.expectedIdxs))
|
expected := make([]*snapshot.Manifest, 0, len(test.expectedIdxs))
|
||||||
for _, i := range test.expectedIdxs {
|
for _, i := range test.expectedIdxs {
|
||||||
expected = append(expected, test.data[i].man)
|
expected = append(expected, test.data[i].man)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.ElementsMatch(t, expected, snaps)
|
got := make([]*snapshot.Manifest, 0, len(snaps))
|
||||||
|
for _, s := range snaps {
|
||||||
|
got = append(got, s.Manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, expected, got)
|
||||||
|
|
||||||
|
// Check the resons for selecting each manifest are correct.
|
||||||
|
expectedReasons := make(map[manifest.ID][]Reason, len(test.expectedReasons))
|
||||||
|
for idx, reason := range test.expectedReasons {
|
||||||
|
expectedReasons[test.data[idx].man.ID] = reason
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, found := range snaps {
|
||||||
|
reason, ok := expectedReasons[found.ID]
|
||||||
|
if !ok {
|
||||||
|
// Missing or extra snapshots will be reported by earlier checks.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(
|
||||||
|
t,
|
||||||
|
reason,
|
||||||
|
found.Reasons,
|
||||||
|
"incorrect reasons for snapshot with ID %s",
|
||||||
|
found.ID,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check number of loads to make sure caching is working properly.
|
||||||
// Need to manually check because we don't know the order the
|
// Need to manually check because we don't know the order the
|
||||||
// user/service/category labels will be iterated over. For some tests this
|
// user/service/category labels will be iterated over. For some tests this
|
||||||
// could cause more loads than the ideal case.
|
// could cause more loads than the ideal case.
|
||||||
@ -542,7 +875,12 @@ func (suite *SnapshotFetchUnitSuite) TestFetchPrevSnapshots_customTags() {
|
|||||||
expected = append(expected, data[i].man)
|
expected = append(expected, data[i].man)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.ElementsMatch(t, expected, snaps)
|
got := make([]*snapshot.Manifest, 0, len(snaps))
|
||||||
|
for _, s := range snaps {
|
||||||
|
got = append(got, s.Manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.ElementsMatch(t, expected, got)
|
||||||
|
|
||||||
// Need to manually check because we don't know the order the
|
// Need to manually check because we don't know the order the
|
||||||
// user/service/category labels will be iterated over. For some tests this
|
// user/service/category labels will be iterated over. For some tests this
|
||||||
|
|||||||
@ -113,7 +113,7 @@ func (w *Wrapper) Close(ctx context.Context) error {
|
|||||||
// TODO(ashmrtn): Use previousSnapshots parameter.
|
// TODO(ashmrtn): Use previousSnapshots parameter.
|
||||||
func (w Wrapper) BackupCollections(
|
func (w Wrapper) BackupCollections(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
previousSnapshots []*snapshot.Manifest,
|
previousSnapshots []*ManifestEntry,
|
||||||
collections []data.Collection,
|
collections []data.Collection,
|
||||||
service path.ServiceType,
|
service path.ServiceType,
|
||||||
oc *OwnersCats,
|
oc *OwnersCats,
|
||||||
@ -156,11 +156,16 @@ func (w Wrapper) makeSnapshotWithRoot(
|
|||||||
addlTags map[string]string,
|
addlTags map[string]string,
|
||||||
) (*BackupStats, error) {
|
) (*BackupStats, error) {
|
||||||
var (
|
var (
|
||||||
man *snapshot.Manifest
|
man *snapshot.Manifest
|
||||||
prevSnaps = fetchPrevSnapshotManifests(ctx, w.c, oc, nil)
|
prevSnapEntries = fetchPrevSnapshotManifests(ctx, w.c, oc, nil)
|
||||||
bc = &stats.ByteCounter{}
|
bc = &stats.ByteCounter{}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
prevSnaps := make([]*snapshot.Manifest, 0, len(prevSnapEntries))
|
||||||
|
for _, ent := range prevSnapEntries {
|
||||||
|
prevSnaps = append(prevSnaps, ent.Manifest)
|
||||||
|
}
|
||||||
|
|
||||||
err := repo.WriteSession(
|
err := repo.WriteSession(
|
||||||
ctx,
|
ctx,
|
||||||
w.c,
|
w.c,
|
||||||
@ -414,7 +419,7 @@ func (w Wrapper) FetchPrevSnapshotManifests(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
oc *OwnersCats,
|
oc *OwnersCats,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
) ([]*snapshot.Manifest, error) {
|
) ([]*ManifestEntry, error) {
|
||||||
if w.c == nil {
|
if w.c == nil {
|
||||||
return nil, errors.WithStack(errNotConnected)
|
return nil, errors.WithStack(errNotConnected)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,7 +6,6 @@ import (
|
|||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
multierror "github.com/hashicorp/go-multierror"
|
multierror "github.com/hashicorp/go-multierror"
|
||||||
"github.com/kopia/kopia/snapshot"
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common"
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
@ -181,7 +180,7 @@ func produceManifestsAndMetadata(
|
|||||||
sw *store.Wrapper,
|
sw *store.Wrapper,
|
||||||
oc *kopia.OwnersCats,
|
oc *kopia.OwnersCats,
|
||||||
acct account.Account,
|
acct account.Account,
|
||||||
) ([]*snapshot.Manifest, []data.Collection, error) {
|
) ([]*kopia.ManifestEntry, []data.Collection, error) {
|
||||||
complete, closer := observe.MessageWithCompletion("Fetching backup heuristics:")
|
complete, closer := observe.MessageWithCompletion("Fetching backup heuristics:")
|
||||||
defer func() {
|
defer func() {
|
||||||
complete <- struct{}{}
|
complete <- struct{}{}
|
||||||
@ -337,7 +336,7 @@ func consumeBackupDataCollections(
|
|||||||
kw *kopia.Wrapper,
|
kw *kopia.Wrapper,
|
||||||
sel selectors.Selector,
|
sel selectors.Selector,
|
||||||
oc *kopia.OwnersCats,
|
oc *kopia.OwnersCats,
|
||||||
mans []*snapshot.Manifest,
|
mans []*kopia.ManifestEntry,
|
||||||
cs []data.Collection,
|
cs []data.Collection,
|
||||||
backupID model.StableID,
|
backupID model.StableID,
|
||||||
) (*kopia.BackupStats, *details.Details, error) {
|
) (*kopia.BackupStats, *details.Details, error) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user