Minor snapshot lookup refactor and logging addition (#2164)
## Description Make some parts of the lookup logic more uniform as to when reasons are added to things. Add logging to make the process easier to debug ## Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No ## Type of change - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [x] 🧹 Tech Debt/Cleanup ## Issue(s) * #1675 * #2149 ## Test Plan - [x] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
e3b6d035fb
commit
383b93f097
@ -117,6 +117,7 @@ func getLastIdx(
|
|||||||
// adds it to the returned list. Otherwise no incomplete manifest is returned.
|
// adds it to the returned list. Otherwise no incomplete manifest is returned.
|
||||||
// Returns nil if there are no complete or incomplete manifests in mans.
|
// Returns nil if there are no complete or incomplete manifests in mans.
|
||||||
func manifestsSinceLastComplete(
|
func manifestsSinceLastComplete(
|
||||||
|
ctx context.Context,
|
||||||
mans []*snapshot.Manifest,
|
mans []*snapshot.Manifest,
|
||||||
) ([]*snapshot.Manifest, bool) {
|
) ([]*snapshot.Manifest, bool) {
|
||||||
var (
|
var (
|
||||||
@ -134,9 +135,10 @@ func manifestsSinceLastComplete(
|
|||||||
|
|
||||||
if len(m.IncompleteReason) > 0 {
|
if len(m.IncompleteReason) > 0 {
|
||||||
if !foundIncomplete {
|
if !foundIncomplete {
|
||||||
|
res = append(res, m)
|
||||||
foundIncomplete = true
|
foundIncomplete = true
|
||||||
|
|
||||||
res = append(res, m)
|
logger.Ctx(ctx).Infow("found incomplete snapshot", "snapshot_id", m.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@ -147,6 +149,8 @@ func manifestsSinceLastComplete(
|
|||||||
res = append(res, m)
|
res = append(res, m)
|
||||||
foundComplete = true
|
foundComplete = true
|
||||||
|
|
||||||
|
logger.Ctx(ctx).Infow("found complete snapshot", "snapshot_id", m.ID)
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -164,7 +168,7 @@ func fetchPrevManifests(
|
|||||||
foundMans map[manifest.ID]*ManifestEntry,
|
foundMans map[manifest.ID]*ManifestEntry,
|
||||||
reason Reason,
|
reason Reason,
|
||||||
tags map[string]string,
|
tags map[string]string,
|
||||||
) ([]*ManifestEntry, error) {
|
) ([]*snapshot.Manifest, error) {
|
||||||
allTags := map[string]string{}
|
allTags := map[string]string{}
|
||||||
|
|
||||||
for _, k := range reason.TagKeys() {
|
for _, k := range reason.TagKeys() {
|
||||||
@ -188,8 +192,7 @@ func fetchPrevManifests(
|
|||||||
// We have a complete cached snapshot and it's the most recent. No need
|
// We have a complete cached snapshot and it's the most recent. No need
|
||||||
// to do anything else.
|
// to do anything else.
|
||||||
if lastCompleteIdx == len(metas)-1 {
|
if lastCompleteIdx == len(metas)-1 {
|
||||||
man.Reasons = append(man.Reasons, reason)
|
return []*snapshot.Manifest{man.Manifest}, nil
|
||||||
return nil, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(ashmrtn): Remainder of the function can be simplified if we can inject
|
// TODO(ashmrtn): Remainder of the function can be simplified if we can inject
|
||||||
@ -209,24 +212,21 @@ func fetchPrevManifests(
|
|||||||
return nil, errors.Wrap(err, "fetching previous manifests")
|
return nil, errors.Wrap(err, "fetching previous manifests")
|
||||||
}
|
}
|
||||||
|
|
||||||
found, hasCompleted := manifestsSinceLastComplete(mans)
|
found, hasCompleted := manifestsSinceLastComplete(ctx, mans)
|
||||||
res := make([]*ManifestEntry, 0, len(found))
|
|
||||||
|
|
||||||
for _, m := range found {
|
|
||||||
res = append(res, &ManifestEntry{
|
|
||||||
Manifest: m,
|
|
||||||
Reasons: []Reason{reason},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we didn't find another complete manifest then we need to mark the
|
// If we didn't find another complete manifest then we need to mark the
|
||||||
// previous complete manifest as having this ResourceOwner, Service, Category
|
// previous complete manifest as having this ResourceOwner, Service, Category
|
||||||
// as the reason as well.
|
// as the reason as well.
|
||||||
if !hasCompleted && man != nil {
|
if !hasCompleted && man != nil {
|
||||||
man.Reasons = append(man.Reasons, reason)
|
found = append(found, man.Manifest)
|
||||||
|
logger.Ctx(ctx).Infow(
|
||||||
|
"reusing cached complete snapshot",
|
||||||
|
"snapshot_id",
|
||||||
|
man.ID,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return res, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchPrevSnapshotManifests returns a set of manifests for complete and maybe
|
// fetchPrevSnapshotManifests returns a set of manifests for complete and maybe
|
||||||
@ -252,6 +252,14 @@ func fetchPrevSnapshotManifests(
|
|||||||
// we can pass in. Can be expanded to return more than the most recent
|
// we can pass in. Can be expanded to return more than the most recent
|
||||||
// snapshots, but may require more memory at runtime.
|
// snapshots, but may require more memory at runtime.
|
||||||
for _, reason := range reasons {
|
for _, reason := range reasons {
|
||||||
|
logger.Ctx(ctx).Infow(
|
||||||
|
"searching for previous manifests for reason",
|
||||||
|
"service",
|
||||||
|
reason.Service.String(),
|
||||||
|
"category",
|
||||||
|
reason.Category.String(),
|
||||||
|
)
|
||||||
|
|
||||||
found, err := fetchPrevManifests(
|
found, err := fetchPrevManifests(
|
||||||
ctx,
|
ctx,
|
||||||
sm,
|
sm,
|
||||||
@ -278,18 +286,16 @@ func fetchPrevSnapshotManifests(
|
|||||||
for _, m := range found {
|
for _, m := range found {
|
||||||
man := mans[m.ID]
|
man := mans[m.ID]
|
||||||
if man == nil {
|
if man == nil {
|
||||||
mans[m.ID] = m
|
mans[m.ID] = &ManifestEntry{
|
||||||
|
Manifest: m,
|
||||||
|
Reasons: []Reason{reason},
|
||||||
|
}
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// If the manifest already exists and it's incomplete then we should
|
// This manifest has multiple reasons for being chosen. Merge them here.
|
||||||
// merge the reasons for consistency. This will become easier to handle
|
man.Reasons = append(man.Reasons, reason)
|
||||||
// once we update how checkpoint manifests are tagged.
|
|
||||||
if len(man.IncompleteReason) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
man.Reasons = append(man.Reasons, m.Reasons...)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user