Log every 1000 items when exporting (#5227)
<!-- PR description--> --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [x] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #<issue> #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
90d6db486b
commit
bf52fdbe6a
@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/dttm"
|
||||
"github.com/alcionai/corso/src/pkg/export"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -56,12 +57,22 @@ func ZipExportCollection(
|
||||
defer wr.Close()
|
||||
|
||||
buf := make([]byte, ZipCopyBufferSize)
|
||||
counted := 0
|
||||
log := logger.Ctx(ctx).
|
||||
With("collection_count", len(expCollections))
|
||||
|
||||
for _, ec := range expCollections {
|
||||
folder := ec.BasePath()
|
||||
items := ec.Items(ctx)
|
||||
|
||||
for item := range items {
|
||||
counted++
|
||||
|
||||
// Log every 1000 items that are processed
|
||||
if counted%1000 == 0 {
|
||||
log.Infow("progress zipping export items", "count_items", counted)
|
||||
}
|
||||
|
||||
err := item.Error
|
||||
if err != nil {
|
||||
writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID))
|
||||
@ -90,6 +101,8 @@ func ZipExportCollection(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Infow("completed zipping export items", "count_items", counted)
|
||||
}()
|
||||
|
||||
return zipCollection{reader}, nil
|
||||
|
||||
@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/internal/observe"
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
func ConsumeExportCollections(
|
||||
@ -19,6 +20,10 @@ func ConsumeExportCollections(
|
||||
errs *fault.Bus,
|
||||
) error {
|
||||
el := errs.Local()
|
||||
counted := 0
|
||||
log := logger.Ctx(ctx).
|
||||
With("export_location", exportLocation,
|
||||
"collection_count", len(expColl))
|
||||
|
||||
for _, col := range expColl {
|
||||
if el.Failure() != nil {
|
||||
@ -29,6 +34,13 @@ func ConsumeExportCollections(
|
||||
ictx := clues.Add(ctx, "dir_name", folder)
|
||||
|
||||
for item := range col.Items(ictx) {
|
||||
counted++
|
||||
|
||||
// Log every 1000 items that are processed
|
||||
if counted%1000 == 0 {
|
||||
log.Infow("progress writing export items", "count_items", counted)
|
||||
}
|
||||
|
||||
if item.Error != nil {
|
||||
el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item"))
|
||||
continue
|
||||
@ -42,6 +54,8 @@ func ConsumeExportCollections(
|
||||
}
|
||||
}
|
||||
|
||||
log.Infow("completed writing export items", "count_items", counted)
|
||||
|
||||
return el.Failure()
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user