Log every 1000 items when exporting (#5227)
<!-- PR description--> --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [x] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [ ] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #<issue> #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
90d6db486b
commit
bf52fdbe6a
@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/pkg/dttm"
|
"github.com/alcionai/corso/src/pkg/dttm"
|
||||||
"github.com/alcionai/corso/src/pkg/export"
|
"github.com/alcionai/corso/src/pkg/export"
|
||||||
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -56,12 +57,22 @@ func ZipExportCollection(
|
|||||||
defer wr.Close()
|
defer wr.Close()
|
||||||
|
|
||||||
buf := make([]byte, ZipCopyBufferSize)
|
buf := make([]byte, ZipCopyBufferSize)
|
||||||
|
counted := 0
|
||||||
|
log := logger.Ctx(ctx).
|
||||||
|
With("collection_count", len(expCollections))
|
||||||
|
|
||||||
for _, ec := range expCollections {
|
for _, ec := range expCollections {
|
||||||
folder := ec.BasePath()
|
folder := ec.BasePath()
|
||||||
items := ec.Items(ctx)
|
items := ec.Items(ctx)
|
||||||
|
|
||||||
for item := range items {
|
for item := range items {
|
||||||
|
counted++
|
||||||
|
|
||||||
|
// Log every 1000 items that are processed
|
||||||
|
if counted%1000 == 0 {
|
||||||
|
log.Infow("progress zipping export items", "count_items", counted)
|
||||||
|
}
|
||||||
|
|
||||||
err := item.Error
|
err := item.Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID))
|
writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID))
|
||||||
@ -90,6 +101,8 @@ func ZipExportCollection(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Infow("completed zipping export items", "count_items", counted)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
return zipCollection{reader}, nil
|
return zipCollection{reader}, nil
|
||||||
|
|||||||
@ -10,6 +10,7 @@ import (
|
|||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/observe"
|
"github.com/alcionai/corso/src/internal/observe"
|
||||||
"github.com/alcionai/corso/src/pkg/fault"
|
"github.com/alcionai/corso/src/pkg/fault"
|
||||||
|
"github.com/alcionai/corso/src/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ConsumeExportCollections(
|
func ConsumeExportCollections(
|
||||||
@ -19,6 +20,10 @@ func ConsumeExportCollections(
|
|||||||
errs *fault.Bus,
|
errs *fault.Bus,
|
||||||
) error {
|
) error {
|
||||||
el := errs.Local()
|
el := errs.Local()
|
||||||
|
counted := 0
|
||||||
|
log := logger.Ctx(ctx).
|
||||||
|
With("export_location", exportLocation,
|
||||||
|
"collection_count", len(expColl))
|
||||||
|
|
||||||
for _, col := range expColl {
|
for _, col := range expColl {
|
||||||
if el.Failure() != nil {
|
if el.Failure() != nil {
|
||||||
@ -29,6 +34,13 @@ func ConsumeExportCollections(
|
|||||||
ictx := clues.Add(ctx, "dir_name", folder)
|
ictx := clues.Add(ctx, "dir_name", folder)
|
||||||
|
|
||||||
for item := range col.Items(ictx) {
|
for item := range col.Items(ictx) {
|
||||||
|
counted++
|
||||||
|
|
||||||
|
// Log every 1000 items that are processed
|
||||||
|
if counted%1000 == 0 {
|
||||||
|
log.Infow("progress writing export items", "count_items", counted)
|
||||||
|
}
|
||||||
|
|
||||||
if item.Error != nil {
|
if item.Error != nil {
|
||||||
el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item"))
|
el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item"))
|
||||||
continue
|
continue
|
||||||
@ -42,6 +54,8 @@ func ConsumeExportCollections(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Infow("completed writing export items", "count_items", counted)
|
||||||
|
|
||||||
return el.Failure()
|
return el.Failure()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user