Log every 1000 items when exporting (#5227)

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [x] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2024-02-15 12:15:16 +05:30 committed by GitHub
parent 90d6db486b
commit bf52fdbe6a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 27 additions and 0 deletions

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/pkg/dttm" "github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/export" "github.com/alcionai/corso/src/pkg/export"
"github.com/alcionai/corso/src/pkg/logger"
) )
const ( const (
@ -56,12 +57,22 @@ func ZipExportCollection(
defer wr.Close() defer wr.Close()
buf := make([]byte, ZipCopyBufferSize) buf := make([]byte, ZipCopyBufferSize)
counted := 0
log := logger.Ctx(ctx).
With("collection_count", len(expCollections))
for _, ec := range expCollections { for _, ec := range expCollections {
folder := ec.BasePath() folder := ec.BasePath()
items := ec.Items(ctx) items := ec.Items(ctx)
for item := range items { for item := range items {
counted++
// Log every 1000 items that are processed
if counted%1000 == 0 {
log.Infow("progress zipping export items", "count_items", counted)
}
err := item.Error err := item.Error
if err != nil { if err != nil {
writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID)) writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID))
@ -90,6 +101,8 @@ func ZipExportCollection(
} }
} }
} }
log.Infow("completed zipping export items", "count_items", counted)
}() }()
return zipCollection{reader}, nil return zipCollection{reader}, nil

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
) )
func ConsumeExportCollections( func ConsumeExportCollections(
@ -19,6 +20,10 @@ func ConsumeExportCollections(
errs *fault.Bus, errs *fault.Bus,
) error { ) error {
el := errs.Local() el := errs.Local()
counted := 0
log := logger.Ctx(ctx).
With("export_location", exportLocation,
"collection_count", len(expColl))
for _, col := range expColl { for _, col := range expColl {
if el.Failure() != nil { if el.Failure() != nil {
@ -29,6 +34,13 @@ func ConsumeExportCollections(
ictx := clues.Add(ctx, "dir_name", folder) ictx := clues.Add(ctx, "dir_name", folder)
for item := range col.Items(ictx) { for item := range col.Items(ictx) {
counted++
// Log every 1000 items that are processed
if counted%1000 == 0 {
log.Infow("progress writing export items", "count_items", counted)
}
if item.Error != nil { if item.Error != nil {
el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item")) el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item"))
continue continue
@ -42,6 +54,8 @@ func ConsumeExportCollections(
} }
} }
log.Infow("completed writing export items", "count_items", counted)
return el.Failure() return el.Failure()
} }