From bf52fdbe6a5cf395551151caf40685727118d37d Mon Sep 17 00:00:00 2001
From: Abin Simon <abinsimon10@gmail.com>
Date: Thu, 15 Feb 2024 12:15:16 +0530
Subject: [PATCH] Log every 1000 items when exporting (#5227)

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [ ] :white_check_mark: Yes, it's included
- [ ] :clock1: Yes, but in a later PR
- [x] :no_entry: No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] :sunflower: Feature
- [ ] :bug: Bugfix
- [ ] :world_map: Documentation
- [x] :robot: Supportability/Tests
- [ ] :computer: CI/Deployment
- [ ] :broom: Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] :muscle: Manual
- [ ] :zap: Unit test
- [ ] :green_heart: E2E
---
 src/internal/archive/zip.go | 13 +++++++++++++
 src/pkg/export/consume.go   | 14 ++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/internal/archive/zip.go b/src/internal/archive/zip.go
index 21357fbd3..3aaa54510 100644
--- a/src/internal/archive/zip.go
+++ b/src/internal/archive/zip.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/alcionai/corso/src/pkg/dttm"
 	"github.com/alcionai/corso/src/pkg/export"
+	"github.com/alcionai/corso/src/pkg/logger"
 )
 
 const (
@@ -56,12 +57,22 @@ func ZipExportCollection(
 		defer wr.Close()
 
 		buf := make([]byte, ZipCopyBufferSize)
+		counted := 0
+		log := logger.Ctx(ctx).
+			With("collection_count", len(expCollections))
 
 		for _, ec := range expCollections {
 			folder := ec.BasePath()
 			items := ec.Items(ctx)
 
 			for item := range items {
+				counted++
+
+				// Log every 1000 items that are processed
+				if counted%1000 == 0 {
+					log.Infow("progress zipping export items", "count_items", counted)
+				}
+
 				err := item.Error
 				if err != nil {
 					writer.CloseWithError(clues.Wrap(err, "getting export item").With("id", item.ID))
@@ -90,6 +101,8 @@ func ZipExportCollection(
 				}
 			}
 		}
+
+		log.Infow("completed zipping export items", "count_items", counted)
 	}()
 
 	return zipCollection{reader}, nil
diff --git a/src/pkg/export/consume.go b/src/pkg/export/consume.go
index c4c5711e3..aaa609500 100644
--- a/src/pkg/export/consume.go
+++ b/src/pkg/export/consume.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/alcionai/corso/src/internal/observe"
 	"github.com/alcionai/corso/src/pkg/fault"
+	"github.com/alcionai/corso/src/pkg/logger"
 )
 
 func ConsumeExportCollections(
@@ -19,6 +20,10 @@ func ConsumeExportCollections(
 	errs *fault.Bus,
 ) error {
 	el := errs.Local()
+	counted := 0
+	log := logger.Ctx(ctx).
+		With("export_location", exportLocation,
+			"collection_count", len(expColl))
 
 	for _, col := range expColl {
 		if el.Failure() != nil {
@@ -29,6 +34,13 @@ func ConsumeExportCollections(
 		ictx := clues.Add(ctx, "dir_name", folder)
 
 		for item := range col.Items(ictx) {
+			counted++
+
+			// Log every 1000 items that are processed
+			if counted%1000 == 0 {
+				log.Infow("progress writing export items", "count_items", counted)
+			}
+
 			if item.Error != nil {
 				el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item"))
 				continue
@@ -42,6 +54,8 @@ func ConsumeExportCollections(
 		}
 	}
 
+	log.Infow("completed writing export items", "count_items", counted)
+
 	return el.Failure()
 }