Add progress logging to uploader (#4672)

Adds a progress log in the kopia upload that logs every 10000 files that have been uploaded.

This is useful to track progress of large backups.

---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [x] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #4670 

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Vaibhav Kamra 2023-11-14 00:18:27 -08:00 committed by GitHub
parent bb1f287e6a
commit 477fc43485
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -57,6 +57,7 @@ type corsoProgress struct {
toMerge *mergeDetails
mu sync.RWMutex
totalBytes int64
totalFiles int64
errs *fault.Bus
counter *count.Bus
// expectedIgnoredErrors is a count of error cases caught in the Error wrapper
@ -91,6 +92,13 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
return
}
atomic.AddInt64(&cp.totalFiles, 1)
// Log every 1000 items uploaded
if cp.totalFiles%1000 == 0 {
logger.Ctx(cp.ctx).Infow("finishedfile", "totalFiles", cp.totalFiles, "totalBytes", cp.totalBytes)
}
d := cp.get(relativePath)
if d == nil {
return