Refactor populateItems (#3732)

<!-- PR description-->

Benefits:
* Narrowed scope & better testability. 
* Preps it for follow up PRs which will inject per-item corso extension interfaces
* Not adding unit tests yet, as the args & outputs of this new function will change in a follow up PR.
---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2023-07-05 11:52:21 -07:00 committed by GitHub
parent b0d3d4f3ee
commit 6fff3f7d1d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -403,17 +403,21 @@ func readItemContents(
return rc, nil return rc, nil
} }
type driveStats struct {
dirsRead int64
itemsRead int64
byteCount int64
itemsFound int64
dirsFound int64
}
// populateItems iterates through items added to the collection // populateItems iterates through items added to the collection
// and uses the collection `itemReader` to read the item // and uses the collection `itemReader` to read the item
func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
var ( var (
byteCount int64
itemsRead int64
dirsRead int64
itemsFound int64
dirsFound int64
wg sync.WaitGroup
el = errs.Local() el = errs.Local()
stats driveStats
wg sync.WaitGroup
) )
// Retrieve the OneDrive folder path to set later in // Retrieve the OneDrive folder path to set later in
@ -445,12 +449,36 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
wg.Add(1) wg.Add(1)
go func(ctx context.Context, item models.DriveItemable) { go func(item models.DriveItemable) {
defer wg.Done() defer wg.Done()
defer func() { <-semaphoreCh }() defer func() { <-semaphoreCh }()
// Read the item // Read the item
oc.populateDriveItem(
ctx,
parentPath,
item,
&stats,
errs)
folderProgress <- struct{}{}
}(item)
}
wg.Wait()
oc.reportAsCompleted(ctx, int(stats.itemsFound), int(stats.itemsRead), stats.byteCount)
}
func (oc *Collection) populateDriveItem(
ctx context.Context,
parentPath *path.Builder,
item models.DriveItemable,
stats *driveStats,
errs *fault.Bus,
) {
var ( var (
el = errs.Local()
itemID = ptr.Val(item.GetId()) itemID = ptr.Val(item.GetId())
itemName = ptr.Val(item.GetName()) itemName = ptr.Val(item.GetName())
itemSize = ptr.Val(item.GetSize()) itemSize = ptr.Val(item.GetSize())
@ -473,12 +501,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
isFile := item.GetFile() != nil isFile := item.GetFile() != nil
if isFile { if isFile {
atomic.AddInt64(&itemsFound, 1) atomic.AddInt64(&stats.itemsFound, 1)
metaFileName = itemID metaFileName = itemID
metaSuffix = metadata.MetaFileSuffix metaSuffix = metadata.MetaFileSuffix
} else { } else {
atomic.AddInt64(&dirsFound, 1) atomic.AddInt64(&stats.dirsFound, 1)
// metaFileName not set for directories so we get just ".dirmeta" // metaFileName not set for directories so we get just ".dirmeta"
metaSuffix = metadata.DirMetaFileSuffix metaSuffix = metadata.DirMetaFileSuffix
@ -546,21 +574,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
// Item read successfully, add to collection // Item read successfully, add to collection
if isFile { if isFile {
atomic.AddInt64(&itemsRead, 1) atomic.AddInt64(&stats.itemsRead, 1)
} else { } else {
atomic.AddInt64(&dirsRead, 1) atomic.AddInt64(&stats.dirsRead, 1)
} }
// byteCount iteration atomic.AddInt64(&stats.byteCount, itemSize)
atomic.AddInt64(&byteCount, itemSize)
folderProgress <- struct{}{}
}(ctx, item)
}
wg.Wait()
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount)
} }
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64) { func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64) {