Refactor populateItems (#3732)
<!-- PR description--> Benefits: * Narrowed scope & better testability. * Preps it for follow up PRs which will inject per-item corso extension interfaces * Not adding unit tests yet, as the args & outputs of this new function will change in a follow up PR. --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [x] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * #<issue> #### Test Plan <!-- How will this be tested prior to merging.--> - [x] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
b0d3d4f3ee
commit
6fff3f7d1d
@ -403,17 +403,21 @@ func readItemContents(
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
type driveStats struct {
|
||||
dirsRead int64
|
||||
itemsRead int64
|
||||
byteCount int64
|
||||
itemsFound int64
|
||||
dirsFound int64
|
||||
}
|
||||
|
||||
// populateItems iterates through items added to the collection
|
||||
// and uses the collection `itemReader` to read the item
|
||||
func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
|
||||
var (
|
||||
byteCount int64
|
||||
itemsRead int64
|
||||
dirsRead int64
|
||||
itemsFound int64
|
||||
dirsFound int64
|
||||
wg sync.WaitGroup
|
||||
el = errs.Local()
|
||||
stats driveStats
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
// Retrieve the OneDrive folder path to set later in
|
||||
@ -445,12 +449,36 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
|
||||
|
||||
wg.Add(1)
|
||||
|
||||
go func(ctx context.Context, item models.DriveItemable) {
|
||||
go func(item models.DriveItemable) {
|
||||
defer wg.Done()
|
||||
defer func() { <-semaphoreCh }()
|
||||
|
||||
// Read the item
|
||||
oc.populateDriveItem(
|
||||
ctx,
|
||||
parentPath,
|
||||
item,
|
||||
&stats,
|
||||
errs)
|
||||
|
||||
folderProgress <- struct{}{}
|
||||
}(item)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
oc.reportAsCompleted(ctx, int(stats.itemsFound), int(stats.itemsRead), stats.byteCount)
|
||||
}
|
||||
|
||||
func (oc *Collection) populateDriveItem(
|
||||
ctx context.Context,
|
||||
parentPath *path.Builder,
|
||||
item models.DriveItemable,
|
||||
stats *driveStats,
|
||||
errs *fault.Bus,
|
||||
) {
|
||||
var (
|
||||
el = errs.Local()
|
||||
itemID = ptr.Val(item.GetId())
|
||||
itemName = ptr.Val(item.GetName())
|
||||
itemSize = ptr.Val(item.GetSize())
|
||||
@ -473,12 +501,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
|
||||
isFile := item.GetFile() != nil
|
||||
|
||||
if isFile {
|
||||
atomic.AddInt64(&itemsFound, 1)
|
||||
atomic.AddInt64(&stats.itemsFound, 1)
|
||||
|
||||
metaFileName = itemID
|
||||
metaSuffix = metadata.MetaFileSuffix
|
||||
} else {
|
||||
atomic.AddInt64(&dirsFound, 1)
|
||||
atomic.AddInt64(&stats.dirsFound, 1)
|
||||
|
||||
// metaFileName not set for directories so we get just ".dirmeta"
|
||||
metaSuffix = metadata.DirMetaFileSuffix
|
||||
@ -546,21 +574,12 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
|
||||
|
||||
// Item read successfully, add to collection
|
||||
if isFile {
|
||||
atomic.AddInt64(&itemsRead, 1)
|
||||
atomic.AddInt64(&stats.itemsRead, 1)
|
||||
} else {
|
||||
atomic.AddInt64(&dirsRead, 1)
|
||||
atomic.AddInt64(&stats.dirsRead, 1)
|
||||
}
|
||||
|
||||
// byteCount iteration
|
||||
atomic.AddInt64(&byteCount, itemSize)
|
||||
|
||||
folderProgress <- struct{}{}
|
||||
}(ctx, item)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount)
|
||||
atomic.AddInt64(&stats.byteCount, itemSize)
|
||||
}
|
||||
|
||||
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user