larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add some fixes

Browse Source
This commit is contained in:
Abhishek Pandey 2023-09-27 03:38:10 +05:30
parent d792eea773
commit a6edf145d6
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/internal/m365/collection/drive/collection.go
View File

@ -334,13 +334,14 @@ func downloadContent(
go func() { go func() {
defer wg.Done() defer wg.Done()
go GetItemContentNTimes(1000, ctx, iaag, driveID, item) go GetItemContentNTimes(500, ctx, iaag, driveID, item)
}() }()
logger.Ctx(ctx).Debug("waiting for goroutines to finish") logger.Ctx(ctx).Debug("waiting for goroutines to finish")
wg.Wait() wg.Wait()
// Reset window such that it doesn't impact GetItem() calls // Reset window such that it doesn't impact GetItem() calls
// if they are not made in parallel with item download calls.
time.Sleep(10 * time.Second) time.Sleep(10 * time.Second)
} }

8
src/internal/m365/graph/concurrency_middleware.go
View File

@ -86,16 +86,16 @@ const (
// If the bucket is full, we can push out 200 calls immediately, which brings // If the bucket is full, we can push out 200 calls immediately, which brings
// the total in the first 10 minutes to 9800. We can toe that line if we want, // the total in the first 10 minutes to 9800. We can toe that line if we want,
// but doing so risks timeouts. It's better to give the limits breathing room. // but doing so risks timeouts. It's better to give the limits breathing room.
defaultPerSecond = 200 // 16 * 60 * 10 = 9600 defaultPerSecond = 16 // 16 * 60 * 10 = 9600
defaultMaxCap = 2000 // real cap is 10k-per-10-minutes defaultMaxCap = 200 // real cap is 10k-per-10-minutes
// since drive runs on a per-minute, rather than per-10-minute bucket, we have // since drive runs on a per-minute, rather than per-10-minute bucket, we have
// to keep the max cap equal to the per-second cap. A large maxCap pool (say, // to keep the max cap equal to the per-second cap. A large maxCap pool (say,
// 1200, similar to the per-minute cap) would allow us to make a flood of 2400 // 1200, similar to the per-minute cap) would allow us to make a flood of 2400
// calls in the first minute, putting us over the per-minute limit. Keeping // calls in the first minute, putting us over the per-minute limit. Keeping
// the cap at the per-second burst means we only dole out a max of 1240 in one // the cap at the per-second burst means we only dole out a max of 1240 in one
// minute (20 cap + 1200 per minute + one burst of padding). // minute (20 cap + 1200 per minute + one burst of padding).
drivePerSecond = 200 // 20 * 60 = 1200 drivePerSecond = 20 // 20 * 60 = 1200
driveMaxCap = 2000 // real cap is 1250-per-minute driveMaxCap = 20 // real cap is 1250-per-minute
) )
var ( var (