c58240ea3d
## Description This improves the initial backup speed for OneDrive. OneDrive backup was mostly slow when we had a lot of tiny files. Case where we had mostly large files was pretty much the best case scenario and we were throttled by purely how fast we can get the files from MS and how fast kopia can process and upload it. But of small files, we were slowed by the loop which was taking quite a bit of time to fetch the download urls. We have now parallelized the query for getting the download URL. Under best case scenarios, I was able to speed it up to under 20s from ~4-5m starting point. That said, MS graph api still seems to throttle us and when that happen we still go back to around ~2m for worst case scenario. I've added 3 retries as some requests were failing when we continuously making many requests. This should also take care of the issue of url expiring mentioned in https://github.com/alcionai/corso/issues/581 as we are only prefetching a few urls ahed of time. ## Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🐹 Trivial/Minor ## Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * https://github.com/alcionai/corso/issues/1595 * fixes https://github.com/alcionai/corso/issues/581 ## Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual - [x] ⚡ Unit test - [x] 💚 E2E
76 lines
1.6 KiB
Makefile
76 lines
1.6 KiB
Makefile
# This must match the version defined in .github/workflows/lint.yaml.
|
|
WANTED_LINT_VERSION := 1.50.1
|
|
LINT_VERSION := $(shell golangci-lint version | cut -d' ' -f4)
|
|
HAS_LINT := $(shell which golangci-lint)
|
|
|
|
INSTALL_LINT_PAGE := "https://golangci-lint.run/usage/install/"
|
|
BAD_LINT_MSG := "Missing golangci-lint version $(WANTED_LINT_VERSION). Visit $(INSTALL_LINT_PAGE) for instructions on how to install"
|
|
|
|
.PHONY: check-lint check-lint-version lint load-test
|
|
|
|
build:
|
|
go build -o corso
|
|
|
|
lint: check-lint-version
|
|
golangci-lint run
|
|
|
|
check-lint-version: check-lint
|
|
@if [ "$(LINT_VERSION)" != "$(WANTED_LINT_VERSION)" ]; then \
|
|
echo >&2 $(BAD_LINT_MSG); \
|
|
false; \
|
|
fi
|
|
|
|
check-lint:
|
|
@if [ -z "$(HAS_LINT)" ]; then \
|
|
echo >&2 $(BAD_LINT_MSG); \
|
|
false; \
|
|
fi
|
|
|
|
build-otel-daemon:
|
|
cd testfiles/otel_daemon; \
|
|
docker build -t xray-daemon .
|
|
|
|
otel-daemon:
|
|
results_dir=$$PWD/test_results; \
|
|
cd ./testfiles/otel_daemon; \
|
|
docker run \
|
|
-d \
|
|
-e AWS_REGION \
|
|
-v ~/.aws/:/root/.aws/:ro \
|
|
--name otel-daemon \
|
|
-p 2000:2000/udp \
|
|
--rm \
|
|
xray-daemon \
|
|
--local-mode \
|
|
--log-level debug
|
|
|
|
local-daemon:
|
|
results_dir=$$PWD/test_results; \
|
|
cd ./testfiles/otel_daemon; \
|
|
docker run \
|
|
--attach STDOUT \
|
|
-e AWS_REGION \
|
|
-v ~/.aws/:/root/.aws/:ro \
|
|
--name otel-daemon \
|
|
-p 2000:2000/udp \
|
|
--rm \
|
|
xray-daemon \
|
|
--local-mode \
|
|
--log-level debug
|
|
|
|
# --net=host \
|
|
|
|
load-test:
|
|
AWS_XRAY_NOOP_ID=False \
|
|
CORSO_LOAD_TESTS=y \
|
|
go test \
|
|
-v \
|
|
-count=1 \
|
|
-timeout 1h \
|
|
-blockprofile=block.prof \
|
|
-cpuprofile=cpu.prof \
|
|
-memprofile=mem.prof \
|
|
-mutexprofile=mutex.prof \
|
|
-trace=trace.out \
|
|
-outputdir=test_results \
|
|
./pkg/repository/repository_load_test.go
|