larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minio custom query params (#4763)

Browse Source 
Add and wire up custom query parameters for minio to add extra fields
to S3 server access log output

** This switches to a custom minio version and updates kopia **

Diffs for the package updates (based on current versions used in corso):
* [minio](https://github.com/minio/minio-go/compare/v7.0.64...alcionai:minio-go:pass-custom-get-query-params-local?expand=1)
* [kopia](https://github.com/alcionai/kopia/compare/corsostaging-dir-iterator...custom-query-params)

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Test Plan

- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2023-12-01 16:22:33 -08:00 committed by GitHub
parent 120219cdfb
commit 08dfe6df66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/go.mod
View File

@ -2,7 +2,11 @@ module github.com/alcionai/corso/src
go 1.21
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db
// No tags in the alcion fork of the repo so use v7 as that's in the import
// path.
replace github.com/minio/minio-go/v7 => github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0

8
src/go.sum
View File

@ -17,8 +17,10 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/alcionai/clues v0.0.0-20231115004051-523cbddac8e8 h1:kdpkhcKWWxnZFteGjglHvFTTPCPsvjhKDDY9J1Od6Sg=
github.com/alcionai/clues v0.0.0-20231115004051-523cbddac8e8/go.mod h1:hGnRqQtV7YoojQSNMtqFK0TvfcwAljGZhUEQwx1lw34=
github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23 h1:ojlHvw65O21oFTC6kQldVDO69VQ6JEzMdD6aq9aOrY4=
github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23/go.mod h1:f4PligAuyEicX+lfTlZltc69nM0eMoXX2nE5sCBdo6Y=
github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db h1:zSaL5qEs8iYL3O40AT5dHfCJejbYQ/7a4YHe+I+wXII=
github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db/go.mod h1:fsJixnpmyK+MoY+P/ULJXiX52WkRoEk6V4bJd0jUW5I=
github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa h1:PHzp5TkXgsardwMG6O2nnyk3zBsGW8CqgsOWQCYkykQ=
github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -208,8 +210,6 @@ github.com/microsoftgraph/msgraph-sdk-go-core v1.0.1 h1:uq4qZD8VXLiNZY0t4NoRpLDo
github.com/microsoftgraph/msgraph-sdk-go-core v1.0.1/go.mod h1:HUITyuFN556+0QZ/IVfH5K4FyJM7kllV6ExKi2ImKhE=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.64 h1:Zdza8HwOzkld0ZG/og50w56fKi6AAyfqfifmasD9n2Q=
github.com/minio/minio-go/v7 v7.0.64/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=

1
src/internal/kopia/s3.go
View File

@ -37,6 +37,7 @@ func s3BlobStorage(
DoNotUseTLS: cfg.DoNotUseTLS,
DoNotVerifyTLS: cfg.DoNotVerifyTLS,
Tags: s.SessionTags,
QueryParams: s.QueryParams,
SessionName: s.SessionName,
RoleARN: s.Role,
RoleDuration: s.SessionDuration,

7
src/pkg/storage/storage.go
View File

@ -48,6 +48,11 @@ type Storage struct {
Role string
SessionName string
SessionDuration string
// QueryParams allows passing custom query parameters to S3 GET object
// requests. Parameters prefixed by `x-` will be ignored by S3 (no functional
// changes) but added to server access logs. These KV-pairs are session
// specific and won't be persisted to or sourced from the corso config file.
QueryParams map[string]string
}
// NewStorage aggregates all the supplied configurations into a single configuration.
@ -67,6 +72,7 @@ func NewStorageUsingRole(
roleARN string,
sessionName string,
sessionTags map[string]string,
queryParams map[string]string,
duration string,
cfgs ...common.StringConfigurer,
) (Storage, error) {
@ -77,6 +83,7 @@ func NewStorageUsingRole(
Config: cs,
Role: roleARN,
SessionTags: sessionTags,
QueryParams: queryParams,
SessionName: sessionName,
SessionDuration: duration,
}, err