From 08dfe6df66657466477308d0ee9fb531044729ef Mon Sep 17 00:00:00 2001 From: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:22:33 -0800 Subject: [PATCH] Minio custom query params (#4763) Add and wire up custom query parameters for minio to add extra fields to S3 server access log output ** This switches to a custom minio version and updates kopia ** Diffs for the package updates (based on current versions used in corso): * [minio](https://github.com/minio/minio-go/compare/v7.0.64...alcionai:minio-go:pass-custom-get-query-params-local?expand=1) * [kopia](https://github.com/alcionai/kopia/compare/corsostaging-dir-iterator...custom-query-params) --- #### Does this PR need a docs update or release note? - [ ] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Test Plan - [x] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- src/go.mod | 6 +++++- src/go.sum | 8 ++++---- src/internal/kopia/s3.go | 1 + src/pkg/storage/storage.go | 7 +++++++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/go.mod b/src/go.mod index 2cba7a205..1d1df2071 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,7 +2,11 @@ module github.com/alcionai/corso/src go 1.21 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23 +replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db + +// No tags in the alcion fork of the repo so use v7 as that's in the import +// path. +replace github.com/minio/minio-go/v7 => github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 diff --git a/src/go.sum b/src/go.sum index 125f68e6d..3a201584f 100644 --- a/src/go.sum +++ b/src/go.sum @@ -17,8 +17,10 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alcionai/clues v0.0.0-20231115004051-523cbddac8e8 h1:kdpkhcKWWxnZFteGjglHvFTTPCPsvjhKDDY9J1Od6Sg= github.com/alcionai/clues v0.0.0-20231115004051-523cbddac8e8/go.mod h1:hGnRqQtV7YoojQSNMtqFK0TvfcwAljGZhUEQwx1lw34= -github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23 h1:ojlHvw65O21oFTC6kQldVDO69VQ6JEzMdD6aq9aOrY4= -github.com/alcionai/kopia v0.12.2-0.20231128180219-e763f7c81f23/go.mod h1:f4PligAuyEicX+lfTlZltc69nM0eMoXX2nE5sCBdo6Y= +github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db h1:zSaL5qEs8iYL3O40AT5dHfCJejbYQ/7a4YHe+I+wXII= +github.com/alcionai/kopia v0.12.2-0.20231130213928-e62eca5816db/go.mod h1:fsJixnpmyK+MoY+P/ULJXiX52WkRoEk6V4bJd0jUW5I= +github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa h1:PHzp5TkXgsardwMG6O2nnyk3zBsGW8CqgsOWQCYkykQ= +github.com/alcionai/minio-go/v7 v7.0.0-20231130221740-c745a3d084aa/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -208,8 +210,6 @@ github.com/microsoftgraph/msgraph-sdk-go-core v1.0.1 h1:uq4qZD8VXLiNZY0t4NoRpLDo github.com/microsoftgraph/msgraph-sdk-go-core v1.0.1/go.mod h1:HUITyuFN556+0QZ/IVfH5K4FyJM7kllV6ExKi2ImKhE= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.64 h1:Zdza8HwOzkld0ZG/og50w56fKi6AAyfqfifmasD9n2Q= -github.com/minio/minio-go/v7 v7.0.64/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= diff --git a/src/internal/kopia/s3.go b/src/internal/kopia/s3.go index 6fcde4827..851322b80 100644 --- a/src/internal/kopia/s3.go +++ b/src/internal/kopia/s3.go @@ -37,6 +37,7 @@ func s3BlobStorage( DoNotUseTLS: cfg.DoNotUseTLS, DoNotVerifyTLS: cfg.DoNotVerifyTLS, Tags: s.SessionTags, + QueryParams: s.QueryParams, SessionName: s.SessionName, RoleARN: s.Role, RoleDuration: s.SessionDuration, diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index 89a713ecb..2c99b521c 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -48,6 +48,11 @@ type Storage struct { Role string SessionName string SessionDuration string + // QueryParams allows passing custom query parameters to S3 GET object + // requests. Parameters prefixed by `x-` will be ignored by S3 (no functional + // changes) but added to server access logs. These KV-pairs are session + // specific and won't be persisted to or sourced from the corso config file. + QueryParams map[string]string } // NewStorage aggregates all the supplied configurations into a single configuration. @@ -67,6 +72,7 @@ func NewStorageUsingRole( roleARN string, sessionName string, sessionTags map[string]string, + queryParams map[string]string, duration string, cfgs ...common.StringConfigurer, ) (Storage, error) { @@ -77,6 +83,7 @@ func NewStorageUsingRole( Config: cs, Role: roleARN, SessionTags: sessionTags, + QueryParams: queryParams, SessionName: sessionName, SessionDuration: duration, }, err