normalize bucket naming during config lookup (#1025)
## Description Ensure s3 bucket name inputs are normalized in all uses. Currently, the normalized format is applied when using storage, but not when passing around --bucket flag values elsewhere. ## Type of change - [x] 🐛 Bugfix ## Issue(s) * #975 ## Test Plan - [x] 💪 Manual - [x] ⚡ Unit test
This commit is contained in:
parent
d60a771024
commit
fe148af455
@ -57,6 +57,10 @@ func configureStorage(
|
|||||||
return store, errors.Wrap(err, "reading s3 configs from corso config file")
|
return store, errors.Wrap(err, "reading s3 configs from corso config file")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if b, ok := overrides[storage.Bucket]; ok {
|
||||||
|
overrides[storage.Bucket] = common.NormalizeBucket(b)
|
||||||
|
}
|
||||||
|
|
||||||
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
|
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
|
||||||
return store, errors.Wrap(err, "verifying s3 configs in corso config file")
|
return store, errors.Wrap(err, "verifying s3 configs in corso config file")
|
||||||
}
|
}
|
||||||
|
|||||||
13
src/internal/common/buckets.go
Normal file
13
src/internal/common/buckets.go
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
package common
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// NormalizeBuckets ensures that bucket names are cleaned and
|
||||||
|
// standardized according to the downstream needs of minio.
|
||||||
|
//
|
||||||
|
// Any url prefixing to location the bucket (ex: s3://bckt)
|
||||||
|
// will be removed, leaving only the bucket name (bckt).
|
||||||
|
// Corso should only utilize or store the normalized name.
|
||||||
|
func NormalizeBucket(b string) string {
|
||||||
|
return strings.TrimPrefix(b, "s3://")
|
||||||
|
}
|
||||||
30
src/internal/common/buckets_test.go
Normal file
30
src/internal/common/buckets_test.go
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
package common_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CommonBucketsSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommonBucketsSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(CommonBucketsSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *CommonBucketsSuite) TestDoesThings() {
|
||||||
|
t := suite.T()
|
||||||
|
trimmablePrefixes := []string{"s3://"}
|
||||||
|
|
||||||
|
for _, pfx := range trimmablePrefixes {
|
||||||
|
assert.Equal(t, "fnords", common.NormalizeBucket(pfx+"fnords"))
|
||||||
|
assert.Equal(t, "smarf", "smarf")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,9 +1,9 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
"github.com/alcionai/corso/src/internal/common"
|
||||||
)
|
)
|
||||||
|
|
||||||
type S3Config struct {
|
type S3Config struct {
|
||||||
@ -28,7 +28,7 @@ const (
|
|||||||
|
|
||||||
func (c S3Config) Normalize() S3Config {
|
func (c S3Config) Normalize() S3Config {
|
||||||
return S3Config{
|
return S3Config{
|
||||||
Bucket: strings.TrimPrefix(c.Bucket, "s3://"),
|
Bucket: common.NormalizeBucket(c.Bucket),
|
||||||
Endpoint: c.Endpoint,
|
Endpoint: c.Endpoint,
|
||||||
Prefix: c.Prefix,
|
Prefix: c.Prefix,
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user