normalize bucket naming during config lookup (#1025)

## Description

Ensure s3 bucket name inputs are normalized
in all uses.  Currently, the normalized format is
applied when using storage, but not when passing
around --bucket flag values elsewhere.

## Type of change

- [x] 🐛 Bugfix

## Issue(s)

* #975

## Test Plan

- [x] 💪 Manual
- [x]  Unit test
This commit is contained in:
Keepers 2022-10-03 15:37:20 -06:00 committed by GitHub
parent d60a771024
commit fe148af455
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 50 additions and 3 deletions

View File

@ -57,6 +57,10 @@ func configureStorage(
return store, errors.Wrap(err, "reading s3 configs from corso config file")
}
if b, ok := overrides[storage.Bucket]; ok {
overrides[storage.Bucket] = common.NormalizeBucket(b)
}
if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil {
return store, errors.Wrap(err, "verifying s3 configs in corso config file")
}

View File

@ -0,0 +1,13 @@
package common
import "strings"
// NormalizeBuckets ensures that bucket names are cleaned and
// standardized according to the downstream needs of minio.
//
// Any url prefixing to location the bucket (ex: s3://bckt)
// will be removed, leaving only the bucket name (bckt).
// Corso should only utilize or store the normalized name.
func NormalizeBucket(b string) string {
return strings.TrimPrefix(b, "s3://")
}

View File

@ -0,0 +1,30 @@
package common_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
)
type CommonBucketsSuite struct {
suite.Suite
ctx context.Context
}
func TestCommonBucketsSuite(t *testing.T) {
suite.Run(t, new(CommonBucketsSuite))
}
func (suite *CommonBucketsSuite) TestDoesThings() {
t := suite.T()
trimmablePrefixes := []string{"s3://"}
for _, pfx := range trimmablePrefixes {
assert.Equal(t, "fnords", common.NormalizeBucket(pfx+"fnords"))
assert.Equal(t, "smarf", "smarf")
}
}

View File

@ -1,9 +1,9 @@
package storage
import (
"strings"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common"
)
type S3Config struct {
@ -28,7 +28,7 @@ const (
func (c S3Config) Normalize() S3Config {
return S3Config{
Bucket: strings.TrimPrefix(c.Bucket, "s3://"),
Bucket: common.NormalizeBucket(c.Bucket),
Endpoint: c.Endpoint,
Prefix: c.Prefix,
}