diff --git a/src/cli/config/config_test.go b/src/cli/config/config_test.go index 474645f02..073709e40 100644 --- a/src/cli/config/config_test.go +++ b/src/cli/config/config_test.go @@ -22,7 +22,7 @@ const ( configFileTemplate = ` ` + BucketNameKey + ` = '%s' ` + EndpointKey + ` = 's3.amazonaws.com' -` + PrefixKey + ` = 'test-prefix' +` + PrefixKey + ` = 'test-prefix/' ` + StorageProviderTypeKey + ` = 'S3' ` + AccountProviderTypeKey + ` = 'M365' ` + TenantIDKey + ` = '%s' @@ -208,7 +208,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() { const ( bkt = "get-storage-and-account-bucket" end = "https://get-storage-and-account.com" - pfx = "get-storage-and-account-prefix" + pfx = "get-storage-and-account-prefix/" tid = "3a2faa4e-a882-445c-9d27-f552ef189381" ) @@ -253,7 +253,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride const ( bkt = "get-storage-and-account-no-file-bucket" end = "https://get-storage-and-account.com/no-file" - pfx = "get-storage-and-account-no-file-prefix" + pfx = "get-storage-and-account-no-file-prefix/" tid = "88f8522b-18e4-4d0f-b514-2d7b34d4c5a1" ) diff --git a/src/cli/config/storage.go b/src/cli/config/storage.go index bc71a40b8..ab3a2807b 100644 --- a/src/cli/config/storage.go +++ b/src/cli/config/storage.go @@ -61,6 +61,10 @@ func configureStorage( overrides[storage.Bucket] = common.NormalizeBucket(b) } + if p, ok := overrides[storage.Prefix]; ok { + overrides[storage.Prefix] = common.NormalizePrefix(p) + } + if err := mustMatchConfig(vpr, s3Overrides(overrides)); err != nil { return store, errors.Wrap(err, "verifying s3 configs in corso config file") } diff --git a/src/internal/common/buckets.go b/src/internal/common/buckets.go index 73e207d6e..90f1ee4fa 100644 --- a/src/internal/common/buckets.go +++ b/src/internal/common/buckets.go @@ -11,3 +11,15 @@ import "strings" func NormalizeBucket(b string) string { return strings.TrimPrefix(b, "s3://") } + +// NormalizePrefix ensures that a bucket prefix is always treated as +// object store folder prefix. +func NormalizePrefix(p string) string { + tp := strings.TrimRight(p, "/") + + if len(tp) > 0 { + tp = tp + "/" + } + + return tp +} diff --git a/src/internal/common/buckets_test.go b/src/internal/common/buckets_test.go index b44149deb..c8c6ee762 100644 --- a/src/internal/common/buckets_test.go +++ b/src/internal/common/buckets_test.go @@ -17,7 +17,7 @@ func TestCommonBucketsSuite(t *testing.T) { suite.Run(t, new(CommonBucketsSuite)) } -func (suite *CommonBucketsSuite) TestDoesThings() { +func (suite *CommonBucketsSuite) TestBucketPrefix() { t := suite.T() trimmablePrefixes := []string{"s3://"} @@ -26,3 +26,17 @@ func (suite *CommonBucketsSuite) TestDoesThings() { assert.Equal(t, "smarf", "smarf") } } + +func (suite *CommonBucketsSuite) TestPrefixSuffix() { + t := suite.T() + + prefixBase := "repo-prefix" + properPrefix := prefixBase + "/" + + assert.Equal(t, properPrefix, common.NormalizePrefix(prefixBase), "Trailing '/' should be added") + assert.Equal(t, properPrefix, common.NormalizePrefix(properPrefix), "Properly formatted prefix should not change") + assert.Equal(t, properPrefix, common.NormalizePrefix(prefixBase+"///"), "Only one trailing / should exist") + assert.Equal(t, properPrefix+"/sub/", common.NormalizePrefix(properPrefix+"/sub"), "Only affect trailing /") + assert.Equal(t, "", common.NormalizePrefix(""), "Only normalize actual prefix.") + assert.Equal(t, "", common.NormalizePrefix("//"), "Only normalize actual prefix.") +} diff --git a/src/pkg/storage/s3.go b/src/pkg/storage/s3.go index cc20e2a36..a6a5674c1 100644 --- a/src/pkg/storage/s3.go +++ b/src/pkg/storage/s3.go @@ -30,7 +30,7 @@ func (c S3Config) Normalize() S3Config { return S3Config{ Bucket: common.NormalizeBucket(c.Bucket), Endpoint: c.Endpoint, - Prefix: c.Prefix, + Prefix: common.NormalizePrefix(c.Prefix), } } diff --git a/src/pkg/storage/s3_test.go b/src/pkg/storage/s3_test.go index b74dadcd0..429196298 100644 --- a/src/pkg/storage/s3_test.go +++ b/src/pkg/storage/s3_test.go @@ -22,13 +22,13 @@ var ( goodS3Config = storage.S3Config{ Bucket: "bkt", Endpoint: "end", - Prefix: "pre", + Prefix: "pre/", } goodS3Map = map[string]string{ "s3_bucket": "bkt", "s3_endpoint": "end", - "s3_prefix": "pre", + "s3_prefix": "pre/", } ) @@ -78,7 +78,7 @@ func (suite *S3CfgSuite) TestStorage_S3Config_invalidCases() { name string cfg storage.S3Config }{ - {"missing bucket", makeTestS3Cfg("", "end", "pre")}, + {"missing bucket", makeTestS3Cfg("", "end", "pre/")}, } for _, test := range table { suite.T().Run(test.name, func(t *testing.T) {