expose s3 endpoint as flag (#87)

Exposes the s3 endpoint option as a cli flag and storage config
property.  Uses the kopia default s3.amazonaws.com as a fallback.

.gitignore

@@ -13,3 +13,6 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+# IDE
+.vscode

src/cli/repo/s3.go

@@ -12,8 +12,9 @@ import (
 
 // s3 bucket info from flags
 var (
-	bucket    string
 	accessKey string
+	bucket    string
+	endpoint  string
 )
 
 // called by repo.go to map parent subcommands to provider-specific handling.
@@ -27,9 +28,10 @@ func addS3Commands(parent *cobra.Command) *cobra.Command {
 	}
 	parent.AddCommand(c)
 	fs := c.Flags()
+	fs.StringVar(&accessKey, "access-key", "", "Access key ID (replaces the AWS_ACCESS_KEY_ID env variable).")
 	fs.StringVar(&bucket, "bucket", "", "Name of the S3 bucket (required).")
 	c.MarkFlagRequired("bucket")
-	fs.StringVar(&accessKey, "access-key", "", "Access key ID (replaces the AWS_ACCESS_KEY_ID env variable).")
+	fs.StringVar(&endpoint, "endpoint", "s3.amazonaws.com", "Server endpoint for S3 communication.")
 	return c
 }
 
@@ -116,6 +118,7 @@ func makeS3Config() storage.S3Config {
 	return storage.S3Config{
 		AccessKey:    ak,
 		Bucket:       bucket,
+		Endpoint:     endpoint,
 		SecretKey:    os.Getenv("AWS_SECRET_ACCESS_KEY"),
 		SessionToken: os.Getenv("AWS_SESSION_TOKEN"),
 	}

src/internal/kopia/s3.go

@@ -14,10 +14,14 @@ const (
 )
 
 func s3BlobStorage(ctx context.Context, cfg storage.S3Config) (blob.Storage, error) {
+	endpoint := defaultS3Endpoint
+	if len(cfg.Endpoint) > 0 {
+		endpoint = cfg.Endpoint
+	}
 	opts := s3.Options{
 		AccessKeyID:     cfg.AccessKey,
 		BucketName:      cfg.Bucket,
-		Endpoint:        defaultS3Endpoint,
+		Endpoint:        endpoint,
 		SecretAccessKey: cfg.SecretKey,
 		SessionToken:    cfg.SessionToken,
 	}

src/pkg/storage/s3.go

@@ -3,6 +3,7 @@ package storage
 type S3Config struct {
 	AccessKey    string
 	Bucket       string
+	Endpoint     string
 	SecretKey    string
 	SessionToken string
 }
@@ -10,6 +11,7 @@ type S3Config struct {
 const (
 	keyS3AccessKey    = "s3_accessKey"
 	keyS3Bucket       = "s3_bucket"
+	keyS3Endpoint     = "s3_endpoint"
 	keyS3SecretKey    = "s3_secretKey"
 	keyS3SessionToken = "s3_sessionToken"
 )
@@ -18,6 +20,7 @@ func (c S3Config) Config() config {
 	return config{
 		keyS3AccessKey:    c.AccessKey,
 		keyS3Bucket:       c.Bucket,
+		keyS3Endpoint:     c.Endpoint,
 		keyS3SecretKey:    c.SecretKey,
 		keyS3SessionToken: c.SessionToken,
 	}
@@ -29,6 +32,7 @@ func (s Storage) S3Config() S3Config {
 	if len(s.Config) > 0 {
 		c.AccessKey = s.Config[keyS3AccessKey].(string)
 		c.Bucket = s.Config[keyS3Bucket].(string)
+		c.Endpoint = s.Config[keyS3Endpoint].(string)
 		c.SecretKey = s.Config[keyS3SecretKey].(string)
 		c.SessionToken = s.Config[keyS3SessionToken].(string)
 	}

src/pkg/storage/s3_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestS3Config_Config(t *testing.T) {
-	s3 := storage.S3Config{"bkt", "ak", "sk", "tkn"}
+	s3 := storage.S3Config{"ak", "bkt", "end", "sk", "tkn"}
 	c := s3.Config()
 	table := []struct {
 		key    string
@@ -15,6 +15,7 @@ func TestS3Config_Config(t *testing.T) {
 	}{
 		{"s3_bucket", s3.Bucket},
 		{"s3_accessKey", s3.AccessKey},
+		{"s3_endpoint", s3.Endpoint},
 		{"s3_secretKey", s3.SecretKey},
 		{"s3_sessionToken", s3.SessionToken},
 	}
@@ -28,7 +29,7 @@ func TestS3Config_Config(t *testing.T) {
 }
 
 func TestStorage_S3Config(t *testing.T) {
-	in := storage.S3Config{"bkt", "ak", "sk", "tkn"}
+	in := storage.S3Config{"ak", "bkt", "end", "sk", "tkn"}
 	s := storage.NewStorage(storage.ProviderS3, in)
 	out := s.S3Config()
 	if in.Bucket != out.Bucket {
@@ -37,6 +38,9 @@ func TestStorage_S3Config(t *testing.T) {
 	if in.AccessKey != out.AccessKey {
 		t.Errorf("expected S3Config.AccessKey to be [%s], got [%s]", in.AccessKey, out.AccessKey)
 	}
+	if in.Endpoint != out.Endpoint {
+		t.Errorf("expected S3Config.Endpoint to be [%s], got [%s]", in.Endpoint, out.Endpoint)
+	}
 	if in.SecretKey != out.SecretKey {
 		t.Errorf("expected S3Config.SecretKey to be [%s], got [%s]", in.SecretKey, out.SecretKey)
 	}