changes requried for manual repo init (#25) (#80)

A small collection of changes and code cleanup to successfully run
'corso repo init s3' manually.  Adds a default s3 url (might want
this configurable for local testing) and aws session token support.

src/cli/repo/s3.go

@@ -47,7 +47,8 @@ func initS3Cmd(cmd *cobra.Command, args []string) {
 	mv := getM365Vars()
 	s3Cfg := makeS3Config()
 	fmt.Printf(
-		"Called -\n`corso repo init s3`\nbucket:\t%s\nkey:\t%s\n356Client:\t%s\nfound 356Secret:\t%v\nfound awsSecret:\t%v\n",
+		"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
+		cmd.CommandPath(),
 		s3Cfg.Bucket,
 		s3Cfg.AccessKey,
 		mv.clientID,
@@ -65,6 +66,8 @@ func initS3Cmd(cmd *cobra.Command, args []string) {
 		fmt.Printf("Failed to initialize a new S3 repository: %v", err)
 		os.Exit(1)
 	}
+
+	fmt.Printf("Initialized a S3 repository within bucket %s.\n", s3Cfg.Bucket)
 }
 
 // `corso repo connect s3 [<flag>...]`
@@ -81,7 +84,8 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
 	mv := getM365Vars()
 	s3Cfg := makeS3Config()
 	fmt.Printf(
-		"Called -\n`corso repo connect s3`\nbucket:\t%s\nkey:\t%s\n356Client:\t%s\nfound 356Secret:\t%v\nfound awsSecret:\t%v\n",
+		"Called - %s\n\tbucket:\t%s\n\tkey:\t%s\n\t356Client:\t%s\n\tfound 356Secret:\t%v\n\tfound awsSecret:\t%v\n",
+		cmd.CommandPath(),
 		s3Cfg.Bucket,
 		s3Cfg.AccessKey,
 		mv.clientID,
@@ -99,6 +103,8 @@ func connectS3Cmd(cmd *cobra.Command, args []string) {
 		fmt.Printf("Failed to connect to the S3 repository: %v", err)
 		os.Exit(1)
 	}
+
+	fmt.Printf("Connected to S3 bucket %s.\n", s3Cfg.Bucket)
 }
 
 // helper for aggregating aws connection details.
@@ -108,8 +114,9 @@ func makeS3Config() storage.S3Config {
 		ak = accessKey
 	}
 	return storage.S3Config{
-		AccessKey: ak,
+		AccessKey:    ak,
-		SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
+		Bucket:       bucket,
-		Bucket:    bucket,
+		SecretKey:    os.Getenv("AWS_SECRET_ACCESS_KEY"),
+		SessionToken: os.Getenv("AWS_SESSION_TOKEN"),
 	}
 }

src/internal/kopia/s3.go

@@ -9,11 +9,17 @@ import (
 	"github.com/alcionai/corso/pkg/storage"
 )
 
+const (
+	defaultS3Endpoint = "s3.amazonaws.com" // matches kopia's default value
+)
+
 func s3BlobStorage(ctx context.Context, cfg storage.S3Config) (blob.Storage, error) {
 	opts := s3.Options{
-		BucketName:      cfg.Bucket,
 		AccessKeyID:     cfg.AccessKey,
+		BucketName:      cfg.Bucket,
+		Endpoint:        defaultS3Endpoint,
 		SecretAccessKey: cfg.SecretKey,
+		SessionToken:    cfg.SessionToken,
 	}
 	return s3.New(ctx, &opts)
 }

src/pkg/repository/s3/s3.go

@@ -1,35 +0,0 @@
-package s3
-
-import (
-	"context"
-
-	"github.com/kopia/kopia/repo/blob"
-	kopiaS3 "github.com/kopia/kopia/repo/blob/s3"
-)
-
-// Config defines communication with a s3 repository provider.
-type Config struct {
-	Bucket          string // the S3 storage bucket name
-	AccessKey       string // access key to the S3 bucket
-	SecretAccessKey string // s3 access key secret
-}
-
-// NewConfig generates a S3 configuration struct to use for interfacing with a s3 storage
-// bucket using a repository.Repository.
-func NewConfig(bucket, accessKey, secretKey string) Config {
-	return Config{
-		Bucket:          bucket,
-		AccessKey:       accessKey,
-		SecretAccessKey: secretKey,
-	}
-}
-
-// KopiaStorage produces a kopia/blob Storage handle for connecting to s3.
-func (c Config) KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) {
-	opts := kopiaS3.Options{
-		BucketName:      c.Bucket,
-		AccessKeyID:     c.AccessKey,
-		SecretAccessKey: c.SecretAccessKey,
-	}
-	return kopiaS3.New(ctx, &opts)
-}

src/pkg/repository/s3/s3_test.go

@@ -1,14 +0,0 @@
-package s3_test
-
-import (
-	"testing"
-
-	"github.com/alcionai/corso/pkg/repository/s3"
-)
-
-func TestNewS3(t *testing.T) {
-	cfg := s3.NewConfig("bucket", "access", "secret")
-	if cfg.Bucket != "bucket" {
-		t.Errorf("expected s3 config bucke to be 'bucket', got '%s'", cfg.Bucket)
-	}
-}

src/pkg/storage/s3.go

@@ -1,22 +1,25 @@
 package storage
 
 type S3Config struct {
-	Bucket    string
+	AccessKey    string
-	AccessKey string
+	Bucket       string
-	SecretKey string
+	SecretKey    string
+	SessionToken string
 }
 
 const (
-	keyS3Bucket    = "s3_bucket"
+	keyS3AccessKey    = "s3_accessKey"
-	keyS3AccessKey = "s3_accessKey"
+	keyS3Bucket       = "s3_bucket"
-	keyS3SecretKey = "s3_secretKey"
+	keyS3SecretKey    = "s3_secretKey"
+	keyS3SessionToken = "s3_sessionToken"
 )
 
 func (c S3Config) Config() config {
 	return config{
-		keyS3Bucket:    c.Bucket,
+		keyS3AccessKey:    c.AccessKey,
-		keyS3AccessKey: c.AccessKey,
+		keyS3Bucket:       c.Bucket,
-		keyS3SecretKey: c.SecretKey,
+		keyS3SecretKey:    c.SecretKey,
+		keyS3SessionToken: c.SessionToken,
 	}
 }
 
@@ -24,9 +27,10 @@ func (c S3Config) Config() config {
 func (s Storage) S3Config() S3Config {
 	c := S3Config{}
 	if len(s.Config) > 0 {
-		c.Bucket = s.Config[keyS3Bucket].(string)
 		c.AccessKey = s.Config[keyS3AccessKey].(string)
+		c.Bucket = s.Config[keyS3Bucket].(string)
 		c.SecretKey = s.Config[keyS3SecretKey].(string)
+		c.SessionToken = s.Config[keyS3SessionToken].(string)
 	}
 	return c
 }

src/pkg/storage/s3_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestS3Config_Config(t *testing.T) {
-	s3 := storage.S3Config{"bkt", "ak", "sk"}
+	s3 := storage.S3Config{"bkt", "ak", "sk", "tkn"}
 	c := s3.Config()
 	table := []struct {
 		key    string
@@ -16,6 +16,7 @@ func TestS3Config_Config(t *testing.T) {
 		{"s3_bucket", s3.Bucket},
 		{"s3_accessKey", s3.AccessKey},
 		{"s3_secretKey", s3.SecretKey},
+		{"s3_sessionToken", s3.SessionToken},
 	}
 	for _, test := range table {
 		key := test.key
@@ -27,7 +28,7 @@ func TestS3Config_Config(t *testing.T) {
 }
 
 func TestStorage_S3Config(t *testing.T) {
-	in := storage.S3Config{"bkt", "ak", "sk"}
+	in := storage.S3Config{"bkt", "ak", "sk", "tkn"}
 	s := storage.NewStorage(storage.ProviderS3, in)
 	out := s.S3Config()
 	if in.Bucket != out.Bucket {
@@ -39,4 +40,7 @@ func TestStorage_S3Config(t *testing.T) {
 	if in.SecretKey != out.SecretKey {
 		t.Errorf("expected S3Config.SecretKey to be [%s], got [%s]", in.SecretKey, out.SecretKey)
 	}
+	if in.SessionToken != out.SessionToken {
+		t.Errorf("expected S3Config.SessionToken to be [%s], got [%s]", in.SessionToken, out.SessionToken)
+	}
 }