introduce repository api scaffold with s3 (#6) (#54)

* introduce repository api scaffold with s3 (#6)

introduces the public repository package in /pkg, with a
skeleton implementation for generating the s3 provider config
and boilerplate for where that usage gets centralized.

* condense init/connect to single action

A double-pass NewRepo() && Connect() allows Repository structs
to exist independent guarantees that their usage will be successful.
This hides the struct generation within the Connect and Initialize
funcs so that the construction and guarantees can occur as
a single process.

src/go.mod

@@ -2,9 +2,32 @@ module github.com/alcionai/corso
 
 go 1.18
 
-require github.com/spf13/cobra v1.4.0
+require (
+	github.com/google/uuid v1.3.0
+	github.com/kopia/kopia v0.10.7
+	github.com/spf13/cobra v1.4.0
+)
 
 require (
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.23 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rs/xid v1.3.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
 )

src/go.sum

@@ -1,10 +1,70 @@
+github.com/aws/aws-sdk-go v1.43.31 h1:yJZIr8nMV1hXjAvvOLUFqZRJcHV7udPQBfhJqawDzI0=
+github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
+github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/kopia/kopia v0.10.7 h1:6s0ZIZW3Ge2ozzefddASy7CIUadp/5tF9yCDKQfAKKI=
+github.com/kopia/kopia v0.10.7/go.mod h1:0d9THPD+jwomPcXvPbCdmLyX6phQVP7AqcCcDEajfNA=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4=
+github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
 github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc=
+golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q=
+golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI=
+gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=

src/pkg/repository/repoprovider_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type=repoProvider"; DO NOT EDIT.
+
+package repository
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[ProviderUnknown-0]
+	_ = x[ProviderS3-1]
+}
+
+const _repoProvider_name = "ProviderUnknownProviderS3"
+
+var _repoProvider_index = [...]uint8{0, 15, 25}
+
+func (i repoProvider) String() string {
+	if i < 0 || i >= repoProvider(len(_repoProvider_index)-1) {
+		return "repoProvider(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _repoProvider_name[_repoProvider_index[i]:_repoProvider_index[i+1]]
+}

src/pkg/repository/repository.go

@@ -0,0 +1,84 @@
+package repository
+
+import (
+	"context"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/kopia/kopia/repo/blob"
+)
+
+type repoProvider int
+
+//go:generate stringer -type=repoProvider
+const (
+	ProviderUnknown repoProvider = iota // Unknown Provider
+	ProviderS3                          // S3
+)
+
+// Repository contains storage provider information.
+type Repository struct {
+	ID        uuid.UUID
+	CreatedAt time.Time
+	Version   string // in case of future breaking changes
+
+	Provider repoProvider // must be repository.S3Provider
+	Account  Account      // the user's m365 account connection details
+	Config   Config       // provider-based configuration details
+}
+
+// Account holds the user's m365 account details.
+type Account struct {
+	TenantID     string
+	ClientID     string
+	ClientSecret string
+}
+
+type Config interface {
+	KopiaStorage(ctx context.Context, create bool) (blob.Storage, error)
+}
+
+// Initialize will:
+//  * validate the m365 account & secrets
+//  * connect to the m365 account to ensure communication capability
+//  * validate the provider config & secrets
+//  * initialize the kopia repo with the provider
+//  * store the configuration details
+//  * connect to the provider
+//  * return the connected repository
+func Initialize(
+	ctx context.Context,
+	provider repoProvider,
+	acct Account,
+	cfg Config,
+) (Repository, error) {
+	r := Repository{
+		ID:       uuid.New(),
+		Version:  "v1",
+		Provider: provider,
+		Account:  acct,
+		Config:   cfg,
+	}
+	return r, nil
+}
+
+// Connect will:
+//  * validate the m365 account details
+//  * connect to the m365 account to ensure communication capability
+//  * connect to the provider storage
+//  * return the connected repository
+func Connect(
+	ctx context.Context,
+	provider repoProvider,
+	acct Account,
+	cfg Config,
+) (Repository, error) {
+	// todo: ID and CreatedAt should get retrieved from a stored kopia config.
+	r := Repository{
+		Version:  "v1",
+		Provider: provider,
+		Account:  acct,
+		Config:   cfg,
+	}
+	return r, nil
+}

src/pkg/repository/repository_test.go

@@ -0,0 +1,38 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kopia/kopia/repo/blob"
+
+	"github.com/alcionai/corso/pkg/repository"
+)
+
+type testConfig struct{}
+
+func (tc testConfig) KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) {
+	return nil, nil
+}
+
+func TestInitialize(t *testing.T) {
+	_, err := repository.Initialize(
+		context.Background(),
+		repository.ProviderUnknown,
+		repository.Account{},
+		testConfig{})
+	if err != nil {
+		t.Fatalf("didn't expect initialize to error, got [%v]", err)
+	}
+}
+
+func TestConnect(t *testing.T) {
+	_, err := repository.Connect(
+		context.Background(),
+		repository.ProviderUnknown,
+		repository.Account{},
+		testConfig{})
+	if err != nil {
+		t.Fatalf("didn't expect connect to error, got [%v]", err)
+	}
+}

src/pkg/repository/s3/s3.go

@@ -0,0 +1,35 @@
+package s3
+
+import (
+	"context"
+
+	"github.com/kopia/kopia/repo/blob"
+	kopiaS3 "github.com/kopia/kopia/repo/blob/s3"
+)
+
+// Config defines communication with a s3 repository provider.
+type Config struct {
+	Bucket          string // the S3 storage bucket name
+	AccessKey       string // access key to the S3 bucket
+	SecretAccessKey string // s3 access key secret
+}
+
+// NewConfig generates a S3 configuration struct to use for interfacing with a s3 storage
+// bucket using a repository.Repository.
+func NewConfig(bucket, accessKey, secretKey string) Config {
+	return Config{
+		Bucket:          bucket,
+		AccessKey:       accessKey,
+		SecretAccessKey: secretKey,
+	}
+}
+
+// KopiaStorage produces a kopia/blob Storage handle for connecting to s3.
+func (c Config) KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) {
+	opts := kopiaS3.Options{
+		BucketName:      c.Bucket,
+		AccessKeyID:     c.AccessKey,
+		SecretAccessKey: c.SecretAccessKey,
+	}
+	return kopiaS3.New(ctx, &opts)
+}

src/pkg/repository/s3/s3_test.go

@@ -0,0 +1,14 @@
+package s3_test
+
+import (
+	"testing"
+
+	"github.com/alcionai/corso/pkg/repository/s3"
+)
+
+func TestNewS3(t *testing.T) {
+	cfg := s3.NewConfig("bucket", "access", "secret")
+	if cfg.Bucket != "bucket" {
+		t.Errorf("expected s3 config bucke to be 'bucket', got '%s'", cfg.Bucket)
+	}
+}