diff --git a/src/go.mod b/src/go.mod index ad764141e..05b42ac02 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,9 +2,32 @@ module github.com/alcionai/corso go 1.18 -require github.com/spf13/cobra v1.4.0 +require ( + github.com/google/uuid v1.3.0 + github.com/kopia/kopia v0.10.7 + github.com/spf13/cobra v1.4.0 +) require ( + github.com/dustin/go-humanize v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.23 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rs/xid v1.3.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect + golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect + golang.org/x/text v0.3.7 // indirect + gopkg.in/ini.v1 v1.66.2 // indirect ) diff --git a/src/go.sum b/src/go.sum index 0dd8697bc..53dbd4612 100644 --- a/src/go.sum +++ b/src/go.sum @@ -1,10 +1,70 @@ +github.com/aws/aws-sdk-go v1.43.31 h1:yJZIr8nMV1hXjAvvOLUFqZRJcHV7udPQBfhJqawDzI0= +github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/kopia/kopia v0.10.7 h1:6s0ZIZW3Ge2ozzefddASy7CIUadp/5tF9yCDKQfAKKI= +github.com/kopia/kopia v0.10.7/go.mod h1:0d9THPD+jwomPcXvPbCdmLyX6phQVP7AqcCcDEajfNA= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4= +github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= diff --git a/src/pkg/repository/repoprovider_string.go b/src/pkg/repository/repoprovider_string.go new file mode 100644 index 000000000..a85ff1f2e --- /dev/null +++ b/src/pkg/repository/repoprovider_string.go @@ -0,0 +1,24 @@ +// Code generated by "stringer -type=repoProvider"; DO NOT EDIT. + +package repository + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ProviderUnknown-0] + _ = x[ProviderS3-1] +} + +const _repoProvider_name = "ProviderUnknownProviderS3" + +var _repoProvider_index = [...]uint8{0, 15, 25} + +func (i repoProvider) String() string { + if i < 0 || i >= repoProvider(len(_repoProvider_index)-1) { + return "repoProvider(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _repoProvider_name[_repoProvider_index[i]:_repoProvider_index[i+1]] +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go new file mode 100644 index 000000000..f61a308e2 --- /dev/null +++ b/src/pkg/repository/repository.go @@ -0,0 +1,84 @@ +package repository + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/kopia/kopia/repo/blob" +) + +type repoProvider int + +//go:generate stringer -type=repoProvider +const ( + ProviderUnknown repoProvider = iota // Unknown Provider + ProviderS3 // S3 +) + +// Repository contains storage provider information. +type Repository struct { + ID uuid.UUID + CreatedAt time.Time + Version string // in case of future breaking changes + + Provider repoProvider // must be repository.S3Provider + Account Account // the user's m365 account connection details + Config Config // provider-based configuration details +} + +// Account holds the user's m365 account details. +type Account struct { + TenantID string + ClientID string + ClientSecret string +} + +type Config interface { + KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) +} + +// Initialize will: +// * validate the m365 account & secrets +// * connect to the m365 account to ensure communication capability +// * validate the provider config & secrets +// * initialize the kopia repo with the provider +// * store the configuration details +// * connect to the provider +// * return the connected repository +func Initialize( + ctx context.Context, + provider repoProvider, + acct Account, + cfg Config, +) (Repository, error) { + r := Repository{ + ID: uuid.New(), + Version: "v1", + Provider: provider, + Account: acct, + Config: cfg, + } + return r, nil +} + +// Connect will: +// * validate the m365 account details +// * connect to the m365 account to ensure communication capability +// * connect to the provider storage +// * return the connected repository +func Connect( + ctx context.Context, + provider repoProvider, + acct Account, + cfg Config, +) (Repository, error) { + // todo: ID and CreatedAt should get retrieved from a stored kopia config. + r := Repository{ + Version: "v1", + Provider: provider, + Account: acct, + Config: cfg, + } + return r, nil +} diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go new file mode 100644 index 000000000..c54cc03b8 --- /dev/null +++ b/src/pkg/repository/repository_test.go @@ -0,0 +1,38 @@ +package repository_test + +import ( + "context" + "testing" + + "github.com/kopia/kopia/repo/blob" + + "github.com/alcionai/corso/pkg/repository" +) + +type testConfig struct{} + +func (tc testConfig) KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) { + return nil, nil +} + +func TestInitialize(t *testing.T) { + _, err := repository.Initialize( + context.Background(), + repository.ProviderUnknown, + repository.Account{}, + testConfig{}) + if err != nil { + t.Fatalf("didn't expect initialize to error, got [%v]", err) + } +} + +func TestConnect(t *testing.T) { + _, err := repository.Connect( + context.Background(), + repository.ProviderUnknown, + repository.Account{}, + testConfig{}) + if err != nil { + t.Fatalf("didn't expect connect to error, got [%v]", err) + } +} diff --git a/src/pkg/repository/s3/s3.go b/src/pkg/repository/s3/s3.go new file mode 100644 index 000000000..73438c202 --- /dev/null +++ b/src/pkg/repository/s3/s3.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "context" + + "github.com/kopia/kopia/repo/blob" + kopiaS3 "github.com/kopia/kopia/repo/blob/s3" +) + +// Config defines communication with a s3 repository provider. +type Config struct { + Bucket string // the S3 storage bucket name + AccessKey string // access key to the S3 bucket + SecretAccessKey string // s3 access key secret +} + +// NewConfig generates a S3 configuration struct to use for interfacing with a s3 storage +// bucket using a repository.Repository. +func NewConfig(bucket, accessKey, secretKey string) Config { + return Config{ + Bucket: bucket, + AccessKey: accessKey, + SecretAccessKey: secretKey, + } +} + +// KopiaStorage produces a kopia/blob Storage handle for connecting to s3. +func (c Config) KopiaStorage(ctx context.Context, create bool) (blob.Storage, error) { + opts := kopiaS3.Options{ + BucketName: c.Bucket, + AccessKeyID: c.AccessKey, + SecretAccessKey: c.SecretAccessKey, + } + return kopiaS3.New(ctx, &opts) +} diff --git a/src/pkg/repository/s3/s3_test.go b/src/pkg/repository/s3/s3_test.go new file mode 100644 index 000000000..f9ed90486 --- /dev/null +++ b/src/pkg/repository/s3/s3_test.go @@ -0,0 +1,14 @@ +package s3_test + +import ( + "testing" + + "github.com/alcionai/corso/pkg/repository/s3" +) + +func TestNewS3(t *testing.T) { + cfg := s3.NewConfig("bucket", "access", "secret") + if cfg.Bucket != "bucket" { + t.Errorf("expected s3 config bucke to be 'bucket', got '%s'", cfg.Bucket) + } +}