larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Azure Repositories

Browse Source
This commit is contained in:
Vaibhav Kamra 2023-05-31 22:36:57 -07:00
parent 2da554f00e
commit c4d94b13ab
13 changed files with 214 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/cli/utils/utils.go
View File

@ -152,7 +152,7 @@ func SendStartCorsoEvent(
repoID string,
opts control.Options,
) {
bus, err := events.NewBus(ctx, s, tenID, opts)
bus, err := events.NewBus(ctx, tenID, opts)
if err != nil {
logger.CtxErr(ctx, err).Info("sending start event")
}

3
src/go.mod
View File

@ -2,7 +2,7 @@ module github.com/alcionai/corso/src
go 1.19
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79
replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6
require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
@ -38,6 +38,7 @@ require (
)
require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/andybalholm/brotli v1.0.4 // indirect

9
src/go.sum
View File

@ -36,12 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -55,8 +59,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c h1:Njdw/Nnq2DN3f8QMaHuZZHdVHTUSxFqPMMxDIInDWB4=
github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8=
github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 h1:Wrl99Y7jftZMnNDiOIcRJrjstZO3IEj3+Q/sip27vmI=
github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI=
github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6 h1:EW+ssK0qU623TxMyOI0rWnU3IoDoSILlkSIT6081jdo=
github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6/go.mod h1:Z2lXGiWHb5vM9NQNAXcOQHEpUldWONMQGfN4zLZ8Y9A=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@ -261,6 +265,7 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=

3
src/internal/events/events.go
View File

@ -17,7 +17,6 @@ import (
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/storage"
)
// keys for ease of use
@ -80,7 +79,7 @@ var (
RudderStackDataPlaneURL string
)
func NewBus(ctx context.Context, s storage.Storage, tenID string, opts control.Options) (Bus, error) {
func NewBus(ctx context.Context, tenID string, opts control.Options) (Bus, error) {
if opts.DisableMetrics {
return Bus{}, nil
}

14
src/internal/events/events_test.go
View File

@ -12,7 +12,6 @@ import (
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
)
type EventsIntegrationSuite struct {
@ -31,15 +30,6 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
ctx, flush := tester.NewContext(t)
defer flush()
s, err := storage.NewStorage(
storage.ProviderS3,
storage.S3Config{
Bucket: "bckt",
Prefix: "prfx",
},
)
require.NoError(t, err, clues.ToCore(err))
a, err := account.NewAccount(
account.ProviderM365,
account.M365Config{
@ -52,14 +42,14 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
)
require.NoError(t, err, clues.ToCore(err))
b, err := events.NewBus(ctx, s, a.ID(), control.Defaults())
b, err := events.NewBus(ctx, a.ID(), control.Defaults())
require.NotEmpty(t, b)
require.NoError(t, err, clues.ToCore(err))
err = b.Close()
require.NoError(t, err, clues.ToCore(err))
b2, err := events.NewBus(ctx, s, a.ID(), control.Options{DisableMetrics: true})
b2, err := events.NewBus(ctx, a.ID(), control.Options{DisableMetrics: true})
require.Empty(t, b2)
require.NoError(t, err, clues.ToCore(err))

30
src/internal/kopia/azure.go Normal file
View File

@ -0,0 +1,30 @@
package kopia
import (
"context"
"github.com/alcionai/clues"
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/azure"
"github.com/alcionai/corso/src/pkg/storage"
)
func azBlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) {
cfg, err := s.AzureConfig()
if err != nil {
return nil, clues.Stack(err).WithClues(ctx)
}
opts := azure.Options{
Container: cfg.Container,
Prefix: cfg.Prefix,
}
store, err := azure.New(ctx, &opts, false)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx)
}
return store, nil
}

2
src/internal/kopia/conn.go
View File

@ -178,6 +178,8 @@ func blobStoreByProvider(ctx context.Context, s storage.Storage) (blob.Storage,
switch s.Provider {
case storage.ProviderS3:
return s3BlobStorage(ctx, s)
case storage.ProviderAzure:
return azBlobStorage(ctx, s)
default:
return nil, clues.New("storage provider details are required").WithClues(ctx)
}

1
src/pkg/filters/filters.go
View File

@ -13,7 +13,6 @@ import (
type comparator string
//go:generate stringer -type=comparator -linecomment
const (
UnknownComparator comparator = ""
// norm(a) == norm(b)

4
src/pkg/repository/repository.go
View File

@ -147,7 +147,7 @@ func Initialize(
return nil, clues.Stack(err).WithClues(ctx)
}
bus, err := events.NewBus(ctx, s, acct.ID(), opts)
bus, err := events.NewBus(ctx, acct.ID(), opts)
if err != nil {
return nil, clues.Wrap(err, "constructing event bus")
}
@ -225,7 +225,7 @@ func Connect(
return nil, clues.Stack(err).WithClues(ctx)
}
bus, err := events.NewBus(ctx, s, acct.ID(), opts)
bus, err := events.NewBus(ctx, acct.ID(), opts)
if err != nil {
return nil, clues.Wrap(err, "constructing event bus")
}

70
src/pkg/storage/azure.go Normal file
View File

@ -0,0 +1,70 @@
package storage
import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common"
)
type AzureConfig struct {
Container string // required
Prefix string
StorageAccount string
StorageKey string
}
// config key consts
const (
keyAzContainer = "az_container"
keyAzPrefix = "az_prefix"
)
// config exported name consts
const (
Container = "container"
)
func (c AzureConfig) Normalize() AzureConfig {
return AzureConfig{
Container: c.Container,
Prefix: common.NormalizePrefix(c.Prefix),
}
}
// StringConfig transforms a s3Config struct into a plain
// map[string]string. All values in the original struct which
// serialize into the map are expected to be strings.
func (c AzureConfig) StringConfig() (map[string]string, error) {
cn := c.Normalize()
cfg := map[string]string{
keyAzContainer: cn.Container,
keyAzPrefix: cn.Prefix,
}
return cfg, c.validate()
}
// S3Config retrieves the S3Config details from the Storage config.
func (s Storage) AzureConfig() (AzureConfig, error) {
c := AzureConfig{}
if len(s.Config) > 0 {
c.Container = orEmptyString(s.Config[keyAzContainer])
c.Prefix = orEmptyString(s.Config[keyAzPrefix])
}
return c, c.validate()
}
func (c AzureConfig) validate() error {
check := map[string]string{
Container: c.Container,
}
for k, v := range check {
if len(v) == 0 {
return clues.Stack(errMissingRequired, clues.New(k))
}
}
return nil
}

93
src/pkg/storage/azure_test.go Normal file
View File

@ -0,0 +1,93 @@
package storage
import (
"reflect"
"testing"
)
func TestAzureConfig_Normalize(t *testing.T) {
type fields struct {
Container string
Prefix string
}
tests := []struct {
name string
fields fields
want AzureConfig
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := AzureConfig{
Container: tt.fields.Container,
Prefix: tt.fields.Prefix,
}
if got := c.Normalize(); !reflect.DeepEqual(got, tt.want) {
t.Errorf("AzureConfig.Normalize() = %v, want %v", got, tt.want)
}
})
}
}
func TestAzureConfig_StringConfig(t *testing.T) {
type fields struct {
Container string
Prefix string
}
tests := []struct {
name string
fields fields
want map[string]string
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := AzureConfig{
Container: tt.fields.Container,
Prefix: tt.fields.Prefix,
}
got, err := c.StringConfig()
if (err != nil) != tt.wantErr {
t.Errorf("AzureConfig.StringConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("AzureConfig.StringConfig() = %v, want %v", got, tt.want)
}
})
}
}
func TestStorage_AzureConfig(t *testing.T) {
type fields struct {
Provider storageProvider
Config map[string]string
}
tests := []struct {
name string
fields fields
want AzureConfig
wantErr bool
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Storage{
Provider: tt.fields.Provider,
Config: tt.fields.Config,
}
got, err := s.AzureConfig()
if (err != nil) != tt.wantErr {
t.Errorf("Storage.AzureConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Storage.AzureConfig() = %v, want %v", got, tt.want)
}
})
}
}

1
src/pkg/storage/storage.go
View File

@ -14,6 +14,7 @@ type storageProvider int
const (
ProviderUnknown storageProvider = iota // Unknown Provider
ProviderS3 // S3
ProviderAzure // Azure
)
// storage parsing errors

5
src/pkg/storage/storageprovider_string.go
View File

@ -10,11 +10,12 @@ func _() {
var x [1]struct{}
_ = x[ProviderUnknown-0]
_ = x[ProviderS3-1]
_ = x[ProviderAzure-2]
}
const _storageProvider_name = "Unknown ProviderS3"
const _storageProvider_name = "Unknown ProviderS3Azure"
var _storageProvider_index = [...]uint8{0, 16, 18}
var _storageProvider_index = [...]uint8{0, 16, 18, 23}
func (i storageProvider) String() string {
if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) {