diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 56a13ad1a..b70a50ef5 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -152,7 +152,7 @@ func SendStartCorsoEvent( repoID string, opts control.Options, ) { - bus, err := events.NewBus(ctx, s, tenID, opts) + bus, err := events.NewBus(ctx, tenID, opts) if err != nil { logger.CtxErr(ctx, err).Info("sending start event") } diff --git a/src/go.mod b/src/go.mod index 4edb223a0..ee1468cf2 100644 --- a/src/go.mod +++ b/src/go.mod @@ -2,7 +2,7 @@ module github.com/alcionai/corso/src go 1.19 -replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 +replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6 require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 @@ -38,6 +38,7 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/andybalholm/brotli v1.0.4 // indirect diff --git a/src/go.sum b/src/go.sum index 9423688ad..e2372e252 100644 --- a/src/go.sum +++ b/src/go.sum @@ -36,12 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -55,8 +59,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c h1:Njdw/Nnq2DN3f8QMaHuZZHdVHTUSxFqPMMxDIInDWB4= github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8= -github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 h1:Wrl99Y7jftZMnNDiOIcRJrjstZO3IEj3+Q/sip27vmI= -github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI= +github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6 h1:EW+ssK0qU623TxMyOI0rWnU3IoDoSILlkSIT6081jdo= +github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6/go.mod h1:Z2lXGiWHb5vM9NQNAXcOQHEpUldWONMQGfN4zLZ8Y9A= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -261,6 +265,7 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= diff --git a/src/internal/events/events.go b/src/internal/events/events.go index 7dc5cbf6f..61947a26d 100644 --- a/src/internal/events/events.go +++ b/src/internal/events/events.go @@ -17,7 +17,6 @@ import ( "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" - "github.com/alcionai/corso/src/pkg/storage" ) // keys for ease of use @@ -80,7 +79,7 @@ var ( RudderStackDataPlaneURL string ) -func NewBus(ctx context.Context, s storage.Storage, tenID string, opts control.Options) (Bus, error) { +func NewBus(ctx context.Context, tenID string, opts control.Options) (Bus, error) { if opts.DisableMetrics { return Bus{}, nil } diff --git a/src/internal/events/events_test.go b/src/internal/events/events_test.go index 7cc47f607..18b8b1cd3 100644 --- a/src/internal/events/events_test.go +++ b/src/internal/events/events_test.go @@ -12,7 +12,6 @@ import ( "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/credentials" - "github.com/alcionai/corso/src/pkg/storage" ) type EventsIntegrationSuite struct { @@ -31,15 +30,6 @@ func (suite *EventsIntegrationSuite) TestNewBus() { ctx, flush := tester.NewContext(t) defer flush() - s, err := storage.NewStorage( - storage.ProviderS3, - storage.S3Config{ - Bucket: "bckt", - Prefix: "prfx", - }, - ) - require.NoError(t, err, clues.ToCore(err)) - a, err := account.NewAccount( account.ProviderM365, account.M365Config{ @@ -52,14 +42,14 @@ func (suite *EventsIntegrationSuite) TestNewBus() { ) require.NoError(t, err, clues.ToCore(err)) - b, err := events.NewBus(ctx, s, a.ID(), control.Defaults()) + b, err := events.NewBus(ctx, a.ID(), control.Defaults()) require.NotEmpty(t, b) require.NoError(t, err, clues.ToCore(err)) err = b.Close() require.NoError(t, err, clues.ToCore(err)) - b2, err := events.NewBus(ctx, s, a.ID(), control.Options{DisableMetrics: true}) + b2, err := events.NewBus(ctx, a.ID(), control.Options{DisableMetrics: true}) require.Empty(t, b2) require.NoError(t, err, clues.ToCore(err)) diff --git a/src/internal/kopia/azure.go b/src/internal/kopia/azure.go new file mode 100644 index 000000000..8da700192 --- /dev/null +++ b/src/internal/kopia/azure.go @@ -0,0 +1,30 @@ +package kopia + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/blob" + "github.com/kopia/kopia/repo/blob/azure" + + "github.com/alcionai/corso/src/pkg/storage" +) + +func azBlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) { + cfg, err := s.AzureConfig() + if err != nil { + return nil, clues.Stack(err).WithClues(ctx) + } + + opts := azure.Options{ + Container: cfg.Container, + Prefix: cfg.Prefix, + } + + store, err := azure.New(ctx, &opts, false) + if err != nil { + return nil, clues.Stack(err).WithClues(ctx) + } + + return store, nil +} diff --git a/src/internal/kopia/conn.go b/src/internal/kopia/conn.go index 3447e810f..6e07f5197 100644 --- a/src/internal/kopia/conn.go +++ b/src/internal/kopia/conn.go @@ -178,6 +178,8 @@ func blobStoreByProvider(ctx context.Context, s storage.Storage) (blob.Storage, switch s.Provider { case storage.ProviderS3: return s3BlobStorage(ctx, s) + case storage.ProviderAzure: + return azBlobStorage(ctx, s) default: return nil, clues.New("storage provider details are required").WithClues(ctx) } diff --git a/src/pkg/filters/filters.go b/src/pkg/filters/filters.go index 638d578db..d8da7d7ce 100644 --- a/src/pkg/filters/filters.go +++ b/src/pkg/filters/filters.go @@ -13,7 +13,6 @@ import ( type comparator string -//go:generate stringer -type=comparator -linecomment const ( UnknownComparator comparator = "" // norm(a) == norm(b) diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 7f37f4eaf..643225e43 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -147,7 +147,7 @@ func Initialize( return nil, clues.Stack(err).WithClues(ctx) } - bus, err := events.NewBus(ctx, s, acct.ID(), opts) + bus, err := events.NewBus(ctx, acct.ID(), opts) if err != nil { return nil, clues.Wrap(err, "constructing event bus") } @@ -225,7 +225,7 @@ func Connect( return nil, clues.Stack(err).WithClues(ctx) } - bus, err := events.NewBus(ctx, s, acct.ID(), opts) + bus, err := events.NewBus(ctx, acct.ID(), opts) if err != nil { return nil, clues.Wrap(err, "constructing event bus") } diff --git a/src/pkg/storage/azure.go b/src/pkg/storage/azure.go new file mode 100644 index 000000000..37ddacca1 --- /dev/null +++ b/src/pkg/storage/azure.go @@ -0,0 +1,70 @@ +package storage + +import ( + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/common" +) + +type AzureConfig struct { + Container string // required + Prefix string + StorageAccount string + StorageKey string +} + +// config key consts +const ( + keyAzContainer = "az_container" + keyAzPrefix = "az_prefix" +) + +// config exported name consts +const ( + Container = "container" +) + +func (c AzureConfig) Normalize() AzureConfig { + return AzureConfig{ + Container: c.Container, + Prefix: common.NormalizePrefix(c.Prefix), + } +} + +// StringConfig transforms a s3Config struct into a plain +// map[string]string. All values in the original struct which +// serialize into the map are expected to be strings. +func (c AzureConfig) StringConfig() (map[string]string, error) { + cn := c.Normalize() + cfg := map[string]string{ + keyAzContainer: cn.Container, + keyAzPrefix: cn.Prefix, + } + + return cfg, c.validate() +} + +// S3Config retrieves the S3Config details from the Storage config. +func (s Storage) AzureConfig() (AzureConfig, error) { + c := AzureConfig{} + + if len(s.Config) > 0 { + c.Container = orEmptyString(s.Config[keyAzContainer]) + c.Prefix = orEmptyString(s.Config[keyAzPrefix]) + } + + return c, c.validate() +} + +func (c AzureConfig) validate() error { + check := map[string]string{ + Container: c.Container, + } + for k, v := range check { + if len(v) == 0 { + return clues.Stack(errMissingRequired, clues.New(k)) + } + } + + return nil +} diff --git a/src/pkg/storage/azure_test.go b/src/pkg/storage/azure_test.go new file mode 100644 index 000000000..c761574c5 --- /dev/null +++ b/src/pkg/storage/azure_test.go @@ -0,0 +1,93 @@ +package storage + +import ( + "reflect" + "testing" +) + +func TestAzureConfig_Normalize(t *testing.T) { + type fields struct { + Container string + Prefix string + } + tests := []struct { + name string + fields fields + want AzureConfig + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := AzureConfig{ + Container: tt.fields.Container, + Prefix: tt.fields.Prefix, + } + if got := c.Normalize(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("AzureConfig.Normalize() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAzureConfig_StringConfig(t *testing.T) { + type fields struct { + Container string + Prefix string + } + tests := []struct { + name string + fields fields + want map[string]string + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := AzureConfig{ + Container: tt.fields.Container, + Prefix: tt.fields.Prefix, + } + got, err := c.StringConfig() + if (err != nil) != tt.wantErr { + t.Errorf("AzureConfig.StringConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("AzureConfig.StringConfig() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestStorage_AzureConfig(t *testing.T) { + type fields struct { + Provider storageProvider + Config map[string]string + } + tests := []struct { + name string + fields fields + want AzureConfig + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Storage{ + Provider: tt.fields.Provider, + Config: tt.fields.Config, + } + got, err := s.AzureConfig() + if (err != nil) != tt.wantErr { + t.Errorf("Storage.AzureConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Storage.AzureConfig() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/src/pkg/storage/storage.go b/src/pkg/storage/storage.go index 19cc9ddc7..baac89827 100644 --- a/src/pkg/storage/storage.go +++ b/src/pkg/storage/storage.go @@ -14,6 +14,7 @@ type storageProvider int const ( ProviderUnknown storageProvider = iota // Unknown Provider ProviderS3 // S3 + ProviderAzure // Azure ) // storage parsing errors diff --git a/src/pkg/storage/storageprovider_string.go b/src/pkg/storage/storageprovider_string.go index 389dee37a..9cf35f815 100644 --- a/src/pkg/storage/storageprovider_string.go +++ b/src/pkg/storage/storageprovider_string.go @@ -10,11 +10,12 @@ func _() { var x [1]struct{} _ = x[ProviderUnknown-0] _ = x[ProviderS3-1] + _ = x[ProviderAzure-2] } -const _storageProvider_name = "Unknown ProviderS3" +const _storageProvider_name = "Unknown ProviderS3Azure" -var _storageProvider_index = [...]uint8{0, 16, 18} +var _storageProvider_index = [...]uint8{0, 16, 18, 23} func (i storageProvider) String() string { if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) {