Support Azure Repositories

src/cli/utils/utils.go

@@ -152,7 +152,7 @@ func SendStartCorsoEvent(
 	repoID string,
 	opts control.Options,
 ) {
-	bus, err := events.NewBus(ctx, s, tenID, opts)
+	bus, err := events.NewBus(ctx, tenID, opts)
 	if err != nil {
 		logger.CtxErr(ctx, err).Info("sending start event")
 	}

src/go.mod

@@ -2,7 +2,7 @@ module github.com/alcionai/corso/src
 
 go 1.19
 
-replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79
+replace github.com/kopia/kopia => github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
@@ -38,6 +38,7 @@ require (
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect

src/go.sum

@@ -36,12 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
+github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
 github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -55,8 +59,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH
 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c h1:Njdw/Nnq2DN3f8QMaHuZZHdVHTUSxFqPMMxDIInDWB4=
 github.com/alcionai/clues v0.0.0-20230406223931-f48777f4773c/go.mod h1:DeaMbAwDvYM6ZfPMR/GUl3hceqI5C8jIQ1lstjB2IW8=
-github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79 h1:Wrl99Y7jftZMnNDiOIcRJrjstZO3IEj3+Q/sip27vmI=
-github.com/alcionai/kopia v0.12.2-0.20230502235504-2509b1d72a79/go.mod h1:Iic7CcKhsq+A7MLR9hh6VJfgpcJhLx3Kn+BgjY+azvI=
+github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6 h1:EW+ssK0qU623TxMyOI0rWnU3IoDoSILlkSIT6081jdo=
+github.com/alcionai/kopia v0.12.2-0.20230609054250-707b0ac13fa6/go.mod h1:Z2lXGiWHb5vM9NQNAXcOQHEpUldWONMQGfN4zLZ8Y9A=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -261,6 +265,7 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=

src/internal/events/events.go

@@ -17,7 +17,6 @@ import (
 	"github.com/alcionai/corso/src/internal/version"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/logger"
-	"github.com/alcionai/corso/src/pkg/storage"
 )
 
 // keys for ease of use
@@ -80,7 +79,7 @@ var (
 	RudderStackDataPlaneURL string
 )
 
-func NewBus(ctx context.Context, s storage.Storage, tenID string, opts control.Options) (Bus, error) {
+func NewBus(ctx context.Context, tenID string, opts control.Options) (Bus, error) {
 	if opts.DisableMetrics {
 		return Bus{}, nil
 	}

src/internal/events/events_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/credentials"
-	"github.com/alcionai/corso/src/pkg/storage"
 )
 
 type EventsIntegrationSuite struct {
@@ -31,15 +30,6 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
-	s, err := storage.NewStorage(
-		storage.ProviderS3,
-		storage.S3Config{
-			Bucket: "bckt",
-			Prefix: "prfx",
-		},
-	)
-	require.NoError(t, err, clues.ToCore(err))
-
 	a, err := account.NewAccount(
 		account.ProviderM365,
 		account.M365Config{
@@ -52,14 +42,14 @@ func (suite *EventsIntegrationSuite) TestNewBus() {
 	)
 	require.NoError(t, err, clues.ToCore(err))
 
-	b, err := events.NewBus(ctx, s, a.ID(), control.Defaults())
+	b, err := events.NewBus(ctx, a.ID(), control.Defaults())
 	require.NotEmpty(t, b)
 	require.NoError(t, err, clues.ToCore(err))
 
 	err = b.Close()
 	require.NoError(t, err, clues.ToCore(err))
 
-	b2, err := events.NewBus(ctx, s, a.ID(), control.Options{DisableMetrics: true})
+	b2, err := events.NewBus(ctx, a.ID(), control.Options{DisableMetrics: true})
 	require.Empty(t, b2)
 	require.NoError(t, err, clues.ToCore(err))
 

src/internal/kopia/azure.go

@@ -0,0 +1,30 @@
+package kopia
+
+import (
+	"context"
+
+	"github.com/alcionai/clues"
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/blob/azure"
+
+	"github.com/alcionai/corso/src/pkg/storage"
+)
+
+func azBlobStorage(ctx context.Context, s storage.Storage) (blob.Storage, error) {
+	cfg, err := s.AzureConfig()
+	if err != nil {
+		return nil, clues.Stack(err).WithClues(ctx)
+	}
+
+	opts := azure.Options{
+		Container: cfg.Container,
+		Prefix:    cfg.Prefix,
+	}
+
+	store, err := azure.New(ctx, &opts, false)
+	if err != nil {
+		return nil, clues.Stack(err).WithClues(ctx)
+	}
+
+	return store, nil
+}

src/internal/kopia/conn.go

@@ -178,6 +178,8 @@ func blobStoreByProvider(ctx context.Context, s storage.Storage) (blob.Storage,
 	switch s.Provider {
 	case storage.ProviderS3:
 		return s3BlobStorage(ctx, s)
+	case storage.ProviderAzure:
+		return azBlobStorage(ctx, s)
 	default:
 		return nil, clues.New("storage provider details are required").WithClues(ctx)
 	}

src/pkg/filters/filters.go

@@ -13,7 +13,6 @@ import (
 
 type comparator string
 
-//go:generate stringer -type=comparator -linecomment
 const (
 	UnknownComparator comparator = ""
 	// norm(a) == norm(b)

src/pkg/repository/repository.go

@@ -147,7 +147,7 @@ func Initialize(
 		return nil, clues.Stack(err).WithClues(ctx)
 	}
 
-	bus, err := events.NewBus(ctx, s, acct.ID(), opts)
+	bus, err := events.NewBus(ctx, acct.ID(), opts)
 	if err != nil {
 		return nil, clues.Wrap(err, "constructing event bus")
 	}
@@ -225,7 +225,7 @@ func Connect(
 		return nil, clues.Stack(err).WithClues(ctx)
 	}
 
-	bus, err := events.NewBus(ctx, s, acct.ID(), opts)
+	bus, err := events.NewBus(ctx, acct.ID(), opts)
 	if err != nil {
 		return nil, clues.Wrap(err, "constructing event bus")
 	}

src/pkg/storage/azure.go

@@ -0,0 +1,70 @@
+package storage
+
+import (
+	"github.com/alcionai/clues"
+
+	"github.com/alcionai/corso/src/internal/common"
+)
+
+type AzureConfig struct {
+	Container      string // required
+	Prefix         string
+	StorageAccount string
+	StorageKey     string
+}
+
+// config key consts
+const (
+	keyAzContainer = "az_container"
+	keyAzPrefix    = "az_prefix"
+)
+
+// config exported name consts
+const (
+	Container = "container"
+)
+
+func (c AzureConfig) Normalize() AzureConfig {
+	return AzureConfig{
+		Container: c.Container,
+		Prefix:    common.NormalizePrefix(c.Prefix),
+	}
+}
+
+// StringConfig transforms a s3Config struct into a plain
+// map[string]string.  All values in the original struct which
+// serialize into the map are expected to be strings.
+func (c AzureConfig) StringConfig() (map[string]string, error) {
+	cn := c.Normalize()
+	cfg := map[string]string{
+		keyAzContainer: cn.Container,
+		keyAzPrefix:    cn.Prefix,
+	}
+
+	return cfg, c.validate()
+}
+
+// S3Config retrieves the S3Config details from the Storage config.
+func (s Storage) AzureConfig() (AzureConfig, error) {
+	c := AzureConfig{}
+
+	if len(s.Config) > 0 {
+		c.Container = orEmptyString(s.Config[keyAzContainer])
+		c.Prefix = orEmptyString(s.Config[keyAzPrefix])
+	}
+
+	return c, c.validate()
+}
+
+func (c AzureConfig) validate() error {
+	check := map[string]string{
+		Container: c.Container,
+	}
+	for k, v := range check {
+		if len(v) == 0 {
+			return clues.Stack(errMissingRequired, clues.New(k))
+		}
+	}
+
+	return nil
+}

src/pkg/storage/azure_test.go

@@ -0,0 +1,93 @@
+package storage
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAzureConfig_Normalize(t *testing.T) {
+	type fields struct {
+		Container string
+		Prefix    string
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   AzureConfig
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := AzureConfig{
+				Container: tt.fields.Container,
+				Prefix:    tt.fields.Prefix,
+			}
+			if got := c.Normalize(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("AzureConfig.Normalize() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAzureConfig_StringConfig(t *testing.T) {
+	type fields struct {
+		Container string
+		Prefix    string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    map[string]string
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := AzureConfig{
+				Container: tt.fields.Container,
+				Prefix:    tt.fields.Prefix,
+			}
+			got, err := c.StringConfig()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("AzureConfig.StringConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("AzureConfig.StringConfig() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestStorage_AzureConfig(t *testing.T) {
+	type fields struct {
+		Provider storageProvider
+		Config   map[string]string
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    AzureConfig
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Storage{
+				Provider: tt.fields.Provider,
+				Config:   tt.fields.Config,
+			}
+			got, err := s.AzureConfig()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Storage.AzureConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Storage.AzureConfig() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/pkg/storage/storage.go

@@ -14,6 +14,7 @@ type storageProvider int
 const (
 	ProviderUnknown storageProvider = iota // Unknown Provider
 	ProviderS3                             // S3
+	ProviderAzure                          // Azure
 )
 
 // storage parsing errors

src/pkg/storage/storageprovider_string.go

@@ -10,11 +10,12 @@ func _() {
 	var x [1]struct{}
 	_ = x[ProviderUnknown-0]
 	_ = x[ProviderS3-1]
+	_ = x[ProviderAzure-2]
 }
 
-const _storageProvider_name = "Unknown ProviderS3"
+const _storageProvider_name = "Unknown ProviderS3Azure"
 
-var _storageProvider_index = [...]uint8{0, 16, 18}
+var _storageProvider_index = [...]uint8{0, 16, 18, 23}
 
 func (i storageProvider) String() string {
 	if i < 0 || i >= storageProvider(len(_storageProvider_index)-1) {