From 0c8a03a8269236996e861dfba89436a0f60b0eb3 Mon Sep 17 00:00:00 2001 From: Keepers Date: Wed, 14 Sep 2022 18:28:02 -0600 Subject: [PATCH] add the metrics package (#839) ## Type of change - [x] :sunflower: Feature ## Issue(s) * #741 ## Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [x] :green_heart: E2E --- .github/workflows/ci.yml | 2 + src/.golangci.yml | 1 + src/go.mod | 6 ++ src/go.sum | 12 ++++ src/internal/events/events.go | 110 +++++++++++++++++++++++++++++ src/internal/events/events_test.go | 32 +++++++++ src/pkg/repository/repository.go | 6 +- 7 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 src/internal/events/events.go create mode 100644 src/internal/events/events_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daf35eada..98c9a8a60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,6 +96,8 @@ jobs: CORSO_CI_TESTS: true CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }} CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }} + RUDDERSTACK_CORSO_WRITE_KEY: ${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }} + RUDDERSTACK_CORSO_DATA_PLANE_URL: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }} TENANT_ID: ${{ secrets.TENANT_ID }} run: | set -euo pipefail diff --git a/src/.golangci.yml b/src/.golangci.yml index 62e9d9f0e..eaa0e359a 100644 --- a/src/.golangci.yml +++ b/src/.golangci.yml @@ -10,6 +10,7 @@ linters: - misspell - revive - wsl + - errcheck linters-settings: gci: diff --git a/src/go.mod b/src/go.mod index 6e3849d42..0452932e8 100644 --- a/src/go.mod +++ b/src/go.mod @@ -17,6 +17,7 @@ require ( github.com/microsoftgraph/msgraph-sdk-go v0.34.0 github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0 github.com/pkg/errors v0.9.1 + github.com/rudderlabs/analytics-go v3.3.3+incompatible github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 @@ -46,6 +47,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/cjlapao/common-go v0.0.25 // indirect @@ -87,7 +89,11 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/xid v1.4.0 // indirect + github.com/segmentio/backo-go v1.0.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/tidwall/gjson v1.14.3 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect go.opentelemetry.io/otel v1.9.0 // indirect diff --git a/src/go.sum b/src/go.sum index 49d2b2422..bd113ee1f 100644 --- a/src/go.sum +++ b/src/go.sum @@ -60,6 +60,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -334,7 +336,11 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= +github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= +github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -365,10 +371,16 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tomlazar/table v0.1.2 h1:DP8f62FzZAZk8oavepm1v/oyf4ni3/LMHWNlOinmleg= github.com/tomlazar/table v0.1.2/go.mod h1:IecZnpep9f/BatHacfh+++ftE+lFONN8BVPi9nx5U1w= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= +github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/src/internal/events/events.go b/src/internal/events/events.go new file mode 100644 index 000000000..f2b094f6e --- /dev/null +++ b/src/internal/events/events.go @@ -0,0 +1,110 @@ +package events + +import ( + "context" + "crypto/md5" + "fmt" + "os" + "time" + + analytics "github.com/rudderlabs/analytics-go" + + "github.com/alcionai/corso/src/pkg/logger" +) + +// keys for ease of use +const ( + corsoVersion = "corso-version" + repoID = "repo-id" + payload = "payload" + + // Event Keys + RepoInit = "repo-init" + BackupStart = "backup-start" + BackupEnd = "backup-end" + + // Event Data Keys + BackupID = "backup-id" + ExchangeResources = "exchange-resources" + ExchangeDataRetrieved = "exchange-data-retrieved" + ExchangeDataStored = "exchange-data-stored" + EndTime = "end-time" + StartTime = "start-time" + Duration = "duration" + Status = "status" +) + +// Bus handles all event communication into the events package. +type Bus struct { + client analytics.Client + + repoID string // one-way hash that uniquely identifies the repo. + version string // the Corso release version +} + +var ( + WriteKey string + DataPlaneURL string +) + +func NewBus(repoProvider, bucket, prefix, tenantID string) Bus { + hash := repoHash(repoProvider, bucket, prefix, tenantID) + + envWK := os.Getenv("RUDDERSTACK_CORSO_WRITE_KEY") + if len(envWK) > 0 { + WriteKey = envWK + } + + envDPU := os.Getenv("RUDDERSTACK_CORSO_DATA_PLANE_URL") + if len(envDPU) > 0 { + DataPlaneURL = envDPU + } + + var client analytics.Client + if len(WriteKey) > 0 && len(DataPlaneURL) > 0 { + client = analytics.New(WriteKey, DataPlaneURL) + } + + return Bus{ + client: client, + repoID: hash, + version: "vTODO", // TODO: corso versioning implementation + } +} + +func (b Bus) Close() error { + if b.client == nil { + return nil + } + + return b.client.Close() +} + +func (b Bus) Event(ctx context.Context, key string, data map[string]any) { + if b.client == nil { + return + } + + props := analytics. + NewProperties(). + Set(repoID, b.repoID). + Set(corsoVersion, b.version). + Set(payload, data) + + err := b.client.Enqueue(analytics.Track{ + Event: key, + Timestamp: time.Now().UTC(), + Properties: props, + }) + if err != nil { + logger.Ctx(ctx).Debugw("analytics event failure", "err", err) + } +} + +func repoHash(repoProvider, bucket, prefix, tenantID string) string { + sum := md5.Sum( + []byte(repoProvider + bucket + prefix + tenantID), + ) + + return fmt.Sprintf("%x", sum) +} diff --git a/src/internal/events/events_test.go b/src/internal/events/events_test.go new file mode 100644 index 000000000..8892283d6 --- /dev/null +++ b/src/internal/events/events_test.go @@ -0,0 +1,32 @@ +package events_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/events" + "github.com/alcionai/corso/src/internal/tester" +) + +type EventsIntegrationSuite struct { + suite.Suite +} + +func TestMetricsIntegrationSuite(t *testing.T) { + if err := tester.RunOnAny(tester.CorsoCITests, "floob"); err != nil { + t.Skip(err) + } + + suite.Run(t, new(EventsIntegrationSuite)) +} + +func (suite *EventsIntegrationSuite) TestNewBus() { + t := suite.T() + + b := events.NewBus("s3", "bckt", "prfx", "tenid") + require.NotEmpty(t, b) + + require.NoError(t, b.Close()) +} diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index c143b6e91..b5405bf28 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -25,8 +25,10 @@ type Repository struct { CreatedAt time.Time Version string // in case of future breaking changes - Account account.Account // the user's m365 account connection details - Storage storage.Storage // the storage provider details and configuration + Account account.Account // the user's m365 account connection details + Storage storage.Storage // the storage provider details and configuration + + // Bus events.Bus dataLayer *kopia.Wrapper modelStore *kopia.ModelStore }