add the metrics package (#839)
## Type of change - [x] 🌻 Feature ## Issue(s) * #741 ## Test Plan - [ ] 💪 Manual - [ ] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
Keepers
GitHub
parent
4cf5fcab5a
commit
0c8a03a826
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
@ -96,6 +96,8 @@ jobs:
|
||||
CORSO_CI_TESTS: true
|
||||
CORSO_M356_TEST_USER_ID: ${{ secrets.CORSO_M356_TEST_USER_ID }}
|
||||
CORSO_PASSPHRASE: ${{ secrets.INTEGRATION_TEST_CORSO_PASSPHRASE }}
|
||||
RUDDERSTACK_CORSO_WRITE_KEY: ${{ secrets.RUDDERSTACK_CORSO_WRITE_KEY }}
|
||||
RUDDERSTACK_CORSO_DATA_PLANE_URL: ${{ secrets.RUDDERSTACK_CORSO_DATA_PLANE_URL }}
|
||||
TENANT_ID: ${{ secrets.TENANT_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@ -10,6 +10,7 @@ linters:
|
||||
- misspell
|
||||
- revive
|
||||
- wsl
|
||||
- errcheck
|
||||
|
||||
linters-settings:
|
||||
gci:
|
||||
|
||||
@ -17,6 +17,7 @@ require (
|
||||
github.com/microsoftgraph/msgraph-sdk-go v0.34.0
|
||||
github.com/microsoftgraph/msgraph-sdk-go-core v0.27.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rudderlabs/analytics-go v3.3.3+incompatible
|
||||
github.com/spf13/cobra v1.4.0
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.12.0
|
||||
@ -46,6 +47,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
|
||||
github.com/cjlapao/common-go v0.0.25 // indirect
|
||||
@ -87,7 +89,11 @@ require (
|
||||
github.com/prometheus/procfs v0.8.0 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/rs/xid v1.4.0 // indirect
|
||||
github.com/segmentio/backo-go v1.0.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/tidwall/gjson v1.14.3 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
go.opentelemetry.io/otel v1.9.0 // indirect
|
||||
|
||||
12
src/go.sum
12
src/go.sum
@ -60,6 +60,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
|
||||
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
|
||||
@ -334,7 +336,11 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
||||
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
|
||||
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY=
|
||||
github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA=
|
||||
github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
@ -365,10 +371,16 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI=
|
||||
github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
|
||||
github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
|
||||
github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tomlazar/table v0.1.2 h1:DP8f62FzZAZk8oavepm1v/oyf4ni3/LMHWNlOinmleg=
|
||||
github.com/tomlazar/table v0.1.2/go.mod h1:IecZnpep9f/BatHacfh+++ftE+lFONN8BVPi9nx5U1w=
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g=
|
||||
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
|
||||
110
src/internal/events/events.go
Normal file
110
src/internal/events/events.go
Normal file
@ -0,0 +1,110 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
analytics "github.com/rudderlabs/analytics-go"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
)
|
||||
|
||||
// keys for ease of use
|
||||
const (
|
||||
corsoVersion = "corso-version"
|
||||
repoID = "repo-id"
|
||||
payload = "payload"
|
||||
|
||||
// Event Keys
|
||||
RepoInit = "repo-init"
|
||||
BackupStart = "backup-start"
|
||||
BackupEnd = "backup-end"
|
||||
|
||||
// Event Data Keys
|
||||
BackupID = "backup-id"
|
||||
ExchangeResources = "exchange-resources"
|
||||
ExchangeDataRetrieved = "exchange-data-retrieved"
|
||||
ExchangeDataStored = "exchange-data-stored"
|
||||
EndTime = "end-time"
|
||||
StartTime = "start-time"
|
||||
Duration = "duration"
|
||||
Status = "status"
|
||||
)
|
||||
|
||||
// Bus handles all event communication into the events package.
|
||||
type Bus struct {
|
||||
client analytics.Client
|
||||
|
||||
repoID string // one-way hash that uniquely identifies the repo.
|
||||
version string // the Corso release version
|
||||
}
|
||||
|
||||
var (
|
||||
WriteKey string
|
||||
DataPlaneURL string
|
||||
)
|
||||
|
||||
func NewBus(repoProvider, bucket, prefix, tenantID string) Bus {
|
||||
hash := repoHash(repoProvider, bucket, prefix, tenantID)
|
||||
|
||||
envWK := os.Getenv("RUDDERSTACK_CORSO_WRITE_KEY")
|
||||
if len(envWK) > 0 {
|
||||
WriteKey = envWK
|
||||
}
|
||||
|
||||
envDPU := os.Getenv("RUDDERSTACK_CORSO_DATA_PLANE_URL")
|
||||
if len(envDPU) > 0 {
|
||||
DataPlaneURL = envDPU
|
||||
}
|
||||
|
||||
var client analytics.Client
|
||||
if len(WriteKey) > 0 && len(DataPlaneURL) > 0 {
|
||||
client = analytics.New(WriteKey, DataPlaneURL)
|
||||
}
|
||||
|
||||
return Bus{
|
||||
client: client,
|
||||
repoID: hash,
|
||||
version: "vTODO", // TODO: corso versioning implementation
|
||||
}
|
||||
}
|
||||
|
||||
func (b Bus) Close() error {
|
||||
if b.client == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return b.client.Close()
|
||||
}
|
||||
|
||||
func (b Bus) Event(ctx context.Context, key string, data map[string]any) {
|
||||
if b.client == nil {
|
||||
return
|
||||
}
|
||||
|
||||
props := analytics.
|
||||
NewProperties().
|
||||
Set(repoID, b.repoID).
|
||||
Set(corsoVersion, b.version).
|
||||
Set(payload, data)
|
||||
|
||||
err := b.client.Enqueue(analytics.Track{
|
||||
Event: key,
|
||||
Timestamp: time.Now().UTC(),
|
||||
Properties: props,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Ctx(ctx).Debugw("analytics event failure", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func repoHash(repoProvider, bucket, prefix, tenantID string) string {
|
||||
sum := md5.Sum(
|
||||
[]byte(repoProvider + bucket + prefix + tenantID),
|
||||
)
|
||||
|
||||
return fmt.Sprintf("%x", sum)
|
||||
}
|
||||
32
src/internal/events/events_test.go
Normal file
32
src/internal/events/events_test.go
Normal file
@ -0,0 +1,32 @@
|
||||
package events_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
"github.com/alcionai/corso/src/internal/events"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
)
|
||||
|
||||
type EventsIntegrationSuite struct {
|
||||
suite.Suite
|
||||
}
|
||||
|
||||
func TestMetricsIntegrationSuite(t *testing.T) {
|
||||
if err := tester.RunOnAny(tester.CorsoCITests, "floob"); err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
|
||||
suite.Run(t, new(EventsIntegrationSuite))
|
||||
}
|
||||
|
||||
func (suite *EventsIntegrationSuite) TestNewBus() {
|
||||
t := suite.T()
|
||||
|
||||
b := events.NewBus("s3", "bckt", "prfx", "tenid")
|
||||
require.NotEmpty(t, b)
|
||||
|
||||
require.NoError(t, b.Close())
|
||||
}
|
||||
@ -27,6 +27,8 @@ type Repository struct {
|
||||
|
||||
Account account.Account // the user's m365 account connection details
|
||||
Storage storage.Storage // the storage provider details and configuration
|
||||
|
||||
// Bus events.Bus
|
||||
dataLayer *kopia.Wrapper
|
||||
modelStore *kopia.ModelStore
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user