generate emails in factory script (#1162)

## Description

Adds scripted production of mock emails for building out
large/rolling datasets for load testing.

## Type of change

- [x] 🌻 Feature
- [x] 🤖 Test

## Issue(s)

* #902

## Test Plan

- [x] 💪 Manual
This commit is contained in:
Keepers 2022-10-14 12:48:00 -06:00 committed by GitHub
parent 7facdf1cc3
commit f737f58a0f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 308 additions and 32 deletions

View File

@ -1,10 +1,18 @@
package main
import (
"time"
"github.com/google/uuid"
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
var (
@ -34,13 +42,72 @@ func addExchangeCommands(parent *cobra.Command) {
}
func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
Err(cmd.Context(), ErrNotYetImplemeted)
var (
ctx = cmd.Context()
service = path.ExchangeService
category = path.EmailCategory
)
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
// generate mocked emails
gc, tenantID, err := getGCAndVerifyUser(ctx, user)
if err != nil {
return Only(ctx, err)
}
items := make([]item, 0, count)
for i := 0; i < count; i++ {
var (
now = common.Now()
nowLegacy = common.FormatLegacyTime(time.Now())
id = uuid.NewString()
subject = "automated " + now[:16] + " - " + id[:8]
body = "automated mail generation for " + user + " at " + now + " - " + id
)
items = append(items, item{
name: id,
// TODO: allow flags that specify a different "from" user, rather than duplicating
data: mockconnector.GetMockMessageWith(
user, user, user,
subject, body,
nowLegacy, nowLegacy, nowLegacy, nowLegacy),
})
}
collections := []collection{{
pathElements: []string{destination},
category: category,
items: items,
}}
// TODO: fit the desination to the containers
dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
dest.ContainerName = destination
dataColls, err := buildCollections(
service,
tenantID, user,
dest,
collections,
)
if err != nil {
return Only(ctx, err)
}
Infof(ctx, "Generating %d emails in %s\n", count, destination)
sel := selectors.NewExchangeRestore().Selector
deets, err := gc.RestoreDataCollections(ctx, sel, dest, dataColls)
if err != nil {
return Only(ctx, err)
}
deets.PrintEntries(ctx)
return nil
}
@ -58,7 +125,6 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
}
func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
//nolint
Err(cmd.Context(), ErrNotYetImplemeted)
if utils.HasNoFlagsAndShownHelp(cmd) {

View File

@ -8,6 +8,15 @@ import (
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
var factoryCmd = &cobra.Command{
@ -29,10 +38,10 @@ var oneDriveCmd = &cobra.Command{
}
var (
count int
container string
tenant string
user string
count int
destination string
tenant string
user string
)
// TODO: ErrGenerating = errors.New("not all items were successfully generated")
@ -44,7 +53,10 @@ var ErrNotYetImplemeted = errors.New("not yet implemented")
// ------------------------------------------------------------------------------------------
func main() {
ctx := SetRootCmd(context.Background(), factoryCmd)
ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
ctx = SetRootCmd(ctx, factoryCmd)
defer logger.Flush(ctx)
// persistent flags that are common to all use cases
fs := factoryCmd.PersistentFlags()
@ -53,8 +65,8 @@ func main() {
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("user"))
fs.IntVar(&count, "count", 0, "count of items to produce")
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("count"))
fs.StringVar(&container, "container", "", "container location of the new data (will create as needed)")
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("container"))
fs.StringVar(&destination, "destination", "", "destination of the new data (will create as needed)")
cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("destination"))
factoryCmd.AddCommand(exchangeCmd)
addExchangeCommands(exchangeCmd)
@ -80,3 +92,102 @@ func handleOneDriveFactory(cmd *cobra.Command, args []string) error {
Err(cmd.Context(), ErrNotYetImplemeted)
return cmd.Help()
}
// ------------------------------------------------------------------------------------------
// Common Helpers
// ------------------------------------------------------------------------------------------
func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, string, error) {
tid := common.First(tenant, os.Getenv(account.TenantID))
// get account info
m365Cfg := account.M365Config{
M365: credentials.GetM365(),
TenantID: tid,
}
acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
if err != nil {
return nil, "", errors.Wrap(err, "finding m365 account details")
}
// build a graph connector
gc, err := connector.NewGraphConnector(ctx, acct)
if err != nil {
return nil, "", errors.Wrap(err, "connecting to graph api")
}
if _, ok := gc.Users[user]; !ok {
return nil, "", errors.New("user not found within tenant")
}
return gc, tid, nil
}
type item struct {
name string
data []byte
}
type collection struct {
// Elements (in order) for the path representing this collection. Should
// only contain elements after the prefix that corso uses for the path. For
// example, a collection for the Inbox folder in exchange mail would just be
// "Inbox".
pathElements []string
category path.CategoryType
items []item
}
func buildCollections(
service path.ServiceType,
tenant, user string,
dest control.RestoreDestination,
colls []collection,
) ([]data.Collection, error) {
collections := make([]data.Collection, 0, len(colls))
for _, c := range colls {
pth, err := toDataLayerPath(
service,
tenant,
user,
c.category,
c.pathElements,
false,
)
if err != nil {
return nil, err
}
mc := mockconnector.NewMockExchangeCollection(pth, len(c.items))
for i := 0; i < len(c.items); i++ {
mc.Names[i] = c.items[i].name
mc.Data[i] = c.items[i].data
}
collections = append(collections, mc)
}
return collections, nil
}
func toDataLayerPath(
service path.ServiceType,
tenant, user string,
category path.CategoryType,
elements []string,
isItem bool,
) (path.Path, error) {
pb := path.Builder{}.Append(elements...)
switch service {
case path.ExchangeService:
return pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem)
case path.OneDriveService:
return pb.ToDataLayerOneDrivePath(tenant, user, isItem)
}
return nil, errors.Errorf("unknown service %s", service.String())
}

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/logger"
)
var purgeCmd = &cobra.Command{
@ -64,7 +65,11 @@ var ErrPurging = errors.New("not all items were successfully purged")
// ------------------------------------------------------------------------------------------
func main() {
ctx := SetRootCmd(context.Background(), purgeCmd)
ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
ctx = SetRootCmd(ctx, purgeCmd)
defer logger.Flush(ctx)
fs := purgeCmd.PersistentFlags()
fs.StringVar(&before, "before", "", "folders older than this date are deleted. (default: now in UTC)")
fs.StringVar(&user, "user", "", "m365 user id whose folders will be deleted")

View File

@ -182,8 +182,7 @@ func RestoreMailMessage(
bits []byte,
service graph.Service,
cp control.CollisionPolicy,
destination,
user string,
destination, user string,
) (*details.ExchangeInfo, error) {
// Creates messageable object from original bytes
originalMessage, err := support.CreateMessageFromBytes(bits)

View File

@ -22,22 +22,73 @@ const (
"\\nThanking you in adv"
// Order of fields to fill in:
// 1. message body
// 2. message preview
// 3. sender user ID
// 4. subject
messageTmpl = "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," +
"\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"<html><head>" +
"\\n<meta http-equiv=\\\"Content-Type\\\" content=\\\"text/html; charset=utf-8\\\"><style type=\\\"text/css\\\" style=\\\"display:none\\\">\\n<!--\\np\\n{margin-top:0;\\nmargin-bottom:0}\\n-->" +
"\\n</style></head><body dir=\\\"ltr\\\"><div class=\\\"elementToProof\\\" style=\\\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0)\\\">%s" +
"</div></body></html>\",\"contentType\":\"html\"}," +
"\"bodyPreview\":\"%s\"," +
"\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," +
"\"from\":{\"emailAddress\":{\"address\":\"%s\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"<SJ0PR17MB562266A1E61A8EA12F5FB17BC3529@SJ0PR17MB5622.namprd17.prod.outlook.com>\"," +
"\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," +
"\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," +
"\"subject\":\"%s\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," +
"\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem\"}"
// 1. created datetime
// 2. modified datetime
// 3. message body
// 4. message preview
// 5. sender user ID
// 6. received datetime
// 7. sender email
// 8. sent datetime
// 9. subject
// 10. recipient user addr
messageTmpl = `{
"id":"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=",
"@odata.context":"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity",
"@odata.etag":"W/\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\"",
"categories":[],
"changeKey":"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU",
"createdDateTime":"%s",
"lastModifiedDateTime":"%s",
"bccRecipients":[],
"body":{
"content":"<html><head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"><style type=\"text/css\" style=\"display:none\">\n<!--\np\n{margin-top:0;\nmargin-bottom:0}\n-->` +
`\n</style></head><body dir=\"ltr\"><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0)\">` +
`%s` +
`</div></body></html>",
"contentType":"html"
},
"bodyPreview":"%s",
"ccRecipients":[],
"conversationId":"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=",
"conversationIndex":"AQHY0f3Ermc1ZF01akakuntelv+bsA==",
"flag":{
"flagStatus":"notFlagged"},
"from":{
"emailAddress":{
"address":"%s",
"name":"A Stranger"
}
},
"hasAttachments":false,
"importance":"normal",
"inferenceClassification":"focused",
"internetMessageId":"<SJ0PR17MB562266A1E61A8EA12F5FB17BC3529@SJ0PR17MB5622.namprd17.prod.outlook.com>",
"isDeliveryReceiptRequested":false,
"isDraft":false,
"isRead":false,
"isReadReceiptRequested":false,
"parentFolderId":"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=",
"receivedDateTime":"%s",
"replyTo":[],
"sender":{
"emailAddress":{
"address":"%s",
"name":"A Stranger"
}
},
"sentDateTime":"%s",
"subject":"%s",
"toRecipients":[
{
"emailAddress":{
"address":"%s",
"name":"A Stranger"
}
}
],
"webLink":"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem"
}`
// Order of fields to fill in:
// 1. start/end date
@ -62,7 +113,7 @@ const (
"\"type\":\"singleInstance\",\"webLink\":\"https://outlook.office365.com/owa/?itemid=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAADSEBNbUIB9RL6ePDeF3FIYAAAAAG76AAA%%3D&exvsurl=1&path=/calendar/item\"}"
)
// GetMockMessageBytes returns bytes for Messageable item.
// GetMockMessageBytes returns bytes for a Messageable item.
// Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
func GetMockMessageBytes(subject string) []byte {
userID := "foobar@8qzvrj.onmicrosoft.com"
@ -70,15 +121,23 @@ func GetMockMessageBytes(subject string) []byte {
message := fmt.Sprintf(
messageTmpl,
"2022-09-26T23:15:50Z", // created
"2022-09-26T23:15:51Z", // modified
defaultMessageBody,
defaultMessagePreview,
userID,
"2022-09-26T23:15:50Z",
"foobar@8qzvrj.onmicrosoft.com",
"2022-09-26T23:15:46Z",
"TPS Report "+subject+timestamp,
)
"LidiaH@8qzvrj.onmicrosoft.com")
return []byte(message)
}
// GetMockMessageBytes returns bytes for a Messageable item.
// Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
// Body must contain a well-formatted string, consumable in a json payload. IE: no unescaped newlines.
func GetMockMessageWithBodyBytes(subject, body string) []byte {
userID := "foobar@8qzvrj.onmicrosoft.com"
preview := body
@ -89,11 +148,47 @@ func GetMockMessageWithBodyBytes(subject, body string) []byte {
message := fmt.Sprintf(
messageTmpl,
"2022-09-26T23:15:50Z", // created
"2022-09-26T23:15:51Z", // modified
body,
preview,
userID,
"2022-09-26T23:15:50Z",
"foobar@8qzvrj.onmicrosoft.com",
"2022-09-26T23:15:46Z",
subject,
)
"LidiaH@8qzvrj.onmicrosoft.com")
return []byte(message)
}
// GetMockMessageBytes returns bytes for a Messageable item.
// Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
// created, modified, sent, and received should be in the format 2006-01-02T15:04:05Z
// Body must contain a well-formatted string, consumable in a json payload. IE: no unescaped newlines.
func GetMockMessageWith(
to, from, sender, // user PNs
subject, body, // arbitrary data
created, modified, sent, received string, // legacy datetimes
) []byte {
preview := body
if len(preview) > 255 {
preview = preview[:256]
}
message := fmt.Sprintf(
messageTmpl,
created,
modified,
body,
preview,
from,
received,
sender,
sent,
subject,
to)
return []byte(message)
}