generate emails in factory script (#1162)

## Description

Adds scripted production of mock emails for building out
large/rolling datasets for load testing.

## Type of change

- [x] 🌻 Feature
- [x] 🤖 Test

## Issue(s)

* #902

## Test Plan

- [x] 💪 Manual

src/cmd/factory/exchange.go

@@ -1,10 +1,18 @@
 package main
 
 import (
+	"time"
+
+	"github.com/google/uuid"
 	"github.com/spf13/cobra"
 
 	. "github.com/alcionai/corso/src/cli/print"
 	"github.com/alcionai/corso/src/cli/utils"
+	"github.com/alcionai/corso/src/internal/common"
+	"github.com/alcionai/corso/src/internal/connector/mockconnector"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/path"
+	"github.com/alcionai/corso/src/pkg/selectors"
 )
 
 var (
@@ -34,13 +42,72 @@ func addExchangeCommands(parent *cobra.Command) {
 }
 
 func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
-	Err(cmd.Context(), ErrNotYetImplemeted)
+	var (
+		ctx      = cmd.Context()
+		service  = path.ExchangeService
+		category = path.EmailCategory
+	)
 
 	if utils.HasNoFlagsAndShownHelp(cmd) {
 		return nil
 	}
 
-	// generate mocked emails
+	gc, tenantID, err := getGCAndVerifyUser(ctx, user)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	items := make([]item, 0, count)
+
+	for i := 0; i < count; i++ {
+		var (
+			now       = common.Now()
+			nowLegacy = common.FormatLegacyTime(time.Now())
+			id        = uuid.NewString()
+			subject   = "automated " + now[:16] + " - " + id[:8]
+			body      = "automated mail generation for " + user + " at " + now + " - " + id
+		)
+
+		items = append(items, item{
+			name: id,
+			// TODO: allow flags that specify a different "from" user, rather than duplicating
+			data: mockconnector.GetMockMessageWith(
+				user, user, user,
+				subject, body,
+				nowLegacy, nowLegacy, nowLegacy, nowLegacy),
+		})
+	}
+
+	collections := []collection{{
+		pathElements: []string{destination},
+		category:     category,
+		items:        items,
+	}}
+
+	// TODO: fit the desination to the containers
+	dest := control.DefaultRestoreDestination(common.SimpleTimeTesting)
+	dest.ContainerName = destination
+
+	dataColls, err := buildCollections(
+		service,
+		tenantID, user,
+		dest,
+		collections,
+	)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	Infof(ctx, "Generating %d emails in %s\n", count, destination)
+
+	sel := selectors.NewExchangeRestore().Selector
+
+	deets, err := gc.RestoreDataCollections(ctx, sel, dest, dataColls)
+	if err != nil {
+		return Only(ctx, err)
+	}
+
+	deets.PrintEntries(ctx)
 
 	return nil
 }
@@ -58,7 +125,6 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
 }
 
 func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
-	//nolint
 	Err(cmd.Context(), ErrNotYetImplemeted)
 
 	if utils.HasNoFlagsAndShownHelp(cmd) {

src/cmd/factory/factory.go

@@ -8,6 +8,15 @@ import (
 	"github.com/spf13/cobra"
 
 	. "github.com/alcionai/corso/src/cli/print"
+	"github.com/alcionai/corso/src/internal/common"
+	"github.com/alcionai/corso/src/internal/connector"
+	"github.com/alcionai/corso/src/internal/connector/mockconnector"
+	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/pkg/account"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/credentials"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/path"
 )
 
 var factoryCmd = &cobra.Command{
@@ -29,10 +38,10 @@ var oneDriveCmd = &cobra.Command{
 }
 
 var (
-	count     int
+	count       int
-	container string
+	destination string
-	tenant    string
+	tenant      string
-	user      string
+	user        string
 )
 
 // TODO: ErrGenerating       = errors.New("not all items were successfully generated")
@@ -44,7 +53,10 @@ var ErrNotYetImplemeted = errors.New("not yet implemented")
 // ------------------------------------------------------------------------------------------
 
 func main() {
-	ctx := SetRootCmd(context.Background(), factoryCmd)
+	ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
+	ctx = SetRootCmd(ctx, factoryCmd)
+
+	defer logger.Flush(ctx)
 
 	// persistent flags that are common to all use cases
 	fs := factoryCmd.PersistentFlags()
@@ -53,8 +65,8 @@ func main() {
 	cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("user"))
 	fs.IntVar(&count, "count", 0, "count of items to produce")
 	cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("count"))
-	fs.StringVar(&container, "container", "", "container location of the new data (will create as needed)")
+	fs.StringVar(&destination, "destination", "", "destination of the new data (will create as needed)")
-	cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("container"))
+	cobra.CheckErr(factoryCmd.MarkPersistentFlagRequired("destination"))
 
 	factoryCmd.AddCommand(exchangeCmd)
 	addExchangeCommands(exchangeCmd)
@@ -80,3 +92,102 @@ func handleOneDriveFactory(cmd *cobra.Command, args []string) error {
 	Err(cmd.Context(), ErrNotYetImplemeted)
 	return cmd.Help()
 }
+
+// ------------------------------------------------------------------------------------------
+// Common Helpers
+// ------------------------------------------------------------------------------------------
+
+func getGCAndVerifyUser(ctx context.Context, userID string) (*connector.GraphConnector, string, error) {
+	tid := common.First(tenant, os.Getenv(account.TenantID))
+
+	// get account info
+	m365Cfg := account.M365Config{
+		M365:     credentials.GetM365(),
+		TenantID: tid,
+	}
+
+	acct, err := account.NewAccount(account.ProviderM365, m365Cfg)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "finding m365 account details")
+	}
+
+	// build a graph connector
+	gc, err := connector.NewGraphConnector(ctx, acct)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "connecting to graph api")
+	}
+
+	if _, ok := gc.Users[user]; !ok {
+		return nil, "", errors.New("user not found within tenant")
+	}
+
+	return gc, tid, nil
+}
+
+type item struct {
+	name string
+	data []byte
+}
+
+type collection struct {
+	// Elements (in order) for the path representing this collection. Should
+	// only contain elements after the prefix that corso uses for the path. For
+	// example, a collection for the Inbox folder in exchange mail would just be
+	// "Inbox".
+	pathElements []string
+	category     path.CategoryType
+	items        []item
+}
+
+func buildCollections(
+	service path.ServiceType,
+	tenant, user string,
+	dest control.RestoreDestination,
+	colls []collection,
+) ([]data.Collection, error) {
+	collections := make([]data.Collection, 0, len(colls))
+
+	for _, c := range colls {
+		pth, err := toDataLayerPath(
+			service,
+			tenant,
+			user,
+			c.category,
+			c.pathElements,
+			false,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		mc := mockconnector.NewMockExchangeCollection(pth, len(c.items))
+
+		for i := 0; i < len(c.items); i++ {
+			mc.Names[i] = c.items[i].name
+			mc.Data[i] = c.items[i].data
+		}
+
+		collections = append(collections, mc)
+	}
+
+	return collections, nil
+}
+
+func toDataLayerPath(
+	service path.ServiceType,
+	tenant, user string,
+	category path.CategoryType,
+	elements []string,
+	isItem bool,
+) (path.Path, error) {
+	pb := path.Builder{}.Append(elements...)
+
+	switch service {
+	case path.ExchangeService:
+		return pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem)
+	case path.OneDriveService:
+		return pb.ToDataLayerOneDrivePath(tenant, user, isItem)
+	}
+
+	return nil, errors.Errorf("unknown service %s", service.String())
+}

src/cmd/purge/purge.go

@@ -18,6 +18,7 @@ import (
 	"github.com/alcionai/corso/src/internal/connector/onedrive"
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/credentials"
+	"github.com/alcionai/corso/src/pkg/logger"
 )
 
 var purgeCmd = &cobra.Command{
@@ -64,7 +65,11 @@ var ErrPurging = errors.New("not all items were successfully purged")
 // ------------------------------------------------------------------------------------------
 
 func main() {
-	ctx := SetRootCmd(context.Background(), purgeCmd)
+	ctx, _ := logger.SeedLevel(context.Background(), logger.Development)
+	ctx = SetRootCmd(ctx, purgeCmd)
+
+	defer logger.Flush(ctx)
+
 	fs := purgeCmd.PersistentFlags()
 	fs.StringVar(&before, "before", "", "folders older than this date are deleted.  (default: now in UTC)")
 	fs.StringVar(&user, "user", "", "m365 user id whose folders will be deleted")

src/internal/connector/exchange/service_restore.go

@@ -182,8 +182,7 @@ func RestoreMailMessage(
 	bits []byte,
 	service graph.Service,
 	cp control.CollisionPolicy,
-	destination,
+	destination, user string,
-	user string,
 ) (*details.ExchangeInfo, error) {
 	// Creates messageable object from original bytes
 	originalMessage, err := support.CreateMessageFromBytes(bits)

src/internal/connector/mockconnector/mock_data_message.go

@@ -22,22 +22,73 @@ const (
 		"\\nThanking you in adv"
 
 	// Order of fields to fill in:
-	//   1. message body
+	//   1. created datetime
-	//   2. message preview
+	//   2. modified datetime
-	//   3. sender user ID
+	//   3. message body
-	//   4. subject
+	//   4. message preview
-	messageTmpl = "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity\"," +
+	//   5. sender user ID
-		"\"@odata.etag\":\"W/\\\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\\\"\",\"categories\":[],\"changeKey\":\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\",\"createdDateTime\":\"2022-09-26T23:15:50Z\",\"lastModifiedDateTime\":\"2022-09-26T23:15:51Z\",\"bccRecipients\":[],\"body\":{\"content\":\"<html><head>" +
+	//   6. received datetime
-		"\\n<meta http-equiv=\\\"Content-Type\\\" content=\\\"text/html; charset=utf-8\\\"><style type=\\\"text/css\\\" style=\\\"display:none\\\">\\n<!--\\np\\n{margin-top:0;\\nmargin-bottom:0}\\n-->" +
+	//   7. sender email
-		"\\n</style></head><body dir=\\\"ltr\\\"><div class=\\\"elementToProof\\\" style=\\\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0)\\\">%s" +
+	//   8. sent datetime
-		"</div></body></html>\",\"contentType\":\"html\"}," +
+	//   9. subject
-		"\"bodyPreview\":\"%s\"," +
+	//   10. recipient user addr
-		"\"ccRecipients\":[],\"conversationId\":\"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=\",\"conversationIndex\":\"AQHY0f3Ermc1ZF01akakuntelv+bsA==\",\"flag\":{\"flagStatus\":\"notFlagged\"}," +
+	messageTmpl = `{
-		"\"from\":{\"emailAddress\":{\"address\":\"%s\",\"name\":\"A Stranger\"}},\"hasAttachments\":false,\"importance\":\"normal\",\"inferenceClassification\":\"focused\",\"internetMessageId\":\"<SJ0PR17MB562266A1E61A8EA12F5FB17BC3529@SJ0PR17MB5622.namprd17.prod.outlook.com>\"," +
+		"id":"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA=",
-		"\"isDeliveryReceiptRequested\":false,\"isDraft\":false,\"isRead\":false,\"isReadReceiptRequested\":false,\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=\",\"receivedDateTime\":\"2022-09-26T23:15:50Z\"," +
+		"@odata.context":"https://graph.microsoft.com/v1.0/$metadata#users('a4a472f8-ccb0-43ec-bf52-3697a91b926c')/messages/$entity",
-		"\"replyTo\":[],\"sender\":{\"emailAddress\":{\"address\":\"foobar@8qzvrj.onmicrosoft.com\",\"name\":\"A Stranger\"}},\"sentDateTime\":\"2022-09-26T23:15:46Z\"," +
+		"@odata.etag":"W/\"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU\"",
-		"\"subject\":\"%s\",\"toRecipients\":[{\"emailAddress\":{\"address\":\"LidiaH@8qzvrj.onmicrosoft.com\",\"name\":\"Lidia Holloway\"}}]," +
+		"categories":[],
-		"\"webLink\":\"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem\"}"
+		"changeKey":"CQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAAB2ZxqU",
+		"createdDateTime":"%s",
+		"lastModifiedDateTime":"%s",
+		"bccRecipients":[],
+		"body":{
+			"content":"<html><head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"><style type=\"text/css\" style=\"display:none\">\n<!--\np\n{margin-top:0;\nmargin-bottom:0}\n-->` +
+		`\n</style></head><body dir=\"ltr\"><div class=\"elementToProof\" style=\"font-family:Calibri,Arial,Helvetica,sans-serif; font-size:12pt; color:rgb(0,0,0)\">` +
+		`%s` +
+		`</div></body></html>",
+			"contentType":"html"
+		},
+		"bodyPreview":"%s",
+		"ccRecipients":[],
+		"conversationId":"AAQkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAQAK5nNWRdNWpGpLp7Xpb-m7A=",
+		"conversationIndex":"AQHY0f3Ermc1ZF01akakuntelv+bsA==",
+		"flag":{
+			"flagStatus":"notFlagged"},
+			"from":{
+				"emailAddress":{
+					"address":"%s",
+					"name":"A Stranger"
+				}
+			},
+		"hasAttachments":false,
+		"importance":"normal",
+		"inferenceClassification":"focused",
+		"internetMessageId":"<SJ0PR17MB562266A1E61A8EA12F5FB17BC3529@SJ0PR17MB5622.namprd17.prod.outlook.com>",
+		"isDeliveryReceiptRequested":false,
+		"isDraft":false,
+		"isRead":false,
+		"isReadReceiptRequested":false,
+		"parentFolderId":"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAAA=",
+		"receivedDateTime":"%s",
+		"replyTo":[],
+		"sender":{
+			"emailAddress":{
+				"address":"%s",
+				"name":"A Stranger"
+			}
+		},
+		"sentDateTime":"%s",
+		"subject":"%s",
+		"toRecipients":[
+			{
+				"emailAddress":{
+					"address":"%s",
+					"name":"A Stranger"
+				}
+			}
+		],
+		"webLink":"https://outlook.office365.com/owa/?ItemID=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEMAADSEBNbUIB9RL6ePDeF3FIYAAB3XwIkAAA%%3D&exvsurl=1&viewmodel=ReadMessageItem"
+	}`
 
 	// Order of fields to fill in:
 	//   1. start/end date
@@ -62,7 +113,7 @@ const (
 		"\"type\":\"singleInstance\",\"webLink\":\"https://outlook.office365.com/owa/?itemid=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAADSEBNbUIB9RL6ePDeF3FIYAAAAAG76AAA%%3D&exvsurl=1&path=/calendar/item\"}"
 )
 
-// GetMockMessageBytes returns bytes for Messageable item.
+// GetMockMessageBytes returns bytes for a Messageable item.
 // Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
 func GetMockMessageBytes(subject string) []byte {
 	userID := "foobar@8qzvrj.onmicrosoft.com"
@@ -70,15 +121,23 @@ func GetMockMessageBytes(subject string) []byte {
 
 	message := fmt.Sprintf(
 		messageTmpl,
+		"2022-09-26T23:15:50Z", // created
+		"2022-09-26T23:15:51Z", // modified
 		defaultMessageBody,
 		defaultMessagePreview,
 		userID,
+		"2022-09-26T23:15:50Z",
+		"foobar@8qzvrj.onmicrosoft.com",
+		"2022-09-26T23:15:46Z",
 		"TPS Report "+subject+timestamp,
-	)
+		"LidiaH@8qzvrj.onmicrosoft.com")
 
 	return []byte(message)
 }
 
+// GetMockMessageBytes returns bytes for a Messageable item.
+// Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
+// Body must contain a well-formatted string, consumable in a json payload.  IE: no unescaped newlines.
 func GetMockMessageWithBodyBytes(subject, body string) []byte {
 	userID := "foobar@8qzvrj.onmicrosoft.com"
 	preview := body
@@ -89,11 +148,47 @@ func GetMockMessageWithBodyBytes(subject, body string) []byte {
 
 	message := fmt.Sprintf(
 		messageTmpl,
+		"2022-09-26T23:15:50Z", // created
+		"2022-09-26T23:15:51Z", // modified
 		body,
 		preview,
 		userID,
+		"2022-09-26T23:15:50Z",
+		"foobar@8qzvrj.onmicrosoft.com",
+		"2022-09-26T23:15:46Z",
 		subject,
-	)
+		"LidiaH@8qzvrj.onmicrosoft.com")
+
+	return []byte(message)
+}
+
+// GetMockMessageBytes returns bytes for a Messageable item.
+// Contents verified as working with sample data from kiota-serialization-json-go v0.5.5
+// created, modified, sent, and received should be in the format 2006-01-02T15:04:05Z
+// Body must contain a well-formatted string, consumable in a json payload.  IE: no unescaped newlines.
+func GetMockMessageWith(
+	to, from, sender, // user PNs
+	subject, body, // arbitrary data
+	created, modified, sent, received string, // legacy datetimes
+) []byte {
+	preview := body
+
+	if len(preview) > 255 {
+		preview = preview[:256]
+	}
+
+	message := fmt.Sprintf(
+		messageTmpl,
+		created,
+		modified,
+		body,
+		preview,
+		from,
+		received,
+		sender,
+		sent,
+		subject,
+		to)
 
 	return []byte(message)
 }