Converter for msg to eml (#4640)

Add logic to convert from msg(backup available in corso) to eml

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/issues/3893

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

src/go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/tidwall/pretty v1.2.1
 	github.com/tomlazar/table v0.1.2
 	github.com/vbauerster/mpb/v8 v8.1.6
+	github.com/xhit/go-simple-mail/v2 v2.16.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
@@ -48,6 +49,7 @@ require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/aws/aws-sdk-go v1.47.9 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-test/deep v1.1.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
@@ -66,6 +68,7 @@ require (
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/std-uritemplate/std-uritemplate/go v0.0.46 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.50.0 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect

src/go.sum

@@ -134,6 +134,8 @@ github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
@@ -434,6 +436,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tomlazar/table v0.1.2 h1:DP8f62FzZAZk8oavepm1v/oyf4ni3/LMHWNlOinmleg=
 github.com/tomlazar/table v0.1.2/go.mod h1:IecZnpep9f/BatHacfh+++ftE+lFONN8BVPi9nx5U1w=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -441,6 +445,8 @@ github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e
 github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
 github.com/vbauerster/mpb/v8 v8.1.6 h1:EswHDkAsy4OQ7QBAmU1MUPz4vHzl6KlINjlh7vJoxvY=
 github.com/vbauerster/mpb/v8 v8.1.6/go.mod h1:O9/Wl8X9dUbR63tZ41MLIAxrtNfwlpwUhGkeYugUPW8=
+github.com/xhit/go-simple-mail/v2 v2.16.0 h1:ouGy/Ww4kuaqu2E2UrDw7SvLaziWTB60ICLkIkNVccA=
+github.com/xhit/go-simple-mail/v2 v2.16.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g=
 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

src/internal/converters/eml/eml.go

@@ -0,0 +1,142 @@
+package eml
+
+// This package helps convert from the json response
+// received from Graph API to .eml format (rfc0822).
+// Ref: https://www.ietf.org/rfc/rfc0822.txt
+// Ref: https://datatracker.ietf.org/doc/html/rfc5322
+// Data missing from backup:
+// SetReturnPath SetPriority SetListUnsubscribe SetDkim
+// AddAlternative SetDSN (and any other X-MS specific headers)
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	mail "github.com/xhit/go-simple-mail/v2"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/pkg/logger"
+)
+
+const (
+	fromFormat = "%s <%s>"
+	dateFormat = "2006-01-02 15:04:05 MST" // from xhit/go-simple-mail
+)
+
+// toEml converts a Messageable to .eml format
+func toEml(data models.Messageable) (string, error) {
+	email := mail.NewMSG()
+
+	if data.GetFrom() != nil {
+		email.SetFrom(
+			fmt.Sprintf(
+				fromFormat,
+				ptr.Val(data.GetFrom().GetEmailAddress().GetName()),
+				ptr.Val(data.GetFrom().GetEmailAddress().GetAddress())))
+	}
+
+	if data.GetToRecipients() != nil {
+		for _, recipient := range data.GetToRecipients() {
+			email.AddTo(
+				fmt.Sprintf(
+					fromFormat,
+					ptr.Val(recipient.GetEmailAddress().GetName()),
+					ptr.Val(recipient.GetEmailAddress().GetAddress())))
+		}
+	}
+
+	if data.GetCcRecipients() != nil {
+		for _, recipient := range data.GetCcRecipients() {
+			email.AddCc(
+				fmt.Sprintf(
+					fromFormat,
+					ptr.Val(recipient.GetEmailAddress().GetName()),
+					ptr.Val(recipient.GetEmailAddress().GetAddress())))
+		}
+	}
+
+	if data.GetBccRecipients() != nil {
+		for _, recipient := range data.GetBccRecipients() {
+			email.AddBcc(
+				fmt.Sprintf(
+					fromFormat,
+					ptr.Val(recipient.GetEmailAddress().GetName()),
+					ptr.Val(recipient.GetEmailAddress().GetAddress())))
+		}
+	}
+
+	if data.GetReplyTo() != nil {
+		rts := data.GetReplyTo()
+		if len(rts) > 1 {
+			logger.Ctx(context.TODO()).
+				With("id", ptr.Val(data.GetId()),
+					"reply_to_count", len(rts)).
+				Warn("more than 1 reply to")
+		} else if len(rts) != 0 {
+			email.SetReplyTo(
+				fmt.Sprintf(
+					fromFormat,
+					ptr.Val(rts[0].GetEmailAddress().GetName()),
+					ptr.Val(rts[0].GetEmailAddress().GetAddress())))
+		}
+	}
+
+	if data.GetSubject() != nil {
+		email.SetSubject(ptr.Val(data.GetSubject()))
+	}
+
+	if data.GetSentDateTime() != nil {
+		email.SetDate(ptr.Val(data.GetSentDateTime()).Format(dateFormat))
+	}
+
+	if data.GetBody() != nil {
+		if data.GetBody().GetContentType() != nil {
+			var contentType mail.ContentType
+
+			switch data.GetBody().GetContentType().String() {
+			case "html":
+				contentType = mail.TextHTML
+			case "text":
+				contentType = mail.TextPlain
+			default:
+				// https://learn.microsoft.com/en-us/graph/api/resources/itembody?view=graph-rest-1.0#properties
+				// This should not be possible according to the documentation
+				logger.Ctx(context.TODO()).
+					With("body_type", data.GetBody().GetContentType().String(),
+						"id", ptr.Val(data.GetId())).
+					Info("unknown body content type")
+
+				contentType = mail.TextPlain
+			}
+
+			email.SetBody(contentType, ptr.Val(data.GetBody().GetContent()))
+		}
+	}
+
+	if data.GetAttachments() != nil {
+		for _, attachment := range data.GetAttachments() {
+			kind := ptr.Val(attachment.GetContentType())
+
+			bytes, err := attachment.GetBackingStore().Get("contentBytes")
+			if err != nil {
+				return "", clues.Wrap(err, "failed to get attachment bytes")
+			}
+
+			bts, ok := bytes.([]byte)
+			if !ok {
+				return "", clues.Wrap(err, "invalid content bytes")
+			}
+
+			email.Attach(&mail.File{
+				Name:     ptr.Val(attachment.GetName()),
+				MimeType: kind,
+				Data:     bts,
+				Inline:   ptr.Val(attachment.GetIsInline()),
+			})
+		}
+	}
+
+	return email.GetMessage(), nil
+}

src/internal/converters/eml/eml_test.go

@@ -0,0 +1,37 @@
+package eml
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
+)
+
+type EMLUnitSuite struct {
+	tester.Suite
+}
+
+func TestEMLUnitSuite(t *testing.T) {
+	suite.Run(t, &EMLUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *EMLUnitSuite) TestConvert_messageble_to_eml() {
+	t := suite.T()
+
+	// read test file into body as []bytes
+	body, err := os.ReadFile("testdata/email-with-attachments.json")
+	require.NoError(t, err, "reading test file")
+
+	msg, err := api.BytesToMessageable(body)
+	require.NoError(t, err, "creating message")
+
+	_, err = toEml(msg)
+	// TODO(meain): add more tests on the generated content
+	// Cannot test output directly as it contains a random boundary
+	assert.NoError(t, err, "converting to eml")
+}