Contacts export for exchange (#4883)

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [x] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes https://github.com/alcionai/corso/issues/3891

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E

CHANGELOG.md

@@ -20,6 +20,9 @@ this case, Corso will skip over the item but report this in the backup summary.
 ### Known issues
 - Restoring OneDrive, SharePoint, or Teams & Groups items shared with external users while the tenant or site is configured to not allow sharing with external users will not restore permissions.
 
+### Added
+- Contacts can now be exported from Exchange backups as .vcf files
+
 ## [v0.17.0] (beta) - 2023-12-11
 
 ### Changed

src/cmd/converter/converter.go

@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/alcionai/corso/src/internal/converters/eml"
+	"github.com/alcionai/corso/src/internal/converters/vcf"
 )
 
 func main() {
@@ -27,13 +28,18 @@ func main() {
 	var out string
 
 	switch from {
-	case "msg":
+	case "json":
 		switch to {
 		case "eml":
 			out, err = eml.FromJSON(context.Background(), body)
 			if err != nil {
 				log.Fatal(err)
 			}
+		case "vcf":
+			out, err = vcf.FromJSON(context.Background(), body)
+			if err != nil {
+				log.Fatal(err)
+			}
 		default:
 			log.Fatal("Unknown target format", to)
 		}

src/go.mod

@@ -53,6 +53,8 @@ require (
 	gotest.tools/v3 v3.5.1
 )
 
+require github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9
+
 require (
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect

src/go.sum

@@ -69,6 +69,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
+github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA=
+github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

src/internal/converters/vcf/testdata/contacts-input.json

@@ -0,0 +1,67 @@
+{
+  "id": "AAMkAGJiZmE2NGU4LTQ4YjktNDI1Mi1iMWQzLTQ1MmMxODJkZmQyNABGAAAAAABFdiK7oifWRb4ADuqgSRcnBwBBFDg0JJk7TY1fmsJrh7tNAAAAAAEOAABBFDg0JJk7TY1fmsJrh7tNAADhLT4uAAA=",
+  "@odata.type": "#microsoft.graph.contact",
+  "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('7ceb8e03-bdc5-4509-a136-457526165ec0')/contacts/$entity",
+  "@odata.etag": "W/\"EQAAABYAAABBFDg0JJk7TY1fmsJrh7tNAAFGKnME\"",
+  "categories": [],
+  "changeKey": "EQAAABYAAABBFDg0JJk7TY1fmsJrh7tNAAFGKnME",
+  "createdDateTime": "2023-07-17T10:42:56Z",
+  "lastModifiedDateTime": "2023-12-18T10:37:50Z",
+  "assistantName": "",
+  "birthday": "2012-09-11T11:59:00Z",
+  "businessAddress": {
+    "city": "Businss City",
+    "countryOrRegion": "Business Country",
+    "postalCode": "888888",
+    "state": "Business State",
+    "street": "Business Stree"
+  },
+  "businessPhones": [],
+  "children": [],
+  "companyName": "Company Name",
+  "department": "Mangrove",
+  "displayName": "Wo. Random Peterson Person Jr.",
+  "emailAddresses": [
+    {
+      "address": "mockemail@provider.com",
+      "name": "mockemail@provider.com"
+    },
+    {
+      "address": "anotheremail@place.com",
+      "name": "anotheremail@place.com"
+    }
+  ],
+  "fileAs": "",
+  "generation": "Jr.",
+  "givenName": "Random",
+  "homeAddress": {
+    "city": "City Name",
+    "countryOrRegion": "Country",
+    "postalCode": "090909",
+    "state": "Province Name",
+    "street": "Street Name"
+  },
+  "homePhones": [
+    "123091230921"
+  ],
+  "imAddresses": [
+    "immmm"
+  ],
+  "jobTitle": "Principal Dough Beater",
+  "manager": "",
+  "middleName": "Peterson",
+  "mobilePhone": "00000111111",
+  "nickName": "Mikey",
+  "officeLocation": "AhemBad",
+  "otherAddress": {
+    "street": "Another street"
+  },
+  "parentFolderId": "AQMkAGJiAGZhNjRlOC00OGI5LTQyNTItYjFkMy00NTJjMTgyZGZkMjQALgAAA0V2IruiJ9ZFvgAO6qBJFycBAEEUODQkmTtNjV_awmuHu00AAAIBDgAAAA==",
+  "personalNotes": "Note on dough stuff",
+  "spouseName": "Nona Ur Business",
+  "surname": "Person",
+  "title": "Wo.",
+  "yomiCompanyName": "Owchshclaw",
+  "yomiGivenName": "Aaaaa",
+  "yomiSurname": "Bbbbb"
+}

src/internal/converters/vcf/testdata/contacts-output.eml

@@ -0,0 +1,18 @@
+BEGIN:VCARD
+VERSION:4.0
+ADR;TYPE=home:;;Street Name;City Name;Province Name;090909;Country
+ADR;TYPE=work:;;Business Stree;Businss City;Business State;888888;Business Country
+ADR;TYPE=other:;;Another street;;;;
+BDAY:2012-09-11
+EMAIL:mockemail@provider.com
+EMAIL:anotheremail@place.com
+IMPP:immmm
+N:Person;Random;Peterson;Wo.;Jr.
+NICKNAME:Mikey
+NOTE:Note on dough stuff
+ORG:Company Name;Mangrove
+RELATED;TYPE=spouse:Nona Ur Business
+TEL;TYPE=cell:00000111111
+TEL;TYPE=home:123091230921
+TITLE:Principal Dough Beater
+END:VCARD

src/internal/converters/vcf/testdata/testdata.go

@@ -0,0 +1,9 @@
+package testdata
+
+import _ "embed"
+
+//go:embed contacts-input.json
+var ContactsInput string
+
+//go:embed contacts-output.eml
+var ContactsOutput string

src/internal/converters/vcf/vcf.go

@@ -0,0 +1,206 @@
+package vcf
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/alcionai/clues"
+	"github.com/emersion/go-vcard"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
+)
+
+// This package helps convert the json response backed up from graph
+// API to a vCard file.
+// Ref: https://learn.microsoft.com/en-us/graph/api/resources/contact?view=graph-rest-1.0
+// Ref: https://datatracker.ietf.org/doc/html/rfc6350
+
+// TODO: items that are only available via beta api and not mapped
+// weddingAnniversary, gender, websites
+
+func addAddress(iaddr models.PhysicalAddressable, addrType string, vc *vcard.Card) {
+	if iaddr == nil {
+		return
+	}
+
+	// return if every value is empty
+	if len(ptr.Val(iaddr.GetStreet())) == 0 &&
+		len(ptr.Val(iaddr.GetCity())) == 0 &&
+		len(ptr.Val(iaddr.GetState())) == 0 &&
+		len(ptr.Val(iaddr.GetPostalCode())) == 0 &&
+		len(ptr.Val(iaddr.GetCountryOrRegion())) == 0 {
+		return
+	}
+
+	addr := vcard.Address{
+		StreetAddress: ptr.Val(iaddr.GetStreet()),
+		Locality:      ptr.Val(iaddr.GetCity()),
+		Region:        ptr.Val(iaddr.GetState()),
+		PostalCode:    ptr.Val(iaddr.GetPostalCode()),
+		Country:       ptr.Val(iaddr.GetCountryOrRegion()),
+	}
+
+	if len(addrType) > 0 {
+		addr.Field = &vcard.Field{}
+		addr.Params = vcard.Params{"TYPE": []string{addrType}}
+	}
+
+	vc.AddAddress(&addr)
+}
+
+func addPhones(phones []string, phoneType string, vc *vcard.Card) {
+	for _, phone := range phones {
+		vc.Add(
+			vcard.FieldTelephone,
+			&vcard.Field{Value: phone, Params: vcard.Params{"TYPE": []string{phoneType}}})
+	}
+}
+
+func addEmails(emails []models.EmailAddressable, vc *vcard.Card) {
+	for _, email := range emails {
+		etype, _ := email.GetBackingStore().Get("type")
+		if etype == "unknown" {
+			etype = nil
+		}
+
+		if etype != nil {
+			vc.Add(
+				vcard.FieldEmail,
+				&vcard.Field{
+					Value:  ptr.Val(email.GetAddress()),
+					Params: vcard.Params{"TYPE": []string{etype.(string)}},
+				})
+		} else {
+			vc.Add(
+				vcard.FieldEmail,
+				&vcard.Field{Value: ptr.Val(email.GetAddress())})
+		}
+	}
+}
+
+func FromJSON(ctx context.Context, body []byte) (string, error) {
+	vc := vcard.Card{}
+	vcard.ToV4(vc)
+
+	data, err := api.BytesToContactable(body)
+	if err != nil {
+		return "", clues.Wrap(err, "converting to contactable")
+	}
+
+	name := vcard.Name{
+		GivenName:       ptr.Val(data.GetGivenName()),
+		FamilyName:      ptr.Val(data.GetSurname()),
+		AdditionalName:  ptr.Val(data.GetMiddleName()),
+		HonorificPrefix: ptr.Val(data.GetTitle()),
+		HonorificSuffix: ptr.Val(data.GetGeneration()),
+	}
+	vc.SetName(&name)
+
+	nick := data.GetNickName()
+	if nick != nil {
+		vc.Set(vcard.FieldNickname, &vcard.Field{Value: *nick})
+	}
+
+	bday := data.GetBirthday()
+	if bday != nil {
+		vc.Set(vcard.FieldBirthday, &vcard.Field{Value: bday.Format("2006-01-02")})
+	}
+
+	addAddress(data.GetHomeAddress(), vcard.TypeHome, &vc)
+	addAddress(data.GetBusinessAddress(), vcard.TypeWork, &vc)
+	addAddress(data.GetOtherAddress(), "other", &vc)
+
+	mob := data.GetMobilePhone()
+	if mob != nil && len(ptr.Val(mob)) > 0 {
+		addPhones([]string{*mob}, vcard.TypeCell, &vc)
+	}
+
+	addPhones(data.GetBusinessPhones(), vcard.TypeWork, &vc)
+	addPhones(data.GetHomePhones(), vcard.TypeHome, &vc)
+
+	addEmails(data.GetEmailAddresses(), &vc) // no type?
+
+	im := data.GetImAddresses()
+	for _, imaddr := range im {
+		vc.Add(vcard.FieldIMPP, &vcard.Field{Value: imaddr})
+	}
+
+	orgFull := ""
+
+	org := data.GetCompanyName()
+	if org != nil {
+		orgFull = *org
+	}
+
+	dept := data.GetDepartment()
+	if dept != nil {
+		if len(orgFull) > 0 {
+			orgFull += ";"
+		}
+
+		orgFull += *dept
+	}
+
+	profession := data.GetProfession()
+	if profession != nil {
+		if len(orgFull) > 0 {
+			orgFull += ";"
+		}
+
+		orgFull += *profession
+	}
+
+	if len(orgFull) > 0 {
+		vc.Set(vcard.FieldOrganization, &vcard.Field{Value: orgFull})
+	}
+
+	job := data.GetJobTitle()
+	if job != nil && len(ptr.Val(job)) > 0 {
+		vc.Set(vcard.FieldTitle, &vcard.Field{Value: *job})
+	}
+
+	children := data.GetChildren()
+	for _, child := range children {
+		vc.Add(
+			vcard.FieldRelated,
+			&vcard.Field{Value: child, Params: vcard.Params{"TYPE": []string{vcard.TypeChild}}})
+	}
+
+	spouse := data.GetSpouseName()
+	if spouse != nil && len(ptr.Val(spouse)) > 0 {
+		vc.Add(
+			vcard.FieldRelated,
+			&vcard.Field{Value: *spouse, Params: vcard.Params{"TYPE": []string{vcard.TypeSpouse}}})
+	}
+
+	manager := data.GetManager()
+	if manager != nil && len(ptr.Val(manager)) > 0 {
+		vc.Add(
+			vcard.FieldRelated,
+			&vcard.Field{Value: *manager, Params: vcard.Params{"TYPE": []string{"manager"}}})
+	}
+
+	assistant := data.GetAssistantName()
+	if assistant != nil && len(ptr.Val(assistant)) > 0 {
+		vc.Add(
+			vcard.FieldRelated,
+			&vcard.Field{Value: *assistant, Params: vcard.Params{"TYPE": []string{"assistant"}}})
+	}
+
+	notes := data.GetPersonalNotes()
+	if notes != nil && len(ptr.Val(notes)) > 0 {
+		vc.Set(vcard.FieldNote, &vcard.Field{Value: *notes})
+	}
+
+	out := bytes.NewBuffer(nil)
+	enc := vcard.NewEncoder(out)
+
+	err = enc.Encode(vc)
+	if err != nil {
+		return "", clues.Wrap(err, "encoding vcard")
+	}
+
+	return out.String(), nil
+}

src/internal/converters/vcf/vcf_test.go

@@ -0,0 +1,161 @@
+package vcf
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	kjson "github.com/microsoft/kiota-serialization-json-go"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/internal/converters/vcf/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+)
+
+type VCFUnitSuite struct {
+	tester.Suite
+}
+
+func TestVCFUnitSuite(t *testing.T) {
+	suite.Run(t, &VCFUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *VCFUnitSuite) TestConvert_contactable_to_vcf() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	body := []byte(testdata.ContactsInput)
+
+	bytes, err := FromJSON(ctx, body)
+	require.NoError(t, err, "convert")
+
+	out := strings.ReplaceAll(string(bytes), "\r", "") // output contains \r
+	assert.Equal(t, strings.TrimSpace(testdata.ContactsOutput), strings.TrimSpace(string(out)))
+}
+
+func (suite *VCFUnitSuite) TestConvert_contactable_cases() {
+	t := suite.T()
+
+	tests := []struct {
+		name           string
+		transformation func(contact models.Contactable)
+		check          string
+	}{
+		{
+			name: "name",
+			transformation: func(contact models.Contactable) {
+				contact.SetGivenName(ptr.To("given"))
+				contact.SetSurname(ptr.To("sur"))
+			},
+			check: "N:sur;given;;;",
+		},
+		{
+			name: "all name related",
+			transformation: func(contact models.Contactable) {
+				contact.SetGivenName(ptr.To("given"))
+				contact.SetSurname(ptr.To("sur"))
+				contact.SetMiddleName(ptr.To("middle"))
+				contact.SetTitle(ptr.To("title"))
+				contact.SetGeneration(ptr.To("gen"))
+			},
+			check: "N:sur;given;middle;title;gen",
+		},
+		{
+			name: "org",
+			transformation: func(contact models.Contactable) {
+				contact.SetCompanyName(ptr.To("org"))
+			},
+			check: "ORG:org",
+		},
+		{
+			name: "org,dept,prof",
+			transformation: func(contact models.Contactable) {
+				contact.SetCompanyName(ptr.To("org"))
+				contact.SetDepartment(ptr.To("dept"))
+				contact.SetProfession(ptr.To("prof"))
+			},
+			check: "ORG:org;dept;prof",
+		},
+		{
+			name: "dept,prof without org name",
+			transformation: func(contact models.Contactable) {
+				contact.SetDepartment(ptr.To("dept"))
+				contact.SetProfession(ptr.To("prof"))
+			},
+			check: "ORG:dept;prof",
+		},
+		{
+			name: "org,prof without dept",
+			transformation: func(contact models.Contactable) {
+				contact.SetCompanyName(ptr.To("org"))
+				contact.SetProfession(ptr.To("prof"))
+			},
+			check: "ORG:org;prof",
+		},
+		{
+			name: "birthday",
+			transformation: func(contact models.Contactable) {
+				date := time.Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
+				contact.SetBirthday(ptr.To(date))
+			},
+			check: "BDAY:2000-01-01",
+		},
+		{
+			name: "address",
+			transformation: func(contact models.Contactable) {
+				add := models.NewPhysicalAddress()
+				add.SetStreet(ptr.To("street"))
+				add.SetCity(ptr.To("city"))
+				add.SetState(ptr.To("state"))
+				add.SetCountryOrRegion(ptr.To("country"))
+				add.SetPostalCode(ptr.To("zip"))
+				contact.SetHomeAddress(add)
+			},
+			check: "ADR;TYPE=home:;;street;city;state;zip;country",
+		},
+		{
+			name: "mobile",
+			transformation: func(contact models.Contactable) {
+				contact.SetMobilePhone(ptr.To("mobile"))
+			},
+			check: "TEL;TYPE=cell:mobile",
+		},
+		{
+			name: "home",
+			transformation: func(contact models.Contactable) {
+				contact.SetHomePhones([]string{"home"})
+			},
+			check: "TEL;TYPE=home:home", // ideally check both
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, flush := tester.NewContext(t)
+			defer flush()
+
+			contact := models.NewContact()
+			tt.transformation(contact)
+
+			writer := kjson.NewJsonSerializationWriter()
+			defer writer.Close()
+
+			err := writer.WriteObjectValue("", contact)
+			require.NoError(t, err, "serializing contact")
+
+			nbody, err := writer.GetSerializedContent()
+			require.NoError(t, err, "getting serialized content")
+
+			bytes, err := FromJSON(ctx, nbody)
+			require.NoError(t, err, "convert")
+
+			assert.Contains(t, string(bytes), tt.check)
+		})
+	}
+}

src/internal/m365/collection/exchange/export.go

@@ -8,6 +8,7 @@ import (
 	"github.com/alcionai/clues"
 
 	"github.com/alcionai/corso/src/internal/converters/eml"
+	"github.com/alcionai/corso/src/internal/converters/vcf"
 	"github.com/alcionai/corso/src/internal/data"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/export"
@@ -48,13 +49,20 @@ func streamItems(
 	for _, rc := range drc {
 		ictx := clues.Add(ctx, "path_short_ref", rc.FullPath().ShortRef())
 
+		ext := ".eml"
+		category := rc.FullPath().Category()
+
+		if category == path.ContactsCategory {
+			ext = ".vcf"
+		}
+
 		for item := range rc.Items(ictx, errs) {
 			id := item.ID()
-			name := id + ".eml"
+			name := id + ext
 
 			itemCtx := clues.Add(ictx, "stream_item_id", id)
 
-			stats.UpdateResourceCount(path.EmailCategory)
+			stats.UpdateResourceCount(category)
 
 			reader := item.ToReader()
 			content, err := io.ReadAll(reader)
@@ -74,22 +82,41 @@ func streamItems(
 				continue
 			}
 
-			email, err := eml.FromJSON(itemCtx, content)
+			var outData string
-			if err != nil {
-				err = clues.Wrap(err, "converting JSON to eml")
 
-				logger.CtxErr(ctx, err).Info("processing collection item")
+			switch category {
+			case path.EmailCategory:
+				outData, err = eml.FromJSON(itemCtx, content)
+				if err != nil {
+					err = clues.Wrap(err, "converting to eml")
 
-				ch <- export.Item{
+					logger.CtxErr(ctx, err).Info("processing collection item")
-					ID:    id,
+
-					Error: err,
+					ch <- export.Item{
+						ID:    id,
+						Error: err,
+					}
+
+					continue
 				}
+			case path.ContactsCategory:
+				outData, err = vcf.FromJSON(ctx, content)
+				if err != nil {
+					err = clues.Wrap(err, "converting to vcf")
 
-				continue
+					logger.CtxErr(ctx, err).Info("processing collection item")
+
+					ch <- export.Item{
+						ID:    id,
+						Error: err,
+					}
+
+					continue
+				}
 			}
 
-			emlReader := io.NopCloser(bytes.NewReader([]byte(email)))
+			emlReader := io.NopCloser(bytes.NewReader([]byte(outData)))
-			body := metrics.ReaderWithStats(emlReader, path.EmailCategory, stats)
+			body := metrics.ReaderWithStats(emlReader, category, stats)
 
 			ch <- export.Item{
 				ID:   id,

src/internal/m365/service/exchange/export.go

@@ -62,9 +62,9 @@ func (h *baseExchangeHandler) ProduceExportCollections(
 		category := dc.FullPath().Category()
 
 		switch category {
-		case path.EmailCategory:
+		case path.ContactsCategory, path.EmailCategory:
 			folders := dc.FullPath().Folders()
-			pth := path.Builder{}.Append(path.EmailCategory.HumanString()).Append(folders...)
+			pth := path.Builder{}.Append(category.HumanString()).Append(folders...)
 
 			ec = append(
 				ec,
@@ -73,7 +73,7 @@ func (h *baseExchangeHandler) ProduceExportCollections(
 					[]data.RestoreCollection{dc},
 					backupVersion,
 					stats))
-		case path.EventsCategory, path.ContactsCategory:
+		case path.EventsCategory:
 			logger.Ctx(ctx).With("category", category.String()).Debugw("Skipping restore for category")
 		default:
 			return nil, clues.NewWC(ctx, "data category not supported").