diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d264d5b8..96153f31d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ this case, Corso will skip over the item but report this in the backup summary. ### Known issues - Restoring OneDrive, SharePoint, or Teams & Groups items shared with external users while the tenant or site is configured to not allow sharing with external users will not restore permissions. +### Added +- Contacts can now be exported from Exchange backups as .vcf files + ## [v0.17.0] (beta) - 2023-12-11 ### Changed diff --git a/src/cmd/converter/converter.go b/src/cmd/converter/converter.go index 0b1d58959..7dd315140 100644 --- a/src/cmd/converter/converter.go +++ b/src/cmd/converter/converter.go @@ -7,6 +7,7 @@ import ( "os" "github.com/alcionai/corso/src/internal/converters/eml" + "github.com/alcionai/corso/src/internal/converters/vcf" ) func main() { @@ -27,13 +28,18 @@ func main() { var out string switch from { - case "msg": + case "json": switch to { case "eml": out, err = eml.FromJSON(context.Background(), body) if err != nil { log.Fatal(err) } + case "vcf": + out, err = vcf.FromJSON(context.Background(), body) + if err != nil { + log.Fatal(err) + } default: log.Fatal("Unknown target format", to) } diff --git a/src/go.mod b/src/go.mod index c3d78b5be..df9a19a8a 100644 --- a/src/go.mod +++ b/src/go.mod @@ -53,6 +53,8 @@ require ( gotest.tools/v3 v3.5.1 ) +require github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 + require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect diff --git a/src/go.sum b/src/go.sum index 33d194127..2c2b102f9 100644 --- a/src/go.sum +++ b/src/go.sum @@ -69,6 +69,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= diff --git a/src/internal/converters/vcf/testdata/contacts-input.json b/src/internal/converters/vcf/testdata/contacts-input.json new file mode 100644 index 000000000..2f229d19c --- /dev/null +++ b/src/internal/converters/vcf/testdata/contacts-input.json @@ -0,0 +1,67 @@ +{ + "id": "AAMkAGJiZmE2NGU4LTQ4YjktNDI1Mi1iMWQzLTQ1MmMxODJkZmQyNABGAAAAAABFdiK7oifWRb4ADuqgSRcnBwBBFDg0JJk7TY1fmsJrh7tNAAAAAAEOAABBFDg0JJk7TY1fmsJrh7tNAADhLT4uAAA=", + "@odata.type": "#microsoft.graph.contact", + "@odata.context": "https://graph.microsoft.com/v1.0/$metadata#users('7ceb8e03-bdc5-4509-a136-457526165ec0')/contacts/$entity", + "@odata.etag": "W/\"EQAAABYAAABBFDg0JJk7TY1fmsJrh7tNAAFGKnME\"", + "categories": [], + "changeKey": "EQAAABYAAABBFDg0JJk7TY1fmsJrh7tNAAFGKnME", + "createdDateTime": "2023-07-17T10:42:56Z", + "lastModifiedDateTime": "2023-12-18T10:37:50Z", + "assistantName": "", + "birthday": "2012-09-11T11:59:00Z", + "businessAddress": { + "city": "Businss City", + "countryOrRegion": "Business Country", + "postalCode": "888888", + "state": "Business State", + "street": "Business Stree" + }, + "businessPhones": [], + "children": [], + "companyName": "Company Name", + "department": "Mangrove", + "displayName": "Wo. Random Peterson Person Jr.", + "emailAddresses": [ + { + "address": "mockemail@provider.com", + "name": "mockemail@provider.com" + }, + { + "address": "anotheremail@place.com", + "name": "anotheremail@place.com" + } + ], + "fileAs": "", + "generation": "Jr.", + "givenName": "Random", + "homeAddress": { + "city": "City Name", + "countryOrRegion": "Country", + "postalCode": "090909", + "state": "Province Name", + "street": "Street Name" + }, + "homePhones": [ + "123091230921" + ], + "imAddresses": [ + "immmm" + ], + "jobTitle": "Principal Dough Beater", + "manager": "", + "middleName": "Peterson", + "mobilePhone": "00000111111", + "nickName": "Mikey", + "officeLocation": "AhemBad", + "otherAddress": { + "street": "Another street" + }, + "parentFolderId": "AQMkAGJiAGZhNjRlOC00OGI5LTQyNTItYjFkMy00NTJjMTgyZGZkMjQALgAAA0V2IruiJ9ZFvgAO6qBJFycBAEEUODQkmTtNjV_awmuHu00AAAIBDgAAAA==", + "personalNotes": "Note on dough stuff", + "spouseName": "Nona Ur Business", + "surname": "Person", + "title": "Wo.", + "yomiCompanyName": "Owchshclaw", + "yomiGivenName": "Aaaaa", + "yomiSurname": "Bbbbb" +} diff --git a/src/internal/converters/vcf/testdata/contacts-output.eml b/src/internal/converters/vcf/testdata/contacts-output.eml new file mode 100644 index 000000000..2fedef3d9 --- /dev/null +++ b/src/internal/converters/vcf/testdata/contacts-output.eml @@ -0,0 +1,18 @@ +BEGIN:VCARD +VERSION:4.0 +ADR;TYPE=home:;;Street Name;City Name;Province Name;090909;Country +ADR;TYPE=work:;;Business Stree;Businss City;Business State;888888;Business Country +ADR;TYPE=other:;;Another street;;;; +BDAY:2012-09-11 +EMAIL:mockemail@provider.com +EMAIL:anotheremail@place.com +IMPP:immmm +N:Person;Random;Peterson;Wo.;Jr. +NICKNAME:Mikey +NOTE:Note on dough stuff +ORG:Company Name;Mangrove +RELATED;TYPE=spouse:Nona Ur Business +TEL;TYPE=cell:00000111111 +TEL;TYPE=home:123091230921 +TITLE:Principal Dough Beater +END:VCARD \ No newline at end of file diff --git a/src/internal/converters/vcf/testdata/testdata.go b/src/internal/converters/vcf/testdata/testdata.go new file mode 100644 index 000000000..16a829eaa --- /dev/null +++ b/src/internal/converters/vcf/testdata/testdata.go @@ -0,0 +1,9 @@ +package testdata + +import _ "embed" + +//go:embed contacts-input.json +var ContactsInput string + +//go:embed contacts-output.eml +var ContactsOutput string diff --git a/src/internal/converters/vcf/vcf.go b/src/internal/converters/vcf/vcf.go new file mode 100644 index 000000000..aa0785cf9 --- /dev/null +++ b/src/internal/converters/vcf/vcf.go @@ -0,0 +1,206 @@ +package vcf + +import ( + "bytes" + "context" + + "github.com/alcionai/clues" + "github.com/emersion/go-vcard" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +// This package helps convert the json response backed up from graph +// API to a vCard file. +// Ref: https://learn.microsoft.com/en-us/graph/api/resources/contact?view=graph-rest-1.0 +// Ref: https://datatracker.ietf.org/doc/html/rfc6350 + +// TODO: items that are only available via beta api and not mapped +// weddingAnniversary, gender, websites + +func addAddress(iaddr models.PhysicalAddressable, addrType string, vc *vcard.Card) { + if iaddr == nil { + return + } + + // return if every value is empty + if len(ptr.Val(iaddr.GetStreet())) == 0 && + len(ptr.Val(iaddr.GetCity())) == 0 && + len(ptr.Val(iaddr.GetState())) == 0 && + len(ptr.Val(iaddr.GetPostalCode())) == 0 && + len(ptr.Val(iaddr.GetCountryOrRegion())) == 0 { + return + } + + addr := vcard.Address{ + StreetAddress: ptr.Val(iaddr.GetStreet()), + Locality: ptr.Val(iaddr.GetCity()), + Region: ptr.Val(iaddr.GetState()), + PostalCode: ptr.Val(iaddr.GetPostalCode()), + Country: ptr.Val(iaddr.GetCountryOrRegion()), + } + + if len(addrType) > 0 { + addr.Field = &vcard.Field{} + addr.Params = vcard.Params{"TYPE": []string{addrType}} + } + + vc.AddAddress(&addr) +} + +func addPhones(phones []string, phoneType string, vc *vcard.Card) { + for _, phone := range phones { + vc.Add( + vcard.FieldTelephone, + &vcard.Field{Value: phone, Params: vcard.Params{"TYPE": []string{phoneType}}}) + } +} + +func addEmails(emails []models.EmailAddressable, vc *vcard.Card) { + for _, email := range emails { + etype, _ := email.GetBackingStore().Get("type") + if etype == "unknown" { + etype = nil + } + + if etype != nil { + vc.Add( + vcard.FieldEmail, + &vcard.Field{ + Value: ptr.Val(email.GetAddress()), + Params: vcard.Params{"TYPE": []string{etype.(string)}}, + }) + } else { + vc.Add( + vcard.FieldEmail, + &vcard.Field{Value: ptr.Val(email.GetAddress())}) + } + } +} + +func FromJSON(ctx context.Context, body []byte) (string, error) { + vc := vcard.Card{} + vcard.ToV4(vc) + + data, err := api.BytesToContactable(body) + if err != nil { + return "", clues.Wrap(err, "converting to contactable") + } + + name := vcard.Name{ + GivenName: ptr.Val(data.GetGivenName()), + FamilyName: ptr.Val(data.GetSurname()), + AdditionalName: ptr.Val(data.GetMiddleName()), + HonorificPrefix: ptr.Val(data.GetTitle()), + HonorificSuffix: ptr.Val(data.GetGeneration()), + } + vc.SetName(&name) + + nick := data.GetNickName() + if nick != nil { + vc.Set(vcard.FieldNickname, &vcard.Field{Value: *nick}) + } + + bday := data.GetBirthday() + if bday != nil { + vc.Set(vcard.FieldBirthday, &vcard.Field{Value: bday.Format("2006-01-02")}) + } + + addAddress(data.GetHomeAddress(), vcard.TypeHome, &vc) + addAddress(data.GetBusinessAddress(), vcard.TypeWork, &vc) + addAddress(data.GetOtherAddress(), "other", &vc) + + mob := data.GetMobilePhone() + if mob != nil && len(ptr.Val(mob)) > 0 { + addPhones([]string{*mob}, vcard.TypeCell, &vc) + } + + addPhones(data.GetBusinessPhones(), vcard.TypeWork, &vc) + addPhones(data.GetHomePhones(), vcard.TypeHome, &vc) + + addEmails(data.GetEmailAddresses(), &vc) // no type? + + im := data.GetImAddresses() + for _, imaddr := range im { + vc.Add(vcard.FieldIMPP, &vcard.Field{Value: imaddr}) + } + + orgFull := "" + + org := data.GetCompanyName() + if org != nil { + orgFull = *org + } + + dept := data.GetDepartment() + if dept != nil { + if len(orgFull) > 0 { + orgFull += ";" + } + + orgFull += *dept + } + + profession := data.GetProfession() + if profession != nil { + if len(orgFull) > 0 { + orgFull += ";" + } + + orgFull += *profession + } + + if len(orgFull) > 0 { + vc.Set(vcard.FieldOrganization, &vcard.Field{Value: orgFull}) + } + + job := data.GetJobTitle() + if job != nil && len(ptr.Val(job)) > 0 { + vc.Set(vcard.FieldTitle, &vcard.Field{Value: *job}) + } + + children := data.GetChildren() + for _, child := range children { + vc.Add( + vcard.FieldRelated, + &vcard.Field{Value: child, Params: vcard.Params{"TYPE": []string{vcard.TypeChild}}}) + } + + spouse := data.GetSpouseName() + if spouse != nil && len(ptr.Val(spouse)) > 0 { + vc.Add( + vcard.FieldRelated, + &vcard.Field{Value: *spouse, Params: vcard.Params{"TYPE": []string{vcard.TypeSpouse}}}) + } + + manager := data.GetManager() + if manager != nil && len(ptr.Val(manager)) > 0 { + vc.Add( + vcard.FieldRelated, + &vcard.Field{Value: *manager, Params: vcard.Params{"TYPE": []string{"manager"}}}) + } + + assistant := data.GetAssistantName() + if assistant != nil && len(ptr.Val(assistant)) > 0 { + vc.Add( + vcard.FieldRelated, + &vcard.Field{Value: *assistant, Params: vcard.Params{"TYPE": []string{"assistant"}}}) + } + + notes := data.GetPersonalNotes() + if notes != nil && len(ptr.Val(notes)) > 0 { + vc.Set(vcard.FieldNote, &vcard.Field{Value: *notes}) + } + + out := bytes.NewBuffer(nil) + enc := vcard.NewEncoder(out) + + err = enc.Encode(vc) + if err != nil { + return "", clues.Wrap(err, "encoding vcard") + } + + return out.String(), nil +} diff --git a/src/internal/converters/vcf/vcf_test.go b/src/internal/converters/vcf/vcf_test.go new file mode 100644 index 000000000..eaa0b0bc3 --- /dev/null +++ b/src/internal/converters/vcf/vcf_test.go @@ -0,0 +1,161 @@ +package vcf + +import ( + "strings" + "testing" + "time" + + kjson "github.com/microsoft/kiota-serialization-json-go" + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/converters/vcf/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type VCFUnitSuite struct { + tester.Suite +} + +func TestVCFUnitSuite(t *testing.T) { + suite.Run(t, &VCFUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *VCFUnitSuite) TestConvert_contactable_to_vcf() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + body := []byte(testdata.ContactsInput) + + bytes, err := FromJSON(ctx, body) + require.NoError(t, err, "convert") + + out := strings.ReplaceAll(string(bytes), "\r", "") // output contains \r + assert.Equal(t, strings.TrimSpace(testdata.ContactsOutput), strings.TrimSpace(string(out))) +} + +func (suite *VCFUnitSuite) TestConvert_contactable_cases() { + t := suite.T() + + tests := []struct { + name string + transformation func(contact models.Contactable) + check string + }{ + { + name: "name", + transformation: func(contact models.Contactable) { + contact.SetGivenName(ptr.To("given")) + contact.SetSurname(ptr.To("sur")) + }, + check: "N:sur;given;;;", + }, + { + name: "all name related", + transformation: func(contact models.Contactable) { + contact.SetGivenName(ptr.To("given")) + contact.SetSurname(ptr.To("sur")) + contact.SetMiddleName(ptr.To("middle")) + contact.SetTitle(ptr.To("title")) + contact.SetGeneration(ptr.To("gen")) + }, + check: "N:sur;given;middle;title;gen", + }, + { + name: "org", + transformation: func(contact models.Contactable) { + contact.SetCompanyName(ptr.To("org")) + }, + check: "ORG:org", + }, + { + name: "org,dept,prof", + transformation: func(contact models.Contactable) { + contact.SetCompanyName(ptr.To("org")) + contact.SetDepartment(ptr.To("dept")) + contact.SetProfession(ptr.To("prof")) + }, + check: "ORG:org;dept;prof", + }, + { + name: "dept,prof without org name", + transformation: func(contact models.Contactable) { + contact.SetDepartment(ptr.To("dept")) + contact.SetProfession(ptr.To("prof")) + }, + check: "ORG:dept;prof", + }, + { + name: "org,prof without dept", + transformation: func(contact models.Contactable) { + contact.SetCompanyName(ptr.To("org")) + contact.SetProfession(ptr.To("prof")) + }, + check: "ORG:org;prof", + }, + { + name: "birthday", + transformation: func(contact models.Contactable) { + date := time.Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) + contact.SetBirthday(ptr.To(date)) + }, + check: "BDAY:2000-01-01", + }, + { + name: "address", + transformation: func(contact models.Contactable) { + add := models.NewPhysicalAddress() + add.SetStreet(ptr.To("street")) + add.SetCity(ptr.To("city")) + add.SetState(ptr.To("state")) + add.SetCountryOrRegion(ptr.To("country")) + add.SetPostalCode(ptr.To("zip")) + contact.SetHomeAddress(add) + }, + check: "ADR;TYPE=home:;;street;city;state;zip;country", + }, + { + name: "mobile", + transformation: func(contact models.Contactable) { + contact.SetMobilePhone(ptr.To("mobile")) + }, + check: "TEL;TYPE=cell:mobile", + }, + { + name: "home", + transformation: func(contact models.Contactable) { + contact.SetHomePhones([]string{"home"}) + }, + check: "TEL;TYPE=home:home", // ideally check both + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, flush := tester.NewContext(t) + defer flush() + + contact := models.NewContact() + tt.transformation(contact) + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + err := writer.WriteObjectValue("", contact) + require.NoError(t, err, "serializing contact") + + nbody, err := writer.GetSerializedContent() + require.NoError(t, err, "getting serialized content") + + bytes, err := FromJSON(ctx, nbody) + require.NoError(t, err, "convert") + + assert.Contains(t, string(bytes), tt.check) + }) + } +} diff --git a/src/internal/m365/collection/exchange/export.go b/src/internal/m365/collection/exchange/export.go index 71319585f..36e2520d9 100644 --- a/src/internal/m365/collection/exchange/export.go +++ b/src/internal/m365/collection/exchange/export.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/internal/converters/eml" + "github.com/alcionai/corso/src/internal/converters/vcf" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/export" @@ -48,13 +49,20 @@ func streamItems( for _, rc := range drc { ictx := clues.Add(ctx, "path_short_ref", rc.FullPath().ShortRef()) + ext := ".eml" + category := rc.FullPath().Category() + + if category == path.ContactsCategory { + ext = ".vcf" + } + for item := range rc.Items(ictx, errs) { id := item.ID() - name := id + ".eml" + name := id + ext itemCtx := clues.Add(ictx, "stream_item_id", id) - stats.UpdateResourceCount(path.EmailCategory) + stats.UpdateResourceCount(category) reader := item.ToReader() content, err := io.ReadAll(reader) @@ -74,22 +82,41 @@ func streamItems( continue } - email, err := eml.FromJSON(itemCtx, content) - if err != nil { - err = clues.Wrap(err, "converting JSON to eml") + var outData string - logger.CtxErr(ctx, err).Info("processing collection item") + switch category { + case path.EmailCategory: + outData, err = eml.FromJSON(itemCtx, content) + if err != nil { + err = clues.Wrap(err, "converting to eml") - ch <- export.Item{ - ID: id, - Error: err, + logger.CtxErr(ctx, err).Info("processing collection item") + + ch <- export.Item{ + ID: id, + Error: err, + } + + continue } + case path.ContactsCategory: + outData, err = vcf.FromJSON(ctx, content) + if err != nil { + err = clues.Wrap(err, "converting to vcf") - continue + logger.CtxErr(ctx, err).Info("processing collection item") + + ch <- export.Item{ + ID: id, + Error: err, + } + + continue + } } - emlReader := io.NopCloser(bytes.NewReader([]byte(email))) - body := metrics.ReaderWithStats(emlReader, path.EmailCategory, stats) + emlReader := io.NopCloser(bytes.NewReader([]byte(outData))) + body := metrics.ReaderWithStats(emlReader, category, stats) ch <- export.Item{ ID: id, diff --git a/src/internal/m365/service/exchange/export.go b/src/internal/m365/service/exchange/export.go index f5ba740e3..d9de3c554 100644 --- a/src/internal/m365/service/exchange/export.go +++ b/src/internal/m365/service/exchange/export.go @@ -62,9 +62,9 @@ func (h *baseExchangeHandler) ProduceExportCollections( category := dc.FullPath().Category() switch category { - case path.EmailCategory: + case path.ContactsCategory, path.EmailCategory: folders := dc.FullPath().Folders() - pth := path.Builder{}.Append(path.EmailCategory.HumanString()).Append(folders...) + pth := path.Builder{}.Append(category.HumanString()).Append(folders...) ec = append( ec, @@ -73,7 +73,7 @@ func (h *baseExchangeHandler) ProduceExportCollections( []data.RestoreCollection{dc}, backupVersion, stats)) - case path.EventsCategory, path.ContactsCategory: + case path.EventsCategory: logger.Ctx(ctx).With("category", category.String()).Debugw("Skipping restore for category") default: return nil, clues.NewWC(ctx, "data category not supported").