larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use recurrence timezone for ics exports (#5206)

Browse Source 
<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [x] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2024-02-10 14:20:38 +05:30 committed by GitHub
parent f0b8041c3f
commit 8502e1fee6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 3294 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Handle the case where an email or event cannot be retrieved from Exchange due to an `ErrorCorruptData` error. Corso will skip over the item but report it in the backup summary.
- Emails attached within other emails are now correctly exported
- Gracefully handle email and post attachments without name when exporting to eml
- Use correct timezone for event start and end times in Exchange exports (helps fix issues in relative recurrence patterns)
## [v0.19.0] (beta) - 2024-02-06

8
src/internal/converters/eml/eml_test.go
View File

@ -266,11 +266,11 @@ func (suite *EMLUnitSuite) TestConvert_eml_ics() {
assert.Equal(
t,
msg.GetCreatedDateTime().Format(ics.ICalDateTimeFormat),
msg.GetCreatedDateTime().Format(ics.ICalDateTimeFormatUTC),
event.GetProperty(ical.ComponentPropertyCreated).Value)
assert.Equal(
t,
msg.GetLastModifiedDateTime().Format(ics.ICalDateTimeFormat),
msg.GetLastModifiedDateTime().Format(ics.ICalDateTimeFormatUTC),
event.GetProperty(ical.ComponentPropertyLastModified).Value)
st, err := ics.GetUTCTime(
@ -285,11 +285,11 @@ func (suite *EMLUnitSuite) TestConvert_eml_ics() {
assert.Equal(
t,
st.Format(ics.ICalDateTimeFormat),
st.Format(ics.ICalDateTimeFormatUTC),
event.GetProperty(ical.ComponentPropertyDtStart).Value)
assert.Equal(
t,
et.Format(ics.ICalDateTimeFormat),
et.Format(ics.ICalDateTimeFormatUTC),
event.GetProperty(ical.ComponentPropertyDtEnd).Value)
tos := msg.GetToRecipients()

17
src/internal/converters/ics/consts.go
View File

@ -166,3 +166,20 @@ var GraphTimeZoneToTZ = map[string]string{
"Yukon Standard Time": "America/Whitehorse",
"tzone://Microsoft/Utc": "Etc/UTC",
}
// Map from alternatives to the canonical time zone name
// There mapping are currently generated by manually going on the
// values in the GraphTimeZoneToTZ which is not available in the tzdb
var CanonicalTimeZoneMap = map[string]string{
"Africa/Asmara": "Africa/Asmera",
"Asia/Calcutta": "Asia/Kolkata",
"Asia/Rangoon": "Asia/Yangon",
"Asia/Saigon": "Asia/Ho_Chi_Minh",
"Europe/Kiev": "Europe/Kyiv",
"Europe/Warsaw": "Europe/Warszawa",
"America/Buenos_Aires": "America/Argentina/Buenos_Aires",
"America/Godthab": "America/Nuuk",
// NOTE: "Atlantic/Raykjavik" missing in tzdb but is in MS list
"Etc/UTC": "UTC", // simplifying the time zone name
}

150
src/internal/converters/ics/ics.go
View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/converters/ics/tzdata"
"github.com/alcionai/corso/src/pkg/dttm"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/services/m365/api"
@ -32,8 +33,9 @@ import (
// TODO locations: https://github.com/alcionai/corso/issues/5003
const (
ICalDateTimeFormat = "20060102T150405Z"
ICalDateFormat = "20060102"
ICalDateTimeFormat = "20060102T150405"
ICalDateTimeFormatUTC = "20060102T150405Z"
ICalDateFormat = "20060102"
)
func keyValues(key, value string) *ics.KeyValues {
@ -173,6 +175,17 @@ func getRecurrencePattern(
recurComponents = append(recurComponents, "BYDAY="+prefix+strings.Join(dowComponents, ","))
}
// This is necessary to compute when weekly events recur
fdow := pat.GetFirstDayOfWeek()
if fdow != nil {
icalday, ok := GraphToICalDOW[fdow.String()]
if !ok {
return "", clues.NewWC(ctx, "unknown first day of week").With("day", fdow)
}
recurComponents = append(recurComponents, "WKST="+icalday)
}
rrange := recurrence.GetRangeEscaped()
if rrange != nil {
switch ptr.Val(rrange.GetTypeEscaped()) {
@ -196,7 +209,7 @@ func getRecurrencePattern(
return "", clues.WrapWC(ctx, err, "parsing end time")
}
recurComponents = append(recurComponents, "UNTIL="+endTime.Format(ICalDateTimeFormat))
recurComponents = append(recurComponents, "UNTIL="+endTime.Format(ICalDateTimeFormatUTC))
}
case models.NOEND_RECURRENCERANGETYPE:
// Nothing to do
@ -225,10 +238,15 @@ func FromEventable(ctx context.Context, event models.Eventable) (string, error)
cal := ics.NewCalendar()
cal.SetProductId("-//Alcion//Corso") // Does this have to be customizable?
err := addTimeZoneComponents(ctx, cal, event)
if err != nil {
return "", clues.Wrap(err, "adding timezone components")
}
id := ptr.Val(event.GetId())
iCalEvent := cal.AddEvent(id)
err := updateEventProperties(ctx, event, iCalEvent)
err = updateEventProperties(ctx, event, iCalEvent)
if err != nil {
return "", clues.Wrap(err, "updating event properties")
}
@ -259,7 +277,7 @@ func FromEventable(ctx context.Context, event models.Eventable) (string, error)
exICalEvent := cal.AddEvent(id)
start := exception.GetOriginalStart() // will always be in UTC
exICalEvent.AddProperty(ics.ComponentProperty(ics.PropertyRecurrenceId), start.Format(ICalDateTimeFormat))
exICalEvent.AddProperty(ics.ComponentProperty(ics.PropertyRecurrenceId), start.Format(ICalDateTimeFormatUTC))
err = updateEventProperties(ctx, exception, exICalEvent)
if err != nil {
@ -270,6 +288,91 @@ func FromEventable(ctx context.Context, event models.Eventable) (string, error)
return cal.Serialize(), nil
}
func getTZDataKeyValues(ctx context.Context, timezone string) (map[string]string, error) {
template, ok := tzdata.TZData[timezone]
if !ok {
return nil, clues.NewWC(ctx, "timezone not found in tz database").
With("timezone", timezone)
}
keyValues := map[string]string{}
for _, line := range strings.Split(template, "\n") {
splits := strings.SplitN(line, ":", 2)
if len(splits) != 2 {
return nil, clues.NewWC(ctx, "invalid tzdata line").
With("line", line).
With("timezone", timezone)
}
keyValues[splits[0]] = splits[1]
}
return keyValues, nil
}
func addTimeZoneComponents(ctx context.Context, cal *ics.Calendar, event models.Eventable) error {
// Handling of timezone get a bit tricky when we have to deal with
// relative recurrence. The issue comes up when we set a recurrence
// to be something like "repeat every 3rd Tuesday". Tuesday in UTC
// and in IST will be different and so we cannot just always use UTC.
//
// The way this is solved is by using the timezone in the
// recurrence for start and end timezones as we have to use UTC
// for UNTIL(mostly).
// https://www.rfc-editor.org/rfc/rfc5545#section-3.3.10
timezone, err := getRecurrenceTimezone(ctx, event)
if err != nil {
return clues.Stack(err)
}
if timezone != time.UTC {
kvs, err := getTZDataKeyValues(ctx, timezone.String())
if err != nil {
return clues.Stack(err)
}
tz := cal.AddTimezone(timezone.String())
for k, v := range kvs {
tz.AddProperty(ics.ComponentProperty(k), v)
}
}
return nil
}
// getRecurrenceTimezone get the timezone specified by the recurrence
// in the calendar. It does a normalization pass where we always convert
// the timezone to the value in tzdb If we don't have a recurrence
// timezone, we don't have to use a specific timezone in the export and
// is safe to return UTC from this method.
func getRecurrenceTimezone(ctx context.Context, event models.Eventable) (*time.Location, error) {
if event.GetRecurrence() != nil {
timezone := ptr.Val(event.GetRecurrence().GetRangeEscaped().GetRecurrenceTimeZone())
ctz, ok := GraphTimeZoneToTZ[timezone]
if ok {
timezone = ctz
}
cannon, ok := CanonicalTimeZoneMap[timezone]
if ok {
timezone = cannon
}
loc, err := time.LoadLocation(timezone)
if err != nil {
return nil, clues.WrapWC(ctx, err, "unknown timezone").
With("timezone", timezone)
}
return loc, nil
}
return time.UTC, nil
}
func isASCII(s string) bool {
for _, c := range s {
if c > unicode.MaxASCII {
@ -299,6 +402,11 @@ func updateEventProperties(ctx context.Context, event models.Eventable, iCalEven
iCalEvent.SetModifiedAt(ptr.Val(modified))
}
timezone, err := getRecurrenceTimezone(ctx, event)
if err != nil {
return err
}
// DTSTART - https://www.rfc-editor.org/rfc/rfc5545#section-3.8.2.4
allDay := ptr.Val(event.GetIsAllDay())
startString := event.GetStart().GetDateTime()
@ -310,11 +418,7 @@ func updateEventProperties(ctx context.Context, event models.Eventable, iCalEven
return clues.WrapWC(ctx, err, "parsing start time")
}
if allDay {
iCalEvent.SetStartAt(start, ics.WithValue(string(ics.ValueDataTypeDate)))
} else {
iCalEvent.SetStartAt(start)
}
addTime(iCalEvent, ics.ComponentPropertyDtStart, start, allDay, timezone)
}
// DTEND - https://www.rfc-editor.org/rfc/rfc5545#section-3.8.2.2
@ -327,11 +431,7 @@ func updateEventProperties(ctx context.Context, event models.Eventable, iCalEven
return clues.WrapWC(ctx, err, "parsing end time")
}
if allDay {
iCalEvent.SetEndAt(end, ics.WithValue(string(ics.ValueDataTypeDate)))
} else {
iCalEvent.SetEndAt(end)
}
addTime(iCalEvent, ics.ComponentPropertyDtEnd, end, allDay, timezone)
}
recurrence := event.GetRecurrence()
@ -630,6 +730,26 @@ func updateEventProperties(ctx context.Context, event models.Eventable, iCalEven
return nil
}
func addTime(iCalEvent *ics.VEvent, prop ics.ComponentProperty, tm time.Time, allDay bool, tzLoc *time.Location) {
if allDay {
if tzLoc == time.UTC {
iCalEvent.SetProperty(prop, tm.Format(ICalDateFormat), ics.WithValue(string(ics.ValueDataTypeDate)))
} else {
iCalEvent.SetProperty(
prop,
tm.In(tzLoc).Format(ICalDateFormat),
ics.WithValue(string(ics.ValueDataTypeDate)),
keyValues("TZID", tzLoc.String()))
}
} else {
if tzLoc == time.UTC {
iCalEvent.SetProperty(prop, tm.Format(ICalDateTimeFormatUTC))
} else {
iCalEvent.SetProperty(prop, tm.In(tzLoc).Format(ICalDateTimeFormat), keyValues("TZID", tzLoc.String()))
}
}
}
func getCancelledDates(ctx context.Context, event models.Eventable) ([]time.Time, error) {
dateStrings, err := api.GetCancelledEventDateStrings(event)
if err != nil {

381
src/internal/converters/ics/ics_test.go
View File

@ -13,6 +13,7 @@ import (
"testing"
"time"
ics "github.com/arran4/golang-ical"
"github.com/microsoft/kiota-abstractions-go/serialization"
kjson "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -21,6 +22,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/converters/ics/tzdata"
"github.com/alcionai/corso/src/internal/tester"
)
@ -32,7 +34,7 @@ func TestICSUnitSuite(t *testing.T) {
suite.Run(t, &ICSUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ICSUnitSuite) TestGetLocationString() {
func (s *ICSUnitSuite) TestGetLocationString() {
table := []struct {
name string
loc func() models.Locationable
@ -110,13 +112,13 @@ func (suite *ICSUnitSuite) TestGetLocationString() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
assert.Equal(suite.T(), tt.expect, getLocationString(tt.loc()))
s.Run(tt.name, func() {
assert.Equal(s.T(), tt.expect, getLocationString(tt.loc()))
})
}
}
func (suite *ICSUnitSuite) TestGetUTCTime() {
func (s *ICSUnitSuite) TestGetUTCTime() {
table := []struct {
name string
timestamp string
@ -162,18 +164,18 @@ func (suite *ICSUnitSuite) TestGetUTCTime() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
s.Run(tt.name, func() {
t, err := GetUTCTime(tt.timestamp, tt.timezone)
tt.errCheck(suite.T(), err)
tt.errCheck(s.T(), err)
if !tt.time.Equal(time.Time{}) {
assert.Equal(suite.T(), tt.time, t)
assert.Equal(s.T(), tt.time, t)
}
})
}
}
func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
func (s *ICSUnitSuite) TestGetRecurrencePattern() {
table := []struct {
name string
recurrence func() models.PatternedRecurrenceable
@ -187,16 +189,37 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("daily")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rec.SetPattern(pat)
return rec
},
expect: "FREQ=DAILY;INTERVAL=1",
expect: "FREQ=DAILY;INTERVAL=1;WKST=SU",
errCheck: require.NoError,
},
{
name: "daily different start of week",
recurrence: func() models.PatternedRecurrenceable {
rec := models.NewPatternedRecurrence()
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("daily")
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.MONDAY_DAYOFWEEK))
rec.SetPattern(pat)
return rec
},
expect: "FREQ=DAILY;INTERVAL=1;WKST=MO",
errCheck: require.NoError,
},
{
@ -206,15 +229,16 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("daily")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rng := models.NewRecurrenceRange()
rrtype, err := models.ParseRecurrenceRangeType("endDate")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
rng.SetTypeEscaped(rrtype.(*models.RecurrenceRangeType))
@ -227,7 +251,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=DAILY;INTERVAL=1;UNTIL=20210101T182959Z",
expect: "FREQ=DAILY;INTERVAL=1;WKST=SU;UNTIL=20210101T182959Z",
errCheck: require.NoError,
},
{
@ -237,16 +261,17 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("weekly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rec.SetPattern(pat)
return rec
},
expect: "FREQ=WEEKLY;INTERVAL=1",
expect: "FREQ=WEEKLY;INTERVAL=1;WKST=SU",
errCheck: require.NoError,
},
{
@ -256,15 +281,16 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("weekly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rng := models.NewRecurrenceRange()
rrtype, err := models.ParseRecurrenceRangeType("endDate")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
rng.SetTypeEscaped(rrtype.(*models.RecurrenceRangeType))
@ -277,7 +303,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=WEEKLY;INTERVAL=1;UNTIL=20210101T235959Z",
expect: "FREQ=WEEKLY;INTERVAL=1;WKST=SU;UNTIL=20210101T235959Z",
errCheck: require.NoError,
},
{
@ -287,15 +313,16 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("weekly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rng := models.NewRecurrenceRange()
rrtype, err := models.ParseRecurrenceRangeType("numbered")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
rng.SetTypeEscaped(rrtype.(*models.RecurrenceRangeType))
@ -307,7 +334,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=WEEKLY;INTERVAL=1;COUNT=10",
expect: "FREQ=WEEKLY;INTERVAL=1;WKST=SU;COUNT=10",
errCheck: require.NoError,
},
{
@ -317,10 +344,11 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("weekly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
days := []models.DayOfWeek{
models.MONDAY_DAYOFWEEK,
@ -334,7 +362,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=WEEKLY;INTERVAL=1;BYDAY=MO,WE,TH",
expect: "FREQ=WEEKLY;INTERVAL=1;BYDAY=MO,WE,TH;WKST=SU",
errCheck: require.NoError,
},
{
@ -344,16 +372,17 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("daily")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(2)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rec.SetPattern(pat)
return rec
},
expect: "FREQ=DAILY;INTERVAL=2",
expect: "FREQ=DAILY;INTERVAL=2;WKST=SU",
errCheck: require.NoError,
},
{
@ -363,10 +392,11 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("absoluteMonthly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
pat.SetDayOfMonth(ptr.To(int32(5)))
@ -374,7 +404,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=5",
expect: "FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=5;WKST=SU",
errCheck: require.NoError,
},
{
@ -384,10 +414,11 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("absoluteYearly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(3)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
pat.SetMonth(ptr.To(int32(8)))
@ -395,7 +426,7 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
return rec
},
expect: "FREQ=YEARLY;INTERVAL=3;BYMONTH=8",
expect: "FREQ=YEARLY;INTERVAL=3;BYMONTH=8;WKST=SU",
errCheck: require.NoError,
},
{
@ -405,37 +436,38 @@ func (suite *ICSUnitSuite) TestGetRecurrencePattern() {
pat := models.NewRecurrencePattern()
typ, err := models.ParseRecurrencePatternType("relativeYearly")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
pat.SetMonth(ptr.To(int32(8)))
pat.SetDaysOfWeek([]models.DayOfWeek{models.FRIDAY_DAYOFWEEK})
wi, err := models.ParseWeekIndex("first")
require.NoError(suite.T(), err)
require.NoError(s.T(), err)
pat.SetIndex(wi.(*models.WeekIndex))
rec.SetPattern(pat)
return rec
},
expect: "FREQ=YEARLY;INTERVAL=1;BYMONTH=8;BYDAY=1FR",
expect: "FREQ=YEARLY;INTERVAL=1;BYMONTH=8;BYDAY=1FR;WKST=SU",
errCheck: require.NoError,
},
// TODO(meain): could still use more tests for edge cases of time
}
for _, tt := range table {
suite.Run(tt.name, func() {
ctx, flush := tester.NewContext(suite.T())
s.Run(tt.name, func() {
ctx, flush := tester.NewContext(s.T())
defer flush()
rec, err := getRecurrencePattern(ctx, tt.recurrence())
tt.errCheck(suite.T(), err)
tt.errCheck(s.T(), err)
assert.Equal(suite.T(), tt.expect, rec)
assert.Equal(s.T(), tt.expect, rec)
})
}
}
@ -460,8 +492,8 @@ func baseEvent() *models.Event {
return e
}
func (suite *ICSUnitSuite) TestEventConversion() {
t := suite.T()
func (s *ICSUnitSuite) TestEventConversion() {
t := s.T()
table := []struct {
name string
@ -546,14 +578,19 @@ func (suite *ICSUnitSuite) TestEventConversion() {
rec := models.NewPatternedRecurrence()
pat := models.NewRecurrencePattern()
rng := models.NewRecurrenceRange()
typ, err := models.ParseRecurrencePatternType("daily")
require.NoError(t, err)
pat.SetTypeEscaped(typ.(*models.RecurrencePatternType))
pat.SetInterval(ptr.To(int32(1)))
pat.SetFirstDayOfWeek(ptr.To(models.SUNDAY_DAYOFWEEK))
rng.SetRecurrenceTimeZone(ptr.To("UTC"))
rec.SetPattern(pat)
rec.SetRangeEscaped(rng)
e.SetRecurrence(rec)
@ -830,8 +867,8 @@ func (suite *ICSUnitSuite) TestEventConversion() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
t := suite.T()
s.Run(tt.name, func() {
t := s.T()
ctx, flush := tester.NewContext(t)
defer flush()
@ -881,8 +918,8 @@ func checkAttendee(t *testing.T, out, check, msg string) {
assert.ElementsMatch(t, as, bs, fmt.Sprintf("fields %s", msg))
}
func (suite *ICSUnitSuite) TestAttendees() {
t := suite.T()
func (s *ICSUnitSuite) TestAttendees() {
t := s.T()
table := []struct {
name string
@ -949,8 +986,8 @@ func (suite *ICSUnitSuite) TestAttendees() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
t := suite.T()
s.Run(tt.name, func() {
t := s.T()
ctx, flush := tester.NewContext(t)
defer flush()
@ -1071,8 +1108,8 @@ func checkAttachment(t *testing.T, out, check, msg string) {
assert.ElementsMatch(t, as, bs, fmt.Sprintf("fields %s", msg))
}
func (suite *ICSUnitSuite) TestAttachments() {
t := suite.T()
func (s *ICSUnitSuite) TestAttachments() {
t := s.T()
type attachment struct {
cid string // contentid
@ -1128,8 +1165,8 @@ func (suite *ICSUnitSuite) TestAttachments() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
t := suite.T()
s.Run(tt.name, func() {
t := s.T()
ctx, flush := tester.NewContext(t)
defer flush()
@ -1172,7 +1209,7 @@ func (suite *ICSUnitSuite) TestAttachments() {
}
}
func (suite *ICSUnitSuite) TestCancellations() {
func (s *ICSUnitSuite) TestCancellations() {
table := []struct {
name string
cancelledIds []string
@ -1196,8 +1233,8 @@ func (suite *ICSUnitSuite) TestCancellations() {
}
for _, tt := range table {
suite.Run(tt.name, func() {
t := suite.T()
s.Run(tt.name, func() {
t := s.T()
ctx, flush := tester.NewContext(t)
defer flush()
@ -1260,7 +1297,7 @@ func eventToJSON(e *models.Event) ([]byte, error) {
return bts, err
}
func (suite *ICSUnitSuite) TestEventExceptions() {
func (s *ICSUnitSuite) TestEventExceptions() {
table := []struct {
name string
event func() *models.Event
@ -1282,7 +1319,7 @@ func (suite *ICSUnitSuite) TestEventExceptions() {
exception.SetEnd(newEnd)
parsed, err := eventToMap(exception)
require.NoError(suite.T(), err, "parsing exception")
require.NoError(s.T(), err, "parsing exception")
// add exception event to additional data
e.SetAdditionalData(map[string]any{
@ -1301,15 +1338,15 @@ func (suite *ICSUnitSuite) TestEventExceptions() {
}
}
assert.Equal(suite.T(), 2, events, "number of events")
assert.Equal(s.T(), 2, events, "number of events")
assert.Contains(suite.T(), out, "RECURRENCE-ID:20210101T120000Z", "recurrence id")
assert.Contains(s.T(), out, "RECURRENCE-ID:20210101T120000Z", "recurrence id")
assert.Contains(suite.T(), out, "SUMMARY:Subject", "original event")
assert.Contains(suite.T(), out, "SUMMARY:Exception", "exception event")
assert.Contains(s.T(), out, "SUMMARY:Subject", "original event")
assert.Contains(s.T(), out, "SUMMARY:Exception", "exception event")
assert.Contains(suite.T(), out, "DTSTART:20210101T130000Z", "new start time")
assert.Contains(suite.T(), out, "DTEND:20210101T140000Z", "new end time")
assert.Contains(s.T(), out, "DTSTART:20210101T130000Z", "new start time")
assert.Contains(s.T(), out, "DTEND:20210101T140000Z", "new end time")
},
},
{
@ -1338,10 +1375,10 @@ func (suite *ICSUnitSuite) TestEventExceptions() {
exception2.SetEnd(newEnd)
parsed1, err := eventToMap(exception1)
require.NoError(suite.T(), err, "parsing exception 1")
require.NoError(s.T(), err, "parsing exception 1")
parsed2, err := eventToMap(exception2)
require.NoError(suite.T(), err, "parsing exception 2")
require.NoError(s.T(), err, "parsing exception 2")
// add exception event to additional data
e.SetAdditionalData(map[string]any{
@ -1360,36 +1397,230 @@ func (suite *ICSUnitSuite) TestEventExceptions() {
}
}
assert.Equal(suite.T(), 3, events, "number of events")
assert.Equal(s.T(), 3, events, "number of events")
assert.Contains(suite.T(), out, "RECURRENCE-ID:20210101T120000Z", "recurrence id 1")
assert.Contains(suite.T(), out, "RECURRENCE-ID:20210102T120000Z", "recurrence id 2")
assert.Contains(s.T(), out, "RECURRENCE-ID:20210101T120000Z", "recurrence id 1")
assert.Contains(s.T(), out, "RECURRENCE-ID:20210102T120000Z", "recurrence id 2")
assert.Contains(suite.T(), out, "SUMMARY:Subject", "original event")
assert.Contains(suite.T(), out, "SUMMARY:Exception 1", "exception event 1")
assert.Contains(suite.T(), out, "SUMMARY:Exception 2", "exception event 2")
assert.Contains(s.T(), out, "SUMMARY:Subject", "original event")
assert.Contains(s.T(), out, "SUMMARY:Exception 1", "exception event 1")
assert.Contains(s.T(), out, "SUMMARY:Exception 2", "exception event 2")
assert.Contains(suite.T(), out, "DTSTART:20210101T130000Z", "new start time 1")
assert.Contains(suite.T(), out, "DTEND:20210101T140000Z", "new end time 1")
assert.Contains(s.T(), out, "DTSTART:20210101T130000Z", "new start time 1")
assert.Contains(s.T(), out, "DTEND:20210101T140000Z", "new end time 1")
assert.Contains(suite.T(), out, "DTSTART:20210102T130000Z", "new start time 2")
assert.Contains(suite.T(), out, "DTEND:20210102T140000Z", "new end time 2")
assert.Contains(s.T(), out, "DTSTART:20210102T130000Z", "new start time 2")
assert.Contains(s.T(), out, "DTEND:20210102T140000Z", "new end time 2")
},
},
}
for _, tt := range table {
suite.Run(tt.name, func() {
ctx, flush := tester.NewContext(suite.T())
s.Run(tt.name, func() {
ctx, flush := tester.NewContext(s.T())
defer flush()
bts, err := eventToJSON(tt.event())
require.NoError(suite.T(), err, "getting serialized content")
require.NoError(s.T(), err, "getting serialized content")
out, err := FromJSON(ctx, bts)
require.NoError(suite.T(), err, "converting to ics")
require.NoError(s.T(), err, "converting to ics")
tt.check(out)
})
}
}
func (s *ICSUnitSuite) TestGetRecurrenceTimezone() {
table := []struct {
name string
intz string
outtz string
}{
{
name: "empty",
intz: "",
outtz: "UTC",
},
{
name: "utc",
intz: "UTC",
outtz: "UTC",
},
{
name: "simple",
intz: "Asia/Kolkata",
outtz: "Asia/Kolkata",
},
{
name: "windows tz",
intz: "India Standard Time",
outtz: "Asia/Kolkata",
},
{
name: "non canonical",
intz: "Asia/Calcutta",
outtz: "Asia/Kolkata",
},
}
for _, tt := range table {
s.Run(tt.name, func() {
ctx, flush := tester.NewContext(s.T())
defer flush()
event := baseEvent()
if len(tt.intz) > 0 {
recur := models.NewPatternedRecurrence()
rp := models.NewRecurrenceRange()
rp.SetRecurrenceTimeZone(ptr.To(tt.intz))
recur.SetRangeEscaped(rp)
event.SetRecurrence(recur)
}
timezone, err := getRecurrenceTimezone(ctx, event)
require.NoError(s.T(), err)
assert.Equal(s.T(), tt.outtz, timezone.String())
})
}
}
func (s *ICSUnitSuite) TestAddTimezoneComponents() {
event := baseEvent()
recur := models.NewPatternedRecurrence()
rp := models.NewRecurrenceRange()
rp.SetRecurrenceTimeZone(ptr.To("Asia/Kolkata"))
recur.SetRangeEscaped(rp)
event.SetRecurrence(recur)
ctx, flush := tester.NewContext(s.T())
defer flush()
cal := ics.NewCalendar()
err := addTimeZoneComponents(ctx, cal, event)
require.NoError(s.T(), err)
text := cal.Serialize()
assert.Contains(s.T(), text, "BEGIN:VTIMEZONE", "beginning of timezone")
assert.Contains(s.T(), text, "TZID:Asia/Kolkata", "timezone id")
assert.Contains(s.T(), text, "END:VTIMEZONE", "end of timezone")
}
func (s *ICSUnitSuite) TestAddTime() {
locak, err := time.LoadLocation("Asia/Kolkata")
require.NoError(s.T(), err)
table := []struct {
name string
prop ics.ComponentProperty
time time.Time
allDay bool
loc *time.Location
exp string
}{
{
name: "utc",
prop: ics.ComponentPropertyDtStart,
time: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC),
allDay: false,
loc: time.UTC,
exp: "DTSTART:20210102T030405Z",
},
{
name: "local",
prop: ics.ComponentPropertyDtStart,
time: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC),
allDay: false,
loc: locak,
exp: "DTSTART;TZID=Asia/Kolkata:20210102T083405",
},
{
name: "all day",
prop: ics.ComponentPropertyDtStart,
time: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC),
allDay: true,
loc: time.UTC,
exp: "DTSTART;VALUE=DATE:20210102",
},
{
name: "all day local",
prop: ics.ComponentPropertyDtStart,
time: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC),
allDay: true,
loc: locak,
exp: "DTSTART;VALUE=DATE;TZID=Asia/Kolkata:20210102",
},
{
name: "end",
prop: ics.ComponentPropertyDtEnd,
time: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC),
allDay: false,
loc: time.UTC,
exp: "DTEND:20210102T030405Z",
},
{
// This won't happen, but a good test to have to test loc handling
name: "windows tz",
prop: ics.ComponentPropertyDtStart,
time: time.Date(2021, 1, 2, 3, 4, 5, 0, time.UTC),
allDay: false,
loc: time.FixedZone("India Standard Time", 5*60*60+30*60),
exp: "DTSTART;TZID=India Standard Time:20210102T083405",
},
}
for _, tt := range table {
s.Run(tt.name, func() {
cal := ics.NewCalendar()
evt := cal.AddEvent("id")
addTime(evt, tt.prop, tt.time, tt.allDay, tt.loc)
expSplits := strings.FieldsFunc(tt.exp, func(c rune) bool {
return c == ':' || c == ';'
})
text := cal.Serialize()
checkLine := ""
for _, l := range strings.Split(text, "\r\n") {
if strings.HasPrefix(l, string(tt.prop)) {
checkLine = l
break
}
}
actSplits := strings.FieldsFunc(checkLine, func(c rune) bool {
return c == ':' || c == ';'
})
assert.Greater(s.T(), len(checkLine), 0, "line not found")
assert.Equal(s.T(), len(expSplits), len(actSplits), "length of fields")
assert.ElementsMatch(s.T(), expSplits, actSplits, "fields")
})
}
}
// This tests and ensures that the generated data is int he format
// that we expect
func (s *ICSUnitSuite) TestGetTZDataKeyValues() {
for key := range tzdata.TZData {
s.Run(key, func() {
ctx, flush := tester.NewContext(s.T())
defer flush()
data, err := getTZDataKeyValues(ctx, key)
require.NoError(s.T(), err)
assert.NotEmpty(s.T(), data, "data")
assert.NotContains(s.T(), data, "BEGIN", "beginning of timezone") // should be stripped
assert.NotContains(s.T(), data, "END", "end of timezone") // should be stripped
assert.NotContains(s.T(), data, "TZID", "timezone id") // should be stripped
assert.Contains(s.T(), data, "DTSTART", "start time")
assert.Contains(s.T(), data, "TZOFFSETFROM", "offset from")
})
}
}

2796
src/internal/converters/ics/tzdata/data.go Normal file
View File

File diff suppressed because it is too large Load Diff

35
src/internal/converters/ics/tzdata/fetch.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/sh
set -eo pipefail
if ! echo "$PWD" | grep -q '/tzdata$'; then
echo "Please run this script from the tzdata dir"
exit 1
fi
# TODO: Generate from https://www.iana.org/time-zones
if [ ! -d /tmp/corso-tzdata ]; then
git clone --depth 1 https://github.com/add2cal/timezones-ical-library.git /tmp/corso-tzdata
else
cd /tmp/corso-tzdata
git pull
cd -
fi
# Generate a huge go file with all the timezones
echo "package tzdata" >data.go
echo "" >>data.go
echo "var TZData = map[string]string{" >>data.go
find /tmp/corso-tzdata/ -name '*.ics' | while read -r f; do
tz=$(echo "$f" | sed 's|/tmp/corso-tzdata/api/||;s|\.ics$||')
echo "Processing $tz"
printf "\t\"%s\": \`" "$tz" >>data.go
cat "$f" | grep -Ev "(BEGIN:|END:|TZID:)" |
sed 's|`|\\`|g;s|\r||;s|TZID:/timezones-ical-library/|TZID:|' |
perl -pe 'chomp if eof' >>data.go
echo "\`," >>data.go
done
echo "}" >>data.go