From 9199db8f458fd4db5285d05bb4d720d9905628d4 Mon Sep 17 00:00:00 2001 From: Abin Simon Date: Mon, 19 Jun 2023 17:27:32 +0530 Subject: [PATCH] Handle event exceptions in caledar (#3589) This ensures that we backup and restore and exceptions in the recurring events. One thing pending here is fixing up the attachments. I hope to create that as a separate PR. This can probably go in separately as well. --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included - [ ] :clock1: Yes, but in a later PR - [ ] :no_entry: No #### Type of change - [x] :sunflower: Feature - [ ] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Supportability/Tests - [ ] :computer: CI/Deployment - [ ] :broom: Tech Debt/Cleanup #### Issue(s) * https://github.com/alcionai/corso/issues/2835 #### Test Plan - [ ] :muscle: Manual - [x] :zap: Unit test - [x] :green_heart: E2E --- CHANGELOG.md | 4 + src/cmd/factory/impl/exchange.go | 5 +- src/internal/m365/exchange/events_restore.go | 184 +++++++++++- src/internal/m365/exchange/mock/event.go | 114 +++++++- src/internal/m365/exchange/restore_test.go | 8 + src/internal/m365/exchange/transform.go | 9 +- src/internal/m365/exchange/transform_test.go | 26 ++ .../operations/backup_integration_test.go | 5 +- src/pkg/services/m365/api/events.go | 274 ++++++++++++++++-- website/docs/support/known-issues.md | 2 + 10 files changed, 580 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d528b18c8..49b6eae30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ProtectedResourceName to the backup list json output. ProtectedResourceName holds either a UPN or a WebURL, depending on the resource type. - Rework base selection logic for incremental backups so it's more likely to find a valid base. - Improve OneDrive restore performance by paralleling item restores +- Exceptions and cancellations for recurring events are now backed up and restored ### Fixed - Fix Exchange folder cache population error when parent folder isn't found. @@ -23,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Do not display all the items that we restored at the end if there are more than 15. You can override this with `--verbose`. +### Known Issues +- Changes to attachments in instances of recurring events compared to the series master aren't restored + ## [v0.8.0] (beta) - 2023-05-15 ### Added diff --git a/src/cmd/factory/impl/exchange.go b/src/cmd/factory/impl/exchange.go index dd304e2e9..2c7f0f4e3 100644 --- a/src/cmd/factory/impl/exchange.go +++ b/src/cmd/factory/impl/exchange.go @@ -114,7 +114,10 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error func(id, now, subject, body string) []byte { return exchMock.EventWith( User, subject, body, body, - now, now, exchMock.NoRecurrence, exchMock.NoAttendees, false) + exchMock.NoOriginalStartDate, now, now, + exchMock.NoRecurrence, exchMock.NoAttendees, + false, exchMock.NoCancelledOccurrences, + exchMock.NoExceptionOccurrences) }, control.Defaults(), errs) diff --git a/src/internal/m365/exchange/events_restore.go b/src/internal/m365/exchange/events_restore.go index 18540ecaf..fcb1273a0 100644 --- a/src/internal/m365/exchange/events_restore.go +++ b/src/internal/m365/exchange/events_restore.go @@ -2,11 +2,16 @@ package exchange import ( "context" + "fmt" + "strings" + "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -82,12 +87,14 @@ func (h eventRestoreHandler) restore( if ptr.Val(event.GetHasAttachments()) { attachments = event.GetAttachments() - event.SetAttachments([]models.Attachmentable{}) + // We cannot use `[]models.Attbachmentable{}` instead of nil + // for beta endpoint. + event.SetAttachments(nil) } item, err := h.ip.PostItem(ctx, userID, destinationID, event) if err != nil { - return nil, graph.Wrap(ctx, err, "restoring mail message") + return nil, graph.Wrap(ctx, err, "restoring calendar item") } err = uploadAttachments( @@ -102,8 +109,181 @@ func (h eventRestoreHandler) restore( return nil, clues.Stack(err) } + // Have to parse event again as we modified the original event and + // removed cancelled and exceptions events form it + event, err = api.BytesToEventable(body) + if err != nil { + return nil, clues.Wrap(err, "creating event from bytes").WithClues(ctx) + } + + // Fix up event instances in case we have a recurring event + err = updateRecurringEvents(ctx, h.ac, userID, destinationID, ptr.Val(item.GetId()), event) + if err != nil { + return nil, clues.Stack(err) + } + info := api.EventInfo(event) info.Size = int64(len(body)) return info, nil } + +func updateRecurringEvents( + ctx context.Context, + ac api.Events, + userID, containerID, itemID string, + event models.Eventable, +) error { + if event.GetRecurrence() == nil { + return nil + } + + // Cancellations and exceptions are currently in additional data + // but will get their own fields once the beta API lands and + // should be moved then + cancelledOccurrences := event.GetAdditionalData()["cancelledOccurrences"] + exceptionOccurrences := event.GetAdditionalData()["exceptionOccurrences"] + + err := updateCancelledOccurrences(ctx, ac, userID, itemID, cancelledOccurrences) + if err != nil { + return clues.Wrap(err, "update cancelled occurrences") + } + + err = updateExceptionOccurrences(ctx, ac, userID, itemID, exceptionOccurrences) + if err != nil { + return clues.Wrap(err, "update exception occurrences") + } + + return nil +} + +// updateExceptionOccurrences take events that have exceptions, uses +// the originalStart date to find the instance and modify it to match +// the backup by updating the instance to match the backed up one +func updateExceptionOccurrences( + ctx context.Context, + ac api.Events, + userID string, + itemID string, + exceptionOccurrences any, +) error { + if exceptionOccurrences == nil { + return nil + } + + eo, ok := exceptionOccurrences.([]any) + if !ok { + return clues.New("converting exceptionOccurrences to []any"). + With("type", fmt.Sprintf("%T", exceptionOccurrences)) + } + + for _, instance := range eo { + instance, ok := instance.(map[string]any) + if !ok { + return clues.New("converting instance to map[string]any"). + With("type", fmt.Sprintf("%T", instance)) + } + + evt, err := api.EventFromMap(instance) + if err != nil { + return clues.Wrap(err, "parsing exception event") + } + + start := ptr.Val(evt.GetOriginalStart()) + startStr := dttm.FormatTo(start, dttm.DateOnly) + endStr := dttm.FormatTo(start.Add(24*time.Hour), dttm.DateOnly) + + // Get all instances on the day of the instance which should + // just the one we need to modify + evts, err := ac.GetItemInstances(ctx, userID, itemID, startStr, endStr) + if err != nil { + return clues.Wrap(err, "getting instances") + } + + // Since the min recurrence interval is 1 day and we are + // querying for only a single day worth of instances, we + // should not have more than one instance here. + if len(evts) != 1 { + return clues.New("invalid number of instances for modified"). + With("instances_count", len(evts), "search_start", startStr, "search_end", endStr) + } + + evt = toEventSimplified(evt) + + // TODO(meain): Update attachments (might have to diff the + // attachments using ids and delete or add). We will have + // to get the id of the existing attachments, diff them + // with what we need a then create/delete items kinda like + // permissions + _, err = ac.PatchItem(ctx, userID, ptr.Val(evts[0].GetId()), evt) + if err != nil { + return clues.Wrap(err, "updating event instance") + } + } + + return nil +} + +// updateCancelledOccurrences get the cancelled occurrences which is a +// list of strings of the format ".", parses the date out of +// that and uses the to get the event instance at that date to delete. +func updateCancelledOccurrences( + ctx context.Context, + ac api.Events, + userID string, + itemID string, + cancelledOccurrences any, +) error { + if cancelledOccurrences == nil { + return nil + } + + co, ok := cancelledOccurrences.([]any) + if !ok { + return clues.New("converting cancelledOccurrences to []any"). + With("type", fmt.Sprintf("%T", cancelledOccurrences)) + } + + // OPTIMIZATION: We can fetch a date range instead of fetching + // instances if we have multiple cancelled events which are nearby + // and reduce the number of API calls that we have to make + for _, instance := range co { + instance, err := str.AnyToString(instance) + if err != nil { + return err + } + + splits := strings.Split(instance, ".") + + startStr := splits[len(splits)-1] + + start, err := dttm.ParseTime(startStr) + if err != nil { + return clues.Wrap(err, "parsing cancelled event date") + } + + endStr := dttm.FormatTo(start.Add(24*time.Hour), dttm.DateOnly) + + // Get all instances on the day of the instance which should + // just the one we need to modify + evts, err := ac.GetItemInstances(ctx, userID, itemID, startStr, endStr) + if err != nil { + return clues.Wrap(err, "getting instances") + } + + // Since the min recurrence interval is 1 day and we are + // querying for only a single day worth of instances, we + // should not have more than one instance here. + if len(evts) != 1 { + return clues.New("invalid number of instances for cancelled"). + With("instances_count", len(evts), "search_start", startStr, "search_end", endStr) + } + + err = ac.DeleteItem(ctx, userID, ptr.Val(evts[0].GetId())) + if err != nil { + return clues.Wrap(err, "deleting event instance") + } + } + + return nil +} diff --git a/src/internal/m365/exchange/mock/event.go b/src/internal/m365/exchange/mock/event.go index 7df667af6..d5711b7ce 100644 --- a/src/internal/m365/exchange/mock/event.go +++ b/src/internal/m365/exchange/mock/event.go @@ -25,11 +25,6 @@ import ( //nolint:lll const ( eventTmpl = `{ - "id":"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAADSEBNbUIB9RL6ePDeF3FIYAAAAAG76AAA=", - "calendar@odata.navigationLink":"https://graph.microsoft.com/v1.0/users('foobar@8qzvrj.onmicrosoft.com')/calendars('AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAAA=')", - "calendar@odata.associationLink":"https://graph.microsoft.com/v1.0/users('foobar@8qzvrj.onmicrosoft.com')/calendars('AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9QmQYWNcI7hCpPrAQDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAAA=')/$ref", - "@odata.etag":"W/\"0hATW1CAfUS+njw3hdxSGAAAJIxNug==\"", - "@odata.context":"https://graph.microsoft.com/v1.0/$metadata#users('foobar%%408qzvrj.onmicrosoft.com')/events/$entity", "categories":[], "changeKey":"0hATW1CAfUS+njw3hdxSGAAAJIxNug==", "createdDateTime":"2022-03-28T03:42:03Z", @@ -46,7 +41,6 @@ const ( "timeZone":"UTC" }, "hideAttendees":false, - "iCalUId":"040000008200E00074C5B7101A82E0080000000035723BC75542D801000000000000000010000000E1E7C8F785242E4894DA13AEFB947B85", "importance":"normal", "isAllDay":false, "isCancelled":false, @@ -75,6 +69,7 @@ const ( "name":"Anu Pierson" } }, + %s "originalEndTimeZone":"UTC", "originalStartTimeZone":"UTC", "reminderMinutesBeforeStart":15, @@ -90,11 +85,13 @@ const ( "timeZone":"UTC" }, "subject":"%s", - "type":"singleInstance", + "type":"%s", "hasAttachments":%v, %s "webLink":"https://outlook.office365.com/owa/?itemid=AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAENAADSEBNbUIB9RL6ePDeF3FIYAAAAAG76AAA%%3D&exvsurl=1&path=/calendar/item", "recurrence":%s, + %s + %s "attendees":%s }` @@ -151,13 +148,16 @@ const ( "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}]," + originalStartDateFormat = `"originalStart": "%s",` + NoOriginalStartDate = `` + NoRecurrence = `null` recurrenceTmpl = `{ "pattern": { "type": "absoluteYearly", "interval": 1, - "month": 1, - "dayOfMonth": 1, + "month": %s, + "dayOfMonth": %s, "firstDayOfWeek": "sunday", "index": "first" }, @@ -170,6 +170,13 @@ const ( } }` + cancelledOccurrencesFormat = `"cancelledOccurrences": [%s],` + cancelledOccurrenceInstanceFormat = `"OID.AAMkAGJiZmE2NGU4LTQ4YjktNDI1Mi1iMWQzLTQ1MmMxODJkZmQyNABGAAAAAABFdiK7oifWRb4ADuqgSRcnBwBBFDg0JJk7TY1fmsJrh7tNAAAAAAENAABBFDg0JJk7TY1fmsJrh7tNAADHGTZoAAA=.%s"` + NoCancelledOccurrences = "" + + exceptionOccurrencesFormat = `"exceptionOccurrences": [%s],` + NoExceptionOccurrences = "" + NoAttendees = `[]` attendeesTmpl = `[{ "emailAddress": { @@ -227,7 +234,8 @@ func EventWithSubjectBytes(subject string) []byte { return EventWith( defaultEventOrganizer, subject, defaultEventBody, defaultEventBodyPreview, - atTime, endTime, NoRecurrence, NoAttendees, false, + NoOriginalStartDate, atTime, endTime, NoRecurrence, NoAttendees, + false, NoCancelledOccurrences, NoExceptionOccurrences, ) } @@ -239,7 +247,8 @@ func EventWithAttachment(subject string) []byte { return EventWith( defaultEventOrganizer, subject, defaultEventBody, defaultEventBodyPreview, - atTime, atTime, NoRecurrence, NoAttendees, true, + NoOriginalStartDate, atTime, atTime, NoRecurrence, NoAttendees, + true, NoCancelledOccurrences, NoExceptionOccurrences, ) } @@ -251,6 +260,8 @@ func EventWithRecurrenceBytes(subject, recurrenceTimeZone string) []byte { recurrence := string(fmt.Sprintf( recurrenceTmpl, + strconv.Itoa(int(at.Month())), + strconv.Itoa(at.Day()), timeSlice[0], recurrenceTimeZone, )) @@ -258,7 +269,70 @@ func EventWithRecurrenceBytes(subject, recurrenceTimeZone string) []byte { return EventWith( defaultEventOrganizer, subject, defaultEventBody, defaultEventBodyPreview, - atTime, atTime, recurrence, attendeesTmpl, true, + NoOriginalStartDate, atTime, atTime, recurrence, attendeesTmpl, + true, NoCancelledOccurrences, NoExceptionOccurrences, + ) +} + +func EventWithRecurrenceAndCancellationBytes(subject string) []byte { + tomorrow := time.Now().UTC().AddDate(0, 0, 1) + at := time.Date(tomorrow.Year(), tomorrow.Month(), tomorrow.Day(), tomorrow.Hour(), 0, 0, 0, time.UTC) + atTime := dttm.Format(at) + timeSlice := strings.Split(atTime, "T") + nextYear := tomorrow.AddDate(1, 0, 0) + + recurrence := string(fmt.Sprintf( + recurrenceTmpl, + strconv.Itoa(int(at.Month())), + strconv.Itoa(at.Day()), + timeSlice[0], + `"UTC"`, + )) + + cancelledInstances := []string{fmt.Sprintf(cancelledOccurrenceInstanceFormat, dttm.FormatTo(nextYear, dttm.DateOnly))} + cancelledOccurrences := fmt.Sprintf(cancelledOccurrencesFormat, strings.Join(cancelledInstances, ",")) + + return EventWith( + defaultEventOrganizer, subject, + defaultEventBody, defaultEventBodyPreview, + NoOriginalStartDate, atTime, atTime, recurrence, attendeesTmpl, + true, cancelledOccurrences, NoExceptionOccurrences, + ) +} + +func EventWithRecurrenceAndExceptionBytes(subject string) []byte { + tomorrow := time.Now().UTC().AddDate(0, 0, 1) + at := time.Date(tomorrow.Year(), tomorrow.Month(), tomorrow.Day(), tomorrow.Hour(), 0, 0, 0, time.UTC) + atTime := dttm.Format(at) + timeSlice := strings.Split(atTime, "T") + newTime := dttm.Format(tomorrow.AddDate(0, 0, 1)) + originalStartDate := dttm.FormatTo(at, dttm.TabularOutput) + + recurrence := string(fmt.Sprintf( + recurrenceTmpl, + strconv.Itoa(int(at.Month())), + strconv.Itoa(at.Day()), + timeSlice[0], + `"UTC"`, + )) + + exceptionEvent := EventWith( + defaultEventOrganizer, subject+"(modified)", + defaultEventBody, defaultEventBodyPreview, + fmt.Sprintf(originalStartDateFormat, originalStartDate), + newTime, newTime, NoRecurrence, attendeesTmpl, + false, NoCancelledOccurrences, NoExceptionOccurrences, + ) + exceptionOccurrences := fmt.Sprintf( + exceptionOccurrencesFormat, + strings.Join([]string{string(exceptionEvent)}, ","), + ) + + return EventWith( + defaultEventOrganizer, subject, + defaultEventBody, defaultEventBodyPreview, + NoOriginalStartDate, atTime, atTime, recurrence, attendeesTmpl, + true, NoCancelledOccurrences, exceptionOccurrences, ) } @@ -270,7 +344,8 @@ func EventWithAttendeesBytes(subject string) []byte { return EventWith( defaultEventOrganizer, subject, defaultEventBody, defaultEventBodyPreview, - atTime, atTime, NoRecurrence, attendeesTmpl, true, + NoOriginalStartDate, atTime, atTime, NoRecurrence, attendeesTmpl, + true, NoCancelledOccurrences, NoExceptionOccurrences, ) } @@ -281,8 +356,8 @@ func EventWithAttendeesBytes(subject string) []byte { // Body must contain a well-formatted string, consumable in a json payload. IE: no unescaped newlines. func EventWith( organizer, subject, body, bodyPreview, - startDateTime, endDateTime, recurrence, attendees string, - hasAttachments bool, + originalStartDate, startDateTime, endDateTime, recurrence, attendees string, + hasAttachments bool, cancelledOccurrences, exceptionOccurrences string, ) []byte { var attachments string if hasAttachments { @@ -300,17 +375,26 @@ func EventWith( endDateTime += ".0000000" } + eventType := "singleInstance" + if recurrence != "null" { + eventType = "seriesMaster" + } + return []byte(fmt.Sprintf( eventTmpl, body, bodyPreview, endDateTime, organizer, + originalStartDate, startDateTime, subject, + eventType, hasAttachments, attachments, recurrence, + cancelledOccurrences, + exceptionOccurrences, attendees, )) } diff --git a/src/internal/m365/exchange/restore_test.go b/src/internal/m365/exchange/restore_test.go index a4e985620..e94c7670b 100644 --- a/src/internal/m365/exchange/restore_test.go +++ b/src/internal/m365/exchange/restore_test.go @@ -116,6 +116,14 @@ func (suite *RestoreIntgSuite) TestRestoreEvent() { name: "Test recurrenceTimeZone: Empty", bytes: exchMock.EventWithRecurrenceBytes(subject, `""`), }, + { + name: "Test cancelledOccurrences", + bytes: exchMock.EventWithRecurrenceAndCancellationBytes(subject), + }, + { + name: "Test exceptionOccurrences", + bytes: exchMock.EventWithRecurrenceAndExceptionBytes(subject), + }, } for _, test := range tests { diff --git a/src/internal/m365/exchange/transform.go b/src/internal/m365/exchange/transform.go index 67b790a9e..2e740549c 100644 --- a/src/internal/m365/exchange/transform.go +++ b/src/internal/m365/exchange/transform.go @@ -70,7 +70,6 @@ func toEventSimplified(orig models.Eventable) models.Eventable { newContent := insertStringToBody(origBody, attendees) newBody := models.NewItemBody() newBody.SetContentType(origBody.GetContentType()) - newBody.SetAdditionalData(origBody.GetAdditionalData()) newBody.SetOdataType(origBody.GetOdataType()) newBody.SetContent(&newContent) orig.SetBody(newBody) @@ -89,6 +88,14 @@ func toEventSimplified(orig models.Eventable) models.Eventable { } } + // Remove exceptions for recurring events + // These will be present in objects once we start using the API + // that is currently in beta + additionalData := orig.GetAdditionalData() + delete(additionalData, "cancelledOccurrences") + delete(additionalData, "exceptionOccurrences") + orig.SetAdditionalData(additionalData) + return orig } diff --git a/src/internal/m365/exchange/transform_test.go b/src/internal/m365/exchange/transform_test.go index 4e3ce4278..823a65f2c 100644 --- a/src/internal/m365/exchange/transform_test.go +++ b/src/internal/m365/exchange/transform_test.go @@ -121,6 +121,32 @@ func (suite *TransformUnitTest) TestToEventSimplified_recurrence() { return ptr.Val(e.GetRecurrence().GetRange().GetRecurrenceTimeZone()) == "Pacific Standard Time" }, }, + { + name: "Test cancelledOccurrences", + event: func() models.Eventable { + bytes := exchMock.EventWithRecurrenceAndCancellationBytes(subject) + event, err := api.BytesToEventable(bytes) + require.NoError(t, err, clues.ToCore(err)) + return event + }, + + validateOutput: func(e models.Eventable) bool { + return e.GetAdditionalData()["cancelledOccurrences"] == nil + }, + }, + { + name: "Test exceptionOccurrences", + event: func() models.Eventable { + bytes := exchMock.EventWithRecurrenceAndExceptionBytes(subject) + event, err := api.BytesToEventable(bytes) + require.NoError(t, err, clues.ToCore(err)) + return event + }, + + validateOutput: func(e models.Eventable) bool { + return e.GetAdditionalData()["exceptionOccurrences"] == nil + }, + }, } for _, test := range tests { diff --git a/src/internal/operations/backup_integration_test.go b/src/internal/operations/backup_integration_test.go index 36ec0cfa3..0c657a56c 100644 --- a/src/internal/operations/backup_integration_test.go +++ b/src/internal/operations/backup_integration_test.go @@ -813,7 +813,10 @@ func testExchangeContinuousBackups(suite *BackupOpIntegrationSuite, toggles cont eventDBF := func(id, timeStamp, subject, body string) []byte { return exchMock.EventWith( suite.user, subject, body, body, - now, now, exchMock.NoRecurrence, exchMock.NoAttendees, false) + exchMock.NoOriginalStartDate, now, now, + exchMock.NoRecurrence, exchMock.NoAttendees, + false, exchMock.NoCancelledOccurrences, + exchMock.NoExceptionOccurrences) } // test data set diff --git a/src/pkg/services/m365/api/events.go b/src/pkg/services/m365/api/events.go index 574e2de21..d24b69259 100644 --- a/src/pkg/services/m365/api/events.go +++ b/src/pkg/services/m365/api/events.go @@ -3,8 +3,10 @@ package api import ( "bytes" "context" + "encoding/json" "fmt" "io" + "strings" "time" "github.com/alcionai/clues" @@ -15,6 +17,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/common/str" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -189,6 +192,14 @@ func (c Events) PatchCalendar( return nil } +const ( + // Beta version cannot have /calendars/%s for get and Patch + // https://stackoverflow.com/questions/50492177/microsoft-graph-get-user-calendar-event-with-beta-version + eventExceptionsBetaURLTemplate = "https://graph.microsoft.com/beta/users/%s/events/%s?$expand=exceptionOccurrences" + eventPostBetaURLTemplate = "https://graph.microsoft.com/beta/users/%s/calendars/%s/events" + eventPatchBetaURLTemplate = "https://graph.microsoft.com/beta/users/%s/events/%s" +) + // --------------------------------------------------------------------------- // items // --------------------------------------------------------------------------- @@ -208,41 +219,216 @@ func (c Events) GetItem( } ) - event, err = c.Stable. + // Beta endpoint helps us fetch the event exceptions, but since we + // don't use the beta SDK, the exceptionOccurrences and + // cancelledOccurrences end up in AdditionalData + // https://learn.microsoft.com/en-us/graph/api/resources/event?view=graph-rest-beta#properties + rawURL := fmt.Sprintf(eventExceptionsBetaURLTemplate, userID, itemID) + builder := users.NewItemEventsEventItemRequestBuilder(rawURL, c.Stable.Adapter()) + + event, err = builder.Get(ctx, config) + if err != nil { + return nil, nil, graph.Stack(ctx, err) + } + + err = validateCancelledOccurrences(event) + if err != nil { + return nil, nil, clues.Wrap(err, "verify cancelled occurrences") + } + + err = fixupExceptionOccurrences(ctx, c, event, immutableIDs, userID) + if err != nil { + return nil, nil, clues.Wrap(err, "fixup exception occurrences") + } + + attachments, err := c.getAttachments(ctx, event, immutableIDs, userID, itemID) + if err != nil { + return nil, nil, err + } + + event.SetAttachments(attachments) + + return event, EventInfo(event), nil +} + +// fixupExceptionOccurrences gets attachments and converts the data +// into a format that gets serialized when storing to kopia +func fixupExceptionOccurrences( + ctx context.Context, + client Events, + event models.Eventable, + immutableIDs bool, + userID string, +) error { + // Fetch attachments for exceptions + exceptionOccurrences := event.GetAdditionalData()["exceptionOccurrences"] + if exceptionOccurrences == nil { + return nil + } + + eo, ok := exceptionOccurrences.([]any) + if !ok { + return clues.New("converting exceptionOccurrences to []any"). + With("type", fmt.Sprintf("%T", exceptionOccurrences)) + } + + for _, instance := range eo { + instance, ok := instance.(map[string]any) + if !ok { + return clues.New("converting instance to map[string]any"). + With("type", fmt.Sprintf("%T", instance)) + } + + evt, err := EventFromMap(instance) + if err != nil { + return clues.Wrap(err, "parsing exception event") + } + + // OPTIMIZATION: We don't have to store any of the + // attachments that carry over from the original + attachments, err := client.getAttachments(ctx, evt, immutableIDs, userID, ptr.Val(evt.GetId())) + if err != nil { + return clues.Wrap(err, "getting exception attachments"). + With("exception_event_id", ptr.Val(evt.GetId())) + } + + // This odd roundabout way of doing this is required as + // the json serialization at the end does not serialize if + // you just pass in a models.Attachmentable + convertedAttachments := []map[string]interface{}{} + + for _, attachment := range attachments { + am, err := parseableToMap(attachment) + if err != nil { + return clues.Wrap(err, "converting attachment") + } + + convertedAttachments = append(convertedAttachments, am) + } + + instance["attachments"] = convertedAttachments + } + + return nil +} + +// Adding checks to ensure that the data is in the format that we expect M365 to return +func validateCancelledOccurrences(event models.Eventable) error { + cancelledOccurrences := event.GetAdditionalData()["cancelledOccurrences"] + if cancelledOccurrences != nil { + co, ok := cancelledOccurrences.([]any) + if !ok { + return clues.New("converting cancelledOccurrences to []any"). + With("type", fmt.Sprintf("%T", cancelledOccurrences)) + } + + for _, instance := range co { + instance, err := str.AnyToString(instance) + if err != nil { + return err + } + + // There might be multiple `.` in the ID and hence >2 + splits := strings.Split(instance, ".") + if len(splits) < 2 { + return clues.New("unexpected cancelled event format"). + With("instance", instance) + } + + startStr := splits[len(splits)-1] + + _, err = dttm.ParseTime(startStr) + if err != nil { + return clues.Wrap(err, "parsing cancelled event date") + } + } + } + + return nil +} + +func parseableToMap(att serialization.Parsable) (map[string]any, error) { + var item map[string]any + + writer := kjson.NewJsonSerializationWriter() + defer writer.Close() + + if err := writer.WriteObjectValue("", att); err != nil { + return nil, err + } + + ats, err := writer.GetSerializedContent() + if err != nil { + return nil, err + } + + err = json.Unmarshal(ats, &item) + if err != nil { + return nil, clues.Wrap(err, "unmarshalling serialized attachment") + } + + return item, nil +} + +func (c Events) getAttachments( + ctx context.Context, + event models.Eventable, + immutableIDs bool, + userID string, + itemID string, +) ([]models.Attachmentable, error) { + if !ptr.Val(event.GetHasAttachments()) && !HasAttachments(event.GetBody()) { + return nil, nil + } + + config := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{ + Expand: []string{"microsoft.graph.itemattachment/item"}, + }, + Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), + } + + attached, err := c.LargeItem. Client(). Users(). ByUserId(userID). Events(). ByEventId(itemID). + Attachments(). Get(ctx, config) if err != nil { - return nil, nil, graph.Stack(ctx, err) + return nil, graph.Wrap(ctx, err, "event attachment download") } - if ptr.Val(event.GetHasAttachments()) || HasAttachments(event.GetBody()) { - config := &users.ItemEventsItemAttachmentsRequestBuilderGetRequestConfiguration{ - QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{ - Expand: []string{"microsoft.graph.itemattachment/item"}, - }, - Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize), preferImmutableIDs(immutableIDs)), - } + return attached.GetValue(), nil +} - attached, err := c.LargeItem. - Client(). - Users(). - ByUserId(userID). - Events(). - ByEventId(itemID). - Attachments(). - Get(ctx, config) - if err != nil { - return nil, nil, graph.Wrap(ctx, err, "event attachment download") - } - - event.SetAttachments(attached.GetValue()) +func (c Events) GetItemInstances( + ctx context.Context, + userID, itemID string, + startDate, endDate string, +) ([]models.Eventable, error) { + config := &users.ItemEventsItemInstancesRequestBuilderGetRequestConfiguration{ + QueryParameters: &users.ItemEventsItemInstancesRequestBuilderGetQueryParameters{ + Select: []string{"id"}, + StartDateTime: ptr.To(startDate), + EndDateTime: ptr.To(endDate), + }, } - return event, EventInfo(event), nil + events, err := c.Stable. + Client(). + Users(). + ByUserId(userID). + Events(). + ByEventId(itemID). + Instances(). + Get(ctx, config) + if err != nil { + return nil, graph.Stack(ctx, err) + } + + return events.GetValue(), nil } func (c Events) PostItem( @@ -250,14 +436,10 @@ func (c Events) PostItem( userID, containerID string, body models.Eventable, ) (models.Eventable, error) { - itm, err := c.Stable. - Client(). - Users(). - ByUserId(userID). - Calendars(). - ByCalendarId(containerID). - Events(). - Post(ctx, body, nil) + rawURL := fmt.Sprintf(eventPostBetaURLTemplate, userID, containerID) + builder := users.NewItemCalendarsItemEventsRequestBuilder(rawURL, c.Stable.Adapter()) + + itm, err := builder.Post(ctx, body, nil) if err != nil { return nil, graph.Wrap(ctx, err, "creating calendar event") } @@ -265,6 +447,22 @@ func (c Events) PostItem( return itm, nil } +func (c Events) PatchItem( + ctx context.Context, + userID, eventID string, + body models.Eventable, +) (models.Eventable, error) { + rawURL := fmt.Sprintf(eventPatchBetaURLTemplate, userID, eventID) + builder := users.NewItemCalendarsItemEventsEventItemRequestBuilder(rawURL, c.Stable.Adapter()) + + itm, err := builder.Patch(ctx, body, nil) + if err != nil { + return nil, graph.Wrap(ctx, err, "updating calendar event") + } + + return itm, nil +} + func (c Events) DeleteItem( ctx context.Context, userID, itemID string, @@ -472,3 +670,17 @@ func EventInfo(evt models.Eventable) *details.ExchangeInfo { Modified: ptr.OrNow(evt.GetLastModifiedDateTime()), } } + +func EventFromMap(ev map[string]any) (models.Eventable, error) { + instBytes, err := json.Marshal(ev) + if err != nil { + return nil, clues.Wrap(err, "marshaling event exception instance") + } + + body, err := BytesToEventable(instBytes) + if err != nil { + return nil, clues.Wrap(err, "converting exception event bytes to Eventable") + } + + return body, nil +} diff --git a/website/docs/support/known-issues.md b/website/docs/support/known-issues.md index 6f98071fb..3b368f256 100644 --- a/website/docs/support/known-issues.md +++ b/website/docs/support/known-issues.md @@ -26,3 +26,5 @@ included in backup and restore. * SharePoint document library data can't be restored after the library has been deleted. * Sharing information of items in OneDrive/SharePoint using sharing links aren't backed up and restored. + +* Changes to attachments in instances of recurring events compared to the series master aren't restored