Add immutable ID header to Get requests (#3157)

This allows using immutable IDs for Exchange objects. It does not add this header to put requests as we don't record the IDs of restored objects anywhere.

May slightly increase performance of events backups as
it now sets the delta query page size to 200 like with
other requests

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #2861

#### Test Plan

- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E

src/cmd/getM365/exchange/get_item.go

@@ -98,11 +98,11 @@ func runDisplayM365JSON(
 
 	switch cat {
 	case path.EmailCategory:
-		bs, err = getItem(ctx, ac.Mail(), user, itemID, errs)
+		bs, err = getItem(ctx, ac.Mail(), user, itemID, true, errs)
 	case path.EventsCategory:
-		bs, err = getItem(ctx, ac.Events(), user, itemID, errs)
+		bs, err = getItem(ctx, ac.Events(), user, itemID, true, errs)
 	case path.ContactsCategory:
-		bs, err = getItem(ctx, ac.Contacts(), user, itemID, errs)
+		bs, err = getItem(ctx, ac.Contacts(), user, itemID, true, errs)
 	default:
 		return fmt.Errorf("unable to process category: %s", cat)
 	}
@@ -132,6 +132,7 @@ type itemer interface {
 	GetItem(
 		ctx context.Context,
 		user, itemID string,
+		immutableID bool,
 		errs *fault.Bus,
 	) (serialization.Parsable, *details.ExchangeInfo, error)
 	Serialize(
@@ -145,9 +146,10 @@ func getItem(
 	ctx context.Context,
 	itm itemer,
 	user, itemID string,
+	immutableIDs bool,
 	errs *fault.Bus,
 ) ([]byte, error) {
-	sp, _, err := itm.GetItem(ctx, user, itemID, errs)
+	sp, _, err := itm.GetItem(ctx, user, itemID, immutableIDs, errs)
 	if err != nil {
 		return nil, clues.Wrap(err, "getting item")
 	}

src/internal/connector/exchange/api/contacts.go

@@ -79,9 +79,14 @@ func (c Contacts) DeleteContainer(
 func (c Contacts) GetItem(
 	ctx context.Context,
 	user, itemID string,
+	immutableIDs bool,
 	_ *fault.Bus, // no attachments to iterate over, so this goes unused
 ) (serialization.Parsable, *details.ExchangeInfo, error) {
-	cont, err := c.Stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil)
+	options := &users.ItemContactsContactItemRequestBuilderGetRequestConfiguration{
+		Headers: buildPreferHeaders(false, immutableIDs),
+	}
+
+	cont, err := c.Stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, options)
 	if err != nil {
 		return nil, nil, graph.Stack(ctx, err)
 	}
@@ -210,6 +215,7 @@ func (p *contactPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, erro
 func (c Contacts) GetAddedAndRemovedItemIDs(
 	ctx context.Context,
 	user, directoryID, oldDelta string,
+	immutableIDs bool,
 ) ([]string, []string, DeltaUpdate, error) {
 	service, err := c.service()
 	if err != nil {
@@ -223,7 +229,9 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
 		"category", selectors.ExchangeContact,
 		"container_id", directoryID)
 
-	options, err := optionsForContactFoldersItemDelta([]string{"parentFolderId"})
+	options, err := optionsForContactFoldersItemDelta(
+		[]string{"parentFolderId"},
+		immutableIDs)
 	if err != nil {
 		return nil,
 			nil,

src/internal/connector/exchange/api/events.go

@@ -103,14 +103,19 @@ func (c Events) GetContainerByID(
 func (c Events) GetItem(
 	ctx context.Context,
 	user, itemID string,
+	immutableIDs bool,
 	errs *fault.Bus,
 ) (serialization.Parsable, *details.ExchangeInfo, error) {
 	var (
-		err   error
-		event models.Eventable
+		err      error
+		event    models.Eventable
+		header   = buildPreferHeaders(false, immutableIDs)
+		itemOpts = &users.ItemEventsEventItemRequestBuilderGetRequestConfiguration{
+			Headers: header,
+		}
 	)
 
-	event, err = c.Stable.Client().UsersById(user).EventsById(itemID).Get(ctx, nil)
+	event, err = c.Stable.Client().UsersById(user).EventsById(itemID).Get(ctx, itemOpts)
 	if err != nil {
 		return nil, nil, graph.Stack(ctx, err)
 	}
@@ -120,6 +125,7 @@ func (c Events) GetItem(
 			QueryParameters: &users.ItemEventsItemAttachmentsRequestBuilderGetQueryParameters{
 				Expand: []string{"microsoft.graph.itemattachment/item"},
 			},
+			Headers: header,
 		}
 
 		attached, err := c.LargeItem.
@@ -245,13 +251,19 @@ func (p *eventPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error)
 func (c Events) GetAddedAndRemovedItemIDs(
 	ctx context.Context,
 	user, calendarID, oldDelta string,
+	immutableIDs bool,
 ) ([]string, []string, DeltaUpdate, error) {
 	service, err := c.service()
 	if err != nil {
 		return nil, nil, DeltaUpdate{}, err
 	}
 
-	var resetDelta bool
+	var (
+		resetDelta bool
+		opts       = &users.ItemCalendarsItemEventsDeltaRequestBuilderGetRequestConfiguration{
+			Headers: buildPreferHeaders(true, immutableIDs),
+		}
+	)
 
 	ctx = clues.Add(
 		ctx,
@@ -260,7 +272,7 @@ func (c Events) GetAddedAndRemovedItemIDs(
 	if len(oldDelta) > 0 {
 		var (
 			builder = users.NewItemCalendarsItemEventsDeltaRequestBuilder(oldDelta, service.Adapter())
-			pgr     = &eventPager{service, builder, nil}
+			pgr     = &eventPager{service, builder, opts}
 		)
 
 		added, removed, deltaURL, err := getItemsAddedAndRemovedFromContainer(ctx, pgr)
@@ -287,7 +299,7 @@ func (c Events) GetAddedAndRemovedItemIDs(
 	// works as intended (until, at least, we want to _not_ call the beta anymore).
 	rawURL := fmt.Sprintf(eventBetaDeltaURLTemplate, user, calendarID)
 	builder := users.NewItemCalendarsItemEventsDeltaRequestBuilder(rawURL, service.Adapter())
-	pgr := &eventPager{service, builder, nil}
+	pgr := &eventPager{service, builder, opts}
 
 	if len(os.Getenv("CORSO_URL_LOGGING")) > 0 {
 		gri, err := builder.ToGetRequestInformation(ctx, nil)

src/internal/connector/exchange/api/mail.go

@@ -131,9 +131,16 @@ func (c Mail) GetContainerByID(
 func (c Mail) GetItem(
 	ctx context.Context,
 	user, itemID string,
+	immutableIDs bool,
 	errs *fault.Bus,
 ) (serialization.Parsable, *details.ExchangeInfo, error) {
-	mail, err := c.Stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil)
+	// Will need adjusted if attachments start allowing paging.
+	headers := buildPreferHeaders(false, immutableIDs)
+	itemOpts := &users.ItemMessagesMessageItemRequestBuilderGetRequestConfiguration{
+		Headers: headers,
+	}
+
+	mail, err := c.Stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, itemOpts)
 	if err != nil {
 		return nil, nil, graph.Stack(ctx, err)
 	}
@@ -146,6 +153,7 @@ func (c Mail) GetItem(
 		QueryParameters: &users.ItemMessagesItemAttachmentsRequestBuilderGetQueryParameters{
 			Expand: []string{"microsoft.graph.itemattachment/item"},
 		},
+		Headers: headers,
 	}
 
 	attached, err := c.LargeItem.
@@ -190,6 +198,7 @@ func (c Mail) GetItem(
 			QueryParameters: &users.ItemMessagesItemAttachmentsAttachmentItemRequestBuilderGetQueryParameters{
 				Expand: []string{"microsoft.graph.itemattachment/item"},
 			},
+			Headers: headers,
 		}
 
 		att, err := c.Stable.
@@ -304,6 +313,7 @@ func (p *mailPager) valuesIn(pl api.DeltaPageLinker) ([]getIDAndAddtler, error)
 func (c Mail) GetAddedAndRemovedItemIDs(
 	ctx context.Context,
 	user, directoryID, oldDelta string,
+	immutableIDs bool,
 ) ([]string, []string, DeltaUpdate, error) {
 	service, err := c.service()
 	if err != nil {
@@ -320,7 +330,7 @@ func (c Mail) GetAddedAndRemovedItemIDs(
 		"category", selectors.ExchangeMail,
 		"container_id", directoryID)
 
-	options, err := optionsForFolderMessagesDelta([]string{"isRead"})
+	options, err := optionsForFolderMessagesDelta([]string{"isRead"}, immutableIDs)
 	if err != nil {
 		return nil,
 			nil,

src/internal/connector/exchange/api/mail_test.go

@@ -342,7 +342,7 @@ func (suite *MailAPIE2ESuite) TestHugeAttachmentListDownload() {
 			defer gock.Off()
 			tt.setupf()
 
-			item, _, err := suite.ac.Mail().GetItem(ctx, "user", mid, fault.New(true))
+			item, _, err := suite.ac.Mail().GetItem(ctx, "user", mid, false, fault.New(true))
 			tt.expect(suite.T(), err)
 
 			it, ok := item.(models.Messageable)

src/internal/connector/exchange/api/options.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/alcionai/clues"
 	abstractions "github.com/microsoft/kiota-abstractions-go"
@@ -63,6 +64,8 @@ const (
 	maxPageSizeHeaderFmt = "odata.maxpagesize=%d"
 	// deltaMaxPageSize is the max page size to use for delta queries
 	deltaMaxPageSize = 200
+	idTypeFmt        = "IdType=%q"
+	immutableIDType  = "ImmutableId"
 )
 
 // -----------------------------------------------------------------------
@@ -74,6 +77,7 @@ const (
 
 func optionsForFolderMessagesDelta(
 	moreOps []string,
+	immutableIDs bool,
 ) (*users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration, error) {
 	selecting, err := buildOptions(moreOps, fieldsForMessages)
 	if err != nil {
@@ -86,7 +90,7 @@ func optionsForFolderMessagesDelta(
 
 	options := &users.ItemMailFoldersItemMessagesDeltaRequestBuilderGetRequestConfiguration{
 		QueryParameters: requestParameters,
-		Headers:         buildDeltaRequestHeaders(),
+		Headers:         buildPreferHeaders(true, immutableIDs),
 	}
 
 	return options, nil
@@ -178,6 +182,7 @@ func optionsForMailFoldersItem(
 
 func optionsForContactFoldersItemDelta(
 	moreOps []string,
+	immutableIDs bool,
 ) (*users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration, error) {
 	selecting, err := buildOptions(moreOps, fieldsForContacts)
 	if err != nil {
@@ -190,7 +195,7 @@ func optionsForContactFoldersItemDelta(
 
 	options := &users.ItemContactFoldersItemContactsDeltaRequestBuilderGetRequestConfiguration{
 		QueryParameters: requestParameters,
-		Headers:         buildDeltaRequestHeaders(),
+		Headers:         buildPreferHeaders(true, immutableIDs),
 	}
 
 	return options, nil
@@ -231,10 +236,21 @@ func buildOptions(fields []string, allowed map[string]struct{}) ([]string, error
 	return append(returnedOptions, fields...), nil
 }
 
-// buildDeltaRequestHeaders returns the headers we add to delta page requests
-func buildDeltaRequestHeaders() *abstractions.RequestHeaders {
+// buildPreferHeaders returns the headers we add to item delta page
+// requests.
+func buildPreferHeaders(pageSize, immutableID bool) *abstractions.RequestHeaders {
+	var allHeaders []string
+
+	if pageSize {
+		allHeaders = append(allHeaders, fmt.Sprintf(maxPageSizeHeaderFmt, deltaMaxPageSize))
+	}
+
+	if immutableID {
+		allHeaders = append(allHeaders, fmt.Sprintf(idTypeFmt, immutableIDType))
+	}
+
 	headers := abstractions.NewRequestHeaders()
-	headers.Add(headerKeyPrefer, fmt.Sprintf(maxPageSizeHeaderFmt, deltaMaxPageSize))
+	headers.Add(headerKeyPrefer, strings.Join(allHeaders, ","))
 
 	return headers
 }

src/internal/connector/exchange/exchange_data_collection.go

@@ -45,6 +45,7 @@ type itemer interface {
 	GetItem(
 		ctx context.Context,
 		user, itemID string,
+		immutableIDs bool,
 		errs *fault.Bus,
 	) (serialization.Parsable, *details.ExchangeInfo, error)
 	Serialize(
@@ -256,6 +257,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
 				ctx,
 				user,
 				id,
+				col.ctrl.ToggleFeatures.ExchangeImmutableIDs,
 				fault.New(true)) // temporary way to force a failFast error
 			if err != nil {
 				// Don't report errors for deleted items as there's no way for us to

src/internal/connector/exchange/exchange_data_collection_test.go

@@ -30,6 +30,7 @@ type mockItemer struct {
 func (mi *mockItemer) GetItem(
 	context.Context,
 	string, string,
+	bool,
 	*fault.Bus,
 ) (serialization.Parsable, *details.ExchangeInfo, error) {
 	mi.getCount++
@@ -228,7 +229,7 @@ func (suite *ExchangeDataCollectionSuite) TestGetItemWithRetries() {
 			defer flush()
 
 			// itemer is mocked, so only the errors are configured atm.
-			_, _, err := test.items.GetItem(ctx, "userID", "itemID", fault.New(true))
+			_, _, err := test.items.GetItem(ctx, "userID", "itemID", false, fault.New(true))
 			test.expectErr(suite.T(), err)
 		})
 	}

src/internal/connector/exchange/service_iterators.go

@@ -22,6 +22,7 @@ type addedAndRemovedItemIDsGetter interface {
 	GetAddedAndRemovedItemIDs(
 		ctx context.Context,
 		user, containerID, oldDeltaToken string,
+		immutableIDs bool,
 	) ([]string, []string, api.DeltaUpdate, error)
 }
 
@@ -108,7 +109,12 @@ func filterContainersAndFillCollections(
 
 		ictx = clues.Add(ictx, "previous_path", prevPath)
 
-		added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ictx, qp.ResourceOwner.ID(), cID, prevDelta)
+		added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(
+			ictx,
+			qp.ResourceOwner.ID(),
+			cID,
+			prevDelta,
+			ctrlOpts.ToggleFeatures.ExchangeImmutableIDs)
 		if err != nil {
 			if !graph.IsErrDeletedInFlight(err) {
 				el.AddRecoverable(clues.Stack(err).Label(fault.LabelForceNoBackupCreation))

src/internal/connector/exchange/service_iterators_test.go

@@ -41,6 +41,7 @@ type (
 func (mg mockGetter) GetAddedAndRemovedItemIDs(
 	ctx context.Context,
 	userID, cID, prevDelta string,
+	_ bool,
 ) (
 	[]string,
 	[]string,

src/internal/operations/backup_integration_test.go

@@ -1078,7 +1078,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
 
 					switch category {
 					case path.EmailCategory:
-						ids, _, _, err := ac.Mail().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "")
+						ids, _, _, err := ac.Mail().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "", false)
 						require.NoError(t, err, "getting message ids", clues.ToCore(err))
 						require.NotEmpty(t, ids, "message ids in folder")
 
@@ -1086,7 +1086,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
 						require.NoError(t, err, "deleting email item", clues.ToCore(err))
 
 					case path.ContactsCategory:
-						ids, _, _, err := ac.Contacts().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "")
+						ids, _, _, err := ac.Contacts().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "", false)
 						require.NoError(t, err, "getting contact ids", clues.ToCore(err))
 						require.NotEmpty(t, ids, "contact ids in folder")
 
@@ -1094,7 +1094,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run_exchangeIncrementals() {
 						require.NoError(t, err, "deleting contact item", clues.ToCore(err))
 
 					case path.EventsCategory:
-						ids, _, _, err := ac.Events().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "")
+						ids, _, _, err := ac.Events().GetAddedAndRemovedItemIDs(ctx, suite.user, containerID, "", false)
 						require.NoError(t, err, "getting event ids", clues.ToCore(err))
 						require.NotEmpty(t, ids, "event ids in folder")
 

src/pkg/control/options.go

@@ -88,4 +88,8 @@ type Toggles struct {
 	// DisableIncrementals prevents backups from using incremental lookups,
 	// forcing a new, complete backup of all data regardless of prior state.
 	DisableIncrementals bool `json:"exchangeIncrementals,omitempty"`
+	// ExchangeImmutableIDs denotes whether Corso should store items with
+	// immutable Exchange IDs. This is only safe to set if the previous backup for
+	// incremental backups used immutable IDs or if a full backup is being done.
+	ExchangeImmutableIDs bool `json:"exchangeImmutableIDs,omitempty"`
 }