Fix uploading large attachments for exchange (#3634)

Upload of large attachments were broken previously.

Previously we were relying on the size reported by Graph API, but
that is not reliable and we were not completing uploads because of that.

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [x] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [x] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [ ] ⚡ Unit test
- [x] 💚 E2E

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix Exchange backup issue caused by incorrect json serialization
 - Fix issues with details model containing duplicate entry for api consumers
 - Handle OLE conversion errors when trying to fetch attachments
+- Fix uploading large attachments for emails and calendar
 
 ### Changed
 - Do not display all the items that we restored at the end if there are more than 15. You can override this with `--verbose`.

src/internal/m365/exchange/attachment.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
 )
 
 type attachmentPoster interface {
@@ -20,15 +21,14 @@ type attachmentPoster interface {
 	PostLargeAttachment(
 		ctx context.Context,
 		userID, containerID, itemID, name string,
-		size int64,
+		content []byte,
-		body models.Attachmentable,
+	) (string, error)
-	) (models.UploadSessionable, error)
 }
 
 const (
 	// Use large attachment logic for attachments > 3MB
 	// https://learn.microsoft.com/en-us/graph/outlook-large-attachments
-	largeAttachmentSize           = int32(3 * 1024 * 1024)
+	largeAttachmentSize           = 3 * 1024 * 1024
 	fileAttachmentOdataValue      = "#microsoft.graph.fileAttachment"
 	itemAttachmentOdataValue      = "#microsoft.graph.itemAttachment"
 	referenceAttachmentOdataValue = "#microsoft.graph.referenceAttachment"
@@ -95,7 +95,15 @@ func uploadAttachment(
 
 	// for file attachments sized >= 3MB
 	if attachmentType == models.FILE_ATTACHMENTTYPE && size >= largeAttachmentSize {
-		_, err := cli.PostLargeAttachment(ctx, userID, containerID, parentItemID, name, int64(size), attachment)
+		// We expect the entire attachment to fit in memory.
+		// Max attachment size is 150MB.
+		content, err := api.GetAttachmentContent(attachment)
+		if err != nil {
+			return clues.Wrap(err, "serializing attachment content").WithClues(ctx)
+		}
+
+		_, err = cli.PostLargeAttachment(ctx, userID, containerID, parentItemID, name, content)
+
 		return err
 	}
 

src/internal/m365/graph/uploadsession.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"github.com/alcionai/clues"
 
@@ -21,8 +22,11 @@ const (
 // Writer implements an io.Writer for a M365
 // UploadSession URL
 type largeItemWriter struct {
+	// ID is the id of the item created.
+	// Will be available after the upload is complete
+	ID string
 	// Identifier
-	id string
+	parentID string
 	// Upload URL for this item
 	url string
 	// Tracks how much data will be written
@@ -32,8 +36,13 @@ type largeItemWriter struct {
 	client            httpWrapper
 }
 
-func NewLargeItemWriter(id, url string, size int64) *largeItemWriter {
+func NewLargeItemWriter(parentID, url string, size int64) *largeItemWriter {
-	return &largeItemWriter{id: id, url: url, contentLength: size, client: *NewNoTimeoutHTTPWrapper()}
+	return &largeItemWriter{
+		parentID:      parentID,
+		url:           url,
+		contentLength: size,
+		client:        *NewNoTimeoutHTTPWrapper(),
+	}
 }
 
 // Write will upload the provided data to M365. It sets the `Content-Length` and `Content-Range` headers based on
@@ -44,7 +53,7 @@ func (iw *largeItemWriter) Write(p []byte) (int, error) {
 
 	logger.Ctx(ctx).
 		Debugf("WRITE for %s. Size:%d, Offset: %d, TotalSize: %d",
-			iw.id, rangeLength, iw.lastWrittenOffset, iw.contentLength)
+			iw.parentID, rangeLength, iw.lastWrittenOffset, iw.contentLength)
 
 	endOffset := iw.lastWrittenOffset + int64(rangeLength)
 
@@ -58,7 +67,7 @@ func (iw *largeItemWriter) Write(p []byte) (int, error) {
 		iw.contentLength)
 	headers[contentLengthHeaderKey] = fmt.Sprintf("%d", rangeLength)
 
-	_, err := iw.client.Request(
+	resp, err := iw.client.Request(
 		ctx,
 		http.MethodPut,
 		iw.url,
@@ -66,7 +75,7 @@ func (iw *largeItemWriter) Write(p []byte) (int, error) {
 		headers)
 	if err != nil {
 		return 0, clues.Wrap(err, "uploading item").With(
-			"upload_id", iw.id,
+			"upload_id", iw.parentID,
 			"upload_chunk_size", rangeLength,
 			"upload_offset", iw.lastWrittenOffset,
 			"upload_size", iw.contentLength)
@@ -75,5 +84,22 @@ func (iw *largeItemWriter) Write(p []byte) (int, error) {
 	// Update last offset
 	iw.lastWrittenOffset = endOffset
 
+	// Once the upload is complete, we get a Location header in the
+	// below format from which we can get the id of the uploaded
+	// item. This will only be available after we have uploaded the
+	// entire content(based on the size in the req header).
+	// https://outlook.office.com/api/v2.0/Users('<user-id>')/Messages('<message-id>')/Attachments('<attachment-id>')
+	// Ref: https://learn.microsoft.com/en-us/graph/outlook-large-attachments?tabs=http
+	loc := resp.Header.Get("Location")
+	if loc != "" {
+		splits := strings.Split(loc, "'")
+		if len(splits) != 7 || splits[4] != ")/Attachments(" || len(splits[5]) == 0 {
+			return 0, clues.New("invalid format for upload completion url").
+				With("location", loc)
+		}
+
+		iw.ID = splits[5]
+	}
+
 	return rangeLength, nil
 }

src/pkg/services/m365/api/events.go

@@ -513,14 +513,9 @@ func (c Events) PostSmallAttachment(
 func (c Events) PostLargeAttachment(
 	ctx context.Context,
 	userID, containerID, parentItemID, itemName string,
-	size int64,
+	content []byte,
-	body models.Attachmentable,
+) (string, error) {
-) (models.UploadSessionable, error) {
+	size := int64(len(content))
-	bs, err := GetAttachmentContent(body)
-	if err != nil {
-		return nil, clues.Wrap(err, "serializing attachment content").WithClues(ctx)
-	}
-
 	session := users.NewItemCalendarEventsItemAttachmentsCreateUploadSessionPostRequestBody()
 	session.SetAttachmentItem(makeSessionAttachment(itemName, size))
 
@@ -536,19 +531,19 @@ func (c Events) PostLargeAttachment(
 		CreateUploadSession().
 		Post(ctx, session, nil)
 	if err != nil {
-		return nil, graph.Wrap(ctx, err, "uploading large event attachment")
+		return "", graph.Wrap(ctx, err, "uploading large event attachment")
 	}
 
 	url := ptr.Val(us.GetUploadUrl())
 	w := graph.NewLargeItemWriter(parentItemID, url, size)
 	copyBuffer := make([]byte, graph.AttachmentChunkSize)
 
-	_, err = io.CopyBuffer(w, bytes.NewReader(bs), copyBuffer)
+	_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
 	if err != nil {
-		return nil, clues.Wrap(err, "buffering large attachment content").WithClues(ctx)
+		return "", clues.Wrap(err, "buffering large attachment content").WithClues(ctx)
 	}
 
-	return us, nil
+	return w.ID, nil
 }
 
 // ---------------------------------------------------------------------------

src/pkg/services/m365/api/events_test.go

@@ -11,9 +11,12 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/alcionai/corso/src/internal/common/dttm"
+	"github.com/alcionai/corso/src/internal/common/ptr"
 	exchMock "github.com/alcionai/corso/src/internal/m365/exchange/mock"
 	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/control/testdata"
 )
 
 type EventsAPIUnitSuite struct {
@@ -212,3 +215,70 @@ func (suite *EventsAPIUnitSuite) TestBytesToEventable() {
 		})
 	}
 }
+
+type EventsAPIIntgSuite struct {
+	tester.Suite
+	credentials account.M365Config
+	ac          Client
+}
+
+func TestEventsAPIntgSuite(t *testing.T) {
+	suite.Run(t, &EventsAPIIntgSuite{
+		Suite: tester.NewIntegrationSuite(
+			t,
+			[][]string{tester.M365AcctCredEnvs}),
+	})
+}
+
+func (suite *EventsAPIIntgSuite) SetupSuite() {
+	t := suite.T()
+
+	a := tester.NewM365Account(t)
+	m365, err := a.M365Config()
+	require.NoError(t, err, clues.ToCore(err))
+
+	suite.credentials = m365
+	suite.ac, err = NewClient(m365)
+	require.NoError(t, err, clues.ToCore(err))
+}
+
+func (suite *EventsAPIIntgSuite) TestRestoreLargeAttachment() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	userID := tester.M365UserID(suite.T())
+
+	folderName := testdata.DefaultRestoreConfig("eventlargeattachmenttest").Location
+	evts := suite.ac.Events()
+	calendar, err := evts.CreateContainer(ctx, userID, folderName, "")
+	require.NoError(t, err, clues.ToCore(err))
+
+	tomorrow := time.Now().Add(24 * time.Hour)
+	evt := models.NewEvent()
+	sdtz := models.NewDateTimeTimeZone()
+	edtz := models.NewDateTimeTimeZone()
+
+	evt.SetSubject(ptr.To("Event with attachment"))
+	sdtz.SetDateTime(ptr.To(dttm.Format(tomorrow)))
+	sdtz.SetTimeZone(ptr.To("UTC"))
+	edtz.SetDateTime(ptr.To(dttm.Format(tomorrow.Add(30 * time.Minute))))
+	edtz.SetTimeZone(ptr.To("UTC"))
+	evt.SetStart(sdtz)
+	evt.SetEnd(edtz)
+
+	item, err := evts.PostItem(ctx, userID, ptr.Val(calendar.GetId()), evt)
+	require.NoError(t, err, clues.ToCore(err))
+
+	id, err := evts.PostLargeAttachment(
+		ctx,
+		userID,
+		ptr.Val(calendar.GetId()),
+		ptr.Val(item.GetId()),
+		"raboganm",
+		[]byte("mangobar"),
+	)
+	require.NoError(t, err, clues.ToCore(err))
+	require.NotEmpty(t, id, "empty id for large attachment")
+}

src/pkg/services/m365/api/mail.go

@@ -63,6 +63,23 @@ func (c Mail) CreateMailFolder(
 	return mdl, nil
 }
 
+func (c Mail) DeleteMailFolder(
+	ctx context.Context,
+	userID, id string,
+) error {
+	err := c.Stable.Client().
+		Users().
+		ByUserId(userID).
+		MailFolders().
+		ByMailFolderId(id).
+		Delete(ctx, nil)
+	if err != nil {
+		return graph.Wrap(ctx, err, "deleting mail folder")
+	}
+
+	return nil
+}
+
 func (c Mail) CreateContainer(
 	ctx context.Context,
 	userID, containerName, parentContainerID string,
@@ -407,14 +424,9 @@ func (c Mail) PostSmallAttachment(
 func (c Mail) PostLargeAttachment(
 	ctx context.Context,
 	userID, containerID, parentItemID, itemName string,
-	size int64,
+	content []byte,
-	body models.Attachmentable,
+) (string, error) {
-) (models.UploadSessionable, error) {
+	size := int64(len(content))
-	bs, err := GetAttachmentContent(body)
-	if err != nil {
-		return nil, clues.Wrap(err, "serializing attachment content").WithClues(ctx)
-	}
-
 	session := users.NewItemMailFoldersItemMessagesItemAttachmentsCreateUploadSessionPostRequestBody()
 	session.SetAttachmentItem(makeSessionAttachment(itemName, size))
 
@@ -430,19 +442,19 @@ func (c Mail) PostLargeAttachment(
 		CreateUploadSession().
 		Post(ctx, session, nil)
 	if err != nil {
-		return nil, graph.Wrap(ctx, err, "uploading large mail attachment")
+		return "", graph.Wrap(ctx, err, "uploading large mail attachment")
 	}
 
 	url := ptr.Val(us.GetUploadUrl())
 	w := graph.NewLargeItemWriter(parentItemID, url, size)
 	copyBuffer := make([]byte, graph.AttachmentChunkSize)
 
-	_, err = io.CopyBuffer(w, bytes.NewReader(bs), copyBuffer)
+	_, err = io.CopyBuffer(w, bytes.NewReader(content), copyBuffer)
 	if err != nil {
-		return nil, clues.Wrap(err, "buffering large attachment content").WithClues(ctx)
+		return "", clues.Wrap(err, "buffering large attachment content").WithClues(ctx)
 	}
 
-	return us, nil
+	return w.ID, nil
 }
 
 // ---------------------------------------------------------------------------

src/pkg/services/m365/api/mail_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/alcionai/corso/src/internal/tester"
 	"github.com/alcionai/corso/src/pkg/account"
 	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/control/testdata"
 	"github.com/alcionai/corso/src/pkg/fault"
 	"github.com/alcionai/corso/src/pkg/services/m365/api"
 	"github.com/alcionai/corso/src/pkg/services/m365/api/mock"
@@ -410,3 +411,34 @@ func (suite *MailAPIIntgSuite) TestHugeAttachmentListDownload() {
 		})
 	}
 }
+
+func (suite *MailAPIIntgSuite) TestRestoreLargeAttachment() {
+	t := suite.T()
+
+	ctx, flush := tester.NewContext(t)
+	defer flush()
+
+	userID := tester.M365UserID(suite.T())
+
+	folderName := testdata.DefaultRestoreConfig("maillargeattachmenttest").Location
+	msgs := suite.ac.Mail()
+	mailfolder, err := msgs.CreateMailFolder(ctx, userID, folderName)
+	require.NoError(t, err, clues.ToCore(err))
+
+	msg := models.NewMessage()
+	msg.SetSubject(ptr.To("Mail with attachment"))
+
+	item, err := msgs.PostItem(ctx, userID, ptr.Val(mailfolder.GetId()), msg)
+	require.NoError(t, err, clues.ToCore(err))
+
+	id, err := msgs.PostLargeAttachment(
+		ctx,
+		userID,
+		ptr.Val(mailfolder.GetId()),
+		ptr.Val(item.GetId()),
+		"raboganm",
+		[]byte("mangobar"),
+	)
+	require.NoError(t, err, clues.ToCore(err))
+	require.NotEmpty(t, id, "empty id for large attachment")
+}