lazily fetch messages in chat

once we're past kopia assits, the chat can
download all of its messages and store them in the body
uploded to kopia.

src/internal/m365/collection/teamsChats/backup_test.go

@@ -96,15 +96,13 @@ func (bh mockBackupHandler) CanonicalPath() (path.Path, error) {
 func (bh mockBackupHandler) getItem(
 	_ context.Context,
 	_ string,
-	itemID string,
+	chat models.Chatable,
 ) (models.Chatable, *details.TeamsChatsInfo, error) {
-	chat := models.NewChat()
+	chatID := ptr.Val(chat.GetId())
 
-	chat.SetId(ptr.To(itemID))
+	chat.SetMessages(bh.chatMessages[chatID])
-	chat.SetTopic(ptr.To(itemID))
-	chat.SetMessages(bh.chatMessages[itemID])
 
-	return chat, bh.info[itemID], bh.getMessageErr[itemID]
+	return chat, bh.info[chatID], bh.getMessageErr[chatID]
 }
 
 // ---------------------------------------------------------------------------

src/internal/m365/collection/teamsChats/chat_handler.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
 	"github.com/alcionai/corso/src/pkg/backup/details"
+	"github.com/alcionai/corso/src/pkg/errs/core"
 	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/selectors"
 	"github.com/alcionai/corso/src/pkg/services/m365/api"
@@ -80,10 +81,26 @@ func (bh usersChatsBackupHandler) CanonicalPath() (path.Path, error) {
 func (bh usersChatsBackupHandler) getItem(
 	ctx context.Context,
 	userID string,
-	chatID string,
+	chat models.Chatable,
 ) (models.Chatable, *details.TeamsChatsInfo, error) {
-	// FIXME: should retrieve and populate all messages in the chat.
+	if chat == nil {
-	return nil, nil, clues.New("not implemented")
+		return nil, nil, clues.Stack(core.ErrNotFound)
+	}
+
+	chatID := ptr.Val(chat.GetId())
+
+	cc := api.CallConfig{
+		Expand: []string{"lastMessagePreview"},
+	}
+
+	msgs, err := bh.ac.GetChatMessages(ctx, chatID, cc)
+	if err != nil {
+		return nil, nil, clues.Stack(err)
+	}
+
+	chat.SetMessages(msgs)
+
+	return chat, api.TeamsChatInfo(chat), nil
 }
 
 //lint:ignore U1000 false linter issue due to generics

src/internal/m365/collection/teamsChats/collection.go

@@ -152,20 +152,17 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
 			break
 		}
 
-		itemID := ptr.Val(item.GetId())
 		modTime := ptr.Val(item.GetLastUpdatedDateTime())
 
 		wg.Add(1)
 		semaphoreCh <- struct{}{}
 
-		go func(id string, modTime time.Time) {
+		go func(item I, modTime time.Time) {
 			defer wg.Done()
 			defer func() { <-semaphoreCh }()
 
-			ictx := clues.Add(
+			itemID := ptr.Val(item.GetId())
-				ctx,
+			ictx := clues.Add(ctx, "item_id", itemID)
-				"item_id", id,
-				"parent_path", path.LoggableDir(col.LocationPath().String()))
 
 			col.stream <- data.NewLazyItemWithInfo(
 				ictx,
@@ -173,12 +170,12 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
 					modTime:       modTime,
 					getAndAugment: col.getAndAugment,
 					resourceID:    col.protectedResource,
-					itemID:        id,
+					item:          item,
 					containerIDs:  col.FullPath().Folders(),
 					contains:      col.contains,
 					parentPath:    col.LocationPath().String(),
 				},
-				id,
+				itemID,
 				modTime,
 				col.Counter,
 				el)
@@ -188,7 +185,7 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
 			if progressMessage != nil {
 				progressMessage <- struct{}{}
 			}
-		}(itemID, modTime)
+		}(item, modTime)
 	}
 
 	wg.Wait()
@@ -197,7 +194,7 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
 type lazyItemGetter[I chatsItemer] struct {
 	getAndAugment getItemAndAugmentInfoer[I]
 	resourceID    string
-	itemID        string
+	item          I
 	parentPath    string
 	containerIDs  path.Elements
 	modTime       time.Time
@@ -214,7 +211,7 @@ func (lig *lazyItemGetter[I]) GetData(
 	item, info, err := lig.getAndAugment.getItem(
 		ctx,
 		lig.resourceID,
-		lig.itemID)
+		lig.item)
 	if err != nil {
 		// For items that were deleted in flight, add the skip label so that
 		// they don't lead to recoverable failures during backup.

src/internal/m365/collection/teamsChats/collection_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/alcionai/clues"
+	"github.com/google/uuid"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -155,11 +156,9 @@ type getAndAugmentChat struct {
 func (m getAndAugmentChat) getItem(
 	_ context.Context,
 	_ string,
-	itemID string,
+	chat models.Chatable,
 ) (models.Chatable, *details.TeamsChatsInfo, error) {
-	chat := models.NewChat()
+	chat.SetTopic(chat.GetId())
-	chat.SetId(ptr.To(itemID))
-	chat.SetTopic(ptr.To(itemID))
 
 	return chat, &details.TeamsChatsInfo{}, m.err
 }
@@ -277,6 +276,8 @@ func (suite *CollectionUnitSuite) TestLazyItem_GetDataErrors() {
 			ctx, flush := tester.NewContext(t)
 			defer flush()
 
+			chat := testdata.StubChats(uuid.NewString())[0]
+
 			m := getAndAugmentChat{
 				err: test.getErr,
 			}
@@ -285,12 +286,12 @@ func (suite *CollectionUnitSuite) TestLazyItem_GetDataErrors() {
 				ctx,
 				&lazyItemGetter[models.Chatable]{
 					resourceID:    "resourceID",
-					itemID:        "itemID",
+					item:          chat,
 					getAndAugment: &m,
 					modTime:       now,
 					parentPath:    parentPath,
 				},
-				"itemID",
+				ptr.Val(chat.GetId()),
 				now,
 				count.New(),
 				fault.New(true))
@@ -319,6 +320,8 @@ func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlig
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
+	chat := testdata.StubChats(uuid.NewString())[0]
+
 	m := getAndAugmentChat{
 		err: core.ErrNotFound,
 	}
@@ -327,12 +330,12 @@ func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlig
 		ctx,
 		&lazyItemGetter[models.Chatable]{
 			resourceID:    "resourceID",
-			itemID:        "itemID",
+			item:          chat,
 			getAndAugment: &m,
 			modTime:       now,
 			parentPath:    parentPath,
 		},
-		"itemID",
+		ptr.Val(chat.GetId()),
 		now,
 		count.New(),
 		fault.New(true))
@@ -359,18 +362,19 @@ func (suite *CollectionUnitSuite) TestLazyItem() {
 	ctx, flush := tester.NewContext(t)
 	defer flush()
 
+	chat := testdata.StubChats(uuid.NewString())[0]
 	m := getAndAugmentChat{}
 
 	li := data.NewLazyItemWithInfo(
 		ctx,
 		&lazyItemGetter[models.Chatable]{
 			resourceID:    "resourceID",
-			itemID:        "itemID",
+			item:          chat,
 			getAndAugment: &m,
 			modTime:       now,
 			parentPath:    parentPath,
 		},
-		"itemID",
+		ptr.Val(chat.GetId()),
 		now,
 		count.New(),
 		fault.New(true))

src/internal/m365/collection/teamsChats/handlers.go

@@ -62,7 +62,7 @@ type getItemer[I chatsItemer] interface {
 	getItem(
 		ctx context.Context,
 		protectedResource string,
-		itemID string,
+		i I,
 	) (I, *details.TeamsChatsInfo, error)
 }
 

src/internal/m365/collection/teamsChats/testdata/chats.go

@@ -11,11 +11,21 @@ func StubChats(ids ...string) []models.Chatable {
 	sl := make([]models.Chatable, 0, len(ids))
 
 	for _, id := range ids {
-		ch := models.NewChat()
+		chat := models.NewChat()
-		ch.SetTopic(ptr.To(id))
+		chat.SetTopic(ptr.To(id))
-		ch.SetId(ptr.To(id))
+		chat.SetId(ptr.To(id))
 
-		sl = append(sl, ch)
+		// we should expect to get the latest message preview by default
+		lastMsgPrv := models.NewChatMessageInfo()
+		lastMsgPrv.SetId(ptr.To(uuid.NewString()))
+
+		body := models.NewItemBody()
+		body.SetContent(ptr.To(id))
+		lastMsgPrv.SetBody(body)
+
+		chat.SetLastMessagePreview(lastMsgPrv)
+
+		sl = append(sl, chat)
 	}
 
 	return sl
@@ -24,17 +34,24 @@ func StubChats(ids ...string) []models.Chatable {
 func StubChatMessages(ids ...string) []models.ChatMessageable {
 	sl := make([]models.ChatMessageable, 0, len(ids))
 
+	var lastMsg models.ChatMessageable
+
 	for _, id := range ids {
-		cm := models.NewChatMessage()
+		msg := models.NewChatMessage()
-		cm.SetId(ptr.To(uuid.NewString()))
+		msg.SetId(ptr.To(uuid.NewString()))
 
 		body := models.NewItemBody()
 		body.SetContent(ptr.To(id))
 
-		cm.SetBody(body)
+		msg.SetBody(body)
 
-		sl = append(sl, cm)
+		sl = append(sl, msg)
+		lastMsg = msg
 	}
 
+	lastMsgPrv := models.NewChatMessageInfo()
+	lastMsgPrv.SetId(lastMsg.GetId())
+	lastMsgPrv.SetBody(lastMsg.GetBody())
+
 	return sl
 }

src/pkg/services/m365/api/channels.go

@@ -162,7 +162,7 @@ func channelMessageInfo(
 		modTime = lastReplyAt
 	}
 
-	preview, contentLen, err := getChatMessageContentPreview(msg)
+	preview, contentLen, err := getChatMessageContentPreview(msg, msg)
 	if err != nil {
 		preview = "malformed or unparseable html" + preview
 	}
@@ -180,7 +180,7 @@ func channelMessageInfo(
 	var lr details.ChannelMessageInfo
 
 	if lastReply != nil {
-		preview, contentLen, err = getChatMessageContentPreview(lastReply)
+		preview, contentLen, err = getChatMessageContentPreview(lastReply, lastReply)
 		if err != nil {
 			preview = "malformed or unparseable html: " + preview
 		}
@@ -239,12 +239,28 @@ func GetChatMessageFrom(msg models.ChatMessageable) string {
 	return ""
 }
 
-func getChatMessageContentPreview(msg models.ChatMessageable) (string, int64, error) {
+// a hack for fulfilling getAttachmentser when the model doesn't
-	content, origSize, err := stripChatMessageHTML(msg)
+// provide GetAttachments()
+type noAttachments struct{}
+
+func (noAttachments) GetAttachments() []models.ChatMessageAttachmentable {
+	return []models.ChatMessageAttachmentable{}
+}
+
+type getBodyer interface {
+	GetBody() models.ItemBodyable
+}
+
+type getAttachmentser interface {
+	GetAttachments() []models.ChatMessageAttachmentable
+}
+
+func getChatMessageContentPreview(msg getBodyer, atts getAttachmentser) (string, int64, error) {
+	content, origSize, err := stripChatMessageHTML(msg, atts)
 	return str.Preview(content, 128), origSize, clues.Stack(err).OrNil()
 }
 
-func stripChatMessageHTML(msg models.ChatMessageable) (string, int64, error) {
+func stripChatMessageHTML(msg getBodyer, atts getAttachmentser) (string, int64, error) {
 	var (
 		content  string
 		origSize int64
@@ -256,7 +272,7 @@ func stripChatMessageHTML(msg models.ChatMessageable) (string, int64, error) {
 
 	origSize = int64(len(content))
 
-	content = replaceAttachmentMarkup(content, msg.GetAttachments())
+	content = replaceAttachmentMarkup(content, atts.GetAttachments())
 	content, err := html2text.FromString(content)
 
 	return content, origSize, clues.Stack(err).OrNil()

src/pkg/services/m365/api/channels_test.go

@@ -712,7 +712,7 @@ func (suite *ChannelsAPIUnitSuite) TestStripChatMessageContent() {
 			msg.SetAttachments(test.attachments)
 
 			// not testing len; it's effectively covered by the content assertion
-			result, _, err := stripChatMessageHTML(msg)
+			result, _, err := stripChatMessageHTML(msg, msg)
 			assert.Equal(t, test.expect, result)
 			test.expectErr(t, err, clues.ToCore(err))
 		})

src/pkg/services/m365/api/teamdChats_test.go

@@ -1,89 +0,0 @@
-package api
-
-import (
-	"testing"
-	"time"
-
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/suite"
-
-	"github.com/alcionai/corso/src/internal/common/ptr"
-	"github.com/alcionai/corso/src/internal/tester"
-	"github.com/alcionai/corso/src/pkg/backup/details"
-)
-
-type ChatsAPIUnitSuite struct {
-	tester.Suite
-}
-
-func TestChatsAPIUnitSuite(t *testing.T) {
-	suite.Run(t, &ChatsAPIUnitSuite{Suite: tester.NewUnitSuite(t)})
-}
-
-func (suite *ChatsAPIUnitSuite) TestChatsInfo() {
-	start := time.Now()
-
-	tests := []struct {
-		name            string
-		chatAndExpected func() (models.Chatable, *details.TeamsChatsInfo)
-	}{
-		{
-			name: "Empty chat",
-			chatAndExpected: func() (models.Chatable, *details.TeamsChatsInfo) {
-				chat := models.NewChat()
-
-				i := &details.TeamsChatsInfo{
-					ItemType: details.TeamsChat,
-					Chat:     details.ChatInfo{},
-				}
-
-				return chat, i
-			},
-		},
-		{
-			name: "All fields",
-			chatAndExpected: func() (models.Chatable, *details.TeamsChatsInfo) {
-				now := time.Now()
-				then := now.Add(1 * time.Hour)
-
-				chat := models.NewChat()
-				chat.SetTopic(ptr.To("Hello world"))
-				chat.SetCreatedDateTime(&now)
-				chat.SetLastUpdatedDateTime(&then)
-
-				i := &details.TeamsChatsInfo{
-					ItemType: details.TeamsChat,
-					Chat: details.ChatInfo{
-						Name: "Hello world",
-					},
-				}
-
-				return chat, i
-			},
-		},
-	}
-	for _, test := range tests {
-		suite.Run(test.name, func() {
-			t := suite.T()
-			chat, expected := test.chatAndExpected()
-			result := TeamsChatInfo(chat)
-
-			assert.Equal(t, expected.Chat.Name, result.Chat.Name)
-
-			expectLastUpdated := chat.GetLastUpdatedDateTime()
-			if expectLastUpdated != nil {
-				assert.Equal(t, ptr.Val(expectLastUpdated), result.Modified)
-			} else {
-				assert.True(t, result.Modified.After(start))
-			}
-
-			expectCreated := chat.GetCreatedDateTime()
-			if expectCreated != nil {
-				assert.Equal(t, ptr.Val(expectCreated), result.Chat.CreatedAt)
-			} else {
-				assert.True(t, result.Chat.CreatedAt.After(start))
-			}
-		})
-	}
-}

src/pkg/services/m365/api/teamsChats.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"time"
 
 	"github.com/microsoftgraph/msgraph-sdk-go/chats"
 	"github.com/microsoftgraph/msgraph-sdk-go/models"
@@ -62,13 +63,47 @@ func (c Chats) GetChatByID(
 // ---------------------------------------------------------------------------
 
 func TeamsChatInfo(chat models.Chatable) *details.TeamsChatsInfo {
+	var (
+		// in case of an empty chat, we want to use Val instead of OrNow
+		lastModTime      = ptr.Val(chat.GetLastUpdatedDateTime())
+		lastMsgPreview   = chat.GetLastMessagePreview()
+		lastMsgCreatedAt time.Time
+		members          = chat.GetMembers()
+		memberNames      = []string{}
+		msgs             = chat.GetMessages()
+		preview          string
+		err              error
+	)
+
+	if lastMsgPreview != nil {
+		preview, _, err = getChatMessageContentPreview(lastMsgPreview, noAttachments{})
+		if err != nil {
+			preview = "malformed or unparseable html" + preview
+		}
+
+		// in case of an empty mod time, we want to use the chat's mod time
+		// therefore Val instaed of OrNow
+		lastMsgCreatedAt = ptr.Val(lastMsgPreview.GetCreatedDateTime())
+		if lastModTime.Before(lastMsgCreatedAt) {
+			lastModTime = lastMsgCreatedAt
+		}
+	}
+
+	for _, m := range members {
+		memberNames = append(memberNames, ptr.Val(m.GetDisplayName()))
+	}
+
 	return &details.TeamsChatsInfo{
 		ItemType: details.TeamsChat,
-		Modified: ptr.OrNow(chat.GetLastUpdatedDateTime()),
+		Modified: lastModTime,
 
 		Chat: details.ChatInfo{
-			CreatedAt: ptr.OrNow(chat.GetCreatedDateTime()),
+			CreatedAt:          ptr.OrNow(chat.GetCreatedDateTime()),
-			Name:      ptr.Val(chat.GetTopic()),
+			LastMessageAt:      lastMsgCreatedAt,
+			LastMessagePreview: preview,
+			Members:            memberNames,
+			MessageCount:       len(msgs),
+			Name:               ptr.Val(chat.GetTopic()),
 		},
 	}
 }

src/pkg/services/m365/api/teamsChats_pager.go

@@ -85,26 +85,6 @@ func (c Chats) GetChatMessages(
 	return items, graph.Stack(ctx, err).OrNil()
 }
 
-// GetChatMessageIDs fetches a delta of all messages in the chat.
-// returns two maps: addedItems, deletedItems
-func (c Chats) GetChatMessageIDs(
-	ctx context.Context,
-	chatID string,
-	cc CallConfig,
-) (pagers.AddedAndRemoved, error) {
-	aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
-		ctx,
-		c.NewChatMessagePager(chatID, CallConfig{}),
-		nil,
-		"",
-		false, // delta queries are not supported
-		0,
-		pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable],
-		IsNotSystemMessage)
-
-	return aar, clues.Stack(err).OrNil()
-}
-
 // ---------------------------------------------------------------------------
 // chat pager
 // ---------------------------------------------------------------------------

src/pkg/services/m365/api/teamsChats_test.go

@@ -0,0 +1,158 @@
+package api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/alcionai/corso/src/internal/common/ptr"
+	"github.com/alcionai/corso/src/internal/m365/collection/teamschats/testdata"
+	"github.com/alcionai/corso/src/internal/tester"
+	"github.com/alcionai/corso/src/pkg/backup/details"
+)
+
+type ChatsAPIUnitSuite struct {
+	tester.Suite
+}
+
+func TestChatsAPIUnitSuite(t *testing.T) {
+	suite.Run(t, &ChatsAPIUnitSuite{Suite: tester.NewUnitSuite(t)})
+}
+
+func (suite *ChatsAPIUnitSuite) TestChatsInfo() {
+	start := time.Now()
+
+	tests := []struct {
+		name     string
+		expected func() (models.Chatable, *details.TeamsChatsInfo)
+	}{
+		{
+			name: "Empty chat",
+			expected: func() (models.Chatable, *details.TeamsChatsInfo) {
+				chat := models.NewChat()
+
+				i := &details.TeamsChatsInfo{
+					ItemType: details.TeamsChat,
+					Modified: ptr.Val(chat.GetLastUpdatedDateTime()),
+					Chat:     details.ChatInfo{},
+				}
+
+				return chat, i
+			},
+		},
+		{
+			name: "All fields",
+			expected: func() (models.Chatable, *details.TeamsChatsInfo) {
+				now := time.Now()
+				then := now.Add(1 * time.Hour)
+				id := uuid.NewString()
+
+				chat := testdata.StubChats(id)[0]
+				chat.SetTopic(ptr.To("Hello world"))
+				chat.SetCreatedDateTime(&now)
+				chat.SetLastUpdatedDateTime(&now)
+				chat.GetLastMessagePreview().SetCreatedDateTime(&then)
+
+				msgs := testdata.StubChatMessages(ptr.Val(chat.GetLastMessagePreview().GetId()))
+				chat.SetMessages(msgs)
+
+				i := &details.TeamsChatsInfo{
+					ItemType: details.TeamsChat,
+					Modified: then,
+					Chat: details.ChatInfo{
+						Name:               "Hello world",
+						LastMessageAt:      then,
+						LastMessagePreview: id,
+						Members:            []string{},
+						MessageCount:       1,
+					},
+				}
+
+				return chat, i
+			},
+		},
+		{
+			name: "last message preview, but no messages",
+			expected: func() (models.Chatable, *details.TeamsChatsInfo) {
+				now := time.Now()
+				then := now.Add(1 * time.Hour)
+				id := uuid.NewString()
+
+				chat := testdata.StubChats(id)[0]
+				chat.SetTopic(ptr.To("Hello world"))
+				chat.SetCreatedDateTime(&now)
+				chat.SetLastUpdatedDateTime(&now)
+				chat.GetLastMessagePreview().SetCreatedDateTime(&then)
+
+				i := &details.TeamsChatsInfo{
+					ItemType: details.TeamsChat,
+					Modified: then,
+					Chat: details.ChatInfo{
+						Name:               "Hello world",
+						LastMessageAt:      then,
+						LastMessagePreview: id,
+						Members:            []string{},
+						MessageCount:       0,
+					},
+				}
+
+				return chat, i
+			},
+		},
+		{
+			name: "chat only, no messages",
+			expected: func() (models.Chatable, *details.TeamsChatsInfo) {
+				now := time.Now()
+				then := now.Add(1 * time.Hour)
+
+				chat := testdata.StubChats(uuid.NewString())[0]
+				chat.SetTopic(ptr.To("Hello world"))
+				chat.SetCreatedDateTime(&now)
+				chat.SetLastUpdatedDateTime(&then)
+				chat.SetLastMessagePreview(nil)
+				chat.SetMessages(nil)
+
+				i := &details.TeamsChatsInfo{
+					ItemType: details.TeamsChat,
+					Modified: then,
+					Chat: details.ChatInfo{
+						Name:               "Hello world",
+						LastMessageAt:      time.Time{},
+						LastMessagePreview: "",
+						Members:            []string{},
+						MessageCount:       0,
+					},
+				}
+
+				return chat, i
+			},
+		},
+	}
+	for _, test := range tests {
+		suite.Run(test.name, func() {
+			t := suite.T()
+			chat, expected := test.expected()
+			result := TeamsChatInfo(chat)
+
+			assert.Equal(t, expected.Chat.Name, result.Chat.Name)
+
+			expectCreated := chat.GetCreatedDateTime()
+			if expectCreated != nil {
+				assert.Equal(t, ptr.Val(expectCreated), result.Chat.CreatedAt)
+			} else {
+				assert.True(t, result.Chat.CreatedAt.After(start))
+			}
+
+			assert.Truef(
+				t,
+				expected.Modified.Equal(result.Modified),
+				"modified time doesn't match\nexpected %v\ngot %v",
+				expected.Modified,
+				result.Modified)
+		})
+	}
+}