larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lazily fetch messages in chat

Browse Source 
once we're past kopia assits, the chat can
download all of its messages and store them in the body
uploded to kopia.
This commit is contained in:
ryanfkeepers 2024-01-24 12:17:40 -07:00
parent 80d7d5c63d
commit 316e5f0195
12 changed files with 291 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/internal/m365/collection/teamschats/backup_test.go
View File

@ -96,15 +96,13 @@ func (bh mockBackupHandler) CanonicalPath() (path.Path, error) {
func (bh mockBackupHandler) getItem(
_ context.Context,
_ string,
itemID string,
chat models.Chatable,
) (models.Chatable, *details.TeamsChatsInfo, error) {
chat := models.NewChat()
chatID := ptr.Val(chat.GetId())
chat.SetId(ptr.To(itemID))
chat.SetTopic(ptr.To(itemID))
chat.SetMessages(bh.chatMessages[itemID])
chat.SetMessages(bh.chatMessages[chatID])
return chat, bh.info[itemID], bh.getMessageErr[itemID]
return chat, bh.info[chatID], bh.getMessageErr[chatID]
}
// ---------------------------------------------------------------------------

23
src/internal/m365/collection/teamschats/chat_handler.go
View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/errs/core"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
@ -80,10 +81,26 @@ func (bh usersChatsBackupHandler) CanonicalPath() (path.Path, error) {
func (bh usersChatsBackupHandler) getItem(
ctx context.Context,
userID string,
chatID string,
chat models.Chatable,
) (models.Chatable, *details.TeamsChatsInfo, error) {
// FIXME: should retrieve and populate all messages in the chat.
return nil, nil, clues.New("not implemented")
if chat == nil {
return nil, nil, clues.Stack(core.ErrNotFound)
}
chatID := ptr.Val(chat.GetId())
cc := api.CallConfig{
Expand: []string{"lastMessagePreview"},
}
msgs, err := bh.ac.GetChatMessages(ctx, chatID, cc)
if err != nil {
return nil, nil, clues.Stack(err)
}
chat.SetMessages(msgs)
return chat, api.TeamsChatInfo(chat), nil
}
//lint:ignore U1000 false linter issue due to generics

19
src/internal/m365/collection/teamschats/collection.go
View File

@ -152,20 +152,17 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
break
}
itemID := ptr.Val(item.GetId())
modTime := ptr.Val(item.GetLastUpdatedDateTime())
wg.Add(1)
semaphoreCh <- struct{}{}
go func(id string, modTime time.Time) {
go func(item I, modTime time.Time) {
defer wg.Done()
defer func() { <-semaphoreCh }()
ictx := clues.Add(
ctx,
"item_id", id,
"parent_path", path.LoggableDir(col.LocationPath().String()))
itemID := ptr.Val(item.GetId())
ictx := clues.Add(ctx, "item_id", itemID)
col.stream <- data.NewLazyItemWithInfo(
ictx,
@ -173,12 +170,12 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
modTime: modTime,
getAndAugment: col.getAndAugment,
resourceID: col.protectedResource,
itemID: id,
item: item,
containerIDs: col.FullPath().Folders(),
contains: col.contains,
parentPath: col.LocationPath().String(),
},
id,
itemID,
modTime,
col.Counter,
el)
@ -188,7 +185,7 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
if progressMessage != nil {
progressMessage <- struct{}{}
}
}(itemID, modTime)
}(item, modTime)
}
wg.Wait()
@ -197,7 +194,7 @@ func (col *lazyFetchCollection[I]) streamItems(ctx context.Context, errs *fault.
type lazyItemGetter[I chatsItemer] struct {
getAndAugment getItemAndAugmentInfoer[I]
resourceID string
itemID string
item I
parentPath string
containerIDs path.Elements
modTime time.Time
@ -214,7 +211,7 @@ func (lig *lazyItemGetter[I]) GetData(
item, info, err := lig.getAndAugment.getItem(
ctx,
lig.resourceID,
lig.itemID)
lig.item)
if err != nil {
// For items that were deleted in flight, add the skip label so that
// they don't lead to recoverable failures during backup.

24
src/internal/m365/collection/teamschats/collection_test.go
View File

@ -9,6 +9,7 @@ import (
"time"
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -155,11 +156,9 @@ type getAndAugmentChat struct {
func (m getAndAugmentChat) getItem(
_ context.Context,
_ string,
itemID string,
chat models.Chatable,
) (models.Chatable, *details.TeamsChatsInfo, error) {
chat := models.NewChat()
chat.SetId(ptr.To(itemID))
chat.SetTopic(ptr.To(itemID))
chat.SetTopic(chat.GetId())
return chat, &details.TeamsChatsInfo{}, m.err
}
@ -277,6 +276,8 @@ func (suite *CollectionUnitSuite) TestLazyItem_GetDataErrors() {
ctx, flush := tester.NewContext(t)
defer flush()
chat := testdata.StubChats(uuid.NewString())[0]
m := getAndAugmentChat{
err: test.getErr,
}
@ -285,12 +286,12 @@ func (suite *CollectionUnitSuite) TestLazyItem_GetDataErrors() {
ctx,
&lazyItemGetter[models.Chatable]{
resourceID: "resourceID",
itemID: "itemID",
item: chat,
getAndAugment: &m,
modTime: now,
parentPath: parentPath,
},
"itemID",
ptr.Val(chat.GetId()),
now,
count.New(),
fault.New(true))
@ -319,6 +320,8 @@ func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlig
ctx, flush := tester.NewContext(t)
defer flush()
chat := testdata.StubChats(uuid.NewString())[0]
m := getAndAugmentChat{
err: core.ErrNotFound,
}
@ -327,12 +330,12 @@ func (suite *CollectionUnitSuite) TestLazyItem_ReturnsEmptyReaderOnDeletedInFlig
ctx,
&lazyItemGetter[models.Chatable]{
resourceID: "resourceID",
itemID: "itemID",
item: chat,
getAndAugment: &m,
modTime: now,
parentPath: parentPath,
},
"itemID",
ptr.Val(chat.GetId()),
now,
count.New(),
fault.New(true))
@ -359,18 +362,19 @@ func (suite *CollectionUnitSuite) TestLazyItem() {
ctx, flush := tester.NewContext(t)
defer flush()
chat := testdata.StubChats(uuid.NewString())[0]
m := getAndAugmentChat{}
li := data.NewLazyItemWithInfo(
ctx,
&lazyItemGetter[models.Chatable]{
resourceID: "resourceID",
itemID: "itemID",
item: chat,
getAndAugment: &m,
modTime: now,
parentPath: parentPath,
},
"itemID",
ptr.Val(chat.GetId()),
now,
count.New(),
fault.New(true))

2
src/internal/m365/collection/teamschats/handlers.go
View File

@ -62,7 +62,7 @@ type getItemer[I chatsItemer] interface {
getItem(
ctx context.Context,
protectedResource string,
itemID string,
i I,
) (I, *details.TeamsChatsInfo, error)
}

33
src/internal/m365/collection/teamschats/testdata/chats.go vendored
View File

@ -11,11 +11,21 @@ func StubChats(ids ...string) []models.Chatable {
sl := make([]models.Chatable, 0, len(ids))
for _, id := range ids {
ch := models.NewChat()
ch.SetTopic(ptr.To(id))
ch.SetId(ptr.To(id))
chat := models.NewChat()
chat.SetTopic(ptr.To(id))
chat.SetId(ptr.To(id))
sl = append(sl, ch)
// we should expect to get the latest message preview by default
lastMsgPrv := models.NewChatMessageInfo()
lastMsgPrv.SetId(ptr.To(uuid.NewString()))
body := models.NewItemBody()
body.SetContent(ptr.To(id))
lastMsgPrv.SetBody(body)
chat.SetLastMessagePreview(lastMsgPrv)
sl = append(sl, chat)
}
return sl
@ -24,17 +34,24 @@ func StubChats(ids ...string) []models.Chatable {
func StubChatMessages(ids ...string) []models.ChatMessageable {
sl := make([]models.ChatMessageable, 0, len(ids))
var lastMsg models.ChatMessageable
for _, id := range ids {
cm := models.NewChatMessage()
cm.SetId(ptr.To(uuid.NewString()))
msg := models.NewChatMessage()
msg.SetId(ptr.To(uuid.NewString()))
body := models.NewItemBody()
body.SetContent(ptr.To(id))
cm.SetBody(body)
msg.SetBody(body)
sl = append(sl, cm)
sl = append(sl, msg)
lastMsg = msg
}
lastMsgPrv := models.NewChatMessageInfo()
lastMsgPrv.SetId(lastMsg.GetId())
lastMsgPrv.SetBody(lastMsg.GetBody())
return sl
}

28
src/pkg/services/m365/api/channels.go
View File

@ -162,7 +162,7 @@ func channelMessageInfo(
modTime = lastReplyAt
}
preview, contentLen, err := getChatMessageContentPreview(msg)
preview, contentLen, err := getChatMessageContentPreview(msg, msg)
if err != nil {
preview = "malformed or unparseable html" + preview
}
@ -180,7 +180,7 @@ func channelMessageInfo(
var lr details.ChannelMessageInfo
if lastReply != nil {
preview, contentLen, err = getChatMessageContentPreview(lastReply)
preview, contentLen, err = getChatMessageContentPreview(lastReply, lastReply)
if err != nil {
preview = "malformed or unparseable html: " + preview
}
@ -239,12 +239,28 @@ func GetChatMessageFrom(msg models.ChatMessageable) string {
return ""
}
func getChatMessageContentPreview(msg models.ChatMessageable) (string, int64, error) {
content, origSize, err := stripChatMessageHTML(msg)
// a hack for fulfilling getAttachmentser when the model doesn't
// provide GetAttachments()
type noAttachments struct{}
func (noAttachments) GetAttachments() []models.ChatMessageAttachmentable {
return []models.ChatMessageAttachmentable{}
}
type getBodyer interface {
GetBody() models.ItemBodyable
}
type getAttachmentser interface {
GetAttachments() []models.ChatMessageAttachmentable
}
func getChatMessageContentPreview(msg getBodyer, atts getAttachmentser) (string, int64, error) {
content, origSize, err := stripChatMessageHTML(msg, atts)
return str.Preview(content, 128), origSize, clues.Stack(err).OrNil()
}
func stripChatMessageHTML(msg models.ChatMessageable) (string, int64, error) {
func stripChatMessageHTML(msg getBodyer, atts getAttachmentser) (string, int64, error) {
var (
content string
origSize int64
@ -256,7 +272,7 @@ func stripChatMessageHTML(msg models.ChatMessageable) (string, int64, error) {
origSize = int64(len(content))
content = replaceAttachmentMarkup(content, msg.GetAttachments())
content = replaceAttachmentMarkup(content, atts.GetAttachments())
content, err := html2text.FromString(content)
return content, origSize, clues.Stack(err).OrNil()

2
src/pkg/services/m365/api/channels_test.go
View File

@ -712,7 +712,7 @@ func (suite *ChannelsAPIUnitSuite) TestStripChatMessageContent() {
msg.SetAttachments(test.attachments)
// not testing len; it's effectively covered by the content assertion
result, _, err := stripChatMessageHTML(msg)
result, _, err := stripChatMessageHTML(msg, msg)
assert.Equal(t, test.expect, result)
test.expectErr(t, err, clues.ToCore(err))
})

89
src/pkg/services/m365/api/teamdChats_test.go
View File

@ -1,89 +0,0 @@
package api
import (
"testing"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
)
type ChatsAPIUnitSuite struct {
tester.Suite
}
func TestChatsAPIUnitSuite(t *testing.T) {
suite.Run(t, &ChatsAPIUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ChatsAPIUnitSuite) TestChatsInfo() {
start := time.Now()
tests := []struct {
name string
chatAndExpected func() (models.Chatable, *details.TeamsChatsInfo)
}{
{
name: "Empty chat",
chatAndExpected: func() (models.Chatable, *details.TeamsChatsInfo) {
chat := models.NewChat()
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Chat: details.ChatInfo{},
}
return chat, i
},
},
{
name: "All fields",
chatAndExpected: func() (models.Chatable, *details.TeamsChatsInfo) {
now := time.Now()
then := now.Add(1 * time.Hour)
chat := models.NewChat()
chat.SetTopic(ptr.To("Hello world"))
chat.SetCreatedDateTime(&now)
chat.SetLastUpdatedDateTime(&then)
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Chat: details.ChatInfo{
Name: "Hello world",
},
}
return chat, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
chat, expected := test.chatAndExpected()
result := TeamsChatInfo(chat)
assert.Equal(t, expected.Chat.Name, result.Chat.Name)
expectLastUpdated := chat.GetLastUpdatedDateTime()
if expectLastUpdated != nil {
assert.Equal(t, ptr.Val(expectLastUpdated), result.Modified)
} else {
assert.True(t, result.Modified.After(start))
}
expectCreated := chat.GetCreatedDateTime()
if expectCreated != nil {
assert.Equal(t, ptr.Val(expectCreated), result.Chat.CreatedAt)
} else {
assert.True(t, result.Chat.CreatedAt.After(start))
}
})
}
}

37
src/pkg/services/m365/api/teamsChats.go
View File

@ -2,6 +2,7 @@ package api
import (
"context"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/chats"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -62,12 +63,46 @@ func (c Chats) GetChatByID(
// ---------------------------------------------------------------------------
func TeamsChatInfo(chat models.Chatable) *details.TeamsChatsInfo {
var (
// in case of an empty chat, we want to use Val instead of OrNow
lastModTime = ptr.Val(chat.GetLastUpdatedDateTime())
lastMsgPreview = chat.GetLastMessagePreview()
lastMsgCreatedAt time.Time
members = chat.GetMembers()
memberNames = []string{}
msgs = chat.GetMessages()
preview string
err error
)
if lastMsgPreview != nil {
preview, _, err = getChatMessageContentPreview(lastMsgPreview, noAttachments{})
if err != nil {
preview = "malformed or unparseable html" + preview
}
// in case of an empty mod time, we want to use the chat's mod time
// therefore Val instaed of OrNow
lastMsgCreatedAt = ptr.Val(lastMsgPreview.GetCreatedDateTime())
if lastModTime.Before(lastMsgCreatedAt) {
lastModTime = lastMsgCreatedAt
}
}
for _, m := range members {
memberNames = append(memberNames, ptr.Val(m.GetDisplayName()))
}
return &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Modified: ptr.OrNow(chat.GetLastUpdatedDateTime()),
Modified: lastModTime,
Chat: details.ChatInfo{
CreatedAt: ptr.OrNow(chat.GetCreatedDateTime()),
LastMessageAt: lastMsgCreatedAt,
LastMessagePreview: preview,
Members: memberNames,
MessageCount: len(msgs),
Name: ptr.Val(chat.GetTopic()),
},
}

20
src/pkg/services/m365/api/teamsChats_pager.go
View File

@ -85,26 +85,6 @@ func (c Chats) GetChatMessages(
return items, graph.Stack(ctx, err).OrNil()
}
// GetChatMessageIDs fetches a delta of all messages in the chat.
// returns two maps: addedItems, deletedItems
func (c Chats) GetChatMessageIDs(
ctx context.Context,
chatID string,
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
ctx,
c.NewChatMessagePager(chatID, CallConfig{}),
nil,
"",
false, // delta queries are not supported
0,
pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable],
IsNotSystemMessage)
return aar, clues.Stack(err).OrNil()
}
// ---------------------------------------------------------------------------
// chat pager
// ---------------------------------------------------------------------------

158
src/pkg/services/m365/api/teamsChats_test.go Normal file
View File

@ -0,0 +1,158 @@
package api
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/m365/collection/teamschats/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
)
type ChatsAPIUnitSuite struct {
tester.Suite
}
func TestChatsAPIUnitSuite(t *testing.T) {
suite.Run(t, &ChatsAPIUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ChatsAPIUnitSuite) TestChatsInfo() {
start := time.Now()
tests := []struct {
name string
expected func() (models.Chatable, *details.TeamsChatsInfo)
}{
{
name: "Empty chat",
expected: func() (models.Chatable, *details.TeamsChatsInfo) {
chat := models.NewChat()
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Modified: ptr.Val(chat.GetLastUpdatedDateTime()),
Chat: details.ChatInfo{},
}
return chat, i
},
},
{
name: "All fields",
expected: func() (models.Chatable, *details.TeamsChatsInfo) {
now := time.Now()
then := now.Add(1 * time.Hour)
id := uuid.NewString()
chat := testdata.StubChats(id)[0]
chat.SetTopic(ptr.To("Hello world"))
chat.SetCreatedDateTime(&now)
chat.SetLastUpdatedDateTime(&now)
chat.GetLastMessagePreview().SetCreatedDateTime(&then)
msgs := testdata.StubChatMessages(ptr.Val(chat.GetLastMessagePreview().GetId()))
chat.SetMessages(msgs)
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Modified: then,
Chat: details.ChatInfo{
Name: "Hello world",
LastMessageAt: then,
LastMessagePreview: id,
Members: []string{},
MessageCount: 1,
},
}
return chat, i
},
},
{
name: "last message preview, but no messages",
expected: func() (models.Chatable, *details.TeamsChatsInfo) {
now := time.Now()
then := now.Add(1 * time.Hour)
id := uuid.NewString()
chat := testdata.StubChats(id)[0]
chat.SetTopic(ptr.To("Hello world"))
chat.SetCreatedDateTime(&now)
chat.SetLastUpdatedDateTime(&now)
chat.GetLastMessagePreview().SetCreatedDateTime(&then)
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Modified: then,
Chat: details.ChatInfo{
Name: "Hello world",
LastMessageAt: then,
LastMessagePreview: id,
Members: []string{},
MessageCount: 0,
},
}
return chat, i
},
},
{
name: "chat only, no messages",
expected: func() (models.Chatable, *details.TeamsChatsInfo) {
now := time.Now()
then := now.Add(1 * time.Hour)
chat := testdata.StubChats(uuid.NewString())[0]
chat.SetTopic(ptr.To("Hello world"))
chat.SetCreatedDateTime(&now)
chat.SetLastUpdatedDateTime(&then)
chat.SetLastMessagePreview(nil)
chat.SetMessages(nil)
i := &details.TeamsChatsInfo{
ItemType: details.TeamsChat,
Modified: then,
Chat: details.ChatInfo{
Name: "Hello world",
LastMessageAt: time.Time{},
LastMessagePreview: "",
Members: []string{},
MessageCount: 0,
},
}
return chat, i
},
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
chat, expected := test.expected()
result := TeamsChatInfo(chat)
assert.Equal(t, expected.Chat.Name, result.Chat.Name)
expectCreated := chat.GetCreatedDateTime()
if expectCreated != nil {
assert.Equal(t, ptr.Val(expectCreated), result.Chat.CreatedAt)
} else {
assert.True(t, result.Chat.CreatedAt.After(start))
}
assert.Truef(
t,
expected.Modified.Equal(result.Modified),
"modified time doesn't match\nexpected %v\ngot %v",
expected.Modified,
result.Modified)
})
}
}