diff --git a/src/pkg/services/m365/api/channels.go b/src/pkg/services/m365/api/channels.go
index e178c2340..835a37b84 100644
--- a/src/pkg/services/m365/api/channels.go
+++ b/src/pkg/services/m365/api/channels.go
@@ -170,7 +170,7 @@ func ChannelMessageInfo(
LastReplyAt: lastReply,
Modified: modTime,
MessageCreator: GetChatMessageFrom(msg),
- MessagePreview: str.Preview(content, 16),
+ MessagePreview: str.Preview(content, 128),
ReplyCount: len(msg.GetReplies()),
Size: int64(len(content)),
}
diff --git a/src/pkg/services/m365/api/channels_pager.go b/src/pkg/services/m365/api/channels_pager.go
index e9fb76637..fe3144de6 100644
--- a/src/pkg/services/m365/api/channels_pager.go
+++ b/src/pkg/services/m365/api/channels_pager.go
@@ -144,6 +144,25 @@ func (c Channels) NewChannelMessageDeltaPager(
}
}
+// this is the message content for system chatMessage entities with type
+// unknownFutureValue.
+const channelMessageSystemMessageContent = ""
+
+func FilterOutSystemMessages(cm models.ChatMessageable) bool {
+ if ptr.Val(cm.GetMessageType()) == models.SYSTEMEVENTMESSAGE_CHATMESSAGETYPE {
+ return false
+ }
+
+ content := ""
+
+ if cm.GetBody() != nil {
+ content = ptr.Val(cm.GetBody().GetContent())
+ }
+
+ return !(ptr.Val(cm.GetMessageType()) == models.UNKNOWNFUTUREVALUE_CHATMESSAGETYPE &&
+ content == channelMessageSystemMessageContent)
+}
+
// GetChannelMessageIDsDelta fetches a delta of all messages in the channel.
// returns two maps: addedItems, deletedItems
func (c Channels) GetChannelMessageIDs(
@@ -157,7 +176,8 @@ func (c Channels) GetChannelMessageIDs(
c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink),
prevDeltaLink,
canMakeDeltaQueries,
- addedAndRemovedByDeletedDateTime[models.ChatMessageable])
+ addedAndRemovedByDeletedDateTime[models.ChatMessageable],
+ FilterOutSystemMessages)
return added, validModTimes, removed, du, clues.Stack(err).OrNil()
}
diff --git a/src/pkg/services/m365/api/channels_pager_test.go b/src/pkg/services/m365/api/channels_pager_test.go
index 2782fd55e..f9ce3c06e 100644
--- a/src/pkg/services/m365/api/channels_pager_test.go
+++ b/src/pkg/services/m365/api/channels_pager_test.go
@@ -5,6 +5,7 @@ import (
"time"
"github.com/alcionai/clues"
+ "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@@ -125,7 +126,7 @@ func testEnumerateChannelMessageReplies(
assert.Equal(t, len(replies), info.ReplyCount)
assert.Equal(t, msg.GetFrom().GetUser().GetDisplayName(), info.MessageCreator)
assert.Equal(t, lastReply, info.LastReplyAt)
- assert.Equal(t, str.Preview(ptr.Val(msg.GetBody().GetContent()), 16), info.MessagePreview)
+ assert.Equal(t, str.Preview(ptr.Val(msg.GetBody().GetContent()), 128), info.MessagePreview)
msgReplyIDs := map[string]struct{}{}
@@ -135,3 +136,67 @@ func testEnumerateChannelMessageReplies(
assert.Equal(t, replyIDs, msgReplyIDs)
}
+
+func (suite *ChannelsPagerIntgSuite) TestFilterOutSystemMessages() {
+ systemMessage := models.NewChatMessage()
+ systemMessage.SetMessageType(ptr.To(models.SYSTEMEVENTMESSAGE_CHATMESSAGETYPE))
+
+ systemMessageBody := models.NewItemBody()
+ systemMessageBody.SetContent(ptr.To(""))
+
+ messageBody := models.NewItemBody()
+ messageBody.SetContent(ptr.To("message"))
+
+ unknownFutureSystemMessage := models.NewChatMessage()
+ unknownFutureSystemMessage.SetMessageType(ptr.To(models.UNKNOWNFUTUREVALUE_CHATMESSAGETYPE))
+ unknownFutureSystemMessage.SetBody(systemMessageBody)
+
+ unknownFutureMessage := models.NewChatMessage()
+ unknownFutureMessage.SetMessageType(ptr.To(models.UNKNOWNFUTUREVALUE_CHATMESSAGETYPE))
+ unknownFutureMessage.SetBody(messageBody)
+
+ regularSystemMessage := models.NewChatMessage()
+ regularSystemMessage.SetMessageType(ptr.To(models.MESSAGE_CHATMESSAGETYPE))
+ regularSystemMessage.SetBody(systemMessageBody)
+
+ regularMessage := models.NewChatMessage()
+ regularMessage.SetMessageType(ptr.To(models.MESSAGE_CHATMESSAGETYPE))
+ regularMessage.SetBody(messageBody)
+
+ table := []struct {
+ name string
+ cm models.ChatMessageable
+ expect assert.BoolAssertionFunc
+ }{
+ {
+ name: "system message type",
+ cm: systemMessage,
+ expect: assert.False,
+ },
+ {
+ name: "unknown future type system body",
+ cm: unknownFutureSystemMessage,
+ expect: assert.False,
+ },
+ {
+ name: "unknown future type message body",
+ cm: unknownFutureMessage,
+ expect: assert.True,
+ },
+ {
+ name: "message type system body",
+ cm: regularSystemMessage,
+ expect: assert.True,
+ },
+ {
+ name: "message type message body",
+ cm: regularMessage,
+ expect: assert.True,
+ },
+ }
+ for _, test := range table {
+ suite.Run(test.name, func() {
+ test.expect(suite.T(), api.FilterOutSystemMessages(test.cm))
+ })
+ }
+}
diff --git a/src/pkg/services/m365/api/item_pager.go b/src/pkg/services/m365/api/item_pager.go
index 5423a9869..4e524a58f 100644
--- a/src/pkg/services/m365/api/item_pager.go
+++ b/src/pkg/services/m365/api/item_pager.go
@@ -209,7 +209,12 @@ func deltaEnumerateItems[T any](
type addedAndRemovedHandler[T any] func(
items []T,
-) (map[string]time.Time, []string, error)
+ filters ...func(T) bool, // false -> remove, true -> keep
+) (
+ map[string]time.Time,
+ []string,
+ error,
+)
func getAddedAndRemovedItemIDs[T any](
ctx context.Context,
@@ -218,6 +223,7 @@ func getAddedAndRemovedItemIDs[T any](
prevDeltaLink string,
canMakeDeltaQueries bool,
aarh addedAndRemovedHandler[T],
+ filters ...func(T) bool,
) (map[string]time.Time, bool, []string, DeltaUpdate, error) {
if canMakeDeltaQueries {
ts, du, err := deltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink)
@@ -226,7 +232,7 @@ func getAddedAndRemovedItemIDs[T any](
}
if err == nil {
- a, r, err := aarh(ts)
+ a, r, err := aarh(ts, filters...)
return a, deltaPager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
}
}
@@ -238,7 +244,7 @@ func getAddedAndRemovedItemIDs[T any](
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err)
}
- a, r, err := aarh(ts)
+ a, r, err := aarh(ts, filters...)
return a, pager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
}
@@ -264,11 +270,22 @@ type getModTimer interface {
func addedAndRemovedByAddtlData[T any](
items []T,
+ filters ...func(T) bool,
) (map[string]time.Time, []string, error) {
added := map[string]time.Time{}
removed := []string{}
for _, item := range items {
+ passAllFilters := true
+
+ for _, passes := range filters {
+ passAllFilters = passAllFilters && passes(item)
+ }
+
+ if !passAllFilters {
+ continue
+ }
+
giaa, ok := any(item).(getIDModAndAddtler)
if !ok {
return nil, nil, clues.New("item does not provide id and additional data getters").
@@ -312,11 +329,22 @@ type getIDModAndDeletedDateTimer interface {
func addedAndRemovedByDeletedDateTime[T any](
items []T,
+ filters ...func(T) bool,
) (map[string]time.Time, []string, error) {
added := map[string]time.Time{}
removed := []string{}
for _, item := range items {
+ passAllFilters := true
+
+ for _, passes := range filters {
+ passAllFilters = passAllFilters && passes(item)
+ }
+
+ if !passAllFilters {
+ continue
+ }
+
giaddt, ok := any(item).(getIDModAndDeletedDateTimer)
if !ok {
return nil, nil, clues.New("item does not provide id and deleted date time getters").
diff --git a/src/pkg/services/m365/api/item_pager_test.go b/src/pkg/services/m365/api/item_pager_test.go
index 63edb4989..01398a343 100644
--- a/src/pkg/services/m365/api/item_pager_test.go
+++ b/src/pkg/services/m365/api/item_pager_test.go
@@ -313,6 +313,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
*testing.T,
) DeltaPager[testItem]
prevDelta string
+ filter func(a testItem) bool
expect expected
canDelta bool
validModTimes bool
@@ -454,6 +455,57 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
},
canDelta: false,
},
+ {
+ name: "no prev delta and fail all filter",
+ pagerGetter: func(t *testing.T) Pager[testItem] {
+ return nil
+ },
+ deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] {
+ return &testIDsDeltaPager{
+ t: t,
+ added: map[string]time.Time{
+ "uno": now.Add(time.Minute),
+ "dos": now.Add(2 * time.Minute),
+ },
+ removed: []string{"tres", "quatro"},
+ validModTimes: true,
+ }
+ },
+ filter: func(testItem) bool { return false },
+ expect: expected{
+ added: map[string]time.Time{},
+ removed: []string{},
+ deltaUpdate: DeltaUpdate{Reset: true},
+ validModTimes: true,
+ },
+ canDelta: true,
+ },
+ {
+ name: "with prev delta and fail all filter",
+ pagerGetter: func(t *testing.T) Pager[testItem] {
+ return nil
+ },
+ deltaPagerGetter: func(t *testing.T) DeltaPager[testItem] {
+ return &testIDsDeltaPager{
+ t: t,
+ added: map[string]time.Time{
+ "uno": now.Add(time.Minute),
+ "dos": now.Add(2 * time.Minute),
+ },
+ removed: []string{"tres", "quatro"},
+ validModTimes: true,
+ }
+ },
+ filter: func(testItem) bool { return false },
+ prevDelta: "delta",
+ expect: expected{
+ added: map[string]time.Time{},
+ removed: []string{},
+ deltaUpdate: DeltaUpdate{Reset: false},
+ validModTimes: true,
+ },
+ canDelta: true,
+ },
}
for _, test := range tests {
@@ -463,13 +515,19 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
ctx, flush := tester.NewContext(t)
defer flush()
+ filters := []func(testItem) bool{}
+ if test.filter != nil {
+ filters = append(filters, test.filter)
+ }
+
added, validModTimes, removed, deltaUpdate, err := getAddedAndRemovedItemIDs[testItem](
ctx,
test.pagerGetter(t),
test.deltaPagerGetter(t),
test.prevDelta,
test.canDelta,
- addedAndRemovedByAddtlData[testItem])
+ addedAndRemovedByAddtlData[testItem],
+ filters...)
require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err))
if validModTimes {