diff --git a/src/internal/m365/collection/drive/collection.go b/src/internal/m365/collection/drive/collection.go index 4c6be1ab7..60010bc8a 100644 --- a/src/internal/m365/collection/drive/collection.go +++ b/src/internal/m365/collection/drive/collection.go @@ -208,7 +208,7 @@ func (oc *Collection) Items( ctx context.Context, errs *fault.Bus, ) <-chan data.Item { - go oc.populateItems(ctx, errs) + go oc.streamItems(ctx, errs) return oc.data } @@ -411,9 +411,9 @@ type driveStats struct { itemsFound int64 } -// populateItems iterates through items added to the collection +// streamItems iterates through items added to the collection // and uses the collection `itemReader` to read the item -func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { +func (oc *Collection) streamItems(ctx context.Context, errs *fault.Bus) { var ( stats driveStats wg sync.WaitGroup @@ -453,7 +453,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { defer func() { <-semaphoreCh }() // Read the item - oc.populateDriveItem( + oc.streamDriveItem( ctx, parentPath, item, @@ -470,7 +470,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { oc.reportAsCompleted(ctx, int(stats.itemsFound), int(stats.itemsRead), stats.byteCount) } -func (oc *Collection) populateDriveItem( +func (oc *Collection) streamDriveItem( ctx context.Context, parentPath *path.Builder, item models.DriveItemable, diff --git a/src/internal/m365/collection/drive/collection_test.go b/src/internal/m365/collection/drive/collection_test.go index 2ec7ea77d..ad2c9cc93 100644 --- a/src/internal/m365/collection/drive/collection_test.go +++ b/src/internal/m365/collection/drive/collection_test.go @@ -38,16 +38,16 @@ import ( // tests // --------------------------------------------------------------------------- -type CollectionUnitTestSuite struct { +type CollectionUnitSuite struct { tester.Suite } -func TestCollectionUnitTestSuite(t *testing.T) { - suite.Run(t, &CollectionUnitTestSuite{Suite: tester.NewUnitSuite(t)}) +func TestCollectionUnitSuite(t *testing.T) { + suite.Run(t, &CollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) } // Returns a status update function that signals the specified WaitGroup when it is done -func (suite *CollectionUnitTestSuite) testStatusUpdater( +func (suite *CollectionUnitSuite) testStatusUpdater( wg *sync.WaitGroup, statusToUpdate *support.ControllerOperationStatus, ) support.StatusUpdater { @@ -59,7 +59,7 @@ func (suite *CollectionUnitTestSuite) testStatusUpdater( } } -func (suite *CollectionUnitTestSuite) TestCollection() { +func (suite *CollectionUnitSuite) TestCollection() { var ( now = time.Now() @@ -281,7 +281,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { } } -func (suite *CollectionUnitTestSuite) TestCollectionReadError() { +func (suite *CollectionUnitSuite) TestCollectionReadError() { var ( t = suite.T() stubItemID = "fakeItemID" @@ -349,7 +349,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() { require.Equal(t, 1, collStatus.Metrics.Successes, "TODO: should be 0, but allowing 1 to reduce async management") } -func (suite *CollectionUnitTestSuite) TestCollectionReadUnauthorizedErrorRetry() { +func (suite *CollectionUnitSuite) TestCollectionReadUnauthorizedErrorRetry() { var ( t = suite.T() stubItemID = "fakeItemID" @@ -417,7 +417,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadUnauthorizedErrorRetry() } // Ensure metadata file always uses latest time for mod time -func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTime() { +func (suite *CollectionUnitSuite) TestCollectionPermissionBackupLatestModTime() { var ( t = suite.T() stubItemID = "fakeItemID" @@ -779,7 +779,7 @@ func (suite *GetDriveItemUnitTestSuite) TestDownloadContent() { } } -func (suite *CollectionUnitTestSuite) TestItemExtensions() { +func (suite *CollectionUnitSuite) TestItemExtensions() { type verifyExtensionOutput func( t *testing.T, info details.ItemInfo, diff --git a/src/internal/m365/collection/exchange/collection_test.go b/src/internal/m365/collection/exchange/collection_test.go index 03becd45f..ed15bfe3e 100644 --- a/src/internal/m365/collection/exchange/collection_test.go +++ b/src/internal/m365/collection/exchange/collection_test.go @@ -2,59 +2,33 @@ package exchange import ( "bytes" - "context" "testing" + "time" "github.com/alcionai/clues" - "github.com/microsoft/kiota-abstractions-go/serialization" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/exchange/mock" "github.com/alcionai/corso/src/internal/m365/graph" + "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) -type mockItemer struct { - getCount int - serializeCount int - getErr error - serializeErr error -} - -func (mi *mockItemer) GetItem( - context.Context, - string, string, - bool, - *fault.Bus, -) (serialization.Parsable, *details.ExchangeInfo, error) { - mi.getCount++ - return nil, nil, mi.getErr -} - -func (mi *mockItemer) Serialize( - context.Context, - serialization.Parsable, - string, string, -) ([]byte, error) { - mi.serializeCount++ - return nil, mi.serializeErr -} - -type CollectionSuite struct { +type CollectionUnitSuite struct { tester.Suite } -func TestCollectionSuite(t *testing.T) { - suite.Run(t, &CollectionSuite{Suite: tester.NewUnitSuite(t)}) +func TestCollectionUnitSuite(t *testing.T) { + suite.Run(t, &CollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *CollectionSuite) TestReader_Valid() { +func (suite *CollectionUnitSuite) TestReader_Valid() { m := []byte("test message") description := "aFile" ed := &Item{id: description, message: m} @@ -66,7 +40,7 @@ func (suite *CollectionSuite) TestReader_Valid() { assert.Equal(suite.T(), description, ed.ID()) } -func (suite *CollectionSuite) TestReader_Empty() { +func (suite *CollectionUnitSuite) TestReader_Empty() { var ( empty []byte expected int64 @@ -81,7 +55,7 @@ func (suite *CollectionSuite) TestReader_Empty() { assert.NoError(t, err, clues.ToCore(err)) } -func (suite *CollectionSuite) TestCollection_NewCollection() { +func (suite *CollectionUnitSuite) TestCollection_NewCollection() { t := suite.T() tenant := "a-tenant" user := "a-user" @@ -105,7 +79,7 @@ func (suite *CollectionSuite) TestCollection_NewCollection() { assert.Equal(t, fullPath, edc.FullPath()) } -func (suite *CollectionSuite) TestNewCollection_state() { +func (suite *CollectionUnitSuite) TestNewCollection_state() { fooP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "foo") require.NoError(suite.T(), err, clues.ToCore(err)) barP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "bar") @@ -154,7 +128,8 @@ func (suite *CollectionSuite) TestNewCollection_state() { "u", test.curr, test.prev, test.loc, 0, - &mockItemer{}, nil, + mock.DefaultItemGetSerialize(), + nil, control.DefaultOptions(), false) assert.Equal(t, test.expect, c.State(), "collection state") @@ -165,16 +140,16 @@ func (suite *CollectionSuite) TestNewCollection_state() { } } -func (suite *CollectionSuite) TestGetItemWithRetries() { +func (suite *CollectionUnitSuite) TestGetItemWithRetries() { table := []struct { name string - items *mockItemer + items *mock.ItemGetSerialize expectErr func(*testing.T, error) expectGetCalls int }{ { name: "happy", - items: &mockItemer{}, + items: mock.DefaultItemGetSerialize(), expectErr: func(t *testing.T, err error) { assert.NoError(t, err, clues.ToCore(err)) }, @@ -182,7 +157,7 @@ func (suite *CollectionSuite) TestGetItemWithRetries() { }, { name: "an error", - items: &mockItemer{getErr: assert.AnError}, + items: &mock.ItemGetSerialize{GetErr: assert.AnError}, expectErr: func(t *testing.T, err error) { assert.Error(t, err, clues.ToCore(err)) }, @@ -190,8 +165,8 @@ func (suite *CollectionSuite) TestGetItemWithRetries() { }, { name: "deleted in flight", - items: &mockItemer{ - getErr: graph.ErrDeletedInFlight, + items: &mock.ItemGetSerialize{ + GetErr: graph.ErrDeletedInFlight, }, expectErr: func(t *testing.T, err error) { assert.True(t, graph.IsErrDeletedInFlight(err), "is ErrDeletedInFlight") @@ -212,3 +187,107 @@ func (suite *CollectionSuite) TestGetItemWithRetries() { }) } } + +func (suite *CollectionUnitSuite) TestCollection_streamItems() { + var ( + t = suite.T() + start = time.Now().Add(-1 * time.Second) + statusUpdater = func(*support.ControllerOperationStatus) {} + ) + + fullPath, err := path.Build("t", "pr", path.ExchangeService, path.EmailCategory, false, "fnords", "smarf") + require.NoError(t, err, clues.ToCore(err)) + + locPath, err := path.Build("t", "pr", path.ExchangeService, path.EmailCategory, false, "fnords", "smarf") + require.NoError(t, err, clues.ToCore(err)) + + table := []struct { + name string + added map[string]struct{} + removed map[string]struct{} + }{ + { + name: "no items", + added: map[string]struct{}{}, + removed: map[string]struct{}{}, + }, + { + name: "only added items", + added: map[string]struct{}{ + "fisher": {}, + "flannigan": {}, + "fitzbog": {}, + }, + removed: map[string]struct{}{}, + }, + { + name: "only removed items", + added: map[string]struct{}{}, + removed: map[string]struct{}{ + "princess": {}, + "poppy": {}, + "petunia": {}, + }, + }, + { + name: "added and removed items", + added: map[string]struct{}{}, + removed: map[string]struct{}{ + "general": {}, + "goose": {}, + "grumbles": {}, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + var ( + t = suite.T() + errs = fault.New(true) + itemCount int + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + col := &Collection{ + added: test.added, + removed: test.removed, + ctrl: control.DefaultOptions(), + getter: &mock.ItemGetSerialize{}, + stream: make(chan data.Item), + fullPath: fullPath, + locationPath: locPath.ToBuilder(), + statusUpdater: statusUpdater, + } + + go col.streamItems(ctx, errs) + + for item := range col.stream { + itemCount++ + + _, aok := test.added[item.ID()] + if aok { + assert.False(t, item.Deleted(), "additions should not be marked as deleted") + } + + _, rok := test.removed[item.ID()] + if rok { + assert.True(t, item.Deleted(), "removals should be marked as deleted") + dimt, ok := item.(data.ItemModTime) + require.True(t, ok, "item implements data.ItemModTime") + assert.True(t, dimt.ModTime().After(start), "deleted items should set mod time to now()") + } + + assert.True(t, aok || rok, "item must be either added or removed: %q", item.ID()) + } + + assert.NoError(t, errs.Failure()) + assert.Equal( + t, + len(test.added)+len(test.removed), + itemCount, + "should see all expected items") + }) + } +} diff --git a/src/internal/m365/collection/exchange/mock/item.go b/src/internal/m365/collection/exchange/mock/item.go new file mode 100644 index 000000000..774a7faa5 --- /dev/null +++ b/src/internal/m365/collection/exchange/mock/item.go @@ -0,0 +1,40 @@ +package mock + +import ( + "context" + + "github.com/microsoft/kiota-abstractions-go/serialization" + + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" +) + +type ItemGetSerialize struct { + GetCount int + GetErr error + SerializeCount int + SerializeErr error +} + +func (m *ItemGetSerialize) GetItem( + context.Context, + string, string, + bool, + *fault.Bus, +) (serialization.Parsable, *details.ExchangeInfo, error) { + m.GetCount++ + return nil, &details.ExchangeInfo{}, m.GetErr +} + +func (m *ItemGetSerialize) Serialize( + context.Context, + serialization.Parsable, + string, string, +) ([]byte, error) { + m.SerializeCount++ + return nil, m.SerializeErr +} + +func DefaultItemGetSerialize() *ItemGetSerialize { + return &ItemGetSerialize{} +} diff --git a/src/internal/m365/collection/groups/backup_test.go b/src/internal/m365/collection/groups/backup_test.go index 60c6b4bf3..64403cd10 100644 --- a/src/internal/m365/collection/groups/backup_test.go +++ b/src/internal/m365/collection/groups/backup_test.go @@ -83,7 +83,7 @@ func (bh mockBackupHandler) canonicalPath( false) } -func (bh mockBackupHandler) getChannelMessage( +func (bh mockBackupHandler) GetChannelMessage( _ context.Context, _, _, itemID string, ) (models.ChatMessageable, *details.GroupsInfo, error) { diff --git a/src/internal/m365/collection/groups/channel_handler.go b/src/internal/m365/collection/groups/channel_handler.go index 1512e2dbf..f56f165c5 100644 --- a/src/internal/m365/collection/groups/channel_handler.go +++ b/src/internal/m365/collection/groups/channel_handler.go @@ -66,7 +66,7 @@ func (bh channelsBackupHandler) canonicalPath( false) } -func (bh channelsBackupHandler) getChannelMessage( +func (bh channelsBackupHandler) GetChannelMessage( ctx context.Context, teamID, channelID, itemID string, ) (models.ChatMessageable, *details.GroupsInfo, error) { diff --git a/src/internal/m365/collection/groups/collection.go b/src/internal/m365/collection/groups/collection.go index 23d0c8512..190c3e688 100644 --- a/src/internal/m365/collection/groups/collection.go +++ b/src/internal/m365/collection/groups/collection.go @@ -142,12 +142,9 @@ func (col Collection) DoNotMergeItems() bool { // Item represents a single item retrieved from exchange type Item struct { - id string - // TODO: We may need this to be a "oneOf" of `message`, `contact`, etc. - // going forward. Using []byte for now but I assume we'll have - // some structured type in here (serialization to []byte can be done in `Read`) + id string message []byte - info *details.GroupsInfo // temporary change to bring populate function into directory + info *details.GroupsInfo // TODO(ashmrtn): Can probably eventually be sourced from info as there's a // request to provide modtime in ItemInfo structs. modTime time.Time @@ -220,31 +217,30 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { semaphoreCh := make(chan struct{}, col.ctrl.Parallelism.ItemFetch) defer close(semaphoreCh) - // TODO: add for v1 with incrementals // delete all removed items - // for id := range col.removed { - // semaphoreCh <- struct{}{} + for id := range col.removed { + semaphoreCh <- struct{}{} - // wg.Add(1) + wg.Add(1) - // go func(id string) { - // defer wg.Done() - // defer func() { <-semaphoreCh }() + go func(id string) { + defer wg.Done() + defer func() { <-semaphoreCh }() - // col.stream <- &Item{ - // id: id, - // modTime: time.Now().UTC(), // removed items have no modTime entry. - // deleted: true, - // } + col.stream <- &Item{ + id: id, + modTime: time.Now().UTC(), // removed items have no modTime entry. + deleted: true, + } - // atomic.AddInt64(&streamedItems, 1) - // atomic.AddInt64(&totalBytes, 0) + atomic.AddInt64(&streamedItems, 1) + atomic.AddInt64(&totalBytes, 0) - // if colProgress != nil { - // colProgress <- struct{}{} - // } - // }(id) - // } + if colProgress != nil { + colProgress <- struct{}{} + } + }(id) + } // add any new items for id := range col.added { @@ -265,7 +261,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) { flds := col.fullPath.Folders() parentFolderID := flds[len(flds)-1] - item, info, err := col.getter.getChannelMessage( + item, info, err := col.getter.GetChannelMessage( ctx, col.protectedResource, parentFolderID, diff --git a/src/internal/m365/collection/groups/collection_test.go b/src/internal/m365/collection/groups/collection_test.go index 5879d64c4..de89a0808 100644 --- a/src/internal/m365/collection/groups/collection_test.go +++ b/src/internal/m365/collection/groups/collection_test.go @@ -3,6 +3,7 @@ package groups import ( "bytes" "testing" + "time" "github.com/alcionai/clues" "github.com/stretchr/testify/assert" @@ -10,20 +11,23 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/m365/collection/groups/mock" + "github.com/alcionai/corso/src/internal/m365/support" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) -type CollectionSuite struct { +type CollectionUnitSuite struct { tester.Suite } -func TestCollectionSuite(t *testing.T) { - suite.Run(t, &CollectionSuite{Suite: tester.NewUnitSuite(t)}) +func TestCollectionUnitSuite(t *testing.T) { + suite.Run(t, &CollectionUnitSuite{Suite: tester.NewUnitSuite(t)}) } -func (suite *CollectionSuite) TestReader_Valid() { +func (suite *CollectionUnitSuite) TestReader_Valid() { m := []byte("test message") description := "aFile" ed := &Item{id: description, message: m} @@ -35,7 +39,7 @@ func (suite *CollectionSuite) TestReader_Valid() { assert.Equal(suite.T(), description, ed.ID()) } -func (suite *CollectionSuite) TestReader_Empty() { +func (suite *CollectionUnitSuite) TestReader_Empty() { var ( empty []byte expected int64 @@ -50,7 +54,7 @@ func (suite *CollectionSuite) TestReader_Empty() { assert.NoError(t, err, clues.ToCore(err)) } -func (suite *CollectionSuite) TestCollection_NewCollection() { +func (suite *CollectionUnitSuite) TestCollection_NewCollection() { t := suite.T() tenant := "a-tenant" protectedResource := "a-protectedResource" @@ -74,7 +78,7 @@ func (suite *CollectionSuite) TestCollection_NewCollection() { assert.Equal(t, fullPath, edc.FullPath()) } -func (suite *CollectionSuite) TestNewCollection_state() { +func (suite *CollectionUnitSuite) TestNewCollection_state() { fooP, err := path.Build("t", "u", path.GroupsService, path.ChannelMessagesCategory, false, "foo") require.NoError(suite.T(), err, clues.ToCore(err)) barP, err := path.Build("t", "u", path.GroupsService, path.ChannelMessagesCategory, false, "bar") @@ -135,3 +139,107 @@ func (suite *CollectionSuite) TestNewCollection_state() { }) } } + +func (suite *CollectionUnitSuite) TestCollection_streamItems() { + var ( + t = suite.T() + start = time.Now().Add(-1 * time.Second) + statusUpdater = func(*support.ControllerOperationStatus) {} + ) + + fullPath, err := path.Build("t", "pr", path.GroupsService, path.ChannelMessagesCategory, false, "fnords", "smarf") + require.NoError(t, err, clues.ToCore(err)) + + locPath, err := path.Build("t", "pr", path.GroupsService, path.ChannelMessagesCategory, false, "fnords", "smarf") + require.NoError(t, err, clues.ToCore(err)) + + table := []struct { + name string + added map[string]struct{} + removed map[string]struct{} + }{ + { + name: "no items", + added: map[string]struct{}{}, + removed: map[string]struct{}{}, + }, + { + name: "only added items", + added: map[string]struct{}{ + "fisher": {}, + "flannigan": {}, + "fitzbog": {}, + }, + removed: map[string]struct{}{}, + }, + { + name: "only removed items", + added: map[string]struct{}{}, + removed: map[string]struct{}{ + "princess": {}, + "poppy": {}, + "petunia": {}, + }, + }, + { + name: "added and removed items", + added: map[string]struct{}{}, + removed: map[string]struct{}{ + "general": {}, + "goose": {}, + "grumbles": {}, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + var ( + t = suite.T() + errs = fault.New(true) + itemCount int + ) + + ctx, flush := tester.NewContext(t) + defer flush() + + col := &Collection{ + added: test.added, + removed: test.removed, + ctrl: control.DefaultOptions(), + getter: mock.GetChannelMessage{}, + stream: make(chan data.Item), + fullPath: fullPath, + locationPath: locPath.ToBuilder(), + statusUpdater: statusUpdater, + } + + go col.streamItems(ctx, errs) + + for item := range col.stream { + itemCount++ + + _, aok := test.added[item.ID()] + if aok { + assert.False(t, item.Deleted(), "additions should not be marked as deleted") + } + + _, rok := test.removed[item.ID()] + if rok { + assert.True(t, item.Deleted(), "removals should be marked as deleted") + dimt, ok := item.(data.ItemModTime) + require.True(t, ok, "item implements data.ItemModTime") + assert.True(t, dimt.ModTime().After(start), "deleted items should set mod time to now()") + } + + assert.True(t, aok || rok, "item must be either added or removed: %q", item.ID()) + } + + assert.NoError(t, errs.Failure()) + assert.Equal( + t, + len(test.added)+len(test.removed), + itemCount, + "should see all expected items") + }) + } +} diff --git a/src/internal/m365/collection/groups/handlers.go b/src/internal/m365/collection/groups/handlers.go index f46b3a099..19389a67a 100644 --- a/src/internal/m365/collection/groups/handlers.go +++ b/src/internal/m365/collection/groups/handlers.go @@ -45,7 +45,7 @@ type backupHandler interface { } type getChannelMessager interface { - getChannelMessage( + GetChannelMessage( ctx context.Context, teamID, channelID, itemID string, ) (models.ChatMessageable, *details.GroupsInfo, error) diff --git a/src/internal/m365/collection/groups/mock/getter.go b/src/internal/m365/collection/groups/mock/getter.go new file mode 100644 index 000000000..cdee35097 --- /dev/null +++ b/src/internal/m365/collection/groups/mock/getter.go @@ -0,0 +1,24 @@ +package mock + +import ( + "context" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/pkg/backup/details" +) + +type GetChannelMessage struct { + Err error +} + +func (m GetChannelMessage) GetChannelMessage( + ctx context.Context, + teamID, channelID, itemID string, +) (models.ChatMessageable, *details.GroupsInfo, error) { + msg := models.NewChatMessage() + msg.SetId(ptr.To(itemID)) + + return msg, &details.GroupsInfo{}, m.Err +} diff --git a/src/internal/operations/test/helper_test.go b/src/internal/operations/test/helper_test.go index 53742bbda..4ae9fad6f 100644 --- a/src/internal/operations/test/helper_test.go +++ b/src/internal/operations/test/helper_test.go @@ -691,11 +691,22 @@ func verifyExtensionData( ) { require.NotNil(t, itemInfo.Extension, "nil extension") assert.NotNil(t, itemInfo.Extension.Data[extensions.KNumBytes], "key not found in extension") - actualSize := int64(itemInfo.Extension.Data[extensions.KNumBytes].(float64)) - if p == path.SharePointService { - assert.Equal(t, itemInfo.SharePoint.Size, actualSize, "incorrect data in extension") - } else { - assert.Equal(t, itemInfo.OneDrive.Size, actualSize, "incorrect data in extension") + var ( + detailsSize int64 + extensionSize = int64(itemInfo.Extension.Data[extensions.KNumBytes].(float64)) + ) + + switch p { + case path.SharePointService: + detailsSize = itemInfo.SharePoint.Size + case path.OneDriveService: + detailsSize = itemInfo.OneDrive.Size + case path.GroupsService: + detailsSize = itemInfo.Groups.Size + default: + assert.Fail(t, "unrecognized data type") } + + assert.Equal(t, extensionSize, detailsSize, "incorrect size in extension") } diff --git a/src/pkg/services/m365/api/channels.go b/src/pkg/services/m365/api/channels.go index 8aab2453a..92d799d60 100644 --- a/src/pkg/services/m365/api/channels.go +++ b/src/pkg/services/m365/api/channels.go @@ -108,8 +108,6 @@ func (c Channels) GetChannelMessage( ctx context.Context, teamID, channelID, messageID string, ) (models.ChatMessageable, *details.GroupsInfo, error) { - var size int64 - message, err := c.Stable. Client(). Teams(). @@ -123,7 +121,14 @@ func (c Channels) GetChannelMessage( return nil, nil, graph.Stack(ctx, err) } - info := ChannelMessageInfo(message, size) + replies, err := c.GetChannelMessageReplies(ctx, teamID, channelID, messageID) + if err != nil { + return nil, nil, graph.Wrap(ctx, err, "retrieving message replies") + } + + message.SetReplies(replies) + + info := ChannelMessageInfo(message) return message, info, nil } @@ -134,12 +139,12 @@ func (c Channels) GetChannelMessage( func ChannelMessageInfo( msg models.ChatMessageable, - size int64, ) *details.GroupsInfo { var ( lastReply time.Time modTime = ptr.OrNow(msg.GetLastModifiedDateTime()) msgCreator string + content string ) for _, r := range msg.GetReplies() { @@ -169,15 +174,19 @@ func ChannelMessageInfo( msgCreator = ptr.Val(from.GetUser().GetDisplayName()) } + if msg.GetBody() != nil { + content = ptr.Val(msg.GetBody().GetContent()) + } + return &details.GroupsInfo{ ItemType: details.GroupsChannelMessage, Created: ptr.Val(msg.GetCreatedDateTime()), LastReplyAt: lastReply, Modified: modTime, MessageCreator: msgCreator, - MessagePreview: str.Preview(ptr.Val(msg.GetBody().GetContent()), 16), + MessagePreview: str.Preview(content, 16), ReplyCount: len(msg.GetReplies()), - Size: size, + Size: int64(len(content)), } } diff --git a/src/pkg/services/m365/api/channels_pager.go b/src/pkg/services/m365/api/channels_pager.go index ae6b0ee19..bd8932c2d 100644 --- a/src/pkg/services/m365/api/channels_pager.go +++ b/src/pkg/services/m365/api/channels_pager.go @@ -214,7 +214,9 @@ func (c Channels) GetChannelMessageReplies( ctx context.Context, teamID, channelID, messageID string, ) ([]models.ChatMessageable, error) { - return enumerateItems[models.ChatMessageable](ctx, c.NewChannelMessageRepliesPager(teamID, channelID, messageID)) + return enumerateItems[models.ChatMessageable]( + ctx, + c.NewChannelMessageRepliesPager(teamID, channelID, messageID)) } // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/channels_test.go b/src/pkg/services/m365/api/channels_test.go new file mode 100644 index 000000000..bb632ac9c --- /dev/null +++ b/src/pkg/services/m365/api/channels_test.go @@ -0,0 +1,241 @@ +package api_test + +import ( + "testing" + "time" + + "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/common/ptr" + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type ChannelsAPIUnitSuite struct { + tester.Suite +} + +func TestChannelsAPIUnitSuitee(t *testing.T) { + suite.Run(t, &ChannelsAPIUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ChannelsAPIUnitSuite) TestChannelMessageInfo() { + var ( + initial = time.Now().Add(-24 * time.Hour) + mid = time.Now().Add(-1 * time.Hour) + curr = time.Now() + + content = "content" + body = models.NewItemBody() + ) + + body.SetContent(ptr.To(content)) + + tests := []struct { + name string + msgAndInfo func() (models.ChatMessageable, *details.GroupsInfo) + }{ + { + name: "No body", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("user")) + + from := models.NewChatMessageFromIdentitySet() + from.SetUser(iden) + + msg.SetFrom(from) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: initial, + LastReplyAt: time.Time{}, + ReplyCount: 0, + MessageCreator: "user", + } + + return msg, i + }, + }, + { + name: "No Replies - created by user", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + msg.SetBody(body) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("user")) + + from := models.NewChatMessageFromIdentitySet() + from.SetUser(iden) + + msg.SetFrom(from) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: initial, + LastReplyAt: time.Time{}, + ReplyCount: 0, + MessageCreator: "user", + Size: int64(len(content)), + MessagePreview: content, + } + + return msg, i + }, + }, + { + name: "No Replies - created by application", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + msg.SetBody(body) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("app")) + + from := models.NewChatMessageFromIdentitySet() + from.SetApplication(iden) + + msg.SetFrom(from) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: initial, + LastReplyAt: time.Time{}, + ReplyCount: 0, + MessageCreator: "app", + Size: int64(len(content)), + MessagePreview: content, + } + + return msg, i + }, + }, + { + name: "No Replies - created by device", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + msg.SetBody(body) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("device")) + + from := models.NewChatMessageFromIdentitySet() + from.SetDevice(iden) + + msg.SetFrom(from) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: initial, + LastReplyAt: time.Time{}, + ReplyCount: 0, + MessageCreator: "device", + Size: int64(len(content)), + MessagePreview: content, + } + + return msg, i + }, + }, + { + name: "One Reply", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + msg.SetBody(body) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("user")) + + from := models.NewChatMessageFromIdentitySet() + from.SetUser(iden) + + msg.SetFrom(from) + + reply := models.NewChatMessage() + reply.SetCreatedDateTime(&curr) + reply.SetLastModifiedDateTime(&curr) + + msg.SetReplies([]models.ChatMessageable{reply}) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: curr, + LastReplyAt: curr, + ReplyCount: 1, + MessageCreator: "user", + Size: int64(len(content)), + MessagePreview: content, + } + + return msg, i + }, + }, + { + name: "Many Replies", + msgAndInfo: func() (models.ChatMessageable, *details.GroupsInfo) { + msg := models.NewChatMessage() + msg.SetCreatedDateTime(&initial) + msg.SetLastModifiedDateTime(&initial) + msg.SetBody(body) + + iden := models.NewIdentity() + iden.SetDisplayName(ptr.To("user")) + + from := models.NewChatMessageFromIdentitySet() + from.SetUser(iden) + + msg.SetFrom(from) + + reply1 := models.NewChatMessage() + reply1.SetCreatedDateTime(&mid) + reply1.SetLastModifiedDateTime(&mid) + + reply2 := models.NewChatMessage() + reply2.SetCreatedDateTime(&curr) + reply2.SetLastModifiedDateTime(&curr) + + msg.SetReplies([]models.ChatMessageable{reply1, reply2}) + + i := &details.GroupsInfo{ + ItemType: details.GroupsChannelMessage, + Created: initial, + Modified: curr, + LastReplyAt: curr, + ReplyCount: 2, + MessageCreator: "user", + Size: int64(len(content)), + MessagePreview: content, + } + + return msg, i + }, + }, + } + for _, test := range tests { + suite.Run(test.name, func() { + chMsg, expected := test.msgAndInfo() + assert.Equal(suite.T(), expected, api.ChannelMessageInfo(chMsg)) + }) + } +} diff --git a/src/pkg/services/m365/api/contacts_test.go b/src/pkg/services/m365/api/contacts_test.go index 55f059d04..402b0f4e9 100644 --- a/src/pkg/services/m365/api/contacts_test.go +++ b/src/pkg/services/m365/api/contacts_test.go @@ -46,9 +46,11 @@ func (suite *ContactsAPIUnitSuite) TestContactInfo() { Created: initial, Modified: initial, } + return contact, i }, - }, { + }, + { name: "Only Name", contactAndRP: func() (models.Contactable, *details.ExchangeInfo) { aPerson := "Whole Person" @@ -56,12 +58,14 @@ func (suite *ContactsAPIUnitSuite) TestContactInfo() { contact.SetCreatedDateTime(&initial) contact.SetLastModifiedDateTime(&initial) contact.SetDisplayName(&aPerson) + i := &details.ExchangeInfo{ ItemType: details.ExchangeContact, ContactName: aPerson, Created: initial, Modified: initial, } + return contact, i }, },