From 49bbdfc09641a4b27fab5549fa86591b80537511 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 7 Nov 2023 14:57:55 -0700 Subject: [PATCH] add getPostIDs in conversations pager (#4578) adds the getPostIDs func to ensure conversations complies with standard data paging patterns --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #4536 #### Test Plan - [x] :zap: Unit test - [x] :green_heart: E2E --- src/cli/utils/groups_test.go | 14 +-- .../m365/collection/exchange/backup.go | 25 ++-- .../m365/collection/exchange/backup_test.go | 22 ++-- .../m365/collection/exchange/handlers.go | 6 +- src/internal/m365/collection/groups/backup.go | 19 +-- .../m365/collection/groups/backup_test.go | 13 +- .../m365/collection/groups/channel_handler.go | 7 +- .../m365/collection/groups/handlers.go | 6 +- src/internal/m365/graph/cache_container.go | 27 +---- src/internal/m365/graph/interfaces.go | 29 +++++ src/internal/operations/test/exchange_test.go | 27 +++-- src/pkg/path/category_type.go | 64 +++++----- src/pkg/path/category_type_test.go | 2 + src/pkg/path/categorytype_string.go | 5 +- src/pkg/selectors/groups.go | 89 +++++++++++--- src/pkg/selectors/groups_test.go | 111 ++++++++++++++---- src/pkg/services/m365/api/channels_pager.go | 11 +- .../services/m365/api/channels_pager_test.go | 30 +++-- src/pkg/services/m365/api/client.go | 6 +- src/pkg/services/m365/api/contacts_pager.go | 12 +- .../services/m365/api/conversations_pager.go | 24 +++- .../m365/api/conversations_pager_test.go | 13 ++ src/pkg/services/m365/api/events_pager.go | 12 +- src/pkg/services/m365/api/interfaces.go | 31 +++++ src/pkg/services/m365/api/mail_pager.go | 12 +- src/pkg/services/m365/api/pagers/pagers.go | 110 +++++++++++++---- .../services/m365/api/pagers/pagers_test.go | 16 +-- 27 files changed, 511 insertions(+), 232 deletions(-) create mode 100644 src/internal/m365/graph/interfaces.go create mode 100644 src/pkg/services/m365/api/interfaces.go diff --git a/src/cli/utils/groups_test.go b/src/cli/utils/groups_test.go index d6239b780..2856de3d3 100644 --- a/src/cli/utils/groups_test.go +++ b/src/cli/utils/groups_test.go @@ -42,28 +42,28 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() { { name: "no inputs", opts: utils.GroupsOpts{}, - expectIncludeLen: 2, + expectIncludeLen: 3, }, { name: "empty", opts: utils.GroupsOpts{ Groups: empty, }, - expectIncludeLen: 2, + expectIncludeLen: 3, }, { name: "single inputs", opts: utils.GroupsOpts{ Groups: single, }, - expectIncludeLen: 2, + expectIncludeLen: 3, }, { name: "multi inputs", opts: utils.GroupsOpts{ Groups: multi, }, - expectIncludeLen: 2, + expectIncludeLen: 3, }, // sharepoint { @@ -120,7 +120,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() { FileName: empty, FolderPath: empty, }, - expectIncludeLen: 2, + expectIncludeLen: 3, }, { name: "library folder suffixes and contains", @@ -128,7 +128,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() { FileName: empty, FolderPath: empty, }, - expectIncludeLen: 2, + expectIncludeLen: 3, }, { name: "Page Folder", @@ -389,7 +389,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() { { name: "none", cats: []string{}, - expectScopeLen: 2, + expectScopeLen: 3, }, { name: "libraries", diff --git a/src/internal/m365/collection/exchange/backup.go b/src/internal/m365/collection/exchange/backup.go index ae6ef7276..7bc210dac 100644 --- a/src/internal/m365/collection/exchange/backup.go +++ b/src/internal/m365/collection/exchange/backup.go @@ -18,6 +18,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -157,14 +158,18 @@ func populateCollections( ictx = clues.Add(ictx, "previous_path", prevPath) - added, _, removed, newDelta, err := bh.itemEnumerator(). + cc := api.CallConfig{ + CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta, + UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs, + } + + addAndRem, err := bh.itemEnumerator(). GetAddedAndRemovedItemIDs( ictx, qp.ProtectedResource.ID(), cID, prevDelta, - ctrlOpts.ToggleFeatures.ExchangeImmutableIDs, - !ctrlOpts.ToggleFeatures.DisableDelta) + cc) if err != nil { if !graph.IsErrDeletedInFlight(err) { el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) @@ -176,12 +181,12 @@ func populateCollections( // to reset. This prevents any old items from being retained in // storage. If the container (or its children) are sill missing // on the next backup, they'll get tombstoned. - newDelta = pagers.DeltaUpdate{Reset: true} + addAndRem.DU = pagers.DeltaUpdate{Reset: true} } - if len(newDelta.URL) > 0 { - deltaURLs[cID] = newDelta.URL - } else if !newDelta.Reset { + if len(addAndRem.DU.URL) > 0 { + deltaURLs[cID] = addAndRem.DU.URL + } else if !addAndRem.DU.Reset { logger.Ctx(ictx).Info("missing delta url") } @@ -191,11 +196,11 @@ func populateCollections( prevPath, locPath, ctrlOpts, - newDelta.Reset), + addAndRem.DU.Reset), qp.ProtectedResource.ID(), bh.itemHandler(), - added, - removed, + addAndRem.Added, + addAndRem.Removed, // TODO: produce a feature flag that allows selective // enabling of valid modTimes. This currently produces // rare-case failures with incorrect details merging. diff --git a/src/internal/m365/collection/exchange/backup_test.go b/src/internal/m365/collection/exchange/backup_test.go index 1c4efc139..58ff0bdf2 100644 --- a/src/internal/m365/collection/exchange/backup_test.go +++ b/src/internal/m365/collection/exchange/backup_test.go @@ -75,18 +75,11 @@ type ( func (mg mockGetter) GetAddedAndRemovedItemIDs( ctx context.Context, userID, cID, prevDelta string, - _ bool, - _ bool, -) ( - map[string]time.Time, - bool, - []string, - pagers.DeltaUpdate, - error, -) { + _ api.CallConfig, +) (pagers.AddedAndRemoved, error) { results, ok := mg.results[cID] if !ok { - return nil, false, nil, pagers.DeltaUpdate{}, clues.New("mock not found for " + cID) + return pagers.AddedAndRemoved{}, clues.New("mock not found for " + cID) } delta := results.newDelta @@ -99,7 +92,14 @@ func (mg mockGetter) GetAddedAndRemovedItemIDs( resAdded[add] = time.Time{} } - return resAdded, false, results.removed, delta, results.err + aar := pagers.AddedAndRemoved{ + Added: resAdded, + Removed: results.removed, + ValidModTimes: false, + DU: delta, + } + + return aar, results.err } var _ graph.ContainerResolver = &mockResolver{} diff --git a/src/internal/m365/collection/exchange/handlers.go b/src/internal/m365/collection/exchange/handlers.go index 9265c4c9b..18b97dcb8 100644 --- a/src/internal/m365/collection/exchange/handlers.go +++ b/src/internal/m365/collection/exchange/handlers.go @@ -2,7 +2,6 @@ package exchange import ( "context" - "time" "github.com/microsoft/kiota-abstractions-go/serialization" @@ -30,9 +29,8 @@ type addedAndRemovedItemGetter interface { GetAddedAndRemovedItemIDs( ctx context.Context, user, containerID, oldDeltaToken string, - immutableIDs bool, - canMakeDeltaQueries bool, - ) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) + cc api.CallConfig, + ) (pagers.AddedAndRemoved, error) } type itemGetterSerializer interface { diff --git a/src/internal/m365/collection/groups/backup.go b/src/internal/m365/collection/groups/backup.go index e805ee3b3..de5814577 100644 --- a/src/internal/m365/collection/groups/backup.go +++ b/src/internal/m365/collection/groups/backup.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" ) // TODO: incremental support @@ -152,20 +153,22 @@ func populateCollections( // if the channel has no email property, it is unable to process delta tokens // and will return an error if a delta token is queried. - canMakeDeltaQueries := len(ptr.Val(c.GetEmail())) > 0 + cc := api.CallConfig{ + CanMakeDeltaQueries: len(ptr.Val(c.GetEmail())) > 0, + } - add, _, rem, du, err := bh.getContainerItemIDs(ctx, cID, prevDelta, canMakeDeltaQueries) + addAndRem, err := bh.getContainerItemIDs(ctx, cID, prevDelta, cc) if err != nil { el.AddRecoverable(ctx, clues.Stack(err)) continue } - added := str.SliceToMap(maps.Keys(add)) - removed := str.SliceToMap(rem) + added := str.SliceToMap(maps.Keys(addAndRem.Added)) + removed := str.SliceToMap(addAndRem.Removed) - if len(du.URL) > 0 { - deltaURLs[cID] = du.URL - } else if !du.Reset { + if len(addAndRem.DU.URL) > 0 { + deltaURLs[cID] = addAndRem.DU.URL + } else if !addAndRem.DU.Reset { logger.Ctx(ictx).Info("missing delta url") } @@ -188,7 +191,7 @@ func populateCollections( prevPath, path.Builder{}.Append(cName), ctrlOpts, - du.Reset), + addAndRem.DU.Reset), bh, qp.ProtectedResource.ID(), added, diff --git a/src/internal/m365/collection/groups/backup_test.go b/src/internal/m365/collection/groups/backup_test.go index 1f2a7568d..a02e83541 100644 --- a/src/internal/m365/collection/groups/backup_test.go +++ b/src/internal/m365/collection/groups/backup_test.go @@ -58,15 +58,22 @@ func (bh mockBackupHandler) getContainers(context.Context) ([]models.Channelable func (bh mockBackupHandler) getContainerItemIDs( _ context.Context, _, _ string, - _ bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { + _ api.CallConfig, +) (pagers.AddedAndRemoved, error) { idRes := make(map[string]time.Time, len(bh.messageIDs)) for _, id := range bh.messageIDs { idRes[id] = time.Time{} } - return idRes, true, bh.deletedMsgIDs, pagers.DeltaUpdate{}, bh.messagesErr + aar := pagers.AddedAndRemoved{ + Added: idRes, + Removed: bh.deletedMsgIDs, + ValidModTimes: true, + DU: pagers.DeltaUpdate{}, + } + + return aar, bh.messagesErr } func (bh mockBackupHandler) includeContainer( diff --git a/src/internal/m365/collection/groups/channel_handler.go b/src/internal/m365/collection/groups/channel_handler.go index f136272f2..e049df042 100644 --- a/src/internal/m365/collection/groups/channel_handler.go +++ b/src/internal/m365/collection/groups/channel_handler.go @@ -2,7 +2,6 @@ package groups import ( "context" - "time" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -41,9 +40,9 @@ func (bh channelsBackupHandler) getContainers( func (bh channelsBackupHandler) getContainerItemIDs( ctx context.Context, channelID, prevDelta string, - canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { - return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, canMakeDeltaQueries) + cc api.CallConfig, +) (pagers.AddedAndRemoved, error) { + return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, cc) } func (bh channelsBackupHandler) includeContainer( diff --git a/src/internal/m365/collection/groups/handlers.go b/src/internal/m365/collection/groups/handlers.go index ce859bad3..c4dc5fdbf 100644 --- a/src/internal/m365/collection/groups/handlers.go +++ b/src/internal/m365/collection/groups/handlers.go @@ -2,7 +2,6 @@ package groups import ( "context" - "time" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -10,6 +9,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" ) @@ -25,8 +25,8 @@ type backupHandler interface { getContainerItemIDs( ctx context.Context, containerID, prevDelta string, - canMakeDeltaQueries bool, - ) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) + cc api.CallConfig, + ) (pagers.AddedAndRemoved, error) // includeContainer evaluates whether the container is included // in the provided scope. diff --git a/src/internal/m365/graph/cache_container.go b/src/internal/m365/graph/cache_container.go index ba8e5b819..ad3866501 100644 --- a/src/internal/m365/graph/cache_container.go +++ b/src/internal/m365/graph/cache_container.go @@ -11,29 +11,10 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) -// Idable represents objects that implement msgraph-sdk-go/models.entityable -// and have the concept of an ID. -type Idable interface { - GetId() *string -} - -// Descendable represents objects that implement msgraph-sdk-go/models.entityable -// and have the concept of a "parent folder". -type Descendable interface { - Idable - GetParentFolderId() *string -} - -// Displayable represents objects that implement msgraph-sdk-go/models.entityable -// and have the concept of a display name. -type Displayable interface { - Idable - GetDisplayName() *string -} - type Container interface { - Descendable - Displayable + GetIDer + GetParentFolderIDer + GetDisplayNamer } // CachedContainer is used for local unit tests but also makes it so that this @@ -41,10 +22,12 @@ type Container interface { // reuse logic in IDToPath. type CachedContainer interface { Container + // Location contains either the display names for the dirs (if this is a calendar) // or nil Location() *path.Builder SetLocation(*path.Builder) + // Path contains either the ids for the dirs (if this is a calendar) // or the display names for the dirs Path() *path.Builder diff --git a/src/internal/m365/graph/interfaces.go b/src/internal/m365/graph/interfaces.go new file mode 100644 index 000000000..875ae652e --- /dev/null +++ b/src/internal/m365/graph/interfaces.go @@ -0,0 +1,29 @@ +package graph + +import ( + "time" +) + +type GetIDer interface { + GetId() *string +} + +type GetLastModifiedDateTimer interface { + GetLastModifiedDateTime() *time.Time +} + +type GetAdditionalDataer interface { + GetAdditionalData() map[string]any +} + +type GetDeletedDateTimer interface { + GetDeletedDateTime() *time.Time +} + +type GetDisplayNamer interface { + GetDisplayName() *string +} + +type GetParentFolderIDer interface { + GetParentFolderId() *string +} diff --git a/src/internal/operations/test/exchange_test.go b/src/internal/operations/test/exchange_test.go index 56ac8ac98..d31e6d0c1 100644 --- a/src/internal/operations/test/exchange_test.go +++ b/src/internal/operations/test/exchange_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "testing" - "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -35,6 +34,7 @@ import ( "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" + "github.com/alcionai/corso/src/pkg/services/m365/api/pagers" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" ) @@ -494,37 +494,38 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr require.True(t, ok, "dir %s found in %s cache", locRef.String(), category) var ( - err error - items map[string]time.Time + err error + aar pagers.AddedAndRemoved + cc = api.CallConfig{ + UseImmutableIDs: toggles.ExchangeImmutableIDs, + CanMakeDeltaQueries: true, + } ) switch category { case path.EmailCategory: - items, _, _, _, err = ac.Mail().GetAddedAndRemovedItemIDs( + aar, err = ac.Mail().GetAddedAndRemovedItemIDs( ctx, uidn.ID(), containerID, "", - toggles.ExchangeImmutableIDs, - true) + cc) case path.EventsCategory: - items, _, _, _, err = ac.Events().GetAddedAndRemovedItemIDs( + aar, err = ac.Events().GetAddedAndRemovedItemIDs( ctx, uidn.ID(), containerID, "", - toggles.ExchangeImmutableIDs, - true) + cc) case path.ContactsCategory: - items, _, _, _, err = ac.Contacts().GetAddedAndRemovedItemIDs( + aar, err = ac.Contacts().GetAddedAndRemovedItemIDs( ctx, uidn.ID(), containerID, "", - toggles.ExchangeImmutableIDs, - true) + cc) } require.NoError( @@ -534,6 +535,8 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr category, locRef.String()) + items := aar.Added + dest := dataset[category].dests[destName] dest.locRef = locRef.String() dest.containerID = containerID diff --git a/src/pkg/path/category_type.go b/src/pkg/path/category_type.go index c403e3c19..6dff9ebeb 100644 --- a/src/pkg/path/category_type.go +++ b/src/pkg/path/category_type.go @@ -17,28 +17,30 @@ type CategoryType int //go:generate stringer -type=CategoryType -linecomment const ( - UnknownCategory CategoryType = 0 - EmailCategory CategoryType = 1 // email - ContactsCategory CategoryType = 2 // contacts - EventsCategory CategoryType = 3 // events - FilesCategory CategoryType = 4 // files - ListsCategory CategoryType = 5 // lists - LibrariesCategory CategoryType = 6 // libraries - PagesCategory CategoryType = 7 // pages - DetailsCategory CategoryType = 8 // details - ChannelMessagesCategory CategoryType = 9 // channelMessages + UnknownCategory CategoryType = 0 + EmailCategory CategoryType = 1 // email + ContactsCategory CategoryType = 2 // contacts + EventsCategory CategoryType = 3 // events + FilesCategory CategoryType = 4 // files + ListsCategory CategoryType = 5 // lists + LibrariesCategory CategoryType = 6 // libraries + PagesCategory CategoryType = 7 // pages + DetailsCategory CategoryType = 8 // details + ChannelMessagesCategory CategoryType = 9 // channelMessages + ConversationPostsCategory CategoryType = 10 // conversationPosts ) var strToCat = map[string]CategoryType{ - strings.ToLower(EmailCategory.String()): EmailCategory, - strings.ToLower(ContactsCategory.String()): ContactsCategory, - strings.ToLower(EventsCategory.String()): EventsCategory, - strings.ToLower(FilesCategory.String()): FilesCategory, - strings.ToLower(LibrariesCategory.String()): LibrariesCategory, - strings.ToLower(ListsCategory.String()): ListsCategory, - strings.ToLower(PagesCategory.String()): PagesCategory, - strings.ToLower(DetailsCategory.String()): DetailsCategory, - strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory, + strings.ToLower(EmailCategory.String()): EmailCategory, + strings.ToLower(ContactsCategory.String()): ContactsCategory, + strings.ToLower(EventsCategory.String()): EventsCategory, + strings.ToLower(FilesCategory.String()): FilesCategory, + strings.ToLower(LibrariesCategory.String()): LibrariesCategory, + strings.ToLower(ListsCategory.String()): ListsCategory, + strings.ToLower(PagesCategory.String()): PagesCategory, + strings.ToLower(DetailsCategory.String()): DetailsCategory, + strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory, + strings.ToLower(ConversationPostsCategory.String()): ConversationPostsCategory, } func ToCategoryType(s string) CategoryType { @@ -51,15 +53,16 @@ func ToCategoryType(s string) CategoryType { } var catToHuman = map[CategoryType]string{ - EmailCategory: "Emails", - ContactsCategory: "Contacts", - EventsCategory: "Events", - FilesCategory: "Files", - LibrariesCategory: "Libraries", - ListsCategory: "Lists", - PagesCategory: "Pages", - DetailsCategory: "Details", - ChannelMessagesCategory: "Messages", + EmailCategory: "Emails", + ContactsCategory: "Contacts", + EventsCategory: "Events", + FilesCategory: "Files", + LibrariesCategory: "Libraries", + ListsCategory: "Lists", + PagesCategory: "Pages", + DetailsCategory: "Details", + ChannelMessagesCategory: "Messages", + ConversationPostsCategory: "Posts", } // HumanString produces a more human-readable string version of the category. @@ -93,8 +96,9 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{ PagesCategory: {}, }, GroupsService: { - ChannelMessagesCategory: {}, - LibrariesCategory: {}, + ChannelMessagesCategory: {}, + ConversationPostsCategory: {}, + LibrariesCategory: {}, }, } diff --git a/src/pkg/path/category_type_test.go b/src/pkg/path/category_type_test.go index 209f69924..639eccb60 100644 --- a/src/pkg/path/category_type_test.go +++ b/src/pkg/path/category_type_test.go @@ -34,6 +34,7 @@ func (suite *CategoryTypeUnitSuite) TestToCategoryType() { {input: "pages", expect: 7}, {input: "details", expect: 8}, {input: "channelmessages", expect: 9}, + {input: "conversationposts", expect: 10}, } for _, test := range table { suite.Run(test.input, func() { @@ -60,6 +61,7 @@ func (suite *CategoryTypeUnitSuite) TestHumanString() { {input: 7, expect: "Pages"}, {input: 8, expect: "Details"}, {input: 9, expect: "Messages"}, + {input: 10, expect: "Posts"}, } for _, test := range table { suite.Run(test.input.String(), func() { diff --git a/src/pkg/path/categorytype_string.go b/src/pkg/path/categorytype_string.go index fb12d8001..98abbbed5 100644 --- a/src/pkg/path/categorytype_string.go +++ b/src/pkg/path/categorytype_string.go @@ -18,11 +18,12 @@ func _() { _ = x[PagesCategory-7] _ = x[DetailsCategory-8] _ = x[ChannelMessagesCategory-9] + _ = x[ConversationPostsCategory-10] } -const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessages" +const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessagesconversationPosts" -var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80} +var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80, 97} func (i CategoryType) String() string { if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) { diff --git a/src/pkg/selectors/groups.go b/src/pkg/selectors/groups.go index d5aae480a..dc9148573 100644 --- a/src/pkg/selectors/groups.go +++ b/src/pkg/selectors/groups.go @@ -217,15 +217,14 @@ func (s *groups) AllData() []GroupsScope { scopes = append( scopes, makeScope[GroupsScope](GroupsLibraryFolder, Any()), - makeScope[GroupsScope](GroupsChannel, Any())) + makeScope[GroupsScope](GroupsChannel, Any()), + makeScope[GroupsScope](GroupsConversation, Any())) return scopes } // Channels produces one or more SharePoint channel scopes, where the channel -// matches upon a given channel by ID or Name. In order to ensure channel selection -// this should always be embedded within the Filter() set; include(channel()) will -// select all items in the channel without further filtering. +// matches upon a given channel by ID or Name. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] @@ -260,6 +259,42 @@ func (s *groups) ChannelMessages(channels, messages []string, opts ...option) [] return scopes } +// Conversations produces one or more SharePoint conversation scopes, where the +// conversation matches with a given conversation by ID or Topic. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (s *groups) Conversation(conversations []string, opts ...option) []GroupsScope { + var ( + scopes = []GroupsScope{} + os = append([]option{pathComparator()}, opts...) + ) + + scopes = append( + scopes, + makeScope[GroupsScope](GroupsConversation, conversations, os...)) + + return scopes +} + +// ConversationPosts produces one or more Groups conversation post scopes. +// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] +// If any slice contains selectors.None, that slice is reduced to [selectors.None] +// If any slice is empty, it defaults to [selectors.None] +func (s *groups) ConversationPosts(conversations, posts []string, opts ...option) []GroupsScope { + var ( + scopes = []GroupsScope{} + os = append([]option{pathComparator()}, opts...) + ) + + scopes = append( + scopes, + makeScope[GroupsScope](GroupsConversationPost, posts, os...). + set(GroupsConversation, conversations, opts...)) + + return scopes +} + // Sites produces one or more Groups site scopes, where the site // matches upon a given site by ID or URL. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] @@ -516,15 +551,17 @@ const ( GroupsCategoryUnknown groupsCategory = "" // types of data in Groups - GroupsGroup groupsCategory = "GroupsGroup" - GroupsChannel groupsCategory = "GroupsChannel" - GroupsChannelMessage groupsCategory = "GroupsChannelMessage" - GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder" - GroupsLibraryItem groupsCategory = "GroupsLibraryItem" - GroupsList groupsCategory = "GroupsList" - GroupsListItem groupsCategory = "GroupsListItem" - GroupsPageFolder groupsCategory = "GroupsPageFolder" - GroupsPage groupsCategory = "GroupsPage" + GroupsGroup groupsCategory = "GroupsGroup" + GroupsChannel groupsCategory = "GroupsChannel" + GroupsChannelMessage groupsCategory = "GroupsChannelMessage" + GroupsConversation groupsCategory = "GroupsConversation" + GroupsConversationPost groupsCategory = "GroupsConversationPost" + GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder" + GroupsLibraryItem groupsCategory = "GroupsLibraryItem" + GroupsList groupsCategory = "GroupsList" + GroupsListItem groupsCategory = "GroupsListItem" + GroupsPageFolder groupsCategory = "GroupsPageFolder" + GroupsPage groupsCategory = "GroupsPage" // details.itemInfo comparables GroupsInfoLibraryItemCreatedAfter groupsCategory = "GroupsInfoLibraryItemCreatedAfter" @@ -546,10 +583,14 @@ const ( // groupsLeafProperties describes common metadata of the leaf categories var groupsLeafProperties = map[categorizer]leafProperty{ - GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf + GroupsChannelMessage: { pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage}, pathType: path.ChannelMessagesCategory, }, + GroupsConversationPost: { + pathKeys: []categorizer{GroupsConversation, GroupsConversationPost}, + pathType: path.ConversationPostsCategory, + }, GroupsLibraryItem: { pathKeys: []categorizer{GroupsLibraryFolder, GroupsLibraryItem}, pathType: path.LibrariesCategory, @@ -571,12 +612,12 @@ func (c groupsCategory) String() string { // Ex: ServiceUser.leafCat() => ServiceUser func (c groupsCategory) leafCat() categorizer { switch c { - // TODO: if channels ever contain more than one type of item, - // we'll need to fix this up. case GroupsChannel, GroupsChannelMessage, GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator, GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore: return GroupsChannelMessage + case GroupsConversation, GroupsConversationPost: + return GroupsConversationPost case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSite, GroupsInfoSiteLibraryDrive, GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore, GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore: @@ -631,6 +672,9 @@ func (c groupsCategory) pathValues( case GroupsChannel, GroupsChannelMessage: folderCat, itemCat = GroupsChannel, GroupsChannelMessage rFld = ent.Groups.ParentPath + case GroupsConversation, GroupsConversationPost: + folderCat, itemCat = GroupsConversation, GroupsConversationPost + rFld = ent.Groups.ParentPath case GroupsLibraryFolder, GroupsLibraryItem: folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem rFld = ent.Groups.ParentPath @@ -729,7 +773,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS os := []option{} switch cat { - case GroupsChannel, GroupsLibraryFolder: + case GroupsChannel, GroupsConversation, GroupsLibraryFolder: os = append(os, pathComparator()) } @@ -742,12 +786,16 @@ func (s GroupsScope) setDefaults() { case GroupsGroup: s[GroupsChannel.String()] = passAny s[GroupsChannelMessage.String()] = passAny + s[GroupsConversation.String()] = passAny + s[GroupsConversationPost.String()] = passAny s[GroupsLibraryFolder.String()] = passAny s[GroupsLibraryItem.String()] = passAny case GroupsChannel: s[GroupsChannelMessage.String()] = passAny case GroupsLibraryFolder: s[GroupsLibraryItem.String()] = passAny + case GroupsConversation: + s[GroupsConversationPost.String()] = passAny } } @@ -767,8 +815,9 @@ func (s groups) Reduce( deets, s.Selector, map[path.CategoryType]groupsCategory{ - path.ChannelMessagesCategory: GroupsChannelMessage, - path.LibrariesCategory: GroupsLibraryItem, + path.ChannelMessagesCategory: GroupsChannelMessage, + path.ConversationPostsCategory: GroupsConversationPost, + path.LibrariesCategory: GroupsLibraryItem, }, errs) } @@ -793,6 +842,8 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool { acceptableItemType = int(details.SharePointLibrary) case GroupsChannelMessage: acceptableItemType = int(details.GroupsChannelMessage) + case GroupsConversationPost: + acceptableItemType = int(details.GroupsConversationPost) } switch infoCat { diff --git a/src/pkg/selectors/groups_test.go b/src/pkg/selectors/groups_test.go index a403ae2f7..63b966abb 100644 --- a/src/pkg/selectors/groups_test.go +++ b/src/pkg/selectors/groups_test.go @@ -112,6 +112,9 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { chanItem = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems1), "chitem") chanItem2 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems2), "chitem2") chanItem3 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems3), "chitem3") + convItem = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems1), "convitem") + convItem2 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems2), "convitem2") + convItem3 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems3), "convitem3") ) deets := &details.Details{ @@ -186,6 +189,39 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { }, }, }, + { + RepoRef: convItem, + ItemRef: "convitem", + LocationRef: strings.Join(itemElems1, "/"), + ItemInfo: details.ItemInfo{ + Groups: &details.GroupsInfo{ + ItemType: details.GroupsConversationPost, + ParentPath: strings.Join(itemElems1, "/"), + }, + }, + }, + { + RepoRef: convItem2, + LocationRef: strings.Join(itemElems2, "/"), + // ItemRef intentionally blank to test fallback case + ItemInfo: details.ItemInfo{ + Groups: &details.GroupsInfo{ + ItemType: details.GroupsConversationPost, + ParentPath: strings.Join(itemElems2, "/"), + }, + }, + }, + { + RepoRef: convItem3, + ItemRef: "convitem3", + LocationRef: strings.Join(itemElems3, "/"), + ItemInfo: details.ItemInfo{ + Groups: &details.GroupsInfo{ + ItemType: details.GroupsConversationPost, + ParentPath: strings.Join(itemElems3, "/"), + }, + }, + }, }, }, } @@ -207,7 +243,10 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { sel.Include(sel.AllData()) return sel }, - expect: arr(libItem, libItem2, libItem3, chanItem, chanItem2, chanItem3), + expect: arr( + libItem, libItem2, libItem3, + chanItem, chanItem2, chanItem3, + convItem, convItem2, convItem3), }, { name: "only match library item", @@ -218,15 +257,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { }, expect: arr(libItem2), }, - { - name: "only match channel item", - makeSelector: func() *GroupsRestore { - sel := NewGroupsRestore(Any()) - sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"})) - return sel - }, - expect: arr(chanItem2), - }, { name: "library id doesn't match name", makeSelector: func() *GroupsRestore { @@ -237,16 +267,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { expect: []string{}, cfg: Config{OnlyMatchItemNames: true}, }, - { - name: "channel id doesn't match name", - makeSelector: func() *GroupsRestore { - sel := NewGroupsRestore(Any()) - sel.Include(sel.ChannelMessages(Any(), []string{"item2"})) - return sel - }, - expect: []string{}, - cfg: Config{OnlyMatchItemNames: true}, - }, { name: "library only match item name", makeSelector: func() *GroupsRestore { @@ -275,6 +295,44 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() { }, expect: arr(libItem, libItem2), }, + { + name: "only match channel item", + makeSelector: func() *GroupsRestore { + sel := NewGroupsRestore(Any()) + sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"})) + return sel + }, + expect: arr(chanItem2), + }, + { + name: "channel id doesn't match name", + makeSelector: func() *GroupsRestore { + sel := NewGroupsRestore(Any()) + sel.Include(sel.ChannelMessages(Any(), []string{"item2"})) + return sel + }, + expect: []string{}, + cfg: Config{OnlyMatchItemNames: true}, + }, + { + name: "only match conversation item", + makeSelector: func() *GroupsRestore { + sel := NewGroupsRestore(Any()) + sel.Include(sel.ConversationPosts(Any(), []string{"convitem2"})) + return sel + }, + expect: arr(convItem2), + }, + { + name: "conversation id doesn't match name", + makeSelector: func() *GroupsRestore { + sel := NewGroupsRestore(Any()) + sel.Include(sel.ConversationPosts(Any(), []string{"item2"})) + return sel + }, + expect: []string{}, + cfg: Config{OnlyMatchItemNames: true}, + }, } for _, test := range table { suite.Run(test.name, func() { @@ -320,6 +378,17 @@ func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() { }, cfg: Config{}, }, + { + name: "Groups Conversation Posts", + sc: GroupsConversationPost, + pathElems: elems, + locRef: "", + expected: map[categorizer][]string{ + GroupsConversation: {""}, + GroupsConversationPost: {itemID, shortRef}, + }, + cfg: Config{}, + }, } for _, test := range table { @@ -476,6 +545,8 @@ func (suite *GroupsSelectorSuite) TestCategory_PathType() { {GroupsCategoryUnknown, path.UnknownCategory}, {GroupsChannel, path.ChannelMessagesCategory}, {GroupsChannelMessage, path.ChannelMessagesCategory}, + {GroupsConversation, path.ConversationPostsCategory}, + {GroupsConversationPost, path.ConversationPostsCategory}, {GroupsInfoChannelMessageCreator, path.ChannelMessagesCategory}, {GroupsInfoChannelMessageCreatedAfter, path.ChannelMessagesCategory}, {GroupsInfoChannelMessageCreatedBefore, path.ChannelMessagesCategory}, diff --git a/src/pkg/services/m365/api/channels_pager.go b/src/pkg/services/m365/api/channels_pager.go index 87b95fe39..ac06204cf 100644 --- a/src/pkg/services/m365/api/channels_pager.go +++ b/src/pkg/services/m365/api/channels_pager.go @@ -2,7 +2,6 @@ package api import ( "context" - "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -186,18 +185,18 @@ func filterOutSystemMessages(cm models.ChatMessageable) bool { func (c Channels) GetChannelMessageIDs( ctx context.Context, teamID, channelID, prevDeltaLink string, - canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { - added, validModTimes, removed, du, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable]( + cc CallConfig, +) (pagers.AddedAndRemoved, error) { + aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable]( ctx, c.NewChannelMessagePager(teamID, channelID, CallConfig{}), c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink), prevDeltaLink, - canMakeDeltaQueries, + cc.CanMakeDeltaQueries, pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable], filterOutSystemMessages) - return added, validModTimes, removed, du, clues.Stack(err).OrNil() + return aar, clues.Stack(err).OrNil() } // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/channels_pager_test.go b/src/pkg/services/m365/api/channels_pager_test.go index ea118a350..a46dc2c63 100644 --- a/src/pkg/services/m365/api/channels_pager_test.go +++ b/src/pkg/services/m365/api/channels_pager_test.go @@ -56,30 +56,34 @@ func (suite *ChannelsPagerIntgSuite) TestEnumerateChannelMessages() { ctx, flush := tester.NewContext(t) defer flush() - addedIDs, _, _, du, err := ac.GetChannelMessageIDs( + cc := CallConfig{ + CanMakeDeltaQueries: true, + } + + aar, err := ac.GetChannelMessageIDs( ctx, suite.its.group.id, suite.its.group.testContainerID, "", - true) + cc) require.NoError(t, err, clues.ToCore(err)) - require.NotEmpty(t, addedIDs) - require.NotZero(t, du.URL, "delta link") - require.True(t, du.Reset, "reset due to empty prev delta link") + require.NotEmpty(t, aar.Added) + require.NotZero(t, aar.DU.URL, "delta link") + require.True(t, aar.DU.Reset, "reset due to empty prev delta link") - addedIDs, _, deletedIDs, du, err := ac.GetChannelMessageIDs( + aar, err = ac.GetChannelMessageIDs( ctx, suite.its.group.id, suite.its.group.testContainerID, - du.URL, - true) + aar.DU.URL, + cc) require.NoError(t, err, clues.ToCore(err)) - require.Empty(t, addedIDs, "should have no new messages from delta") - require.Empty(t, deletedIDs, "should have no deleted messages from delta") - require.NotZero(t, du.URL, "delta link") - require.False(t, du.Reset, "prev delta link should be valid") + require.Empty(t, aar.Added, "should have no new messages from delta") + require.Empty(t, aar.Removed, "should have no deleted messages from delta") + require.NotZero(t, aar.DU.URL, "delta link") + require.False(t, aar.DU.Reset, "prev delta link should be valid") - for id := range addedIDs { + for id := range aar.Added { suite.Run(id+"-replies", func() { testEnumerateChannelMessageReplies( suite.T(), diff --git a/src/pkg/services/m365/api/client.go b/src/pkg/services/m365/api/client.go index f35232098..adcd08e05 100644 --- a/src/pkg/services/m365/api/client.go +++ b/src/pkg/services/m365/api/client.go @@ -159,8 +159,10 @@ func (c Client) Post( // --------------------------------------------------------------------------- type CallConfig struct { - Expand []string - Select []string + Expand []string + Select []string + CanMakeDeltaQueries bool + UseImmutableIDs bool } // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/contacts_pager.go b/src/pkg/services/m365/api/contacts_pager.go index e2dfe6622..3e1f4a315 100644 --- a/src/pkg/services/m365/api/contacts_pager.go +++ b/src/pkg/services/m365/api/contacts_pager.go @@ -2,7 +2,6 @@ package api import ( "context" - "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -250,9 +249,8 @@ func (p *contactDeltaPager) ValidModTimes() bool { func (c Contacts) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - immutableIDs bool, - canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { + cc CallConfig, +) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, "data_category", path.ContactsCategory, @@ -263,12 +261,12 @@ func (c Contacts) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - immutableIDs, + cc.UseImmutableIDs, idAnd(lastModifiedDateTime)...) pager := c.NewContactsPager( userID, containerID, - immutableIDs, + cc.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Contactable]( @@ -276,6 +274,6 @@ func (c Contacts) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - canMakeDeltaQueries, + cc.CanMakeDeltaQueries, pagers.AddedAndRemovedByAddtlData[models.Contactable]) } diff --git a/src/pkg/services/m365/api/conversations_pager.go b/src/pkg/services/m365/api/conversations_pager.go index bc9cd2a32..93a80d446 100644 --- a/src/pkg/services/m365/api/conversations_pager.go +++ b/src/pkg/services/m365/api/conversations_pager.go @@ -140,7 +140,7 @@ func (c Conversations) NewConversationThreadsPager( } } -// GetConversations fetches all conversations in the group. +// GetConversations fetches all conversation threads in the group. func (c Conversations) GetConversationThreads( ctx context.Context, groupID, conversationID string, @@ -218,7 +218,7 @@ func (c Conversations) NewConversationThreadPostsPager( } } -// GetConversations fetches all conversations in the group. +// GetConversations fetches all conversation posts in the group. func (c Conversations) GetConversationThreadPosts( ctx context.Context, groupID, conversationID, threadID string, @@ -230,3 +230,23 @@ func (c Conversations) GetConversationThreadPosts( return items, graph.Stack(ctx, err).OrNil() } + +// GetConversations fetches all added and deleted conversation posts in the group. +func (c Conversations) GetConversationThreadPostIDs( + ctx context.Context, + groupID, conversationID, threadID string, + cc CallConfig, +) (pagers.AddedAndRemoved, error) { + canMakeDeltaQueries := false + + aarh, err := pagers.GetAddedAndRemovedItemIDs[models.Postable]( + ctx, + c.NewConversationThreadPostsPager(groupID, conversationID, threadID, CallConfig{}), + nil, + "", + canMakeDeltaQueries, + pagers.AddedAndRemovedAddAll[models.Postable], + pagers.FilterIncludeAll[models.Postable]) + + return aarh, clues.Stack(err).OrNil() +} diff --git a/src/pkg/services/m365/api/conversations_pager_test.go b/src/pkg/services/m365/api/conversations_pager_test.go index 43023ad65..d96ca1298 100644 --- a/src/pkg/services/m365/api/conversations_pager_test.go +++ b/src/pkg/services/m365/api/conversations_pager_test.go @@ -5,6 +5,7 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -49,6 +50,18 @@ func (suite *ConversationsPagerIntgSuite) TestEnumerateConversations_withThreads for _, thread := range threads { posts := testEnumerateConvPosts(suite, conv, thread) + aar, err := ac.GetConversationThreadPostIDs( + ctx, + suite.its.group.id, + ptr.Val(conv.GetId()), + ptr.Val(thread.GetId()), + CallConfig{}) + require.NoError(t, err, clues.ToCore(err)) + require.Equal(t, len(posts), len(aar.Added), "added the same number of ids and posts") + assert.True(t, aar.ValidModTimes, "mod times should be valid") + assert.Empty(t, aar.Removed, "no items should get removed") + assert.Empty(t, aar.DU.URL, "no delta update token should be provided") + for _, post := range posts { testGetPostByID(suite, conv, thread, post) } diff --git a/src/pkg/services/m365/api/events_pager.go b/src/pkg/services/m365/api/events_pager.go index 278125b53..e58fe321b 100644 --- a/src/pkg/services/m365/api/events_pager.go +++ b/src/pkg/services/m365/api/events_pager.go @@ -3,7 +3,6 @@ package api import ( "context" "fmt" - "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -245,9 +244,8 @@ func (p *eventDeltaPager) ValidModTimes() bool { func (c Events) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - immutableIDs bool, - canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { + cc CallConfig, +) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, "data_category", path.EventsCategory, @@ -258,12 +256,12 @@ func (c Events) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - immutableIDs, + cc.UseImmutableIDs, idAnd()...) pager := c.NewEventsPager( userID, containerID, - immutableIDs, + cc.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Eventable]( @@ -271,6 +269,6 @@ func (c Events) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - canMakeDeltaQueries, + cc.CanMakeDeltaQueries, pagers.AddedAndRemovedByAddtlData[models.Eventable]) } diff --git a/src/pkg/services/m365/api/interfaces.go b/src/pkg/services/m365/api/interfaces.go new file mode 100644 index 000000000..df8160e1d --- /dev/null +++ b/src/pkg/services/m365/api/interfaces.go @@ -0,0 +1,31 @@ +package api + +import ( + "context" + + "github.com/microsoft/kiota-abstractions-go/serialization" + + "github.com/alcionai/corso/src/pkg/fault" +) + +type GetAndSerializeItemer[INFO any] interface { + GetItemer[INFO] + Serializer +} + +type GetItemer[INFO any] interface { + GetItem( + ctx context.Context, + user, itemID string, + immutableIDs bool, + errs *fault.Bus, + ) (serialization.Parsable, *INFO, error) +} + +type Serializer interface { + Serialize( + ctx context.Context, + item serialization.Parsable, + protectedResource, itemID string, + ) ([]byte, error) +} diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index 13f9e849e..50ad1ea43 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -3,7 +3,6 @@ package api import ( "context" "fmt" - "time" "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" @@ -247,9 +246,8 @@ func (p *mailDeltaPager) ValidModTimes() bool { func (c Mail) GetAddedAndRemovedItemIDs( ctx context.Context, userID, containerID, prevDeltaLink string, - immutableIDs bool, - canMakeDeltaQueries bool, -) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { + cc CallConfig, +) (pagers.AddedAndRemoved, error) { ctx = clues.Add( ctx, "data_category", path.EmailCategory, @@ -260,12 +258,12 @@ func (c Mail) GetAddedAndRemovedItemIDs( userID, containerID, prevDeltaLink, - immutableIDs, + cc.UseImmutableIDs, idAnd(lastModifiedDateTime)...) pager := c.NewMailPager( userID, containerID, - immutableIDs, + cc.UseImmutableIDs, idAnd(lastModifiedDateTime)...) return pagers.GetAddedAndRemovedItemIDs[models.Messageable]( @@ -273,6 +271,6 @@ func (c Mail) GetAddedAndRemovedItemIDs( pager, deltaPager, prevDeltaLink, - canMakeDeltaQueries, + cc.CanMakeDeltaQueries, pagers.AddedAndRemovedByAddtlData[models.Messageable]) } diff --git a/src/pkg/services/m365/api/pagers/pagers.go b/src/pkg/services/m365/api/pagers/pagers.go index daf31dce5..34715501f 100644 --- a/src/pkg/services/m365/api/pagers/pagers.go +++ b/src/pkg/services/m365/api/pagers/pagers.go @@ -366,13 +366,28 @@ func batchDeltaEnumerateItems[T any]( return results, du, clues.Stack(err).OrNil() } +// --------------------------------------------------------------------------- +// filter funcs +// --------------------------------------------------------------------------- + +func FilterIncludeAll[T any](_ T) bool { + return true +} + // --------------------------------------------------------------------------- // shared enumeration runner funcs // --------------------------------------------------------------------------- +type AddedAndRemoved struct { + Added map[string]time.Time + Removed []string + DU DeltaUpdate + ValidModTimes bool +} + type addedAndRemovedHandler[T any] func( items []T, - filters ...func(T) bool, // false -> remove, true -> keep + filters ...func(T) bool, ) ( map[string]time.Time, []string, @@ -387,16 +402,23 @@ func GetAddedAndRemovedItemIDs[T any]( canMakeDeltaQueries bool, aarh addedAndRemovedHandler[T], filters ...func(T) bool, -) (map[string]time.Time, bool, []string, DeltaUpdate, error) { +) (AddedAndRemoved, error) { if canMakeDeltaQueries { ts, du, err := batchDeltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink) if err != nil && !graph.IsErrInvalidDelta(err) && !graph.IsErrDeltaNotSupported(err) { - return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err) + return AddedAndRemoved{}, graph.Stack(ctx, err) } if err == nil { a, r, err := aarh(ts, filters...) - return a, deltaPager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil() + aar := AddedAndRemoved{ + Added: a, + Removed: r, + DU: du, + ValidModTimes: deltaPager.ValidModTimes(), + } + + return aar, graph.Stack(ctx, err).OrNil() } } @@ -404,31 +426,61 @@ func GetAddedAndRemovedItemIDs[T any]( ts, err := BatchEnumerateItems(ctx, pager) if err != nil { - return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err) + return AddedAndRemoved{}, graph.Stack(ctx, err) } a, r, err := aarh(ts, filters...) + aar := AddedAndRemoved{ + Added: a, + Removed: r, + DU: du, + ValidModTimes: pager.ValidModTimes(), + } - return a, pager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil() + return aar, graph.Stack(ctx, err).OrNil() } -type getIDer interface { - GetId() *string +type getIDAndModDateTimer interface { + graph.GetIDer + graph.GetLastModifiedDateTimer +} + +// AddedAndRemovedAddAll indiscriminately adds every item to the added list, deleting nothing. +func AddedAndRemovedAddAll[T any]( + items []T, + filters ...func(T) bool, +) (map[string]time.Time, []string, error) { + added := map[string]time.Time{} + + for _, item := range items { + passAllFilters := true + + for _, passes := range filters { + passAllFilters = passAllFilters && passes(item) + } + + if !passAllFilters { + continue + } + + giamdt, ok := any(item).(getIDAndModDateTimer) + if !ok { + return nil, nil, clues.New("item does not provide id and modified date time getters"). + With("item_type", fmt.Sprintf("%T", item)) + } + + added[ptr.Val(giamdt.GetId())] = dttm.OrNow(ptr.Val(giamdt.GetLastModifiedDateTime())) + } + + return added, []string{}, nil } // for added and removed by additionalData[@removed] type getIDModAndAddtler interface { - getIDer - getModTimer - GetAdditionalData() map[string]any -} - -// for types that are non-compliant with this interface, -// pagers will need to wrap the return value in a struct -// that provides this compliance. -type getModTimer interface { - GetLastModifiedDateTime() *time.Time + graph.GetIDer + graph.GetLastModifiedDateTimer + graph.GetAdditionalDataer } func AddedAndRemovedByAddtlData[T any]( @@ -451,7 +503,7 @@ func AddedAndRemovedByAddtlData[T any]( giaa, ok := any(item).(getIDModAndAddtler) if !ok { - return nil, nil, clues.New("item does not provide id and additional data getters"). + return nil, nil, clues.New("item does not provide id, modified date time, and additional data getters"). With("item_type", fmt.Sprintf("%T", item)) } @@ -461,7 +513,11 @@ func AddedAndRemovedByAddtlData[T any]( if giaa.GetAdditionalData()[graph.AddtlDataRemoved] == nil { var modTime time.Time - if mt, ok := giaa.(getModTimer); ok { + // not all items comply with last modified date time, and not all + // items can be wrapped in a way that produces a valid value for + // the func. That's why this isn't packed in to the expected + // interfaace composition. + if mt, ok := giaa.(graph.GetLastModifiedDateTimer); ok { // Make sure to get a non-zero mod time if the item doesn't have one for // some reason. Otherwise we can hit an issue where kopia has a // different mod time for the file than the details does. This occurs @@ -485,9 +541,9 @@ func AddedAndRemovedByAddtlData[T any]( // for added and removed by GetDeletedDateTime() type getIDModAndDeletedDateTimer interface { - getIDer - getModTimer - GetDeletedDateTime() *time.Time + graph.GetIDer + graph.GetLastModifiedDateTimer + graph.GetDeletedDateTimer } func AddedAndRemovedByDeletedDateTime[T any]( @@ -510,14 +566,18 @@ func AddedAndRemovedByDeletedDateTime[T any]( giaddt, ok := any(item).(getIDModAndDeletedDateTimer) if !ok { - return nil, nil, clues.New("item does not provide id and deleted date time getters"). + return nil, nil, clues.New("item does not provide id, modified, and deleted date time getters"). With("item_type", fmt.Sprintf("%T", item)) } if giaddt.GetDeletedDateTime() == nil { var modTime time.Time - if mt, ok := giaddt.(getModTimer); ok { + // not all items comply with last modified date time, and not all + // items can be wrapped in a way that produces a valid value for + // the func. That's why this isn't packed in to the expected + // interfaace composition. + if mt, ok := giaddt.(graph.GetLastModifiedDateTimer); ok { // Make sure to get a non-zero mod time if the item doesn't have one for // some reason. Otherwise we can hit an issue where kopia has a // different mod time for the file than the details does. This occurs diff --git a/src/pkg/services/m365/api/pagers/pagers_test.go b/src/pkg/services/m365/api/pagers/pagers_test.go index c62f04ab2..afe5a5298 100644 --- a/src/pkg/services/m365/api/pagers/pagers_test.go +++ b/src/pkg/services/m365/api/pagers/pagers_test.go @@ -520,7 +520,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { filters = append(filters, test.filter) } - added, validModTimes, removed, deltaUpdate, err := GetAddedAndRemovedItemIDs[testItem]( + aar, err := GetAddedAndRemovedItemIDs[testItem]( ctx, test.pagerGetter(t), test.deltaPagerGetter(t), @@ -530,18 +530,18 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() { filters...) require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err)) - if validModTimes { - assert.Equal(t, test.expect.added, added, "added item IDs and mod times") + if aar.ValidModTimes { + assert.Equal(t, test.expect.added, aar.Added, "added item IDs and mod times") } else { - assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(added), "added item IDs") - for _, modtime := range added { + assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(aar.Added), "added item IDs") + for _, modtime := range aar.Added { assert.True(t, modtime.After(epoch), "mod time after epoch") assert.False(t, modtime.Equal(time.Time{}), "non-zero mod time") } } - assert.Equal(t, test.expect.validModTimes, validModTimes, "valid mod times") - assert.EqualValues(t, test.expect.removed, removed, "removed item IDs") - assert.Equal(t, test.expect.deltaUpdate, deltaUpdate, "delta update") + assert.Equal(t, test.expect.validModTimes, aar.ValidModTimes, "valid mod times") + assert.EqualValues(t, test.expect.removed, aar.Removed, "removed item IDs") + assert.Equal(t, test.expect.deltaUpdate, aar.DU, "delta update") }) } }