add getPostIDs in conversations pager (#4578)

adds the getPostIDs func to ensure conversations
complies with standard data paging patterns

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #4536

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-11-07 14:57:55 -07:00 committed by GitHub
parent ec1afa8c84
commit 49bbdfc096
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 511 additions and 232 deletions

View File

@ -42,28 +42,28 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
{
name: "no inputs",
opts: utils.GroupsOpts{},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
{
name: "empty",
opts: utils.GroupsOpts{
Groups: empty,
},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
{
name: "single inputs",
opts: utils.GroupsOpts{
Groups: single,
},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
{
name: "multi inputs",
opts: utils.GroupsOpts{
Groups: multi,
},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
// sharepoint
{
@ -120,7 +120,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
FileName: empty,
FolderPath: empty,
},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
{
name: "library folder suffixes and contains",
@ -128,7 +128,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
FileName: empty,
FolderPath: empty,
},
expectIncludeLen: 2,
expectIncludeLen: 3,
},
{
name: "Page Folder",
@ -389,7 +389,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
{
name: "none",
cats: []string{},
expectScopeLen: 2,
expectScopeLen: 3,
},
{
name: "libraries",

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
)
@ -157,14 +158,18 @@ func populateCollections(
ictx = clues.Add(ictx, "previous_path", prevPath)
added, _, removed, newDelta, err := bh.itemEnumerator().
cc := api.CallConfig{
CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta,
UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs,
}
addAndRem, err := bh.itemEnumerator().
GetAddedAndRemovedItemIDs(
ictx,
qp.ProtectedResource.ID(),
cID,
prevDelta,
ctrlOpts.ToggleFeatures.ExchangeImmutableIDs,
!ctrlOpts.ToggleFeatures.DisableDelta)
cc)
if err != nil {
if !graph.IsErrDeletedInFlight(err) {
el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
@ -176,12 +181,12 @@ func populateCollections(
// to reset. This prevents any old items from being retained in
// storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned.
newDelta = pagers.DeltaUpdate{Reset: true}
addAndRem.DU = pagers.DeltaUpdate{Reset: true}
}
if len(newDelta.URL) > 0 {
deltaURLs[cID] = newDelta.URL
} else if !newDelta.Reset {
if len(addAndRem.DU.URL) > 0 {
deltaURLs[cID] = addAndRem.DU.URL
} else if !addAndRem.DU.Reset {
logger.Ctx(ictx).Info("missing delta url")
}
@ -191,11 +196,11 @@ func populateCollections(
prevPath,
locPath,
ctrlOpts,
newDelta.Reset),
addAndRem.DU.Reset),
qp.ProtectedResource.ID(),
bh.itemHandler(),
added,
removed,
addAndRem.Added,
addAndRem.Removed,
// TODO: produce a feature flag that allows selective
// enabling of valid modTimes. This currently produces
// rare-case failures with incorrect details merging.

View File

@ -75,18 +75,11 @@ type (
func (mg mockGetter) GetAddedAndRemovedItemIDs(
ctx context.Context,
userID, cID, prevDelta string,
_ bool,
_ bool,
) (
map[string]time.Time,
bool,
[]string,
pagers.DeltaUpdate,
error,
) {
_ api.CallConfig,
) (pagers.AddedAndRemoved, error) {
results, ok := mg.results[cID]
if !ok {
return nil, false, nil, pagers.DeltaUpdate{}, clues.New("mock not found for " + cID)
return pagers.AddedAndRemoved{}, clues.New("mock not found for " + cID)
}
delta := results.newDelta
@ -99,7 +92,14 @@ func (mg mockGetter) GetAddedAndRemovedItemIDs(
resAdded[add] = time.Time{}
}
return resAdded, false, results.removed, delta, results.err
aar := pagers.AddedAndRemoved{
Added: resAdded,
Removed: results.removed,
ValidModTimes: false,
DU: delta,
}
return aar, results.err
}
var _ graph.ContainerResolver = &mockResolver{}

View File

@ -2,7 +2,6 @@ package exchange
import (
"context"
"time"
"github.com/microsoft/kiota-abstractions-go/serialization"
@ -30,9 +29,8 @@ type addedAndRemovedItemGetter interface {
GetAddedAndRemovedItemIDs(
ctx context.Context,
user, containerID, oldDeltaToken string,
immutableIDs bool,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error)
cc api.CallConfig,
) (pagers.AddedAndRemoved, error)
}
type itemGetterSerializer interface {

View File

@ -20,6 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
// TODO: incremental support
@ -152,20 +153,22 @@ func populateCollections(
// if the channel has no email property, it is unable to process delta tokens
// and will return an error if a delta token is queried.
canMakeDeltaQueries := len(ptr.Val(c.GetEmail())) > 0
cc := api.CallConfig{
CanMakeDeltaQueries: len(ptr.Val(c.GetEmail())) > 0,
}
add, _, rem, du, err := bh.getContainerItemIDs(ctx, cID, prevDelta, canMakeDeltaQueries)
addAndRem, err := bh.getContainerItemIDs(ctx, cID, prevDelta, cc)
if err != nil {
el.AddRecoverable(ctx, clues.Stack(err))
continue
}
added := str.SliceToMap(maps.Keys(add))
removed := str.SliceToMap(rem)
added := str.SliceToMap(maps.Keys(addAndRem.Added))
removed := str.SliceToMap(addAndRem.Removed)
if len(du.URL) > 0 {
deltaURLs[cID] = du.URL
} else if !du.Reset {
if len(addAndRem.DU.URL) > 0 {
deltaURLs[cID] = addAndRem.DU.URL
} else if !addAndRem.DU.Reset {
logger.Ctx(ictx).Info("missing delta url")
}
@ -188,7 +191,7 @@ func populateCollections(
prevPath,
path.Builder{}.Append(cName),
ctrlOpts,
du.Reset),
addAndRem.DU.Reset),
bh,
qp.ProtectedResource.ID(),
added,

View File

@ -58,15 +58,22 @@ func (bh mockBackupHandler) getContainers(context.Context) ([]models.Channelable
func (bh mockBackupHandler) getContainerItemIDs(
_ context.Context,
_, _ string,
_ bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
_ api.CallConfig,
) (pagers.AddedAndRemoved, error) {
idRes := make(map[string]time.Time, len(bh.messageIDs))
for _, id := range bh.messageIDs {
idRes[id] = time.Time{}
}
return idRes, true, bh.deletedMsgIDs, pagers.DeltaUpdate{}, bh.messagesErr
aar := pagers.AddedAndRemoved{
Added: idRes,
Removed: bh.deletedMsgIDs,
ValidModTimes: true,
DU: pagers.DeltaUpdate{},
}
return aar, bh.messagesErr
}
func (bh mockBackupHandler) includeContainer(

View File

@ -2,7 +2,6 @@ package groups
import (
"context"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -41,9 +40,9 @@ func (bh channelsBackupHandler) getContainers(
func (bh channelsBackupHandler) getContainerItemIDs(
ctx context.Context,
channelID, prevDelta string,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, canMakeDeltaQueries)
cc api.CallConfig,
) (pagers.AddedAndRemoved, error) {
return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, cc)
}
func (bh channelsBackupHandler) includeContainer(

View File

@ -2,7 +2,6 @@ package groups
import (
"context"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -10,6 +9,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
)
@ -25,8 +25,8 @@ type backupHandler interface {
getContainerItemIDs(
ctx context.Context,
containerID, prevDelta string,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error)
cc api.CallConfig,
) (pagers.AddedAndRemoved, error)
// includeContainer evaluates whether the container is included
// in the provided scope.

View File

@ -11,29 +11,10 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
// Idable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of an ID.
type Idable interface {
GetId() *string
}
// Descendable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a "parent folder".
type Descendable interface {
Idable
GetParentFolderId() *string
}
// Displayable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a display name.
type Displayable interface {
Idable
GetDisplayName() *string
}
type Container interface {
Descendable
Displayable
GetIDer
GetParentFolderIDer
GetDisplayNamer
}
// CachedContainer is used for local unit tests but also makes it so that this
@ -41,10 +22,12 @@ type Container interface {
// reuse logic in IDToPath.
type CachedContainer interface {
Container
// Location contains either the display names for the dirs (if this is a calendar)
// or nil
Location() *path.Builder
SetLocation(*path.Builder)
// Path contains either the ids for the dirs (if this is a calendar)
// or the display names for the dirs
Path() *path.Builder

View File

@ -0,0 +1,29 @@
package graph
import (
"time"
)
type GetIDer interface {
GetId() *string
}
type GetLastModifiedDateTimer interface {
GetLastModifiedDateTime() *time.Time
}
type GetAdditionalDataer interface {
GetAdditionalData() map[string]any
}
type GetDeletedDateTimer interface {
GetDeletedDateTime() *time.Time
}
type GetDisplayNamer interface {
GetDisplayName() *string
}
type GetParentFolderIDer interface {
GetParentFolderId() *string
}

View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"testing"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -35,6 +34,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
)
@ -495,36 +495,37 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
var (
err error
items map[string]time.Time
aar pagers.AddedAndRemoved
cc = api.CallConfig{
UseImmutableIDs: toggles.ExchangeImmutableIDs,
CanMakeDeltaQueries: true,
}
)
switch category {
case path.EmailCategory:
items, _, _, _, err = ac.Mail().GetAddedAndRemovedItemIDs(
aar, err = ac.Mail().GetAddedAndRemovedItemIDs(
ctx,
uidn.ID(),
containerID,
"",
toggles.ExchangeImmutableIDs,
true)
cc)
case path.EventsCategory:
items, _, _, _, err = ac.Events().GetAddedAndRemovedItemIDs(
aar, err = ac.Events().GetAddedAndRemovedItemIDs(
ctx,
uidn.ID(),
containerID,
"",
toggles.ExchangeImmutableIDs,
true)
cc)
case path.ContactsCategory:
items, _, _, _, err = ac.Contacts().GetAddedAndRemovedItemIDs(
aar, err = ac.Contacts().GetAddedAndRemovedItemIDs(
ctx,
uidn.ID(),
containerID,
"",
toggles.ExchangeImmutableIDs,
true)
cc)
}
require.NoError(
@ -534,6 +535,8 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
category,
locRef.String())
items := aar.Added
dest := dataset[category].dests[destName]
dest.locRef = locRef.String()
dest.containerID = containerID

View File

@ -27,6 +27,7 @@ const (
PagesCategory CategoryType = 7 // pages
DetailsCategory CategoryType = 8 // details
ChannelMessagesCategory CategoryType = 9 // channelMessages
ConversationPostsCategory CategoryType = 10 // conversationPosts
)
var strToCat = map[string]CategoryType{
@ -39,6 +40,7 @@ var strToCat = map[string]CategoryType{
strings.ToLower(PagesCategory.String()): PagesCategory,
strings.ToLower(DetailsCategory.String()): DetailsCategory,
strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory,
strings.ToLower(ConversationPostsCategory.String()): ConversationPostsCategory,
}
func ToCategoryType(s string) CategoryType {
@ -60,6 +62,7 @@ var catToHuman = map[CategoryType]string{
PagesCategory: "Pages",
DetailsCategory: "Details",
ChannelMessagesCategory: "Messages",
ConversationPostsCategory: "Posts",
}
// HumanString produces a more human-readable string version of the category.
@ -94,6 +97,7 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
},
GroupsService: {
ChannelMessagesCategory: {},
ConversationPostsCategory: {},
LibrariesCategory: {},
},
}

View File

@ -34,6 +34,7 @@ func (suite *CategoryTypeUnitSuite) TestToCategoryType() {
{input: "pages", expect: 7},
{input: "details", expect: 8},
{input: "channelmessages", expect: 9},
{input: "conversationposts", expect: 10},
}
for _, test := range table {
suite.Run(test.input, func() {
@ -60,6 +61,7 @@ func (suite *CategoryTypeUnitSuite) TestHumanString() {
{input: 7, expect: "Pages"},
{input: 8, expect: "Details"},
{input: 9, expect: "Messages"},
{input: 10, expect: "Posts"},
}
for _, test := range table {
suite.Run(test.input.String(), func() {

View File

@ -18,11 +18,12 @@ func _() {
_ = x[PagesCategory-7]
_ = x[DetailsCategory-8]
_ = x[ChannelMessagesCategory-9]
_ = x[ConversationPostsCategory-10]
}
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessages"
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessagesconversationPosts"
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80}
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80, 97}
func (i CategoryType) String() string {
if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) {

View File

@ -217,15 +217,14 @@ func (s *groups) AllData() []GroupsScope {
scopes = append(
scopes,
makeScope[GroupsScope](GroupsLibraryFolder, Any()),
makeScope[GroupsScope](GroupsChannel, Any()))
makeScope[GroupsScope](GroupsChannel, Any()),
makeScope[GroupsScope](GroupsConversation, Any()))
return scopes
}
// Channels produces one or more SharePoint channel scopes, where the channel
// matches upon a given channel by ID or Name. In order to ensure channel selection
// this should always be embedded within the Filter() set; include(channel()) will
// select all items in the channel without further filtering.
// matches upon a given channel by ID or Name.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
@ -260,6 +259,42 @@ func (s *groups) ChannelMessages(channels, messages []string, opts ...option) []
return scopes
}
// Conversations produces one or more SharePoint conversation scopes, where the
// conversation matches with a given conversation by ID or Topic.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) Conversation(conversations []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(
scopes,
makeScope[GroupsScope](GroupsConversation, conversations, os...))
return scopes
}
// ConversationPosts produces one or more Groups conversation post scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) ConversationPosts(conversations, posts []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(
scopes,
makeScope[GroupsScope](GroupsConversationPost, posts, os...).
set(GroupsConversation, conversations, opts...))
return scopes
}
// Sites produces one or more Groups site scopes, where the site
// matches upon a given site by ID or URL.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
@ -519,6 +554,8 @@ const (
GroupsGroup groupsCategory = "GroupsGroup"
GroupsChannel groupsCategory = "GroupsChannel"
GroupsChannelMessage groupsCategory = "GroupsChannelMessage"
GroupsConversation groupsCategory = "GroupsConversation"
GroupsConversationPost groupsCategory = "GroupsConversationPost"
GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder"
GroupsLibraryItem groupsCategory = "GroupsLibraryItem"
GroupsList groupsCategory = "GroupsList"
@ -546,10 +583,14 @@ const (
// groupsLeafProperties describes common metadata of the leaf categories
var groupsLeafProperties = map[categorizer]leafProperty{
GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf
GroupsChannelMessage: {
pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage},
pathType: path.ChannelMessagesCategory,
},
GroupsConversationPost: {
pathKeys: []categorizer{GroupsConversation, GroupsConversationPost},
pathType: path.ConversationPostsCategory,
},
GroupsLibraryItem: {
pathKeys: []categorizer{GroupsLibraryFolder, GroupsLibraryItem},
pathType: path.LibrariesCategory,
@ -571,12 +612,12 @@ func (c groupsCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser
func (c groupsCategory) leafCat() categorizer {
switch c {
// TODO: if channels ever contain more than one type of item,
// we'll need to fix this up.
case GroupsChannel, GroupsChannelMessage,
GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator,
GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore:
return GroupsChannelMessage
case GroupsConversation, GroupsConversationPost:
return GroupsConversationPost
case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSite, GroupsInfoSiteLibraryDrive,
GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore,
GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore:
@ -631,6 +672,9 @@ func (c groupsCategory) pathValues(
case GroupsChannel, GroupsChannelMessage:
folderCat, itemCat = GroupsChannel, GroupsChannelMessage
rFld = ent.Groups.ParentPath
case GroupsConversation, GroupsConversationPost:
folderCat, itemCat = GroupsConversation, GroupsConversationPost
rFld = ent.Groups.ParentPath
case GroupsLibraryFolder, GroupsLibraryItem:
folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem
rFld = ent.Groups.ParentPath
@ -729,7 +773,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS
os := []option{}
switch cat {
case GroupsChannel, GroupsLibraryFolder:
case GroupsChannel, GroupsConversation, GroupsLibraryFolder:
os = append(os, pathComparator())
}
@ -742,12 +786,16 @@ func (s GroupsScope) setDefaults() {
case GroupsGroup:
s[GroupsChannel.String()] = passAny
s[GroupsChannelMessage.String()] = passAny
s[GroupsConversation.String()] = passAny
s[GroupsConversationPost.String()] = passAny
s[GroupsLibraryFolder.String()] = passAny
s[GroupsLibraryItem.String()] = passAny
case GroupsChannel:
s[GroupsChannelMessage.String()] = passAny
case GroupsLibraryFolder:
s[GroupsLibraryItem.String()] = passAny
case GroupsConversation:
s[GroupsConversationPost.String()] = passAny
}
}
@ -768,6 +816,7 @@ func (s groups) Reduce(
s.Selector,
map[path.CategoryType]groupsCategory{
path.ChannelMessagesCategory: GroupsChannelMessage,
path.ConversationPostsCategory: GroupsConversationPost,
path.LibrariesCategory: GroupsLibraryItem,
},
errs)
@ -793,6 +842,8 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool {
acceptableItemType = int(details.SharePointLibrary)
case GroupsChannelMessage:
acceptableItemType = int(details.GroupsChannelMessage)
case GroupsConversationPost:
acceptableItemType = int(details.GroupsConversationPost)
}
switch infoCat {

View File

@ -112,6 +112,9 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
chanItem = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems1), "chitem")
chanItem2 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems2), "chitem2")
chanItem3 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems3), "chitem3")
convItem = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems1), "convitem")
convItem2 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems2), "convitem2")
convItem3 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems3), "convitem3")
)
deets := &details.Details{
@ -186,6 +189,39 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
},
},
},
{
RepoRef: convItem,
ItemRef: "convitem",
LocationRef: strings.Join(itemElems1, "/"),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems1, "/"),
},
},
},
{
RepoRef: convItem2,
LocationRef: strings.Join(itemElems2, "/"),
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems2, "/"),
},
},
},
{
RepoRef: convItem3,
ItemRef: "convitem3",
LocationRef: strings.Join(itemElems3, "/"),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems3, "/"),
},
},
},
},
},
}
@ -207,7 +243,10 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
sel.Include(sel.AllData())
return sel
},
expect: arr(libItem, libItem2, libItem3, chanItem, chanItem2, chanItem3),
expect: arr(
libItem, libItem2, libItem3,
chanItem, chanItem2, chanItem3,
convItem, convItem2, convItem3),
},
{
name: "only match library item",
@ -218,15 +257,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
},
expect: arr(libItem2),
},
{
name: "only match channel item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
return sel
},
expect: arr(chanItem2),
},
{
name: "library id doesn't match name",
makeSelector: func() *GroupsRestore {
@ -237,16 +267,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "channel id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "library only match item name",
makeSelector: func() *GroupsRestore {
@ -275,6 +295,44 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
},
expect: arr(libItem, libItem2),
},
{
name: "only match channel item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
return sel
},
expect: arr(chanItem2),
},
{
name: "channel id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match conversation item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ConversationPosts(Any(), []string{"convitem2"}))
return sel
},
expect: arr(convItem2),
},
{
name: "conversation id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ConversationPosts(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -320,6 +378,17 @@ func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() {
},
cfg: Config{},
},
{
name: "Groups Conversation Posts",
sc: GroupsConversationPost,
pathElems: elems,
locRef: "",
expected: map[categorizer][]string{
GroupsConversation: {""},
GroupsConversationPost: {itemID, shortRef},
},
cfg: Config{},
},
}
for _, test := range table {
@ -476,6 +545,8 @@ func (suite *GroupsSelectorSuite) TestCategory_PathType() {
{GroupsCategoryUnknown, path.UnknownCategory},
{GroupsChannel, path.ChannelMessagesCategory},
{GroupsChannelMessage, path.ChannelMessagesCategory},
{GroupsConversation, path.ConversationPostsCategory},
{GroupsConversationPost, path.ConversationPostsCategory},
{GroupsInfoChannelMessageCreator, path.ChannelMessagesCategory},
{GroupsInfoChannelMessageCreatedAfter, path.ChannelMessagesCategory},
{GroupsInfoChannelMessageCreatedBefore, path.ChannelMessagesCategory},

View File

@ -2,7 +2,6 @@ package api
import (
"context"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -186,18 +185,18 @@ func filterOutSystemMessages(cm models.ChatMessageable) bool {
func (c Channels) GetChannelMessageIDs(
ctx context.Context,
teamID, channelID, prevDeltaLink string,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
added, validModTimes, removed, du, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
ctx,
c.NewChannelMessagePager(teamID, channelID, CallConfig{}),
c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink),
prevDeltaLink,
canMakeDeltaQueries,
cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable],
filterOutSystemMessages)
return added, validModTimes, removed, du, clues.Stack(err).OrNil()
return aar, clues.Stack(err).OrNil()
}
// ---------------------------------------------------------------------------

View File

@ -56,30 +56,34 @@ func (suite *ChannelsPagerIntgSuite) TestEnumerateChannelMessages() {
ctx, flush := tester.NewContext(t)
defer flush()
addedIDs, _, _, du, err := ac.GetChannelMessageIDs(
cc := CallConfig{
CanMakeDeltaQueries: true,
}
aar, err := ac.GetChannelMessageIDs(
ctx,
suite.its.group.id,
suite.its.group.testContainerID,
"",
true)
cc)
require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, addedIDs)
require.NotZero(t, du.URL, "delta link")
require.True(t, du.Reset, "reset due to empty prev delta link")
require.NotEmpty(t, aar.Added)
require.NotZero(t, aar.DU.URL, "delta link")
require.True(t, aar.DU.Reset, "reset due to empty prev delta link")
addedIDs, _, deletedIDs, du, err := ac.GetChannelMessageIDs(
aar, err = ac.GetChannelMessageIDs(
ctx,
suite.its.group.id,
suite.its.group.testContainerID,
du.URL,
true)
aar.DU.URL,
cc)
require.NoError(t, err, clues.ToCore(err))
require.Empty(t, addedIDs, "should have no new messages from delta")
require.Empty(t, deletedIDs, "should have no deleted messages from delta")
require.NotZero(t, du.URL, "delta link")
require.False(t, du.Reset, "prev delta link should be valid")
require.Empty(t, aar.Added, "should have no new messages from delta")
require.Empty(t, aar.Removed, "should have no deleted messages from delta")
require.NotZero(t, aar.DU.URL, "delta link")
require.False(t, aar.DU.Reset, "prev delta link should be valid")
for id := range addedIDs {
for id := range aar.Added {
suite.Run(id+"-replies", func() {
testEnumerateChannelMessageReplies(
suite.T(),

View File

@ -161,6 +161,8 @@ func (c Client) Post(
type CallConfig struct {
Expand []string
Select []string
CanMakeDeltaQueries bool
UseImmutableIDs bool
}
// ---------------------------------------------------------------------------

View File

@ -2,7 +2,6 @@ package api
import (
"context"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -250,9 +249,8 @@ func (p *contactDeltaPager) ValidModTimes() bool {
func (c Contacts) GetAddedAndRemovedItemIDs(
ctx context.Context,
userID, containerID, prevDeltaLink string,
immutableIDs bool,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
ctx = clues.Add(
ctx,
"data_category", path.ContactsCategory,
@ -263,12 +261,12 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
userID,
containerID,
prevDeltaLink,
immutableIDs,
cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...)
pager := c.NewContactsPager(
userID,
containerID,
immutableIDs,
cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Contactable](
@ -276,6 +274,6 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
pager,
deltaPager,
prevDeltaLink,
canMakeDeltaQueries,
cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Contactable])
}

View File

@ -140,7 +140,7 @@ func (c Conversations) NewConversationThreadsPager(
}
}
// GetConversations fetches all conversations in the group.
// GetConversations fetches all conversation threads in the group.
func (c Conversations) GetConversationThreads(
ctx context.Context,
groupID, conversationID string,
@ -218,7 +218,7 @@ func (c Conversations) NewConversationThreadPostsPager(
}
}
// GetConversations fetches all conversations in the group.
// GetConversations fetches all conversation posts in the group.
func (c Conversations) GetConversationThreadPosts(
ctx context.Context,
groupID, conversationID, threadID string,
@ -230,3 +230,23 @@ func (c Conversations) GetConversationThreadPosts(
return items, graph.Stack(ctx, err).OrNil()
}
// GetConversations fetches all added and deleted conversation posts in the group.
func (c Conversations) GetConversationThreadPostIDs(
ctx context.Context,
groupID, conversationID, threadID string,
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
canMakeDeltaQueries := false
aarh, err := pagers.GetAddedAndRemovedItemIDs[models.Postable](
ctx,
c.NewConversationThreadPostsPager(groupID, conversationID, threadID, CallConfig{}),
nil,
"",
canMakeDeltaQueries,
pagers.AddedAndRemovedAddAll[models.Postable],
pagers.FilterIncludeAll[models.Postable])
return aarh, clues.Stack(err).OrNil()
}

View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -49,6 +50,18 @@ func (suite *ConversationsPagerIntgSuite) TestEnumerateConversations_withThreads
for _, thread := range threads {
posts := testEnumerateConvPosts(suite, conv, thread)
aar, err := ac.GetConversationThreadPostIDs(
ctx,
suite.its.group.id,
ptr.Val(conv.GetId()),
ptr.Val(thread.GetId()),
CallConfig{})
require.NoError(t, err, clues.ToCore(err))
require.Equal(t, len(posts), len(aar.Added), "added the same number of ids and posts")
assert.True(t, aar.ValidModTimes, "mod times should be valid")
assert.Empty(t, aar.Removed, "no items should get removed")
assert.Empty(t, aar.DU.URL, "no delta update token should be provided")
for _, post := range posts {
testGetPostByID(suite, conv, thread, post)
}

View File

@ -3,7 +3,6 @@ package api
import (
"context"
"fmt"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -245,9 +244,8 @@ func (p *eventDeltaPager) ValidModTimes() bool {
func (c Events) GetAddedAndRemovedItemIDs(
ctx context.Context,
userID, containerID, prevDeltaLink string,
immutableIDs bool,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
ctx = clues.Add(
ctx,
"data_category", path.EventsCategory,
@ -258,12 +256,12 @@ func (c Events) GetAddedAndRemovedItemIDs(
userID,
containerID,
prevDeltaLink,
immutableIDs,
cc.UseImmutableIDs,
idAnd()...)
pager := c.NewEventsPager(
userID,
containerID,
immutableIDs,
cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Eventable](
@ -271,6 +269,6 @@ func (c Events) GetAddedAndRemovedItemIDs(
pager,
deltaPager,
prevDeltaLink,
canMakeDeltaQueries,
cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Eventable])
}

View File

@ -0,0 +1,31 @@
package api
import (
"context"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/pkg/fault"
)
type GetAndSerializeItemer[INFO any] interface {
GetItemer[INFO]
Serializer
}
type GetItemer[INFO any] interface {
GetItem(
ctx context.Context,
user, itemID string,
immutableIDs bool,
errs *fault.Bus,
) (serialization.Parsable, *INFO, error)
}
type Serializer interface {
Serialize(
ctx context.Context,
item serialization.Parsable,
protectedResource, itemID string,
) ([]byte, error)
}

View File

@ -3,7 +3,6 @@ package api
import (
"context"
"fmt"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
@ -247,9 +246,8 @@ func (p *mailDeltaPager) ValidModTimes() bool {
func (c Mail) GetAddedAndRemovedItemIDs(
ctx context.Context,
userID, containerID, prevDeltaLink string,
immutableIDs bool,
canMakeDeltaQueries bool,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
ctx = clues.Add(
ctx,
"data_category", path.EmailCategory,
@ -260,12 +258,12 @@ func (c Mail) GetAddedAndRemovedItemIDs(
userID,
containerID,
prevDeltaLink,
immutableIDs,
cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...)
pager := c.NewMailPager(
userID,
containerID,
immutableIDs,
cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Messageable](
@ -273,6 +271,6 @@ func (c Mail) GetAddedAndRemovedItemIDs(
pager,
deltaPager,
prevDeltaLink,
canMakeDeltaQueries,
cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Messageable])
}

View File

@ -366,13 +366,28 @@ func batchDeltaEnumerateItems[T any](
return results, du, clues.Stack(err).OrNil()
}
// ---------------------------------------------------------------------------
// filter funcs
// ---------------------------------------------------------------------------
func FilterIncludeAll[T any](_ T) bool {
return true
}
// ---------------------------------------------------------------------------
// shared enumeration runner funcs
// ---------------------------------------------------------------------------
type AddedAndRemoved struct {
Added map[string]time.Time
Removed []string
DU DeltaUpdate
ValidModTimes bool
}
type addedAndRemovedHandler[T any] func(
items []T,
filters ...func(T) bool, // false -> remove, true -> keep
filters ...func(T) bool,
) (
map[string]time.Time,
[]string,
@ -387,16 +402,23 @@ func GetAddedAndRemovedItemIDs[T any](
canMakeDeltaQueries bool,
aarh addedAndRemovedHandler[T],
filters ...func(T) bool,
) (map[string]time.Time, bool, []string, DeltaUpdate, error) {
) (AddedAndRemoved, error) {
if canMakeDeltaQueries {
ts, du, err := batchDeltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink)
if err != nil && !graph.IsErrInvalidDelta(err) && !graph.IsErrDeltaNotSupported(err) {
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err)
return AddedAndRemoved{}, graph.Stack(ctx, err)
}
if err == nil {
a, r, err := aarh(ts, filters...)
return a, deltaPager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
aar := AddedAndRemoved{
Added: a,
Removed: r,
DU: du,
ValidModTimes: deltaPager.ValidModTimes(),
}
return aar, graph.Stack(ctx, err).OrNil()
}
}
@ -404,31 +426,61 @@ func GetAddedAndRemovedItemIDs[T any](
ts, err := BatchEnumerateItems(ctx, pager)
if err != nil {
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err)
return AddedAndRemoved{}, graph.Stack(ctx, err)
}
a, r, err := aarh(ts, filters...)
return a, pager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
aar := AddedAndRemoved{
Added: a,
Removed: r,
DU: du,
ValidModTimes: pager.ValidModTimes(),
}
type getIDer interface {
GetId() *string
return aar, graph.Stack(ctx, err).OrNil()
}
type getIDAndModDateTimer interface {
graph.GetIDer
graph.GetLastModifiedDateTimer
}
// AddedAndRemovedAddAll indiscriminately adds every item to the added list, deleting nothing.
func AddedAndRemovedAddAll[T any](
items []T,
filters ...func(T) bool,
) (map[string]time.Time, []string, error) {
added := map[string]time.Time{}
for _, item := range items {
passAllFilters := true
for _, passes := range filters {
passAllFilters = passAllFilters && passes(item)
}
if !passAllFilters {
continue
}
giamdt, ok := any(item).(getIDAndModDateTimer)
if !ok {
return nil, nil, clues.New("item does not provide id and modified date time getters").
With("item_type", fmt.Sprintf("%T", item))
}
added[ptr.Val(giamdt.GetId())] = dttm.OrNow(ptr.Val(giamdt.GetLastModifiedDateTime()))
}
return added, []string{}, nil
}
// for added and removed by additionalData[@removed]
type getIDModAndAddtler interface {
getIDer
getModTimer
GetAdditionalData() map[string]any
}
// for types that are non-compliant with this interface,
// pagers will need to wrap the return value in a struct
// that provides this compliance.
type getModTimer interface {
GetLastModifiedDateTime() *time.Time
graph.GetIDer
graph.GetLastModifiedDateTimer
graph.GetAdditionalDataer
}
func AddedAndRemovedByAddtlData[T any](
@ -451,7 +503,7 @@ func AddedAndRemovedByAddtlData[T any](
giaa, ok := any(item).(getIDModAndAddtler)
if !ok {
return nil, nil, clues.New("item does not provide id and additional data getters").
return nil, nil, clues.New("item does not provide id, modified date time, and additional data getters").
With("item_type", fmt.Sprintf("%T", item))
}
@ -461,7 +513,11 @@ func AddedAndRemovedByAddtlData[T any](
if giaa.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
var modTime time.Time
if mt, ok := giaa.(getModTimer); ok {
// not all items comply with last modified date time, and not all
// items can be wrapped in a way that produces a valid value for
// the func. That's why this isn't packed in to the expected
// interfaace composition.
if mt, ok := giaa.(graph.GetLastModifiedDateTimer); ok {
// Make sure to get a non-zero mod time if the item doesn't have one for
// some reason. Otherwise we can hit an issue where kopia has a
// different mod time for the file than the details does. This occurs
@ -485,9 +541,9 @@ func AddedAndRemovedByAddtlData[T any](
// for added and removed by GetDeletedDateTime()
type getIDModAndDeletedDateTimer interface {
getIDer
getModTimer
GetDeletedDateTime() *time.Time
graph.GetIDer
graph.GetLastModifiedDateTimer
graph.GetDeletedDateTimer
}
func AddedAndRemovedByDeletedDateTime[T any](
@ -510,14 +566,18 @@ func AddedAndRemovedByDeletedDateTime[T any](
giaddt, ok := any(item).(getIDModAndDeletedDateTimer)
if !ok {
return nil, nil, clues.New("item does not provide id and deleted date time getters").
return nil, nil, clues.New("item does not provide id, modified, and deleted date time getters").
With("item_type", fmt.Sprintf("%T", item))
}
if giaddt.GetDeletedDateTime() == nil {
var modTime time.Time
if mt, ok := giaddt.(getModTimer); ok {
// not all items comply with last modified date time, and not all
// items can be wrapped in a way that produces a valid value for
// the func. That's why this isn't packed in to the expected
// interfaace composition.
if mt, ok := giaddt.(graph.GetLastModifiedDateTimer); ok {
// Make sure to get a non-zero mod time if the item doesn't have one for
// some reason. Otherwise we can hit an issue where kopia has a
// different mod time for the file than the details does. This occurs

View File

@ -520,7 +520,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
filters = append(filters, test.filter)
}
added, validModTimes, removed, deltaUpdate, err := GetAddedAndRemovedItemIDs[testItem](
aar, err := GetAddedAndRemovedItemIDs[testItem](
ctx,
test.pagerGetter(t),
test.deltaPagerGetter(t),
@ -530,18 +530,18 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
filters...)
require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err))
if validModTimes {
assert.Equal(t, test.expect.added, added, "added item IDs and mod times")
if aar.ValidModTimes {
assert.Equal(t, test.expect.added, aar.Added, "added item IDs and mod times")
} else {
assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(added), "added item IDs")
for _, modtime := range added {
assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(aar.Added), "added item IDs")
for _, modtime := range aar.Added {
assert.True(t, modtime.After(epoch), "mod time after epoch")
assert.False(t, modtime.Equal(time.Time{}), "non-zero mod time")
}
}
assert.Equal(t, test.expect.validModTimes, validModTimes, "valid mod times")
assert.EqualValues(t, test.expect.removed, removed, "removed item IDs")
assert.Equal(t, test.expect.deltaUpdate, deltaUpdate, "delta update")
assert.Equal(t, test.expect.validModTimes, aar.ValidModTimes, "valid mod times")
assert.EqualValues(t, test.expect.removed, aar.Removed, "removed item IDs")
assert.Equal(t, test.expect.deltaUpdate, aar.DU, "delta update")
})
}
}