add getPostIDs in conversations pager (#4578)

adds the getPostIDs func to ensure conversations
complies with standard data paging patterns

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #4536

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-11-07 14:57:55 -07:00 committed by GitHub
parent ec1afa8c84
commit 49bbdfc096
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 511 additions and 232 deletions

View File

@ -42,28 +42,28 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
{ {
name: "no inputs", name: "no inputs",
opts: utils.GroupsOpts{}, opts: utils.GroupsOpts{},
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
{ {
name: "empty", name: "empty",
opts: utils.GroupsOpts{ opts: utils.GroupsOpts{
Groups: empty, Groups: empty,
}, },
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
{ {
name: "single inputs", name: "single inputs",
opts: utils.GroupsOpts{ opts: utils.GroupsOpts{
Groups: single, Groups: single,
}, },
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
{ {
name: "multi inputs", name: "multi inputs",
opts: utils.GroupsOpts{ opts: utils.GroupsOpts{
Groups: multi, Groups: multi,
}, },
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
// sharepoint // sharepoint
{ {
@ -120,7 +120,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
FileName: empty, FileName: empty,
FolderPath: empty, FolderPath: empty,
}, },
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
{ {
name: "library folder suffixes and contains", name: "library folder suffixes and contains",
@ -128,7 +128,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
FileName: empty, FileName: empty,
FolderPath: empty, FolderPath: empty,
}, },
expectIncludeLen: 2, expectIncludeLen: 3,
}, },
{ {
name: "Page Folder", name: "Page Folder",
@ -389,7 +389,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
{ {
name: "none", name: "none",
cats: []string{}, cats: []string{},
expectScopeLen: 2, expectScopeLen: 3,
}, },
{ {
name: "libraries", name: "libraries",

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
) )
@ -157,14 +158,18 @@ func populateCollections(
ictx = clues.Add(ictx, "previous_path", prevPath) ictx = clues.Add(ictx, "previous_path", prevPath)
added, _, removed, newDelta, err := bh.itemEnumerator(). cc := api.CallConfig{
CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta,
UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs,
}
addAndRem, err := bh.itemEnumerator().
GetAddedAndRemovedItemIDs( GetAddedAndRemovedItemIDs(
ictx, ictx,
qp.ProtectedResource.ID(), qp.ProtectedResource.ID(),
cID, cID,
prevDelta, prevDelta,
ctrlOpts.ToggleFeatures.ExchangeImmutableIDs, cc)
!ctrlOpts.ToggleFeatures.DisableDelta)
if err != nil { if err != nil {
if !graph.IsErrDeletedInFlight(err) { if !graph.IsErrDeletedInFlight(err) {
el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation)) el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
@ -176,12 +181,12 @@ func populateCollections(
// to reset. This prevents any old items from being retained in // to reset. This prevents any old items from being retained in
// storage. If the container (or its children) are sill missing // storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned. // on the next backup, they'll get tombstoned.
newDelta = pagers.DeltaUpdate{Reset: true} addAndRem.DU = pagers.DeltaUpdate{Reset: true}
} }
if len(newDelta.URL) > 0 { if len(addAndRem.DU.URL) > 0 {
deltaURLs[cID] = newDelta.URL deltaURLs[cID] = addAndRem.DU.URL
} else if !newDelta.Reset { } else if !addAndRem.DU.Reset {
logger.Ctx(ictx).Info("missing delta url") logger.Ctx(ictx).Info("missing delta url")
} }
@ -191,11 +196,11 @@ func populateCollections(
prevPath, prevPath,
locPath, locPath,
ctrlOpts, ctrlOpts,
newDelta.Reset), addAndRem.DU.Reset),
qp.ProtectedResource.ID(), qp.ProtectedResource.ID(),
bh.itemHandler(), bh.itemHandler(),
added, addAndRem.Added,
removed, addAndRem.Removed,
// TODO: produce a feature flag that allows selective // TODO: produce a feature flag that allows selective
// enabling of valid modTimes. This currently produces // enabling of valid modTimes. This currently produces
// rare-case failures with incorrect details merging. // rare-case failures with incorrect details merging.

View File

@ -75,18 +75,11 @@ type (
func (mg mockGetter) GetAddedAndRemovedItemIDs( func (mg mockGetter) GetAddedAndRemovedItemIDs(
ctx context.Context, ctx context.Context,
userID, cID, prevDelta string, userID, cID, prevDelta string,
_ bool, _ api.CallConfig,
_ bool, ) (pagers.AddedAndRemoved, error) {
) (
map[string]time.Time,
bool,
[]string,
pagers.DeltaUpdate,
error,
) {
results, ok := mg.results[cID] results, ok := mg.results[cID]
if !ok { if !ok {
return nil, false, nil, pagers.DeltaUpdate{}, clues.New("mock not found for " + cID) return pagers.AddedAndRemoved{}, clues.New("mock not found for " + cID)
} }
delta := results.newDelta delta := results.newDelta
@ -99,7 +92,14 @@ func (mg mockGetter) GetAddedAndRemovedItemIDs(
resAdded[add] = time.Time{} resAdded[add] = time.Time{}
} }
return resAdded, false, results.removed, delta, results.err aar := pagers.AddedAndRemoved{
Added: resAdded,
Removed: results.removed,
ValidModTimes: false,
DU: delta,
}
return aar, results.err
} }
var _ graph.ContainerResolver = &mockResolver{} var _ graph.ContainerResolver = &mockResolver{}

View File

@ -2,7 +2,6 @@ package exchange
import ( import (
"context" "context"
"time"
"github.com/microsoft/kiota-abstractions-go/serialization" "github.com/microsoft/kiota-abstractions-go/serialization"
@ -30,9 +29,8 @@ type addedAndRemovedItemGetter interface {
GetAddedAndRemovedItemIDs( GetAddedAndRemovedItemIDs(
ctx context.Context, ctx context.Context,
user, containerID, oldDeltaToken string, user, containerID, oldDeltaToken string,
immutableIDs bool, cc api.CallConfig,
canMakeDeltaQueries bool, ) (pagers.AddedAndRemoved, error)
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error)
} }
type itemGetterSerializer interface { type itemGetterSerializer interface {

View File

@ -20,6 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
) )
// TODO: incremental support // TODO: incremental support
@ -152,20 +153,22 @@ func populateCollections(
// if the channel has no email property, it is unable to process delta tokens // if the channel has no email property, it is unable to process delta tokens
// and will return an error if a delta token is queried. // and will return an error if a delta token is queried.
canMakeDeltaQueries := len(ptr.Val(c.GetEmail())) > 0 cc := api.CallConfig{
CanMakeDeltaQueries: len(ptr.Val(c.GetEmail())) > 0,
}
add, _, rem, du, err := bh.getContainerItemIDs(ctx, cID, prevDelta, canMakeDeltaQueries) addAndRem, err := bh.getContainerItemIDs(ctx, cID, prevDelta, cc)
if err != nil { if err != nil {
el.AddRecoverable(ctx, clues.Stack(err)) el.AddRecoverable(ctx, clues.Stack(err))
continue continue
} }
added := str.SliceToMap(maps.Keys(add)) added := str.SliceToMap(maps.Keys(addAndRem.Added))
removed := str.SliceToMap(rem) removed := str.SliceToMap(addAndRem.Removed)
if len(du.URL) > 0 { if len(addAndRem.DU.URL) > 0 {
deltaURLs[cID] = du.URL deltaURLs[cID] = addAndRem.DU.URL
} else if !du.Reset { } else if !addAndRem.DU.Reset {
logger.Ctx(ictx).Info("missing delta url") logger.Ctx(ictx).Info("missing delta url")
} }
@ -188,7 +191,7 @@ func populateCollections(
prevPath, prevPath,
path.Builder{}.Append(cName), path.Builder{}.Append(cName),
ctrlOpts, ctrlOpts,
du.Reset), addAndRem.DU.Reset),
bh, bh,
qp.ProtectedResource.ID(), qp.ProtectedResource.ID(),
added, added,

View File

@ -58,15 +58,22 @@ func (bh mockBackupHandler) getContainers(context.Context) ([]models.Channelable
func (bh mockBackupHandler) getContainerItemIDs( func (bh mockBackupHandler) getContainerItemIDs(
_ context.Context, _ context.Context,
_, _ string, _, _ string,
_ bool, _ api.CallConfig,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ) (pagers.AddedAndRemoved, error) {
idRes := make(map[string]time.Time, len(bh.messageIDs)) idRes := make(map[string]time.Time, len(bh.messageIDs))
for _, id := range bh.messageIDs { for _, id := range bh.messageIDs {
idRes[id] = time.Time{} idRes[id] = time.Time{}
} }
return idRes, true, bh.deletedMsgIDs, pagers.DeltaUpdate{}, bh.messagesErr aar := pagers.AddedAndRemoved{
Added: idRes,
Removed: bh.deletedMsgIDs,
ValidModTimes: true,
DU: pagers.DeltaUpdate{},
}
return aar, bh.messagesErr
} }
func (bh mockBackupHandler) includeContainer( func (bh mockBackupHandler) includeContainer(

View File

@ -2,7 +2,6 @@ package groups
import ( import (
"context" "context"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -41,9 +40,9 @@ func (bh channelsBackupHandler) getContainers(
func (bh channelsBackupHandler) getContainerItemIDs( func (bh channelsBackupHandler) getContainerItemIDs(
ctx context.Context, ctx context.Context,
channelID, prevDelta string, channelID, prevDelta string,
canMakeDeltaQueries bool, cc api.CallConfig,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ) (pagers.AddedAndRemoved, error) {
return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, canMakeDeltaQueries) return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, cc)
} }
func (bh channelsBackupHandler) includeContainer( func (bh channelsBackupHandler) includeContainer(

View File

@ -2,7 +2,6 @@ package groups
import ( import (
"context" "context"
"time"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -10,6 +9,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers" "github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
) )
@ -25,8 +25,8 @@ type backupHandler interface {
getContainerItemIDs( getContainerItemIDs(
ctx context.Context, ctx context.Context,
containerID, prevDelta string, containerID, prevDelta string,
canMakeDeltaQueries bool, cc api.CallConfig,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) ) (pagers.AddedAndRemoved, error)
// includeContainer evaluates whether the container is included // includeContainer evaluates whether the container is included
// in the provided scope. // in the provided scope.

View File

@ -11,29 +11,10 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
// Idable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of an ID.
type Idable interface {
GetId() *string
}
// Descendable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a "parent folder".
type Descendable interface {
Idable
GetParentFolderId() *string
}
// Displayable represents objects that implement msgraph-sdk-go/models.entityable
// and have the concept of a display name.
type Displayable interface {
Idable
GetDisplayName() *string
}
type Container interface { type Container interface {
Descendable GetIDer
Displayable GetParentFolderIDer
GetDisplayNamer
} }
// CachedContainer is used for local unit tests but also makes it so that this // CachedContainer is used for local unit tests but also makes it so that this
@ -41,10 +22,12 @@ type Container interface {
// reuse logic in IDToPath. // reuse logic in IDToPath.
type CachedContainer interface { type CachedContainer interface {
Container Container
// Location contains either the display names for the dirs (if this is a calendar) // Location contains either the display names for the dirs (if this is a calendar)
// or nil // or nil
Location() *path.Builder Location() *path.Builder
SetLocation(*path.Builder) SetLocation(*path.Builder)
// Path contains either the ids for the dirs (if this is a calendar) // Path contains either the ids for the dirs (if this is a calendar)
// or the display names for the dirs // or the display names for the dirs
Path() *path.Builder Path() *path.Builder

View File

@ -0,0 +1,29 @@
package graph
import (
"time"
)
type GetIDer interface {
GetId() *string
}
type GetLastModifiedDateTimer interface {
GetLastModifiedDateTime() *time.Time
}
type GetAdditionalDataer interface {
GetAdditionalData() map[string]any
}
type GetDeletedDateTimer interface {
GetDeletedDateTime() *time.Time
}
type GetDisplayNamer interface {
GetDisplayName() *string
}
type GetParentFolderIDer interface {
GetParentFolderId() *string
}

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"fmt" "fmt"
"testing" "testing"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -35,6 +34,7 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
) )
@ -495,36 +495,37 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
var ( var (
err error err error
items map[string]time.Time aar pagers.AddedAndRemoved
cc = api.CallConfig{
UseImmutableIDs: toggles.ExchangeImmutableIDs,
CanMakeDeltaQueries: true,
}
) )
switch category { switch category {
case path.EmailCategory: case path.EmailCategory:
items, _, _, _, err = ac.Mail().GetAddedAndRemovedItemIDs( aar, err = ac.Mail().GetAddedAndRemovedItemIDs(
ctx, ctx,
uidn.ID(), uidn.ID(),
containerID, containerID,
"", "",
toggles.ExchangeImmutableIDs, cc)
true)
case path.EventsCategory: case path.EventsCategory:
items, _, _, _, err = ac.Events().GetAddedAndRemovedItemIDs( aar, err = ac.Events().GetAddedAndRemovedItemIDs(
ctx, ctx,
uidn.ID(), uidn.ID(),
containerID, containerID,
"", "",
toggles.ExchangeImmutableIDs, cc)
true)
case path.ContactsCategory: case path.ContactsCategory:
items, _, _, _, err = ac.Contacts().GetAddedAndRemovedItemIDs( aar, err = ac.Contacts().GetAddedAndRemovedItemIDs(
ctx, ctx,
uidn.ID(), uidn.ID(),
containerID, containerID,
"", "",
toggles.ExchangeImmutableIDs, cc)
true)
} }
require.NoError( require.NoError(
@ -534,6 +535,8 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
category, category,
locRef.String()) locRef.String())
items := aar.Added
dest := dataset[category].dests[destName] dest := dataset[category].dests[destName]
dest.locRef = locRef.String() dest.locRef = locRef.String()
dest.containerID = containerID dest.containerID = containerID

View File

@ -27,6 +27,7 @@ const (
PagesCategory CategoryType = 7 // pages PagesCategory CategoryType = 7 // pages
DetailsCategory CategoryType = 8 // details DetailsCategory CategoryType = 8 // details
ChannelMessagesCategory CategoryType = 9 // channelMessages ChannelMessagesCategory CategoryType = 9 // channelMessages
ConversationPostsCategory CategoryType = 10 // conversationPosts
) )
var strToCat = map[string]CategoryType{ var strToCat = map[string]CategoryType{
@ -39,6 +40,7 @@ var strToCat = map[string]CategoryType{
strings.ToLower(PagesCategory.String()): PagesCategory, strings.ToLower(PagesCategory.String()): PagesCategory,
strings.ToLower(DetailsCategory.String()): DetailsCategory, strings.ToLower(DetailsCategory.String()): DetailsCategory,
strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory, strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory,
strings.ToLower(ConversationPostsCategory.String()): ConversationPostsCategory,
} }
func ToCategoryType(s string) CategoryType { func ToCategoryType(s string) CategoryType {
@ -60,6 +62,7 @@ var catToHuman = map[CategoryType]string{
PagesCategory: "Pages", PagesCategory: "Pages",
DetailsCategory: "Details", DetailsCategory: "Details",
ChannelMessagesCategory: "Messages", ChannelMessagesCategory: "Messages",
ConversationPostsCategory: "Posts",
} }
// HumanString produces a more human-readable string version of the category. // HumanString produces a more human-readable string version of the category.
@ -94,6 +97,7 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
}, },
GroupsService: { GroupsService: {
ChannelMessagesCategory: {}, ChannelMessagesCategory: {},
ConversationPostsCategory: {},
LibrariesCategory: {}, LibrariesCategory: {},
}, },
} }

View File

@ -34,6 +34,7 @@ func (suite *CategoryTypeUnitSuite) TestToCategoryType() {
{input: "pages", expect: 7}, {input: "pages", expect: 7},
{input: "details", expect: 8}, {input: "details", expect: 8},
{input: "channelmessages", expect: 9}, {input: "channelmessages", expect: 9},
{input: "conversationposts", expect: 10},
} }
for _, test := range table { for _, test := range table {
suite.Run(test.input, func() { suite.Run(test.input, func() {
@ -60,6 +61,7 @@ func (suite *CategoryTypeUnitSuite) TestHumanString() {
{input: 7, expect: "Pages"}, {input: 7, expect: "Pages"},
{input: 8, expect: "Details"}, {input: 8, expect: "Details"},
{input: 9, expect: "Messages"}, {input: 9, expect: "Messages"},
{input: 10, expect: "Posts"},
} }
for _, test := range table { for _, test := range table {
suite.Run(test.input.String(), func() { suite.Run(test.input.String(), func() {

View File

@ -18,11 +18,12 @@ func _() {
_ = x[PagesCategory-7] _ = x[PagesCategory-7]
_ = x[DetailsCategory-8] _ = x[DetailsCategory-8]
_ = x[ChannelMessagesCategory-9] _ = x[ChannelMessagesCategory-9]
_ = x[ConversationPostsCategory-10]
} }
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessages" const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessagesconversationPosts"
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80} var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80, 97}
func (i CategoryType) String() string { func (i CategoryType) String() string {
if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) { if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) {

View File

@ -217,15 +217,14 @@ func (s *groups) AllData() []GroupsScope {
scopes = append( scopes = append(
scopes, scopes,
makeScope[GroupsScope](GroupsLibraryFolder, Any()), makeScope[GroupsScope](GroupsLibraryFolder, Any()),
makeScope[GroupsScope](GroupsChannel, Any())) makeScope[GroupsScope](GroupsChannel, Any()),
makeScope[GroupsScope](GroupsConversation, Any()))
return scopes return scopes
} }
// Channels produces one or more SharePoint channel scopes, where the channel // Channels produces one or more SharePoint channel scopes, where the channel
// matches upon a given channel by ID or Name. In order to ensure channel selection // matches upon a given channel by ID or Name.
// this should always be embedded within the Filter() set; include(channel()) will
// select all items in the channel without further filtering.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None] // If any slice is empty, it defaults to [selectors.None]
@ -260,6 +259,42 @@ func (s *groups) ChannelMessages(channels, messages []string, opts ...option) []
return scopes return scopes
} }
// Conversations produces one or more SharePoint conversation scopes, where the
// conversation matches with a given conversation by ID or Topic.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) Conversation(conversations []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(
scopes,
makeScope[GroupsScope](GroupsConversation, conversations, os...))
return scopes
}
// ConversationPosts produces one or more Groups conversation post scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) ConversationPosts(conversations, posts []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(
scopes,
makeScope[GroupsScope](GroupsConversationPost, posts, os...).
set(GroupsConversation, conversations, opts...))
return scopes
}
// Sites produces one or more Groups site scopes, where the site // Sites produces one or more Groups site scopes, where the site
// matches upon a given site by ID or URL. // matches upon a given site by ID or URL.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
@ -519,6 +554,8 @@ const (
GroupsGroup groupsCategory = "GroupsGroup" GroupsGroup groupsCategory = "GroupsGroup"
GroupsChannel groupsCategory = "GroupsChannel" GroupsChannel groupsCategory = "GroupsChannel"
GroupsChannelMessage groupsCategory = "GroupsChannelMessage" GroupsChannelMessage groupsCategory = "GroupsChannelMessage"
GroupsConversation groupsCategory = "GroupsConversation"
GroupsConversationPost groupsCategory = "GroupsConversationPost"
GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder" GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder"
GroupsLibraryItem groupsCategory = "GroupsLibraryItem" GroupsLibraryItem groupsCategory = "GroupsLibraryItem"
GroupsList groupsCategory = "GroupsList" GroupsList groupsCategory = "GroupsList"
@ -546,10 +583,14 @@ const (
// groupsLeafProperties describes common metadata of the leaf categories // groupsLeafProperties describes common metadata of the leaf categories
var groupsLeafProperties = map[categorizer]leafProperty{ var groupsLeafProperties = map[categorizer]leafProperty{
GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf GroupsChannelMessage: {
pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage}, pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage},
pathType: path.ChannelMessagesCategory, pathType: path.ChannelMessagesCategory,
}, },
GroupsConversationPost: {
pathKeys: []categorizer{GroupsConversation, GroupsConversationPost},
pathType: path.ConversationPostsCategory,
},
GroupsLibraryItem: { GroupsLibraryItem: {
pathKeys: []categorizer{GroupsLibraryFolder, GroupsLibraryItem}, pathKeys: []categorizer{GroupsLibraryFolder, GroupsLibraryItem},
pathType: path.LibrariesCategory, pathType: path.LibrariesCategory,
@ -571,12 +612,12 @@ func (c groupsCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser // Ex: ServiceUser.leafCat() => ServiceUser
func (c groupsCategory) leafCat() categorizer { func (c groupsCategory) leafCat() categorizer {
switch c { switch c {
// TODO: if channels ever contain more than one type of item,
// we'll need to fix this up.
case GroupsChannel, GroupsChannelMessage, case GroupsChannel, GroupsChannelMessage,
GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator, GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator,
GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore: GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore:
return GroupsChannelMessage return GroupsChannelMessage
case GroupsConversation, GroupsConversationPost:
return GroupsConversationPost
case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSite, GroupsInfoSiteLibraryDrive, case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSite, GroupsInfoSiteLibraryDrive,
GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore, GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore,
GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore: GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore:
@ -631,6 +672,9 @@ func (c groupsCategory) pathValues(
case GroupsChannel, GroupsChannelMessage: case GroupsChannel, GroupsChannelMessage:
folderCat, itemCat = GroupsChannel, GroupsChannelMessage folderCat, itemCat = GroupsChannel, GroupsChannelMessage
rFld = ent.Groups.ParentPath rFld = ent.Groups.ParentPath
case GroupsConversation, GroupsConversationPost:
folderCat, itemCat = GroupsConversation, GroupsConversationPost
rFld = ent.Groups.ParentPath
case GroupsLibraryFolder, GroupsLibraryItem: case GroupsLibraryFolder, GroupsLibraryItem:
folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem
rFld = ent.Groups.ParentPath rFld = ent.Groups.ParentPath
@ -729,7 +773,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS
os := []option{} os := []option{}
switch cat { switch cat {
case GroupsChannel, GroupsLibraryFolder: case GroupsChannel, GroupsConversation, GroupsLibraryFolder:
os = append(os, pathComparator()) os = append(os, pathComparator())
} }
@ -742,12 +786,16 @@ func (s GroupsScope) setDefaults() {
case GroupsGroup: case GroupsGroup:
s[GroupsChannel.String()] = passAny s[GroupsChannel.String()] = passAny
s[GroupsChannelMessage.String()] = passAny s[GroupsChannelMessage.String()] = passAny
s[GroupsConversation.String()] = passAny
s[GroupsConversationPost.String()] = passAny
s[GroupsLibraryFolder.String()] = passAny s[GroupsLibraryFolder.String()] = passAny
s[GroupsLibraryItem.String()] = passAny s[GroupsLibraryItem.String()] = passAny
case GroupsChannel: case GroupsChannel:
s[GroupsChannelMessage.String()] = passAny s[GroupsChannelMessage.String()] = passAny
case GroupsLibraryFolder: case GroupsLibraryFolder:
s[GroupsLibraryItem.String()] = passAny s[GroupsLibraryItem.String()] = passAny
case GroupsConversation:
s[GroupsConversationPost.String()] = passAny
} }
} }
@ -768,6 +816,7 @@ func (s groups) Reduce(
s.Selector, s.Selector,
map[path.CategoryType]groupsCategory{ map[path.CategoryType]groupsCategory{
path.ChannelMessagesCategory: GroupsChannelMessage, path.ChannelMessagesCategory: GroupsChannelMessage,
path.ConversationPostsCategory: GroupsConversationPost,
path.LibrariesCategory: GroupsLibraryItem, path.LibrariesCategory: GroupsLibraryItem,
}, },
errs) errs)
@ -793,6 +842,8 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool {
acceptableItemType = int(details.SharePointLibrary) acceptableItemType = int(details.SharePointLibrary)
case GroupsChannelMessage: case GroupsChannelMessage:
acceptableItemType = int(details.GroupsChannelMessage) acceptableItemType = int(details.GroupsChannelMessage)
case GroupsConversationPost:
acceptableItemType = int(details.GroupsConversationPost)
} }
switch infoCat { switch infoCat {

View File

@ -112,6 +112,9 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
chanItem = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems1), "chitem") chanItem = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems1), "chitem")
chanItem2 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems2), "chitem2") chanItem2 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems2), "chitem2")
chanItem3 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems3), "chitem3") chanItem3 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems3), "chitem3")
convItem = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems1), "convitem")
convItem2 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems2), "convitem2")
convItem3 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems3), "convitem3")
) )
deets := &details.Details{ deets := &details.Details{
@ -186,6 +189,39 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
}, },
}, },
}, },
{
RepoRef: convItem,
ItemRef: "convitem",
LocationRef: strings.Join(itemElems1, "/"),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems1, "/"),
},
},
},
{
RepoRef: convItem2,
LocationRef: strings.Join(itemElems2, "/"),
// ItemRef intentionally blank to test fallback case
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems2, "/"),
},
},
},
{
RepoRef: convItem3,
ItemRef: "convitem3",
LocationRef: strings.Join(itemElems3, "/"),
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemType: details.GroupsConversationPost,
ParentPath: strings.Join(itemElems3, "/"),
},
},
},
}, },
}, },
} }
@ -207,7 +243,10 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
sel.Include(sel.AllData()) sel.Include(sel.AllData())
return sel return sel
}, },
expect: arr(libItem, libItem2, libItem3, chanItem, chanItem2, chanItem3), expect: arr(
libItem, libItem2, libItem3,
chanItem, chanItem2, chanItem3,
convItem, convItem2, convItem3),
}, },
{ {
name: "only match library item", name: "only match library item",
@ -218,15 +257,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
}, },
expect: arr(libItem2), expect: arr(libItem2),
}, },
{
name: "only match channel item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
return sel
},
expect: arr(chanItem2),
},
{ {
name: "library id doesn't match name", name: "library id doesn't match name",
makeSelector: func() *GroupsRestore { makeSelector: func() *GroupsRestore {
@ -237,16 +267,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
expect: []string{}, expect: []string{},
cfg: Config{OnlyMatchItemNames: true}, cfg: Config{OnlyMatchItemNames: true},
}, },
{
name: "channel id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{ {
name: "library only match item name", name: "library only match item name",
makeSelector: func() *GroupsRestore { makeSelector: func() *GroupsRestore {
@ -275,6 +295,44 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
}, },
expect: arr(libItem, libItem2), expect: arr(libItem, libItem2),
}, },
{
name: "only match channel item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
return sel
},
expect: arr(chanItem2),
},
{
name: "channel id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
{
name: "only match conversation item",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ConversationPosts(Any(), []string{"convitem2"}))
return sel
},
expect: arr(convItem2),
},
{
name: "conversation id doesn't match name",
makeSelector: func() *GroupsRestore {
sel := NewGroupsRestore(Any())
sel.Include(sel.ConversationPosts(Any(), []string{"item2"}))
return sel
},
expect: []string{},
cfg: Config{OnlyMatchItemNames: true},
},
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
@ -320,6 +378,17 @@ func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() {
}, },
cfg: Config{}, cfg: Config{},
}, },
{
name: "Groups Conversation Posts",
sc: GroupsConversationPost,
pathElems: elems,
locRef: "",
expected: map[categorizer][]string{
GroupsConversation: {""},
GroupsConversationPost: {itemID, shortRef},
},
cfg: Config{},
},
} }
for _, test := range table { for _, test := range table {
@ -476,6 +545,8 @@ func (suite *GroupsSelectorSuite) TestCategory_PathType() {
{GroupsCategoryUnknown, path.UnknownCategory}, {GroupsCategoryUnknown, path.UnknownCategory},
{GroupsChannel, path.ChannelMessagesCategory}, {GroupsChannel, path.ChannelMessagesCategory},
{GroupsChannelMessage, path.ChannelMessagesCategory}, {GroupsChannelMessage, path.ChannelMessagesCategory},
{GroupsConversation, path.ConversationPostsCategory},
{GroupsConversationPost, path.ConversationPostsCategory},
{GroupsInfoChannelMessageCreator, path.ChannelMessagesCategory}, {GroupsInfoChannelMessageCreator, path.ChannelMessagesCategory},
{GroupsInfoChannelMessageCreatedAfter, path.ChannelMessagesCategory}, {GroupsInfoChannelMessageCreatedAfter, path.ChannelMessagesCategory},
{GroupsInfoChannelMessageCreatedBefore, path.ChannelMessagesCategory}, {GroupsInfoChannelMessageCreatedBefore, path.ChannelMessagesCategory},

View File

@ -2,7 +2,6 @@ package api
import ( import (
"context" "context"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -186,18 +185,18 @@ func filterOutSystemMessages(cm models.ChatMessageable) bool {
func (c Channels) GetChannelMessageIDs( func (c Channels) GetChannelMessageIDs(
ctx context.Context, ctx context.Context,
teamID, channelID, prevDeltaLink string, teamID, channelID, prevDeltaLink string,
canMakeDeltaQueries bool, cc CallConfig,
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) { ) (pagers.AddedAndRemoved, error) {
added, validModTimes, removed, du, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable]( aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
ctx, ctx,
c.NewChannelMessagePager(teamID, channelID, CallConfig{}), c.NewChannelMessagePager(teamID, channelID, CallConfig{}),
c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink), c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink),
prevDeltaLink, prevDeltaLink,
canMakeDeltaQueries, cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable], pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable],
filterOutSystemMessages) filterOutSystemMessages)
return added, validModTimes, removed, du, clues.Stack(err).OrNil() return aar, clues.Stack(err).OrNil()
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -56,30 +56,34 @@ func (suite *ChannelsPagerIntgSuite) TestEnumerateChannelMessages() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
addedIDs, _, _, du, err := ac.GetChannelMessageIDs( cc := CallConfig{
CanMakeDeltaQueries: true,
}
aar, err := ac.GetChannelMessageIDs(
ctx, ctx,
suite.its.group.id, suite.its.group.id,
suite.its.group.testContainerID, suite.its.group.testContainerID,
"", "",
true) cc)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
require.NotEmpty(t, addedIDs) require.NotEmpty(t, aar.Added)
require.NotZero(t, du.URL, "delta link") require.NotZero(t, aar.DU.URL, "delta link")
require.True(t, du.Reset, "reset due to empty prev delta link") require.True(t, aar.DU.Reset, "reset due to empty prev delta link")
addedIDs, _, deletedIDs, du, err := ac.GetChannelMessageIDs( aar, err = ac.GetChannelMessageIDs(
ctx, ctx,
suite.its.group.id, suite.its.group.id,
suite.its.group.testContainerID, suite.its.group.testContainerID,
du.URL, aar.DU.URL,
true) cc)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
require.Empty(t, addedIDs, "should have no new messages from delta") require.Empty(t, aar.Added, "should have no new messages from delta")
require.Empty(t, deletedIDs, "should have no deleted messages from delta") require.Empty(t, aar.Removed, "should have no deleted messages from delta")
require.NotZero(t, du.URL, "delta link") require.NotZero(t, aar.DU.URL, "delta link")
require.False(t, du.Reset, "prev delta link should be valid") require.False(t, aar.DU.Reset, "prev delta link should be valid")
for id := range addedIDs { for id := range aar.Added {
suite.Run(id+"-replies", func() { suite.Run(id+"-replies", func() {
testEnumerateChannelMessageReplies( testEnumerateChannelMessageReplies(
suite.T(), suite.T(),

View File

@ -161,6 +161,8 @@ func (c Client) Post(
type CallConfig struct { type CallConfig struct {
Expand []string Expand []string
Select []string Select []string
CanMakeDeltaQueries bool
UseImmutableIDs bool
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -2,7 +2,6 @@ package api
import ( import (
"context" "context"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -250,9 +249,8 @@ func (p *contactDeltaPager) ValidModTimes() bool {
func (c Contacts) GetAddedAndRemovedItemIDs( func (c Contacts) GetAddedAndRemovedItemIDs(
ctx context.Context, ctx context.Context,
userID, containerID, prevDeltaLink string, userID, containerID, prevDeltaLink string,
immutableIDs bool, cc CallConfig,
canMakeDeltaQueries bool, ) (pagers.AddedAndRemoved, error) {
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
ctx = clues.Add( ctx = clues.Add(
ctx, ctx,
"data_category", path.ContactsCategory, "data_category", path.ContactsCategory,
@ -263,12 +261,12 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
userID, userID,
containerID, containerID,
prevDeltaLink, prevDeltaLink,
immutableIDs, cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...) idAnd(lastModifiedDateTime)...)
pager := c.NewContactsPager( pager := c.NewContactsPager(
userID, userID,
containerID, containerID,
immutableIDs, cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...) idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Contactable]( return pagers.GetAddedAndRemovedItemIDs[models.Contactable](
@ -276,6 +274,6 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
pager, pager,
deltaPager, deltaPager,
prevDeltaLink, prevDeltaLink,
canMakeDeltaQueries, cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Contactable]) pagers.AddedAndRemovedByAddtlData[models.Contactable])
} }

View File

@ -140,7 +140,7 @@ func (c Conversations) NewConversationThreadsPager(
} }
} }
// GetConversations fetches all conversations in the group. // GetConversations fetches all conversation threads in the group.
func (c Conversations) GetConversationThreads( func (c Conversations) GetConversationThreads(
ctx context.Context, ctx context.Context,
groupID, conversationID string, groupID, conversationID string,
@ -218,7 +218,7 @@ func (c Conversations) NewConversationThreadPostsPager(
} }
} }
// GetConversations fetches all conversations in the group. // GetConversations fetches all conversation posts in the group.
func (c Conversations) GetConversationThreadPosts( func (c Conversations) GetConversationThreadPosts(
ctx context.Context, ctx context.Context,
groupID, conversationID, threadID string, groupID, conversationID, threadID string,
@ -230,3 +230,23 @@ func (c Conversations) GetConversationThreadPosts(
return items, graph.Stack(ctx, err).OrNil() return items, graph.Stack(ctx, err).OrNil()
} }
// GetConversations fetches all added and deleted conversation posts in the group.
func (c Conversations) GetConversationThreadPostIDs(
ctx context.Context,
groupID, conversationID, threadID string,
cc CallConfig,
) (pagers.AddedAndRemoved, error) {
canMakeDeltaQueries := false
aarh, err := pagers.GetAddedAndRemovedItemIDs[models.Postable](
ctx,
c.NewConversationThreadPostsPager(groupID, conversationID, threadID, CallConfig{}),
nil,
"",
canMakeDeltaQueries,
pagers.AddedAndRemovedAddAll[models.Postable],
pagers.FilterIncludeAll[models.Postable])
return aarh, clues.Stack(err).OrNil()
}

View File

@ -5,6 +5,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -49,6 +50,18 @@ func (suite *ConversationsPagerIntgSuite) TestEnumerateConversations_withThreads
for _, thread := range threads { for _, thread := range threads {
posts := testEnumerateConvPosts(suite, conv, thread) posts := testEnumerateConvPosts(suite, conv, thread)
aar, err := ac.GetConversationThreadPostIDs(
ctx,
suite.its.group.id,
ptr.Val(conv.GetId()),
ptr.Val(thread.GetId()),
CallConfig{})
require.NoError(t, err, clues.ToCore(err))
require.Equal(t, len(posts), len(aar.Added), "added the same number of ids and posts")
assert.True(t, aar.ValidModTimes, "mod times should be valid")
assert.Empty(t, aar.Removed, "no items should get removed")
assert.Empty(t, aar.DU.URL, "no delta update token should be provided")
for _, post := range posts { for _, post := range posts {
testGetPostByID(suite, conv, thread, post) testGetPostByID(suite, conv, thread, post)
} }

View File

@ -3,7 +3,6 @@ package api
import ( import (
"context" "context"
"fmt" "fmt"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -245,9 +244,8 @@ func (p *eventDeltaPager) ValidModTimes() bool {
func (c Events) GetAddedAndRemovedItemIDs( func (c Events) GetAddedAndRemovedItemIDs(
ctx context.Context, ctx context.Context,
userID, containerID, prevDeltaLink string, userID, containerID, prevDeltaLink string,
immutableIDs bool, cc CallConfig,
canMakeDeltaQueries bool, ) (pagers.AddedAndRemoved, error) {
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
ctx = clues.Add( ctx = clues.Add(
ctx, ctx,
"data_category", path.EventsCategory, "data_category", path.EventsCategory,
@ -258,12 +256,12 @@ func (c Events) GetAddedAndRemovedItemIDs(
userID, userID,
containerID, containerID,
prevDeltaLink, prevDeltaLink,
immutableIDs, cc.UseImmutableIDs,
idAnd()...) idAnd()...)
pager := c.NewEventsPager( pager := c.NewEventsPager(
userID, userID,
containerID, containerID,
immutableIDs, cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...) idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Eventable]( return pagers.GetAddedAndRemovedItemIDs[models.Eventable](
@ -271,6 +269,6 @@ func (c Events) GetAddedAndRemovedItemIDs(
pager, pager,
deltaPager, deltaPager,
prevDeltaLink, prevDeltaLink,
canMakeDeltaQueries, cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Eventable]) pagers.AddedAndRemovedByAddtlData[models.Eventable])
} }

View File

@ -0,0 +1,31 @@
package api
import (
"context"
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/alcionai/corso/src/pkg/fault"
)
type GetAndSerializeItemer[INFO any] interface {
GetItemer[INFO]
Serializer
}
type GetItemer[INFO any] interface {
GetItem(
ctx context.Context,
user, itemID string,
immutableIDs bool,
errs *fault.Bus,
) (serialization.Parsable, *INFO, error)
}
type Serializer interface {
Serialize(
ctx context.Context,
item serialization.Parsable,
protectedResource, itemID string,
) ([]byte, error)
}

View File

@ -3,7 +3,6 @@ package api
import ( import (
"context" "context"
"fmt" "fmt"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -247,9 +246,8 @@ func (p *mailDeltaPager) ValidModTimes() bool {
func (c Mail) GetAddedAndRemovedItemIDs( func (c Mail) GetAddedAndRemovedItemIDs(
ctx context.Context, ctx context.Context,
userID, containerID, prevDeltaLink string, userID, containerID, prevDeltaLink string,
immutableIDs bool, cc CallConfig,
canMakeDeltaQueries bool, ) (pagers.AddedAndRemoved, error) {
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
ctx = clues.Add( ctx = clues.Add(
ctx, ctx,
"data_category", path.EmailCategory, "data_category", path.EmailCategory,
@ -260,12 +258,12 @@ func (c Mail) GetAddedAndRemovedItemIDs(
userID, userID,
containerID, containerID,
prevDeltaLink, prevDeltaLink,
immutableIDs, cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...) idAnd(lastModifiedDateTime)...)
pager := c.NewMailPager( pager := c.NewMailPager(
userID, userID,
containerID, containerID,
immutableIDs, cc.UseImmutableIDs,
idAnd(lastModifiedDateTime)...) idAnd(lastModifiedDateTime)...)
return pagers.GetAddedAndRemovedItemIDs[models.Messageable]( return pagers.GetAddedAndRemovedItemIDs[models.Messageable](
@ -273,6 +271,6 @@ func (c Mail) GetAddedAndRemovedItemIDs(
pager, pager,
deltaPager, deltaPager,
prevDeltaLink, prevDeltaLink,
canMakeDeltaQueries, cc.CanMakeDeltaQueries,
pagers.AddedAndRemovedByAddtlData[models.Messageable]) pagers.AddedAndRemovedByAddtlData[models.Messageable])
} }

View File

@ -366,13 +366,28 @@ func batchDeltaEnumerateItems[T any](
return results, du, clues.Stack(err).OrNil() return results, du, clues.Stack(err).OrNil()
} }
// ---------------------------------------------------------------------------
// filter funcs
// ---------------------------------------------------------------------------
func FilterIncludeAll[T any](_ T) bool {
return true
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// shared enumeration runner funcs // shared enumeration runner funcs
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type AddedAndRemoved struct {
Added map[string]time.Time
Removed []string
DU DeltaUpdate
ValidModTimes bool
}
type addedAndRemovedHandler[T any] func( type addedAndRemovedHandler[T any] func(
items []T, items []T,
filters ...func(T) bool, // false -> remove, true -> keep filters ...func(T) bool,
) ( ) (
map[string]time.Time, map[string]time.Time,
[]string, []string,
@ -387,16 +402,23 @@ func GetAddedAndRemovedItemIDs[T any](
canMakeDeltaQueries bool, canMakeDeltaQueries bool,
aarh addedAndRemovedHandler[T], aarh addedAndRemovedHandler[T],
filters ...func(T) bool, filters ...func(T) bool,
) (map[string]time.Time, bool, []string, DeltaUpdate, error) { ) (AddedAndRemoved, error) {
if canMakeDeltaQueries { if canMakeDeltaQueries {
ts, du, err := batchDeltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink) ts, du, err := batchDeltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink)
if err != nil && !graph.IsErrInvalidDelta(err) && !graph.IsErrDeltaNotSupported(err) { if err != nil && !graph.IsErrInvalidDelta(err) && !graph.IsErrDeltaNotSupported(err) {
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err) return AddedAndRemoved{}, graph.Stack(ctx, err)
} }
if err == nil { if err == nil {
a, r, err := aarh(ts, filters...) a, r, err := aarh(ts, filters...)
return a, deltaPager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil() aar := AddedAndRemoved{
Added: a,
Removed: r,
DU: du,
ValidModTimes: deltaPager.ValidModTimes(),
}
return aar, graph.Stack(ctx, err).OrNil()
} }
} }
@ -404,31 +426,61 @@ func GetAddedAndRemovedItemIDs[T any](
ts, err := BatchEnumerateItems(ctx, pager) ts, err := BatchEnumerateItems(ctx, pager)
if err != nil { if err != nil {
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err) return AddedAndRemoved{}, graph.Stack(ctx, err)
} }
a, r, err := aarh(ts, filters...) a, r, err := aarh(ts, filters...)
aar := AddedAndRemoved{
return a, pager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil() Added: a,
Removed: r,
DU: du,
ValidModTimes: pager.ValidModTimes(),
} }
type getIDer interface { return aar, graph.Stack(ctx, err).OrNil()
GetId() *string }
type getIDAndModDateTimer interface {
graph.GetIDer
graph.GetLastModifiedDateTimer
}
// AddedAndRemovedAddAll indiscriminately adds every item to the added list, deleting nothing.
func AddedAndRemovedAddAll[T any](
items []T,
filters ...func(T) bool,
) (map[string]time.Time, []string, error) {
added := map[string]time.Time{}
for _, item := range items {
passAllFilters := true
for _, passes := range filters {
passAllFilters = passAllFilters && passes(item)
}
if !passAllFilters {
continue
}
giamdt, ok := any(item).(getIDAndModDateTimer)
if !ok {
return nil, nil, clues.New("item does not provide id and modified date time getters").
With("item_type", fmt.Sprintf("%T", item))
}
added[ptr.Val(giamdt.GetId())] = dttm.OrNow(ptr.Val(giamdt.GetLastModifiedDateTime()))
}
return added, []string{}, nil
} }
// for added and removed by additionalData[@removed] // for added and removed by additionalData[@removed]
type getIDModAndAddtler interface { type getIDModAndAddtler interface {
getIDer graph.GetIDer
getModTimer graph.GetLastModifiedDateTimer
GetAdditionalData() map[string]any graph.GetAdditionalDataer
}
// for types that are non-compliant with this interface,
// pagers will need to wrap the return value in a struct
// that provides this compliance.
type getModTimer interface {
GetLastModifiedDateTime() *time.Time
} }
func AddedAndRemovedByAddtlData[T any]( func AddedAndRemovedByAddtlData[T any](
@ -451,7 +503,7 @@ func AddedAndRemovedByAddtlData[T any](
giaa, ok := any(item).(getIDModAndAddtler) giaa, ok := any(item).(getIDModAndAddtler)
if !ok { if !ok {
return nil, nil, clues.New("item does not provide id and additional data getters"). return nil, nil, clues.New("item does not provide id, modified date time, and additional data getters").
With("item_type", fmt.Sprintf("%T", item)) With("item_type", fmt.Sprintf("%T", item))
} }
@ -461,7 +513,11 @@ func AddedAndRemovedByAddtlData[T any](
if giaa.GetAdditionalData()[graph.AddtlDataRemoved] == nil { if giaa.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
var modTime time.Time var modTime time.Time
if mt, ok := giaa.(getModTimer); ok { // not all items comply with last modified date time, and not all
// items can be wrapped in a way that produces a valid value for
// the func. That's why this isn't packed in to the expected
// interfaace composition.
if mt, ok := giaa.(graph.GetLastModifiedDateTimer); ok {
// Make sure to get a non-zero mod time if the item doesn't have one for // Make sure to get a non-zero mod time if the item doesn't have one for
// some reason. Otherwise we can hit an issue where kopia has a // some reason. Otherwise we can hit an issue where kopia has a
// different mod time for the file than the details does. This occurs // different mod time for the file than the details does. This occurs
@ -485,9 +541,9 @@ func AddedAndRemovedByAddtlData[T any](
// for added and removed by GetDeletedDateTime() // for added and removed by GetDeletedDateTime()
type getIDModAndDeletedDateTimer interface { type getIDModAndDeletedDateTimer interface {
getIDer graph.GetIDer
getModTimer graph.GetLastModifiedDateTimer
GetDeletedDateTime() *time.Time graph.GetDeletedDateTimer
} }
func AddedAndRemovedByDeletedDateTime[T any]( func AddedAndRemovedByDeletedDateTime[T any](
@ -510,14 +566,18 @@ func AddedAndRemovedByDeletedDateTime[T any](
giaddt, ok := any(item).(getIDModAndDeletedDateTimer) giaddt, ok := any(item).(getIDModAndDeletedDateTimer)
if !ok { if !ok {
return nil, nil, clues.New("item does not provide id and deleted date time getters"). return nil, nil, clues.New("item does not provide id, modified, and deleted date time getters").
With("item_type", fmt.Sprintf("%T", item)) With("item_type", fmt.Sprintf("%T", item))
} }
if giaddt.GetDeletedDateTime() == nil { if giaddt.GetDeletedDateTime() == nil {
var modTime time.Time var modTime time.Time
if mt, ok := giaddt.(getModTimer); ok { // not all items comply with last modified date time, and not all
// items can be wrapped in a way that produces a valid value for
// the func. That's why this isn't packed in to the expected
// interfaace composition.
if mt, ok := giaddt.(graph.GetLastModifiedDateTimer); ok {
// Make sure to get a non-zero mod time if the item doesn't have one for // Make sure to get a non-zero mod time if the item doesn't have one for
// some reason. Otherwise we can hit an issue where kopia has a // some reason. Otherwise we can hit an issue where kopia has a
// different mod time for the file than the details does. This occurs // different mod time for the file than the details does. This occurs

View File

@ -520,7 +520,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
filters = append(filters, test.filter) filters = append(filters, test.filter)
} }
added, validModTimes, removed, deltaUpdate, err := GetAddedAndRemovedItemIDs[testItem]( aar, err := GetAddedAndRemovedItemIDs[testItem](
ctx, ctx,
test.pagerGetter(t), test.pagerGetter(t),
test.deltaPagerGetter(t), test.deltaPagerGetter(t),
@ -530,18 +530,18 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
filters...) filters...)
require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err)) require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err))
if validModTimes { if aar.ValidModTimes {
assert.Equal(t, test.expect.added, added, "added item IDs and mod times") assert.Equal(t, test.expect.added, aar.Added, "added item IDs and mod times")
} else { } else {
assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(added), "added item IDs") assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(aar.Added), "added item IDs")
for _, modtime := range added { for _, modtime := range aar.Added {
assert.True(t, modtime.After(epoch), "mod time after epoch") assert.True(t, modtime.After(epoch), "mod time after epoch")
assert.False(t, modtime.Equal(time.Time{}), "non-zero mod time") assert.False(t, modtime.Equal(time.Time{}), "non-zero mod time")
} }
} }
assert.Equal(t, test.expect.validModTimes, validModTimes, "valid mod times") assert.Equal(t, test.expect.validModTimes, aar.ValidModTimes, "valid mod times")
assert.EqualValues(t, test.expect.removed, removed, "removed item IDs") assert.EqualValues(t, test.expect.removed, aar.Removed, "removed item IDs")
assert.Equal(t, test.expect.deltaUpdate, deltaUpdate, "delta update") assert.Equal(t, test.expect.deltaUpdate, aar.DU, "delta update")
}) })
} }
} }