add getPostIDs in conversations pager (#4578)
adds the getPostIDs func to ensure conversations complies with standard data paging patterns --- #### Does this PR need a docs update or release note? - [x] ⛔ No #### Type of change - [x] 🌻 Feature #### Issue(s) * #4536 #### Test Plan - [x] ⚡ Unit test - [x] 💚 E2E
This commit is contained in:
parent
ec1afa8c84
commit
49bbdfc096
@ -42,28 +42,28 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
{
|
||||
name: "no inputs",
|
||||
opts: utils.GroupsOpts{},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: empty,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "single inputs",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: single,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "multi inputs",
|
||||
opts: utils.GroupsOpts{
|
||||
Groups: multi,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
// sharepoint
|
||||
{
|
||||
@ -120,7 +120,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
FileName: empty,
|
||||
FolderPath: empty,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "library folder suffixes and contains",
|
||||
@ -128,7 +128,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
|
||||
FileName: empty,
|
||||
FolderPath: empty,
|
||||
},
|
||||
expectIncludeLen: 2,
|
||||
expectIncludeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "Page Folder",
|
||||
@ -389,7 +389,7 @@ func (suite *GroupsUtilsSuite) TestAddGroupsCategories() {
|
||||
{
|
||||
name: "none",
|
||||
cats: []string{},
|
||||
expectScopeLen: 2,
|
||||
expectScopeLen: 3,
|
||||
},
|
||||
{
|
||||
name: "libraries",
|
||||
|
||||
@ -18,6 +18,7 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||
)
|
||||
|
||||
@ -157,14 +158,18 @@ func populateCollections(
|
||||
|
||||
ictx = clues.Add(ictx, "previous_path", prevPath)
|
||||
|
||||
added, _, removed, newDelta, err := bh.itemEnumerator().
|
||||
cc := api.CallConfig{
|
||||
CanMakeDeltaQueries: !ctrlOpts.ToggleFeatures.DisableDelta,
|
||||
UseImmutableIDs: ctrlOpts.ToggleFeatures.ExchangeImmutableIDs,
|
||||
}
|
||||
|
||||
addAndRem, err := bh.itemEnumerator().
|
||||
GetAddedAndRemovedItemIDs(
|
||||
ictx,
|
||||
qp.ProtectedResource.ID(),
|
||||
cID,
|
||||
prevDelta,
|
||||
ctrlOpts.ToggleFeatures.ExchangeImmutableIDs,
|
||||
!ctrlOpts.ToggleFeatures.DisableDelta)
|
||||
cc)
|
||||
if err != nil {
|
||||
if !graph.IsErrDeletedInFlight(err) {
|
||||
el.AddRecoverable(ctx, clues.Stack(err).Label(fault.LabelForceNoBackupCreation))
|
||||
@ -176,12 +181,12 @@ func populateCollections(
|
||||
// to reset. This prevents any old items from being retained in
|
||||
// storage. If the container (or its children) are sill missing
|
||||
// on the next backup, they'll get tombstoned.
|
||||
newDelta = pagers.DeltaUpdate{Reset: true}
|
||||
addAndRem.DU = pagers.DeltaUpdate{Reset: true}
|
||||
}
|
||||
|
||||
if len(newDelta.URL) > 0 {
|
||||
deltaURLs[cID] = newDelta.URL
|
||||
} else if !newDelta.Reset {
|
||||
if len(addAndRem.DU.URL) > 0 {
|
||||
deltaURLs[cID] = addAndRem.DU.URL
|
||||
} else if !addAndRem.DU.Reset {
|
||||
logger.Ctx(ictx).Info("missing delta url")
|
||||
}
|
||||
|
||||
@ -191,11 +196,11 @@ func populateCollections(
|
||||
prevPath,
|
||||
locPath,
|
||||
ctrlOpts,
|
||||
newDelta.Reset),
|
||||
addAndRem.DU.Reset),
|
||||
qp.ProtectedResource.ID(),
|
||||
bh.itemHandler(),
|
||||
added,
|
||||
removed,
|
||||
addAndRem.Added,
|
||||
addAndRem.Removed,
|
||||
// TODO: produce a feature flag that allows selective
|
||||
// enabling of valid modTimes. This currently produces
|
||||
// rare-case failures with incorrect details merging.
|
||||
|
||||
@ -75,18 +75,11 @@ type (
|
||||
func (mg mockGetter) GetAddedAndRemovedItemIDs(
|
||||
ctx context.Context,
|
||||
userID, cID, prevDelta string,
|
||||
_ bool,
|
||||
_ bool,
|
||||
) (
|
||||
map[string]time.Time,
|
||||
bool,
|
||||
[]string,
|
||||
pagers.DeltaUpdate,
|
||||
error,
|
||||
) {
|
||||
_ api.CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
results, ok := mg.results[cID]
|
||||
if !ok {
|
||||
return nil, false, nil, pagers.DeltaUpdate{}, clues.New("mock not found for " + cID)
|
||||
return pagers.AddedAndRemoved{}, clues.New("mock not found for " + cID)
|
||||
}
|
||||
|
||||
delta := results.newDelta
|
||||
@ -99,7 +92,14 @@ func (mg mockGetter) GetAddedAndRemovedItemIDs(
|
||||
resAdded[add] = time.Time{}
|
||||
}
|
||||
|
||||
return resAdded, false, results.removed, delta, results.err
|
||||
aar := pagers.AddedAndRemoved{
|
||||
Added: resAdded,
|
||||
Removed: results.removed,
|
||||
ValidModTimes: false,
|
||||
DU: delta,
|
||||
}
|
||||
|
||||
return aar, results.err
|
||||
}
|
||||
|
||||
var _ graph.ContainerResolver = &mockResolver{}
|
||||
|
||||
@ -2,7 +2,6 @@ package exchange
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/microsoft/kiota-abstractions-go/serialization"
|
||||
|
||||
@ -30,9 +29,8 @@ type addedAndRemovedItemGetter interface {
|
||||
GetAddedAndRemovedItemIDs(
|
||||
ctx context.Context,
|
||||
user, containerID, oldDeltaToken string,
|
||||
immutableIDs bool,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error)
|
||||
cc api.CallConfig,
|
||||
) (pagers.AddedAndRemoved, error)
|
||||
}
|
||||
|
||||
type itemGetterSerializer interface {
|
||||
|
||||
@ -20,6 +20,7 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
)
|
||||
|
||||
// TODO: incremental support
|
||||
@ -152,20 +153,22 @@ func populateCollections(
|
||||
|
||||
// if the channel has no email property, it is unable to process delta tokens
|
||||
// and will return an error if a delta token is queried.
|
||||
canMakeDeltaQueries := len(ptr.Val(c.GetEmail())) > 0
|
||||
cc := api.CallConfig{
|
||||
CanMakeDeltaQueries: len(ptr.Val(c.GetEmail())) > 0,
|
||||
}
|
||||
|
||||
add, _, rem, du, err := bh.getContainerItemIDs(ctx, cID, prevDelta, canMakeDeltaQueries)
|
||||
addAndRem, err := bh.getContainerItemIDs(ctx, cID, prevDelta, cc)
|
||||
if err != nil {
|
||||
el.AddRecoverable(ctx, clues.Stack(err))
|
||||
continue
|
||||
}
|
||||
|
||||
added := str.SliceToMap(maps.Keys(add))
|
||||
removed := str.SliceToMap(rem)
|
||||
added := str.SliceToMap(maps.Keys(addAndRem.Added))
|
||||
removed := str.SliceToMap(addAndRem.Removed)
|
||||
|
||||
if len(du.URL) > 0 {
|
||||
deltaURLs[cID] = du.URL
|
||||
} else if !du.Reset {
|
||||
if len(addAndRem.DU.URL) > 0 {
|
||||
deltaURLs[cID] = addAndRem.DU.URL
|
||||
} else if !addAndRem.DU.Reset {
|
||||
logger.Ctx(ictx).Info("missing delta url")
|
||||
}
|
||||
|
||||
@ -188,7 +191,7 @@ func populateCollections(
|
||||
prevPath,
|
||||
path.Builder{}.Append(cName),
|
||||
ctrlOpts,
|
||||
du.Reset),
|
||||
addAndRem.DU.Reset),
|
||||
bh,
|
||||
qp.ProtectedResource.ID(),
|
||||
added,
|
||||
|
||||
@ -58,15 +58,22 @@ func (bh mockBackupHandler) getContainers(context.Context) ([]models.Channelable
|
||||
func (bh mockBackupHandler) getContainerItemIDs(
|
||||
_ context.Context,
|
||||
_, _ string,
|
||||
_ bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
_ api.CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
idRes := make(map[string]time.Time, len(bh.messageIDs))
|
||||
|
||||
for _, id := range bh.messageIDs {
|
||||
idRes[id] = time.Time{}
|
||||
}
|
||||
|
||||
return idRes, true, bh.deletedMsgIDs, pagers.DeltaUpdate{}, bh.messagesErr
|
||||
aar := pagers.AddedAndRemoved{
|
||||
Added: idRes,
|
||||
Removed: bh.deletedMsgIDs,
|
||||
ValidModTimes: true,
|
||||
DU: pagers.DeltaUpdate{},
|
||||
}
|
||||
|
||||
return aar, bh.messagesErr
|
||||
}
|
||||
|
||||
func (bh mockBackupHandler) includeContainer(
|
||||
|
||||
@ -2,7 +2,6 @@ package groups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
|
||||
@ -41,9 +40,9 @@ func (bh channelsBackupHandler) getContainers(
|
||||
func (bh channelsBackupHandler) getContainerItemIDs(
|
||||
ctx context.Context,
|
||||
channelID, prevDelta string,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, canMakeDeltaQueries)
|
||||
cc api.CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
return bh.ac.GetChannelMessageIDs(ctx, bh.protectedResource, channelID, prevDelta, cc)
|
||||
}
|
||||
|
||||
func (bh channelsBackupHandler) includeContainer(
|
||||
|
||||
@ -2,7 +2,6 @@ package groups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
|
||||
@ -10,6 +9,7 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/backup/details"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||
)
|
||||
|
||||
@ -25,8 +25,8 @@ type backupHandler interface {
|
||||
getContainerItemIDs(
|
||||
ctx context.Context,
|
||||
containerID, prevDelta string,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error)
|
||||
cc api.CallConfig,
|
||||
) (pagers.AddedAndRemoved, error)
|
||||
|
||||
// includeContainer evaluates whether the container is included
|
||||
// in the provided scope.
|
||||
|
||||
@ -11,29 +11,10 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
)
|
||||
|
||||
// Idable represents objects that implement msgraph-sdk-go/models.entityable
|
||||
// and have the concept of an ID.
|
||||
type Idable interface {
|
||||
GetId() *string
|
||||
}
|
||||
|
||||
// Descendable represents objects that implement msgraph-sdk-go/models.entityable
|
||||
// and have the concept of a "parent folder".
|
||||
type Descendable interface {
|
||||
Idable
|
||||
GetParentFolderId() *string
|
||||
}
|
||||
|
||||
// Displayable represents objects that implement msgraph-sdk-go/models.entityable
|
||||
// and have the concept of a display name.
|
||||
type Displayable interface {
|
||||
Idable
|
||||
GetDisplayName() *string
|
||||
}
|
||||
|
||||
type Container interface {
|
||||
Descendable
|
||||
Displayable
|
||||
GetIDer
|
||||
GetParentFolderIDer
|
||||
GetDisplayNamer
|
||||
}
|
||||
|
||||
// CachedContainer is used for local unit tests but also makes it so that this
|
||||
@ -41,10 +22,12 @@ type Container interface {
|
||||
// reuse logic in IDToPath.
|
||||
type CachedContainer interface {
|
||||
Container
|
||||
|
||||
// Location contains either the display names for the dirs (if this is a calendar)
|
||||
// or nil
|
||||
Location() *path.Builder
|
||||
SetLocation(*path.Builder)
|
||||
|
||||
// Path contains either the ids for the dirs (if this is a calendar)
|
||||
// or the display names for the dirs
|
||||
Path() *path.Builder
|
||||
|
||||
29
src/internal/m365/graph/interfaces.go
Normal file
29
src/internal/m365/graph/interfaces.go
Normal file
@ -0,0 +1,29 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type GetIDer interface {
|
||||
GetId() *string
|
||||
}
|
||||
|
||||
type GetLastModifiedDateTimer interface {
|
||||
GetLastModifiedDateTime() *time.Time
|
||||
}
|
||||
|
||||
type GetAdditionalDataer interface {
|
||||
GetAdditionalData() map[string]any
|
||||
}
|
||||
|
||||
type GetDeletedDateTimer interface {
|
||||
GetDeletedDateTime() *time.Time
|
||||
}
|
||||
|
||||
type GetDisplayNamer interface {
|
||||
GetDisplayName() *string
|
||||
}
|
||||
|
||||
type GetParentFolderIDer interface {
|
||||
GetParentFolderId() *string
|
||||
}
|
||||
@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -35,6 +34,7 @@ import (
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api"
|
||||
"github.com/alcionai/corso/src/pkg/services/m365/api/pagers"
|
||||
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
|
||||
)
|
||||
|
||||
@ -495,36 +495,37 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
|
||||
|
||||
var (
|
||||
err error
|
||||
items map[string]time.Time
|
||||
aar pagers.AddedAndRemoved
|
||||
cc = api.CallConfig{
|
||||
UseImmutableIDs: toggles.ExchangeImmutableIDs,
|
||||
CanMakeDeltaQueries: true,
|
||||
}
|
||||
)
|
||||
|
||||
switch category {
|
||||
case path.EmailCategory:
|
||||
items, _, _, _, err = ac.Mail().GetAddedAndRemovedItemIDs(
|
||||
aar, err = ac.Mail().GetAddedAndRemovedItemIDs(
|
||||
ctx,
|
||||
uidn.ID(),
|
||||
containerID,
|
||||
"",
|
||||
toggles.ExchangeImmutableIDs,
|
||||
true)
|
||||
cc)
|
||||
|
||||
case path.EventsCategory:
|
||||
items, _, _, _, err = ac.Events().GetAddedAndRemovedItemIDs(
|
||||
aar, err = ac.Events().GetAddedAndRemovedItemIDs(
|
||||
ctx,
|
||||
uidn.ID(),
|
||||
containerID,
|
||||
"",
|
||||
toggles.ExchangeImmutableIDs,
|
||||
true)
|
||||
cc)
|
||||
|
||||
case path.ContactsCategory:
|
||||
items, _, _, _, err = ac.Contacts().GetAddedAndRemovedItemIDs(
|
||||
aar, err = ac.Contacts().GetAddedAndRemovedItemIDs(
|
||||
ctx,
|
||||
uidn.ID(),
|
||||
containerID,
|
||||
"",
|
||||
toggles.ExchangeImmutableIDs,
|
||||
true)
|
||||
cc)
|
||||
}
|
||||
|
||||
require.NoError(
|
||||
@ -534,6 +535,8 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
|
||||
category,
|
||||
locRef.String())
|
||||
|
||||
items := aar.Added
|
||||
|
||||
dest := dataset[category].dests[destName]
|
||||
dest.locRef = locRef.String()
|
||||
dest.containerID = containerID
|
||||
|
||||
@ -27,6 +27,7 @@ const (
|
||||
PagesCategory CategoryType = 7 // pages
|
||||
DetailsCategory CategoryType = 8 // details
|
||||
ChannelMessagesCategory CategoryType = 9 // channelMessages
|
||||
ConversationPostsCategory CategoryType = 10 // conversationPosts
|
||||
)
|
||||
|
||||
var strToCat = map[string]CategoryType{
|
||||
@ -39,6 +40,7 @@ var strToCat = map[string]CategoryType{
|
||||
strings.ToLower(PagesCategory.String()): PagesCategory,
|
||||
strings.ToLower(DetailsCategory.String()): DetailsCategory,
|
||||
strings.ToLower(ChannelMessagesCategory.String()): ChannelMessagesCategory,
|
||||
strings.ToLower(ConversationPostsCategory.String()): ConversationPostsCategory,
|
||||
}
|
||||
|
||||
func ToCategoryType(s string) CategoryType {
|
||||
@ -60,6 +62,7 @@ var catToHuman = map[CategoryType]string{
|
||||
PagesCategory: "Pages",
|
||||
DetailsCategory: "Details",
|
||||
ChannelMessagesCategory: "Messages",
|
||||
ConversationPostsCategory: "Posts",
|
||||
}
|
||||
|
||||
// HumanString produces a more human-readable string version of the category.
|
||||
@ -94,6 +97,7 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
|
||||
},
|
||||
GroupsService: {
|
||||
ChannelMessagesCategory: {},
|
||||
ConversationPostsCategory: {},
|
||||
LibrariesCategory: {},
|
||||
},
|
||||
}
|
||||
|
||||
@ -34,6 +34,7 @@ func (suite *CategoryTypeUnitSuite) TestToCategoryType() {
|
||||
{input: "pages", expect: 7},
|
||||
{input: "details", expect: 8},
|
||||
{input: "channelmessages", expect: 9},
|
||||
{input: "conversationposts", expect: 10},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.input, func() {
|
||||
@ -60,6 +61,7 @@ func (suite *CategoryTypeUnitSuite) TestHumanString() {
|
||||
{input: 7, expect: "Pages"},
|
||||
{input: 8, expect: "Details"},
|
||||
{input: 9, expect: "Messages"},
|
||||
{input: 10, expect: "Posts"},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.input.String(), func() {
|
||||
|
||||
@ -18,11 +18,12 @@ func _() {
|
||||
_ = x[PagesCategory-7]
|
||||
_ = x[DetailsCategory-8]
|
||||
_ = x[ChannelMessagesCategory-9]
|
||||
_ = x[ConversationPostsCategory-10]
|
||||
}
|
||||
|
||||
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessages"
|
||||
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannelMessagesconversationPosts"
|
||||
|
||||
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80}
|
||||
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 80, 97}
|
||||
|
||||
func (i CategoryType) String() string {
|
||||
if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) {
|
||||
|
||||
@ -217,15 +217,14 @@ func (s *groups) AllData() []GroupsScope {
|
||||
scopes = append(
|
||||
scopes,
|
||||
makeScope[GroupsScope](GroupsLibraryFolder, Any()),
|
||||
makeScope[GroupsScope](GroupsChannel, Any()))
|
||||
makeScope[GroupsScope](GroupsChannel, Any()),
|
||||
makeScope[GroupsScope](GroupsConversation, Any()))
|
||||
|
||||
return scopes
|
||||
}
|
||||
|
||||
// Channels produces one or more SharePoint channel scopes, where the channel
|
||||
// matches upon a given channel by ID or Name. In order to ensure channel selection
|
||||
// this should always be embedded within the Filter() set; include(channel()) will
|
||||
// select all items in the channel without further filtering.
|
||||
// matches upon a given channel by ID or Name.
|
||||
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
||||
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
|
||||
// If any slice is empty, it defaults to [selectors.None]
|
||||
@ -260,6 +259,42 @@ func (s *groups) ChannelMessages(channels, messages []string, opts ...option) []
|
||||
return scopes
|
||||
}
|
||||
|
||||
// Conversations produces one or more SharePoint conversation scopes, where the
|
||||
// conversation matches with a given conversation by ID or Topic.
|
||||
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
||||
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
|
||||
// If any slice is empty, it defaults to [selectors.None]
|
||||
func (s *groups) Conversation(conversations []string, opts ...option) []GroupsScope {
|
||||
var (
|
||||
scopes = []GroupsScope{}
|
||||
os = append([]option{pathComparator()}, opts...)
|
||||
)
|
||||
|
||||
scopes = append(
|
||||
scopes,
|
||||
makeScope[GroupsScope](GroupsConversation, conversations, os...))
|
||||
|
||||
return scopes
|
||||
}
|
||||
|
||||
// ConversationPosts produces one or more Groups conversation post scopes.
|
||||
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
||||
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
|
||||
// If any slice is empty, it defaults to [selectors.None]
|
||||
func (s *groups) ConversationPosts(conversations, posts []string, opts ...option) []GroupsScope {
|
||||
var (
|
||||
scopes = []GroupsScope{}
|
||||
os = append([]option{pathComparator()}, opts...)
|
||||
)
|
||||
|
||||
scopes = append(
|
||||
scopes,
|
||||
makeScope[GroupsScope](GroupsConversationPost, posts, os...).
|
||||
set(GroupsConversation, conversations, opts...))
|
||||
|
||||
return scopes
|
||||
}
|
||||
|
||||
// Sites produces one or more Groups site scopes, where the site
|
||||
// matches upon a given site by ID or URL.
|
||||
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
|
||||
@ -519,6 +554,8 @@ const (
|
||||
GroupsGroup groupsCategory = "GroupsGroup"
|
||||
GroupsChannel groupsCategory = "GroupsChannel"
|
||||
GroupsChannelMessage groupsCategory = "GroupsChannelMessage"
|
||||
GroupsConversation groupsCategory = "GroupsConversation"
|
||||
GroupsConversationPost groupsCategory = "GroupsConversationPost"
|
||||
GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder"
|
||||
GroupsLibraryItem groupsCategory = "GroupsLibraryItem"
|
||||
GroupsList groupsCategory = "GroupsList"
|
||||
@ -546,10 +583,14 @@ const (
|
||||
|
||||
// groupsLeafProperties describes common metadata of the leaf categories
|
||||
var groupsLeafProperties = map[categorizer]leafProperty{
|
||||
GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf
|
||||
GroupsChannelMessage: {
|
||||
pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage},
|
||||
pathType: path.ChannelMessagesCategory,
|
||||
},
|
||||
GroupsConversationPost: {
|
||||
pathKeys: []categorizer{GroupsConversation, GroupsConversationPost},
|
||||
pathType: path.ConversationPostsCategory,
|
||||
},
|
||||
GroupsLibraryItem: {
|
||||
pathKeys: []categorizer{GroupsLibraryFolder, GroupsLibraryItem},
|
||||
pathType: path.LibrariesCategory,
|
||||
@ -571,12 +612,12 @@ func (c groupsCategory) String() string {
|
||||
// Ex: ServiceUser.leafCat() => ServiceUser
|
||||
func (c groupsCategory) leafCat() categorizer {
|
||||
switch c {
|
||||
// TODO: if channels ever contain more than one type of item,
|
||||
// we'll need to fix this up.
|
||||
case GroupsChannel, GroupsChannelMessage,
|
||||
GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator,
|
||||
GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore:
|
||||
return GroupsChannelMessage
|
||||
case GroupsConversation, GroupsConversationPost:
|
||||
return GroupsConversationPost
|
||||
case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSite, GroupsInfoSiteLibraryDrive,
|
||||
GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore,
|
||||
GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore:
|
||||
@ -631,6 +672,9 @@ func (c groupsCategory) pathValues(
|
||||
case GroupsChannel, GroupsChannelMessage:
|
||||
folderCat, itemCat = GroupsChannel, GroupsChannelMessage
|
||||
rFld = ent.Groups.ParentPath
|
||||
case GroupsConversation, GroupsConversationPost:
|
||||
folderCat, itemCat = GroupsConversation, GroupsConversationPost
|
||||
rFld = ent.Groups.ParentPath
|
||||
case GroupsLibraryFolder, GroupsLibraryItem:
|
||||
folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem
|
||||
rFld = ent.Groups.ParentPath
|
||||
@ -729,7 +773,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS
|
||||
os := []option{}
|
||||
|
||||
switch cat {
|
||||
case GroupsChannel, GroupsLibraryFolder:
|
||||
case GroupsChannel, GroupsConversation, GroupsLibraryFolder:
|
||||
os = append(os, pathComparator())
|
||||
}
|
||||
|
||||
@ -742,12 +786,16 @@ func (s GroupsScope) setDefaults() {
|
||||
case GroupsGroup:
|
||||
s[GroupsChannel.String()] = passAny
|
||||
s[GroupsChannelMessage.String()] = passAny
|
||||
s[GroupsConversation.String()] = passAny
|
||||
s[GroupsConversationPost.String()] = passAny
|
||||
s[GroupsLibraryFolder.String()] = passAny
|
||||
s[GroupsLibraryItem.String()] = passAny
|
||||
case GroupsChannel:
|
||||
s[GroupsChannelMessage.String()] = passAny
|
||||
case GroupsLibraryFolder:
|
||||
s[GroupsLibraryItem.String()] = passAny
|
||||
case GroupsConversation:
|
||||
s[GroupsConversationPost.String()] = passAny
|
||||
}
|
||||
}
|
||||
|
||||
@ -768,6 +816,7 @@ func (s groups) Reduce(
|
||||
s.Selector,
|
||||
map[path.CategoryType]groupsCategory{
|
||||
path.ChannelMessagesCategory: GroupsChannelMessage,
|
||||
path.ConversationPostsCategory: GroupsConversationPost,
|
||||
path.LibrariesCategory: GroupsLibraryItem,
|
||||
},
|
||||
errs)
|
||||
@ -793,6 +842,8 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool {
|
||||
acceptableItemType = int(details.SharePointLibrary)
|
||||
case GroupsChannelMessage:
|
||||
acceptableItemType = int(details.GroupsChannelMessage)
|
||||
case GroupsConversationPost:
|
||||
acceptableItemType = int(details.GroupsConversationPost)
|
||||
}
|
||||
|
||||
switch infoCat {
|
||||
|
||||
@ -112,6 +112,9 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
chanItem = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems1), "chitem")
|
||||
chanItem2 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems2), "chitem2")
|
||||
chanItem3 = toRR(path.ChannelMessagesCategory, "gid", slices.Clone(itemElems3), "chitem3")
|
||||
convItem = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems1), "convitem")
|
||||
convItem2 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems2), "convitem2")
|
||||
convItem3 = toRR(path.ConversationPostsCategory, "gid", slices.Clone(itemElems3), "convitem3")
|
||||
)
|
||||
|
||||
deets := &details.Details{
|
||||
@ -186,6 +189,39 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
RepoRef: convItem,
|
||||
ItemRef: "convitem",
|
||||
LocationRef: strings.Join(itemElems1, "/"),
|
||||
ItemInfo: details.ItemInfo{
|
||||
Groups: &details.GroupsInfo{
|
||||
ItemType: details.GroupsConversationPost,
|
||||
ParentPath: strings.Join(itemElems1, "/"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
RepoRef: convItem2,
|
||||
LocationRef: strings.Join(itemElems2, "/"),
|
||||
// ItemRef intentionally blank to test fallback case
|
||||
ItemInfo: details.ItemInfo{
|
||||
Groups: &details.GroupsInfo{
|
||||
ItemType: details.GroupsConversationPost,
|
||||
ParentPath: strings.Join(itemElems2, "/"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
RepoRef: convItem3,
|
||||
ItemRef: "convitem3",
|
||||
LocationRef: strings.Join(itemElems3, "/"),
|
||||
ItemInfo: details.ItemInfo{
|
||||
Groups: &details.GroupsInfo{
|
||||
ItemType: details.GroupsConversationPost,
|
||||
ParentPath: strings.Join(itemElems3, "/"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -207,7 +243,10 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
sel.Include(sel.AllData())
|
||||
return sel
|
||||
},
|
||||
expect: arr(libItem, libItem2, libItem3, chanItem, chanItem2, chanItem3),
|
||||
expect: arr(
|
||||
libItem, libItem2, libItem3,
|
||||
chanItem, chanItem2, chanItem3,
|
||||
convItem, convItem2, convItem3),
|
||||
},
|
||||
{
|
||||
name: "only match library item",
|
||||
@ -218,15 +257,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
},
|
||||
expect: arr(libItem2),
|
||||
},
|
||||
{
|
||||
name: "only match channel item",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
|
||||
return sel
|
||||
},
|
||||
expect: arr(chanItem2),
|
||||
},
|
||||
{
|
||||
name: "library id doesn't match name",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
@ -237,16 +267,6 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
expect: []string{},
|
||||
cfg: Config{OnlyMatchItemNames: true},
|
||||
},
|
||||
{
|
||||
name: "channel id doesn't match name",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
|
||||
return sel
|
||||
},
|
||||
expect: []string{},
|
||||
cfg: Config{OnlyMatchItemNames: true},
|
||||
},
|
||||
{
|
||||
name: "library only match item name",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
@ -275,6 +295,44 @@ func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
|
||||
},
|
||||
expect: arr(libItem, libItem2),
|
||||
},
|
||||
{
|
||||
name: "only match channel item",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ChannelMessages(Any(), []string{"chitem2"}))
|
||||
return sel
|
||||
},
|
||||
expect: arr(chanItem2),
|
||||
},
|
||||
{
|
||||
name: "channel id doesn't match name",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ChannelMessages(Any(), []string{"item2"}))
|
||||
return sel
|
||||
},
|
||||
expect: []string{},
|
||||
cfg: Config{OnlyMatchItemNames: true},
|
||||
},
|
||||
{
|
||||
name: "only match conversation item",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ConversationPosts(Any(), []string{"convitem2"}))
|
||||
return sel
|
||||
},
|
||||
expect: arr(convItem2),
|
||||
},
|
||||
{
|
||||
name: "conversation id doesn't match name",
|
||||
makeSelector: func() *GroupsRestore {
|
||||
sel := NewGroupsRestore(Any())
|
||||
sel.Include(sel.ConversationPosts(Any(), []string{"item2"}))
|
||||
return sel
|
||||
},
|
||||
expect: []string{},
|
||||
cfg: Config{OnlyMatchItemNames: true},
|
||||
},
|
||||
}
|
||||
for _, test := range table {
|
||||
suite.Run(test.name, func() {
|
||||
@ -320,6 +378,17 @@ func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() {
|
||||
},
|
||||
cfg: Config{},
|
||||
},
|
||||
{
|
||||
name: "Groups Conversation Posts",
|
||||
sc: GroupsConversationPost,
|
||||
pathElems: elems,
|
||||
locRef: "",
|
||||
expected: map[categorizer][]string{
|
||||
GroupsConversation: {""},
|
||||
GroupsConversationPost: {itemID, shortRef},
|
||||
},
|
||||
cfg: Config{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range table {
|
||||
@ -476,6 +545,8 @@ func (suite *GroupsSelectorSuite) TestCategory_PathType() {
|
||||
{GroupsCategoryUnknown, path.UnknownCategory},
|
||||
{GroupsChannel, path.ChannelMessagesCategory},
|
||||
{GroupsChannelMessage, path.ChannelMessagesCategory},
|
||||
{GroupsConversation, path.ConversationPostsCategory},
|
||||
{GroupsConversationPost, path.ConversationPostsCategory},
|
||||
{GroupsInfoChannelMessageCreator, path.ChannelMessagesCategory},
|
||||
{GroupsInfoChannelMessageCreatedAfter, path.ChannelMessagesCategory},
|
||||
{GroupsInfoChannelMessageCreatedBefore, path.ChannelMessagesCategory},
|
||||
|
||||
@ -2,7 +2,6 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -186,18 +185,18 @@ func filterOutSystemMessages(cm models.ChatMessageable) bool {
|
||||
func (c Channels) GetChannelMessageIDs(
|
||||
ctx context.Context,
|
||||
teamID, channelID, prevDeltaLink string,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
added, validModTimes, removed, du, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
|
||||
cc CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
aar, err := pagers.GetAddedAndRemovedItemIDs[models.ChatMessageable](
|
||||
ctx,
|
||||
c.NewChannelMessagePager(teamID, channelID, CallConfig{}),
|
||||
c.NewChannelMessageDeltaPager(teamID, channelID, prevDeltaLink),
|
||||
prevDeltaLink,
|
||||
canMakeDeltaQueries,
|
||||
cc.CanMakeDeltaQueries,
|
||||
pagers.AddedAndRemovedByDeletedDateTime[models.ChatMessageable],
|
||||
filterOutSystemMessages)
|
||||
|
||||
return added, validModTimes, removed, du, clues.Stack(err).OrNil()
|
||||
return aar, clues.Stack(err).OrNil()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@ -56,30 +56,34 @@ func (suite *ChannelsPagerIntgSuite) TestEnumerateChannelMessages() {
|
||||
ctx, flush := tester.NewContext(t)
|
||||
defer flush()
|
||||
|
||||
addedIDs, _, _, du, err := ac.GetChannelMessageIDs(
|
||||
cc := CallConfig{
|
||||
CanMakeDeltaQueries: true,
|
||||
}
|
||||
|
||||
aar, err := ac.GetChannelMessageIDs(
|
||||
ctx,
|
||||
suite.its.group.id,
|
||||
suite.its.group.testContainerID,
|
||||
"",
|
||||
true)
|
||||
cc)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
require.NotEmpty(t, addedIDs)
|
||||
require.NotZero(t, du.URL, "delta link")
|
||||
require.True(t, du.Reset, "reset due to empty prev delta link")
|
||||
require.NotEmpty(t, aar.Added)
|
||||
require.NotZero(t, aar.DU.URL, "delta link")
|
||||
require.True(t, aar.DU.Reset, "reset due to empty prev delta link")
|
||||
|
||||
addedIDs, _, deletedIDs, du, err := ac.GetChannelMessageIDs(
|
||||
aar, err = ac.GetChannelMessageIDs(
|
||||
ctx,
|
||||
suite.its.group.id,
|
||||
suite.its.group.testContainerID,
|
||||
du.URL,
|
||||
true)
|
||||
aar.DU.URL,
|
||||
cc)
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
require.Empty(t, addedIDs, "should have no new messages from delta")
|
||||
require.Empty(t, deletedIDs, "should have no deleted messages from delta")
|
||||
require.NotZero(t, du.URL, "delta link")
|
||||
require.False(t, du.Reset, "prev delta link should be valid")
|
||||
require.Empty(t, aar.Added, "should have no new messages from delta")
|
||||
require.Empty(t, aar.Removed, "should have no deleted messages from delta")
|
||||
require.NotZero(t, aar.DU.URL, "delta link")
|
||||
require.False(t, aar.DU.Reset, "prev delta link should be valid")
|
||||
|
||||
for id := range addedIDs {
|
||||
for id := range aar.Added {
|
||||
suite.Run(id+"-replies", func() {
|
||||
testEnumerateChannelMessageReplies(
|
||||
suite.T(),
|
||||
|
||||
@ -161,6 +161,8 @@ func (c Client) Post(
|
||||
type CallConfig struct {
|
||||
Expand []string
|
||||
Select []string
|
||||
CanMakeDeltaQueries bool
|
||||
UseImmutableIDs bool
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@ -2,7 +2,6 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -250,9 +249,8 @@ func (p *contactDeltaPager) ValidModTimes() bool {
|
||||
func (c Contacts) GetAddedAndRemovedItemIDs(
|
||||
ctx context.Context,
|
||||
userID, containerID, prevDeltaLink string,
|
||||
immutableIDs bool,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
cc CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"data_category", path.ContactsCategory,
|
||||
@ -263,12 +261,12 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
|
||||
userID,
|
||||
containerID,
|
||||
prevDeltaLink,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd(lastModifiedDateTime)...)
|
||||
pager := c.NewContactsPager(
|
||||
userID,
|
||||
containerID,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd(lastModifiedDateTime)...)
|
||||
|
||||
return pagers.GetAddedAndRemovedItemIDs[models.Contactable](
|
||||
@ -276,6 +274,6 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
|
||||
pager,
|
||||
deltaPager,
|
||||
prevDeltaLink,
|
||||
canMakeDeltaQueries,
|
||||
cc.CanMakeDeltaQueries,
|
||||
pagers.AddedAndRemovedByAddtlData[models.Contactable])
|
||||
}
|
||||
|
||||
@ -140,7 +140,7 @@ func (c Conversations) NewConversationThreadsPager(
|
||||
}
|
||||
}
|
||||
|
||||
// GetConversations fetches all conversations in the group.
|
||||
// GetConversations fetches all conversation threads in the group.
|
||||
func (c Conversations) GetConversationThreads(
|
||||
ctx context.Context,
|
||||
groupID, conversationID string,
|
||||
@ -218,7 +218,7 @@ func (c Conversations) NewConversationThreadPostsPager(
|
||||
}
|
||||
}
|
||||
|
||||
// GetConversations fetches all conversations in the group.
|
||||
// GetConversations fetches all conversation posts in the group.
|
||||
func (c Conversations) GetConversationThreadPosts(
|
||||
ctx context.Context,
|
||||
groupID, conversationID, threadID string,
|
||||
@ -230,3 +230,23 @@ func (c Conversations) GetConversationThreadPosts(
|
||||
|
||||
return items, graph.Stack(ctx, err).OrNil()
|
||||
}
|
||||
|
||||
// GetConversations fetches all added and deleted conversation posts in the group.
|
||||
func (c Conversations) GetConversationThreadPostIDs(
|
||||
ctx context.Context,
|
||||
groupID, conversationID, threadID string,
|
||||
cc CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
canMakeDeltaQueries := false
|
||||
|
||||
aarh, err := pagers.GetAddedAndRemovedItemIDs[models.Postable](
|
||||
ctx,
|
||||
c.NewConversationThreadPostsPager(groupID, conversationID, threadID, CallConfig{}),
|
||||
nil,
|
||||
"",
|
||||
canMakeDeltaQueries,
|
||||
pagers.AddedAndRemovedAddAll[models.Postable],
|
||||
pagers.FilterIncludeAll[models.Postable])
|
||||
|
||||
return aarh, clues.Stack(err).OrNil()
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/stretchr/testify/suite"
|
||||
|
||||
@ -49,6 +50,18 @@ func (suite *ConversationsPagerIntgSuite) TestEnumerateConversations_withThreads
|
||||
for _, thread := range threads {
|
||||
posts := testEnumerateConvPosts(suite, conv, thread)
|
||||
|
||||
aar, err := ac.GetConversationThreadPostIDs(
|
||||
ctx,
|
||||
suite.its.group.id,
|
||||
ptr.Val(conv.GetId()),
|
||||
ptr.Val(thread.GetId()),
|
||||
CallConfig{})
|
||||
require.NoError(t, err, clues.ToCore(err))
|
||||
require.Equal(t, len(posts), len(aar.Added), "added the same number of ids and posts")
|
||||
assert.True(t, aar.ValidModTimes, "mod times should be valid")
|
||||
assert.Empty(t, aar.Removed, "no items should get removed")
|
||||
assert.Empty(t, aar.DU.URL, "no delta update token should be provided")
|
||||
|
||||
for _, post := range posts {
|
||||
testGetPostByID(suite, conv, thread, post)
|
||||
}
|
||||
|
||||
@ -3,7 +3,6 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -245,9 +244,8 @@ func (p *eventDeltaPager) ValidModTimes() bool {
|
||||
func (c Events) GetAddedAndRemovedItemIDs(
|
||||
ctx context.Context,
|
||||
userID, containerID, prevDeltaLink string,
|
||||
immutableIDs bool,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
cc CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"data_category", path.EventsCategory,
|
||||
@ -258,12 +256,12 @@ func (c Events) GetAddedAndRemovedItemIDs(
|
||||
userID,
|
||||
containerID,
|
||||
prevDeltaLink,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd()...)
|
||||
pager := c.NewEventsPager(
|
||||
userID,
|
||||
containerID,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd(lastModifiedDateTime)...)
|
||||
|
||||
return pagers.GetAddedAndRemovedItemIDs[models.Eventable](
|
||||
@ -271,6 +269,6 @@ func (c Events) GetAddedAndRemovedItemIDs(
|
||||
pager,
|
||||
deltaPager,
|
||||
prevDeltaLink,
|
||||
canMakeDeltaQueries,
|
||||
cc.CanMakeDeltaQueries,
|
||||
pagers.AddedAndRemovedByAddtlData[models.Eventable])
|
||||
}
|
||||
|
||||
31
src/pkg/services/m365/api/interfaces.go
Normal file
31
src/pkg/services/m365/api/interfaces.go
Normal file
@ -0,0 +1,31 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/microsoft/kiota-abstractions-go/serialization"
|
||||
|
||||
"github.com/alcionai/corso/src/pkg/fault"
|
||||
)
|
||||
|
||||
type GetAndSerializeItemer[INFO any] interface {
|
||||
GetItemer[INFO]
|
||||
Serializer
|
||||
}
|
||||
|
||||
type GetItemer[INFO any] interface {
|
||||
GetItem(
|
||||
ctx context.Context,
|
||||
user, itemID string,
|
||||
immutableIDs bool,
|
||||
errs *fault.Bus,
|
||||
) (serialization.Parsable, *INFO, error)
|
||||
}
|
||||
|
||||
type Serializer interface {
|
||||
Serialize(
|
||||
ctx context.Context,
|
||||
item serialization.Parsable,
|
||||
protectedResource, itemID string,
|
||||
) ([]byte, error)
|
||||
}
|
||||
@ -3,7 +3,6 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/alcionai/clues"
|
||||
"github.com/microsoftgraph/msgraph-sdk-go/models"
|
||||
@ -247,9 +246,8 @@ func (p *mailDeltaPager) ValidModTimes() bool {
|
||||
func (c Mail) GetAddedAndRemovedItemIDs(
|
||||
ctx context.Context,
|
||||
userID, containerID, prevDeltaLink string,
|
||||
immutableIDs bool,
|
||||
canMakeDeltaQueries bool,
|
||||
) (map[string]time.Time, bool, []string, pagers.DeltaUpdate, error) {
|
||||
cc CallConfig,
|
||||
) (pagers.AddedAndRemoved, error) {
|
||||
ctx = clues.Add(
|
||||
ctx,
|
||||
"data_category", path.EmailCategory,
|
||||
@ -260,12 +258,12 @@ func (c Mail) GetAddedAndRemovedItemIDs(
|
||||
userID,
|
||||
containerID,
|
||||
prevDeltaLink,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd(lastModifiedDateTime)...)
|
||||
pager := c.NewMailPager(
|
||||
userID,
|
||||
containerID,
|
||||
immutableIDs,
|
||||
cc.UseImmutableIDs,
|
||||
idAnd(lastModifiedDateTime)...)
|
||||
|
||||
return pagers.GetAddedAndRemovedItemIDs[models.Messageable](
|
||||
@ -273,6 +271,6 @@ func (c Mail) GetAddedAndRemovedItemIDs(
|
||||
pager,
|
||||
deltaPager,
|
||||
prevDeltaLink,
|
||||
canMakeDeltaQueries,
|
||||
cc.CanMakeDeltaQueries,
|
||||
pagers.AddedAndRemovedByAddtlData[models.Messageable])
|
||||
}
|
||||
|
||||
@ -366,13 +366,28 @@ func batchDeltaEnumerateItems[T any](
|
||||
return results, du, clues.Stack(err).OrNil()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// filter funcs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func FilterIncludeAll[T any](_ T) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// shared enumeration runner funcs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type AddedAndRemoved struct {
|
||||
Added map[string]time.Time
|
||||
Removed []string
|
||||
DU DeltaUpdate
|
||||
ValidModTimes bool
|
||||
}
|
||||
|
||||
type addedAndRemovedHandler[T any] func(
|
||||
items []T,
|
||||
filters ...func(T) bool, // false -> remove, true -> keep
|
||||
filters ...func(T) bool,
|
||||
) (
|
||||
map[string]time.Time,
|
||||
[]string,
|
||||
@ -387,16 +402,23 @@ func GetAddedAndRemovedItemIDs[T any](
|
||||
canMakeDeltaQueries bool,
|
||||
aarh addedAndRemovedHandler[T],
|
||||
filters ...func(T) bool,
|
||||
) (map[string]time.Time, bool, []string, DeltaUpdate, error) {
|
||||
) (AddedAndRemoved, error) {
|
||||
if canMakeDeltaQueries {
|
||||
ts, du, err := batchDeltaEnumerateItems[T](ctx, deltaPager, prevDeltaLink)
|
||||
if err != nil && !graph.IsErrInvalidDelta(err) && !graph.IsErrDeltaNotSupported(err) {
|
||||
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err)
|
||||
return AddedAndRemoved{}, graph.Stack(ctx, err)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
a, r, err := aarh(ts, filters...)
|
||||
return a, deltaPager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
|
||||
aar := AddedAndRemoved{
|
||||
Added: a,
|
||||
Removed: r,
|
||||
DU: du,
|
||||
ValidModTimes: deltaPager.ValidModTimes(),
|
||||
}
|
||||
|
||||
return aar, graph.Stack(ctx, err).OrNil()
|
||||
}
|
||||
}
|
||||
|
||||
@ -404,31 +426,61 @@ func GetAddedAndRemovedItemIDs[T any](
|
||||
|
||||
ts, err := BatchEnumerateItems(ctx, pager)
|
||||
if err != nil {
|
||||
return nil, false, nil, DeltaUpdate{}, graph.Stack(ctx, err)
|
||||
return AddedAndRemoved{}, graph.Stack(ctx, err)
|
||||
}
|
||||
|
||||
a, r, err := aarh(ts, filters...)
|
||||
|
||||
return a, pager.ValidModTimes(), r, du, graph.Stack(ctx, err).OrNil()
|
||||
aar := AddedAndRemoved{
|
||||
Added: a,
|
||||
Removed: r,
|
||||
DU: du,
|
||||
ValidModTimes: pager.ValidModTimes(),
|
||||
}
|
||||
|
||||
type getIDer interface {
|
||||
GetId() *string
|
||||
return aar, graph.Stack(ctx, err).OrNil()
|
||||
}
|
||||
|
||||
type getIDAndModDateTimer interface {
|
||||
graph.GetIDer
|
||||
graph.GetLastModifiedDateTimer
|
||||
}
|
||||
|
||||
// AddedAndRemovedAddAll indiscriminately adds every item to the added list, deleting nothing.
|
||||
func AddedAndRemovedAddAll[T any](
|
||||
items []T,
|
||||
filters ...func(T) bool,
|
||||
) (map[string]time.Time, []string, error) {
|
||||
added := map[string]time.Time{}
|
||||
|
||||
for _, item := range items {
|
||||
passAllFilters := true
|
||||
|
||||
for _, passes := range filters {
|
||||
passAllFilters = passAllFilters && passes(item)
|
||||
}
|
||||
|
||||
if !passAllFilters {
|
||||
continue
|
||||
}
|
||||
|
||||
giamdt, ok := any(item).(getIDAndModDateTimer)
|
||||
if !ok {
|
||||
return nil, nil, clues.New("item does not provide id and modified date time getters").
|
||||
With("item_type", fmt.Sprintf("%T", item))
|
||||
}
|
||||
|
||||
added[ptr.Val(giamdt.GetId())] = dttm.OrNow(ptr.Val(giamdt.GetLastModifiedDateTime()))
|
||||
}
|
||||
|
||||
return added, []string{}, nil
|
||||
}
|
||||
|
||||
// for added and removed by additionalData[@removed]
|
||||
|
||||
type getIDModAndAddtler interface {
|
||||
getIDer
|
||||
getModTimer
|
||||
GetAdditionalData() map[string]any
|
||||
}
|
||||
|
||||
// for types that are non-compliant with this interface,
|
||||
// pagers will need to wrap the return value in a struct
|
||||
// that provides this compliance.
|
||||
type getModTimer interface {
|
||||
GetLastModifiedDateTime() *time.Time
|
||||
graph.GetIDer
|
||||
graph.GetLastModifiedDateTimer
|
||||
graph.GetAdditionalDataer
|
||||
}
|
||||
|
||||
func AddedAndRemovedByAddtlData[T any](
|
||||
@ -451,7 +503,7 @@ func AddedAndRemovedByAddtlData[T any](
|
||||
|
||||
giaa, ok := any(item).(getIDModAndAddtler)
|
||||
if !ok {
|
||||
return nil, nil, clues.New("item does not provide id and additional data getters").
|
||||
return nil, nil, clues.New("item does not provide id, modified date time, and additional data getters").
|
||||
With("item_type", fmt.Sprintf("%T", item))
|
||||
}
|
||||
|
||||
@ -461,7 +513,11 @@ func AddedAndRemovedByAddtlData[T any](
|
||||
if giaa.GetAdditionalData()[graph.AddtlDataRemoved] == nil {
|
||||
var modTime time.Time
|
||||
|
||||
if mt, ok := giaa.(getModTimer); ok {
|
||||
// not all items comply with last modified date time, and not all
|
||||
// items can be wrapped in a way that produces a valid value for
|
||||
// the func. That's why this isn't packed in to the expected
|
||||
// interfaace composition.
|
||||
if mt, ok := giaa.(graph.GetLastModifiedDateTimer); ok {
|
||||
// Make sure to get a non-zero mod time if the item doesn't have one for
|
||||
// some reason. Otherwise we can hit an issue where kopia has a
|
||||
// different mod time for the file than the details does. This occurs
|
||||
@ -485,9 +541,9 @@ func AddedAndRemovedByAddtlData[T any](
|
||||
// for added and removed by GetDeletedDateTime()
|
||||
|
||||
type getIDModAndDeletedDateTimer interface {
|
||||
getIDer
|
||||
getModTimer
|
||||
GetDeletedDateTime() *time.Time
|
||||
graph.GetIDer
|
||||
graph.GetLastModifiedDateTimer
|
||||
graph.GetDeletedDateTimer
|
||||
}
|
||||
|
||||
func AddedAndRemovedByDeletedDateTime[T any](
|
||||
@ -510,14 +566,18 @@ func AddedAndRemovedByDeletedDateTime[T any](
|
||||
|
||||
giaddt, ok := any(item).(getIDModAndDeletedDateTimer)
|
||||
if !ok {
|
||||
return nil, nil, clues.New("item does not provide id and deleted date time getters").
|
||||
return nil, nil, clues.New("item does not provide id, modified, and deleted date time getters").
|
||||
With("item_type", fmt.Sprintf("%T", item))
|
||||
}
|
||||
|
||||
if giaddt.GetDeletedDateTime() == nil {
|
||||
var modTime time.Time
|
||||
|
||||
if mt, ok := giaddt.(getModTimer); ok {
|
||||
// not all items comply with last modified date time, and not all
|
||||
// items can be wrapped in a way that produces a valid value for
|
||||
// the func. That's why this isn't packed in to the expected
|
||||
// interfaace composition.
|
||||
if mt, ok := giaddt.(graph.GetLastModifiedDateTimer); ok {
|
||||
// Make sure to get a non-zero mod time if the item doesn't have one for
|
||||
// some reason. Otherwise we can hit an issue where kopia has a
|
||||
// different mod time for the file than the details does. This occurs
|
||||
|
||||
@ -520,7 +520,7 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
|
||||
filters = append(filters, test.filter)
|
||||
}
|
||||
|
||||
added, validModTimes, removed, deltaUpdate, err := GetAddedAndRemovedItemIDs[testItem](
|
||||
aar, err := GetAddedAndRemovedItemIDs[testItem](
|
||||
ctx,
|
||||
test.pagerGetter(t),
|
||||
test.deltaPagerGetter(t),
|
||||
@ -530,18 +530,18 @@ func (suite *PagerUnitSuite) TestGetAddedAndRemovedItemIDs() {
|
||||
filters...)
|
||||
|
||||
require.NoErrorf(t, err, "getting added and removed item IDs: %+v", clues.ToCore(err))
|
||||
if validModTimes {
|
||||
assert.Equal(t, test.expect.added, added, "added item IDs and mod times")
|
||||
if aar.ValidModTimes {
|
||||
assert.Equal(t, test.expect.added, aar.Added, "added item IDs and mod times")
|
||||
} else {
|
||||
assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(added), "added item IDs")
|
||||
for _, modtime := range added {
|
||||
assert.ElementsMatch(t, maps.Keys(test.expect.added), maps.Keys(aar.Added), "added item IDs")
|
||||
for _, modtime := range aar.Added {
|
||||
assert.True(t, modtime.After(epoch), "mod time after epoch")
|
||||
assert.False(t, modtime.Equal(time.Time{}), "non-zero mod time")
|
||||
}
|
||||
}
|
||||
assert.Equal(t, test.expect.validModTimes, validModTimes, "valid mod times")
|
||||
assert.EqualValues(t, test.expect.removed, removed, "removed item IDs")
|
||||
assert.Equal(t, test.expect.deltaUpdate, deltaUpdate, "delta update")
|
||||
assert.Equal(t, test.expect.validModTimes, aar.ValidModTimes, "valid mod times")
|
||||
assert.EqualValues(t, test.expect.removed, aar.Removed, "removed item IDs")
|
||||
assert.Equal(t, test.expect.deltaUpdate, aar.DU, "delta update")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user