add per-container item collision enumeration (#3629)

populates the item collision cache generators for all exchange data types.  Next we'll use these to compare the items being restored for collision before making post requests, by populating this cache at the top of each collection restore process.

---

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3562

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-06-22 14:54:06 -06:00 committed by GitHub
parent 65200121b8
commit f4847404c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 490 additions and 86 deletions

View File

@ -228,7 +228,7 @@ func (m GetsItemPermission) GetItemPermission(
// ---------------------------------------------------------------------------
// Restore Handler
// ---------------------------------------------------------------------------
// --------------------------------------------------------------------------
type RestoreHandler struct {
ItemInfo details.ItemInfo

View File

@ -17,8 +17,15 @@ const (
// get easily misspelled.
// eg: we don't need a const for "id"
const (
parentFolderID = "parentFolderId"
attendees = "attendees"
bccRecipients = "bccRecipients"
ccRecipients = "ccRecipients"
createdDateTime = "createdDateTime"
displayName = "displayName"
givenName = "givenName"
parentFolderID = "parentFolderId"
surname = "surname"
toRecipients = "toRecipients"
userPrincipalName = "userPrincipalName"
)

View File

@ -265,3 +265,17 @@ func ContactInfo(contact models.Contactable) *details.ExchangeInfo {
Modified: ptr.OrNow(contact.GetLastModifiedDateTime()),
}
}
func contactCollisionKeyProps() []string {
return idAnd(givenName)
}
// ContactCollisionKey constructs a key from the contactable's creation time and either displayName or given+surname.
// collision keys are used to identify duplicate item conflicts for handling advanced restoration config.
func ContactCollisionKey(item models.Contactable) string {
if item == nil {
return ""
}
return ptr.Val(item.GetId())
}

View File

@ -90,32 +90,74 @@ func (c Contacts) EnumerateContainers(
// item pager
// ---------------------------------------------------------------------------
var _ itemPager[models.Contactable] = &contactsPager{}
var _ itemPager[models.Contactable] = &contactsPageCtrl{}
type contactsPager struct {
// TODO(rkeeprs)
type contactsPageCtrl struct {
gs graph.Servicer
builder *users.ItemContactFoldersItemContactsRequestBuilder
options *users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration
}
func (c Contacts) NewContactsPager() itemPager[models.Contactable] {
// TODO(rkeepers)
return nil
func (c Contacts) NewContactsPager(
userID, containerID string,
selectProps ...string,
) itemPager[models.Contactable] {
options := &users.ItemContactFoldersItemContactsRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)),
}
if len(selectProps) > 0 {
options.QueryParameters = &users.ItemContactFoldersItemContactsRequestBuilderGetQueryParameters{
Select: selectProps,
}
}
builder := c.Stable.
Client().
Users().
ByUserId(userID).
ContactFolders().
ByContactFolderId(containerID).
Contacts()
return &contactsPageCtrl{c.Stable, builder, options}
}
//lint:ignore U1000 False Positive
func (p *contactsPager) getPage(ctx context.Context) (PageLinker, error) {
// TODO(rkeepers)
return nil, nil
func (p *contactsPageCtrl) getPage(ctx context.Context) (PageLinkValuer[models.Contactable], error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return EmptyDeltaLinker[models.Contactable]{PageLinkValuer: resp}, nil
}
//lint:ignore U1000 False Positive
func (p *contactsPager) setNext(nextLink string) {
// TODO(rkeepers)
func (p *contactsPageCtrl) setNext(nextLink string) {
p.builder = users.NewItemContactFoldersItemContactsRequestBuilder(nextLink, p.gs.Adapter())
}
//lint:ignore U1000 False Positive
func (p *contactsPager) valuesIn(pl PageLinker) ([]models.Contactable, error) {
// TODO(rkeepers)
return nil, nil
func (c Contacts) GetItemsInContainerByCollisionKey(
ctx context.Context,
userID, containerID string,
) (map[string]string, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewContactsPager(userID, containerID, contactCollisionKeyProps()...)
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating contacts")
}
m := map[string]string{}
for _, item := range items {
m[ContactCollisionKey(item)] = ptr.Val(item.GetId())
}
return m, nil
}
// ---------------------------------------------------------------------------

View File

@ -0,0 +1,73 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type ContactsPagerIntgSuite struct {
tester.Suite
cts clientTesterSetup
}
func TestContactsPagerIntgSuite(t *testing.T) {
suite.Run(t, &ContactsPagerIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tester.M365AcctCredEnvs}),
})
}
func (suite *ContactsPagerIntgSuite) SetupSuite() {
suite.cts = newClientTesterSetup(suite.T())
}
func (suite *ContactsPagerIntgSuite) TestGetItemsInContainerByCollisionKey() {
t := suite.T()
ac := suite.cts.ac.Contacts()
ctx, flush := tester.NewContext(t)
defer flush()
container, err := ac.GetContainerByID(ctx, suite.cts.userID, "contacts")
require.NoError(t, err, clues.ToCore(err))
conts, err := ac.Stable.
Client().
Users().
ByUserId(suite.cts.userID).
ContactFolders().
ByContactFolderId(ptr.Val(container.GetId())).
Contacts().
Get(ctx, nil)
require.NoError(t, err, clues.ToCore(err))
cs := conts.GetValue()
expect := make([]string, 0, len(cs))
for _, c := range cs {
expect = append(expect, api.ContactCollisionKey(c))
}
results, err := ac.GetItemsInContainerByCollisionKey(ctx, suite.cts.userID, "contacts")
require.NoError(t, err, clues.ToCore(err))
require.Less(t, 0, len(results), "requires at least one result")
for k, v := range results {
assert.NotEmpty(t, k, "all keys should be populated")
assert.NotEmpty(t, v, "all values should be populated")
}
for _, e := range expect {
_, ok := results[e]
assert.Truef(t, ok, "expected results to contain collision key: %s", e)
}
}

View File

@ -698,3 +698,17 @@ func EventFromMap(ev map[string]any) (models.Eventable, error) {
return body, nil
}
func eventCollisionKeyProps() []string {
return idAnd("subject")
}
// EventCollisionKey constructs a key from the eventable's creation time, subject, and organizer.
// collision keys are used to identify duplicate item conflicts for handling advanced restoration config.
func EventCollisionKey(item models.Eventable) string {
if item == nil {
return ""
}
return ptr.Val(item.GetSubject())
}

View File

@ -98,32 +98,74 @@ func (c Events) EnumerateContainers(
// item pager
// ---------------------------------------------------------------------------
var _ itemPager[models.Eventable] = &eventsPager{}
var _ itemPager[models.Eventable] = &eventsPageCtrl{}
type eventsPager struct {
// TODO(rkeeprs)
type eventsPageCtrl struct {
gs graph.Servicer
builder *users.ItemCalendarsItemEventsRequestBuilder
options *users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration
}
func (c Events) NewEventsPager() itemPager[models.Eventable] {
// TODO(rkeepers)
return nil
func (c Events) NewEventsPager(
userID, containerID string,
selectProps ...string,
) itemPager[models.Eventable] {
options := &users.ItemCalendarsItemEventsRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)),
}
if len(selectProps) > 0 {
options.QueryParameters = &users.ItemCalendarsItemEventsRequestBuilderGetQueryParameters{
Select: selectProps,
}
}
builder := c.Stable.
Client().
Users().
ByUserId(userID).
Calendars().
ByCalendarId(containerID).
Events()
return &eventsPageCtrl{c.Stable, builder, options}
}
//lint:ignore U1000 False Positive
func (p *eventsPager) getPage(ctx context.Context) (PageLinker, error) {
// TODO(rkeepers)
return nil, nil
func (p *eventsPageCtrl) getPage(ctx context.Context) (PageLinkValuer[models.Eventable], error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, nil
}
//lint:ignore U1000 False Positive
func (p *eventsPager) setNext(nextLink string) {
// TODO(rkeepers)
func (p *eventsPageCtrl) setNext(nextLink string) {
p.builder = users.NewItemCalendarsItemEventsRequestBuilder(nextLink, p.gs.Adapter())
}
//lint:ignore U1000 False Positive
func (p *eventsPager) valuesIn(pl PageLinker) ([]models.Eventable, error) {
// TODO(rkeepers)
return nil, nil
func (c Events) GetItemsInContainerByCollisionKey(
ctx context.Context,
userID, containerID string,
) (map[string]string, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewEventsPager(userID, containerID, eventCollisionKeyProps()...)
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating events")
}
m := map[string]string{}
for _, item := range items {
m[EventCollisionKey(item)] = ptr.Val(item.GetId())
}
return m, nil
}
// ---------------------------------------------------------------------------

View File

@ -0,0 +1,73 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type EventsPagerIntgSuite struct {
tester.Suite
cts clientTesterSetup
}
func TestEventsPagerIntgSuite(t *testing.T) {
suite.Run(t, &EventsPagerIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tester.M365AcctCredEnvs}),
})
}
func (suite *EventsPagerIntgSuite) SetupSuite() {
suite.cts = newClientTesterSetup(suite.T())
}
func (suite *EventsPagerIntgSuite) TestGetItemsInContainerByCollisionKey() {
t := suite.T()
ac := suite.cts.ac.Events()
ctx, flush := tester.NewContext(t)
defer flush()
container, err := ac.GetContainerByID(ctx, suite.cts.userID, "calendar")
require.NoError(t, err, clues.ToCore(err))
evts, err := ac.Stable.
Client().
Users().
ByUserId(suite.cts.userID).
Calendars().
ByCalendarId(ptr.Val(container.GetId())).
Events().
Get(ctx, nil)
require.NoError(t, err, clues.ToCore(err))
es := evts.GetValue()
expect := make([]string, 0, len(es))
for _, e := range es {
expect = append(expect, api.EventCollisionKey(e))
}
results, err := ac.GetItemsInContainerByCollisionKey(ctx, suite.cts.userID, "calendar")
require.NoError(t, err, clues.ToCore(err))
require.Less(t, 0, len(results), "requires at least one result")
for k, v := range results {
assert.NotEmpty(t, k, "all keys should be populated")
assert.NotEmpty(t, v, "all values should be populated")
}
for _, e := range expect {
_, ok := results[e]
assert.Truef(t, ok, "expected results to contain collision key: %s", e)
}
}

View File

@ -0,0 +1,34 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type clientTesterSetup struct {
ac api.Client
userID string
}
func newClientTesterSetup(t *testing.T) clientTesterSetup {
cts := clientTesterSetup{}
ctx, flush := tester.NewContext(t)
defer flush()
a := tester.NewM365Account(t)
creds, err := a.M365Config()
require.NoError(t, err, clues.ToCore(err))
cts.ac, err = api.NewClient(creds)
require.NoError(t, err, clues.ToCore(err))
cts.userID = tester.GetM365UserID(ctx)
return cts
}

View File

@ -66,11 +66,9 @@ func (e EmptyDeltaLinker[T]) GetValue() []T {
type itemPager[T any] interface {
// getPage get a page with the specified options from graph
getPage(context.Context) (PageLinker, error)
getPage(context.Context) (PageLinkValuer[T], error)
// setNext is used to pass in the next url got from graph
setNext(string)
// valuesIn gets us the values in a page
valuesIn(PageLinker) ([]T, error)
}
func enumerateItems[T any](
@ -90,14 +88,7 @@ func enumerateItems[T any](
return nil, graph.Stack(ctx, err)
}
// each category type responds with a different interface, but all
// of them comply with GetValue, which is where we'll get our item data.
items, err := pager.valuesIn(resp)
if err != nil {
return nil, graph.Stack(ctx, err)
}
result = append(result, items...)
result = append(result, resp.GetValue()...)
nextLink = NextLink(resp)
pager.setNext(nextLink)

View File

@ -57,7 +57,9 @@ func (v testPagerValue) GetAdditionalData() map[string]any {
// mock page
type testPage struct{}
type testPage struct {
values []any
}
func (p testPage) GetOdataNextLink() *string {
// no next, just one page
@ -69,30 +71,28 @@ func (p testPage) GetOdataDeltaLink() *string {
return ptr.To("")
}
func (p testPage) GetValue() []any {
return p.values
}
// mock item pager
var _ itemPager[any] = &testPager{}
type testPager struct {
t *testing.T
items []any
pageErr error
valuesErr error
t *testing.T
pager testPage
pageErr error
}
//lint:ignore U1000 False Positive
func (p *testPager) getPage(ctx context.Context) (PageLinker, error) {
return testPage{}, p.pageErr
func (p *testPager) getPage(ctx context.Context) (PageLinkValuer[any], error) {
return p.pager, p.pageErr
}
//lint:ignore U1000 False Positive
func (p *testPager) setNext(nextLink string) {}
//lint:ignore U1000 False Positive
func (p *testPager) valuesIn(pl PageLinker) ([]any, error) {
return p.items, p.valuesErr
}
// mock id pager
var _ itemIDPager = &testIDsPager{}
@ -169,26 +169,12 @@ func (suite *ItemPagerUnitSuite) TestEnumerateItems() {
) itemPager[any] {
return &testPager{
t: t,
items: []any{"foo", "bar"},
pager: testPage{[]any{"foo", "bar"}},
}
},
expect: []any{"foo", "bar"},
expectErr: require.NoError,
},
{
name: "get values err",
getPager: func(
t *testing.T,
ctx context.Context,
) itemPager[any] {
return &testPager{
t: t,
valuesErr: assert.AnError,
}
},
expect: nil,
expectErr: require.Error,
},
{
name: "next page err",
getPager: func(
@ -212,7 +198,7 @@ func (suite *ItemPagerUnitSuite) TestEnumerateItems() {
ctx, flush := tester.NewContext(t)
defer flush()
result, err := enumerateItems[any](ctx, test.getPager(t, ctx))
result, err := enumerateItems(ctx, test.getPager(t, ctx))
test.expectErr(t, err, clues.ToCore(err))
require.EqualValues(t, test.expect, result)

View File

@ -540,3 +540,17 @@ func UnwrapEmailAddress(contact models.Recipientable) string {
return ptr.Val(contact.GetEmailAddress().GetAddress())
}
func mailCollisionKeyProps() []string {
return idAnd("subject")
}
// MailCollisionKey constructs a key from the messageable's subject, sender, and recipients (to, cc, bcc).
// collision keys are used to identify duplicate item conflicts for handling advanced restoration config.
func MailCollisionKey(item models.Messageable) string {
if item == nil {
return ""
}
return ptr.Val(item.GetSubject())
}

View File

@ -121,32 +121,52 @@ func (c Mail) EnumerateContainers(
// item pager
// ---------------------------------------------------------------------------
var _ itemPager[models.Messageable] = &mailPager{}
var _ itemPager[models.Messageable] = &mailPageCtrl{}
type mailPager struct {
// TODO(rkeeprs)
type mailPageCtrl struct {
gs graph.Servicer
builder *users.ItemMailFoldersItemMessagesRequestBuilder
options *users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration
}
func (c Mail) NewMailPager() itemPager[models.Messageable] {
// TODO(rkeepers)
return nil
func (c Mail) NewMailPager(
userID, containerID string,
selectProps ...string,
) itemPager[models.Messageable] {
options := &users.ItemMailFoldersItemMessagesRequestBuilderGetRequestConfiguration{
Headers: newPreferHeaders(preferPageSize(maxNonDeltaPageSize)),
}
if len(selectProps) > 0 {
options.QueryParameters = &users.ItemMailFoldersItemMessagesRequestBuilderGetQueryParameters{
Select: selectProps,
}
}
builder := c.Stable.
Client().
Users().
ByUserId(userID).
MailFolders().
ByMailFolderId(containerID).
Messages()
return &mailPageCtrl{c.Stable, builder, options}
}
//lint:ignore U1000 False Positive
func (p *mailPager) getPage(ctx context.Context) (PageLinker, error) {
// TODO(rkeepers)
return nil, nil
func (p *mailPageCtrl) getPage(ctx context.Context) (PageLinkValuer[models.Messageable], error) {
page, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return EmptyDeltaLinker[models.Messageable]{PageLinkValuer: page}, nil
}
//lint:ignore U1000 False Positive
func (p *mailPager) setNext(nextLink string) {
// TODO(rkeepers)
}
//lint:ignore U1000 False Positive
func (p *mailPager) valuesIn(pl PageLinker) ([]models.Messageable, error) {
// TODO(rkeepers)
return nil, nil
func (p *mailPageCtrl) setNext(nextLink string) {
p.builder = users.NewItemMailFoldersItemMessagesRequestBuilder(nextLink, p.gs.Adapter())
}
// ---------------------------------------------------------------------------
@ -204,6 +224,27 @@ func (p *mailIDPager) valuesIn(pl PageLinker) ([]getIDAndAddtler, error) {
return toValues[models.Messageable](pl)
}
func (c Mail) GetItemsInContainerByCollisionKey(
ctx context.Context,
userID, containerID string,
) (map[string]string, error) {
ctx = clues.Add(ctx, "container_id", containerID)
pager := c.NewMailPager(userID, containerID, mailCollisionKeyProps()...)
items, err := enumerateItems(ctx, pager)
if err != nil {
return nil, graph.Wrap(ctx, err, "enumerating mail")
}
m := map[string]string{}
for _, item := range items {
m[MailCollisionKey(item)] = ptr.Val(item.GetId())
}
return m, nil
}
// ---------------------------------------------------------------------------
// delta item ID pager
// ---------------------------------------------------------------------------

View File

@ -0,0 +1,73 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type MailPagerIntgSuite struct {
tester.Suite
cts clientTesterSetup
}
func TestMailPagerIntgSuite(t *testing.T) {
suite.Run(t, &MailPagerIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tester.M365AcctCredEnvs}),
})
}
func (suite *MailPagerIntgSuite) SetupSuite() {
suite.cts = newClientTesterSetup(suite.T())
}
func (suite *MailPagerIntgSuite) TestGetItemsInContainerByCollisionKey() {
t := suite.T()
ac := suite.cts.ac.Mail()
ctx, flush := tester.NewContext(t)
defer flush()
container, err := ac.GetContainerByID(ctx, suite.cts.userID, "inbox")
require.NoError(t, err, clues.ToCore(err))
msgs, err := ac.Stable.
Client().
Users().
ByUserId(suite.cts.userID).
MailFolders().
ByMailFolderId(ptr.Val(container.GetId())).
Messages().
Get(ctx, nil)
require.NoError(t, err, clues.ToCore(err))
ms := msgs.GetValue()
expect := make([]string, 0, len(ms))
for _, m := range ms {
expect = append(expect, api.MailCollisionKey(m))
}
results, err := ac.GetItemsInContainerByCollisionKey(ctx, suite.cts.userID, "inbox")
require.NoError(t, err, clues.ToCore(err))
require.Less(t, 0, len(results), "requires at least one result")
for k, v := range results {
assert.NotEmpty(t, k, "all keys should be populated")
assert.NotEmpty(t, v, "all values should be populated")
}
for _, e := range expect {
_, ok := results[e]
assert.Truef(t, ok, "expected results to contain collision key: %s", e)
}
}