larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

populate the location path values (#2430)

Browse Source 
Adds location path values to all exchange types.
Only propagates these values if the collection is
an event category.
This commit is contained in:
Keepers 2023-02-08 14:58:16 -07:00 committed by ryanfkeepers
parent 52add8dad6
commit 210b543280
15 changed files with 426 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/internal/connector/exchange/api/contacts.go
View File

@ -169,10 +169,8 @@ func (c Contacts) EnumerateContainers(
continue
}
temp := graph.NewCacheFolder(fold, nil)
err = fn(temp)
if err != nil {
temp := graph.NewCacheFolder(fold, nil, nil)
if err := fn(temp); err != nil {
errs = multierror.Append(err, errs)
continue
}

9
src/internal/connector/exchange/api/events.go
View File

@ -209,10 +209,11 @@ func (c Events) EnumerateContainers(
continue
}
temp := graph.NewCacheFolder(cd, path.Builder{}.Append(*cd.GetDisplayName()))
err = fn(temp)
if err != nil {
temp := graph.NewCacheFolder(
cd,
path.Builder{}.Append(*cd.GetDisplayName()),
path.Builder{}.Append(*cd.GetDisplayName()))
if err := fn(temp); err != nil {
errs = multierror.Append(err, errs)
continue
}

3
src/internal/connector/exchange/api/mail.go
View File

@ -198,8 +198,7 @@ func (c Mail) EnumerateContainers(
}
for _, v := range resp.GetValue() {
temp := graph.NewCacheFolder(v, nil)
temp := graph.NewCacheFolder(v, nil, nil)
if err := fn(temp); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "iterating mail folders delta"))
continue

6
src/internal/connector/exchange/contact_folder_cache.go
View File

@ -29,8 +29,10 @@ func (cfc *contactFolderCache) populateContactRoot(
return support.ConnectorStackErrorTraceWrap(err, "fetching root folder")
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(baseContainerPath...))
temp := graph.NewCacheFolder(
f,
path.Builder{}.Append(baseContainerPath...), // storage path
path.Builder{}.Append(baseContainerPath...)) // display location
if err := cfc.addFolder(temp); err != nil {
return errors.Wrap(err, "adding resolver dir")
}

47
src/internal/connector/exchange/container_resolver_test.go
View File

@ -26,16 +26,19 @@ type mockContainer struct {
displayName *string
parentID *string
p *path.Builder
l *path.Builder
}
//nolint:revive
func (m mockContainer) GetId() *string { return m.id }
//nolint:revive
func (m mockContainer) GetParentFolderId() *string { return m.parentID }
func (m mockContainer) GetDisplayName() *string { return m.displayName }
func (m mockContainer) Path() *path.Builder { return m.p }
func (m mockContainer) SetPath(p *path.Builder) {}
func (m mockContainer) GetParentFolderId() *string { return m.parentID }
func (m mockContainer) GetDisplayName() *string { return m.displayName }
func (m mockContainer) Location() *path.Builder { return m.l }
func (m mockContainer) SetLocation(p *path.Builder) {}
func (m mockContainer) Path() *path.Builder { return m.p }
func (m mockContainer) SetPath(p *path.Builder) {}
func strPtr(s string) *string {
return &s
@ -168,7 +171,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
parentID: nil,
},
nil,
),
nil),
check: assert.Error,
},
{
@ -180,7 +183,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
parentID: nil,
},
path.Builder{}.Append("foo"),
),
path.Builder{}.Append("loc")),
check: assert.NoError,
},
{
@ -192,7 +195,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
parentID: &testParentID,
},
path.Builder{}.Append("foo"),
),
path.Builder{}.Append("loc")),
check: assert.Error,
},
{
@ -204,7 +207,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
parentID: &testParentID,
},
path.Builder{}.Append("foo"),
),
path.Builder{}.Append("loc")),
check: assert.Error,
},
{
@ -216,7 +219,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
parentID: &testParentID,
},
nil,
),
nil),
check: assert.NoError,
},
}
@ -241,31 +244,21 @@ type mockCachedContainer struct {
id string
parentID string
displayName string
l *path.Builder
p *path.Builder
expectedPath string
}
//nolint:revive
func (m mockCachedContainer) GetId() *string {
return &m.id
}
func (m mockCachedContainer) GetId() *string { return &m.id }
//nolint:revive
func (m mockCachedContainer) GetParentFolderId() *string {
return &m.parentID
}
func (m mockCachedContainer) GetDisplayName() *string {
return &m.displayName
}
func (m mockCachedContainer) Path() *path.Builder {
return m.p
}
func (m *mockCachedContainer) SetPath(newPath *path.Builder) {
m.p = newPath
}
func (m mockCachedContainer) GetParentFolderId() *string { return &m.parentID }
func (m mockCachedContainer) GetDisplayName() *string { return &m.displayName }
func (m mockCachedContainer) Location() *path.Builder { return m.l }
func (m *mockCachedContainer) SetLocation(newLoc *path.Builder) { m.l = newLoc }
func (m mockCachedContainer) Path() *path.Builder { return m.p }
func (m *mockCachedContainer) SetPath(newPath *path.Builder) { m.p = newPath }
func resolverWithContainers(numContainers int) (*containerResolver, []*mockCachedContainer) {
containers := make([]*mockCachedContainer, 0, numContainers)

10
src/internal/connector/exchange/event_calendar_cache.go
View File

@ -44,7 +44,10 @@ func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error {
return errors.Wrap(err, "fetching calendar "+support.ConnectorStackErrorTrace(err))
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(container))
temp := graph.NewCacheFolder(
f,
path.Builder{}.Append(container), // storage path
path.Builder{}.Append(container)) // display location
if err := ecc.addFolder(temp); err != nil {
return errors.Wrap(err, "initializing calendar resolver")
}
@ -91,7 +94,10 @@ func (ecc *eventCalendarCache) AddToCache(ctx context.Context, f graph.Container
return errors.Wrap(err, "validating container")
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(calendarOthersFolder, *f.GetDisplayName()))
temp := graph.NewCacheFolder(
f,
path.Builder{}.Append(calendarOthersFolder, *f.GetDisplayName()), // storage path
path.Builder{}.Append(calendarOthersFolder, *f.GetDisplayName())) // display location
if err := ecc.addFolder(temp); err != nil {
return errors.Wrap(err, "adding container")

20
src/internal/connector/exchange/exchange_data_collection.go
View File

@ -77,6 +77,11 @@ type Collection struct {
// moved. It will be empty on its first retrieval.
prevPath path.Path
// LocationPath contains the path with human-readable display names.
// IE: "/Inbox/Important" instead of "/abcdxyz123/algha=lgkhal=t"
// Currently only implemented for Exchange Calendars.
locationPath path.Path
state data.CollectionState
// doNotMergeItems should only be true if the old delta token expired.
@ -91,7 +96,7 @@ type Collection struct {
// or notMoved (if they match).
func NewCollection(
user string,
curr, prev path.Path,
curr, prev, location path.Path,
category path.CategoryType,
items itemer,
statusUpdater support.StatusUpdater,
@ -99,18 +104,19 @@ func NewCollection(
doNotMergeItems bool,
) Collection {
collection := Collection{
added: make(map[string]struct{}, 0),
category: category,
ctrl: ctrlOpts,
data: make(chan data.Stream, collectionChannelBufferSize),
doNotMergeItems: doNotMergeItems,
fullPath: curr,
added: make(map[string]struct{}, 0),
removed: make(map[string]struct{}, 0),
items: items,
locationPath: location,
prevPath: prev,
removed: make(map[string]struct{}, 0),
state: stateOf(prev, curr),
statusUpdater: statusUpdater,
user: user,
items: items,
}
return collection
@ -144,6 +150,12 @@ func (col *Collection) FullPath() path.Path {
return col.fullPath
}
// LocationPath produces the Collection's full path, but with display names
// instead of IDs in the folders. Only populated for Calendars.
func (col *Collection) LocationPath() path.Path {
return col.locationPath
}
// TODO(ashmrtn): Fill in with previous path once GraphConnector compares old
// and new folder hierarchies.
func (col Collection) PreviousPath() path.Path {

15
src/internal/connector/exchange/exchange_data_collection_test.go
View File

@ -127,28 +127,36 @@ func (suite *ExchangeDataCollectionSuite) TestNewCollection_state() {
Append("bar").
ToDataLayerExchangePathForCategory("t", "u", path.EmailCategory, false)
require.NoError(suite.T(), err)
locP, err := path.Builder{}.
Append("human-readable").
ToDataLayerExchangePathForCategory("t", "u", path.EmailCategory, false)
require.NoError(suite.T(), err)
table := []struct {
name string
prev path.Path
curr path.Path
loc path.Path
expect data.CollectionState
}{
{
name: "new",
curr: fooP,
loc: locP,
expect: data.NewState,
},
{
name: "not moved",
prev: fooP,
curr: fooP,
loc: locP,
expect: data.NotMovedState,
},
{
name: "moved",
prev: fooP,
curr: barP,
loc: locP,
expect: data.MovedState,
},
{
@ -161,12 +169,15 @@ func (suite *ExchangeDataCollectionSuite) TestNewCollection_state() {
suite.T().Run(test.name, func(t *testing.T) {
c := NewCollection(
"u",
test.curr, test.prev,
test.curr, test.prev, test.loc,
0,
&mockItemer{}, nil,
control.Options{},
false)
assert.Equal(t, test.expect, c.State())
assert.Equal(t, test.expect, c.State(), "collection state")
assert.Equal(t, test.curr, c.fullPath, "full path")
assert.Equal(t, test.prev, c.prevPath, "prev path")
assert.Equal(t, test.loc, c.locationPath, "location path")
})
}
}

4
src/internal/connector/exchange/mail_folder_cache.go
View File

@ -53,7 +53,9 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error {
directory = DefaultMailFolder
}
temp := graph.NewCacheFolder(f, path.Builder{}.Append(directory))
temp := graph.NewCacheFolder(f,
path.Builder{}.Append(directory), // storage path
path.Builder{}.Append(directory)) // display location
if err := mc.addFolder(temp); err != nil {
return errors.Wrap(err, "adding resolver dir")
}

50
src/internal/connector/exchange/service_functions.go
View File

@ -86,44 +86,70 @@ func PopulateExchangeContainerResolver(
}
// Returns true if the container passes the scope comparison and should be included.
// Also returns the path representing the directory.
// Returns:
// - the path representing the directory as it should be stored in the repository.
// - the human-readable path using display names.
// - true if the path passes the scope comparison.
func includeContainer(
qp graph.QueryParams,
c graph.CachedContainer,
scope selectors.ExchangeScope,
) (path.Path, bool) {
) (path.Path, path.Path, bool) {
var (
category = scope.Category().PathType()
directory string
locPath path.Path
category = scope.Category().PathType()
pb = c.Path()
loc = c.Location()
)
// Clause ensures that DefaultContactFolder is inspected properly
if category == path.ContactsCategory && *c.GetDisplayName() == DefaultContactFolder {
pb = c.Path().Append(DefaultContactFolder)
pb = pb.Append(DefaultContactFolder)
if loc != nil {
loc = loc.Append(DefaultContactFolder)
}
}
dirPath, err := pb.ToDataLayerExchangePathForCategory(
qp.Credentials.AzureTenantID,
qp.ResourceOwner,
category,
false,
)
false)
// Containers without a path (e.g. Root mail folder) always err here.
if err != nil {
return nil, false
return nil, nil, false
}
directory = pb.String()
directory = dirPath.Folder()
if loc != nil {
locPath, err = loc.ToDataLayerExchangePathForCategory(
qp.Credentials.AzureTenantID,
qp.ResourceOwner,
category,
false)
// Containers without a path (e.g. Root mail folder) always err here.
if err != nil {
return nil, nil, false
}
directory = locPath.Folder()
}
var ok bool
switch category {
case path.EmailCategory:
return dirPath, scope.Matches(selectors.ExchangeMailFolder, directory)
ok = scope.Matches(selectors.ExchangeMailFolder, directory)
case path.ContactsCategory:
return dirPath, scope.Matches(selectors.ExchangeContactFolder, directory)
ok = scope.Matches(selectors.ExchangeContactFolder, directory)
case path.EventsCategory:
return dirPath, scope.Matches(selectors.ExchangeEventCalendar, directory)
ok = scope.Matches(selectors.ExchangeEventCalendar, directory)
default:
return dirPath, false
return nil, nil, false
}
return dirPath, locPath, ok
}

8
src/internal/connector/exchange/service_iterators.go
View File

@ -70,7 +70,7 @@ func filterContainersAndFillCollections(
cID := *c.GetId()
delete(tombstones, cID)
currPath, ok := includeContainer(qp, c, scope)
currPath, locPath, ok := includeContainer(qp, c, scope)
// Only create a collection if the path matches the scope.
if !ok {
continue
@ -110,10 +110,15 @@ func filterContainersAndFillCollections(
deltaURLs[cID] = newDelta.URL
}
if qp.Category != path.EventsCategory {
locPath = nil
}
edc := NewCollection(
qp.ResourceOwner,
currPath,
prevPath,
locPath,
scope.Category().PathType(),
ibt,
statusUpdater,
@ -167,6 +172,7 @@ func filterContainersAndFillCollections(
qp.ResourceOwner,
nil, // marks the collection as deleted
prevPath,
nil, // tombstones don't need a location
scope.Category().PathType(),
ibt,
statusUpdater,

18
src/internal/connector/graph/cache_container.go
View File

@ -12,6 +12,12 @@ import (
// reuse logic in IDToPath.
type CachedContainer interface {
Container
// Location contains either the display names for the dirs (if this is a calendar)
// or nil
Location() *path.Builder
SetLocation(*path.Builder)
// Path contains either the ids for the dirs (if this is a calendar)
// or the display names for the dirs
Path() *path.Builder
SetPath(*path.Builder)
}
@ -45,13 +51,15 @@ var _ CachedContainer = &CacheFolder{}
type CacheFolder struct {
Container
l *path.Builder
p *path.Builder
}
// NewCacheFolder public constructor for struct
func NewCacheFolder(c Container, pb *path.Builder) CacheFolder {
func NewCacheFolder(c Container, pb, lpb *path.Builder) CacheFolder {
cf := CacheFolder{
Container: c,
l: lpb,
p: pb,
}
@ -62,6 +70,14 @@ func NewCacheFolder(c Container, pb *path.Builder) CacheFolder {
// Required Functions to satisfy interfaces
// =========================================
func (cf CacheFolder) Location() *path.Builder {
return cf.l
}
func (cf *CacheFolder) SetLocation(newLocation *path.Builder) {
cf.l = newLocation
}
func (cf CacheFolder) Path() *path.Builder {
return cf.p
}

227
src/pkg/selectors/exchange_test.go
View File

@ -1041,6 +1041,233 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
}
}
func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce_locationRef() {
var (
contact = stubRepoRef(path.ExchangeService, path.ContactsCategory, "uid", "id5/id6", "cid")
contactLocation = "conts/my_cont"
event = stubRepoRef(path.ExchangeService, path.EventsCategory, "uid", "id1/id2", "eid")
eventLocation = "cal/my_cal"
mail = stubRepoRef(path.ExchangeService, path.EmailCategory, "uid", "id3/id4", "mid")
mailLocation = "inbx/my_mail"
)
makeDeets := func(refs ...string) *details.Details {
deets := &details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.DetailsEntry{},
},
}
for _, r := range refs {
var (
location string
itype = details.UnknownType
)
switch r {
case contact:
itype = details.ExchangeContact
location = contactLocation
case event:
itype = details.ExchangeEvent
location = eventLocation
case mail:
itype = details.ExchangeMail
location = mailLocation
}
deets.Entries = append(deets.Entries, details.DetailsEntry{
RepoRef: r,
LocationRef: location,
ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{
ItemType: itype,
},
},
})
}
return deets
}
arr := func(s ...string) []string {
return s
}
table := []struct {
name string
deets *details.Details
makeSelector func() *ExchangeRestore
expect []string
}{
{
"no refs",
makeDeets(),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
return er
},
[]string{},
},
{
"contact only",
makeDeets(contact),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
return er
},
arr(contact),
},
{
"event only",
makeDeets(event),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
return er
},
arr(event),
},
{
"mail only",
makeDeets(mail),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
return er
},
arr(mail),
},
{
"all",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
return er
},
arr(contact, event, mail),
},
{
"only match contact",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore([]string{"uid"})
er.Include(er.Contacts([]string{contactLocation}, []string{"cid"}))
return er
},
arr(contact),
},
{
"only match event",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore([]string{"uid"})
er.Include(er.Events([]string{eventLocation}, []string{"eid"}))
return er
},
arr(event),
},
{
"only match mail",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore([]string{"uid"})
er.Include(er.Mails([]string{mailLocation}, []string{"mid"}))
return er
},
arr(mail),
},
{
"exclude contact",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
er.Exclude(er.Contacts([]string{contactLocation}, []string{"cid"}))
return er
},
arr(event, mail),
},
{
"exclude event",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
er.Exclude(er.Events([]string{eventLocation}, []string{"eid"}))
return er
},
arr(contact, mail),
},
{
"exclude mail",
makeDeets(contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
er.Exclude(er.Mails([]string{mailLocation}, []string{"mid"}))
return er
},
arr(contact, event),
},
{
"filter on mail subject",
func() *details.Details {
ds := makeDeets(mail)
for i := range ds.Entries {
ds.Entries[i].Exchange.Subject = "has a subject"
}
return ds
}(),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
er.Filter(er.MailSubject("subj"))
return er
},
arr(mail),
},
{
"filter on mail subject multiple input categories",
func() *details.Details {
mds := makeDeets(mail)
for i := range mds.Entries {
mds.Entries[i].Exchange.Subject = "has a subject"
}
ds := makeDeets(contact, event)
ds.Entries = append(ds.Entries, mds.Entries...)
return ds
}(),
func() *ExchangeRestore {
er := NewExchangeRestore(Any())
er.Include(er.AllData())
er.Filter(er.MailSubject("subj"))
return er
},
arr(mail),
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
errs := mock.NewAdder()
sel := test.makeSelector()
results := sel.Reduce(ctx, test.deets, errs)
paths := results.Paths()
assert.Equal(t, test.expect, paths)
assert.Empty(t, errs.Errs)
})
}
}
func (suite *ExchangeSelectorSuite) TestScopesByCategory() {
var (
es = NewExchangeRestore(Any())

21
src/pkg/selectors/scopes.go
View File

@ -317,6 +317,27 @@ func reduce[T scopeT, C categoryT](
continue
}
// if the details entry has a locationRef specified, use those folders in place
// of the repoRef folders, so that scopes can match against the display names
// instead of container IDs.
if len(ent.LocationRef) > 0 {
pb, err := path.Builder{}.
Append(path.Split(ent.LocationRef)...).
Append(repoPath.Item()).
ToDataLayerPath(
repoPath.Tenant(),
repoPath.ResourceOwner(),
repoPath.Service(),
repoPath.Category(),
true)
if err != nil {
errs.Add(clues.Wrap(err, "transforming locationRef to path").WithClues(ctx))
continue
}
repoPath = pb
}
// first check, every entry needs to match the selector's resource owners.
if !matchesResourceOwner.Compare(repoPath.ResourceOwner()) {
continue

44
src/pkg/selectors/scopes_test.go
View File

@ -290,6 +290,50 @@ func (suite *SelectorScopesSuite) TestReduce() {
}
}
func (suite *SelectorScopesSuite) TestReduce_locationRef() {
deets := func() details.Details {
return details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.DetailsEntry{
{
RepoRef: stubRepoRef(
pathServiceStub,
pathCatStub,
rootCatStub.String(),
"stub",
leafCatStub.String(),
),
LocationRef: "a/b/c//defg",
},
},
},
}
}
dataCats := map[path.CategoryType]mockCategorizer{
pathCatStub: rootCatStub,
}
for _, test := range reduceTestTable {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
errs := mock.NewAdder()
ds := deets()
result := reduce[mockScope](
ctx,
&ds,
test.sel().Selector,
dataCats,
errs)
require.NotNil(t, result)
require.Empty(t, errs.Errs, "iteration errors")
assert.Len(t, result.Entries, test.expectLen)
})
}
}
func (suite *SelectorScopesSuite) TestScopesByCategory() {
t := suite.T()
s1 := stubScope("")