larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

selectors use mulit-value short-refs (#2736)

Browse Source 
With onedrive storage file names being changed from the file display name to the file id, we need a more granular form of indentification when using selectors to choose which values count as matchable fields.

This change modifies the selector PathValues to return slices of strings for each category instead of a single string, and the reducer matches on any. This will allow each service to decide what values are considered equivalent (id, shortRef, a value inside the info, etc) for each property.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #2708

#### Test Plan

- [x] ⚡ Unit test
This commit is contained in:
Keepers 2023-03-08 16:51:49 -07:00 committed by GitHub
parent 07b4900f58
commit 23b90f9bec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 139 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/pkg/selectors/exchange.go
View File

@ -580,7 +580,7 @@ func (ec exchangeCategory) isLeaf() bool {
// Example:
// [tenantID, service, userPN, category, mailFolder, mailID]
// => {exchMailFolder: mailFolder, exchMail: mailID}
func (ec exchangeCategory) pathValues(repo, location path.Path) (map[categorizer]string, map[categorizer]string) {
func (ec exchangeCategory) pathValues(repo path.Path, ent details.DetailsEntry) map[categorizer][]string {
var folderCat, itemCat categorizer
switch ec {
@ -594,24 +594,19 @@ func (ec exchangeCategory) pathValues(repo, location path.Path) (map[categorizer
folderCat, itemCat = ExchangeMailFolder, ExchangeMail
default:
return map[categorizer]string{}, map[categorizer]string{}
return map[categorizer][]string{}
}
rv := map[categorizer]string{
folderCat: repo.Folder(false),
itemCat: repo.Item(),
result := map[categorizer][]string{
folderCat: {repo.Folder(false)},
itemCat: {repo.Item(), ent.ShortRef},
}
lv := map[categorizer]string{}
if location != nil {
lv = map[categorizer]string{
folderCat: location.Folder(false),
itemCat: location.Item(),
}
if len(ent.LocationRef) > 0 {
result[folderCat] = append(result[folderCat], ent.LocationRef)
}
return rv, lv
return result
}
// pathKeys returns the path keys recognized by the receiver's leaf type.

54
src/pkg/selectors/exchange_test.go
View File

@ -1,6 +1,7 @@
package selectors
import (
"strings"
"testing"
"time"
@ -716,9 +717,14 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
var (
repo = stubPath(suite.T(), usr, []string{fID1, fID2, mail}, path.EmailCategory)
loc = stubPath(suite.T(), usr, []string{fld1, fld2, mail}, path.EmailCategory)
loc = strings.Join([]string{fld1, fld2, mail}, "/")
short = "thisisahashofsomekind"
es = NewExchangeRestore(Any())
ent = details.DetailsEntry{
RepoRef: repo.String(),
ShortRef: short,
LocationRef: loc,
}
)
table := []struct {
@ -758,12 +764,12 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
scopes := setScopesToDefault(test.scope)
var aMatch bool
for _, scope := range scopes {
repoVals, locVals := ExchangeMail.pathValues(repo, loc)
if matchesPathValues(scope, ExchangeMail, repoVals, short) {
pvs := ExchangeMail.pathValues(repo, ent)
if matchesPathValues(scope, ExchangeMail, pvs) {
aMatch = true
break
}
if matchesPathValues(scope, ExchangeMail, locVals, short) {
if matchesPathValues(scope, ExchangeMail, pvs) {
aMatch = true
break
}
@ -1313,7 +1319,10 @@ func (suite *ExchangeSelectorSuite) TestPasses() {
mail = setScopesToDefault(es.Mails(Any(), []string{mid}))
noMail = setScopesToDefault(es.Mails(Any(), None()))
allMail = setScopesToDefault(es.Mails(Any(), Any()))
pth = stubPath(suite.T(), "user", []string{"folder", mid}, path.EmailCategory)
repo = stubPath(suite.T(), "user", []string{"folder", mid}, path.EmailCategory)
ent = details.DetailsEntry{
RepoRef: repo.String(),
}
)
table := []struct {
@ -1336,12 +1345,11 @@ func (suite *ExchangeSelectorSuite) TestPasses() {
suite.Run(test.name, func() {
t := suite.T()
repoVals, locVals := cat.pathValues(pth, pth)
pvs := cat.pathValues(repo, ent)
result := passes(
cat,
repoVals,
locVals,
pvs,
entry,
test.excludes,
test.filters,
@ -1447,25 +1455,25 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() {
t := suite.T()
contactPath := stubPath(t, "user", []string{"cfolder", "contactitem"}, path.ContactsCategory)
contactMap := map[categorizer]string{
ExchangeContactFolder: contactPath.Folder(false),
ExchangeContact: contactPath.Item(),
contactMap := map[categorizer][]string{
ExchangeContactFolder: {contactPath.Folder(false)},
ExchangeContact: {contactPath.Item(), "short"},
}
eventPath := stubPath(t, "user", []string{"ecalendar", "eventitem"}, path.EventsCategory)
eventMap := map[categorizer]string{
ExchangeEventCalendar: eventPath.Folder(false),
ExchangeEvent: eventPath.Item(),
eventMap := map[categorizer][]string{
ExchangeEventCalendar: {eventPath.Folder(false)},
ExchangeEvent: {eventPath.Item(), "short"},
}
mailPath := stubPath(t, "user", []string{"mfolder", "mailitem"}, path.EmailCategory)
mailMap := map[categorizer]string{
ExchangeMailFolder: mailPath.Folder(false),
ExchangeMail: mailPath.Item(),
mailMap := map[categorizer][]string{
ExchangeMailFolder: {mailPath.Folder(false)},
ExchangeMail: {mailPath.Item(), "short"},
}
table := []struct {
cat exchangeCategory
path path.Path
expect map[categorizer]string
expect map[categorizer][]string
}{
{ExchangeContact, contactPath, contactMap},
{ExchangeEvent, eventPath, eventMap},
@ -1473,9 +1481,13 @@ func (suite *ExchangeSelectorSuite) TestExchangeCategory_PathValues() {
}
for _, test := range table {
suite.T().Run(string(test.cat), func(t *testing.T) {
r, l := test.cat.pathValues(test.path, test.path)
assert.Equal(t, test.expect, r)
assert.Equal(t, test.expect, l)
ent := details.DetailsEntry{
RepoRef: test.path.String(),
ShortRef: "short",
}
pvs := test.cat.pathValues(test.path, ent)
assert.Equal(t, test.expect, pvs)
})
}
}

18
src/pkg/selectors/helpers_test.go
View File

@ -55,13 +55,11 @@ func (mc mockCategorizer) isLeaf() bool {
return mc == leafCatStub
}
func (mc mockCategorizer) pathValues(repo, location path.Path) (map[categorizer]string, map[categorizer]string) {
pv := map[categorizer]string{
rootCatStub: "root",
leafCatStub: "leaf",
func (mc mockCategorizer) pathValues(repo path.Path, ent details.DetailsEntry) map[categorizer][]string {
return map[categorizer][]string{
rootCatStub: {"root"},
leafCatStub: {"leaf"},
}
return pv, pv
}
func (mc mockCategorizer) pathKeys() []categorizer {
@ -77,10 +75,10 @@ func (mc mockCategorizer) PathType() path.CategoryType {
}
}
func stubPathValues() map[categorizer]string {
return map[categorizer]string{
rootCatStub: rootCatStub.String(),
leafCatStub: leafCatStub.String(),
func stubPathValues() map[categorizer][]string {
return map[categorizer][]string{
rootCatStub: {rootCatStub.String()},
leafCatStub: {leafCatStub.String()},
}
}

25
src/pkg/selectors/onedrive.go
View File

@ -65,7 +65,7 @@ func (s Selector) ToOneDriveBackup() (*OneDriveBackup, error) {
}
func (s OneDriveBackup) SplitByResourceOwner(users []string) []OneDriveBackup {
sels := splitByResourceOwner[ExchangeScope](s.Selector, users, OneDriveUser)
sels := splitByResourceOwner[OneDriveScope](s.Selector, users, OneDriveUser)
ss := make([]OneDriveBackup, 0, len(sels))
for _, sel := range sels {
@ -99,7 +99,7 @@ func (s Selector) ToOneDriveRestore() (*OneDriveRestore, error) {
}
func (s OneDriveRestore) SplitByResourceOwner(users []string) []OneDriveRestore {
sels := splitByResourceOwner[ExchangeScope](s.Selector, users, ExchangeUser)
sels := splitByResourceOwner[OneDriveScope](s.Selector, users, OneDriveUser)
ss := make([]OneDriveRestore, 0, len(sels))
for _, sel := range sels {
@ -376,25 +376,20 @@ func (c oneDriveCategory) isLeaf() bool {
// Example:
// [tenantID, service, userPN, category, folder, fileID]
// => {odFolder: folder, odFileID: fileID}
func (c oneDriveCategory) pathValues(repo, location path.Path) (map[categorizer]string, map[categorizer]string) {
func (c oneDriveCategory) pathValues(repo path.Path, ent details.DetailsEntry) map[categorizer][]string {
// Ignore `drives/<driveID>/root:` for folder comparison
rFld := path.Builder{}.Append(repo.Folders()...).PopFront().PopFront().PopFront().String()
rv := map[categorizer]string{
OneDriveFolder: rFld,
OneDriveItem: repo.Item(),
result := map[categorizer][]string{
OneDriveFolder: {rFld},
OneDriveItem: {repo.Item(), ent.ShortRef},
}
lv := map[categorizer]string{}
if location != nil {
lFld := path.Builder{}.Append(location.Folders()...).PopFront().PopFront().PopFront().String()
lv = map[categorizer]string{
OneDriveFolder: lFld,
OneDriveItem: location.Item(),
}
if len(ent.LocationRef) > 0 {
result[OneDriveFolder] = append(result[OneDriveFolder], ent.LocationRef)
}
return rv, lv
return result
}
// pathKeys returns the path keys recognized by the receiver's leaf type.

14
src/pkg/selectors/onedrive_test.go
View File

@ -261,14 +261,18 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
filePath, err := pathBuilder.ToDataLayerOneDrivePath("tenant", "user", true)
require.NoError(t, err)
expected := map[categorizer]string{
OneDriveFolder: "dir1/dir2",
OneDriveItem: "file",
expected := map[categorizer][]string{
OneDriveFolder: {"dir1/dir2"},
OneDriveItem: {"file", "short"},
}
r, l := OneDriveItem.pathValues(filePath, filePath)
ent := details.DetailsEntry{
RepoRef: filePath.String(),
ShortRef: "short",
}
r := OneDriveItem.pathValues(filePath, ent)
assert.Equal(t, expected, r)
assert.Equal(t, expected, l)
}
func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() {

96
src/pkg/selectors/scopes.go
View File

@ -88,7 +88,7 @@ type (
// folderCat: folder,
// itemCat: itemID,
// }
pathValues(path.Path, path.Path) (map[categorizer]string, map[categorizer]string)
pathValues(path.Path, details.DetailsEntry) map[categorizer][]string
// pathKeys produces a list of categorizers that can be used as keys in the pathValues
// map. The combination of the two funcs generically interprets the context of the
@ -212,6 +212,20 @@ func matches[T scopeT, C categoryT](s T, cat C, inpt string) bool {
return s[cat.String()].Compare(inpt)
}
// matchesAny returns true if the category is included in the scope's
// data type, and any one of the input strings passes the scope's filter.
func matchesAny[T scopeT, C categoryT](s T, cat C, inpts []string) bool {
if !typeAndCategoryMatches(cat, s.categorizer()) {
return false
}
if len(inpts) == 0 {
return false
}
return s[cat.String()].CompareAny(inpts...)
}
// getCategory returns the scope's category value.
// if s is a filter-type scope, returns the filter category.
func getCategory[T scopeT](s T) string {
@ -297,6 +311,8 @@ func reduce[T scopeT, C categoryT](
return nil
}
el := errs.Local()
// if a DiscreteOwner is specified, only match details for that owner.
matchesResourceOwner := s.ResourceOwners
if len(s.DiscreteOwner) > 0 {
@ -314,35 +330,10 @@ func reduce[T scopeT, C categoryT](
for _, ent := range deets.Items() {
repoPath, err := path.FromDataLayerPath(ent.RepoRef, true)
if err != nil {
errs.AddRecoverable(clues.Wrap(err, "transforming repoRef to path").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "transforming repoRef to path").WithClues(ctx))
continue
}
var locationPath path.Path
// if the details entry has a locationRef specified, use those folders in place
// of the repoRef folders, so that scopes can match against the display names
// instead of container IDs.
if len(ent.LocationRef) > 0 {
pb, err := path.Builder{}.SplitUnescapeAppend(ent.LocationRef)
if err != nil {
errs.AddRecoverable(clues.Wrap(err, "transforming locationRef to path").WithClues(ctx))
continue
}
locationPath, err = pb.Append(repoPath.Item()).
ToDataLayerPath(
repoPath.Tenant(),
repoPath.ResourceOwner(),
repoPath.Service(),
repoPath.Category(),
true)
if err != nil {
errs.AddRecoverable(clues.Wrap(err, "transforming locationRef to path").WithClues(ctx))
continue
}
}
// first check, every entry needs to match the selector's resource owners.
if !matchesResourceOwner.Compare(repoPath.ResourceOwner()) {
continue
@ -360,9 +351,9 @@ func reduce[T scopeT, C categoryT](
continue
}
rv, lv := dc.pathValues(repoPath, locationPath)
pv := dc.pathValues(repoPath, *ent)
passed := passes(dc, rv, lv, *ent, e, f, i)
passed := passes(dc, pv, *ent, e, f, i)
if passed {
ents = append(ents, *ent)
}
@ -407,7 +398,7 @@ func scopesByCategory[T scopeT, C categoryT](
// if the path is included, passes filters, and not excluded.
func passes[T scopeT, C categoryT](
cat C,
repoValues, locationValues map[categorizer]string,
pathValues map[categorizer][]string,
entry details.DetailsEntry,
excs, filts, incs []T,
) bool {
@ -423,7 +414,7 @@ func passes[T scopeT, C categoryT](
var included bool
for _, inc := range incs {
if matchesEntry(inc, cat, repoValues, locationValues, entry) {
if matchesEntry(inc, cat, pathValues, entry) {
included = true
break
}
@ -436,14 +427,14 @@ func passes[T scopeT, C categoryT](
// all filters must pass
for _, filt := range filts {
if !matchesEntry(filt, cat, repoValues, locationValues, entry) {
if !matchesEntry(filt, cat, pathValues, entry) {
return false
}
}
// any matching exclusion means failure
for _, exc := range excs {
if matchesEntry(exc, cat, repoValues, locationValues, entry) {
if matchesEntry(exc, cat, pathValues, entry) {
return false
}
}
@ -456,7 +447,7 @@ func passes[T scopeT, C categoryT](
func matchesEntry[T scopeT, C categoryT](
sc T,
cat C,
repoValues, locationValues map[categorizer]string,
pathValues map[categorizer][]string,
entry details.DetailsEntry,
) bool {
// filterCategory requires matching against service-specific info values
@ -464,11 +455,7 @@ func matchesEntry[T scopeT, C categoryT](
return sc.matchesInfo(entry.ItemInfo)
}
if len(locationValues) > 0 && matchesPathValues(sc, cat, locationValues, entry.ShortRef) {
return true
}
return matchesPathValues(sc, cat, repoValues, entry.ShortRef)
return matchesPathValues(sc, cat, pathValues)
}
// matchesPathValues will check whether the pathValues have matching entries
@ -479,8 +466,7 @@ func matchesEntry[T scopeT, C categoryT](
func matchesPathValues[T scopeT, C categoryT](
sc T,
cat C,
pathValues map[categorizer]string,
shortRef string,
pathValues map[categorizer][]string,
) bool {
for _, c := range cat.pathKeys() {
// resourceOwners are now checked at the beginning of the reduction.
@ -488,12 +474,6 @@ func matchesPathValues[T scopeT, C categoryT](
continue
}
// the pathValues must have an entry for the given categorizer
pathVal, ok := pathValues[c]
if !ok {
return false
}
cc := c.(C)
if isNoneTarget(sc, cc) {
@ -505,23 +485,13 @@ func matchesPathValues[T scopeT, C categoryT](
continue
}
var (
match bool
isLeaf = c.isLeaf()
)
switch {
// Leaf category - the scope can match either the path value (the item ID itself),
// or the shortRef hash representing the item.
case isLeaf && len(shortRef) > 0:
match = matches(sc, cc, pathVal) || matches(sc, cc, shortRef)
// all other categories (root, folder, etc) just need to pass the filter
default:
match = matches(sc, cc, pathVal)
// the pathValues must have an entry for the given categorizer
pathVals, ok := pathValues[c]
if !ok || len(pathVals) == 0 {
return false
}
if !match {
if !matchesAny(sc, cc, pathVals) {
return false
}
}
@ -530,7 +500,7 @@ func matchesPathValues[T scopeT, C categoryT](
}
// ---------------------------------------------------------------------------
// categorizer funcs
// helper funcs
// ---------------------------------------------------------------------------
// categoryMatches returns true if:

20
src/pkg/selectors/scopes_test.go
View File

@ -354,10 +354,14 @@ func (suite *SelectorScopesSuite) TestScopesByCategory() {
}
func (suite *SelectorScopesSuite) TestPasses() {
cat := rootCatStub
pth := stubPath(suite.T(), "uid", []string{"fld"}, path.EventsCategory)
repoVals, locVals := cat.pathValues(pth, pth)
entry := details.DetailsEntry{}
var (
cat = rootCatStub
pth = stubPath(suite.T(), "uid", []string{"fld"}, path.EventsCategory)
entry = details.DetailsEntry{
RepoRef: pth.String(),
}
pvs = cat.pathValues(pth, entry)
)
for _, test := range reduceTestTable {
suite.Run(test.name, func() {
@ -369,8 +373,7 @@ func (suite *SelectorScopesSuite) TestPasses() {
incl := toMockScope(sel.Includes)
result := passes(
cat,
repoVals,
locVals,
pvs,
entry,
excl, filt, incl)
test.expectPasses(t, result)
@ -394,7 +397,6 @@ func toMockScope(sc []scope) []mockScope {
func (suite *SelectorScopesSuite) TestMatchesPathValues() {
cat := rootCatStub
pvs := stubPathValues()
short := "brunheelda"
table := []struct {
@ -440,12 +442,14 @@ func (suite *SelectorScopesSuite) TestMatchesPathValues() {
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
pvs := stubPathValues()
pvs[leafCatStub] = append(pvs[leafCatStub], test.shortRef)
sc := stubScope("")
sc[rootCatStub.String()] = filterize(scopeConfig{}, test.rootVal)
sc[leafCatStub.String()] = filterize(scopeConfig{}, test.leafVal)
test.expect(t, matchesPathValues(sc, cat, pvs, test.shortRef))
test.expect(t, matchesPathValues(sc, cat, pvs))
})
}
}

27
src/pkg/selectors/sharepoint.go
View File

@ -65,7 +65,7 @@ func (s Selector) ToSharePointBackup() (*SharePointBackup, error) {
}
func (s SharePointBackup) SplitByResourceOwner(sites []string) []SharePointBackup {
sels := splitByResourceOwner[ExchangeScope](s.Selector, sites, SharePointSite)
sels := splitByResourceOwner[SharePointScope](s.Selector, sites, SharePointSite)
ss := make([]SharePointBackup, 0, len(sels))
for _, sel := range sels {
@ -98,8 +98,8 @@ func (s Selector) ToSharePointRestore() (*SharePointRestore, error) {
return &src, nil
}
func (s SharePointRestore) SplitByResourceOwner(users []string) []SharePointRestore {
sels := splitByResourceOwner[ExchangeScope](s.Selector, users, ExchangeUser)
func (s SharePointRestore) SplitByResourceOwner(sites []string) []SharePointRestore {
sels := splitByResourceOwner[SharePointScope](s.Selector, sites, SharePointSite)
ss := make([]SharePointRestore, 0, len(sels))
for _, sel := range sels {
@ -476,7 +476,7 @@ func (c sharePointCategory) isLeaf() bool {
// Example:
// [tenantID, service, siteID, category, folder, itemID]
// => {spFolder: folder, spItemID: itemID}
func (c sharePointCategory) pathValues(repo, location path.Path) (map[categorizer]string, map[categorizer]string) {
func (c sharePointCategory) pathValues(repo path.Path, ent details.DetailsEntry) map[categorizer][]string {
var folderCat, itemCat categorizer
switch c {
@ -487,24 +487,19 @@ func (c sharePointCategory) pathValues(repo, location path.Path) (map[categorize
case SharePointPage, SharePointPageFolder:
folderCat, itemCat = SharePointPageFolder, SharePointPage
default:
return map[categorizer]string{}, map[categorizer]string{}
return map[categorizer][]string{}
}
rv := map[categorizer]string{
folderCat: repo.Folder(false),
itemCat: repo.Item(),
result := map[categorizer][]string{
folderCat: {repo.Folder(false)},
itemCat: {repo.Item(), ent.ShortRef},
}
lv := map[categorizer]string{}
if location != nil {
lv = map[categorizer]string{
folderCat: location.Folder(false),
itemCat: location.Item(),
}
if len(ent.LocationRef) > 0 {
result[folderCat] = append(result[folderCat], ent.LocationRef)
}
return rv, lv
return result
}
// pathKeys returns the path keys recognized by the receiver's leaf type.

25
src/pkg/selectors/sharepoint_test.go
View File

@ -326,22 +326,22 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
table := []struct {
name string
sc sharePointCategory
expected map[categorizer]string
expected map[categorizer][]string
}{
{
name: "SharePoint Libraries",
sc: SharePointLibraryItem,
expected: map[categorizer]string{
SharePointLibrary: "dir1/dir2",
SharePointLibraryItem: "item",
expected: map[categorizer][]string{
SharePointLibrary: {"dir1/dir2"},
SharePointLibraryItem: {"item", "short"},
},
},
{
name: "SharePoint Lists",
sc: SharePointListItem,
expected: map[categorizer]string{
SharePointList: "dir1/dir2",
SharePointListItem: "item",
expected: map[categorizer][]string{
SharePointList: {"dir1/dir2"},
SharePointListItem: {"item", "short"},
},
},
}
@ -356,9 +356,14 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
test.sc.PathType(),
true)
require.NoError(t, err)
r, l := test.sc.pathValues(itemPath, itemPath)
assert.Equal(t, test.expected, r)
assert.Equal(t, test.expected, l)
ent := details.DetailsEntry{
RepoRef: itemPath.String(),
ShortRef: "short",
}
pv := test.sc.pathValues(itemPath, ent)
assert.Equal(t, test.expected, pv)
})
}
}