sharepoint webURL selector scopes (#1665)

## Description

Adds webUrl scopes to the sharepoint selector.
Also introduces the idea of a scope categories
that can broadly match across all leaf types
within a service.  This category union should be
reserved for root categories, and properties
that can be used interchangably with the root.

This is part 2 of exposing webURLs as an alternative
to siteIDs for sharepoint backup and restore.  The
next change providing a webURL => siteID lookup
within the graph package.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1616

## Test Plan

- [x] ⚡ Unit test

src/pkg/selectors/exchange.go

@@ -556,6 +556,11 @@ func (ec exchangeCategory) unknownCat() categorizer {
 	return ExchangeCategoryUnknown
 }
 
+// isUnion returns true if c is a user
+func (ec exchangeCategory) isUnion() bool {
+	return ec == ec.rootCat()
+}
+
 // isLeaf is true if the category is a mail, event, or contact category.
 func (ec exchangeCategory) isLeaf() bool {
 	return ec == ec.leafCat()

src/pkg/selectors/exchange_test.go

@@ -588,7 +588,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_IncludesCategory() {
 		check  assert.BoolAssertionFunc
 	}{
 		{ExchangeCategoryUnknown, ExchangeCategoryUnknown, assert.False},
-		{ExchangeCategoryUnknown, ExchangeUser, assert.False},
+		{ExchangeCategoryUnknown, ExchangeUser, assert.True},
 		{ExchangeContact, ExchangeContactFolder, assert.True},
 		{ExchangeContact, ExchangeMailFolder, assert.False},
 		{ExchangeContactFolder, ExchangeContact, assert.True},
@@ -608,7 +608,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_IncludesCategory() {
 		{ExchangeMailFolder, ExchangeContactFolder, assert.False},
 		{ExchangeMailFolder, ExchangeEventCalendar, assert.False},
 		{ExchangeUser, ExchangeUser, assert.True},
-		{ExchangeUser, ExchangeCategoryUnknown, assert.False},
+		{ExchangeUser, ExchangeCategoryUnknown, assert.True},
 		{ExchangeUser, ExchangeMail, assert.True},
 		{ExchangeUser, ExchangeEventCalendar, assert.True},
 	}

src/pkg/selectors/helpers_test.go

@@ -47,6 +47,10 @@ func (mc mockCategorizer) unknownCat() categorizer {
 	return unknownCatStub
 }
 
+func (mc mockCategorizer) isUnion() bool {
+	return mc == rootCatStub
+}
+
 func (mc mockCategorizer) isLeaf() bool {
 	return mc == leafCatStub
 }

src/pkg/selectors/onedrive.go

@@ -351,6 +351,11 @@ func (c oneDriveCategory) unknownCat() categorizer {
 	return OneDriveCategoryUnknown
 }
 
+// isUnion returns true if c is a user
+func (c oneDriveCategory) isUnion() bool {
+	return c == c.rootCat()
+}
+
 // isLeaf is true if the category is a OneDriveItem category.
 func (c oneDriveCategory) isLeaf() bool {
 	// return c == c.leafCat()??

src/pkg/selectors/scopes.go

@@ -63,9 +63,13 @@ type (
 		// rootCat returns the root category for the categorizer
 		rootCat() categorizer
 
-		// unknownType returns the unknown category value
+		// unknownCat returns the unknown category value
 		unknownCat() categorizer
 
+		// isUnion returns true if the category can be used to match against any leaf category.
+		// This can occur when an itemInfo property is used as an alternative resourceOwner id.
+		isUnion() bool
+
 		// isLeaf returns true if the category is one of the leaf categories.
 		// eg: in a resourceOwner/folder/item structure, the item is the leaf.
 		isLeaf() bool
@@ -487,6 +491,10 @@ func matchesPathValues[T scopeT, C categoryT](
 // - either type is the root type
 // - the leaf types match
 func categoryMatches[C categoryT](a, b C) bool {
+	if a.isUnion() || b.isUnion() {
+		return true
+	}
+
 	u := a.unknownCat()
 	if a == u || b == u {
 		return false

src/pkg/selectors/scopes_test.go

@@ -273,7 +273,7 @@ func (suite *SelectorScopesSuite) TestScopesByCategory() {
 		},
 		false)
 	assert.Len(t, result, 1)
-	assert.Len(t, result[rootCatStub], 1)
+	assert.Len(t, result[rootCatStub], 2)
 	assert.Empty(t, result[leafCatStub])
 }
 

src/pkg/selectors/selectors.go

@@ -395,6 +395,7 @@ func resourceOwnersIn(s []scope, rootCat string) []string {
 type scopeConfig struct {
 	usePathFilter   bool
 	usePrefixFilter bool
+	useSuffixFilter bool
 }
 
 type option func(*scopeConfig)
@@ -414,6 +415,15 @@ func PrefixMatch() option {
 	}
 }
 
+// SuffixMatch ensures the selector uses a Suffix comparator, instead
+// of contains or equals.  Will not override a default Any() or None()
+// comparator.
+func SuffixMatch() option {
+	return func(sc *scopeConfig) {
+		sc.useSuffixFilter = true
+	}
+}
+
 // pathType is an internal-facing option.  It is assumed that scope
 // constructors will provide the pathType option whenever a folder-
 // level scope (ie, a scope that compares path hierarchies) is created.
@@ -480,6 +490,10 @@ func filterize(sc scopeConfig, s ...string) filters.Filter {
 			return filters.PathPrefix(s)
 		}
 
+		if sc.useSuffixFilter {
+			return filters.PathSuffix(s)
+		}
+
 		return filters.PathContains(s)
 	}
 
@@ -487,6 +501,10 @@ func filterize(sc scopeConfig, s ...string) filters.Filter {
 		return filters.Prefix(join(s...))
 	}
 
+	if sc.useSuffixFilter {
+		return filters.Suffix(join(s...))
+	}
+
 	if len(s) == 1 {
 		return filters.Equal(s[0])
 	}
@@ -496,6 +514,24 @@ func filterize(sc scopeConfig, s ...string) filters.Filter {
 
 type filterFunc func(string) filters.Filter
 
+// pathFilterFactory returns the appropriate path filter
+// (contains, prefix, or suffix) for the provided options.
+// If multiple options are flagged, Prefix takes priority.
+// If no options are provided, returns PathContains.
+func pathFilterFactory(opts ...option) func([]string) filters.Filter {
+	sc := &scopeConfig{}
+	sc.populate(opts...)
+
+	switch true {
+	case sc.usePrefixFilter:
+		return filters.PathPrefix
+	case sc.useSuffixFilter:
+		return filters.PathSuffix
+	default:
+		return filters.PathContains
+	}
+}
+
 // wrapFilter produces a func that filterizes the input by:
 // - cleans the input string
 // - normalizes the cleaned input (returns anyFail if empty, allFail if *)

src/pkg/selectors/sharepoint.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"github.com/alcionai/corso/src/pkg/backup/details"
-	"github.com/alcionai/corso/src/pkg/filters"
 	"github.com/alcionai/corso/src/pkg/path"
 )
 
@@ -175,6 +174,22 @@ func (s *sharePoint) DiscreteScopes(siteIDs []string) []SharePointScope {
 // -------------------
 // Scope Factories
 
+// Produces one or more SharePoint webURL scopes.
+// One scope is created per webURL entry.
+// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
+// If any slice contains selectors.None, that slice is reduced to [selectors.None]
+// If any slice is empty, it defaults to [selectors.None]
+func (s *SharePointRestore) WebURL(urlSuffixes []string, opts ...option) []SharePointScope {
+	return []SharePointScope{
+		makeFilterScope[SharePointScope](
+			SharePointLibraryItem,
+			SharePointWebURL,
+			urlSuffixes,
+			pathFilterFactory(opts...)),
+		// TODO: list scope
+	}
+}
+
 // Produces one or more SharePoint site scopes.
 // One scope is created per site entry.
 // If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
@@ -226,20 +241,6 @@ func (s *sharePoint) LibraryItems(sites, libraries, items []string, opts ...opti
 // -------------------
 // Filter Factories
 
-// WebURL produces a SharePoint item webURL filter scope.
-// Matches any item where the webURL contains the substring.
-// If the input equals selectors.Any, the scope will match all times.
-// If the input is empty or selectors.None, the scope will always fail comparisons.
-func (s *sharePoint) WebURL(substring string) []SharePointScope {
-	return []SharePointScope{
-		makeFilterScope[SharePointScope](
-			SharePointLibraryItem,
-			SharePointFilterWebURL,
-			[]string{substring},
-			wrapFilter(filters.Less)),
-	}
-}
-
 // ---------------------------------------------------------------------------
 // Categories
 // ---------------------------------------------------------------------------
@@ -253,13 +254,14 @@ var _ categorizer = SharePointCategoryUnknown
 
 const (
 	SharePointCategoryUnknown sharePointCategory = ""
+
 	// types of data identified by SharePoint
+	SharePointWebURL      sharePointCategory = "SharePointWebURL"
 	SharePointSite        sharePointCategory = "SharePointSite"
 	SharePointLibrary     sharePointCategory = "SharePointLibrary"
 	SharePointLibraryItem sharePointCategory = "SharePointLibraryItem"
 
 	// filterable topics identified by SharePoint
-	SharePointFilterWebURL sharePointCategory = "SharePointFilterWebURL"
 )
 
 // sharePointLeafProperties describes common metadata of the leaf categories
@@ -285,8 +287,7 @@ func (c sharePointCategory) String() string {
 // Ex: ServiceUser.leafCat() => ServiceUser
 func (c sharePointCategory) leafCat() categorizer {
 	switch c {
-	case SharePointLibrary, SharePointLibraryItem,
+	case SharePointLibrary, SharePointLibraryItem:
-		SharePointFilterWebURL:
 		return SharePointLibraryItem
 	}
 
@@ -303,6 +304,12 @@ func (c sharePointCategory) unknownCat() categorizer {
 	return SharePointCategoryUnknown
 }
 
+// isUnion returns true if the category is a site or a webURL, which
+// can act as an alternative identifier to siteID across all site types.
+func (c sharePointCategory) isUnion() bool {
+	return c == SharePointWebURL || c == c.rootCat()
+}
+
 // isLeaf is true if the category is a SharePointItem category.
 func (c sharePointCategory) isLeaf() bool {
 	return c == c.leafCat()
@@ -434,21 +441,20 @@ func (s sharePoint) Reduce(ctx context.Context, deets *details.Details) *details
 // matchesInfo handles the standard behavior when comparing a scope and an sharePointInfo
 // returns true if the scope and info match for the provided category.
 func (s SharePointScope) matchesInfo(dii details.ItemInfo) bool {
-	// info := dii.SharePoint
-	// if info == nil {
-	// 	return false
-	// }
 	var (
 		filterCat = s.FilterCategory()
 		i         = ""
+		info      = dii.SharePoint
 	)
 
-	// switch filterCat {
+	if info == nil {
-	// case FileFilterCreatedAfter, FileFilterCreatedBefore:
+		return false
-	// 	i = common.FormatTime(info.Created)
+	}
-	// case FileFilterModifiedAfter, FileFilterModifiedBefore:
+
-	// 	i = common.FormatTime(info.Modified)
+	switch filterCat {
-	// }
+	case SharePointWebURL:
+		i = info.WebURL
+	}
 
 	return s.Matches(filterCat, i)
 }

src/pkg/selectors/sharepoint_test.go

@@ -118,6 +118,50 @@ func (suite *SharePointSelectorSuite) TestSharePointSelector_Sites() {
 	}
 }
 
+func (suite *SharePointSelectorSuite) TestSharePointSelector_Include_WebURLs() {
+	t := suite.T()
+	sel := NewSharePointRestore()
+
+	const (
+		s1 = "s1"
+		s2 = "s2"
+	)
+
+	sel.Include(sel.WebURL([]string{s1, s2}))
+	scopes := sel.Includes
+	require.Len(t, scopes, 1)
+
+	for _, sc := range scopes {
+		scopeMustHave(
+			t,
+			SharePointScope(sc),
+			map[categorizer]string{SharePointWebURL: join(s1, s2)},
+		)
+	}
+}
+
+func (suite *SharePointSelectorSuite) TestSharePointSelector_Exclude_WebURLs() {
+	t := suite.T()
+	sel := NewSharePointRestore()
+
+	const (
+		s1 = "s1"
+		s2 = "s2"
+	)
+
+	sel.Exclude(sel.WebURL([]string{s1, s2}))
+	scopes := sel.Excludes
+	require.Len(t, scopes, 1)
+
+	for _, sc := range scopes {
+		scopeMustHave(
+			t,
+			SharePointScope(sc),
+			map[categorizer]string{SharePointWebURL: join(s1, s2)},
+		)
+	}
+}
+
 func (suite *SharePointSelectorSuite) TestSharePointSelector_Include_Sites() {
 	t := suite.T()
 	sel := NewSharePointBackup()
@@ -288,28 +332,39 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
 }
 
 func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
-	ods := NewSharePointRestore()
+	var (
-
+		ods  = NewSharePointRestore()
-	// var url = "www.website.com"
+		host = "www.website.com"
-
+		pth  = "/foo"
-	itemInfo := details.ItemInfo{
+		url  = host + pth
-		SharePoint: &details.SharePointInfo{
+	)
-			ItemType: details.SharePointItem,
-			// WebURL:   "www.website.com",
-		},
-	}
 
 	table := []struct {
-		name   string
+		name    string
-		scope  []SharePointScope
+		infoURL string
-		expect assert.BoolAssertionFunc
+		scope   []SharePointScope
+		expect  assert.BoolAssertionFunc
 	}{
-		// {"item webURL match", ods.WebURL(url), assert.True},
+		{"host match", host, ods.WebURL([]string{host}), assert.True},
-		// {"item webURL substring", ods.WebURL("website"), assert.True},
+		{"url match", url, ods.WebURL([]string{url}), assert.True},
-		{"item webURL mismatch", ods.WebURL("google"), assert.False},
+		{"url contains host", url, ods.WebURL([]string{host}), assert.True},
+		{"host suffixes host", host, ods.WebURL([]string{host}, SuffixMatch()), assert.True},
+		{"url does not suffix host", url, ods.WebURL([]string{host}, SuffixMatch()), assert.False},
+		{"url contains path", url, ods.WebURL([]string{pth}), assert.True},
+		{"url has path suffix", url, ods.WebURL([]string{pth}, SuffixMatch()), assert.True},
+		{"host does not contain substring", host, ods.WebURL([]string{"website"}), assert.False},
+		{"url does not suffix substring", url, ods.WebURL([]string{"oo"}), assert.False},
+		{"host mismatch", host, ods.WebURL([]string{"www.google.com"}), assert.False},
 	}
 	for _, test := range table {
 		suite.T().Run(test.name, func(t *testing.T) {
+			itemInfo := details.ItemInfo{
+				SharePoint: &details.SharePointInfo{
+					ItemType: details.SharePointItem,
+					WebURL:   test.infoURL,
+				},
+			}
+
 			scopes := setScopesToDefault(test.scope)
 			for _, scope := range scopes {
 				test.expect(t, scope.matchesInfo(itemInfo))
@@ -324,10 +379,10 @@ func (suite *SharePointSelectorSuite) TestCategory_PathType() {
 		pathType path.CategoryType
 	}{
 		{SharePointCategoryUnknown, path.UnknownCategory},
+		{SharePointWebURL, path.UnknownCategory},
 		{SharePointSite, path.UnknownCategory},
 		{SharePointLibrary, path.LibrariesCategory},
 		{SharePointLibraryItem, path.LibrariesCategory},
-		{SharePointFilterWebURL, path.LibrariesCategory},
 	}
 	for _, test := range table {
 		suite.T().Run(test.cat.String(), func(t *testing.T) {