larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add groups selectors for channels and messages (#4071)

Browse Source 
#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3989

#### Test Plan

- [x] ⚡ Unit test
This commit is contained in:
Keepers 2023-08-21 14:44:24 -06:00 committed by GitHub
parent 90ac62ab14
commit 6a83d23ff2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 489 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/pkg/path/category_type.go
View File

@ -17,15 +17,16 @@ type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = 0
EmailCategory CategoryType = 1 // email
ContactsCategory CategoryType = 2 // contacts
EventsCategory CategoryType = 3 // events
FilesCategory CategoryType = 4 // files
ListsCategory CategoryType = 5 // lists
LibrariesCategory CategoryType = 6 // libraries
PagesCategory CategoryType = 7 // pages
DetailsCategory CategoryType = 8 // details
UnknownCategory CategoryType = 0
EmailCategory CategoryType = 1 // email
ContactsCategory CategoryType = 2 // contacts
EventsCategory CategoryType = 3 // events
FilesCategory CategoryType = 4 // files
ListsCategory CategoryType = 5 // lists
LibrariesCategory CategoryType = 6 // libraries
PagesCategory CategoryType = 7 // pages
DetailsCategory CategoryType = 8 // details
ChannelMessagesCategory CategoryType = 9 // channel messages
)
func ToCategoryType(category string) CategoryType {
@ -48,6 +49,8 @@ func ToCategoryType(category string) CategoryType {
return PagesCategory
case strings.ToLower(DetailsCategory.String()):
return DetailsCategory
case strings.ToLower(ChannelMessagesCategory.String()):
return ChannelMessagesCategory
default:
return UnknownCategory
}
@ -73,6 +76,12 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
ListsCategory: {},
PagesCategory: {},
},
GroupsService: {
ChannelMessagesCategory: {},
},
TeamsService: {
ChannelMessagesCategory: {},
},
}
func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType, error) {

5
src/pkg/path/categorytype_string.go
View File

@ -17,11 +17,12 @@ func _() {
_ = x[LibrariesCategory-6]
_ = x[PagesCategory-7]
_ = x[DetailsCategory-8]
_ = x[ChannelMessagesCategory-9]
}
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetails"
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslistslibrariespagesdetailschannel messages"
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65}
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 44, 53, 58, 65, 81}
func (i CategoryType) String() string {
if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) {

83
src/pkg/selectors/groups.go
View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
)
@ -214,38 +215,42 @@ func (s *groups) AllData() []GroupsScope {
scopes = append(
scopes,
makeScope[GroupsScope](GroupsTODOContainer, Any()))
makeScope[GroupsScope](GroupsChannel, Any()))
return scopes
}
// TODO produces one or more Groups TODO scopes.
// Channel produces one or more SharePoint channel scopes, where the channel
// matches upon a given channel by ID or Name. In order to ensure channel selection
// this should always be embedded within the Filter() set; include(channel()) will
// select all items in the channel without further filtering.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// Any empty slice defaults to [selectors.None]
func (s *groups) TODO(lists []string, opts ...option) []GroupsScope {
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) Channel(channel string) []GroupsScope {
return []GroupsScope{
makeInfoScope[GroupsScope](
GroupsChannel,
GroupsInfoChannel,
[]string{channel},
filters.Equal),
}
}
// ChannelMessages produces one or more Groups channel message scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *sharePoint) ChannelMessages(channels, messages []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(scopes, makeScope[GroupsScope](GroupsTODOContainer, lists, os...))
return scopes
}
// ListTODOItemsItems produces one or more Groups TODO item scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
// options are only applied to the list scopes.
func (s *groups) TODOItems(lists, items []string, opts ...option) []GroupsScope {
scopes := []GroupsScope{}
scopes = append(
scopes,
makeScope[GroupsScope](GroupsTODOItem, items, defaultItemOptions(s.Cfg)...).
set(GroupsTODOContainer, lists, opts...))
makeScope[GroupsScope](GroupsChannelMessage, messages, os...).
set(GroupsChannel, channels, opts...))
return scopes
}
@ -270,21 +275,22 @@ const (
GroupsCategoryUnknown groupsCategory = ""
// types of data in Groups
GroupsGroup groupsCategory = "GroupsGroup"
GroupsTODOContainer groupsCategory = "GroupsTODOContainer"
GroupsTODOItem groupsCategory = "GroupsTODOItem"
GroupsGroup groupsCategory = "GroupsGroup"
GroupsChannel groupsCategory = "GroupsChannel"
GroupsChannelMessage groupsCategory = "GroupsChannelMessage"
// details.itemInfo comparables
// library drive selection
// channel drive selection
GroupsInfoSiteLibraryDrive groupsCategory = "GroupsInfoSiteLibraryDrive"
GroupsInfoChannel groupsCategory = "GroupsInfoChannel"
)
// groupsLeafProperties describes common metadata of the leaf categories
var groupsLeafProperties = map[categorizer]leafProperty{
GroupsTODOItem: { // the root category must be represented, even though it isn't a leaf
pathKeys: []categorizer{GroupsTODOContainer, GroupsTODOItem},
pathType: path.UnknownCategory,
GroupsChannelMessage: { // the root category must be represented, even though it isn't a leaf
pathKeys: []categorizer{GroupsChannel, GroupsChannelMessage},
pathType: path.ChannelMessagesCategory,
},
GroupsGroup: { // the root category must be represented, even though it isn't a leaf
pathKeys: []categorizer{GroupsGroup},
@ -303,8 +309,10 @@ func (c groupsCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser
func (c groupsCategory) leafCat() categorizer {
switch c {
case GroupsTODOContainer, GroupsInfoSiteLibraryDrive:
return GroupsTODOItem
// TODO: if channels ever contain more than one type of item,
// we'll need to fix this up.
case GroupsChannel, GroupsChannelMessage, GroupsInfoSiteLibraryDrive:
return GroupsChannelMessage
}
return c
@ -348,12 +356,12 @@ func (c groupsCategory) pathValues(
)
switch c {
case GroupsTODOContainer, GroupsTODOItem:
case GroupsChannel, GroupsChannelMessage:
if ent.Groups == nil {
return nil, clues.New("no Groups ItemInfo in details")
}
folderCat, itemCat = GroupsTODOContainer, GroupsTODOItem
folderCat, itemCat = GroupsChannel, GroupsChannelMessage
rFld = ent.Groups.ParentPath
default:
@ -451,7 +459,7 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS
os := []option{}
switch cat {
case GroupsTODOContainer:
case GroupsChannel:
os = append(os, pathComparator())
}
@ -462,10 +470,10 @@ func (s GroupsScope) set(cat groupsCategory, v []string, opts ...option) GroupsS
func (s GroupsScope) setDefaults() {
switch s.Category() {
case GroupsGroup:
s[GroupsTODOContainer.String()] = passAny
s[GroupsTODOItem.String()] = passAny
case GroupsTODOContainer:
s[GroupsTODOItem.String()] = passAny
s[GroupsChannel.String()] = passAny
s[GroupsChannelMessage.String()] = passAny
case GroupsChannel:
s[GroupsChannelMessage.String()] = passAny
}
}
@ -485,7 +493,7 @@ func (s groups) Reduce(
deets,
s.Selector,
map[path.CategoryType]groupsCategory{
path.UnknownCategory: GroupsTODOItem,
path.ChannelMessagesCategory: GroupsChannelMessage,
},
errs)
}
@ -516,6 +524,9 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool {
}
return matchesAny(s, GroupsInfoSiteLibraryDrive, ds)
case GroupsInfoChannel:
ds := Any()
return matchesAny(s, GroupsInfoChannel, ds)
}
return s.Matches(infoCat, i)

421
src/pkg/selectors/groups_test.go Normal file
View File

@ -0,0 +1,421 @@
package selectors
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path"
)
type GroupsSelectorSuite struct {
tester.Suite
}
func TestGroupsSelectorSuite(t *testing.T) {
suite.Run(t, &GroupsSelectorSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *GroupsSelectorSuite) TestNewGroupsBackup() {
t := suite.T()
ob := NewGroupsBackup(nil)
assert.Equal(t, ob.Service, ServiceGroups)
assert.NotZero(t, ob.Scopes())
}
func (suite *GroupsSelectorSuite) TestToGroupsBackup() {
t := suite.T()
ob := NewGroupsBackup(nil)
s := ob.Selector
ob, err := s.ToGroupsBackup()
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, ob.Service, ServiceGroups)
assert.NotZero(t, ob.Scopes())
}
func (suite *GroupsSelectorSuite) TestNewGroupsRestore() {
t := suite.T()
or := NewGroupsRestore(nil)
assert.Equal(t, or.Service, ServiceGroups)
assert.NotZero(t, or.Scopes())
}
func (suite *GroupsSelectorSuite) TestToGroupsRestore() {
t := suite.T()
eb := NewGroupsRestore(nil)
s := eb.Selector
or, err := s.ToGroupsRestore()
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, or.Service, ServiceGroups)
assert.NotZero(t, or.Scopes())
}
// TODO(rkeepers): implement
// func (suite *GroupsSelectorSuite) TestGroupsRestore_Reduce() {
// toRR := func(cat path.CategoryType, siteID string, folders []string, item string) string {
// folderElems := make([]string, 0, len(folders))
// for _, f := range folders {
// folderElems = append(folderElems, f+".d")
// }
// return stubRepoRef(
// path.GroupsService,
// cat,
// siteID,
// strings.Join(folderElems, "/"),
// item)
// }
// var (
// prefixElems = []string{
// odConsts.DrivesPathDir,
// "drive!id",
// odConsts.RootPathDir,
// }
// itemElems1 = []string{"folderA", "folderB"}
// itemElems2 = []string{"folderA", "folderC"}
// itemElems3 = []string{"folderD", "folderE"}
// pairAC = "folderA/folderC"
// pairGH = "folderG/folderH"
// item = toRR(
// path.LibrariesCategory,
// "sid",
// append(slices.Clone(prefixElems), itemElems1...),
// "item")
// item2 = toRR(
// path.LibrariesCategory,
// "sid",
// append(slices.Clone(prefixElems), itemElems2...),
// "item2")
// item3 = toRR(
// path.LibrariesCategory,
// "sid",
// append(slices.Clone(prefixElems), itemElems3...),
// "item3")
// item4 = stubRepoRef(path.GroupsService, path.PagesCategory, "sid", pairGH, "item4")
// item5 = stubRepoRef(path.GroupsService, path.PagesCategory, "sid", pairGH, "item5")
// )
// deets := &details.Details{
// DetailsModel: details.DetailsModel{
// Entries: []details.Entry{
// {
// RepoRef: item,
// ItemRef: "item",
// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems1...), "/"),
// ItemInfo: details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsLibrary,
// ItemName: "itemName",
// ParentPath: strings.Join(itemElems1, "/"),
// },
// },
// },
// {
// RepoRef: item2,
// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems2...), "/"),
// // ItemRef intentionally blank to test fallback case
// ItemInfo: details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsLibrary,
// ItemName: "itemName2",
// ParentPath: strings.Join(itemElems2, "/"),
// },
// },
// },
// {
// RepoRef: item3,
// ItemRef: "item3",
// LocationRef: strings.Join(append([]string{odConsts.RootPathDir}, itemElems3...), "/"),
// ItemInfo: details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsLibrary,
// ItemName: "itemName3",
// ParentPath: strings.Join(itemElems3, "/"),
// },
// },
// },
// {
// RepoRef: item4,
// LocationRef: pairGH,
// ItemRef: "item4",
// ItemInfo: details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsPage,
// ItemName: "itemName4",
// ParentPath: pairGH,
// },
// },
// },
// {
// RepoRef: item5,
// LocationRef: pairGH,
// // ItemRef intentionally blank to test fallback case
// ItemInfo: details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsPage,
// ItemName: "itemName5",
// ParentPath: pairGH,
// },
// },
// },
// },
// },
// }
// arr := func(s ...string) []string {
// return s
// }
// table := []struct {
// name string
// makeSelector func() *GroupsRestore
// expect []string
// cfg Config
// }{
// {
// name: "all",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore(Any())
// odr.Include(odr.AllData())
// return odr
// },
// expect: arr(item, item2, item3, item4, item5),
// },
// {
// name: "only match item",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore(Any())
// odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
// return odr
// },
// expect: arr(item2),
// },
// {
// name: "id doesn't match name",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore(Any())
// odr.Include(odr.LibraryItems(Any(), []string{"item2"}))
// return odr
// },
// expect: []string{},
// cfg: Config{OnlyMatchItemNames: true},
// },
// {
// name: "only match item name",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore(Any())
// odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
// return odr
// },
// expect: arr(item2),
// cfg: Config{OnlyMatchItemNames: true},
// },
// {
// name: "name doesn't match",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore(Any())
// odr.Include(odr.LibraryItems(Any(), []string{"itemName2"}))
// return odr
// },
// expect: []string{},
// },
// {
// name: "only match folder",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore([]string{"sid"})
// odr.Include(odr.LibraryFolders([]string{"folderA/folderB", pairAC}))
// return odr
// },
// expect: arr(item, item2),
// },
// {
// name: "pages match folder",
// makeSelector: func() *GroupsRestore {
// odr := NewGroupsRestore([]string{"sid"})
// odr.Include(odr.Pages([]string{pairGH, pairAC}))
// return odr
// },
// expect: arr(item4, item5),
// },
// }
// for _, test := range table {
// suite.Run(test.name, func() {
// t := suite.T()
// ctx, flush := tester.NewContext(t)
// defer flush()
// sel := test.makeSelector()
// sel.Configure(test.cfg)
// results := sel.Reduce(ctx, deets, fault.New(true))
// paths := results.Paths()
// assert.Equal(t, test.expect, paths)
// })
// }
// }
func (suite *GroupsSelectorSuite) TestGroupsCategory_PathValues() {
var (
itemName = "item"
itemID = "item-id"
shortRef = "short"
elems = []string{itemID}
)
table := []struct {
name string
sc groupsCategory
pathElems []string
locRef string
parentPath string
expected map[categorizer][]string
cfg Config
}{
{
name: "Groups Channel Messages",
sc: GroupsChannelMessage,
pathElems: elems,
locRef: "",
expected: map[categorizer][]string{
GroupsChannel: {""},
GroupsChannelMessage: {itemID, shortRef},
},
cfg: Config{},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
itemPath, err := path.Build(
"tenant",
"site",
path.GroupsService,
test.sc.PathType(),
true,
test.pathElems...)
require.NoError(t, err, clues.ToCore(err))
ent := details.Entry{
RepoRef: itemPath.String(),
ShortRef: shortRef,
ItemRef: itemPath.Item(),
LocationRef: test.locRef,
ItemInfo: details.ItemInfo{
Groups: &details.GroupsInfo{
ItemName: itemName,
ParentPath: test.parentPath,
},
},
}
pv, err := test.sc.pathValues(itemPath, ent, test.cfg)
require.NoError(t, err)
assert.Equal(t, test.expected, pv)
})
}
}
// TODO(abin): implement
// func (suite *GroupsSelectorSuite) TestGroupsScope_MatchesInfo() {
// var (
// sel = NewGroupsRestore(Any())
// host = "www.website.com"
// pth = "/foo"
// url = host + pth
// epoch = time.Time{}
// now = time.Now()
// modification = now.Add(15 * time.Minute)
// future = now.Add(45 * time.Minute)
// )
// table := []struct {
// name string
// infoURL string
// scope []GroupsScope
// expect assert.BoolAssertionFunc
// }{
// {"host match", host, sel.WebURL([]string{host}), assert.True},
// {"url match", url, sel.WebURL([]string{url}), assert.True},
// {"host suffixes host", host, sel.WebURL([]string{host}, SuffixMatch()), assert.True},
// {"url does not suffix host", url, sel.WebURL([]string{host}, SuffixMatch()), assert.False},
// {"url has path suffix", url, sel.WebURL([]string{pth}, SuffixMatch()), assert.True},
// {"host does not contain substring", host, sel.WebURL([]string{"website"}), assert.False},
// {"url does not suffix substring", url, sel.WebURL([]string{"oo"}, SuffixMatch()), assert.False},
// {"host mismatch", host, sel.WebURL([]string{"www.google.com"}), assert.False},
// {"file create after the epoch", host, sel.CreatedAfter(dttm.Format(epoch)), assert.True},
// {"file create after now", host, sel.CreatedAfter(dttm.Format(now)), assert.False},
// {"file create after later", url, sel.CreatedAfter(dttm.Format(future)), assert.False},
// {"file create before future", host, sel.CreatedBefore(dttm.Format(future)), assert.True},
// {"file create before now", host, sel.CreatedBefore(dttm.Format(now)), assert.False},
// {"file create before modification", host, sel.CreatedBefore(dttm.Format(modification)), assert.True},
// {"file create before epoch", host, sel.CreatedBefore(dttm.Format(now)), assert.False},
// {"file modified after the epoch", host, sel.ModifiedAfter(dttm.Format(epoch)), assert.True},
// {"file modified after now", host, sel.ModifiedAfter(dttm.Format(now)), assert.True},
// {"file modified after later", host, sel.ModifiedAfter(dttm.Format(future)), assert.False},
// {"file modified before future", host, sel.ModifiedBefore(dttm.Format(future)), assert.True},
// {"file modified before now", host, sel.ModifiedBefore(dttm.Format(now)), assert.False},
// {"file modified before epoch", host, sel.ModifiedBefore(dttm.Format(now)), assert.False},
// {"in library", host, sel.Library("included-library"), assert.True},
// {"not in library", host, sel.Library("not-included-library"), assert.False},
// {"library id", host, sel.Library("1234"), assert.True},
// {"not library id", host, sel.Library("abcd"), assert.False},
// }
// for _, test := range table {
// suite.Run(test.name, func() {
// t := suite.T()
// itemInfo := details.ItemInfo{
// Groups: &details.GroupsInfo{
// ItemType: details.GroupsPage,
// WebURL: test.infoURL,
// Created: now,
// Modified: modification,
// DriveName: "included-library",
// DriveID: "1234",
// },
// }
// scopes := setScopesToDefault(test.scope)
// for _, scope := range scopes {
// test.expect(t, scope.matchesInfo(itemInfo))
// }
// })
// }
// }
func (suite *GroupsSelectorSuite) TestCategory_PathType() {
table := []struct {
cat groupsCategory
pathType path.CategoryType
}{
{
cat: GroupsCategoryUnknown,
pathType: path.UnknownCategory,
},
{
cat: GroupsChannel,
pathType: path.ChannelMessagesCategory,
},
{
cat: GroupsChannelMessage,
pathType: path.ChannelMessagesCategory,
},
}
for _, test := range table {
suite.Run(test.cat.String(), func() {
assert.Equal(
suite.T(),
test.pathType.String(),
test.cat.PathType().String())
})
}
}