larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate sharepoint cat from file to library (#1555)

Browse Source 
## Description

Adds a LibraryCategory in paths, and changes
the current sharepoint code to use that cat
instead of the onedrive files cat.  This is mostly to keep sharepoint library drives separate from
onedrive and sharepoint site list items.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1506

## Test Plan

- [x] ⚡ Unit test
This commit is contained in:
Keepers 2022-11-22 12:01:05 -07:00 committed by GitHub
parent 10acf0ccf6
commit 5b0549fb32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 113 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/internal/connector/data_collections_test.go
View File

@ -172,7 +172,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
name: "Items - TODO: actual sharepoint categories",
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup()
sel.Include(sel.Folders([]string{suite.site}, selectors.Any()))
sel.Include(sel.Libraries([]string{suite.site}, selectors.Any()))
return sel.Selector
},
@ -509,7 +509,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
sel = selectors.NewSharePointBackup()
)
sel.Include(sel.Folders(
sel.Include(sel.Libraries(
[]string{siteID},
[]string{"foo"},
selectors.PrefixMatch(),

2
src/internal/connector/graph/service_helper.go
View File

@ -81,6 +81,8 @@ func StringToPathCategory(input string) path.CategoryType {
return path.EventsCategory
case "files":
return path.FilesCategory
case "libraries":
return path.LibrariesCategory
default:
return path.UnknownCategory
}

8
src/internal/connector/graph_connector_helper_test.go
View File

@ -23,7 +23,7 @@ import (
func mustToDataLayerPath(
t *testing.T,
service path.ServiceType,
tenant, user string,
tenant, resourceOwner string,
category path.CategoryType,
elements []string,
isItem bool,
@ -37,11 +37,13 @@ func mustToDataLayerPath(
switch service {
case path.ExchangeService:
res, err = pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem)
res, err = pb.ToDataLayerExchangePathForCategory(tenant, resourceOwner, category, isItem)
case path.OneDriveService:
require.Equal(t, path.FilesCategory, category)
res, err = pb.ToDataLayerOneDrivePath(tenant, user, isItem)
res, err = pb.ToDataLayerOneDrivePath(tenant, resourceOwner, isItem)
case path.SharePointService:
res, err = pb.ToDataLayerSharePointPath(tenant, resourceOwner, category, isItem)
default:
err = errors.Errorf("bad service type %s", service.String())

2
src/internal/connector/onedrive/collections.go
View File

@ -170,7 +170,7 @@ func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path
case OneDriveSource:
result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false)
case SharePointSource:
result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false)
result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, path.LibrariesCategory, false)
default:
return nil, errors.Errorf("unrecognized drive data source")
}

3
src/internal/connector/onedrive/collections_test.go
View File

@ -59,7 +59,7 @@ func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() {
name: "sharepoint",
source: SharePointSource,
dir: []string{"sharepoint"},
expect: "tenant/sharepoint/resourceOwner/files/sharepoint",
expect: "tenant/sharepoint/resourceOwner/libraries/sharepoint",
expectErr: assert.NoError,
},
{
@ -256,6 +256,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
defer flush()
c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil)
err := c.UpdateCollections(ctx, "driveID", tt.items)
tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")

6
src/internal/connector/sharepoint/data_collections.go
View File

@ -57,7 +57,7 @@ func DataCollections(
defer close(foldersComplete)
switch scope.Category().PathType() {
case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory
case path.LibrariesCategory:
spcs, err := collectLibraries(
ctx,
serv,
@ -116,9 +116,9 @@ type folderMatcher struct {
}
func (fm folderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.SharePointFolder)
return fm.scope.IsAny(selectors.SharePointLibrary)
}
func (fm folderMatcher) Matches(dir string) bool {
return fm.scope.Matches(selectors.SharePointFolder, dir)
return fm.scope.Matches(selectors.SharePointLibrary, dir)
}

6
src/internal/connector/sharepoint/data_collections_test.go
View File

@ -28,11 +28,11 @@ type testFolderMatcher struct {
}
func (fm testFolderMatcher) IsAny() bool {
return fm.scope.IsAny(selectors.SharePointFolder)
return fm.scope.IsAny(selectors.SharePointLibrary)
}
func (fm testFolderMatcher) Matches(path string) bool {
return fm.scope.Matches(selectors.SharePointFolder, path)
return fm.scope.Matches(selectors.SharePointLibrary, path)
}
type MockGraphService struct{}
@ -63,7 +63,7 @@ func TestSharePointLibrariesSuite(t *testing.T) {
}
func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0]
anyFolder := (&selectors.SharePointBackup{}).Libraries(selectors.Any(), selectors.Any())[0]
const (
tenant = "tenant"

5
src/pkg/path/categorytype_string.go
View File

@ -13,11 +13,12 @@ func _() {
_ = x[ContactsCategory-2]
_ = x[EventsCategory-3]
_ = x[FilesCategory-4]
_ = x[LibrariesCategory-5]
}
const _CategoryType_name = "UnknownCategoryemailcontactseventsfiles"
const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslibraries"
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39}
var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 48}
func (i CategoryType) String() string {
if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) {

52
src/pkg/path/path.go
View File

@ -10,29 +10,44 @@
//
// Examples of paths splitting by elements and canonicalization with escaping:
// 1.
// input path: `this/is/a/path`
// elements of path: `this`, `is`, `a`, `path`
//
// input path: `this/is/a/path`
// elements of path: `this`, `is`, `a`, `path`
//
// 2.
// input path: `this/is\/a/path`
// elements of path: `this`, `is/a`, `path`
//
// input path: `this/is\/a/path`
// elements of path: `this`, `is/a`, `path`
//
// 3.
// input path: `this/is\\/a/path`
// elements of path: `this`, `is\`, `a`, `path`
//
// input path: `this/is\\/a/path`
// elements of path: `this`, `is\`, `a`, `path`
//
// 4.
// input path: `this/is\\\/a/path`
// elements of path: `this`, `is\/a`, `path`
//
// input path: `this/is\\\/a/path`
// elements of path: `this`, `is\/a`, `path`
//
// 5.
// input path: `this/is//a/path`
// elements of path: `this`, `is`, `a`, `path`
//
// input path: `this/is//a/path`
// elements of path: `this`, `is`, `a`, `path`
//
// 6.
// input path: `this/is\//a/path`
// elements of path: `this`, `is/`, `a`, `path`
//
// input path: `this/is\//a/path`
// elements of path: `this`, `is/`, `a`, `path`
//
// 7.
// input path: `this/is/a/path/`
// elements of path: `this`, `is`, `a`, `path`
//
// input path: `this/is/a/path/`
// elements of path: `this`, `is`, `a`, `path`
//
// 8.
// input path: `this/is/a/path\/`
// elements of path: `this`, `is`, `a`, `path/`
//
// input path: `this/is/a/path\/`
// elements of path: `this`, `is`, `a`, `path/`
package path
import (
@ -307,6 +322,7 @@ func (pb Builder) ToDataLayerOneDrivePath(
func (pb Builder) ToDataLayerSharePointPath(
tenant, site string,
category CategoryType,
isItem bool,
) (Path, error) {
if err := pb.verifyPrefix(tenant, site); err != nil {
@ -318,10 +334,10 @@ func (pb Builder) ToDataLayerSharePointPath(
tenant,
SharePointService.String(),
site,
FilesCategory.String(),
category.String(),
),
service: SharePointService,
category: FilesCategory,
category: category,
hasItem: isItem,
}, nil
}

17
src/pkg/path/resource_path.go
View File

@ -35,11 +35,12 @@ type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
LibrariesCategory // libraries
)
func ToCategoryType(category string) CategoryType {
@ -52,6 +53,8 @@ func ToCategoryType(category string) CategoryType {
return EventsCategory
case FilesCategory.String():
return FilesCategory
case LibrariesCategory.String():
return LibrariesCategory
default:
return UnknownCategory
}
@ -68,8 +71,8 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
FilesCategory: {},
},
SharePointService: {
// TODO: need to figure out the service Category(s) for sharepoint.
FilesCategory: {},
LibrariesCategory: {},
// TODO: Lists
},
}

6
src/pkg/path/resource_path_test.go
View File

@ -104,9 +104,9 @@ var (
},
{
service: path.SharePointService,
category: path.FilesCategory,
pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) {
return pb.ToDataLayerSharePointPath(tenant, user, isItem)
category: path.LibrariesCategory,
pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) {
return pb.ToDataLayerSharePointPath(tenant, site, path.LibrariesCategory, isItem)
},
},
}

6
src/pkg/path/service_category_test.go
View File

@ -106,11 +106,11 @@ func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategory() {
check: assert.NoError,
},
{
name: "SharePointFiles",
name: "SharePointLibraries",
service: SharePointService.String(),
category: FilesCategory.String(),
category: LibrariesCategory.String(),
expectedService: SharePointService,
expectedCategory: FilesCategory,
expectedCategory: LibrariesCategory,
check: assert.NoError,
},
}

59
src/pkg/selectors/sharepoint.go
View File

@ -183,16 +183,16 @@ func (s *sharePoint) DiscreteScopes(siteIDs []string) []SharePointScope {
func (s *sharePoint) Sites(sites []string) []SharePointScope {
scopes := []SharePointScope{}
scopes = append(scopes, makeScope[SharePointScope](SharePointFolder, sites, Any()))
scopes = append(scopes, makeScope[SharePointScope](SharePointLibrary, sites, Any()))
return scopes
}
// Folders produces one or more SharePoint folder scopes.
// Libraries produces one or more SharePoint library scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePointScope {
func (s *sharePoint) Libraries(sites, libraries []string, opts ...option) []SharePointScope {
var (
scopes = []SharePointScope{}
os = append([]option{pathType()}, opts...)
@ -200,24 +200,24 @@ func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePoi
scopes = append(
scopes,
makeScope[SharePointScope](SharePointFolder, sites, folders, os...),
makeScope[SharePointScope](SharePointLibrary, sites, libraries, os...),
)
return scopes
}
// Items produces one or more SharePoint item scopes.
// LibraryItems produces one or more SharePoint library item scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
// options are only applied to the folder scopes.
func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []SharePointScope {
// options are only applied to the library scopes.
func (s *sharePoint) LibraryItems(sites, libraries, items []string, opts ...option) []SharePointScope {
scopes := []SharePointScope{}
scopes = append(
scopes,
makeScope[SharePointScope](SharePointItem, sites, items).
set(SharePointFolder, folders, opts...),
makeScope[SharePointScope](SharePointLibraryItem, sites, items).
set(SharePointLibrary, libraries, opts...),
)
return scopes
@ -233,7 +233,7 @@ func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []Sha
func (s *sharePoint) WebURL(substring string) []SharePointScope {
return []SharePointScope{
makeFilterScope[SharePointScope](
SharePointItem,
SharePointLibraryItem,
SharePointFilterWebURL,
[]string{substring},
wrapFilter(filters.Less)),
@ -254,9 +254,9 @@ var _ categorizer = SharePointCategoryUnknown
const (
SharePointCategoryUnknown sharePointCategory = ""
// types of data identified by SharePoint
SharePointSite sharePointCategory = "SharePointSite"
SharePointFolder sharePointCategory = "SharePointFolder"
SharePointItem sharePointCategory = "SharePointItem"
SharePointSite sharePointCategory = "SharePointSite"
SharePointLibrary sharePointCategory = "SharePointLibrary"
SharePointLibraryItem sharePointCategory = "SharePointLibraryItem"
// filterable topics identified by SharePoint
SharePointFilterWebURL sharePointCategory = "SharePointFilterWebURL"
@ -264,9 +264,9 @@ const (
// sharePointLeafProperties describes common metadata of the leaf categories
var sharePointLeafProperties = map[categorizer]leafProperty{
SharePointItem: {
pathKeys: []categorizer{SharePointSite, SharePointFolder, SharePointItem},
pathType: path.FilesCategory,
SharePointLibraryItem: {
pathKeys: []categorizer{SharePointSite, SharePointLibrary, SharePointLibraryItem},
pathType: path.LibrariesCategory,
},
SharePointSite: { // the root category must be represented, even though it isn't a leaf
pathKeys: []categorizer{SharePointSite},
@ -285,9 +285,9 @@ func (c sharePointCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser
func (c sharePointCategory) leafCat() categorizer {
switch c {
case SharePointFolder, SharePointItem,
case SharePointLibrary, SharePointLibraryItem,
SharePointFilterWebURL:
return SharePointItem
return SharePointLibraryItem
}
return c
@ -305,8 +305,7 @@ func (c sharePointCategory) unknownCat() categorizer {
// isLeaf is true if the category is a SharePointItem category.
func (c sharePointCategory) isLeaf() bool {
// return c == c.leafCat()??
return c == SharePointItem
return c == c.leafCat()
}
// pathValues transforms a path to a map of identified properties.
@ -316,9 +315,9 @@ func (c sharePointCategory) isLeaf() bool {
// => {spSite: siteID, spFolder: folder, spItemID: itemID}
func (c sharePointCategory) pathValues(p path.Path) map[categorizer]string {
return map[categorizer]string{
SharePointSite: p.ResourceOwner(),
SharePointFolder: p.Folder(),
SharePointItem: p.Item(),
SharePointSite: p.ResourceOwner(),
SharePointLibrary: p.Folder(),
SharePointLibraryItem: p.Item(),
}
}
@ -390,7 +389,7 @@ func (s SharePointScope) Get(cat sharePointCategory) []string {
// sets a value by category to the scope. Only intended for internal use.
func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option) SharePointScope {
os := []option{}
if cat == SharePointFolder {
if cat == SharePointLibrary {
os = append(os, pathType())
}
@ -401,10 +400,10 @@ func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option)
func (s SharePointScope) setDefaults() {
switch s.Category() {
case SharePointSite:
s[SharePointFolder.String()] = passAny
s[SharePointItem.String()] = passAny
case SharePointFolder:
s[SharePointItem.String()] = passAny
s[SharePointLibrary.String()] = passAny
s[SharePointLibraryItem.String()] = passAny
case SharePointLibrary:
s[SharePointLibraryItem.String()] = passAny
}
}
@ -442,8 +441,8 @@ func (s sharePoint) Reduce(ctx context.Context, deets *details.Details) *details
deets,
s.Selector,
map[path.CategoryType]sharePointCategory{
// TODO: need to figure out the path Category(s) for sharepoint.
path.FilesCategory: SharePointItem,
path.LibrariesCategory: SharePointLibraryItem,
// TODO: list category type
},
)
}

26
src/pkg/selectors/sharepoint_test.go
View File

@ -181,9 +181,9 @@ func (suite *SharePointSelectorSuite) TestToSharePointRestore() {
func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
var (
item = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderB", "item")
item2 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderC", "item2")
item3 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderD/folderE", "item3")
item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderB", "item")
item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderC", "item2")
item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderD/folderE", "item3")
)
deets := &details.Details{
@ -242,7 +242,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets,
func() *SharePointRestore {
odr := NewSharePointRestore()
odr.Include(odr.Items(Any(), Any(), []string{"item2"}))
odr.Include(odr.LibraryItems(Any(), Any(), []string{"item2"}))
return odr
},
arr(item2),
@ -252,7 +252,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
deets,
func() *SharePointRestore {
odr := NewSharePointRestore()
odr.Include(odr.Folders([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"}))
odr.Include(odr.Libraries([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"}))
return odr
},
arr(item, item2),
@ -275,16 +275,16 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
t := suite.T()
pathBuilder := path.Builder{}.Append("dir1", "dir2", "item")
itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", true)
itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", path.LibrariesCategory, true)
require.NoError(t, err)
expected := map[categorizer]string{
SharePointSite: "site",
SharePointFolder: "dir1/dir2",
SharePointItem: "item",
SharePointSite: "site",
SharePointLibrary: "dir1/dir2",
SharePointLibraryItem: "item",
}
assert.Equal(t, expected, SharePointItem.pathValues(itemPath))
assert.Equal(t, expected, SharePointLibraryItem.pathValues(itemPath))
}
func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
@ -325,9 +325,9 @@ func (suite *SharePointSelectorSuite) TestCategory_PathType() {
}{
{SharePointCategoryUnknown, path.UnknownCategory},
{SharePointSite, path.UnknownCategory},
{SharePointFolder, path.FilesCategory},
{SharePointItem, path.FilesCategory},
{SharePointFilterWebURL, path.FilesCategory},
{SharePointLibrary, path.LibrariesCategory},
{SharePointLibraryItem, path.LibrariesCategory},
{SharePointFilterWebURL, path.LibrariesCategory},
}
for _, test := range table {
suite.T().Run(test.cat.String(), func(t *testing.T) {