From 5b0549fb32be4507d9e787aac94f59708d220035 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 22 Nov 2022 12:01:05 -0700 Subject: [PATCH] migrate sharepoint cat from file to library (#1555) ## Description Adds a LibraryCategory in paths, and changes the current sharepoint code to use that cat instead of the onedrive files cat. This is mostly to keep sharepoint library drives separate from onedrive and sharepoint site list items. ## Type of change - [x] :sunflower: Feature ## Issue(s) * #1506 ## Test Plan - [x] :zap: Unit test --- .../connector/data_collections_test.go | 4 +- .../connector/graph/service_helper.go | 2 + .../connector/graph_connector_helper_test.go | 8 ++- .../connector/onedrive/collections.go | 2 +- .../connector/onedrive/collections_test.go | 3 +- .../connector/sharepoint/data_collections.go | 6 +- .../sharepoint/data_collections_test.go | 6 +- src/pkg/path/categorytype_string.go | 5 +- src/pkg/path/path.go | 52 ++++++++++------ src/pkg/path/resource_path.go | 17 +++--- src/pkg/path/resource_path_test.go | 6 +- src/pkg/path/service_category_test.go | 6 +- src/pkg/selectors/sharepoint.go | 59 +++++++++---------- src/pkg/selectors/sharepoint_test.go | 26 ++++---- 14 files changed, 113 insertions(+), 89 deletions(-) diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 9ee159340..8020a1587 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -172,7 +172,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti name: "Items - TODO: actual sharepoint categories", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup() - sel.Include(sel.Folders([]string{suite.site}, selectors.Any())) + sel.Include(sel.Libraries([]string{suite.site}, selectors.Any())) return sel.Selector }, @@ -509,7 +509,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar sel = selectors.NewSharePointBackup() ) - sel.Include(sel.Folders( + sel.Include(sel.Libraries( []string{siteID}, []string{"foo"}, selectors.PrefixMatch(), diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go index c4e86858d..23fad0274 100644 --- a/src/internal/connector/graph/service_helper.go +++ b/src/internal/connector/graph/service_helper.go @@ -81,6 +81,8 @@ func StringToPathCategory(input string) path.CategoryType { return path.EventsCategory case "files": return path.FilesCategory + case "libraries": + return path.LibrariesCategory default: return path.UnknownCategory } diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 9d63952f8..e1bc6fd56 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -23,7 +23,7 @@ import ( func mustToDataLayerPath( t *testing.T, service path.ServiceType, - tenant, user string, + tenant, resourceOwner string, category path.CategoryType, elements []string, isItem bool, @@ -37,11 +37,13 @@ func mustToDataLayerPath( switch service { case path.ExchangeService: - res, err = pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem) + res, err = pb.ToDataLayerExchangePathForCategory(tenant, resourceOwner, category, isItem) case path.OneDriveService: require.Equal(t, path.FilesCategory, category) - res, err = pb.ToDataLayerOneDrivePath(tenant, user, isItem) + res, err = pb.ToDataLayerOneDrivePath(tenant, resourceOwner, isItem) + case path.SharePointService: + res, err = pb.ToDataLayerSharePointPath(tenant, resourceOwner, category, isItem) default: err = errors.Errorf("bad service type %s", service.String()) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 39c25e296..ad8602f19 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -170,7 +170,7 @@ func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path case OneDriveSource: result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false) case SharePointSource: - result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false) + result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, path.LibrariesCategory, false) default: return nil, errors.Errorf("unrecognized drive data source") } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index a1a00c4ca..78148efcc 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -59,7 +59,7 @@ func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() { name: "sharepoint", source: SharePointSource, dir: []string{"sharepoint"}, - expect: "tenant/sharepoint/resourceOwner/files/sharepoint", + expect: "tenant/sharepoint/resourceOwner/libraries/sharepoint", expectErr: assert.NoError, }, { @@ -256,6 +256,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { defer flush() c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil) + err := c.UpdateCollections(ctx, "driveID", tt.items) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index c1b7e1b3f..16bea69d9 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -57,7 +57,7 @@ func DataCollections( defer close(foldersComplete) switch scope.Category().PathType() { - case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory + case path.LibrariesCategory: spcs, err := collectLibraries( ctx, serv, @@ -116,9 +116,9 @@ type folderMatcher struct { } func (fm folderMatcher) IsAny() bool { - return fm.scope.IsAny(selectors.SharePointFolder) + return fm.scope.IsAny(selectors.SharePointLibrary) } func (fm folderMatcher) Matches(dir string) bool { - return fm.scope.Matches(selectors.SharePointFolder, dir) + return fm.scope.Matches(selectors.SharePointLibrary, dir) } diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 15d0007e3..fbb428f52 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -28,11 +28,11 @@ type testFolderMatcher struct { } func (fm testFolderMatcher) IsAny() bool { - return fm.scope.IsAny(selectors.SharePointFolder) + return fm.scope.IsAny(selectors.SharePointLibrary) } func (fm testFolderMatcher) Matches(path string) bool { - return fm.scope.Matches(selectors.SharePointFolder, path) + return fm.scope.Matches(selectors.SharePointLibrary, path) } type MockGraphService struct{} @@ -63,7 +63,7 @@ func TestSharePointLibrariesSuite(t *testing.T) { } func (suite *SharePointLibrariesSuite) TestUpdateCollections() { - anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0] + anyFolder := (&selectors.SharePointBackup{}).Libraries(selectors.Any(), selectors.Any())[0] const ( tenant = "tenant" diff --git a/src/pkg/path/categorytype_string.go b/src/pkg/path/categorytype_string.go index 5ac1c3b21..e3797e1ba 100644 --- a/src/pkg/path/categorytype_string.go +++ b/src/pkg/path/categorytype_string.go @@ -13,11 +13,12 @@ func _() { _ = x[ContactsCategory-2] _ = x[EventsCategory-3] _ = x[FilesCategory-4] + _ = x[LibrariesCategory-5] } -const _CategoryType_name = "UnknownCategoryemailcontactseventsfiles" +const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslibraries" -var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39} +var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 48} func (i CategoryType) String() string { if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) { diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index a0e9b426f..5a06d3d74 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -10,29 +10,44 @@ // // Examples of paths splitting by elements and canonicalization with escaping: // 1. -// input path: `this/is/a/path` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is/a/path` +// elements of path: `this`, `is`, `a`, `path` +// // 2. -// input path: `this/is\/a/path` -// elements of path: `this`, `is/a`, `path` +// +// input path: `this/is\/a/path` +// elements of path: `this`, `is/a`, `path` +// // 3. -// input path: `this/is\\/a/path` -// elements of path: `this`, `is\`, `a`, `path` +// +// input path: `this/is\\/a/path` +// elements of path: `this`, `is\`, `a`, `path` +// // 4. -// input path: `this/is\\\/a/path` -// elements of path: `this`, `is\/a`, `path` +// +// input path: `this/is\\\/a/path` +// elements of path: `this`, `is\/a`, `path` +// // 5. -// input path: `this/is//a/path` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is//a/path` +// elements of path: `this`, `is`, `a`, `path` +// // 6. -// input path: `this/is\//a/path` -// elements of path: `this`, `is/`, `a`, `path` +// +// input path: `this/is\//a/path` +// elements of path: `this`, `is/`, `a`, `path` +// // 7. -// input path: `this/is/a/path/` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is/a/path/` +// elements of path: `this`, `is`, `a`, `path` +// // 8. -// input path: `this/is/a/path\/` -// elements of path: `this`, `is`, `a`, `path/` +// +// input path: `this/is/a/path\/` +// elements of path: `this`, `is`, `a`, `path/` package path import ( @@ -307,6 +322,7 @@ func (pb Builder) ToDataLayerOneDrivePath( func (pb Builder) ToDataLayerSharePointPath( tenant, site string, + category CategoryType, isItem bool, ) (Path, error) { if err := pb.verifyPrefix(tenant, site); err != nil { @@ -318,10 +334,10 @@ func (pb Builder) ToDataLayerSharePointPath( tenant, SharePointService.String(), site, - FilesCategory.String(), + category.String(), ), service: SharePointService, - category: FilesCategory, + category: category, hasItem: isItem, }, nil } diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 9dc386f99..18e7a883f 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -35,11 +35,12 @@ type CategoryType int //go:generate stringer -type=CategoryType -linecomment const ( - UnknownCategory CategoryType = iota - EmailCategory // email - ContactsCategory // contacts - EventsCategory // events - FilesCategory // files + UnknownCategory CategoryType = iota + EmailCategory // email + ContactsCategory // contacts + EventsCategory // events + FilesCategory // files + LibrariesCategory // libraries ) func ToCategoryType(category string) CategoryType { @@ -52,6 +53,8 @@ func ToCategoryType(category string) CategoryType { return EventsCategory case FilesCategory.String(): return FilesCategory + case LibrariesCategory.String(): + return LibrariesCategory default: return UnknownCategory } @@ -68,8 +71,8 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{ FilesCategory: {}, }, SharePointService: { - // TODO: need to figure out the service Category(s) for sharepoint. - FilesCategory: {}, + LibrariesCategory: {}, + // TODO: Lists }, } diff --git a/src/pkg/path/resource_path_test.go b/src/pkg/path/resource_path_test.go index 4d3a85d64..2e2b2b6b6 100644 --- a/src/pkg/path/resource_path_test.go +++ b/src/pkg/path/resource_path_test.go @@ -104,9 +104,9 @@ var ( }, { service: path.SharePointService, - category: path.FilesCategory, - pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { - return pb.ToDataLayerSharePointPath(tenant, user, isItem) + category: path.LibrariesCategory, + pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) { + return pb.ToDataLayerSharePointPath(tenant, site, path.LibrariesCategory, isItem) }, }, } diff --git a/src/pkg/path/service_category_test.go b/src/pkg/path/service_category_test.go index 91f79204f..915e48dc5 100644 --- a/src/pkg/path/service_category_test.go +++ b/src/pkg/path/service_category_test.go @@ -106,11 +106,11 @@ func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategory() { check: assert.NoError, }, { - name: "SharePointFiles", + name: "SharePointLibraries", service: SharePointService.String(), - category: FilesCategory.String(), + category: LibrariesCategory.String(), expectedService: SharePointService, - expectedCategory: FilesCategory, + expectedCategory: LibrariesCategory, check: assert.NoError, }, } diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 7aecab5e3..fb1df33f4 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -183,16 +183,16 @@ func (s *sharePoint) DiscreteScopes(siteIDs []string) []SharePointScope { func (s *sharePoint) Sites(sites []string) []SharePointScope { scopes := []SharePointScope{} - scopes = append(scopes, makeScope[SharePointScope](SharePointFolder, sites, Any())) + scopes = append(scopes, makeScope[SharePointScope](SharePointLibrary, sites, Any())) return scopes } -// Folders produces one or more SharePoint folder scopes. +// Libraries produces one or more SharePoint library scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] -func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePointScope { +func (s *sharePoint) Libraries(sites, libraries []string, opts ...option) []SharePointScope { var ( scopes = []SharePointScope{} os = append([]option{pathType()}, opts...) @@ -200,24 +200,24 @@ func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePoi scopes = append( scopes, - makeScope[SharePointScope](SharePointFolder, sites, folders, os...), + makeScope[SharePointScope](SharePointLibrary, sites, libraries, os...), ) return scopes } -// Items produces one or more SharePoint item scopes. +// LibraryItems produces one or more SharePoint library item scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] -// options are only applied to the folder scopes. -func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []SharePointScope { +// options are only applied to the library scopes. +func (s *sharePoint) LibraryItems(sites, libraries, items []string, opts ...option) []SharePointScope { scopes := []SharePointScope{} scopes = append( scopes, - makeScope[SharePointScope](SharePointItem, sites, items). - set(SharePointFolder, folders, opts...), + makeScope[SharePointScope](SharePointLibraryItem, sites, items). + set(SharePointLibrary, libraries, opts...), ) return scopes @@ -233,7 +233,7 @@ func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []Sha func (s *sharePoint) WebURL(substring string) []SharePointScope { return []SharePointScope{ makeFilterScope[SharePointScope]( - SharePointItem, + SharePointLibraryItem, SharePointFilterWebURL, []string{substring}, wrapFilter(filters.Less)), @@ -254,9 +254,9 @@ var _ categorizer = SharePointCategoryUnknown const ( SharePointCategoryUnknown sharePointCategory = "" // types of data identified by SharePoint - SharePointSite sharePointCategory = "SharePointSite" - SharePointFolder sharePointCategory = "SharePointFolder" - SharePointItem sharePointCategory = "SharePointItem" + SharePointSite sharePointCategory = "SharePointSite" + SharePointLibrary sharePointCategory = "SharePointLibrary" + SharePointLibraryItem sharePointCategory = "SharePointLibraryItem" // filterable topics identified by SharePoint SharePointFilterWebURL sharePointCategory = "SharePointFilterWebURL" @@ -264,9 +264,9 @@ const ( // sharePointLeafProperties describes common metadata of the leaf categories var sharePointLeafProperties = map[categorizer]leafProperty{ - SharePointItem: { - pathKeys: []categorizer{SharePointSite, SharePointFolder, SharePointItem}, - pathType: path.FilesCategory, + SharePointLibraryItem: { + pathKeys: []categorizer{SharePointSite, SharePointLibrary, SharePointLibraryItem}, + pathType: path.LibrariesCategory, }, SharePointSite: { // the root category must be represented, even though it isn't a leaf pathKeys: []categorizer{SharePointSite}, @@ -285,9 +285,9 @@ func (c sharePointCategory) String() string { // Ex: ServiceUser.leafCat() => ServiceUser func (c sharePointCategory) leafCat() categorizer { switch c { - case SharePointFolder, SharePointItem, + case SharePointLibrary, SharePointLibraryItem, SharePointFilterWebURL: - return SharePointItem + return SharePointLibraryItem } return c @@ -305,8 +305,7 @@ func (c sharePointCategory) unknownCat() categorizer { // isLeaf is true if the category is a SharePointItem category. func (c sharePointCategory) isLeaf() bool { - // return c == c.leafCat()?? - return c == SharePointItem + return c == c.leafCat() } // pathValues transforms a path to a map of identified properties. @@ -316,9 +315,9 @@ func (c sharePointCategory) isLeaf() bool { // => {spSite: siteID, spFolder: folder, spItemID: itemID} func (c sharePointCategory) pathValues(p path.Path) map[categorizer]string { return map[categorizer]string{ - SharePointSite: p.ResourceOwner(), - SharePointFolder: p.Folder(), - SharePointItem: p.Item(), + SharePointSite: p.ResourceOwner(), + SharePointLibrary: p.Folder(), + SharePointLibraryItem: p.Item(), } } @@ -390,7 +389,7 @@ func (s SharePointScope) Get(cat sharePointCategory) []string { // sets a value by category to the scope. Only intended for internal use. func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option) SharePointScope { os := []option{} - if cat == SharePointFolder { + if cat == SharePointLibrary { os = append(os, pathType()) } @@ -401,10 +400,10 @@ func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option) func (s SharePointScope) setDefaults() { switch s.Category() { case SharePointSite: - s[SharePointFolder.String()] = passAny - s[SharePointItem.String()] = passAny - case SharePointFolder: - s[SharePointItem.String()] = passAny + s[SharePointLibrary.String()] = passAny + s[SharePointLibraryItem.String()] = passAny + case SharePointLibrary: + s[SharePointLibraryItem.String()] = passAny } } @@ -442,8 +441,8 @@ func (s sharePoint) Reduce(ctx context.Context, deets *details.Details) *details deets, s.Selector, map[path.CategoryType]sharePointCategory{ - // TODO: need to figure out the path Category(s) for sharepoint. - path.FilesCategory: SharePointItem, + path.LibrariesCategory: SharePointLibraryItem, + // TODO: list category type }, ) } diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 07bbe4dba..1c7f87755 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -181,9 +181,9 @@ func (suite *SharePointSelectorSuite) TestToSharePointRestore() { func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( - item = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderB", "item") - item2 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderC", "item2") - item3 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderD/folderE", "item3") + item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderB", "item") + item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderC", "item2") + item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderD/folderE", "item3") ) deets := &details.Details{ @@ -242,7 +242,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets, func() *SharePointRestore { odr := NewSharePointRestore() - odr.Include(odr.Items(Any(), Any(), []string{"item2"})) + odr.Include(odr.LibraryItems(Any(), Any(), []string{"item2"})) return odr }, arr(item2), @@ -252,7 +252,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets, func() *SharePointRestore { odr := NewSharePointRestore() - odr.Include(odr.Folders([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"})) + odr.Include(odr.Libraries([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"})) return odr }, arr(item, item2), @@ -275,16 +275,16 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { t := suite.T() pathBuilder := path.Builder{}.Append("dir1", "dir2", "item") - itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", true) + itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", path.LibrariesCategory, true) require.NoError(t, err) expected := map[categorizer]string{ - SharePointSite: "site", - SharePointFolder: "dir1/dir2", - SharePointItem: "item", + SharePointSite: "site", + SharePointLibrary: "dir1/dir2", + SharePointLibraryItem: "item", } - assert.Equal(t, expected, SharePointItem.pathValues(itemPath)) + assert.Equal(t, expected, SharePointLibraryItem.pathValues(itemPath)) } func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() { @@ -325,9 +325,9 @@ func (suite *SharePointSelectorSuite) TestCategory_PathType() { }{ {SharePointCategoryUnknown, path.UnknownCategory}, {SharePointSite, path.UnknownCategory}, - {SharePointFolder, path.FilesCategory}, - {SharePointItem, path.FilesCategory}, - {SharePointFilterWebURL, path.FilesCategory}, + {SharePointLibrary, path.LibrariesCategory}, + {SharePointLibraryItem, path.LibrariesCategory}, + {SharePointFilterWebURL, path.LibrariesCategory}, } for _, test := range table { suite.T().Run(test.cat.String(), func(t *testing.T) {