diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 9ee159340..8020a1587 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -172,7 +172,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti name: "Items - TODO: actual sharepoint categories", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewSharePointBackup() - sel.Include(sel.Folders([]string{suite.site}, selectors.Any())) + sel.Include(sel.Libraries([]string{suite.site}, selectors.Any())) return sel.Selector }, @@ -509,7 +509,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar sel = selectors.NewSharePointBackup() ) - sel.Include(sel.Folders( + sel.Include(sel.Libraries( []string{siteID}, []string{"foo"}, selectors.PrefixMatch(), diff --git a/src/internal/connector/graph/service_helper.go b/src/internal/connector/graph/service_helper.go index c4e86858d..23fad0274 100644 --- a/src/internal/connector/graph/service_helper.go +++ b/src/internal/connector/graph/service_helper.go @@ -81,6 +81,8 @@ func StringToPathCategory(input string) path.CategoryType { return path.EventsCategory case "files": return path.FilesCategory + case "libraries": + return path.LibrariesCategory default: return path.UnknownCategory } diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index 9d63952f8..e1bc6fd56 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -23,7 +23,7 @@ import ( func mustToDataLayerPath( t *testing.T, service path.ServiceType, - tenant, user string, + tenant, resourceOwner string, category path.CategoryType, elements []string, isItem bool, @@ -37,11 +37,13 @@ func mustToDataLayerPath( switch service { case path.ExchangeService: - res, err = pb.ToDataLayerExchangePathForCategory(tenant, user, category, isItem) + res, err = pb.ToDataLayerExchangePathForCategory(tenant, resourceOwner, category, isItem) case path.OneDriveService: require.Equal(t, path.FilesCategory, category) - res, err = pb.ToDataLayerOneDrivePath(tenant, user, isItem) + res, err = pb.ToDataLayerOneDrivePath(tenant, resourceOwner, isItem) + case path.SharePointService: + res, err = pb.ToDataLayerSharePointPath(tenant, resourceOwner, category, isItem) default: err = errors.Errorf("bad service type %s", service.String()) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index 39c25e296..ad8602f19 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -170,7 +170,7 @@ func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path case OneDriveSource: result, err = pathBuilder.ToDataLayerOneDrivePath(tenant, resourceOwner, false) case SharePointSource: - result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, false) + result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, path.LibrariesCategory, false) default: return nil, errors.Errorf("unrecognized drive data source") } diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index a1a00c4ca..78148efcc 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -59,7 +59,7 @@ func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() { name: "sharepoint", source: SharePointSource, dir: []string{"sharepoint"}, - expect: "tenant/sharepoint/resourceOwner/files/sharepoint", + expect: "tenant/sharepoint/resourceOwner/libraries/sharepoint", expectErr: assert.NoError, }, { @@ -256,6 +256,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { defer flush() c := NewCollections(tenant, user, OneDriveSource, testFolderMatcher{tt.scope}, &MockGraphService{}, nil) + err := c.UpdateCollections(ctx, "driveID", tt.items) tt.expect(t, err) assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths") diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index c1b7e1b3f..16bea69d9 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -57,7 +57,7 @@ func DataCollections( defer close(foldersComplete) switch scope.Category().PathType() { - case path.FilesCategory: // TODO: better category for sp drives, eg: LibrariesCategory + case path.LibrariesCategory: spcs, err := collectLibraries( ctx, serv, @@ -116,9 +116,9 @@ type folderMatcher struct { } func (fm folderMatcher) IsAny() bool { - return fm.scope.IsAny(selectors.SharePointFolder) + return fm.scope.IsAny(selectors.SharePointLibrary) } func (fm folderMatcher) Matches(dir string) bool { - return fm.scope.Matches(selectors.SharePointFolder, dir) + return fm.scope.Matches(selectors.SharePointLibrary, dir) } diff --git a/src/internal/connector/sharepoint/data_collections_test.go b/src/internal/connector/sharepoint/data_collections_test.go index 15d0007e3..fbb428f52 100644 --- a/src/internal/connector/sharepoint/data_collections_test.go +++ b/src/internal/connector/sharepoint/data_collections_test.go @@ -28,11 +28,11 @@ type testFolderMatcher struct { } func (fm testFolderMatcher) IsAny() bool { - return fm.scope.IsAny(selectors.SharePointFolder) + return fm.scope.IsAny(selectors.SharePointLibrary) } func (fm testFolderMatcher) Matches(path string) bool { - return fm.scope.Matches(selectors.SharePointFolder, path) + return fm.scope.Matches(selectors.SharePointLibrary, path) } type MockGraphService struct{} @@ -63,7 +63,7 @@ func TestSharePointLibrariesSuite(t *testing.T) { } func (suite *SharePointLibrariesSuite) TestUpdateCollections() { - anyFolder := (&selectors.SharePointBackup{}).Folders(selectors.Any(), selectors.Any())[0] + anyFolder := (&selectors.SharePointBackup{}).Libraries(selectors.Any(), selectors.Any())[0] const ( tenant = "tenant" diff --git a/src/pkg/path/categorytype_string.go b/src/pkg/path/categorytype_string.go index 5ac1c3b21..e3797e1ba 100644 --- a/src/pkg/path/categorytype_string.go +++ b/src/pkg/path/categorytype_string.go @@ -13,11 +13,12 @@ func _() { _ = x[ContactsCategory-2] _ = x[EventsCategory-3] _ = x[FilesCategory-4] + _ = x[LibrariesCategory-5] } -const _CategoryType_name = "UnknownCategoryemailcontactseventsfiles" +const _CategoryType_name = "UnknownCategoryemailcontactseventsfileslibraries" -var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39} +var _CategoryType_index = [...]uint8{0, 15, 20, 28, 34, 39, 48} func (i CategoryType) String() string { if i < 0 || i >= CategoryType(len(_CategoryType_index)-1) { diff --git a/src/pkg/path/path.go b/src/pkg/path/path.go index a0e9b426f..5a06d3d74 100644 --- a/src/pkg/path/path.go +++ b/src/pkg/path/path.go @@ -10,29 +10,44 @@ // // Examples of paths splitting by elements and canonicalization with escaping: // 1. -// input path: `this/is/a/path` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is/a/path` +// elements of path: `this`, `is`, `a`, `path` +// // 2. -// input path: `this/is\/a/path` -// elements of path: `this`, `is/a`, `path` +// +// input path: `this/is\/a/path` +// elements of path: `this`, `is/a`, `path` +// // 3. -// input path: `this/is\\/a/path` -// elements of path: `this`, `is\`, `a`, `path` +// +// input path: `this/is\\/a/path` +// elements of path: `this`, `is\`, `a`, `path` +// // 4. -// input path: `this/is\\\/a/path` -// elements of path: `this`, `is\/a`, `path` +// +// input path: `this/is\\\/a/path` +// elements of path: `this`, `is\/a`, `path` +// // 5. -// input path: `this/is//a/path` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is//a/path` +// elements of path: `this`, `is`, `a`, `path` +// // 6. -// input path: `this/is\//a/path` -// elements of path: `this`, `is/`, `a`, `path` +// +// input path: `this/is\//a/path` +// elements of path: `this`, `is/`, `a`, `path` +// // 7. -// input path: `this/is/a/path/` -// elements of path: `this`, `is`, `a`, `path` +// +// input path: `this/is/a/path/` +// elements of path: `this`, `is`, `a`, `path` +// // 8. -// input path: `this/is/a/path\/` -// elements of path: `this`, `is`, `a`, `path/` +// +// input path: `this/is/a/path\/` +// elements of path: `this`, `is`, `a`, `path/` package path import ( @@ -307,6 +322,7 @@ func (pb Builder) ToDataLayerOneDrivePath( func (pb Builder) ToDataLayerSharePointPath( tenant, site string, + category CategoryType, isItem bool, ) (Path, error) { if err := pb.verifyPrefix(tenant, site); err != nil { @@ -318,10 +334,10 @@ func (pb Builder) ToDataLayerSharePointPath( tenant, SharePointService.String(), site, - FilesCategory.String(), + category.String(), ), service: SharePointService, - category: FilesCategory, + category: category, hasItem: isItem, }, nil } diff --git a/src/pkg/path/resource_path.go b/src/pkg/path/resource_path.go index 9dc386f99..18e7a883f 100644 --- a/src/pkg/path/resource_path.go +++ b/src/pkg/path/resource_path.go @@ -35,11 +35,12 @@ type CategoryType int //go:generate stringer -type=CategoryType -linecomment const ( - UnknownCategory CategoryType = iota - EmailCategory // email - ContactsCategory // contacts - EventsCategory // events - FilesCategory // files + UnknownCategory CategoryType = iota + EmailCategory // email + ContactsCategory // contacts + EventsCategory // events + FilesCategory // files + LibrariesCategory // libraries ) func ToCategoryType(category string) CategoryType { @@ -52,6 +53,8 @@ func ToCategoryType(category string) CategoryType { return EventsCategory case FilesCategory.String(): return FilesCategory + case LibrariesCategory.String(): + return LibrariesCategory default: return UnknownCategory } @@ -68,8 +71,8 @@ var serviceCategories = map[ServiceType]map[CategoryType]struct{}{ FilesCategory: {}, }, SharePointService: { - // TODO: need to figure out the service Category(s) for sharepoint. - FilesCategory: {}, + LibrariesCategory: {}, + // TODO: Lists }, } diff --git a/src/pkg/path/resource_path_test.go b/src/pkg/path/resource_path_test.go index 4d3a85d64..2e2b2b6b6 100644 --- a/src/pkg/path/resource_path_test.go +++ b/src/pkg/path/resource_path_test.go @@ -104,9 +104,9 @@ var ( }, { service: path.SharePointService, - category: path.FilesCategory, - pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { - return pb.ToDataLayerSharePointPath(tenant, user, isItem) + category: path.LibrariesCategory, + pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) { + return pb.ToDataLayerSharePointPath(tenant, site, path.LibrariesCategory, isItem) }, }, } diff --git a/src/pkg/path/service_category_test.go b/src/pkg/path/service_category_test.go index 91f79204f..915e48dc5 100644 --- a/src/pkg/path/service_category_test.go +++ b/src/pkg/path/service_category_test.go @@ -106,11 +106,11 @@ func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategory() { check: assert.NoError, }, { - name: "SharePointFiles", + name: "SharePointLibraries", service: SharePointService.String(), - category: FilesCategory.String(), + category: LibrariesCategory.String(), expectedService: SharePointService, - expectedCategory: FilesCategory, + expectedCategory: LibrariesCategory, check: assert.NoError, }, } diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 7aecab5e3..fb1df33f4 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -183,16 +183,16 @@ func (s *sharePoint) DiscreteScopes(siteIDs []string) []SharePointScope { func (s *sharePoint) Sites(sites []string) []SharePointScope { scopes := []SharePointScope{} - scopes = append(scopes, makeScope[SharePointScope](SharePointFolder, sites, Any())) + scopes = append(scopes, makeScope[SharePointScope](SharePointLibrary, sites, Any())) return scopes } -// Folders produces one or more SharePoint folder scopes. +// Libraries produces one or more SharePoint library scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] -func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePointScope { +func (s *sharePoint) Libraries(sites, libraries []string, opts ...option) []SharePointScope { var ( scopes = []SharePointScope{} os = append([]option{pathType()}, opts...) @@ -200,24 +200,24 @@ func (s *sharePoint) Folders(sites, folders []string, opts ...option) []SharePoi scopes = append( scopes, - makeScope[SharePointScope](SharePointFolder, sites, folders, os...), + makeScope[SharePointScope](SharePointLibrary, sites, libraries, os...), ) return scopes } -// Items produces one or more SharePoint item scopes. +// LibraryItems produces one or more SharePoint library item scopes. // If any slice contains selectors.Any, that slice is reduced to [selectors.Any] // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] -// options are only applied to the folder scopes. -func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []SharePointScope { +// options are only applied to the library scopes. +func (s *sharePoint) LibraryItems(sites, libraries, items []string, opts ...option) []SharePointScope { scopes := []SharePointScope{} scopes = append( scopes, - makeScope[SharePointScope](SharePointItem, sites, items). - set(SharePointFolder, folders, opts...), + makeScope[SharePointScope](SharePointLibraryItem, sites, items). + set(SharePointLibrary, libraries, opts...), ) return scopes @@ -233,7 +233,7 @@ func (s *sharePoint) Items(sites, folders, items []string, opts ...option) []Sha func (s *sharePoint) WebURL(substring string) []SharePointScope { return []SharePointScope{ makeFilterScope[SharePointScope]( - SharePointItem, + SharePointLibraryItem, SharePointFilterWebURL, []string{substring}, wrapFilter(filters.Less)), @@ -254,9 +254,9 @@ var _ categorizer = SharePointCategoryUnknown const ( SharePointCategoryUnknown sharePointCategory = "" // types of data identified by SharePoint - SharePointSite sharePointCategory = "SharePointSite" - SharePointFolder sharePointCategory = "SharePointFolder" - SharePointItem sharePointCategory = "SharePointItem" + SharePointSite sharePointCategory = "SharePointSite" + SharePointLibrary sharePointCategory = "SharePointLibrary" + SharePointLibraryItem sharePointCategory = "SharePointLibraryItem" // filterable topics identified by SharePoint SharePointFilterWebURL sharePointCategory = "SharePointFilterWebURL" @@ -264,9 +264,9 @@ const ( // sharePointLeafProperties describes common metadata of the leaf categories var sharePointLeafProperties = map[categorizer]leafProperty{ - SharePointItem: { - pathKeys: []categorizer{SharePointSite, SharePointFolder, SharePointItem}, - pathType: path.FilesCategory, + SharePointLibraryItem: { + pathKeys: []categorizer{SharePointSite, SharePointLibrary, SharePointLibraryItem}, + pathType: path.LibrariesCategory, }, SharePointSite: { // the root category must be represented, even though it isn't a leaf pathKeys: []categorizer{SharePointSite}, @@ -285,9 +285,9 @@ func (c sharePointCategory) String() string { // Ex: ServiceUser.leafCat() => ServiceUser func (c sharePointCategory) leafCat() categorizer { switch c { - case SharePointFolder, SharePointItem, + case SharePointLibrary, SharePointLibraryItem, SharePointFilterWebURL: - return SharePointItem + return SharePointLibraryItem } return c @@ -305,8 +305,7 @@ func (c sharePointCategory) unknownCat() categorizer { // isLeaf is true if the category is a SharePointItem category. func (c sharePointCategory) isLeaf() bool { - // return c == c.leafCat()?? - return c == SharePointItem + return c == c.leafCat() } // pathValues transforms a path to a map of identified properties. @@ -316,9 +315,9 @@ func (c sharePointCategory) isLeaf() bool { // => {spSite: siteID, spFolder: folder, spItemID: itemID} func (c sharePointCategory) pathValues(p path.Path) map[categorizer]string { return map[categorizer]string{ - SharePointSite: p.ResourceOwner(), - SharePointFolder: p.Folder(), - SharePointItem: p.Item(), + SharePointSite: p.ResourceOwner(), + SharePointLibrary: p.Folder(), + SharePointLibraryItem: p.Item(), } } @@ -390,7 +389,7 @@ func (s SharePointScope) Get(cat sharePointCategory) []string { // sets a value by category to the scope. Only intended for internal use. func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option) SharePointScope { os := []option{} - if cat == SharePointFolder { + if cat == SharePointLibrary { os = append(os, pathType()) } @@ -401,10 +400,10 @@ func (s SharePointScope) set(cat sharePointCategory, v []string, opts ...option) func (s SharePointScope) setDefaults() { switch s.Category() { case SharePointSite: - s[SharePointFolder.String()] = passAny - s[SharePointItem.String()] = passAny - case SharePointFolder: - s[SharePointItem.String()] = passAny + s[SharePointLibrary.String()] = passAny + s[SharePointLibraryItem.String()] = passAny + case SharePointLibrary: + s[SharePointLibraryItem.String()] = passAny } } @@ -442,8 +441,8 @@ func (s sharePoint) Reduce(ctx context.Context, deets *details.Details) *details deets, s.Selector, map[path.CategoryType]sharePointCategory{ - // TODO: need to figure out the path Category(s) for sharepoint. - path.FilesCategory: SharePointItem, + path.LibrariesCategory: SharePointLibraryItem, + // TODO: list category type }, ) } diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index 07bbe4dba..1c7f87755 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -181,9 +181,9 @@ func (suite *SharePointSelectorSuite) TestToSharePointRestore() { func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { var ( - item = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderB", "item") - item2 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderA/folderC", "item2") - item3 = stubRepoRef(path.SharePointService, path.FilesCategory, "uid", "/folderD/folderE", "item3") + item = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderB", "item") + item2 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderA/folderC", "item2") + item3 = stubRepoRef(path.SharePointService, path.LibrariesCategory, "uid", "/folderD/folderE", "item3") ) deets := &details.Details{ @@ -242,7 +242,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets, func() *SharePointRestore { odr := NewSharePointRestore() - odr.Include(odr.Items(Any(), Any(), []string{"item2"})) + odr.Include(odr.LibraryItems(Any(), Any(), []string{"item2"})) return odr }, arr(item2), @@ -252,7 +252,7 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() { deets, func() *SharePointRestore { odr := NewSharePointRestore() - odr.Include(odr.Folders([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"})) + odr.Include(odr.Libraries([]string{"uid"}, []string{"folderA/folderB", "folderA/folderC"})) return odr }, arr(item, item2), @@ -275,16 +275,16 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { t := suite.T() pathBuilder := path.Builder{}.Append("dir1", "dir2", "item") - itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", true) + itemPath, err := pathBuilder.ToDataLayerSharePointPath("tenant", "site", path.LibrariesCategory, true) require.NoError(t, err) expected := map[categorizer]string{ - SharePointSite: "site", - SharePointFolder: "dir1/dir2", - SharePointItem: "item", + SharePointSite: "site", + SharePointLibrary: "dir1/dir2", + SharePointLibraryItem: "item", } - assert.Equal(t, expected, SharePointItem.pathValues(itemPath)) + assert.Equal(t, expected, SharePointLibraryItem.pathValues(itemPath)) } func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() { @@ -325,9 +325,9 @@ func (suite *SharePointSelectorSuite) TestCategory_PathType() { }{ {SharePointCategoryUnknown, path.UnknownCategory}, {SharePointSite, path.UnknownCategory}, - {SharePointFolder, path.FilesCategory}, - {SharePointItem, path.FilesCategory}, - {SharePointFilterWebURL, path.FilesCategory}, + {SharePointLibrary, path.LibrariesCategory}, + {SharePointLibraryItem, path.LibrariesCategory}, + {SharePointFilterWebURL, path.LibrariesCategory}, } for _, test := range table { suite.T().Run(test.cat.String(), func(t *testing.T) {