Compare commits

...

9 Commits

Author SHA1 Message Date
ryanfkeepers
3d15a0d649 first pass on compliance with the reason
establishes behavior around using the reasoner interface
in a world where paths can contain multiple services.  Primarily
focused on ensuring the reasoner clearly guides maintainers
towards proper usage.
2023-08-15 13:29:14 -06:00
ryanfkeepers
48aae5d485 all non-resourcer compliance
Updates all path package uses that does not involve
a Resourcer.  Third to last step to wrapping up compliance.
Second step is to update the Resourcer interface to
comply with ServiceResources.  Last step is to clean up
any lingering linters, bugs, tests, and other things needed
to make the build green.
2023-08-11 16:16:40 -06:00
ryanfkeepers
bf398f6c8d fixing up selectors
adds a set of fixes targeted at the selectors package.  Largely
updates testing code.  But also updates the resource check
at the beginning of the Reduce flow to check for a match on
any of the resources in the ServiceResources slice.
2023-08-11 11:50:11 -06:00
ryanfkeepers
7c242819bc Add 'halves()' to path interface
Adds a func to the path interface which breaks up the path
into its two distinct halves: the prefix builder, and the folder
and item elements.  Likely to be useful in the future where
prefixes will be of arbitrary length, and downstream consumers
are primarly focused on the non-prefix portion of the path.
2023-08-11 11:14:27 -06:00
ryanfkeepers
6ed54fad9c first pass on ServiceResource tuple compliance
First pass for updating code outside of the path package
to comply with ServiceResource tuple slices.  Compliance
is still incomplete, so the build and tests
will still fail.  Changes
in this PR are focused on the easier-to-make changes, mostly
generating ServiceResources where there were
previously manual declarations of service and resource.
2023-08-10 17:55:34 -06:00
ryanfkeepers
dfd486cd41 distrubutes ServiceResource throughout path
Adds utilization of the ServiceResource struct to all
of the path package functionality.  In general, this
takes most instances where a service and resource
are requested, and replaces those values with a
slice of ServiceResource tuples.

To keep the review smaller, this change does not update
any packages outside of path.  Therefore the tests and
build are guaranteed to fail.  The next PR, which
updates all other packages with the changes, is
necessary for both PRs to move forward.
2023-08-10 14:57:04 -06:00
ryanfkeepers
5f7f1092ec introduce service_resource tuple
the service resource tuple will form the basis for
having paths contain multiple nested service
declarations, by allowing it to swap out the
Service() and ResourceOwner() funcs in favor of
a func that returns the ordered list of service-resource
tuples found in the path.
2023-08-10 12:51:42 -06:00
ryanfkeepers
859b8fafc7 small paths code rearrangement
small cleanup in paths, primarily splitting files
so that file contents are more clearly owned, which
should be a little better for readability and code
placement.

Also renames `ServicePrefix` to `BuildPrefix` in
anticipation of multi-service prefixes.
2023-08-10 12:50:56 -06:00
ryanfkeepers
89e8af74bc move selectorsToReason into selectors 2023-08-10 12:50:17 -06:00
78 changed files with 3261 additions and 1397 deletions

View File

@ -177,7 +177,7 @@ type collection struct {
func buildCollections(
service path.ServiceType,
tenant, user string,
tenant, protectedResource string,
restoreCfg control.RestoreConfig,
colls []collection,
) ([]data.RestoreCollection, error) {
@ -186,8 +186,10 @@ func buildCollections(
for _, c := range colls {
pth, err := path.Build(
tenant,
user,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: protectedResource,
}},
c.category,
false,
c.PathElements...)

View File

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
)
func AnyValueToT[T any](k string, m map[string]any) (T, error) {
@ -23,3 +24,21 @@ func AnyValueToT[T any](k string, m map[string]any) (T, error) {
return vt, nil
}
func AnyToT[T any](a any) (T, error) {
if a == nil {
return *new(T), clues.New("missing value")
}
pt, ok := a.(*T)
if ok {
return ptr.Val(pt), nil
}
t, ok := a.(T)
if ok {
return t, nil
}
return *new(T), clues.New(fmt.Sprintf("unexpected type: %T", a))
}

View File

@ -21,11 +21,35 @@ func TestDataCollectionSuite(t *testing.T) {
}
func (suite *DataCollectionSuite) TestStateOf() {
fooP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "foo")
fooP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err))
barP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "bar")
barP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"bar")
require.NoError(suite.T(), err, clues.ToCore(err))
preP, err := path.Build("_t", "_u", path.ExchangeService, path.EmailCategory, false, "foo")
preP, err := path.Build(
"_t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err))
table := []struct {

View File

@ -111,6 +111,12 @@ func (bb *backupBases) ClearAssistBases() {
bb.assistBases = nil
}
// BaseKeyServiceCategory makes a backup base key using
// the reasoner's Service and Category.
func BaseKeyServiceCategory(br identity.Reasoner) string {
return br.Service().String() + br.Category().String()
}
// MergeBackupBases reduces the two BackupBases into a single BackupBase.
// Assumes the passed in BackupBases represents a prior backup version (across
// some migration that disrupts lookup), and that the BackupBases used to call

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/identity"
idMock "github.com/alcionai/corso/src/pkg/backup/identity/mock"
"github.com/alcionai/corso/src/pkg/path"
)
@ -460,18 +461,12 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
bb := makeBackupBases(test.merge, test.assist)
other := makeBackupBases(test.otherMerge, test.otherAssist)
expected := test.expect()
ctx, flush := tester.NewContext(t)
defer flush()
got := bb.MergeBackupBases(
ctx,
other,
func(r identity.Reasoner) string {
return r.Service().String() + r.Category().String()
})
AssertBackupBasesEqual(t, expected, got)
got := bb.MergeBackupBases(ctx, other, BaseKeyServiceCategory)
AssertBackupBasesEqual(t, test.expect(), got)
})
}
}
@ -843,3 +838,37 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
})
}
}
func (suite *BackupBasesUnitSuite) TestBaseKeyServiceCategory() {
table := []struct {
name string
service path.ServiceType
category path.CategoryType
expect string
}{
{
name: "unknown",
service: path.UnknownService,
category: path.UnknownCategory,
expect: "unknown",
},
{
name: "known service",
service: path.ExchangeService,
category: path.EmailCategory,
expect: "exchangeEmail",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result := BaseKeyServiceCategory(idMock.Reason{
TenantID: "tid",
Cat: test.category,
Svc: test.service,
})
assert.Equal(t, test.expect, result)
})
}
}

View File

@ -70,11 +70,14 @@ func (r reason) Category() path.CategoryType {
}
func (r reason) SubtreePath() (path.Path, error) {
p, err := path.ServicePrefix(
r.Tenant(),
r.ProtectedResource(),
srs, err := path.NewServiceResources(
r.Service(),
r.Category())
r.ProtectedResource())
if err != nil {
return nil, clues.Wrap(err, "building path service prefix")
}
p, err := path.BuildPrefix(r.Tenant(), srs, r.Category())
return p, clues.Wrap(err, "building path").OrNil()
}

View File

@ -78,8 +78,10 @@ func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
pth, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data")
@ -330,8 +332,10 @@ func (suite *KopiaDataCollectionUnitSuite) TestFetchItemByName() {
pth, err := path.Build(
tenant,
user,
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category,
false,
folder1, folder2)

View File

@ -32,8 +32,10 @@ func (suite *MergeCollectionUnitSuite) TestReturnsPath() {
pth, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data")
@ -61,8 +63,10 @@ func (suite *MergeCollectionUnitSuite) TestItems() {
pth, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data")
@ -101,8 +105,10 @@ func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() {
pth1, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data")
@ -110,8 +116,10 @@ func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() {
pth2, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data2")
@ -142,8 +150,10 @@ func (suite *MergeCollectionUnitSuite) TestFetchItemByName() {
pth, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"some", "path", "for", "data")

View File

@ -196,14 +196,17 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
return
}
ctx := clues.Add(
cp.ctx,
"services", path.ServiceResourcesToServices(d.repoPath.ServiceResources()),
"category", d.repoPath.Category().String())
// These items were sourced from a base snapshot or were cached in kopia so we
// never had to materialize their details in-memory.
if d.info == nil || d.cached {
if d.prevPath == nil {
cp.errs.AddRecoverable(cp.ctx, clues.New("item sourced from previous backup with no previous path").
With(
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
WithClues(ctx).
Label(fault.LabelForceNoBackupCreation))
return
@ -219,9 +222,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
d.locationPath)
if err != nil {
cp.errs.AddRecoverable(cp.ctx, clues.Wrap(err, "adding item to merge list").
With(
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
WithClues(ctx).
Label(fault.LabelForceNoBackupCreation))
}
@ -235,9 +236,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
*d.info)
if err != nil {
cp.errs.AddRecoverable(cp.ctx, clues.New("adding item to details").
With(
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
WithClues(ctx).
Label(fault.LabelForceNoBackupCreation))
return
@ -515,7 +514,7 @@ func streamBaseEntries(
// TODO(ashmrtn): We may eventually want to make this a function that is
// passed in so that we can more easily switch it between different external
// service provider implementations.
if !metadata.IsMetadataFile(itemPath) {
if !metadata.IsMetadataFilePath(itemPath) {
// All items have item info in the base backup. However, we need to make
// sure we have enough metadata to find those entries. To do that we add
// the item to progress and having progress aggregate everything for

View File

@ -364,8 +364,10 @@ func TestCorsoProgressUnitSuite(t *testing.T) {
func (suite *CorsoProgressUnitSuite) SetupSuite() {
p, err := path.Build(
testTenant,
testUser,
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.EmailCategory,
true,
testInboxDir, "testFile")
@ -2867,16 +2869,40 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt
migratedUser = "user_migrate"
)
oldPrefixPathEmail, err := path.ServicePrefix(testTenant, testUser, path.ExchangeService, path.EmailCategory)
oldPrefixPathEmail, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err))
newPrefixPathEmail, err := path.ServicePrefix(testTenant, migratedUser, path.ExchangeService, path.EmailCategory)
newPrefixPathEmail, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: migratedUser,
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err))
oldPrefixPathCont, err := path.ServicePrefix(testTenant, testUser, path.ExchangeService, path.ContactsCategory)
oldPrefixPathCont, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err))
newPrefixPathCont, err := path.ServicePrefix(testTenant, migratedUser, path.ExchangeService, path.ContactsCategory)
newPrefixPathCont, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: migratedUser,
}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err))
var (

View File

@ -403,17 +403,19 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
// No excludes yet as this isn't an incremental backup.
assert.True(t, excludes.Empty())
// assume the last service in the path is sharepoint.
srs := cols[0].FullPath().ServiceResources()
service := srs[len(srs)-1].Service
t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String())
assert.Equal(
t,
path.SharePointMetadataService.String(),
cols[0].FullPath().Service().String())
assert.Equal(t, path.SharePointMetadataService, service)
// assume the last service in the path is sharepoint.
srs = cols[1].FullPath().ServiceResources()
service = srs[len(srs)-1].Service
t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String())
assert.Equal(
t,
path.SharePointService.String(),
cols[1].FullPath().Service().String())
assert.Equal(t, path.SharePointService, service)
}
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {

View File

@ -1248,8 +1248,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
metadataPath, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant,
user,
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: user,
}},
path.FilesCategory,
false)
require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err))

View File

@ -46,8 +46,10 @@ func (h itemBackupHandler) PathPrefix(
) (path.Path, error) {
return path.Build(
tenantID,
resourceOwner,
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: resourceOwner,
}},
path.FilesCategory,
false,
odConsts.DrivesPathDir,

View File

@ -41,8 +41,10 @@ func (h libraryBackupHandler) PathPrefix(
) (path.Path, error) {
return path.Build(
tenantID,
resourceOwner,
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: resourceOwner,
}},
path.LibrariesCategory,
false,
odConsts.DrivesPathDir,

View File

@ -42,8 +42,10 @@ func runComputeParentPermissionsTest(
entry, err := path.Build(
"tenant",
resourceOwner,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}},
category,
false,
strings.Split(entryPath, "/")...)
@ -51,8 +53,10 @@ func runComputeParentPermissionsTest(
rootEntry, err := path.Build(
"tenant",
resourceOwner,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}},
category,
false,
strings.Split(rootEntryPath, "/")...)

View File

@ -93,8 +93,10 @@ func CollectPages(
dir, err := path.Build(
creds.AzureTenantID,
bpc.ProtectedResource.ID(),
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.PagesCategory,
false,
tuple.Name)
@ -144,8 +146,10 @@ func CollectLists(
dir, err := path.Build(
tenantID,
bpc.ProtectedResource.ID(),
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.ListsCategory,
false,
tuple.Name)

View File

@ -218,12 +218,16 @@ func (sc *Collection) retrieveLists(
var (
metrics support.CollectionMetrics
el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = sc.fullPath.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
)
lists, err := loadSiteLists(
ctx,
sc.client.Stable,
sc.fullPath.ResourceOwner(),
protectedResource,
sc.jobs,
errs)
if err != nil {
@ -279,6 +283,10 @@ func (sc *Collection) retrievePages(
var (
metrics support.CollectionMetrics
el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = sc.fullPath.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
)
betaService := sc.betaService
@ -286,14 +294,14 @@ func (sc *Collection) retrievePages(
return metrics, clues.New("beta service required").WithClues(ctx)
}
parent, err := as.GetByID(ctx, sc.fullPath.ResourceOwner())
parent, err := as.GetByID(ctx, protectedResource)
if err != nil {
return metrics, err
}
root := ptr.Val(parent.GetWebUrl())
pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs)
pages, err := betaAPI.GetSitePages(ctx, betaService, protectedResource, sc.jobs, errs)
if err != nil {
return metrics, err
}

View File

@ -95,8 +95,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
getDir: func(t *testing.T) path.Path {
dir, err := path.Build(
tenant,
user,
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
path.ListsCategory,
false,
dirRoot)
@ -131,8 +133,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
getDir: func(t *testing.T) path.Path {
dir, err := path.Build(
tenant,
user,
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
path.PagesCategory,
false,
dirRoot)

View File

@ -69,7 +69,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx,
"category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath())
)
@ -219,9 +221,12 @@ func RestoreListCollection(
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
siteID = directory.ResourceOwner()
items = dc.Items(ctx, errs)
el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = directory.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
)
trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String())
@ -245,7 +250,7 @@ func RestoreListCollection(
ctx,
service,
itemData,
siteID,
protectedResource,
restoreContainerName)
if err != nil {
el.AddRecoverable(ctx, err)
@ -292,7 +297,10 @@ func RestorePageCollection(
var (
metrics = support.CollectionMetrics{}
directory = dc.FullPath()
siteID = directory.ResourceOwner()
// todo: pass in the resourceOwner as an idname.Provider
srs = directory.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
)
trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String())
@ -325,7 +333,7 @@ func RestorePageCollection(
ctx,
service,
itemData,
siteID,
protectedResource,
restoreContainerName)
if err != nil {
el.AddRecoverable(ctx, err)

View File

@ -1111,9 +1111,11 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
})
totalItems, _, collections, expectedData, err := stub.CollectionsForInfo(
test.service,
suite.ctrl.tenant,
suite.user,
[]path.ServiceResource{{
Service: test.service,
ProtectedResource: suite.user,
}},
restoreCfg,
[]stub.ColInfo{collection},
version.Backup,
@ -1247,11 +1249,11 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
table := []struct {
name string
resourceCat resource.Category
selectorFunc func(t *testing.T) selectors.Selector
service path.ServiceType
categories []string
name string
resourceCat resource.Category
selectorFunc func(t *testing.T) selectors.Selector
metadataServices []path.ServiceType
categories []string
}{
{
name: "Exchange",
@ -1265,7 +1267,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector
},
service: path.ExchangeService,
metadataServices: []path.ServiceType{path.ExchangeMetadataService},
categories: []string{
path.EmailCategory.String(),
path.ContactsCategory.String(),
@ -1281,7 +1283,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector
},
service: path.OneDriveService,
metadataServices: []path.ServiceType{path.OneDriveMetadataService},
categories: []string{
path.FilesCategory.String(),
},
@ -1300,7 +1302,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector
},
service: path.SharePointService,
metadataServices: []path.ServiceType{path.SharePointMetadataService},
categories: []string{
path.LibrariesCategory.String(),
// not yet in use
@ -1363,7 +1365,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
// Ignore metadata collections.
fullPath := col.FullPath()
if fullPath.Service() != test.service {
if path.ServiceResourcesMatchServices(fullPath.ServiceResources(), test.metadataServices) {
continue
}

View File

@ -79,7 +79,13 @@ func BaseCollections(
for cat := range categories {
ictx := clues.Add(ctx, "base_service", service, "base_category", cat)
full, err := path.ServicePrefix(tenant, rOwner, service, cat)
full, err := path.BuildPrefix(
tenant,
[]path.ServiceResource{{
Service: service,
ProtectedResource: rOwner,
}},
cat)
if err != nil {
// Shouldn't happen.
err = clues.Wrap(err, "making path").WithClues(ictx)

View File

@ -24,16 +24,44 @@ func (suite *CollectionsUnitSuite) TestNewPrefixCollection() {
serv := path.OneDriveService
cat := path.FilesCategory
p1, err := path.ServicePrefix("t", "ro1", serv, cat)
p1, err := path.BuildPrefix(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro1",
}},
cat)
require.NoError(t, err, clues.ToCore(err))
p2, err := path.ServicePrefix("t", "ro2", serv, cat)
p2, err := path.BuildPrefix(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro2",
}},
cat)
require.NoError(t, err, clues.ToCore(err))
items, err := path.Build("t", "ro", serv, cat, true, "fld", "itm")
items, err := path.Build(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro",
}},
cat,
true,
"fld", "itm")
require.NoError(t, err, clues.ToCore(err))
folders, err := path.Build("t", "ro", serv, cat, false, "fld")
folders, err := path.Build(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro",
}},
cat,
false,
"fld")
require.NoError(t, err, clues.ToCore(err))
table := []struct {

View File

@ -5,13 +5,30 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
func IsMetadataFile(p path.Path) bool {
switch p.Service() {
// IsMetadataFilePath checks whether the LAST service in the path
// supports metadata file types and, if so, whether the item has
// a meta suffix.
func IsMetadataFilePath(p path.Path) bool {
return IsMetadataFile(
p.ServiceResources(),
p.Category(),
p.Item())
}
// IsMetadataFile accepts the ServiceResources, cat, and Item values from
// a path (or equivalent representation) and returns true if the item
// is a Metadata entry.
func IsMetadataFile(
srs []path.ServiceResource,
cat path.CategoryType,
itemID string,
) bool {
switch srs[len(srs)-1].Service {
case path.OneDriveService:
return metadata.HasMetaSuffix(p.Item())
return metadata.HasMetaSuffix(itemID)
case path.SharePointService:
return p.Category() == path.LibrariesCategory && metadata.HasMetaSuffix(p.Item())
return cat == path.LibrariesCategory && metadata.HasMetaSuffix(itemID)
default:
return false

View File

@ -1,7 +1,7 @@
package metadata_test
import (
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
@ -18,7 +18,7 @@ import (
type boolfAssertionFunc func(assert.TestingT, bool, string, ...any) bool
type testCase struct {
service path.ServiceType
srs []path.ServiceResource
category path.CategoryType
expected boolfAssertionFunc
}
@ -39,40 +39,89 @@ var (
cases = []testCase{
{
service: path.ExchangeService,
srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.EmailCategory,
expected: assert.Falsef,
},
{
service: path.ExchangeService,
srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.ContactsCategory,
expected: assert.Falsef,
},
{
service: path.ExchangeService,
srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.EventsCategory,
expected: assert.Falsef,
},
{
service: path.OneDriveService,
srs: []path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: user,
}},
category: path.FilesCategory,
expected: assert.Truef,
},
{
service: path.SharePointService,
srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.LibrariesCategory,
expected: assert.Truef,
},
{
service: path.SharePointService,
srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.ListsCategory,
expected: assert.Falsef,
},
{
service: path.SharePointService,
srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.PagesCategory,
expected: assert.Falsef,
},
{
srs: []path.ServiceResource{
{
Service: path.OneDriveService,
ProtectedResource: user,
},
{
Service: path.ExchangeService,
ProtectedResource: user,
},
},
category: path.EventsCategory,
expected: assert.Falsef,
},
{
srs: []path.ServiceResource{
{
Service: path.ExchangeService,
ProtectedResource: user,
},
{
Service: path.OneDriveService,
ProtectedResource: user,
},
},
category: path.FilesCategory,
expected: assert.Truef,
},
}
)
@ -87,19 +136,26 @@ func TestMetadataUnitSuite(t *testing.T) {
func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() {
for _, test := range cases {
for _, ext := range metaSuffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() {
name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T()
p, err := path.Build(
tenant,
user,
test.service,
test.srs,
test.category,
true,
"file"+ext)
require.NoError(t, err, clues.ToCore(err))
test.expected(t, metadata.IsMetadataFile(p), "extension %s", ext)
test.expected(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
})
}
}
@ -108,19 +164,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() {
func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_NotMetaSuffixes() {
for _, test := range cases {
for _, ext := range notMetaSuffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() {
name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T()
p, err := path.Build(
tenant,
user,
test.service,
test.srs,
test.category,
true,
"file"+ext)
require.NoError(t, err, clues.ToCore(err))
assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext)
assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
})
}
}
@ -131,19 +194,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() {
for _, test := range cases {
for _, ext := range suffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() {
name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T()
p, err := path.Build(
tenant,
user,
test.service,
test.srs,
test.category,
false,
"file"+ext)
require.NoError(t, err, clues.ToCore(err))
assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext)
assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
})
}
}

View File

@ -75,8 +75,10 @@ func MakeMetadataCollection(
p, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant,
resourceOwner,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}},
cat,
false)
if err != nil {

View File

@ -30,8 +30,10 @@ func (suite *MetadataCollectionUnitSuite) TestFullPath() {
p, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"foo")
@ -72,8 +74,10 @@ func (suite *MetadataCollectionUnitSuite) TestItems() {
p, err := path.Build(
"a-tenant",
"a-user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory,
false,
"foo")

View File

@ -865,7 +865,6 @@ func compareItem(
t *testing.T,
colPath path.Path,
expected map[string][]byte,
service path.ServiceType,
category path.CategoryType,
item data.Stream,
mci m365Stub.ConfigInfo,
@ -875,7 +874,11 @@ func compareItem(
assert.NotZero(t, mt.ModTime())
}
switch service {
// assume the last service in the path is the data owner
srs := colPath.ServiceResources()
lastService := srs[len(srs)-1].Service
switch lastService {
case path.ExchangeService:
switch category {
case path.EmailCategory:
@ -900,7 +903,7 @@ func compareItem(
return compareDriveItem(t, expected, item, mci, rootDir)
default:
assert.FailNowf(t, "unexpected service: %s", service.String())
assert.FailNowf(t, "unexpected service: %s", lastService.String())
}
return true
@ -929,9 +932,12 @@ func checkHasCollections(
fp := g.FullPath()
loc := g.(data.LocationPather).LocationPath()
// take the last service, since it should be the one owning data
srs := fp.ServiceResources()
service := srs[len(srs)-1].Service
if fp.Service() == path.OneDriveService ||
(fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) {
if service == path.OneDriveService ||
(service == path.SharePointService && fp.Category() == path.LibrariesCategory) {
dp, err := path.ToDrivePath(fp)
if !assert.NoError(t, err, clues.ToCore(err)) {
continue
@ -942,8 +948,7 @@ func checkHasCollections(
p, err := loc.ToDataLayerPath(
fp.Tenant(),
fp.ResourceOwner(),
fp.Service(),
fp.ServiceResources(),
fp.Category(),
false)
if !assert.NoError(t, err, clues.ToCore(err)) {
@ -972,11 +977,12 @@ func checkCollections(
for _, returned := range got {
var (
hasItems bool
service = returned.FullPath().Service()
category = returned.FullPath().Category()
expectedColData = expected[returned.FullPath().String()]
folders = returned.FullPath().Elements()
rootDir = folders[len(folders)-1] == mci.RestoreCfg.Location
srs = returned.FullPath().ServiceResources()
lastService = srs[len(srs)-1].Service
)
// Need to iterate through all items even if we don't expect to find a match
@ -988,9 +994,9 @@ func checkCollections(
// is for actual pull items.
// TODO(ashmrtn): Should probably eventually check some data in metadata
// collections.
if service == path.ExchangeMetadataService ||
service == path.OneDriveMetadataService ||
service == path.SharePointMetadataService {
if lastService == path.ExchangeMetadataService ||
lastService == path.OneDriveMetadataService ||
lastService == path.SharePointMetadataService {
skipped++
continue
}
@ -1006,7 +1012,6 @@ func checkCollections(
t,
returned.FullPath(),
expectedColData,
service,
category,
item,
mci,

View File

@ -343,8 +343,10 @@ func (f failingColl) Items(ctx context.Context, errs *fault.Bus) <-chan data.Str
func (f failingColl) FullPath() path.Path {
tmp, err := path.Build(
"tenant",
"user",
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "user",
}},
path.EmailCategory,
false,
"inbox")
@ -489,7 +491,9 @@ func (suite *BackupIntgSuite) TestMailFetch() {
require.NoError(t, err, clues.ToCore(err))
for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
continue
}
@ -575,7 +579,9 @@ func (suite *BackupIntgSuite) TestDelta() {
var metadata data.BackupCollection
for _, coll := range collections {
if coll.FullPath().Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
coll.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadata = coll
}
}
@ -609,7 +615,9 @@ func (suite *BackupIntgSuite) TestDelta() {
// Delta usage is commented out at the moment, anyway. So this is currently
// a sanity check that the minimum behavior won't break.
for _, coll := range collections {
if coll.FullPath().Service() != path.ExchangeMetadataService {
if !path.ServiceResourcesMatchServices(
coll.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
ec, ok := coll.(*Collection)
require.True(t, ok, "collection is *Collection")
assert.NotNil(t, ec)
@ -664,7 +672,9 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() {
ctx, flush := tester.NewContext(t)
defer flush()
isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService
isMetadata := path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService})
streamChannel := edc.Items(ctx, fault.New(true))
// Verify that each message can be restored
@ -742,7 +752,9 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() {
require.GreaterOrEqual(t, 2, len(edcs), "expected 1 <= num collections <= 2")
for _, edc := range edcs {
isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService
isMetadata := path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService})
count := 0
for stream := range edc.Items(ctx, fault.New(true)) {
@ -872,7 +884,10 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() {
for _, edc := range collections {
var isMetadata bool
if edc.FullPath().Service() != path.ExchangeMetadataService {
// FIXME: this doesn't seem right, it's saying "if not metadata, isMetadata = true"
if !path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
isMetadata = true
assert.Equal(t, test.expected, edc.FullPath().Folder(false))
} else {
@ -1138,7 +1153,9 @@ func (suite *CollectionPopulationSuite) TestPopulateCollections() {
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++
continue
}
@ -1267,8 +1284,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
oldPath1 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := location.Append("1").ToDataLayerPath(
suite.creds.AzureTenantID,
qp.ProtectedResource.ID(),
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat,
false)
require.NoError(t, err, clues.ToCore(err))
@ -1279,8 +1298,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
oldPath2 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := location.Append("2").ToDataLayerPath(
suite.creds.AzureTenantID,
qp.ProtectedResource.ID(),
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat,
false)
require.NoError(t, err, clues.ToCore(err))
@ -1291,8 +1312,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
idPath1 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := path.Builder{}.Append("1").ToDataLayerPath(
suite.creds.AzureTenantID,
qp.ProtectedResource.ID(),
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat,
false)
require.NoError(t, err, clues.ToCore(err))
@ -1303,8 +1326,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
idPath2 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := path.Builder{}.Append("2").ToDataLayerPath(
suite.creds.AzureTenantID,
qp.ProtectedResource.ID(),
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat,
false)
require.NoError(t, err, clues.ToCore(err))
@ -1481,7 +1506,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
continue
}
if c.FullPath().Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++
checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c)
continue
@ -1640,7 +1667,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_r
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++
continue
}
@ -1706,7 +1735,15 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
)
prevPath := func(t *testing.T, at ...string) path.Path {
p, err := path.Build(tenantID, userID, path.ExchangeService, cat, false, at...)
p, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: userID,
}},
cat,
false,
at...)
require.NoError(t, err, clues.ToCore(err))
return p
@ -2064,7 +2101,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
require.NotNil(t, p)
if p.Service() == path.ExchangeMetadataService {
if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++
continue
}

View File

@ -89,8 +89,10 @@ func (suite *CollectionSuite) TestColleciton_FullPath() {
fullPath, err := path.Build(
tenant,
user,
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
path.EmailCategory,
false,
folder)
@ -113,8 +115,10 @@ func (suite *CollectionSuite) TestCollection_NewCollection() {
fullPath, err := path.Build(
tenant,
user,
path.ExchangeService,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
path.EmailCategory,
false,
folder)
@ -129,9 +133,25 @@ func (suite *CollectionSuite) TestCollection_NewCollection() {
}
func (suite *CollectionSuite) TestNewCollection_state() {
fooP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "foo")
fooP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err))
barP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "bar")
barP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"bar")
require.NoError(suite.T(), err, clues.ToCore(err))
locPB := path.Builder{}.Append("human-readable")

View File

@ -812,8 +812,10 @@ func runCreateDestinationTest(
path1, err := path.Build(
tenantID,
userID,
svc,
[]path.ServiceResource{{
Service: svc,
ProtectedResource: userID,
}},
category,
false,
containerNames1...)
@ -833,8 +835,10 @@ func runCreateDestinationTest(
path2, err := path.Build(
tenantID,
userID,
svc,
[]path.ServiceResource{{
Service: svc,
ProtectedResource: userID,
}},
category,
false,
containerNames2...)

View File

@ -57,7 +57,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx,
"category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location),
"protected_resource", clues.Hide(dc.FullPath().ResourceOwner()),
"resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath())
)

View File

@ -109,19 +109,23 @@ func migrationCollections(
// unlike exchange, which enumerates all folders on every
// backup, onedrive needs to force the owner PN -> ID migration
mc, err := path.ServicePrefix(
mc, err := path.BuildPrefix(
tenant,
bpc.ProtectedResource.ID(),
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.FilesCategory)
if err != nil {
return nil, clues.Wrap(err, "creating user id migration path")
}
mpc, err := path.ServicePrefix(
mpc, err := path.BuildPrefix(
tenant,
bpc.ProtectedResource.Name(),
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: bpc.ProtectedResource.Name(),
}},
path.FilesCategory)
if err != nil {
return nil, clues.Wrap(err, "creating user name migration path")

View File

@ -161,8 +161,10 @@ type pathPrefixer func(tID, ro, driveID string) (path.Path, error)
var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, error) {
return path.Build(
tID,
ro,
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: ro,
}},
path.FilesCategory,
false,
odConsts.DrivesPathDir,
@ -173,8 +175,10 @@ var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, erro
var defaultSharePointPathPrefixer = func(tID, ro, driveID string) (path.Path, error) {
return path.Build(
tID,
ro,
path.SharePointService,
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: ro,
}},
path.LibrariesCategory,
false,
odConsts.DrivesPathDir,

View File

@ -61,7 +61,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx,
"category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()),
"resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath())
)

View File

@ -7,7 +7,7 @@ import (
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/m365/graph/metadata"
"github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/resource"
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
@ -63,9 +63,11 @@ func GetCollectionsAndExpected(
for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo(
config.Service,
config.Tenant,
owner,
[]path.ServiceResource{{
Service: config.Service,
ProtectedResource: owner,
}},
config.RestoreCfg,
testCollections,
backupVersion)
@ -84,8 +86,8 @@ func GetCollectionsAndExpected(
}
func CollectionsForInfo(
service path.ServiceType,
tenant, user string,
tenant string,
srs []path.ServiceResource,
restoreCfg control.RestoreConfig,
allInfo []ColInfo,
backupVersion int,
@ -100,8 +102,7 @@ func CollectionsForInfo(
for _, info := range allInfo {
pth, err := path.Build(
tenant,
user,
service,
srs,
info.Category,
false,
info.PathElements...)
@ -129,9 +130,7 @@ func CollectionsForInfo(
baseExpected[info.Items[i].LookupKey] = info.Items[i].Data
// We do not count metadata files against item count
if backupVersion > 0 &&
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].Name) {
if backupVersion > 0 && metadata.IsMetadataFile(srs, info.Category, info.Items[i].Name) {
continue
}
@ -166,9 +165,13 @@ func backupOutputPathFromRestore(
inputPath path.Path,
) (path.Path, error) {
base := []string{restoreCfg.Location}
srs := inputPath.ServiceResources()
// only the last service is checked, because that should be the service
// whose data is stored..
lastService := srs[len(srs)-1].Service
// OneDrive has leading information like the drive ID.
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService {
if lastService == path.OneDriveService || lastService == path.SharePointService {
folders := inputPath.Folders()
base = append(append([]string{}, folders[:3]...), restoreCfg.Location)
@ -177,14 +180,13 @@ func backupOutputPathFromRestore(
}
}
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory {
if lastService == path.ExchangeService && inputPath.Category() == path.EmailCategory {
base = append(base, inputPath.Folders()...)
}
return path.Build(
inputPath.Tenant(),
inputPath.ResourceOwner(),
inputPath.Service(),
inputPath.ServiceResources(),
inputPath.Category(),
false,
base...)

View File

@ -326,11 +326,17 @@ func (op *BackupOperation) do(
detailsStore streamstore.Streamer,
backupID model.StableID,
) (*details.Builder, error) {
var (
reasons = selectorToReasons(op.account.ID(), op.Selectors, false)
fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors)
lastBackupVersion = version.NoBackup
)
lastBackupVersion := version.NoBackup
reasons, err := op.Selectors.Reasons(op.account.ID(), false)
if err != nil {
return nil, clues.Wrap(err, "getting reasons")
}
fallbackReasons, err := makeFallbackReasons(op.account.ID(), op.Selectors)
if err != nil {
return nil, clues.Wrap(err, "getting fallback reasons")
}
logger.Ctx(ctx).With(
"control_options", op.Options,
@ -424,13 +430,14 @@ func (op *BackupOperation) do(
return deets, nil
}
func makeFallbackReasons(tenant string, sel selectors.Selector) []identity.Reasoner {
func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) {
if sel.PathService() != path.SharePointService &&
sel.DiscreteOwner != sel.DiscreteOwnerName {
return selectorToReasons(tenant, sel, true)
return sel.Reasons(tenant, true)
}
return nil
// return nil for fallback reasons since a nil value will no-op.
return nil, nil
}
// checker to see if conditions are correct for incremental backup behavior such as
@ -472,35 +479,6 @@ func produceBackupDataCollections(
// Consumer funcs
// ---------------------------------------------------------------------------
func selectorToReasons(
tenant string,
sel selectors.Selector,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := []identity.Reasoner{}
pcs, err := sel.PathCategories()
if err != nil {
// This is technically safe, it's just that the resulting backup won't be
// usable as a base for future incremental backups.
return nil
}
owner := sel.DiscreteOwner
if useOwnerNameForID {
owner = sel.DiscreteOwnerName
}
for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} {
for _, cat := range sl {
reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat))
}
}
return reasons
}
// calls kopia to backup the collections of data
func consumeBackupCollections(
ctx context.Context,
@ -588,7 +566,10 @@ func getNewPathRefs(
// TODO(ashmrtn): In the future we can remove this first check as we'll be
// able to assume we always have the location in the previous entry. We'll end
// up doing some extra parsing, but it will simplify this code.
if repoRef.Service() == path.ExchangeService {
//
// Currently safe to check only the 0th SR, since exchange had no subservices
// at the time of the locations addition.
if repoRef.ServiceResources()[0].Service == path.ExchangeService {
newPath, newLoc, err := dataFromBackup.GetNewPathRefs(
repoRef.ToBuilder(),
entry.Modified(),

View File

@ -218,8 +218,10 @@ func makeMetadataBasePath(
p, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant,
resourceOwner,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}},
category,
false)
require.NoError(t, err, clues.ToCore(err))
@ -264,6 +266,7 @@ func makePath(t *testing.T, elements []string, isItem bool) path.Path {
return p
}
// FIXME: out of date, does not contain sharepoint support
func makeDetailsEntry(
t *testing.T,
p path.Path,
@ -288,7 +291,10 @@ func makeDetailsEntry(
Updated: updated,
}
switch p.Service() {
srs := p.ServiceResources()
lastService := srs[len(srs)-1].Service
switch lastService {
case path.ExchangeService:
if p.Category() != path.EmailCategory {
assert.FailNowf(
@ -319,7 +325,7 @@ func makeDetailsEntry(
assert.FailNowf(
t,
"service %s not supported in helper function",
p.Service().String())
lastService.String())
}
return res
@ -527,6 +533,23 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
fault.New(true))
}
// makeElements allows creation of repoRefs that wouldn't
// pass paths package validators.
func makeElements(
tenant string,
srs []path.ServiceResource,
cat path.CategoryType,
suffix ...string,
) path.Elements {
elems := append(
path.Elements{tenant},
path.ServiceResourcesToElements(srs)...)
elems = append(elems, cat.String())
elems = append(elems, suffix...)
return elems
}
func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() {
var (
tenant = "a-tenant"
@ -707,17 +730,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
DetailsModel: details.DetailsModel{
Entries: []details.Entry{
{
RepoRef: stdpath.Join(
append(
[]string{
itemPath1.Tenant(),
itemPath1.Service().String(),
itemPath1.ResourceOwner(),
path.UnknownCategory.String(),
},
itemPath1.Folders()...,
)...,
),
RepoRef: stdpath.Join(makeElements(
itemPath1.Tenant(),
itemPath1.ServiceResources(),
path.UnknownCategory,
itemPath1.Folders()...)...),
ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
@ -738,16 +755,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
res := newMockDetailsMergeInfoer()
p := makePath(
suite.T(),
[]string{
makeElements(
itemPath1.Tenant(),
path.OneDriveService.String(),
itemPath1.ResourceOwner(),
path.FilesCategory.String(),
itemPath1.ServiceResources(),
path.FilesCategory,
"personal",
"item1",
},
true,
)
"item1"),
true)
res.add(itemPath1, p, nil)
@ -1635,7 +1649,15 @@ func (suite *AssistBackupIntegrationSuite) TestBackupTypesForFailureModes() {
pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID}
tmp, err := path.Build(tenantID, userID, path.OneDriveService, path.FilesCategory, false, pathElements...)
tmp, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: userID,
}},
path.FilesCategory,
false,
pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err))
locPath := path.Builder{}.Append(tmp.Folders()...)
@ -1916,7 +1938,15 @@ func (suite *AssistBackupIntegrationSuite) TestExtensionsIncrementals() {
pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID}
tmp, err := path.Build(tenantID, userID, path.OneDriveService, path.FilesCategory, false, pathElements...)
tmp, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: userID,
}},
path.FilesCategory,
false,
pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err))
locPath := path.Builder{}.Append(tmp.Folders()...)

View File

@ -80,12 +80,7 @@ func getManifestsAndMetadata(
// 3. the current reasons contain all the necessary manifests.
// Note: This is not relevant for assist backups, since they are newly introduced
// and they don't exist with fallback reasons.
bb = bb.MergeBackupBases(
ctx,
fbb,
func(r identity.Reasoner) string {
return r.Service().String() + r.Category().String()
})
bb = bb.MergeBackupBases(ctx, fbb, kopia.BaseKeyServiceCategory)
if !getMetadata {
return bb, nil, false, nil

View File

@ -344,7 +344,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
assert.Equal(
t,
path.ExchangeMetadataService,
p.Service(),
p.ServiceResources()[0].Service,
"read data service")
assert.Contains(
@ -354,8 +354,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
path.ContactsCategory,
},
p.Category(),
"read data category doesn't match a given reason",
)
"read data category doesn't match a given reason")
}
},
expectMans: kopia.NewMockBackupBases().WithMergeBases(

View File

@ -42,10 +42,9 @@ func locationRef(
func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, error) {
if len(locRef.Elements()) == 0 {
res, err := path.ServicePrefix(
res, err := path.BuildPrefix(
repoRef.Tenant(),
repoRef.ResourceOwner(),
repoRef.Service(),
repoRef.ServiceResources(),
repoRef.Category())
if err != nil {
return nil, clues.Wrap(err, "getting prefix for empty location")
@ -56,8 +55,7 @@ func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, erro
return locRef.ToDataLayerPath(
repoRef.Tenant(),
repoRef.ResourceOwner(),
repoRef.Service(),
repoRef.ServiceResources(),
repoRef.Category(),
false)
}

View File

@ -36,12 +36,8 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() {
repoRef path.Path,
unescapedFolders ...string,
) string {
return path.Builder{}.
Append(
repoRef.Tenant(),
repoRef.Service().String(),
repoRef.ResourceOwner(),
repoRef.Category().String()).
pfx, _ := repoRef.Halves()
return pfx.
Append(unescapedFolders...).
String()
}

View File

@ -347,7 +347,13 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
// },
}
rrPfx, err := path.ServicePrefix(acct.ID(), uidn.ID(), service, path.EmailCategory)
rrPfx, err := path.BuildPrefix(
acct.ID(),
[]path.ServiceResource{{
Service: service,
ProtectedResource: uidn.ID(),
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err))
// strip the category from the prefix; we primarily want the tenant and resource owner.

View File

@ -33,6 +33,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
idMock "github.com/alcionai/corso/src/pkg/backup/identity/mock"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
@ -244,7 +245,11 @@ func checkBackupIsInManifests(
for _, category := range categories {
t.Run(category.String(), func(t *testing.T) {
var (
r = kopia.NewReason("", resourceOwner, sel.PathService(), category)
r = idMock.Reason{
Cat: category,
Svc: sel.PathService(),
Resource: resourceOwner,
}
tags = map[string]string{kopia.TagBackupCategory: ""}
found bool
)
@ -291,11 +296,15 @@ func checkMetadataFilesExist(
paths := []path.RestorePaths{}
pathsByRef := map[string][]string{}
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}}
for _, fName := range files {
p, err := path.Builder{}.
Append(fName).
ToServiceCategoryMetadataPath(tenant, resourceOwner, service, category, true)
ToServiceCategoryMetadataPath(tenant, srs, category, true)
if !assert.NoError(t, err, "bad metadata path", clues.ToCore(err)) {
continue
}

View File

@ -213,7 +213,13 @@ func runDriveIncrementalTest(
}
)
rrPfx, err := path.ServicePrefix(atid, roidn.ID(), service, category)
rrPfx, err := path.BuildPrefix(
atid,
[]path.ServiceResource{{
Service: service,
ProtectedResource: roidn.ID(),
}},
category)
require.NoError(t, err, clues.ToCore(err))
// strip the category from the prefix; we primarily want the tenant and resource owner.

View File

@ -198,8 +198,13 @@ func collect(
service path.ServiceType,
col Collectable,
) (data.BackupCollection, error) {
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: col.purpose,
}}
// construct the path of the container
p, err := path.Builder{}.ToStreamStorePath(tenantID, col.purpose, service, false)
p, err := path.Builder{}.ToStreamStorePath(tenantID, srs, false)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx)
}
@ -257,10 +262,15 @@ func read(
rer inject.RestoreProducer,
errs *fault.Bus,
) error {
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: col.purpose,
}}
// construct the path of the container
p, err := path.Builder{}.
Append(col.itemName).
ToStreamStorePath(tenantID, col.purpose, service, true)
ToStreamStorePath(tenantID, srs, true)
if err != nil {
return clues.Stack(err).WithClues(ctx)
}

View File

@ -1122,8 +1122,10 @@ func makeItemPath(
p, err := path.Build(
tenant,
resourceOwner,
service,
[]path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}},
category,
true,
elems...)

View File

@ -5,9 +5,20 @@ import "github.com/alcionai/corso/src/pkg/path"
// Reasoner describes the parts of the backup that make up its
// data identity: the tenant, protected resources, services, and
// categories which are held within the backup.
//
// Reasoner only recognizes the "primary" protected resource and
// service. IE: subservice resources and services are not recognized
// as part of the backup Reason.
type Reasoner interface {
Tenant() string
// ProtectedResource represents the Primary protected resource.
// IE: if a path or backup supports subservices, this value
// should only provide the first service's resource, and not the
// resource for any subservice.
ProtectedResource() string
// Service represents the Primary service.
// IE: if a path or backup supports subservices, this value
// should only provide the first service; not a subservice.
Service() path.ServiceType
Category() path.CategoryType
// SubtreePath returns the path prefix for data in existing backups that have

View File

@ -0,0 +1,47 @@
package mock
import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/path"
)
type Reason struct {
TenantID string
Cat path.CategoryType
Svc path.ServiceType
Resource string
SubtreeErr error
}
func (r Reason) Tenant() string {
return r.TenantID
}
func (r Reason) Category() path.CategoryType {
return r.Cat
}
func (r Reason) Service() path.ServiceType {
return r.Svc
}
func (r Reason) ProtectedResource() string {
return r.Resource
}
func (r Reason) SubtreePath() (path.Path, error) {
if r.SubtreeErr != nil {
return nil, r.SubtreeErr
}
p, err := path.BuildPrefix(
r.Tenant(),
[]path.ServiceResource{{
ProtectedResource: r.Resource,
Service: r.Svc,
}},
r.Category())
return p, clues.Wrap(err, "building path").OrNil()
}

View File

@ -104,7 +104,17 @@ func (suite *RestoreUnitSuite) TestEnsureRestoreConfigDefaults() {
}
func (suite *RestoreUnitSuite) TestRestoreConfig_piiHandling() {
p, err := path.Build("tid", "ro", path.ExchangeService, path.EmailCategory, true, "foo", "bar", "baz")
p, err := path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "ro",
}},
path.EmailCategory,
true,
"foo",
"bar",
"baz")
require.NoError(suite.T(), err, clues.ToCore(err))
cdrc := control.DefaultRestoreConfig(dttm.HumanReadable)

View File

@ -19,7 +19,15 @@ const itemID = "item_id"
var (
err error
itemPath, _ = path.Build("tid", "own", path.ExchangeService, path.ContactsCategory, false, "foo")
itemPath, _ = path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "own",
}},
path.ContactsCategory,
false,
"foo")
)
// ---------------------------------------------------------------------------

346
src/pkg/path/builder.go Normal file
View File

@ -0,0 +1,346 @@
package path
import (
"bytes"
"crypto/sha256"
"fmt"
"github.com/alcionai/clues"
)
// interface compliance required for handling PII
var (
_ clues.Concealer = &Builder{}
_ fmt.Stringer = &Builder{}
)
// Builder is a simple path representation that only tracks path elements. It
// can join, escape, and unescape elements. Higher-level packages are expected
// to wrap this struct to build resource-specific contexts (e.x. an
// ExchangeMailPath).
// Resource-specific paths allow access to more information like segments in the
// path. Builders that are turned into resource paths later on do not need to
// manually add prefixes for items that normally appear in the data layer (ex.
// tenant ID, service, user ID, etc).
type Builder struct {
// Unescaped version of elements.
elements Elements
}
// Append creates a copy of this Builder and adds the given elements them to the
// end of the new Builder. Elements are added in the order they are passed.
func (pb Builder) Append(elements ...string) *Builder {
res := &Builder{elements: make([]string, len(pb.elements))}
copy(res.elements, pb.elements)
// Unescaped elements can't fail validation.
//nolint:errcheck
res.appendElements(false, elements)
return res
}
func (pb *Builder) appendElements(escaped bool, elements []string) error {
for _, e := range elements {
if len(e) == 0 {
continue
}
tmp := e
if escaped {
tmp = TrimTrailingSlash(tmp)
// If tmp was just the path separator then it will be empty now.
if len(tmp) == 0 {
continue
}
if err := validateEscapedElement(tmp); err != nil {
return err
}
tmp = unescape(tmp)
}
pb.elements = append(pb.elements, tmp)
}
return nil
}
// UnescapeAndAppend creates a copy of this Builder and adds one or more already
// escaped path elements to the end of the new Builder. Elements are added in
// the order they are passed.
func (pb Builder) UnescapeAndAppend(elements ...string) (*Builder, error) {
res := &Builder{elements: make([]string, 0, len(pb.elements))}
copy(res.elements, pb.elements)
if err := res.appendElements(true, elements); err != nil {
return nil, err
}
return res, nil
}
// SplitUnescapeAppend takes in an escaped string representing a directory
// path, splits the string, and appends it to the current builder.
func (pb Builder) SplitUnescapeAppend(s string) (*Builder, error) {
elems := Split(TrimTrailingSlash(s))
return pb.UnescapeAndAppend(elems...)
}
func (pb Builder) PopFront() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
elements := make([]string, len(pb.elements)-1)
copy(elements, pb.elements[1:])
return &Builder{
elements: elements,
}
}
// Dir removes the last element from the builder.
func (pb Builder) Dir() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
return &Builder{
// Safe to use the same elements because Builders are immutable.
elements: pb.elements[:len(pb.elements)-1],
}
}
// HeadElem returns the first element in the Builder.
func (pb Builder) HeadElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[0]
}
// LastElem returns the last element in the Builder.
func (pb Builder) LastElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[len(pb.elements)-1]
}
// UpdateParent updates leading elements matching prev to be cur and returns
// true if it was updated. If prev is not a prefix of this Builder changes
// nothing and returns false. If either prev or cur is nil does nothing and
// returns false.
func (pb *Builder) UpdateParent(prev, cur *Builder) bool {
if prev == cur || prev == nil || cur == nil || len(prev.Elements()) > len(pb.Elements()) {
return false
}
parent := true
for i, e := range prev.Elements() {
if pb.elements[i] != e {
parent = false
break
}
}
if !parent {
return false
}
pb.elements = append(cur.Elements(), pb.elements[len(prev.Elements()):]...)
return true
}
// ShortRef produces a truncated hash of the builder that
// acts as a unique identifier.
func (pb Builder) ShortRef() string {
if len(pb.elements) == 0 {
return ""
}
data := bytes.Buffer{}
for _, element := range pb.elements {
data.WriteString(element)
}
sum := sha256.Sum256(data.Bytes())
// Some conversions to get the right number of characters in the output. This
// outputs hex, so we need to take the target number of characters and do the
// equivalent of (shortRefCharacters * 4) / 8. This is
// <number of bits represented> / <bits per byte> which gets us how many bytes
// to give to our format command.
numBytes := shortRefCharacters / 2
return fmt.Sprintf("%x", sum[:numBytes])
}
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
func (pb Builder) Elements() Elements {
return append(Elements{}, pb.elements...)
}
// withPrefix creates a Builder prefixed with the parameter values, and
// concatenated with the current builder elements.
func (pb Builder) withPrefix(elements ...string) *Builder {
res := Builder{}.Append(elements...)
res.elements = append(res.elements, pb.elements...)
return res
}
// verifyPrefix ensures that the tenant and resourceOwner are valid
// values, and that the builder has some directory structure.
// func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
// if err := verifyPrefixValues(tenant, resourceOwner); err != nil {
// return err
// }
// if len(pb.elements) == 0 {
// return clues.New("missing path beyond prefix")
// }
// return nil
// }
// ---------------------------------------------------------------------------
// Data Layer Path Transformers
// ---------------------------------------------------------------------------
func (pb Builder) ToStreamStorePath(
tenant string,
srs []ServiceResource,
isItem bool,
) (Path, error) {
cat := DetailsCategory
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
dlrp := newDataLayerResourcePath(pb, tenant, toMetadataServices(srs), cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToServiceCategoryMetadataPath(
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
dlrp := newDataLayerResourcePath(pb, tenant, toMetadataServices(srs), cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToDataLayerPath(
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
dlrp := newDataLayerResourcePath(pb, tenant, srs, cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToDataLayerExchangePathForCategory(
tenant, mailboxID string,
category CategoryType,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(ExchangeService, mailboxID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, category, isItem)
}
func (pb Builder) ToDataLayerOneDrivePath(
tenant, userID string,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(OneDriveService, userID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, FilesCategory, isItem)
}
func (pb Builder) ToDataLayerSharePointPath(
tenant, siteID string,
category CategoryType,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(SharePointService, siteID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, category, isItem)
}
// TODO: ToDataLayerGroupsPath()
// ---------------------------------------------------------------------------
// Stringers and PII Concealer Compliance
// ---------------------------------------------------------------------------
// Conceal produces a concealed representation of the builder, suitable for
// logging, storing in errors, and other output.
func (pb Builder) Conceal() string {
return pb.elements.Conceal()
}
// Format produces a concealed representation of the builder, even when
// used within a PrintF, suitable for logging, storing in errors,
// and other output.
func (pb Builder) Format(fs fmt.State, _ rune) {
fmt.Fprint(fs, pb.Conceal())
}
// String returns a string that contains all path elements joined together.
// Elements of the path that need escaping are escaped.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) String() string {
return pb.elements.String()
}
// PlainString returns an unescaped, unmodified string of the builder.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) PlainString() string {
return pb.elements.PlainString()
}

View File

@ -0,0 +1,377 @@
package path
import (
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type BuilderUnitSuite struct {
tester.Suite
}
func TestBuilderUnitSuite(t *testing.T) {
suite.Run(t, &BuilderUnitSuite{Suite: tester.NewUnitSuite(t)})
}
// set the clues hashing to mask for the span of this suite
func (suite *BuilderUnitSuite) SetupSuite() {
clues.SetHasher(clues.HashCfg{HashAlg: clues.Flatmask})
}
// revert clues hashing to plaintext for all other tests
func (suite *BuilderUnitSuite) TeardownSuite() {
clues.SetHasher(clues.NoHash())
}
func (suite *BuilderUnitSuite) TestAppend() {
table := append(append([]testData{}, genericCases...), basicUnescapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := Builder{}.Append(test.input...)
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *BuilderUnitSuite) TestAppendItem() {
t := suite.T()
srs, err := NewServiceResources(ExchangeService, "ro")
require.NoError(t, err, clues.ToCore(err))
p, err := Build("t", srs, EmailCategory, false, "foo", "bar")
require.NoError(t, err, clues.ToCore(err))
pb := p.ToBuilder()
assert.Equal(t, pb.String(), p.String())
pb = pb.Append("qux")
p, err = p.AppendItem("qux")
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, pb.String(), p.String())
_, err = p.AppendItem("fnords")
require.Error(t, err, clues.ToCore(err))
}
func (suite *BuilderUnitSuite) TestUnescapeAndAppend() {
table := append(append([]testData{}, genericCases...), basicEscapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := Builder{}.UnescapeAndAppend(test.input...)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *BuilderUnitSuite) TestEscapedFailure() {
target := "i_s"
for c := range charactersToEscape {
suite.Run(fmt.Sprintf("Unescaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(suite.T(), err, "path with unescaped %s did not error", string(c))
})
}
}
func (suite *BuilderUnitSuite) TestBadEscapeSequenceErrors() {
target := `i\_s/a`
notEscapes := []rune{'a', 'b', '#', '%'}
for _, c := range notEscapes {
suite.Run(fmt.Sprintf("Escaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(
suite.T(),
err,
"path with bad escape sequence %c%c did not error",
escapeCharacter,
c)
})
}
}
func (suite *BuilderUnitSuite) TestTrailingEscapeChar() {
base := []string{"this", "is", "a", "path"}
for i := 0; i < len(base); i++ {
suite.Run(fmt.Sprintf("Element%v", i), func() {
path := make([]string, len(base))
copy(path, base)
path[i] = path[i] + string(escapeCharacter)
_, err := Builder{}.UnescapeAndAppend(path...)
assert.Error(
suite.T(),
err,
"path with trailing escape character did not error")
})
}
}
func (suite *BuilderUnitSuite) TestElements() {
table := []struct {
name string
input []string
output []string
pathFunc func(elements []string) (*Builder, error)
}{
{
name: "SimpleEscapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
{
name: "SimpleUnescapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.Append(elements...), nil
},
},
{
name: "EscapedPath",
input: []string{"this", `is\/`, "a", "path"},
output: []string{"this", "is/", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := test.pathFunc(test.input)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, Elements(test.output), p.Elements())
})
}
}
func (suite *BuilderUnitSuite) TestPopFront() {
table := []struct {
name string
base *Builder
expectedString string
}{
{
name: "Empty",
base: &Builder{},
expectedString: "",
},
{
name: "OneElement",
base: Builder{}.Append("something"),
expectedString: "",
},
{
name: "TwoElements",
base: Builder{}.Append("something", "else"),
expectedString: "else",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expectedString, test.base.PopFront().String())
})
}
}
func (suite *BuilderUnitSuite) TestShortRef() {
table := []struct {
name string
inputElements []string
expectedLen int
}{
{
name: "PopulatedPath",
inputElements: []string{"this", "is", "a", "path"},
expectedLen: shortRefCharacters,
},
{
name: "EmptyPath",
inputElements: nil,
expectedLen: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
pb := Builder{}.Append(test.inputElements...)
ref := pb.ShortRef()
assert.Len(suite.T(), ref, test.expectedLen)
})
}
}
func (suite *BuilderUnitSuite) TestShortRefIsStable() {
t := suite.T()
pb := Builder{}.Append("this", "is", "a", "path")
prevRef := pb.ShortRef()
assert.Len(t, prevRef, shortRefCharacters)
for i := 0; i < 5; i++ {
ref := pb.ShortRef()
assert.Len(t, ref, shortRefCharacters)
assert.Equal(t, prevRef, ref, "ShortRef changed between calls")
prevRef = ref
}
}
func (suite *BuilderUnitSuite) TestShortRefIsUnique() {
pb1 := Builder{}.Append("this", "is", "a", "path")
pb2 := pb1.Append("also")
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
// TestShortRefUniqueWithEscaping tests that two paths that output the same
// unescaped string but different escaped strings have different shortrefs. This
// situation can occur when one path has embedded path separators while the
// other does not but contains the same characters.
func (suite *BuilderUnitSuite) TestShortRefUniqueWithEscaping() {
pb1 := Builder{}.Append(`this`, `is`, `a`, `path`)
pb2 := Builder{}.Append(`this`, `is/a`, `path`)
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
func (suite *BuilderUnitSuite) TestFolder() {
table := []struct {
name string
p func(t *testing.T) Path
escape bool
expectFolder string
expectSplit []string
}{
{
name: "clean path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "clean path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a//b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a\\//b/c",
expectSplit: []string{"a\\/", "b", "c"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := test.p(t)
result := p.Folder(test.escape)
assert.Equal(t, test.expectFolder, result)
assert.Equal(t, test.expectSplit, Split(result))
})
}
}
func (suite *BuilderUnitSuite) TestPIIHandling() {
t := suite.T()
srs, err := NewServiceResources(ExchangeService, "ro")
require.NoError(t, err, clues.ToCore(err))
p, err := Build("t", srs, EventsCategory, true, "dir", "item")
require.NoError(t, err)
table := []struct {
name string
p Path
expect string
expectPlain string
}{
{
name: "standard path",
p: p,
expect: "***/exchange/***/events/***/***",
expectPlain: "t/exchange/ro/events/dir/item",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expect, test.p.Conceal(), "conceal")
assert.Equal(t, test.expectPlain, test.p.String(), "string")
assert.Equal(t, test.expect, fmt.Sprintf("%s", test.p), "fmt %%s")
assert.Equal(t, test.expect, fmt.Sprintf("%+v", test.p), "fmt %%+v")
assert.Equal(t, test.expectPlain, test.p.PlainString(), "plain")
})
}
}

View File

@ -0,0 +1,90 @@
package path
import (
"fmt"
"strings"
"github.com/alcionai/clues"
)
var ErrorUnknownCategory = clues.New("unknown category string")
// CategoryType denotes what category of data the path corresponds to. The order
// of the enums below can be changed, but the string representation of each enum
// must remain the same or migration code needs to be added to handle changes to
// the string format.
type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
ListsCategory // lists
LibrariesCategory // libraries
PagesCategory // pages
DetailsCategory // details
)
func ToCategoryType(category string) CategoryType {
cat := strings.ToLower(category)
switch cat {
case strings.ToLower(EmailCategory.String()):
return EmailCategory
case strings.ToLower(ContactsCategory.String()):
return ContactsCategory
case strings.ToLower(EventsCategory.String()):
return EventsCategory
case strings.ToLower(FilesCategory.String()):
return FilesCategory
case strings.ToLower(LibrariesCategory.String()):
return LibrariesCategory
case strings.ToLower(ListsCategory.String()):
return ListsCategory
case strings.ToLower(PagesCategory.String()):
return PagesCategory
case strings.ToLower(DetailsCategory.String()):
return DetailsCategory
default:
return UnknownCategory
}
}
// ---------------------------------------------------------------------------
// Service-Category pairings
// ---------------------------------------------------------------------------
// serviceCategories is a mapping of all valid service/category pairs for
// non-metadata paths.
var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
ExchangeService: {
EmailCategory: {},
ContactsCategory: {},
EventsCategory: {},
},
OneDriveService: {
FilesCategory: {},
},
SharePointService: {
LibrariesCategory: {},
ListsCategory: {},
PagesCategory: {},
},
}
func ValidateServiceAndCategory(service ServiceType, category CategoryType) error {
cats, ok := serviceCategories[service]
if !ok {
return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))
}
if _, ok := cats[category]; !ok {
return clues.New("unknown service/category combination").
With("service", fmt.Sprintf("%q", service), "category", fmt.Sprintf("%q", category))
}
return nil
}

View File

@ -2,6 +2,7 @@ package path
import "github.com/alcionai/clues"
// TODO: Move this into m365/collection/drive
// drivePath is used to represent path components
// of an item within the drive i.e.
// Given `drives/b!X_8Z2zuXpkKkXZsr7gThk9oJpuj0yXVGnK5_VjRRPK-q725SX_8ZQJgFDK8PlFxA/root:/Folder1/Folder2/file`

View File

@ -59,7 +59,10 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() {
suite.Run(tt.name, func() {
t := suite.T()
p, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, false, tt.pathElements...)
srs, err := path.NewServiceResources(path.OneDriveService, "user")
require.NoError(t, err, clues.ToCore(err))
p, err := path.Build("tenant", srs, path.FilesCategory, false, tt.pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err))
got, err := path.ToDrivePath(p)

View File

@ -51,8 +51,6 @@
package path
import (
"bytes"
"crypto/sha256"
"fmt"
"strings"
@ -81,10 +79,12 @@ var (
// string.
type Path interface {
String() string
Service() ServiceType
// ServiceResources produces all of the services and subservices, along with
// the protected resource paired with the service, as contained in the path,
// in their order of appearance.
ServiceResources() []ServiceResource
Category() CategoryType
Tenant() string
ResourceOwner() string
Folder(escaped bool) string
Folders() Elements
Item() string
@ -115,6 +115,10 @@ type Path interface {
ShortRef() string
// ToBuilder returns a Builder instance that represents the current Path.
ToBuilder() *Builder
// Halves breaks the path into its prefix (tenant, services, resources, category)
// and suffix (all parts after the prefix). If either half is empty, that half
// returns an empty, non-nil, value.
Halves() (*Builder, Elements)
// Every path needs to comply with these funcs to ensure that PII
// is appropriately hidden from logging, errors, and other outputs.
@ -122,12 +126,6 @@ type Path interface {
fmt.Stringer
}
// interface compliance required for handling PII
var (
_ clues.Concealer = &Builder{}
_ fmt.Stringer = &Builder{}
)
// RestorePaths denotes the location to find an item in kopia and the path of
// the collection to place the item in for restore.
type RestorePaths struct {
@ -135,396 +133,34 @@ type RestorePaths struct {
RestorePath Path
}
// Builder is a simple path representation that only tracks path elements. It
// can join, escape, and unescape elements. Higher-level packages are expected
// to wrap this struct to build resource-specific contexts (e.x. an
// ExchangeMailPath).
// Resource-specific paths allow access to more information like segments in the
// path. Builders that are turned into resource paths later on do not need to
// manually add prefixes for items that normally appear in the data layer (ex.
// tenant ID, service, user ID, etc).
type Builder struct {
// Unescaped version of elements.
elements Elements
}
// Append creates a copy of this Builder and adds the given elements them to the
// end of the new Builder. Elements are added in the order they are passed.
func (pb Builder) Append(elements ...string) *Builder {
res := &Builder{elements: make([]string, len(pb.elements))}
copy(res.elements, pb.elements)
// Unescaped elements can't fail validation.
//nolint:errcheck
res.appendElements(false, elements)
return res
}
func (pb *Builder) appendElements(escaped bool, elements []string) error {
for _, e := range elements {
if len(e) == 0 {
continue
}
tmp := e
if escaped {
tmp = TrimTrailingSlash(tmp)
// If tmp was just the path separator then it will be empty now.
if len(tmp) == 0 {
continue
}
if err := validateEscapedElement(tmp); err != nil {
return err
}
tmp = unescape(tmp)
}
pb.elements = append(pb.elements, tmp)
}
return nil
}
// UnescapeAndAppend creates a copy of this Builder and adds one or more already
// escaped path elements to the end of the new Builder. Elements are added in
// the order they are passed.
func (pb Builder) UnescapeAndAppend(elements ...string) (*Builder, error) {
res := &Builder{elements: make([]string, 0, len(pb.elements))}
copy(res.elements, pb.elements)
if err := res.appendElements(true, elements); err != nil {
return nil, err
}
return res, nil
}
// SplitUnescapeAppend takes in an escaped string representing a directory
// path, splits the string, and appends it to the current builder.
func (pb Builder) SplitUnescapeAppend(s string) (*Builder, error) {
elems := Split(TrimTrailingSlash(s))
return pb.UnescapeAndAppend(elems...)
}
func (pb Builder) PopFront() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
elements := make([]string, len(pb.elements)-1)
copy(elements, pb.elements[1:])
return &Builder{
elements: elements,
}
}
// Dir removes the last element from the builder.
func (pb Builder) Dir() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
return &Builder{
// Safe to use the same elements because Builders are immutable.
elements: pb.elements[:len(pb.elements)-1],
}
}
// HeadElem returns the first element in the Builder.
func (pb Builder) HeadElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[0]
}
// LastElem returns the last element in the Builder.
func (pb Builder) LastElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[len(pb.elements)-1]
}
// UpdateParent updates leading elements matching prev to be cur and returns
// true if it was updated. If prev is not a prefix of this Builder changes
// nothing and returns false. If either prev or cur is nil does nothing and
// returns false.
func (pb *Builder) UpdateParent(prev, cur *Builder) bool {
if prev == cur || prev == nil || cur == nil || len(prev.Elements()) > len(pb.Elements()) {
return false
}
parent := true
for i, e := range prev.Elements() {
if pb.elements[i] != e {
parent = false
break
}
}
if !parent {
return false
}
pb.elements = append(cur.Elements(), pb.elements[len(prev.Elements()):]...)
return true
}
// ShortRef produces a truncated hash of the builder that
// acts as a unique identifier.
func (pb Builder) ShortRef() string {
if len(pb.elements) == 0 {
return ""
}
data := bytes.Buffer{}
for _, element := range pb.elements {
data.WriteString(element)
}
sum := sha256.Sum256(data.Bytes())
// Some conversions to get the right number of characters in the output. This
// outputs hex, so we need to take the target number of characters and do the
// equivalent of (shortRefCharacters * 4) / 8. This is
// <number of bits represented> / <bits per byte> which gets us how many bytes
// to give to our format command.
numBytes := shortRefCharacters / 2
return fmt.Sprintf("%x", sum[:numBytes])
}
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
func (pb Builder) Elements() Elements {
return append(Elements{}, pb.elements...)
}
func ServicePrefix(
tenant, resourceOwner string,
s ServiceType,
c CategoryType,
) (Path, error) {
pb := Builder{}
if err := ValidateServiceAndCategory(s, c); err != nil {
return nil, err
}
if err := verifyInputValues(tenant, resourceOwner); err != nil {
return nil, err
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(tenant, s.String(), resourceOwner, c.String()),
service: s,
category: c,
hasItem: false,
}, nil
}
// withPrefix creates a Builder prefixed with the parameter values, and
// concatenated with the current builder elements.
func (pb Builder) withPrefix(elements ...string) *Builder {
res := Builder{}.Append(elements...)
res.elements = append(res.elements, pb.elements...)
return res
}
// ---------------------------------------------------------------------------
// Data Layer Path Transformers
// ---------------------------------------------------------------------------
func (pb Builder) ToStreamStorePath(
tenant, purpose string,
service ServiceType,
isItem bool,
) (Path, error) {
if err := verifyInputValues(tenant, purpose); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
metadataService := UnknownService
switch service {
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
metadataService.String(),
purpose,
DetailsCategory.String()),
service: metadataService,
category: DetailsCategory,
hasItem: isItem,
}, nil
}
func (pb Builder) ToServiceCategoryMetadataPath(
tenant, user string,
service ServiceType,
category CategoryType,
isItem bool,
) (Path, error) {
if err := ValidateServiceAndCategory(service, category); err != nil {
return nil, err
}
if err := verifyInputValues(tenant, user); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
metadataService := UnknownService
switch service {
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
metadataService.String(),
user,
category.String(),
),
service: metadataService,
category: category,
hasItem: isItem,
}, nil
}
func (pb Builder) ToDataLayerPath(
tenant, user string,
service ServiceType,
category CategoryType,
isItem bool,
) (Path, error) {
if err := ValidateServiceAndCategory(service, category); err != nil {
return nil, err
}
if err := pb.verifyPrefix(tenant, user); err != nil {
return nil, err
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
service.String(),
user,
category.String()),
service: service,
category: category,
hasItem: isItem,
}, nil
}
func (pb Builder) ToDataLayerExchangePathForCategory(
tenant, user string,
category CategoryType,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, user, ExchangeService, category, isItem)
}
func (pb Builder) ToDataLayerOneDrivePath(
tenant, user string,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, user, OneDriveService, FilesCategory, isItem)
}
func (pb Builder) ToDataLayerSharePointPath(
tenant, site string,
category CategoryType,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, site, SharePointService, category, isItem)
}
// ---------------------------------------------------------------------------
// Stringers and PII Concealer Compliance
// ---------------------------------------------------------------------------
// Conceal produces a concealed representation of the builder, suitable for
// logging, storing in errors, and other output.
func (pb Builder) Conceal() string {
return pb.elements.Conceal()
}
// Format produces a concealed representation of the builder, even when
// used within a PrintF, suitable for logging, storing in errors,
// and other output.
func (pb Builder) Format(fs fmt.State, _ rune) {
fmt.Fprint(fs, pb.Conceal())
}
// String returns a string that contains all path elements joined together.
// Elements of the path that need escaping are escaped.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) String() string {
return pb.elements.String()
}
// PlainString returns an unescaped, unmodified string of the builder.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) PlainString() string {
return pb.elements.PlainString()
}
// ---------------------------------------------------------------------------
// Exported Helpers
// ---------------------------------------------------------------------------
func Build(
tenant, resourceOwner string,
service ServiceType,
tenant string,
srs []ServiceResource,
category CategoryType,
hasItem bool,
elements ...string,
) (Path, error) {
b := Builder{}.Append(elements...)
return Builder{}.
Append(elements...).
ToDataLayerPath(tenant, srs, category, hasItem)
}
return b.ToDataLayerPath(
tenant, resourceOwner,
service, category,
hasItem)
func BuildPrefix(
tenant string,
srs []ServiceResource,
cat CategoryType,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
dlrp := newDataLayerResourcePath(Builder{}, tenant, srs, cat, false)
return &dlrp, nil
}
// FromDataLayerPath parses the escaped path p, validates the elements in p
@ -544,24 +180,37 @@ func FromDataLayerPath(p string, isItem bool) (Path, error) {
return nil, clues.Stack(errParsingPath, err).With("path_string", p)
}
// initial check for minimum required elements:
// tenant, service, resource, category, container/item
if len(pb.elements) < 5 {
return nil, clues.New("path has too few segments").With("path_string", p)
}
service, category, err := validateServiceAndCategoryStrings(
pb.elements[1],
pb.elements[3],
)
srs, catIdx, err := elementsToServiceResources(pb.elements[1:])
if err != nil {
return nil, clues.Stack(err)
}
// follow-up check: if more than one service exists, revisit the len check.
if len(srs) > 1 && len(pb.elements) < 3+(2*len(srs)) {
return nil, clues.New("path has too few segments").With("path_string", p)
}
// +1 to account for slicing the tenant when calling the transformer func.
category := ToCategoryType(pb.elements[catIdx+1])
if err := verifyPrefixValues(pb.elements[0], srs, category); err != nil {
return nil, clues.Stack(errParsingPath, err).With("path_string", p)
}
return &dataLayerResourcePath{
Builder: *pb,
service: service,
category: category,
hasItem: isItem,
}, nil
dlrp := dataLayerResourcePath{
Builder: *pb,
serviceResources: srs,
category: category,
hasItem: isItem,
}
return &dlrp, nil
}
// TrimTrailingSlash takes an escaped path element and returns an escaped path
@ -648,16 +297,21 @@ func Split(segment string) []string {
// Unexported Helpers
// ---------------------------------------------------------------------------
func verifyInputValues(tenant, resourceOwner string) error {
func verifyPrefixValues(
tenant string,
srs []ServiceResource,
cat CategoryType,
) error {
if len(tenant) == 0 {
return clues.Stack(errMissingSegment, clues.New("tenant"))
}
if len(resourceOwner) == 0 {
return clues.Stack(errMissingSegment, clues.New("resourceOwner"))
if err := validateServiceResources(srs); err != nil {
return err
}
return nil
// only the final service is checked for its category validity
return ValidateServiceAndCategory(srs[len(srs)-1].Service, cat)
}
// escapeElement takes a single path element and escapes all characters that
@ -762,17 +416,3 @@ func join(elements []string) string {
// '\' according to the escaping rules.
return strings.Join(elements, string(PathSeparator))
}
// verifyPrefix ensures that the tenant and resourceOwner are valid
// values, and that the builder has some directory structure.
func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
if err := verifyInputValues(tenant, resourceOwner); err != nil {
return err
}
if len(pb.elements) == 0 {
return clues.New("missing path beyond prefix")
}
return nil
}

View File

@ -2,7 +2,6 @@ package path
import (
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
@ -233,239 +232,7 @@ func (suite *PathUnitSuite) TeardownSuite() {
clues.SetHasher(clues.NoHash())
}
func (suite *PathUnitSuite) TestAppend() {
table := append(append([]testData{}, genericCases...), basicUnescapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := Builder{}.Append(test.input...)
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *PathUnitSuite) TestAppendItem() {
t := suite.T()
p, err := Build("t", "ro", ExchangeService, EmailCategory, false, "foo", "bar")
require.NoError(t, err, clues.ToCore(err))
pb := p.ToBuilder()
assert.Equal(t, pb.String(), p.String())
pb = pb.Append("qux")
p, err = p.AppendItem("qux")
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, pb.String(), p.String())
_, err = p.AppendItem("fnords")
require.Error(t, err, clues.ToCore(err))
}
func (suite *PathUnitSuite) TestUnescapeAndAppend() {
table := append(append([]testData{}, genericCases...), basicEscapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := Builder{}.UnescapeAndAppend(test.input...)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *PathUnitSuite) TestEscapedFailure() {
target := "i_s"
for c := range charactersToEscape {
suite.Run(fmt.Sprintf("Unescaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(suite.T(), err, "path with unescaped %s did not error", string(c))
})
}
}
func (suite *PathUnitSuite) TestBadEscapeSequenceErrors() {
target := `i\_s/a`
notEscapes := []rune{'a', 'b', '#', '%'}
for _, c := range notEscapes {
suite.Run(fmt.Sprintf("Escaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(
suite.T(),
err,
"path with bad escape sequence %c%c did not error",
escapeCharacter,
c)
})
}
}
func (suite *PathUnitSuite) TestTrailingEscapeChar() {
base := []string{"this", "is", "a", "path"}
for i := 0; i < len(base); i++ {
suite.Run(fmt.Sprintf("Element%v", i), func() {
path := make([]string, len(base))
copy(path, base)
path[i] = path[i] + string(escapeCharacter)
_, err := Builder{}.UnescapeAndAppend(path...)
assert.Error(
suite.T(),
err,
"path with trailing escape character did not error")
})
}
}
func (suite *PathUnitSuite) TestElements() {
table := []struct {
name string
input []string
output []string
pathFunc func(elements []string) (*Builder, error)
}{
{
name: "SimpleEscapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
{
name: "SimpleUnescapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.Append(elements...), nil
},
},
{
name: "EscapedPath",
input: []string{"this", `is\/`, "a", "path"},
output: []string{"this", "is/", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := test.pathFunc(test.input)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, Elements(test.output), p.Elements())
})
}
}
func (suite *PathUnitSuite) TestPopFront() {
table := []struct {
name string
base *Builder
expectedString string
}{
{
name: "Empty",
base: &Builder{},
expectedString: "",
},
{
name: "OneElement",
base: Builder{}.Append("something"),
expectedString: "",
},
{
name: "TwoElements",
base: Builder{}.Append("something", "else"),
expectedString: "else",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expectedString, test.base.PopFront().String())
})
}
}
func (suite *PathUnitSuite) TestShortRef() {
table := []struct {
name string
inputElements []string
expectedLen int
}{
{
name: "PopulatedPath",
inputElements: []string{"this", "is", "a", "path"},
expectedLen: shortRefCharacters,
},
{
name: "EmptyPath",
inputElements: nil,
expectedLen: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
pb := Builder{}.Append(test.inputElements...)
ref := pb.ShortRef()
assert.Len(suite.T(), ref, test.expectedLen)
})
}
}
func (suite *PathUnitSuite) TestShortRefIsStable() {
t := suite.T()
pb := Builder{}.Append("this", "is", "a", "path")
prevRef := pb.ShortRef()
assert.Len(t, prevRef, shortRefCharacters)
for i := 0; i < 5; i++ {
ref := pb.ShortRef()
assert.Len(t, ref, shortRefCharacters)
assert.Equal(t, prevRef, ref, "ShortRef changed between calls")
prevRef = ref
}
}
func (suite *PathUnitSuite) TestShortRefIsUnique() {
pb1 := Builder{}.Append("this", "is", "a", "path")
pb2 := pb1.Append("also")
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
// TestShortRefUniqueWithEscaping tests that two paths that output the same
// unescaped string but different escaped strings have different shortrefs. This
// situation can occur when one path has embedded path separators while the
// other does not but contains the same characters.
func (suite *PathUnitSuite) TestShortRefUniqueWithEscaping() {
pb1 := Builder{}.Append(`this`, `is`, `a`, `path`)
pb2 := Builder{}.Append(`this`, `is/a`, `path`)
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
func (suite *PathUnitSuite) TestFromStringErrors() {
func (suite *PathUnitSuite) TestFromDataLayerPathErrors() {
table := []struct {
name string
escapedPath string
@ -521,82 +288,7 @@ func (suite *PathUnitSuite) TestFromStringErrors() {
}
}
func (suite *PathUnitSuite) TestFolder() {
table := []struct {
name string
p func(t *testing.T) Path
escape bool
expectFolder string
expectSplit []string
}{
{
name: "clean path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "clean path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a//b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a\\//b/c",
expectSplit: []string{"a\\/", "b", "c"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := test.p(t)
result := p.Folder(test.escape)
assert.Equal(t, test.expectFolder, result)
assert.Equal(t, test.expectSplit, Split(result))
})
}
}
func (suite *PathUnitSuite) TestFromString() {
func (suite *PathUnitSuite) TestFromDataLayerPath() {
const (
testTenant = "tenant"
testUser = "user"
@ -642,14 +334,12 @@ func (suite *PathUnitSuite) TestFromString() {
testUser,
testElement1,
testElement2,
testElement3,
),
testElement3),
expectedFolder: fmt.Sprintf(
"%s/%s/%s",
testElementTrimmed,
testElement2,
testElement3,
),
testElement3),
expectedSplit: []string{
testElementTrimmed,
testElement2,
@ -659,8 +349,7 @@ func (suite *PathUnitSuite) TestFromString() {
expectedItemFolder: fmt.Sprintf(
"%s/%s",
testElementTrimmed,
testElement2,
),
testElement2),
expectedItemSplit: []string{
testElementTrimmed,
testElement2,
@ -674,14 +363,12 @@ func (suite *PathUnitSuite) TestFromString() {
testUser,
testElementTrimmed,
testElement2,
testElement3,
),
testElement3),
expectedFolder: fmt.Sprintf(
"%s/%s/%s",
testElementTrimmed,
testElement2,
testElement3,
),
testElement3),
expectedSplit: []string{
testElementTrimmed,
testElement2,
@ -691,8 +378,7 @@ func (suite *PathUnitSuite) TestFromString() {
expectedItemFolder: fmt.Sprintf(
"%s/%s",
testElementTrimmed,
testElement2,
),
testElement2),
expectedItemSplit: []string{
testElementTrimmed,
testElement2,
@ -706,16 +392,19 @@ func (suite *PathUnitSuite) TestFromString() {
suite.Run(fmt.Sprintf("%s-%s-%s", service, cat, item.name), func() {
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
testPath := fmt.Sprintf(test.unescapedPath, service, cat)
var (
t = suite.T()
testPath = fmt.Sprintf(test.unescapedPath, service, cat)
sr = ServiceResource{service, testUser}
)
p, err := FromDataLayerPath(testPath, item.isItem)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, service, p.Service(), "service")
assert.Len(t, p.ServiceResources(), 1, "service resources")
assert.Equal(t, sr, p.ServiceResources()[0], "service resource")
assert.Equal(t, cat, p.Category(), "category")
assert.Equal(t, testTenant, p.Tenant(), "tenant")
assert.Equal(t, testUser, p.ResourceOwner(), "resource owner")
fld := p.Folder(false)
escfld := p.Folder(true)
@ -740,77 +429,78 @@ func (suite *PathUnitSuite) TestFromString() {
}
}
func (suite *PathUnitSuite) TestPath_piiHandling() {
p, err := Build("t", "ro", ExchangeService, EventsCategory, true, "dir", "item")
require.NoError(suite.T(), err)
table := []struct {
name string
p Path
expect string
expectPlain string
}{
{
name: "standard path",
p: p,
expect: "***/exchange/***/events/***/***",
expectPlain: "t/exchange/ro/events/dir/item",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expect, test.p.Conceal(), "conceal")
assert.Equal(t, test.expectPlain, test.p.String(), "string")
assert.Equal(t, test.expect, fmt.Sprintf("%s", test.p), "fmt %%s")
assert.Equal(t, test.expect, fmt.Sprintf("%+v", test.p), "fmt %%+v")
assert.Equal(t, test.expectPlain, test.p.PlainString(), "plain")
})
}
}
func (suite *PathUnitSuite) TestToServicePrefix() {
func (suite *PathUnitSuite) TestBuildPrefix() {
table := []struct {
name string
service ServiceType
category CategoryType
tenant string
owner string
srs []ServiceResource
category CategoryType
expect string
expectErr require.ErrorAssertionFunc
}{
{
name: "ok",
service: ExchangeService,
category: ContactsCategory,
tenant: "t",
owner: "ro",
expect: join([]string{"t", ExchangeService.String(), "ro", ContactsCategory.String()}),
srs: []ServiceResource{{ExchangeService, "roo"}},
category: ContactsCategory,
expect: join([]string{"t", ExchangeService.String(), "roo", ContactsCategory.String()}),
expectErr: require.NoError,
},
{
name: "ok with subservice",
tenant: "t",
srs: []ServiceResource{
{GroupsService, "roo"},
{SharePointService, "oor"},
},
category: LibrariesCategory,
expect: join([]string{
"t",
GroupsService.String(), "roo",
SharePointService.String(), "oor",
LibrariesCategory.String(),
}),
expectErr: require.NoError,
},
{
name: "bad category",
service: ExchangeService,
srs: []ServiceResource{{ExchangeService, "roo"}},
category: FilesCategory,
tenant: "t",
owner: "ro",
expectErr: require.Error,
},
{
name: "bad tenant",
service: ExchangeService,
category: ContactsCategory,
tenant: "",
owner: "ro",
srs: []ServiceResource{{ExchangeService, "roo"}},
category: ContactsCategory,
expectErr: require.Error,
},
{
name: "bad owner",
service: ExchangeService,
category: ContactsCategory,
name: "bad resource",
tenant: "t",
owner: "",
srs: []ServiceResource{{ExchangeService, ""}},
category: ContactsCategory,
expectErr: require.Error,
},
{
name: "bad subservice",
tenant: "t",
srs: []ServiceResource{
{ExchangeService, "roo"},
{OneDriveService, "oor"},
},
category: FilesCategory,
expectErr: require.Error,
},
{
name: "bad subservice resource",
tenant: "t",
srs: []ServiceResource{
{GroupsService, "roo"},
{SharePointService, ""},
},
category: LibrariesCategory,
expectErr: require.Error,
},
}
@ -818,7 +508,7 @@ func (suite *PathUnitSuite) TestToServicePrefix() {
suite.Run(test.name, func() {
t := suite.T()
r, err := ServicePrefix(test.tenant, test.owner, test.service, test.category)
r, err := BuildPrefix(test.tenant, test.srs, test.category)
test.expectErr(t, err, clues.ToCore(err))
if r == nil {

View File

@ -1,154 +1,9 @@
package path
import (
"fmt"
"strings"
"github.com/alcionai/clues"
)
var ErrorUnknownService = clues.New("unknown service string")
// ServiceType denotes what service the path corresponds to. Metadata services
// are also included though they are only used for paths that house metadata for
// Corso backups.
//
// Metadata services are not considered valid service types for resource paths
// though they can be used for metadata paths.
//
// The order of the enums below can be changed, but the string representation of
// each enum must remain the same or migration code needs to be added to handle
// changes to the string format.
type ServiceType int
//go:generate stringer -type=ServiceType -linecomment
const (
UnknownService ServiceType = iota
ExchangeService // exchange
OneDriveService // onedrive
SharePointService // sharepoint
ExchangeMetadataService // exchangeMetadata
OneDriveMetadataService // onedriveMetadata
SharePointMetadataService // sharepointMetadata
)
func toServiceType(service string) ServiceType {
s := strings.ToLower(service)
switch s {
case strings.ToLower(ExchangeService.String()):
return ExchangeService
case strings.ToLower(OneDriveService.String()):
return OneDriveService
case strings.ToLower(SharePointService.String()):
return SharePointService
case strings.ToLower(ExchangeMetadataService.String()):
return ExchangeMetadataService
case strings.ToLower(OneDriveMetadataService.String()):
return OneDriveMetadataService
case strings.ToLower(SharePointMetadataService.String()):
return SharePointMetadataService
default:
return UnknownService
}
}
var ErrorUnknownCategory = clues.New("unknown category string")
// CategoryType denotes what category of data the path corresponds to. The order
// of the enums below can be changed, but the string representation of each enum
// must remain the same or migration code needs to be added to handle changes to
// the string format.
type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
ListsCategory // lists
LibrariesCategory // libraries
PagesCategory // pages
DetailsCategory // details
)
func ToCategoryType(category string) CategoryType {
cat := strings.ToLower(category)
switch cat {
case strings.ToLower(EmailCategory.String()):
return EmailCategory
case strings.ToLower(ContactsCategory.String()):
return ContactsCategory
case strings.ToLower(EventsCategory.String()):
return EventsCategory
case strings.ToLower(FilesCategory.String()):
return FilesCategory
case strings.ToLower(LibrariesCategory.String()):
return LibrariesCategory
case strings.ToLower(ListsCategory.String()):
return ListsCategory
case strings.ToLower(PagesCategory.String()):
return PagesCategory
case strings.ToLower(DetailsCategory.String()):
return DetailsCategory
default:
return UnknownCategory
}
}
// serviceCategories is a mapping of all valid service/category pairs for
// non-metadata paths.
var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
ExchangeService: {
EmailCategory: {},
ContactsCategory: {},
EventsCategory: {},
},
OneDriveService: {
FilesCategory: {},
},
SharePointService: {
LibrariesCategory: {},
ListsCategory: {},
PagesCategory: {},
},
}
func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType, error) {
service := toServiceType(s)
if service == UnknownService {
return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("service", fmt.Sprintf("%q", s))
}
category := ToCategoryType(c)
if category == UnknownCategory {
return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("category", fmt.Sprintf("%q", c))
}
if err := ValidateServiceAndCategory(service, category); err != nil {
return UnknownService, UnknownCategory, err
}
return service, category, nil
}
func ValidateServiceAndCategory(service ServiceType, category CategoryType) error {
cats, ok := serviceCategories[service]
if !ok {
return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))
}
if _, ok := cats[category]; !ok {
return clues.New("unknown service/category combination").
With("service", fmt.Sprintf("%q", service), "category", fmt.Sprintf("%q", category))
}
return nil
}
// dataLayerResourcePath allows callers to extract information from a
// resource-specific path. This struct is unexported so that callers are
// forced to use the pre-defined constructors, making it impossible to create a
@ -163,9 +18,28 @@ func ValidateServiceAndCategory(service ServiceType, category CategoryType) erro
// element after the prefix.
type dataLayerResourcePath struct {
Builder
category CategoryType
service ServiceType
hasItem bool
category CategoryType
serviceResources []ServiceResource
hasItem bool
}
// performs no validation, assumes the caller has validated the inputs.
func newDataLayerResourcePath(
pb Builder,
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) dataLayerResourcePath {
pfx := append([]string{tenant}, ServiceResourcesToElements(srs)...)
pfx = append(pfx, cat.String())
return dataLayerResourcePath{
Builder: *pb.withPrefix(pfx...),
serviceResources: srs,
category: cat,
hasItem: isItem,
}
}
// Tenant returns the tenant ID embedded in the dataLayerResourcePath.
@ -173,9 +47,8 @@ func (rp dataLayerResourcePath) Tenant() string {
return rp.Builder.elements[0]
}
// Service returns the ServiceType embedded in the dataLayerResourcePath.
func (rp dataLayerResourcePath) Service() ServiceType {
return rp.service
func (rp dataLayerResourcePath) ServiceResources() []ServiceResource {
return rp.serviceResources
}
// Category returns the CategoryType embedded in the dataLayerResourcePath.
@ -240,15 +113,18 @@ func (rp dataLayerResourcePath) Item() string {
// Dir removes the last element from the path. If this would remove a
// value that is part of the standard prefix structure, an error is returned.
func (rp dataLayerResourcePath) Dir() (Path, error) {
if len(rp.elements) <= 4 {
// Dir is not allowed to slice off any prefix values.
// The prefix len is determined by the length of the number of
// service+resource tuples, plus 2 (tenant and category).
if len(rp.elements) <= 2+(2*len(rp.serviceResources)) {
return nil, clues.New("unable to shorten path").With("path", rp)
}
return &dataLayerResourcePath{
Builder: *rp.Builder.Dir(),
service: rp.service,
category: rp.category,
hasItem: false,
Builder: *rp.Builder.Dir(),
serviceResources: rp.serviceResources,
category: rp.category,
hasItem: false,
}, nil
}
@ -261,10 +137,10 @@ func (rp dataLayerResourcePath) Append(
}
return &dataLayerResourcePath{
Builder: *rp.Builder.Append(elems...),
service: rp.service,
category: rp.category,
hasItem: isItem,
Builder: *rp.Builder.Append(elems...),
serviceResources: rp.serviceResources,
category: rp.category,
hasItem: isItem,
}, nil
}
@ -280,3 +156,17 @@ func (rp dataLayerResourcePath) ToBuilder() *Builder {
func (rp *dataLayerResourcePath) UpdateParent(prev, cur Path) bool {
return rp.Builder.UpdateParent(prev.ToBuilder(), cur.ToBuilder())
}
func (rp *dataLayerResourcePath) Halves() (*Builder, Elements) {
pfx, sfx := &Builder{}, Elements{}
b := rp.Builder
if len(b.elements) > 0 {
lenPfx := 2 + (len(rp.serviceResources) * 2)
pfx = &Builder{elements: append(Elements{}, b.elements[:lenPfx]...)}
sfx = append(Elements{}, b.elements[lenPfx-1:]...)
}
return pfx, sfx
}

View File

@ -256,10 +256,10 @@ func (suite *DataLayerResourcePath) TestDir() {
func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
tenant := "a-tenant"
user := "a-user"
resource := "a-resource"
table := []struct {
name string
service path.ServiceType
srs []path.ServiceResource
category path.CategoryType
postfix []string
expectedService path.ServiceType
@ -267,14 +267,14 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
}{
{
name: "NoPostfixPasses",
service: path.ExchangeService,
srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EmailCategory,
expectedService: path.ExchangeMetadataService,
check: assert.NoError,
},
{
name: "PostfixPasses",
service: path.ExchangeService,
srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EmailCategory,
postfix: []string{"a", "b"},
expectedService: path.ExchangeMetadataService,
@ -282,48 +282,48 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
},
{
name: "Fails",
service: path.ExchangeService,
srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.FilesCategory,
check: assert.Error,
},
{
name: "Passes",
service: path.ExchangeService,
srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.ContactsCategory,
expectedService: path.ExchangeMetadataService,
check: assert.NoError,
},
{
name: "Passes",
service: path.ExchangeService,
srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EventsCategory,
expectedService: path.ExchangeMetadataService,
check: assert.NoError,
},
{
name: "Passes",
service: path.OneDriveService,
srs: []path.ServiceResource{{path.OneDriveService, resource}},
category: path.FilesCategory,
expectedService: path.OneDriveMetadataService,
check: assert.NoError,
},
{
name: "Passes",
service: path.SharePointService,
srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.LibrariesCategory,
expectedService: path.SharePointMetadataService,
check: assert.NoError,
},
{
name: "Passes",
service: path.SharePointService,
srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.ListsCategory,
expectedService: path.SharePointMetadataService,
check: assert.NoError,
},
{
name: "Passes",
service: path.SharePointService,
srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.PagesCategory,
expectedService: path.SharePointMetadataService,
check: assert.NoError,
@ -331,27 +331,26 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
}
for _, test := range table {
suite.Run(strings.Join([]string{
name := strings.Join([]string{
test.name,
test.service.String(),
test.srs[0].Service.String(),
test.category.String(),
}, "_"), func() {
}, "_")
suite.Run(name, func() {
t := suite.T()
pb := path.Builder{}.Append(test.postfix...)
p, err := pb.ToServiceCategoryMetadataPath(
tenant,
user,
test.service,
test.srs,
test.category,
false)
test.check(t, err, clues.ToCore(err))
if err != nil {
return
if err == nil {
assert.Equal(t, test.expectedService, p.ServiceResources()[0])
}
assert.Equal(t, test.expectedService, p.Service())
})
}
}
@ -402,9 +401,9 @@ func (suite *DataLayerResourcePath) TestToExchangePathForCategory() {
}
assert.Equal(t, testTenant, p.Tenant())
assert.Equal(t, path.ExchangeService, p.Service())
assert.Equal(t, path.ExchangeService, p.ServiceResources()[0].Service)
assert.Equal(t, test.category, p.Category())
assert.Equal(t, testUser, p.ResourceOwner())
assert.Equal(t, testUser, p.ServiceResources()[0].ProtectedResource)
assert.Equal(t, strings.Join(m.expectedFolders, "/"), p.Folder(false))
assert.Equal(t, path.Elements(m.expectedFolders), p.Folders())
assert.Equal(t, m.expectedItem, p.Item())
@ -456,7 +455,10 @@ func (suite *PopulatedDataLayerResourcePath) TestService() {
suite.Run(m.name, func() {
t := suite.T()
assert.Equal(t, path.ExchangeService, suite.paths[m.isItem].Service())
assert.Equal(
t,
path.ExchangeService,
suite.paths[m.isItem].ServiceResources()[0].Service)
})
}
}
@ -476,7 +478,10 @@ func (suite *PopulatedDataLayerResourcePath) TestResourceOwner() {
suite.Run(m.name, func() {
t := suite.T()
assert.Equal(t, testUser, suite.paths[m.isItem].ResourceOwner())
assert.Equal(
t,
testUser,
suite.paths[m.isItem].ServiceResources()[0].ProtectedResource)
})
}
}
@ -673,3 +678,57 @@ func (suite *PopulatedDataLayerResourcePath) TestUpdateParent_NoopsNils() {
})
}
}
func (suite *PopulatedDataLayerResourcePath) TestHalves() {
t := suite.T()
onlyPrefix, err := path.BuildPrefix(
"titd",
[]path.ServiceResource{{path.ExchangeService, "pr"}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err))
fullPath, err := path.Build(
"tid",
[]path.ServiceResource{{path.ExchangeService, "pr"}},
path.ContactsCategory,
true,
"fld", "item")
require.NoError(t, err, clues.ToCore(err))
table := []struct {
name string
dlrp path.Path
expectPfx *path.Builder
expectSfx path.Elements
}{
{
name: "only prefix",
dlrp: onlyPrefix,
expectPfx: path.Builder{}.Append(
"tid",
path.ExchangeService.String(),
"pr",
path.ContactsCategory.String()),
expectSfx: path.Elements{},
},
{
name: "full path",
dlrp: fullPath,
expectPfx: path.Builder{}.Append(
"tid",
path.ExchangeService.String(),
"pr",
path.ContactsCategory.String()),
expectSfx: path.Elements{"foo", "bar"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
pfx, sfx := test.dlrp.Halves()
assert.Equal(t, test.expectPfx, pfx, "prefix")
assert.Equal(t, test.expectSfx, sfx, "suffix")
})
}
}

View File

@ -20,118 +20,76 @@ func TestServiceCategoryUnitSuite(t *testing.T) {
suite.Run(t, s)
}
func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategoryBadStringErrors() {
func (suite *ServiceCategoryUnitSuite) TestVerifyPrefixValues() {
table := []struct {
name string
service string
category string
}{
{
name: "Service",
service: "foo",
category: EmailCategory.String(),
},
{
name: "Category",
service: ExchangeService.String(),
category: "foo",
},
}
for _, test := range table {
suite.Run(test.name, func() {
_, _, err := validateServiceAndCategoryStrings(test.service, test.category)
assert.Error(suite.T(), err)
})
}
}
func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategory() {
table := []struct {
name string
service string
category string
expectedService ServiceType
expectedCategory CategoryType
check assert.ErrorAssertionFunc
service ServiceType
category CategoryType
check assert.ErrorAssertionFunc
}{
{
name: "UnknownService",
service: UnknownService.String(),
category: EmailCategory.String(),
service: UnknownService,
category: EmailCategory,
check: assert.Error,
},
{
name: "UnknownCategory",
service: ExchangeService.String(),
category: UnknownCategory.String(),
service: ExchangeService,
category: UnknownCategory,
check: assert.Error,
},
{
name: "BadServiceString",
service: "foo",
category: EmailCategory.String(),
name: "BadServiceType",
service: ServiceType(-1),
category: EmailCategory,
check: assert.Error,
},
{
name: "BadCategoryString",
service: ExchangeService.String(),
category: "foo",
name: "BadCategoryType",
service: ExchangeService,
category: CategoryType(-1),
check: assert.Error,
},
{
name: "ExchangeEmail",
service: ExchangeService.String(),
category: EmailCategory.String(),
expectedService: ExchangeService,
expectedCategory: EmailCategory,
check: assert.NoError,
name: "ExchangeEmail",
service: ExchangeService,
category: EmailCategory,
check: assert.NoError,
},
{
name: "ExchangeContacts",
service: ExchangeService.String(),
category: ContactsCategory.String(),
expectedService: ExchangeService,
expectedCategory: ContactsCategory,
check: assert.NoError,
name: "ExchangeContacts",
service: ExchangeService,
category: ContactsCategory,
check: assert.NoError,
},
{
name: "ExchangeEvents",
service: ExchangeService.String(),
category: EventsCategory.String(),
expectedService: ExchangeService,
expectedCategory: EventsCategory,
check: assert.NoError,
name: "ExchangeEvents",
service: ExchangeService,
category: EventsCategory,
check: assert.NoError,
},
{
name: "OneDriveFiles",
service: OneDriveService.String(),
category: FilesCategory.String(),
expectedService: OneDriveService,
expectedCategory: FilesCategory,
check: assert.NoError,
name: "OneDriveFiles",
service: OneDriveService,
category: FilesCategory,
check: assert.NoError,
},
{
name: "SharePointLibraries",
service: SharePointService.String(),
category: LibrariesCategory.String(),
expectedService: SharePointService,
expectedCategory: LibrariesCategory,
check: assert.NoError,
name: "SharePointLibraries",
service: SharePointService,
category: LibrariesCategory,
check: assert.NoError,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
s, c, err := validateServiceAndCategoryStrings(test.service, test.category)
srs := []ServiceResource{{test.service, "resource"}}
err := verifyPrefixValues("tid", srs, test.category)
test.check(t, err, clues.ToCore(err))
if err != nil {
return
}
assert.Equal(t, test.expectedService, s)
assert.Equal(t, test.expectedCategory, c)
})
}
}
@ -155,9 +113,10 @@ func (suite *ServiceCategoryUnitSuite) TestToServiceType() {
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expected, toServiceType(test.service))
assert.Equal(
suite.T(),
test.expected,
ToServiceType(test.service))
})
}
}

View File

@ -0,0 +1,205 @@
package path
import (
"github.com/alcionai/clues"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/common/tform"
)
// ---------------------------------------------------------------------------
// Tuple
// ---------------------------------------------------------------------------
// ServiceResource holds a service + resource tuple. The tuple implies
// that the resource owns some data in the given service.
type ServiceResource struct {
Service ServiceType
ProtectedResource string
}
func MakeServiceResource(
st ServiceType,
protectedResource string,
) ServiceResource {
return ServiceResource{
Service: st,
ProtectedResource: protectedResource,
}
}
func (sr ServiceResource) validate() error {
if len(sr.ProtectedResource) == 0 {
return clues.Stack(errMissingSegment, clues.New("protected resource"))
}
return nil
}
// ---------------------------------------------------------------------------
// Exported Helpers
// ---------------------------------------------------------------------------
// NewServiceResources is a lenient constructor for building a
// new []ServiceResource. It allows the caller to pass in any
// number of arbitrary values, but will require the following:
// 1. even values must be path.ServiceType typed
// 2. odd values must be string typed
// 3. a non-zero, even number of values must be provided
func NewServiceResources(elems ...any) ([]ServiceResource, error) {
if len(elems) == 0 {
return nil, clues.New("missing service resources")
}
if len(elems)%2 == 1 {
return nil, clues.New("odd number of service resources")
}
srs := make([]ServiceResource, 0, len(elems)/2)
for i, j := 0, 1; i < len(elems); i, j = i+2, j+2 {
srv, err := tform.AnyToT[ServiceType](elems[i])
if err != nil {
return nil, clues.Wrap(err, "service")
}
pr, err := str.AnyToString(elems[j])
if err != nil {
return nil, clues.Wrap(err, "protected resource")
}
srs = append(srs, MakeServiceResource(srv, pr))
}
return srs, nil
}
func ServiceResourcesToResources(srs []ServiceResource) []string {
prs := make([]string, len(srs))
for i := range srs {
prs[i] = srs[i].ProtectedResource
}
return prs
}
func ServiceResourcesToServices(srs []ServiceResource) []ServiceType {
sts := make([]ServiceType, len(srs))
for i := range srs {
sts[i] = srs[i].Service
}
return sts
}
func ServiceResourcesMatchServices(srs []ServiceResource, sts []ServiceType) bool {
return slices.EqualFunc(srs, sts, func(sr ServiceResource, st ServiceType) bool {
return sr.Service == st
})
}
func ServiceResourcesToElements(srs []ServiceResource) Elements {
es := make(Elements, 0, len(srs)*2)
for _, tuple := range srs {
es = append(es, tuple.Service.String())
es = append(es, tuple.ProtectedResource)
}
return es
}
// ---------------------------------------------------------------------------
// Unexported Helpers
// ---------------------------------------------------------------------------
// elementsToServiceResources turns as many pairs of elems as possible
// into ServiceResource tuples. Elems must begin with a service, but
// may contain more entries than there are service/resource pairs.
// This transformer will continue consuming elements until it finds an
// even-numbered index that cannot be cast to a ServiceType.
// Returns the serviceResource pairs, the first index containing element
// that is not part of a service/resource pair, and an error if elems is
// len==0 or contains no services.
func elementsToServiceResources(elems Elements) ([]ServiceResource, int, error) {
if len(elems) == 0 {
return nil, -1, clues.Wrap(errMissingSegment, "service")
}
var (
srs = make([]ServiceResource, 0)
i int
)
for j := 1; i < len(elems); i, j = i+2, j+2 {
service := ToServiceType(elems[i])
if service == UnknownService {
if i == 0 {
return nil, -1, clues.Wrap(errMissingSegment, "service")
}
break
}
srs = append(srs, ServiceResource{service, elems[j]})
}
return srs, i, nil
}
// checks for the following:
// 1. each ServiceResource is valid
// 2. if len(srs) > 1, srs[i], srs[i+1] pass subservice checks.
func validateServiceResources(srs []ServiceResource) error {
switch len(srs) {
case 0:
return clues.Stack(errMissingSegment, clues.New("service"))
case 1:
return srs[0].validate()
}
for i, tuple := range srs {
if err := tuple.validate(); err != nil {
return err
}
if i+1 >= len(srs) {
continue
}
if err := ValidateServiceAndSubService(tuple.Service, srs[i+1].Service); err != nil {
return err
}
}
return nil
}
// makes a copy of the slice with all of the Services swapped for their
// metadata service countterparts.
func toMetadataServices(srs []ServiceResource) []ServiceResource {
msrs := make([]ServiceResource, 0, len(srs))
for _, sr := range srs {
msr := sr
metadataService := UnknownService
switch sr.Service {
// TODO: add groups
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
msr.Service = metadataService
msrs = append(msrs, msr)
}
return msrs
}

View File

@ -0,0 +1,243 @@
package path
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type ServiceResourceUnitSuite struct {
tester.Suite
}
func TestServiceResourceUnitSuite(t *testing.T) {
suite.Run(t, &ServiceResourceUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ServiceResourceUnitSuite) TestNewServiceResource() {
table := []struct {
name string
input []any
expectErr assert.ErrorAssertionFunc
expectResult []ServiceResource
}{
{
name: "empty",
input: []any{},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "odd elems: 1",
input: []any{ExchangeService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "odd elems: 3",
input: []any{ExchangeService, "mailbox", OneDriveService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "non-service even index",
input: []any{"foo", "bar"},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "non-string odd index",
input: []any{ExchangeService, OneDriveService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "valid single",
input: []any{ExchangeService, "mailbox"},
expectErr: assert.NoError,
expectResult: []ServiceResource{{ExchangeService, "mailbox"}},
},
{
name: "valid multiple",
input: []any{ExchangeService, "mailbox", OneDriveService, "user"},
expectErr: assert.NoError,
expectResult: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result, err := NewServiceResources(test.input...)
test.expectErr(t, err, clues.ToCore(err))
assert.Equal(t, test.expectResult, result)
})
}
}
func (suite *ServiceResourceUnitSuite) TestValidateServiceResources() {
table := []struct {
name string
srs []ServiceResource
expect assert.ErrorAssertionFunc
}{
{
name: "empty",
srs: []ServiceResource{},
expect: assert.Error,
},
{
name: "invalid resource",
srs: []ServiceResource{{ExchangeService, ""}},
expect: assert.Error,
},
{
name: "invalid subservice",
srs: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
expect: assert.Error,
},
{
name: "valid",
srs: []ServiceResource{
{GroupsService, "group"},
{SharePointService, "site"},
},
expect: assert.NoError,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
err := validateServiceResources(test.srs)
test.expect(t, err, clues.ToCore(err))
})
}
}
func (suite *ServiceResourceUnitSuite) TestServiceResourceToElements() {
table := []struct {
name string
srs []ServiceResource
expect Elements
}{
{
name: "empty",
srs: []ServiceResource{},
expect: Elements{},
},
{
name: "single",
srs: []ServiceResource{{ExchangeService, "user"}},
expect: Elements{ExchangeService.String(), "user"},
},
{
name: "multiple",
srs: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
expect: Elements{
ExchangeService.String(), "mailbox",
OneDriveService.String(), "user",
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result := ServiceResourcesToElements(test.srs)
// not ElementsMatch, order matters
assert.Equal(t, test.expect, result)
})
}
}
func (suite *ServiceResourceUnitSuite) TestElementsToServiceResource() {
table := []struct {
name string
elems Elements
expectErr assert.ErrorAssertionFunc
expectIdx int
expectSRS []ServiceResource
}{
{
name: "empty",
elems: Elements{},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "nil",
elems: nil,
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "non-service 0th elem",
elems: Elements{"fnords"},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "non-service 2nd elem",
elems: Elements{ExchangeService.String(), "fnords", "smarf"},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "single serviceResource",
elems: Elements{ExchangeService.String(), "fnords"},
expectErr: assert.NoError,
expectIdx: 2,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}},
},
{
name: "single serviceResource and extra value",
elems: Elements{ExchangeService.String(), "fnords", "smarf"},
expectErr: assert.NoError,
expectIdx: 2,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}},
},
{
name: "multiple serviceResource",
elems: Elements{ExchangeService.String(), "fnords", OneDriveService.String(), "smarf"},
expectErr: assert.NoError,
expectIdx: 4,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}, {OneDriveService, "smarf"}},
},
{
name: "multiple serviceResource and extra value",
elems: Elements{ExchangeService.String(), "fnords", OneDriveService.String(), "smarf", "flaboigans"},
expectErr: assert.NoError,
expectIdx: 4,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}, {OneDriveService, "smarf"}},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
srs, idx, err := elementsToServiceResources(test.elems)
test.expectErr(t, err, clues.ToCore(err))
assert.Equal(t, test.expectIdx, idx)
assert.Equal(t, test.expectSRS, srs)
})
}
}

View File

@ -0,0 +1,88 @@
package path
import (
"strings"
"github.com/alcionai/clues"
)
var ErrorUnknownService = clues.New("unknown service string")
// ServiceType denotes what service the path corresponds to. Metadata services
// are also included though they are only used for paths that house metadata for
// Corso backups.
//
// Metadata services are not considered valid service types for resource paths
// though they can be used for metadata paths.
//
// The order of the enums below can be changed, but the string representation of
// each enum must remain the same or migration code needs to be added to handle
// changes to the string format.
type ServiceType int
//go:generate stringer -type=ServiceType -linecomment
const (
UnknownService ServiceType = iota
ExchangeService // exchange
OneDriveService // onedrive
SharePointService // sharepoint
ExchangeMetadataService // exchangeMetadata
OneDriveMetadataService // onedriveMetadata
SharePointMetadataService // sharepointMetadata
GroupsService // groups
GroupsMetadataService // groupsMetadata
)
func ToServiceType(service string) ServiceType {
s := strings.ToLower(service)
switch s {
case strings.ToLower(ExchangeService.String()):
return ExchangeService
case strings.ToLower(OneDriveService.String()):
return OneDriveService
case strings.ToLower(SharePointService.String()):
return SharePointService
case strings.ToLower(ExchangeMetadataService.String()):
return ExchangeMetadataService
case strings.ToLower(OneDriveMetadataService.String()):
return OneDriveMetadataService
case strings.ToLower(SharePointMetadataService.String()):
return SharePointMetadataService
default:
return UnknownService
}
}
// subServices is a mapping of all valid service/subService pairs.
// a subService pair occurs when one service contains a reference
// to a protected resource of another service type, and the resource
// for that second service is the identifier which is used to discover
// data. A subService relationship may imply that the subservice data
// is wholly replicated/owned by the primary service, or it may not,
// each case differs.
//
// Ex:
// - groups/<gID>/sharepoint/<siteID> => each team in groups contains a
// complete sharepoint site.
// - groups/<gID>/member/<userID> => each user in a team can own one or
// more Chats. But the group does not contain the complete user data.
var subServices = map[ServiceType]map[ServiceType]struct{}{
GroupsService: {
SharePointService: {},
},
}
func ValidateServiceAndSubService(service, subService ServiceType) error {
subs, ok := subServices[service]
if !ok {
return clues.New("unsupported service").With("service", service)
}
if _, ok := subs[subService]; !ok {
return clues.New("unknown service/subService combination").
With("service", service, "subService", subService)
}
return nil
}

View File

@ -0,0 +1,54 @@
package path
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type ServiceTypeUnitSuite struct {
tester.Suite
}
func TestServiceTypeUnitSuite(t *testing.T) {
suite.Run(t, &ServiceTypeUnitSuite{Suite: tester.NewUnitSuite(t)})
}
var knownServices = []ServiceType{
UnknownService,
ExchangeService,
OneDriveService,
SharePointService,
ExchangeMetadataService,
OneDriveMetadataService,
SharePointMetadataService,
GroupsService,
GroupsMetadataService,
}
func (suite *ServiceTypeUnitSuite) TestValildateServiceAndSubService() {
table := map[ServiceType]map[ServiceType]assert.ErrorAssertionFunc{}
for _, si := range knownServices {
table[si] = map[ServiceType]assert.ErrorAssertionFunc{}
for _, sj := range knownServices {
table[si][sj] = assert.Error
}
}
// expected successful
table[GroupsService][SharePointService] = assert.NoError
for srv, ti := range table {
for sub, expect := range ti {
suite.Run(srv.String()+"-"+sub.String(), func() {
err := ValidateServiceAndSubService(srv, sub)
expect(suite.T(), err, clues.ToCore(err))
})
}
}
}

View File

@ -348,7 +348,7 @@ func ensureAllUsersInDetails(
continue
}
ro := p.ResourceOwner()
ro := p.ServiceResources()[0].ProtectedResource
if !assert.NotEmpty(t, ro, "resource owner in path: "+rr) {
continue
}

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -43,6 +44,7 @@ type (
var (
_ Reducer = &ExchangeRestore{}
_ pathCategorier = &ExchangeRestore{}
_ reasoner = &ExchangeRestore{}
)
// NewExchange produces a new Selector with the service set to ServiceExchange.
@ -122,6 +124,13 @@ func (s exchange) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s exchange) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -815,8 +815,12 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
}
joinedFldrs := strings.Join(newElems, "/")
return stubRepoRef(p.Service(), p.Category(), p.ResourceOwner(), joinedFldrs, p.Item())
return stubRepoRef(
suite.T(),
p.ServiceResources(),
p.Category(),
joinedFldrs,
p.Item())
}
makeDeets := func(refs ...path.Path) *details.Details {
@ -1058,12 +1062,36 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce_locationRef() {
var (
contact = stubRepoRef(path.ExchangeService, path.ContactsCategory, "uid", "id5/id6", "cid")
contactLocation = "conts/my_cont"
event = stubRepoRef(path.ExchangeService, path.EventsCategory, "uid", "id1/id2", "eid")
eventLocation = "cal/my_cal"
mail = stubRepoRef(path.ExchangeService, path.EmailCategory, "uid", "id3/id4", "mid")
mailLocation = "inbx/my_mail"
contact = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.ContactsCategory,
"id5/id6",
"cid")
event = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.EventsCategory,
"id1/id2",
"eid")
mail = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.EmailCategory,
"id3/id4",
"mid")
)
makeDeets := func(refs ...string) *details.Details {

View File

@ -7,6 +7,7 @@ import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -40,6 +41,7 @@ type (
var (
_ Reducer = &GroupsRestore{}
_ pathCategorier = &GroupsRestore{}
_ reasoner = &GroupsRestore{}
)
// NewGroupsBackup produces a new Selector with the service set to ServiceGroups.
@ -119,6 +121,13 @@ func (s groups) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s groups) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -2,7 +2,6 @@ package selectors
import (
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
@ -167,6 +166,8 @@ func (s mockScope) PlainString() string { return plainString(s) }
// selectors
// ---------------------------------------------------------------------------
var _ servicerCategorizerProvider = &mockSel{}
type mockSel struct {
Selector
}
@ -183,6 +184,10 @@ func stubSelector(resourceOwners []string) mockSel {
}
}
func (m mockSel) PathCategories() selectorPathCategories {
return m.PathCategories()
}
// ---------------------------------------------------------------------------
// helper funcs
// ---------------------------------------------------------------------------
@ -207,7 +212,15 @@ func scopeMustHave[T scopeT](t *testing.T, sc T, m map[categorizer][]string) {
// stubPath ensures test path production matches that of fullPath design,
// stubbing out static values where necessary.
func stubPath(t *testing.T, user string, s []string, cat path.CategoryType) path.Path {
pth, err := path.Build("tid", user, path.ExchangeService, cat, true, s...)
pth, err := path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
cat,
true,
s...)
require.NoError(t, err, clues.ToCore(err))
return pth
@ -215,6 +228,22 @@ func stubPath(t *testing.T, user string, s []string, cat path.CategoryType) path
// stubRepoRef ensures test path production matches that of repoRef design,
// stubbing out static values where necessary.
func stubRepoRef(service path.ServiceType, data path.CategoryType, resourceOwner, folders, item string) string {
return strings.Join([]string{"tid", service.String(), resourceOwner, data.String(), folders, item}, "/")
func stubRepoRef(
t *testing.T,
srs []path.ServiceResource,
cat path.CategoryType,
folders, item string,
) string {
fs := path.Split(folders)
fs = append(fs, item)
pb, err := path.Build(
"tid",
srs,
cat,
true,
fs...)
require.NoError(t, err, clues.ToCore(err))
return pb.String()
}

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var (
_ Reducer = &OneDriveRestore{}
_ pathCategorier = &OneDriveRestore{}
_ reasoner = &OneDriveRestore{}
)
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
@ -121,6 +123,13 @@ func (s oneDrive) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s oneDrive) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -161,23 +161,32 @@ func (suite *OneDriveSelectorSuite) TestToOneDriveRestore() {
func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
var (
file = stubRepoRef(
path.OneDriveService,
suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory,
"uid",
"drive/driveID/root:/folderA.d/folderB.d",
"file")
fileParent = "folderA/folderB"
file2 = stubRepoRef(
path.OneDriveService,
suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory,
"uid",
"drive/driveID/root:/folderA.d/folderC.d",
"file2")
fileParent2 = "folderA/folderC"
file3 = stubRepoRef(
path.OneDriveService,
suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory,
"uid",
"drive/driveID/root:/folderD.d/folderE.d",
"file3")
fileParent3 = "folderD/folderE"
@ -314,7 +323,15 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
shortRef := "short"
elems := []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "dir1.d", "dir2.d", fileID}
filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...)
filePath, err := path.Build(
"tenant",
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "user",
}},
path.FilesCategory,
true,
elems...)
require.NoError(t, err, clues.ToCore(err))
fileLoc := path.Builder{}.Append("dir1", "dir2")
@ -351,8 +368,10 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
itemPath, err := path.Build(
"tenant",
"site",
path.OneDriveService,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "site",
}},
path.FilesCategory,
true,
test.pathElems...)

View File

@ -0,0 +1,102 @@
package selectors
import (
"golang.org/x/exp/maps"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// reasoner interface compliance
// ---------------------------------------------------------------------------
var _ identity.Reasoner = &backupReason{}
type backupReason struct {
category path.CategoryType
resource string
service path.ServiceType
tenant string
}
func (br backupReason) Tenant() string {
return br.tenant
}
func (br backupReason) ProtectedResource() string {
return br.resource
}
func (br backupReason) Service() path.ServiceType {
return br.service
}
func (br backupReason) Category() path.CategoryType {
return br.category
}
func (br backupReason) SubtreePath() (path.Path, error) {
srs, err := path.NewServiceResources(br.service, br.resource)
if err != nil {
return nil, clues.Wrap(err, "building path prefix services")
}
return path.BuildPrefix(br.tenant, srs, br.category)
}
func (br backupReason) key() string {
return br.category.String() + br.resource + br.service.String() + br.tenant
}
// ---------------------------------------------------------------------------
// common transformer
// ---------------------------------------------------------------------------
type servicerCategorizerProvider interface {
pathServicer
pathCategorier
idname.Provider
}
// produces the Reasoner basis described by the selector.
// In cases of reasons with subservices (ie, multiple
// services described by a backup or path), the selector
// will only ever generate a ServiceResource for the first
// service+resource pair in the set.
//
// TODO: it may be possible, if necessary, to add subservice
// recognition to the service via additional scopes.
func reasonsFor(
sel servicerCategorizerProvider,
tenantID string,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := map[string]identity.Reasoner{}
resource := sel.ID()
if useOwnerNameForID {
resource = sel.Name()
}
pc := sel.PathCategories()
for _, sl := range [][]path.CategoryType{pc.Includes, pc.Filters} {
for _, cat := range sl {
br := backupReason{
category: cat,
resource: resource,
service: service,
tenant: tenantID,
}
reasons[br.key()] = br
}
}
return maps.Values(reasons)
}

View File

@ -0,0 +1,406 @@
package selectors
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/path"
)
type ReasonsUnitSuite struct {
tester.Suite
}
func TestReasonsUnitSuite(t *testing.T) {
suite.Run(t, &ReasonsUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "use name",
sel: func() ExchangeRestore {
sel := NewExchangeRestore([]string{"timbo"})
sel.Include(sel.MailFolders(Any()))
plainSel := sel.SetDiscreteOwnerIDName("timbo", "timbubba")
sel, err := plainSel.ToExchangeRestore()
require.NoError(suite.T(), err, clues.ToCore(err))
return *sel
},
useName: true,
expect: []expect{
{
tenant: tenantID,
resource: "timbubba",
category: email,
service: exchange,
subtreePath: stpFor("timbubba", email, exchange),
},
},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}
func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}

View File

@ -544,8 +544,11 @@ func reduce[T scopeT, C categoryT](
continue
}
// first check, every entry needs to match the selector's resource owners.
if !matchesResourceOwner.Compare(repoPath.ResourceOwner()) {
// first check, every entry needs to have at least one protected resource
// that matches the selector's protected resources.
if !matchesResourceOwner.CompareAny(
path.ServiceResourcesToResources(
repoPath.ServiceResources())...) {
continue
}

View File

@ -22,15 +22,15 @@ import (
// tests
// ---------------------------------------------------------------------------
type SelectorScopesSuite struct {
type ScopesUnitSuite struct {
tester.Suite
}
func TestSelectorScopesSuite(t *testing.T) {
suite.Run(t, &SelectorScopesSuite{Suite: tester.NewUnitSuite(t)})
func TestScopesUnitSuite(t *testing.T) {
suite.Run(t, &ScopesUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *SelectorScopesSuite) TestContains() {
func (suite *ScopesUnitSuite) TestContains() {
table := []struct {
name string
scope func() mockScope
@ -108,7 +108,7 @@ func (suite *SelectorScopesSuite) TestContains() {
}
}
func (suite *SelectorScopesSuite) TestGetCatValue() {
func (suite *ScopesUnitSuite) TestGetCatValue() {
t := suite.T()
stub := stubScope("")
@ -122,7 +122,7 @@ func (suite *SelectorScopesSuite) TestGetCatValue() {
getCatValue(stub, mockCategorizer("foo")))
}
func (suite *SelectorScopesSuite) TestIsAnyTarget() {
func (suite *ScopesUnitSuite) TestIsAnyTarget() {
t := suite.T()
stub := stubScope("")
assert.True(t, isAnyTarget(stub, rootCatStub))
@ -253,16 +253,19 @@ var reduceTestTable = []struct {
},
}
func (suite *SelectorScopesSuite) TestReduce() {
func (suite *ScopesUnitSuite) TestReduce() {
deets := func() details.Details {
return details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.Entry{
{
RepoRef: stubRepoRef(
pathServiceStub,
suite.T(),
[]path.ServiceResource{{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
}},
pathCatStub,
rootCatStub.String(),
"stub",
leafCatStub.String(),
),
@ -298,16 +301,90 @@ func (suite *SelectorScopesSuite) TestReduce() {
}
}
func (suite *SelectorScopesSuite) TestReduce_locationRef() {
func (suite *ScopesUnitSuite) TestReduce_locationRef() {
deets := func() details.Details {
return details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.Entry{{
RepoRef: stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
}},
pathCatStub,
"stub",
leafCatStub.String(),
),
LocationRef: "a/b/c//defg",
}},
},
}
}
dataCats := map[path.CategoryType]mockCategorizer{
pathCatStub: rootCatStub,
}
for _, test := range reduceTestTable {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
ds := deets()
result := reduce[mockScope](
ctx,
&ds,
test.sel().Selector,
dataCats,
fault.New(true))
require.NotNil(t, result)
assert.Len(t, result.Entries, test.expectLen)
})
}
}
func (suite *ScopesUnitSuite) TestReduce_multipleServiceResources() {
deets := func() details.Details {
return details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.Entry{
// tid/serv/"matching-id"/subserv/"non-matching-id"/...
{
RepoRef: stubRepoRef(
pathServiceStub,
suite.T(),
[]path.ServiceResource{
{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
},
{
Service: pathServiceStub,
ProtectedResource: "foo",
},
},
pathCatStub,
"stub",
leafCatStub.String(),
),
LocationRef: "a/b/c//defg",
},
// tid/serv/"non-matching-id"/subserv/"matching-id"/...
{
RepoRef: stubRepoRef(
suite.T(),
[]path.ServiceResource{
{
Service: pathServiceStub,
ProtectedResource: "foo",
},
{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
},
},
pathCatStub,
rootCatStub.String(),
"stub",
leafCatStub.String(),
),
@ -341,7 +418,7 @@ func (suite *SelectorScopesSuite) TestReduce_locationRef() {
}
}
func (suite *SelectorScopesSuite) TestScopesByCategory() {
func (suite *ScopesUnitSuite) TestScopesByCategory() {
t := suite.T()
s1 := stubScope("")
s2 := stubScope("")
@ -357,7 +434,7 @@ func (suite *SelectorScopesSuite) TestScopesByCategory() {
assert.Empty(t, result[leafCatStub])
}
func (suite *SelectorScopesSuite) TestPasses() {
func (suite *ScopesUnitSuite) TestPasses() {
var (
cat = rootCatStub
pth = stubPath(suite.T(), "uid", []string{"fld"}, path.EventsCategory)
@ -401,7 +478,7 @@ func toMockScope(sc []scope) []mockScope {
return ms
}
func (suite *SelectorScopesSuite) TestMatchesPathValues() {
func (suite *ScopesUnitSuite) TestMatchesPathValues() {
cat := rootCatStub
short := "brunheelda"
@ -460,7 +537,7 @@ func (suite *SelectorScopesSuite) TestMatchesPathValues() {
}
}
func (suite *SelectorScopesSuite) TestDefaultItemOptions() {
func (suite *ScopesUnitSuite) TestDefaultItemOptions() {
table := []struct {
name string
cfg Config
@ -521,7 +598,7 @@ func (suite *SelectorScopesSuite) TestDefaultItemOptions() {
}
}
func (suite *SelectorScopesSuite) TestClean() {
func (suite *ScopesUnitSuite) TestClean() {
table := []struct {
name string
input []string
@ -568,7 +645,7 @@ func (suite *SelectorScopesSuite) TestClean() {
}
}
func (suite *SelectorScopesSuite) TestScopeConfig() {
func (suite *ScopesUnitSuite) TestScopeConfig() {
input := "input"
table := []struct {
@ -608,7 +685,7 @@ func (ms mockFMTState) Width() (int, bool) { return 0, false }
func (ms mockFMTState) Precision() (int, bool) { return 0, false }
func (ms mockFMTState) Flag(int) bool { return false }
func (suite *SelectorScopesSuite) TestScopesPII() {
func (suite *ScopesUnitSuite) TestScopesPII() {
table := []struct {
name string
s mockScope

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -87,6 +88,14 @@ type pathCategorier interface {
PathCategories() selectorPathCategories
}
type pathServicer interface {
PathService() path.ServiceType
}
type reasoner interface {
Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner
}
// ---------------------------------------------------------------------------
// Selector
// ---------------------------------------------------------------------------
@ -273,7 +282,7 @@ func (s Selector) Reduce(
return r.Reduce(ctx, deets, errs), nil
}
// returns the sets of path categories identified in each scope set.
// PathCategories returns the sets of path categories identified in each scope set.
func (s Selector) PathCategories() (selectorPathCategories, error) {
ro, err := selectorAsIface[pathCategorier](s)
if err != nil {
@ -283,6 +292,18 @@ func (s Selector) PathCategories() (selectorPathCategories, error) {
return ro.PathCategories(), nil
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s Selector) Reasons(tenantID string, useOwnerNameForID bool) ([]identity.Reasoner, error) {
ro, err := selectorAsIface[reasoner](s)
if err != nil {
return nil, err
}
return ro.Reasons(tenantID, useOwnerNameForID), nil
}
// transformer for arbitrary selector interfaces
func selectorAsIface[T any](s Selector) (T, error) {
var (

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var (
_ Reducer = &SharePointRestore{}
_ pathCategorier = &SharePointRestore{}
_ reasoner = &SharePointRestore{}
)
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
@ -121,6 +123,13 @@ func (s sharePoint) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s sharePoint) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -155,9 +155,12 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
}
return stubRepoRef(
path.SharePointService,
suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: siteID,
}},
cat,
siteID,
strings.Join(folderElems, "/"),
item)
}
@ -188,8 +191,24 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
"sid",
append(slices.Clone(prefixElems), itemElems3...),
"item3")
item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item4")
item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item5")
item4 = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: "sid",
}},
path.PagesCategory,
pairGH,
"item4")
item5 = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: "sid",
}},
path.PagesCategory,
pairGH,
"item5")
)
deets := &details.Details{
@ -417,10 +436,12 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
suite.Run(test.name, func() {
t := suite.T()
srs, err := path.NewServiceResources(path.SharePointService, "site")
require.NoError(t, err, clues.ToCore(err))
itemPath, err := path.Build(
"tenant",
"site",
path.SharePointService,
srs,
test.sc.PathType(),
true,
test.pathElems...)