Compare commits

...

10 Commits

Author SHA1 Message Date
ryanfkeepers
f5400f9e31 return path service and resource
Returns the path interface funcs for Service and Resource,
this time as "primaryFoo" funcs, to indicate that these are the
primary service and resource values.  This functionality is
mostly for quality of life.
2023-08-15 15:59:09 -06:00
ryanfkeepers
3d15a0d649 first pass on compliance with the reason
establishes behavior around using the reasoner interface
in a world where paths can contain multiple services.  Primarily
focused on ensuring the reasoner clearly guides maintainers
towards proper usage.
2023-08-15 13:29:14 -06:00
ryanfkeepers
48aae5d485 all non-resourcer compliance
Updates all path package uses that does not involve
a Resourcer.  Third to last step to wrapping up compliance.
Second step is to update the Resourcer interface to
comply with ServiceResources.  Last step is to clean up
any lingering linters, bugs, tests, and other things needed
to make the build green.
2023-08-11 16:16:40 -06:00
ryanfkeepers
bf398f6c8d fixing up selectors
adds a set of fixes targeted at the selectors package.  Largely
updates testing code.  But also updates the resource check
at the beginning of the Reduce flow to check for a match on
any of the resources in the ServiceResources slice.
2023-08-11 11:50:11 -06:00
ryanfkeepers
7c242819bc Add 'halves()' to path interface
Adds a func to the path interface which breaks up the path
into its two distinct halves: the prefix builder, and the folder
and item elements.  Likely to be useful in the future where
prefixes will be of arbitrary length, and downstream consumers
are primarly focused on the non-prefix portion of the path.
2023-08-11 11:14:27 -06:00
ryanfkeepers
6ed54fad9c first pass on ServiceResource tuple compliance
First pass for updating code outside of the path package
to comply with ServiceResource tuple slices.  Compliance
is still incomplete, so the build and tests
will still fail.  Changes
in this PR are focused on the easier-to-make changes, mostly
generating ServiceResources where there were
previously manual declarations of service and resource.
2023-08-10 17:55:34 -06:00
ryanfkeepers
dfd486cd41 distrubutes ServiceResource throughout path
Adds utilization of the ServiceResource struct to all
of the path package functionality.  In general, this
takes most instances where a service and resource
are requested, and replaces those values with a
slice of ServiceResource tuples.

To keep the review smaller, this change does not update
any packages outside of path.  Therefore the tests and
build are guaranteed to fail.  The next PR, which
updates all other packages with the changes, is
necessary for both PRs to move forward.
2023-08-10 14:57:04 -06:00
ryanfkeepers
5f7f1092ec introduce service_resource tuple
the service resource tuple will form the basis for
having paths contain multiple nested service
declarations, by allowing it to swap out the
Service() and ResourceOwner() funcs in favor of
a func that returns the ordered list of service-resource
tuples found in the path.
2023-08-10 12:51:42 -06:00
ryanfkeepers
859b8fafc7 small paths code rearrangement
small cleanup in paths, primarily splitting files
so that file contents are more clearly owned, which
should be a little better for readability and code
placement.

Also renames `ServicePrefix` to `BuildPrefix` in
anticipation of multi-service prefixes.
2023-08-10 12:50:56 -06:00
ryanfkeepers
89e8af74bc move selectorsToReason into selectors 2023-08-10 12:50:17 -06:00
79 changed files with 3640 additions and 1571 deletions

View File

@ -177,7 +177,7 @@ type collection struct {
func buildCollections( func buildCollections(
service path.ServiceType, service path.ServiceType,
tenant, user string, tenant, protectedResource string,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
colls []collection, colls []collection,
) ([]data.RestoreCollection, error) { ) ([]data.RestoreCollection, error) {
@ -186,8 +186,10 @@ func buildCollections(
for _, c := range colls { for _, c := range colls {
pth, err := path.Build( pth, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
service, Service: service,
ProtectedResource: protectedResource,
}},
c.category, c.category,
false, false,
c.PathElements...) c.PathElements...)

View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
) )
func AnyValueToT[T any](k string, m map[string]any) (T, error) { func AnyValueToT[T any](k string, m map[string]any) (T, error) {
@ -23,3 +24,21 @@ func AnyValueToT[T any](k string, m map[string]any) (T, error) {
return vt, nil return vt, nil
} }
func AnyToT[T any](a any) (T, error) {
if a == nil {
return *new(T), clues.New("missing value")
}
pt, ok := a.(*T)
if ok {
return ptr.Val(pt), nil
}
t, ok := a.(T)
if ok {
return t, nil
}
return *new(T), clues.New(fmt.Sprintf("unexpected type: %T", a))
}

View File

@ -21,11 +21,35 @@ func TestDataCollectionSuite(t *testing.T) {
} }
func (suite *DataCollectionSuite) TestStateOf() { func (suite *DataCollectionSuite) TestStateOf() {
fooP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "foo") fooP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
barP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "bar") barP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"bar")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
preP, err := path.Build("_t", "_u", path.ExchangeService, path.EmailCategory, false, "foo") preP, err := path.Build(
"_t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
table := []struct { table := []struct {

View File

@ -111,6 +111,12 @@ func (bb *backupBases) ClearAssistBases() {
bb.assistBases = nil bb.assistBases = nil
} }
// BaseKeyServiceCategory makes a backup base key using
// the reasoner's Service and Category.
func BaseKeyServiceCategory(br identity.Reasoner) string {
return br.Service().String() + br.Category().String()
}
// MergeBackupBases reduces the two BackupBases into a single BackupBase. // MergeBackupBases reduces the two BackupBases into a single BackupBase.
// Assumes the passed in BackupBases represents a prior backup version (across // Assumes the passed in BackupBases represents a prior backup version (across
// some migration that disrupts lookup), and that the BackupBases used to call // some migration that disrupts lookup), and that the BackupBases used to call

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/backup/identity"
idMock "github.com/alcionai/corso/src/pkg/backup/identity/mock"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -460,18 +461,12 @@ func (suite *BackupBasesUnitSuite) TestMergeBackupBases() {
bb := makeBackupBases(test.merge, test.assist) bb := makeBackupBases(test.merge, test.assist)
other := makeBackupBases(test.otherMerge, test.otherAssist) other := makeBackupBases(test.otherMerge, test.otherAssist)
expected := test.expect()
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
got := bb.MergeBackupBases( got := bb.MergeBackupBases(ctx, other, BaseKeyServiceCategory)
ctx, AssertBackupBasesEqual(t, test.expect(), got)
other,
func(r identity.Reasoner) string {
return r.Service().String() + r.Category().String()
})
AssertBackupBasesEqual(t, expected, got)
}) })
} }
} }
@ -843,3 +838,37 @@ func (suite *BackupBasesUnitSuite) TestFixupAndVerify() {
}) })
} }
} }
func (suite *BackupBasesUnitSuite) TestBaseKeyServiceCategory() {
table := []struct {
name string
service path.ServiceType
category path.CategoryType
expect string
}{
{
name: "unknown",
service: path.UnknownService,
category: path.UnknownCategory,
expect: "unknown",
},
{
name: "known service",
service: path.ExchangeService,
category: path.EmailCategory,
expect: "exchangeEmail",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result := BaseKeyServiceCategory(idMock.Reason{
TenantID: "tid",
Cat: test.category,
Svc: test.service,
})
assert.Equal(t, test.expect, result)
})
}
}

View File

@ -70,11 +70,14 @@ func (r reason) Category() path.CategoryType {
} }
func (r reason) SubtreePath() (path.Path, error) { func (r reason) SubtreePath() (path.Path, error) {
p, err := path.ServicePrefix( srs, err := path.NewServiceResources(
r.Tenant(),
r.ProtectedResource(),
r.Service(), r.Service(),
r.Category()) r.ProtectedResource())
if err != nil {
return nil, clues.Wrap(err, "building path service prefix")
}
p, err := path.BuildPrefix(r.Tenant(), srs, r.Category())
return p, clues.Wrap(err, "building path").OrNil() return p, clues.Wrap(err, "building path").OrNil()
} }

View File

@ -78,8 +78,10 @@ func (suite *KopiaDataCollectionUnitSuite) TestReturnsPath() {
pth, err := path.Build( pth, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data") "some", "path", "for", "data")
@ -330,8 +332,10 @@ func (suite *KopiaDataCollectionUnitSuite) TestFetchItemByName() {
pth, err := path.Build( pth, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: user,
}},
category, category,
false, false,
folder1, folder2) folder1, folder2)

View File

@ -32,8 +32,10 @@ func (suite *MergeCollectionUnitSuite) TestReturnsPath() {
pth, err := path.Build( pth, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data") "some", "path", "for", "data")
@ -61,8 +63,10 @@ func (suite *MergeCollectionUnitSuite) TestItems() {
pth, err := path.Build( pth, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data") "some", "path", "for", "data")
@ -101,8 +105,10 @@ func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() {
pth1, err := path.Build( pth1, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data") "some", "path", "for", "data")
@ -110,8 +116,10 @@ func (suite *MergeCollectionUnitSuite) TestAddCollection_DifferentPathFails() {
pth2, err := path.Build( pth2, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data2") "some", "path", "for", "data2")
@ -142,8 +150,10 @@ func (suite *MergeCollectionUnitSuite) TestFetchItemByName() {
pth, err := path.Build( pth, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"some", "path", "for", "data") "some", "path", "for", "data")

View File

@ -196,14 +196,17 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
return return
} }
ctx := clues.Add(
cp.ctx,
"services", path.ServiceResourcesToServices(d.repoPath.ServiceResources()),
"category", d.repoPath.Category().String())
// These items were sourced from a base snapshot or were cached in kopia so we // These items were sourced from a base snapshot or were cached in kopia so we
// never had to materialize their details in-memory. // never had to materialize their details in-memory.
if d.info == nil || d.cached { if d.info == nil || d.cached {
if d.prevPath == nil { if d.prevPath == nil {
cp.errs.AddRecoverable(cp.ctx, clues.New("item sourced from previous backup with no previous path"). cp.errs.AddRecoverable(cp.ctx, clues.New("item sourced from previous backup with no previous path").
With( WithClues(ctx).
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
Label(fault.LabelForceNoBackupCreation)) Label(fault.LabelForceNoBackupCreation))
return return
@ -219,9 +222,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
d.locationPath) d.locationPath)
if err != nil { if err != nil {
cp.errs.AddRecoverable(cp.ctx, clues.Wrap(err, "adding item to merge list"). cp.errs.AddRecoverable(cp.ctx, clues.Wrap(err, "adding item to merge list").
With( WithClues(ctx).
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
Label(fault.LabelForceNoBackupCreation)) Label(fault.LabelForceNoBackupCreation))
} }
@ -235,9 +236,7 @@ func (cp *corsoProgress) FinishedFile(relativePath string, err error) {
*d.info) *d.info)
if err != nil { if err != nil {
cp.errs.AddRecoverable(cp.ctx, clues.New("adding item to details"). cp.errs.AddRecoverable(cp.ctx, clues.New("adding item to details").
With( WithClues(ctx).
"service", d.repoPath.Service().String(),
"category", d.repoPath.Category().String()).
Label(fault.LabelForceNoBackupCreation)) Label(fault.LabelForceNoBackupCreation))
return return
@ -515,7 +514,7 @@ func streamBaseEntries(
// TODO(ashmrtn): We may eventually want to make this a function that is // TODO(ashmrtn): We may eventually want to make this a function that is
// passed in so that we can more easily switch it between different external // passed in so that we can more easily switch it between different external
// service provider implementations. // service provider implementations.
if !metadata.IsMetadataFile(itemPath) { if !metadata.IsMetadataFilePath(itemPath) {
// All items have item info in the base backup. However, we need to make // All items have item info in the base backup. However, we need to make
// sure we have enough metadata to find those entries. To do that we add // sure we have enough metadata to find those entries. To do that we add
// the item to progress and having progress aggregate everything for // the item to progress and having progress aggregate everything for

View File

@ -364,8 +364,10 @@ func TestCorsoProgressUnitSuite(t *testing.T) {
func (suite *CorsoProgressUnitSuite) SetupSuite() { func (suite *CorsoProgressUnitSuite) SetupSuite() {
p, err := path.Build( p, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.EmailCategory, path.EmailCategory,
true, true,
testInboxDir, "testFile") testInboxDir, "testFile")
@ -2867,16 +2869,40 @@ func (suite *HierarchyBuilderUnitSuite) TestBuildDirectoryTreeSelectsMigrateSubt
migratedUser = "user_migrate" migratedUser = "user_migrate"
) )
oldPrefixPathEmail, err := path.ServicePrefix(testTenant, testUser, path.ExchangeService, path.EmailCategory) oldPrefixPathEmail, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
newPrefixPathEmail, err := path.ServicePrefix(testTenant, migratedUser, path.ExchangeService, path.EmailCategory) newPrefixPathEmail, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: migratedUser,
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
oldPrefixPathCont, err := path.ServicePrefix(testTenant, testUser, path.ExchangeService, path.ContactsCategory) oldPrefixPathCont, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: testUser,
}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
newPrefixPathCont, err := path.ServicePrefix(testTenant, migratedUser, path.ExchangeService, path.ContactsCategory) newPrefixPathCont, err := path.BuildPrefix(
testTenant,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: migratedUser,
}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
var ( var (

View File

@ -724,8 +724,10 @@ func TestKopiaIntegrationSuite(t *testing.T) {
func (suite *KopiaIntegrationSuite) SetupSuite() { func (suite *KopiaIntegrationSuite) SetupSuite() {
tmp, err := path.Build( tmp, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
testInboxDir) testInboxDir)
@ -736,8 +738,10 @@ func (suite *KopiaIntegrationSuite) SetupSuite() {
tmp, err = path.Build( tmp, err = path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
testArchiveDir) testArchiveDir)
@ -804,14 +808,14 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
reasons := []identity.Reasoner{ reasons := []identity.Reasoner{
NewReason( NewReason(
testTenant, testTenant,
suite.storePath1.ResourceOwner(), suite.storePath1.PrimaryProtectedResource(),
suite.storePath1.Service(), suite.storePath1.PrimaryService(),
suite.storePath1.Category(), suite.storePath1.Category(),
), ),
NewReason( NewReason(
testTenant, testTenant,
suite.storePath2.ResourceOwner(), suite.storePath2.PrimaryProtectedResource(),
suite.storePath2.Service(), suite.storePath2.PrimaryService(),
suite.storePath2.Category(), suite.storePath2.Category(),
), ),
} }
@ -1052,8 +1056,10 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() { func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
tmp, err := path.Build( tmp, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.OneDriveService, ProtectedResource: testUser,
Service: path.OneDriveService,
}},
path.FilesCategory, path.FilesCategory,
false, false,
testInboxDir) testInboxDir)
@ -1079,8 +1085,8 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_NoDetailsForMeta() {
reasons := []identity.Reasoner{ reasons := []identity.Reasoner{
NewReason( NewReason(
testTenant, testTenant,
storePath.ResourceOwner(), storePath.PrimaryProtectedResource(),
storePath.Service(), storePath.PrimaryService(),
storePath.Category()), storePath.Category()),
} }
@ -1507,8 +1513,10 @@ func TestKopiaSimpleRepoIntegrationSuite(t *testing.T) {
func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() { func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() {
tmp, err := path.Build( tmp, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
testInboxDir) testInboxDir)
@ -1518,8 +1526,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupSuite() {
tmp, err = path.Build( tmp, err = path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
testArchiveDir) testArchiveDir)
@ -1800,8 +1810,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestBackupExcludeItem() {
func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() { func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() {
doesntExist, err := path.Build( doesntExist, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
true, true,
"subdir", "foo") "subdir", "foo")
@ -1934,8 +1946,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections() {
func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_PathChanges() { func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_PathChanges() {
rp1, err := path.Build( rp1, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
"corso_restore", "Inbox") "corso_restore", "Inbox")
@ -1943,8 +1957,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Path
rp2, err := path.Build( rp2, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
"corso_restore", "Archive") "corso_restore", "Archive")
@ -2057,8 +2073,10 @@ func (suite *KopiaSimpleRepoIntegrationSuite) TestProduceRestoreCollections_Fetc
rp1, err := path.Build( rp1, err := path.Build(
testTenant, testTenant,
testUser, []path.ServiceResource{{
path.ExchangeService, ProtectedResource: testUser,
Service: path.ExchangeService,
}},
path.EmailCategory, path.EmailCategory,
false, false,
"corso_restore", "Inbox") "corso_restore", "Inbox")

View File

@ -403,17 +403,19 @@ func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Libraries() {
// No excludes yet as this isn't an incremental backup. // No excludes yet as this isn't an incremental backup.
assert.True(t, excludes.Empty()) assert.True(t, excludes.Empty())
// assume the last service in the path is sharepoint.
srs := cols[0].FullPath().ServiceResources()
service := srs[len(srs)-1].Service
t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String()) t.Logf("cols[0] Path: %s\n", cols[0].FullPath().String())
assert.Equal( assert.Equal(t, path.SharePointMetadataService, service)
t,
path.SharePointMetadataService.String(), // assume the last service in the path is sharepoint.
cols[0].FullPath().Service().String()) srs = cols[1].FullPath().ServiceResources()
service = srs[len(srs)-1].Service
t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String()) t.Logf("cols[1] Path: %s\n", cols[1].FullPath().String())
assert.Equal( assert.Equal(t, path.SharePointService, service)
t,
path.SharePointService.String(),
cols[1].FullPath().Service().String())
} }
func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() { func (suite *SPCollectionIntgSuite) TestCreateSharePointCollection_Lists() {

View File

@ -1248,8 +1248,10 @@ func (suite *OneDriveCollectionsUnitSuite) TestGet() {
metadataPath, err := path.Builder{}.ToServiceCategoryMetadataPath( metadataPath, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant, tenant,
user, []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: user,
}},
path.FilesCategory, path.FilesCategory,
false) false)
require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err)) require.NoError(suite.T(), err, "making metadata path", clues.ToCore(err))

View File

@ -46,8 +46,10 @@ func (h itemBackupHandler) PathPrefix(
) (path.Path, error) { ) (path.Path, error) {
return path.Build( return path.Build(
tenantID, tenantID,
resourceOwner, []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: resourceOwner,
}},
path.FilesCategory, path.FilesCategory,
false, false,
odConsts.DrivesPathDir, odConsts.DrivesPathDir,

View File

@ -41,8 +41,10 @@ func (h libraryBackupHandler) PathPrefix(
) (path.Path, error) { ) (path.Path, error) {
return path.Build( return path.Build(
tenantID, tenantID,
resourceOwner, []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: resourceOwner,
}},
path.LibrariesCategory, path.LibrariesCategory,
false, false,
odConsts.DrivesPathDir, odConsts.DrivesPathDir,

View File

@ -42,8 +42,10 @@ func runComputeParentPermissionsTest(
entry, err := path.Build( entry, err := path.Build(
"tenant", "tenant",
resourceOwner, []path.ServiceResource{{
service, Service: service,
ProtectedResource: resourceOwner,
}},
category, category,
false, false,
strings.Split(entryPath, "/")...) strings.Split(entryPath, "/")...)
@ -51,8 +53,10 @@ func runComputeParentPermissionsTest(
rootEntry, err := path.Build( rootEntry, err := path.Build(
"tenant", "tenant",
resourceOwner, []path.ServiceResource{{
service, Service: service,
ProtectedResource: resourceOwner,
}},
category, category,
false, false,
strings.Split(rootEntryPath, "/")...) strings.Split(rootEntryPath, "/")...)

View File

@ -93,8 +93,10 @@ func CollectPages(
dir, err := path.Build( dir, err := path.Build(
creds.AzureTenantID, creds.AzureTenantID,
bpc.ProtectedResource.ID(), []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.PagesCategory, path.PagesCategory,
false, false,
tuple.Name) tuple.Name)
@ -144,8 +146,10 @@ func CollectLists(
dir, err := path.Build( dir, err := path.Build(
tenantID, tenantID,
bpc.ProtectedResource.ID(), []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.ListsCategory, path.ListsCategory,
false, false,
tuple.Name) tuple.Name)

View File

@ -218,12 +218,16 @@ func (sc *Collection) retrieveLists(
var ( var (
metrics support.CollectionMetrics metrics support.CollectionMetrics
el = errs.Local() el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = sc.fullPath.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
) )
lists, err := loadSiteLists( lists, err := loadSiteLists(
ctx, ctx,
sc.client.Stable, sc.client.Stable,
sc.fullPath.ResourceOwner(), protectedResource,
sc.jobs, sc.jobs,
errs) errs)
if err != nil { if err != nil {
@ -279,6 +283,10 @@ func (sc *Collection) retrievePages(
var ( var (
metrics support.CollectionMetrics metrics support.CollectionMetrics
el = errs.Local() el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = sc.fullPath.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
) )
betaService := sc.betaService betaService := sc.betaService
@ -286,14 +294,14 @@ func (sc *Collection) retrievePages(
return metrics, clues.New("beta service required").WithClues(ctx) return metrics, clues.New("beta service required").WithClues(ctx)
} }
parent, err := as.GetByID(ctx, sc.fullPath.ResourceOwner()) parent, err := as.GetByID(ctx, protectedResource)
if err != nil { if err != nil {
return metrics, err return metrics, err
} }
root := ptr.Val(parent.GetWebUrl()) root := ptr.Val(parent.GetWebUrl())
pages, err := betaAPI.GetSitePages(ctx, betaService, sc.fullPath.ResourceOwner(), sc.jobs, errs) pages, err := betaAPI.GetSitePages(ctx, betaService, protectedResource, sc.jobs, errs)
if err != nil { if err != nil {
return metrics, err return metrics, err
} }

View File

@ -95,8 +95,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
getDir: func(t *testing.T) path.Path { getDir: func(t *testing.T) path.Path {
dir, err := path.Build( dir, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: user,
}},
path.ListsCategory, path.ListsCategory,
false, false,
dirRoot) dirRoot)
@ -131,8 +133,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
getDir: func(t *testing.T) path.Path { getDir: func(t *testing.T) path.Path {
dir, err := path.Build( dir, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: user,
}},
path.PagesCategory, path.PagesCategory,
false, false,
dirRoot) dirRoot)

View File

@ -69,7 +69,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx, ictx = clues.Add(ctx,
"category", category, "category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location), "restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath()) "full_path", dc.FullPath())
) )
@ -219,9 +221,12 @@ func RestoreListCollection(
var ( var (
metrics = support.CollectionMetrics{} metrics = support.CollectionMetrics{}
directory = dc.FullPath() directory = dc.FullPath()
siteID = directory.ResourceOwner()
items = dc.Items(ctx, errs) items = dc.Items(ctx, errs)
el = errs.Local() el = errs.Local()
// todo: pass in the resourceOwner as an idname.Provider
srs = directory.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
) )
trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String()) trace.Log(ctx, "m365:sharepoint:restoreListCollection", directory.String())
@ -245,7 +250,7 @@ func RestoreListCollection(
ctx, ctx,
service, service,
itemData, itemData,
siteID, protectedResource,
restoreContainerName) restoreContainerName)
if err != nil { if err != nil {
el.AddRecoverable(ctx, err) el.AddRecoverable(ctx, err)
@ -292,7 +297,10 @@ func RestorePageCollection(
var ( var (
metrics = support.CollectionMetrics{} metrics = support.CollectionMetrics{}
directory = dc.FullPath() directory = dc.FullPath()
siteID = directory.ResourceOwner() // todo: pass in the resourceOwner as an idname.Provider
srs = directory.ServiceResources()
// take the last resource in srs, since that should be the data owner
protectedResource = srs[len(srs)-1].ProtectedResource
) )
trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String()) trace.Log(ctx, "m365:sharepoint:restorePageCollection", directory.String())
@ -325,7 +333,7 @@ func RestorePageCollection(
ctx, ctx,
service, service,
itemData, itemData,
siteID, protectedResource,
restoreContainerName) restoreContainerName)
if err != nil { if err != nil {
el.AddRecoverable(ctx, err) el.AddRecoverable(ctx, err)

View File

@ -1111,9 +1111,11 @@ func (suite *ControllerIntegrationSuite) TestMultiFolderBackupDifferentNames() {
}) })
totalItems, _, collections, expectedData, err := stub.CollectionsForInfo( totalItems, _, collections, expectedData, err := stub.CollectionsForInfo(
test.service,
suite.ctrl.tenant, suite.ctrl.tenant,
suite.user, []path.ServiceResource{{
Service: test.service,
ProtectedResource: suite.user,
}},
restoreCfg, restoreCfg,
[]stub.ColInfo{collection}, []stub.ColInfo{collection},
version.Backup, version.Backup,
@ -1247,11 +1249,11 @@ func (suite *ControllerIntegrationSuite) TestRestoreAndBackup_largeMailAttachmen
func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() { func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
table := []struct { table := []struct {
name string name string
resourceCat resource.Category resourceCat resource.Category
selectorFunc func(t *testing.T) selectors.Selector selectorFunc func(t *testing.T) selectors.Selector
service path.ServiceType metadataServices []path.ServiceType
categories []string categories []string
}{ }{
{ {
name: "Exchange", name: "Exchange",
@ -1265,7 +1267,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector return sel.Selector
}, },
service: path.ExchangeService, metadataServices: []path.ServiceType{path.ExchangeMetadataService},
categories: []string{ categories: []string{
path.EmailCategory.String(), path.EmailCategory.String(),
path.ContactsCategory.String(), path.ContactsCategory.String(),
@ -1281,7 +1283,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector return sel.Selector
}, },
service: path.OneDriveService, metadataServices: []path.ServiceType{path.OneDriveMetadataService},
categories: []string{ categories: []string{
path.FilesCategory.String(), path.FilesCategory.String(),
}, },
@ -1300,7 +1302,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
return sel.Selector return sel.Selector
}, },
service: path.SharePointService, metadataServices: []path.ServiceType{path.SharePointMetadataService},
categories: []string{ categories: []string{
path.LibrariesCategory.String(), path.LibrariesCategory.String(),
// not yet in use // not yet in use
@ -1363,7 +1365,7 @@ func (suite *ControllerIntegrationSuite) TestBackup_CreatesPrefixCollections() {
// Ignore metadata collections. // Ignore metadata collections.
fullPath := col.FullPath() fullPath := col.FullPath()
if fullPath.Service() != test.service { if path.ServiceResourcesMatchServices(fullPath.ServiceResources(), test.metadataServices) {
continue continue
} }

View File

@ -79,7 +79,13 @@ func BaseCollections(
for cat := range categories { for cat := range categories {
ictx := clues.Add(ctx, "base_service", service, "base_category", cat) ictx := clues.Add(ctx, "base_service", service, "base_category", cat)
full, err := path.ServicePrefix(tenant, rOwner, service, cat) full, err := path.BuildPrefix(
tenant,
[]path.ServiceResource{{
Service: service,
ProtectedResource: rOwner,
}},
cat)
if err != nil { if err != nil {
// Shouldn't happen. // Shouldn't happen.
err = clues.Wrap(err, "making path").WithClues(ictx) err = clues.Wrap(err, "making path").WithClues(ictx)

View File

@ -24,16 +24,44 @@ func (suite *CollectionsUnitSuite) TestNewPrefixCollection() {
serv := path.OneDriveService serv := path.OneDriveService
cat := path.FilesCategory cat := path.FilesCategory
p1, err := path.ServicePrefix("t", "ro1", serv, cat) p1, err := path.BuildPrefix(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro1",
}},
cat)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
p2, err := path.ServicePrefix("t", "ro2", serv, cat) p2, err := path.BuildPrefix(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro2",
}},
cat)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
items, err := path.Build("t", "ro", serv, cat, true, "fld", "itm") items, err := path.Build(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro",
}},
cat,
true,
"fld", "itm")
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
folders, err := path.Build("t", "ro", serv, cat, false, "fld") folders, err := path.Build(
"t",
[]path.ServiceResource{{
Service: serv,
ProtectedResource: "ro",
}},
cat,
false,
"fld")
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
table := []struct { table := []struct {

View File

@ -5,13 +5,30 @@ import (
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
func IsMetadataFile(p path.Path) bool { // IsMetadataFilePath checks whether the LAST service in the path
switch p.Service() { // supports metadata file types and, if so, whether the item has
// a meta suffix.
func IsMetadataFilePath(p path.Path) bool {
return IsMetadataFile(
p.ServiceResources(),
p.Category(),
p.Item())
}
// IsMetadataFile accepts the ServiceResources, cat, and Item values from
// a path (or equivalent representation) and returns true if the item
// is a Metadata entry.
func IsMetadataFile(
srs []path.ServiceResource,
cat path.CategoryType,
itemID string,
) bool {
switch srs[len(srs)-1].Service {
case path.OneDriveService: case path.OneDriveService:
return metadata.HasMetaSuffix(p.Item()) return metadata.HasMetaSuffix(itemID)
case path.SharePointService: case path.SharePointService:
return p.Category() == path.LibrariesCategory && metadata.HasMetaSuffix(p.Item()) return cat == path.LibrariesCategory && metadata.HasMetaSuffix(itemID)
default: default:
return false return false

View File

@ -1,7 +1,7 @@
package metadata_test package metadata_test
import ( import (
"fmt" "strings"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -18,7 +18,7 @@ import (
type boolfAssertionFunc func(assert.TestingT, bool, string, ...any) bool type boolfAssertionFunc func(assert.TestingT, bool, string, ...any) bool
type testCase struct { type testCase struct {
service path.ServiceType srs []path.ServiceResource
category path.CategoryType category path.CategoryType
expected boolfAssertionFunc expected boolfAssertionFunc
} }
@ -39,40 +39,89 @@ var (
cases = []testCase{ cases = []testCase{
{ {
service: path.ExchangeService, srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.EmailCategory, category: path.EmailCategory,
expected: assert.Falsef, expected: assert.Falsef,
}, },
{ {
service: path.ExchangeService, srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.ContactsCategory, category: path.ContactsCategory,
expected: assert.Falsef, expected: assert.Falsef,
}, },
{ {
service: path.ExchangeService, srs: []path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
category: path.EventsCategory, category: path.EventsCategory,
expected: assert.Falsef, expected: assert.Falsef,
}, },
{ {
service: path.OneDriveService, srs: []path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: user,
}},
category: path.FilesCategory, category: path.FilesCategory,
expected: assert.Truef, expected: assert.Truef,
}, },
{ {
service: path.SharePointService, srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.LibrariesCategory, category: path.LibrariesCategory,
expected: assert.Truef, expected: assert.Truef,
}, },
{ {
service: path.SharePointService, srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.ListsCategory, category: path.ListsCategory,
expected: assert.Falsef, expected: assert.Falsef,
}, },
{ {
service: path.SharePointService, srs: []path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: user,
}},
category: path.PagesCategory, category: path.PagesCategory,
expected: assert.Falsef, expected: assert.Falsef,
}, },
{
srs: []path.ServiceResource{
{
Service: path.OneDriveService,
ProtectedResource: user,
},
{
Service: path.ExchangeService,
ProtectedResource: user,
},
},
category: path.EventsCategory,
expected: assert.Falsef,
},
{
srs: []path.ServiceResource{
{
Service: path.ExchangeService,
ProtectedResource: user,
},
{
Service: path.OneDriveService,
ProtectedResource: user,
},
},
category: path.FilesCategory,
expected: assert.Truef,
},
} }
) )
@ -87,19 +136,26 @@ func TestMetadataUnitSuite(t *testing.T) {
func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() {
for _, test := range cases { for _, test := range cases {
for _, ext := range metaSuffixes { for _, ext := range metaSuffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T() t := suite.T()
p, err := path.Build( p, err := path.Build(
tenant, tenant,
user, test.srs,
test.service,
test.category, test.category,
true, true,
"file"+ext) "file"+ext)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
test.expected(t, metadata.IsMetadataFile(p), "extension %s", ext) test.expected(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
}) })
} }
} }
@ -108,19 +164,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_MetaSuffixes() {
func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_NotMetaSuffixes() { func (suite *MetadataUnitSuite) TestIsMetadataFile_Files_NotMetaSuffixes() {
for _, test := range cases { for _, test := range cases {
for _, ext := range notMetaSuffixes { for _, ext := range notMetaSuffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T() t := suite.T()
p, err := path.Build( p, err := path.Build(
tenant, tenant,
user, test.srs,
test.service,
test.category, test.category,
true, true,
"file"+ext) "file"+ext)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
}) })
} }
} }
@ -131,19 +194,26 @@ func (suite *MetadataUnitSuite) TestIsMetadataFile_Directories() {
for _, test := range cases { for _, test := range cases {
for _, ext := range suffixes { for _, ext := range suffixes {
suite.Run(fmt.Sprintf("%s %s %s", test.service, test.category, ext), func() { name := []string{}
for _, sr := range test.srs {
name = append(name, sr.Service.String())
}
name = append(name, test.category.String(), ext)
suite.Run(strings.Join(name, " "), func() {
t := suite.T() t := suite.T()
p, err := path.Build( p, err := path.Build(
tenant, tenant,
user, test.srs,
test.service,
test.category, test.category,
false, false,
"file"+ext) "file"+ext)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Falsef(t, metadata.IsMetadataFile(p), "extension %s", ext) assert.Falsef(t, metadata.IsMetadataFilePath(p), "extension %s", ext)
}) })
} }
} }

View File

@ -75,8 +75,10 @@ func MakeMetadataCollection(
p, err := path.Builder{}.ToServiceCategoryMetadataPath( p, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant, tenant,
resourceOwner, []path.ServiceResource{{
service, Service: service,
ProtectedResource: resourceOwner,
}},
cat, cat,
false) false)
if err != nil { if err != nil {

View File

@ -30,8 +30,10 @@ func (suite *MetadataCollectionUnitSuite) TestFullPath() {
p, err := path.Build( p, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"foo") "foo")
@ -72,8 +74,10 @@ func (suite *MetadataCollectionUnitSuite) TestItems() {
p, err := path.Build( p, err := path.Build(
"a-tenant", "a-tenant",
"a-user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "a-user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"foo") "foo")

View File

@ -865,7 +865,6 @@ func compareItem(
t *testing.T, t *testing.T,
colPath path.Path, colPath path.Path,
expected map[string][]byte, expected map[string][]byte,
service path.ServiceType,
category path.CategoryType, category path.CategoryType,
item data.Stream, item data.Stream,
mci m365Stub.ConfigInfo, mci m365Stub.ConfigInfo,
@ -875,7 +874,11 @@ func compareItem(
assert.NotZero(t, mt.ModTime()) assert.NotZero(t, mt.ModTime())
} }
switch service { // assume the last service in the path is the data owner
srs := colPath.ServiceResources()
lastService := srs[len(srs)-1].Service
switch lastService {
case path.ExchangeService: case path.ExchangeService:
switch category { switch category {
case path.EmailCategory: case path.EmailCategory:
@ -900,7 +903,7 @@ func compareItem(
return compareDriveItem(t, expected, item, mci, rootDir) return compareDriveItem(t, expected, item, mci, rootDir)
default: default:
assert.FailNowf(t, "unexpected service: %s", service.String()) assert.FailNowf(t, "unexpected service: %s", lastService.String())
} }
return true return true
@ -929,9 +932,12 @@ func checkHasCollections(
fp := g.FullPath() fp := g.FullPath()
loc := g.(data.LocationPather).LocationPath() loc := g.(data.LocationPather).LocationPath()
// take the last service, since it should be the one owning data
srs := fp.ServiceResources()
service := srs[len(srs)-1].Service
if fp.Service() == path.OneDriveService || if service == path.OneDriveService ||
(fp.Service() == path.SharePointService && fp.Category() == path.LibrariesCategory) { (service == path.SharePointService && fp.Category() == path.LibrariesCategory) {
dp, err := path.ToDrivePath(fp) dp, err := path.ToDrivePath(fp)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
continue continue
@ -942,8 +948,7 @@ func checkHasCollections(
p, err := loc.ToDataLayerPath( p, err := loc.ToDataLayerPath(
fp.Tenant(), fp.Tenant(),
fp.ResourceOwner(), fp.ServiceResources(),
fp.Service(),
fp.Category(), fp.Category(),
false) false)
if !assert.NoError(t, err, clues.ToCore(err)) { if !assert.NoError(t, err, clues.ToCore(err)) {
@ -972,11 +977,12 @@ func checkCollections(
for _, returned := range got { for _, returned := range got {
var ( var (
hasItems bool hasItems bool
service = returned.FullPath().Service()
category = returned.FullPath().Category() category = returned.FullPath().Category()
expectedColData = expected[returned.FullPath().String()] expectedColData = expected[returned.FullPath().String()]
folders = returned.FullPath().Elements() folders = returned.FullPath().Elements()
rootDir = folders[len(folders)-1] == mci.RestoreCfg.Location rootDir = folders[len(folders)-1] == mci.RestoreCfg.Location
srs = returned.FullPath().ServiceResources()
lastService = srs[len(srs)-1].Service
) )
// Need to iterate through all items even if we don't expect to find a match // Need to iterate through all items even if we don't expect to find a match
@ -988,9 +994,9 @@ func checkCollections(
// is for actual pull items. // is for actual pull items.
// TODO(ashmrtn): Should probably eventually check some data in metadata // TODO(ashmrtn): Should probably eventually check some data in metadata
// collections. // collections.
if service == path.ExchangeMetadataService || if lastService == path.ExchangeMetadataService ||
service == path.OneDriveMetadataService || lastService == path.OneDriveMetadataService ||
service == path.SharePointMetadataService { lastService == path.SharePointMetadataService {
skipped++ skipped++
continue continue
} }
@ -1006,7 +1012,6 @@ func checkCollections(
t, t,
returned.FullPath(), returned.FullPath(),
expectedColData, expectedColData,
service,
category, category,
item, item,
mci, mci,

View File

@ -343,8 +343,10 @@ func (f failingColl) Items(ctx context.Context, errs *fault.Bus) <-chan data.Str
func (f failingColl) FullPath() path.Path { func (f failingColl) FullPath() path.Path {
tmp, err := path.Build( tmp, err := path.Build(
"tenant", "tenant",
"user", []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: "user",
}},
path.EmailCategory, path.EmailCategory,
false, false,
"inbox") "inbox")
@ -489,7 +491,9 @@ func (suite *BackupIntgSuite) TestMailFetch() {
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
for _, c := range collections { for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
continue continue
} }
@ -575,7 +579,9 @@ func (suite *BackupIntgSuite) TestDelta() {
var metadata data.BackupCollection var metadata data.BackupCollection
for _, coll := range collections { for _, coll := range collections {
if coll.FullPath().Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
coll.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadata = coll metadata = coll
} }
} }
@ -609,7 +615,9 @@ func (suite *BackupIntgSuite) TestDelta() {
// Delta usage is commented out at the moment, anyway. So this is currently // Delta usage is commented out at the moment, anyway. So this is currently
// a sanity check that the minimum behavior won't break. // a sanity check that the minimum behavior won't break.
for _, coll := range collections { for _, coll := range collections {
if coll.FullPath().Service() != path.ExchangeMetadataService { if !path.ServiceResourcesMatchServices(
coll.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
ec, ok := coll.(*Collection) ec, ok := coll.(*Collection)
require.True(t, ok, "collection is *Collection") require.True(t, ok, "collection is *Collection")
assert.NotNil(t, ec) assert.NotNil(t, ec)
@ -664,7 +672,9 @@ func (suite *BackupIntgSuite) TestMailSerializationRegression() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService isMetadata := path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService})
streamChannel := edc.Items(ctx, fault.New(true)) streamChannel := edc.Items(ctx, fault.New(true))
// Verify that each message can be restored // Verify that each message can be restored
@ -742,7 +752,9 @@ func (suite *BackupIntgSuite) TestContactSerializationRegression() {
require.GreaterOrEqual(t, 2, len(edcs), "expected 1 <= num collections <= 2") require.GreaterOrEqual(t, 2, len(edcs), "expected 1 <= num collections <= 2")
for _, edc := range edcs { for _, edc := range edcs {
isMetadata := edc.FullPath().Service() == path.ExchangeMetadataService isMetadata := path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService})
count := 0 count := 0
for stream := range edc.Items(ctx, fault.New(true)) { for stream := range edc.Items(ctx, fault.New(true)) {
@ -872,7 +884,10 @@ func (suite *BackupIntgSuite) TestEventsSerializationRegression() {
for _, edc := range collections { for _, edc := range collections {
var isMetadata bool var isMetadata bool
if edc.FullPath().Service() != path.ExchangeMetadataService { // FIXME: this doesn't seem right, it's saying "if not metadata, isMetadata = true"
if !path.ServiceResourcesMatchServices(
edc.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
isMetadata = true isMetadata = true
assert.Equal(t, test.expected, edc.FullPath().Folder(false)) assert.Equal(t, test.expected, edc.FullPath().Folder(false))
} else { } else {
@ -1138,7 +1153,9 @@ func (suite *CollectionPopulationSuite) TestPopulateCollections() {
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
for _, c := range collections { for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++ metadatas++
continue continue
} }
@ -1267,8 +1284,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
oldPath1 := func(t *testing.T, cat path.CategoryType) path.Path { oldPath1 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := location.Append("1").ToDataLayerPath( res, err := location.Append("1").ToDataLayerPath(
suite.creds.AzureTenantID, suite.creds.AzureTenantID,
qp.ProtectedResource.ID(), []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat, cat,
false) false)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -1279,8 +1298,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
oldPath2 := func(t *testing.T, cat path.CategoryType) path.Path { oldPath2 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := location.Append("2").ToDataLayerPath( res, err := location.Append("2").ToDataLayerPath(
suite.creds.AzureTenantID, suite.creds.AzureTenantID,
qp.ProtectedResource.ID(), []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat, cat,
false) false)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -1291,8 +1312,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
idPath1 := func(t *testing.T, cat path.CategoryType) path.Path { idPath1 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := path.Builder{}.Append("1").ToDataLayerPath( res, err := path.Builder{}.Append("1").ToDataLayerPath(
suite.creds.AzureTenantID, suite.creds.AzureTenantID,
qp.ProtectedResource.ID(), []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat, cat,
false) false)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -1303,8 +1326,10 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
idPath2 := func(t *testing.T, cat path.CategoryType) path.Path { idPath2 := func(t *testing.T, cat path.CategoryType) path.Path {
res, err := path.Builder{}.Append("2").ToDataLayerPath( res, err := path.Builder{}.Append("2").ToDataLayerPath(
suite.creds.AzureTenantID, suite.creds.AzureTenantID,
qp.ProtectedResource.ID(), []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: qp.ProtectedResource.ID(),
}},
cat, cat,
false) false)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -1481,7 +1506,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_D
continue continue
} }
if c.FullPath().Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++ metadatas++
checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c) checkMetadata(t, ctx, qp.Category, test.expectMetadata(t, qp.Category), c)
continue continue
@ -1640,7 +1667,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_r
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0 deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
for _, c := range collections { for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++ metadatas++
continue continue
} }
@ -1706,7 +1735,15 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
) )
prevPath := func(t *testing.T, at ...string) path.Path { prevPath := func(t *testing.T, at ...string) path.Path {
p, err := path.Build(tenantID, userID, path.ExchangeService, cat, false, at...) p, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: userID,
}},
cat,
false,
at...)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
return p return p
@ -2064,7 +2101,9 @@ func (suite *CollectionPopulationSuite) TestFilterContainersAndFillCollections_i
require.NotNil(t, p) require.NotNil(t, p)
if p.Service() == path.ExchangeMetadataService { if path.ServiceResourcesMatchServices(
c.FullPath().ServiceResources(),
[]path.ServiceType{path.ExchangeMetadataService}) {
metadatas++ metadatas++
continue continue
} }

View File

@ -89,8 +89,10 @@ func (suite *CollectionSuite) TestColleciton_FullPath() {
fullPath, err := path.Build( fullPath, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: user,
}},
path.EmailCategory, path.EmailCategory,
false, false,
folder) folder)
@ -113,8 +115,10 @@ func (suite *CollectionSuite) TestCollection_NewCollection() {
fullPath, err := path.Build( fullPath, err := path.Build(
tenant, tenant,
user, []path.ServiceResource{{
path.ExchangeService, Service: path.ExchangeService,
ProtectedResource: user,
}},
path.EmailCategory, path.EmailCategory,
false, false,
folder) folder)
@ -129,9 +133,25 @@ func (suite *CollectionSuite) TestCollection_NewCollection() {
} }
func (suite *CollectionSuite) TestNewCollection_state() { func (suite *CollectionSuite) TestNewCollection_state() {
fooP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "foo") fooP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"foo")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
barP, err := path.Build("t", "u", path.ExchangeService, path.EmailCategory, false, "bar") barP, err := path.Build(
"t",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "u",
}},
path.EmailCategory,
false,
"bar")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
locPB := path.Builder{}.Append("human-readable") locPB := path.Builder{}.Append("human-readable")

View File

@ -812,8 +812,10 @@ func runCreateDestinationTest(
path1, err := path.Build( path1, err := path.Build(
tenantID, tenantID,
userID, []path.ServiceResource{{
svc, Service: svc,
ProtectedResource: userID,
}},
category, category,
false, false,
containerNames1...) containerNames1...)
@ -833,8 +835,10 @@ func runCreateDestinationTest(
path2, err := path.Build( path2, err := path.Build(
tenantID, tenantID,
userID, []path.ServiceResource{{
svc, Service: svc,
ProtectedResource: userID,
}},
category, category,
false, false,
containerNames2...) containerNames2...)

View File

@ -57,7 +57,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx, ictx = clues.Add(ctx,
"category", category, "category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location), "restore_location", clues.Hide(rcc.RestoreConfig.Location),
"protected_resource", clues.Hide(dc.FullPath().ResourceOwner()), "resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath()) "full_path", dc.FullPath())
) )

View File

@ -109,19 +109,23 @@ func migrationCollections(
// unlike exchange, which enumerates all folders on every // unlike exchange, which enumerates all folders on every
// backup, onedrive needs to force the owner PN -> ID migration // backup, onedrive needs to force the owner PN -> ID migration
mc, err := path.ServicePrefix( mc, err := path.BuildPrefix(
tenant, tenant,
bpc.ProtectedResource.ID(), []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: bpc.ProtectedResource.ID(),
}},
path.FilesCategory) path.FilesCategory)
if err != nil { if err != nil {
return nil, clues.Wrap(err, "creating user id migration path") return nil, clues.Wrap(err, "creating user id migration path")
} }
mpc, err := path.ServicePrefix( mpc, err := path.BuildPrefix(
tenant, tenant,
bpc.ProtectedResource.Name(), []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: bpc.ProtectedResource.Name(),
}},
path.FilesCategory) path.FilesCategory)
if err != nil { if err != nil {
return nil, clues.Wrap(err, "creating user name migration path") return nil, clues.Wrap(err, "creating user name migration path")

View File

@ -161,8 +161,10 @@ type pathPrefixer func(tID, ro, driveID string) (path.Path, error)
var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, error) { var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, error) {
return path.Build( return path.Build(
tID, tID,
ro, []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: ro,
}},
path.FilesCategory, path.FilesCategory,
false, false,
odConsts.DrivesPathDir, odConsts.DrivesPathDir,
@ -173,8 +175,10 @@ var defaultOneDrivePathPrefixer = func(tID, ro, driveID string) (path.Path, erro
var defaultSharePointPathPrefixer = func(tID, ro, driveID string) (path.Path, error) { var defaultSharePointPathPrefixer = func(tID, ro, driveID string) (path.Path, error) {
return path.Build( return path.Build(
tID, tID,
ro, []path.ServiceResource{{
path.SharePointService, Service: path.SharePointService,
ProtectedResource: ro,
}},
path.LibrariesCategory, path.LibrariesCategory,
false, false,
odConsts.DrivesPathDir, odConsts.DrivesPathDir,

View File

@ -61,7 +61,9 @@ func ConsumeRestoreCollections(
ictx = clues.Add(ctx, ictx = clues.Add(ctx,
"category", category, "category", category,
"restore_location", clues.Hide(rcc.RestoreConfig.Location), "restore_location", clues.Hide(rcc.RestoreConfig.Location),
"resource_owner", clues.Hide(dc.FullPath().ResourceOwner()), "resource_owners", clues.Hide(
path.ServiceResourcesToResources(
dc.FullPath().ServiceResources())),
"full_path", dc.FullPath()) "full_path", dc.FullPath())
) )

View File

@ -7,7 +7,7 @@ import (
"golang.org/x/exp/maps" "golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/m365/graph/metadata"
"github.com/alcionai/corso/src/internal/m365/mock" "github.com/alcionai/corso/src/internal/m365/mock"
"github.com/alcionai/corso/src/internal/m365/resource" "github.com/alcionai/corso/src/internal/m365/resource"
exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock" exchMock "github.com/alcionai/corso/src/internal/m365/service/exchange/mock"
@ -63,9 +63,11 @@ func GetCollectionsAndExpected(
for _, owner := range config.ResourceOwners { for _, owner := range config.ResourceOwners {
numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo( numItems, kopiaItems, ownerCollections, userExpectedData, err := CollectionsForInfo(
config.Service,
config.Tenant, config.Tenant,
owner, []path.ServiceResource{{
Service: config.Service,
ProtectedResource: owner,
}},
config.RestoreCfg, config.RestoreCfg,
testCollections, testCollections,
backupVersion) backupVersion)
@ -84,8 +86,8 @@ func GetCollectionsAndExpected(
} }
func CollectionsForInfo( func CollectionsForInfo(
service path.ServiceType, tenant string,
tenant, user string, srs []path.ServiceResource,
restoreCfg control.RestoreConfig, restoreCfg control.RestoreConfig,
allInfo []ColInfo, allInfo []ColInfo,
backupVersion int, backupVersion int,
@ -100,8 +102,7 @@ func CollectionsForInfo(
for _, info := range allInfo { for _, info := range allInfo {
pth, err := path.Build( pth, err := path.Build(
tenant, tenant,
user, srs,
service,
info.Category, info.Category,
false, false,
info.PathElements...) info.PathElements...)
@ -129,9 +130,7 @@ func CollectionsForInfo(
baseExpected[info.Items[i].LookupKey] = info.Items[i].Data baseExpected[info.Items[i].LookupKey] = info.Items[i].Data
// We do not count metadata files against item count // We do not count metadata files against item count
if backupVersion > 0 && if backupVersion > 0 && metadata.IsMetadataFile(srs, info.Category, info.Items[i].Name) {
(service == path.OneDriveService || service == path.SharePointService) &&
metadata.HasMetaSuffix(info.Items[i].Name) {
continue continue
} }
@ -166,9 +165,13 @@ func backupOutputPathFromRestore(
inputPath path.Path, inputPath path.Path,
) (path.Path, error) { ) (path.Path, error) {
base := []string{restoreCfg.Location} base := []string{restoreCfg.Location}
srs := inputPath.ServiceResources()
// only the last service is checked, because that should be the service
// whose data is stored..
lastService := srs[len(srs)-1].Service
// OneDrive has leading information like the drive ID. // OneDrive has leading information like the drive ID.
if inputPath.Service() == path.OneDriveService || inputPath.Service() == path.SharePointService { if lastService == path.OneDriveService || lastService == path.SharePointService {
folders := inputPath.Folders() folders := inputPath.Folders()
base = append(append([]string{}, folders[:3]...), restoreCfg.Location) base = append(append([]string{}, folders[:3]...), restoreCfg.Location)
@ -177,14 +180,13 @@ func backupOutputPathFromRestore(
} }
} }
if inputPath.Service() == path.ExchangeService && inputPath.Category() == path.EmailCategory { if lastService == path.ExchangeService && inputPath.Category() == path.EmailCategory {
base = append(base, inputPath.Folders()...) base = append(base, inputPath.Folders()...)
} }
return path.Build( return path.Build(
inputPath.Tenant(), inputPath.Tenant(),
inputPath.ResourceOwner(), inputPath.ServiceResources(),
inputPath.Service(),
inputPath.Category(), inputPath.Category(),
false, false,
base...) base...)

View File

@ -326,11 +326,17 @@ func (op *BackupOperation) do(
detailsStore streamstore.Streamer, detailsStore streamstore.Streamer,
backupID model.StableID, backupID model.StableID,
) (*details.Builder, error) { ) (*details.Builder, error) {
var ( lastBackupVersion := version.NoBackup
reasons = selectorToReasons(op.account.ID(), op.Selectors, false)
fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors) reasons, err := op.Selectors.Reasons(op.account.ID(), false)
lastBackupVersion = version.NoBackup if err != nil {
) return nil, clues.Wrap(err, "getting reasons")
}
fallbackReasons, err := makeFallbackReasons(op.account.ID(), op.Selectors)
if err != nil {
return nil, clues.Wrap(err, "getting fallback reasons")
}
logger.Ctx(ctx).With( logger.Ctx(ctx).With(
"control_options", op.Options, "control_options", op.Options,
@ -424,13 +430,14 @@ func (op *BackupOperation) do(
return deets, nil return deets, nil
} }
func makeFallbackReasons(tenant string, sel selectors.Selector) []identity.Reasoner { func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) {
if sel.PathService() != path.SharePointService && if sel.PathService() != path.SharePointService &&
sel.DiscreteOwner != sel.DiscreteOwnerName { sel.DiscreteOwner != sel.DiscreteOwnerName {
return selectorToReasons(tenant, sel, true) return sel.Reasons(tenant, true)
} }
return nil // return nil for fallback reasons since a nil value will no-op.
return nil, nil
} }
// checker to see if conditions are correct for incremental backup behavior such as // checker to see if conditions are correct for incremental backup behavior such as
@ -472,35 +479,6 @@ func produceBackupDataCollections(
// Consumer funcs // Consumer funcs
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func selectorToReasons(
tenant string,
sel selectors.Selector,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := []identity.Reasoner{}
pcs, err := sel.PathCategories()
if err != nil {
// This is technically safe, it's just that the resulting backup won't be
// usable as a base for future incremental backups.
return nil
}
owner := sel.DiscreteOwner
if useOwnerNameForID {
owner = sel.DiscreteOwnerName
}
for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} {
for _, cat := range sl {
reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat))
}
}
return reasons
}
// calls kopia to backup the collections of data // calls kopia to backup the collections of data
func consumeBackupCollections( func consumeBackupCollections(
ctx context.Context, ctx context.Context,
@ -560,8 +538,8 @@ func consumeBackupCollections(
func matchesReason(reasons []identity.Reasoner, p path.Path) bool { func matchesReason(reasons []identity.Reasoner, p path.Path) bool {
for _, reason := range reasons { for _, reason := range reasons {
if p.ResourceOwner() == reason.ProtectedResource() && if p.PrimaryProtectedResource() == reason.ProtectedResource() &&
p.Service() == reason.Service() && p.PrimaryService() == reason.Service() &&
p.Category() == reason.Category() { p.Category() == reason.Category() {
return true return true
} }
@ -588,7 +566,10 @@ func getNewPathRefs(
// TODO(ashmrtn): In the future we can remove this first check as we'll be // TODO(ashmrtn): In the future we can remove this first check as we'll be
// able to assume we always have the location in the previous entry. We'll end // able to assume we always have the location in the previous entry. We'll end
// up doing some extra parsing, but it will simplify this code. // up doing some extra parsing, but it will simplify this code.
if repoRef.Service() == path.ExchangeService { //
// Currently safe to check only the 0th SR, since exchange had no subservices
// at the time of the locations addition.
if repoRef.ServiceResources()[0].Service == path.ExchangeService {
newPath, newLoc, err := dataFromBackup.GetNewPathRefs( newPath, newLoc, err := dataFromBackup.GetNewPathRefs(
repoRef.ToBuilder(), repoRef.ToBuilder(),
entry.Modified(), entry.Modified(),

View File

@ -218,8 +218,10 @@ func makeMetadataBasePath(
p, err := path.Builder{}.ToServiceCategoryMetadataPath( p, err := path.Builder{}.ToServiceCategoryMetadataPath(
tenant, tenant,
resourceOwner, []path.ServiceResource{{
service, Service: service,
ProtectedResource: resourceOwner,
}},
category, category,
false) false)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
@ -264,6 +266,7 @@ func makePath(t *testing.T, elements []string, isItem bool) path.Path {
return p return p
} }
// FIXME: out of date, does not contain sharepoint support
func makeDetailsEntry( func makeDetailsEntry(
t *testing.T, t *testing.T,
p path.Path, p path.Path,
@ -288,7 +291,10 @@ func makeDetailsEntry(
Updated: updated, Updated: updated,
} }
switch p.Service() { srs := p.ServiceResources()
lastService := srs[len(srs)-1].Service
switch lastService {
case path.ExchangeService: case path.ExchangeService:
if p.Category() != path.EmailCategory { if p.Category() != path.EmailCategory {
assert.FailNowf( assert.FailNowf(
@ -319,7 +325,7 @@ func makeDetailsEntry(
assert.FailNowf( assert.FailNowf(
t, t,
"service %s not supported in helper function", "service %s not supported in helper function",
p.Service().String()) lastService.String())
} }
return res return res
@ -527,6 +533,23 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_ConsumeBackupDataCollections
fault.New(true)) fault.New(true))
} }
// makeElements allows creation of repoRefs that wouldn't
// pass paths package validators.
func makeElements(
tenant string,
srs []path.ServiceResource,
cat path.CategoryType,
suffix ...string,
) path.Elements {
elems := append(
path.Elements{tenant},
path.ServiceResourcesToElements(srs)...)
elems = append(elems, cat.String())
elems = append(elems, suffix...)
return elems
}
func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() { func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems() {
var ( var (
tenant = "a-tenant" tenant = "a-tenant"
@ -594,13 +617,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
pathReason1 = kopia.NewReason( pathReason1 = kopia.NewReason(
"", "",
itemPath1.ResourceOwner(), itemPath1.PrimaryProtectedResource(),
itemPath1.Service(), itemPath1.PrimaryService(),
itemPath1.Category()) itemPath1.Category())
pathReason3 = kopia.NewReason( pathReason3 = kopia.NewReason(
"", "",
itemPath3.ResourceOwner(), itemPath3.PrimaryProtectedResource(),
itemPath3.Service(), itemPath3.PrimaryService(),
itemPath3.Category()) itemPath3.Category())
time1 = time.Now() time1 = time.Now()
@ -707,17 +730,11 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
DetailsModel: details.DetailsModel{ DetailsModel: details.DetailsModel{
Entries: []details.Entry{ Entries: []details.Entry{
{ {
RepoRef: stdpath.Join( RepoRef: stdpath.Join(makeElements(
append( itemPath1.Tenant(),
[]string{ itemPath1.ServiceResources(),
itemPath1.Tenant(), path.UnknownCategory,
itemPath1.Service().String(), itemPath1.Folders()...)...),
itemPath1.ResourceOwner(),
path.UnknownCategory.String(),
},
itemPath1.Folders()...,
)...,
),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
OneDrive: &details.OneDriveInfo{ OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem, ItemType: details.OneDriveItem,
@ -738,16 +755,13 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsItems
res := newMockDetailsMergeInfoer() res := newMockDetailsMergeInfoer()
p := makePath( p := makePath(
suite.T(), suite.T(),
[]string{ makeElements(
itemPath1.Tenant(), itemPath1.Tenant(),
path.OneDriveService.String(), itemPath1.ServiceResources(),
itemPath1.ResourceOwner(), path.FilesCategory,
path.FilesCategory.String(),
"personal", "personal",
"item1", "item1"),
}, true)
true,
)
res.add(itemPath1, p, nil) res.add(itemPath1, p, nil)
@ -1226,8 +1240,8 @@ func (suite *BackupOpUnitSuite) TestBackupOperation_MergeBackupDetails_AddsFolde
pathReason1 = kopia.NewReason( pathReason1 = kopia.NewReason(
"", "",
itemPath1.ResourceOwner(), itemPath1.PrimaryProtectedResource(),
itemPath1.Service(), itemPath1.PrimaryService(),
itemPath1.Category()) itemPath1.Category())
backup1 = kopia.BackupEntry{ backup1 = kopia.BackupEntry{
@ -1635,7 +1649,15 @@ func (suite *AssistBackupIntegrationSuite) TestBackupTypesForFailureModes() {
pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID} pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID}
tmp, err := path.Build(tenantID, userID, path.OneDriveService, path.FilesCategory, false, pathElements...) tmp, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: userID,
}},
path.FilesCategory,
false,
pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
locPath := path.Builder{}.Append(tmp.Folders()...) locPath := path.Builder{}.Append(tmp.Folders()...)
@ -1916,7 +1938,15 @@ func (suite *AssistBackupIntegrationSuite) TestExtensionsIncrementals() {
pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID} pathElements := []string{odConsts.DrivesPathDir, "drive-id", odConsts.RootPathDir, folderID}
tmp, err := path.Build(tenantID, userID, path.OneDriveService, path.FilesCategory, false, pathElements...) tmp, err := path.Build(
tenantID,
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: userID,
}},
path.FilesCategory,
false,
pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
locPath := path.Builder{}.Append(tmp.Folders()...) locPath := path.Builder{}.Append(tmp.Folders()...)

View File

@ -80,12 +80,7 @@ func getManifestsAndMetadata(
// 3. the current reasons contain all the necessary manifests. // 3. the current reasons contain all the necessary manifests.
// Note: This is not relevant for assist backups, since they are newly introduced // Note: This is not relevant for assist backups, since they are newly introduced
// and they don't exist with fallback reasons. // and they don't exist with fallback reasons.
bb = bb.MergeBackupBases( bb = bb.MergeBackupBases(ctx, fbb, kopia.BaseKeyServiceCategory)
ctx,
fbb,
func(r identity.Reasoner) string {
return r.Service().String() + r.Category().String()
})
if !getMetadata { if !getMetadata {
return bb, nil, false, nil return bb, nil, false, nil

View File

@ -344,7 +344,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
assert.Equal( assert.Equal(
t, t,
path.ExchangeMetadataService, path.ExchangeMetadataService,
p.Service(), p.ServiceResources()[0].Service,
"read data service") "read data service")
assert.Contains( assert.Contains(
@ -354,8 +354,7 @@ func (suite *OperationsManifestsUnitSuite) TestProduceManifestsAndMetadata() {
path.ContactsCategory, path.ContactsCategory,
}, },
p.Category(), p.Category(),
"read data category doesn't match a given reason", "read data category doesn't match a given reason")
)
} }
}, },
expectMans: kopia.NewMockBackupBases().WithMergeBases( expectMans: kopia.NewMockBackupBases().WithMergeBases(

View File

@ -42,10 +42,9 @@ func locationRef(
func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, error) { func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, error) {
if len(locRef.Elements()) == 0 { if len(locRef.Elements()) == 0 {
res, err := path.ServicePrefix( res, err := path.BuildPrefix(
repoRef.Tenant(), repoRef.Tenant(),
repoRef.ResourceOwner(), repoRef.ServiceResources(),
repoRef.Service(),
repoRef.Category()) repoRef.Category())
if err != nil { if err != nil {
return nil, clues.Wrap(err, "getting prefix for empty location") return nil, clues.Wrap(err, "getting prefix for empty location")
@ -56,8 +55,7 @@ func basicLocationPath(repoRef path.Path, locRef *path.Builder) (path.Path, erro
return locRef.ToDataLayerPath( return locRef.ToDataLayerPath(
repoRef.Tenant(), repoRef.Tenant(),
repoRef.ResourceOwner(), repoRef.ServiceResources(),
repoRef.Service(),
repoRef.Category(), repoRef.Category(),
false) false)
} }

View File

@ -36,12 +36,8 @@ func (suite *RestorePathTransformerUnitSuite) TestGetPaths() {
repoRef path.Path, repoRef path.Path,
unescapedFolders ...string, unescapedFolders ...string,
) string { ) string {
return path.Builder{}. pfx, _ := repoRef.Halves()
Append( return pfx.
repoRef.Tenant(),
repoRef.Service().String(),
repoRef.ResourceOwner(),
repoRef.Category().String()).
Append(unescapedFolders...). Append(unescapedFolders...).
String() String()
} }

View File

@ -347,7 +347,13 @@ func testExchangeContinuousBackups(suite *ExchangeBackupIntgSuite, toggles contr
// }, // },
} }
rrPfx, err := path.ServicePrefix(acct.ID(), uidn.ID(), service, path.EmailCategory) rrPfx, err := path.BuildPrefix(
acct.ID(),
[]path.ServiceResource{{
Service: service,
ProtectedResource: uidn.ID(),
}},
path.EmailCategory)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// strip the category from the prefix; we primarily want the tenant and resource owner. // strip the category from the prefix; we primarily want the tenant and resource owner.

View File

@ -33,6 +33,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/backup/identity"
idMock "github.com/alcionai/corso/src/pkg/backup/identity/mock"
"github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count" "github.com/alcionai/corso/src/pkg/count"
@ -244,7 +245,11 @@ func checkBackupIsInManifests(
for _, category := range categories { for _, category := range categories {
t.Run(category.String(), func(t *testing.T) { t.Run(category.String(), func(t *testing.T) {
var ( var (
r = kopia.NewReason("", resourceOwner, sel.PathService(), category) r = idMock.Reason{
Cat: category,
Svc: sel.PathService(),
Resource: resourceOwner,
}
tags = map[string]string{kopia.TagBackupCategory: ""} tags = map[string]string{kopia.TagBackupCategory: ""}
found bool found bool
) )
@ -291,11 +296,15 @@ func checkMetadataFilesExist(
paths := []path.RestorePaths{} paths := []path.RestorePaths{}
pathsByRef := map[string][]string{} pathsByRef := map[string][]string{}
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: resourceOwner,
}}
for _, fName := range files { for _, fName := range files {
p, err := path.Builder{}. p, err := path.Builder{}.
Append(fName). Append(fName).
ToServiceCategoryMetadataPath(tenant, resourceOwner, service, category, true) ToServiceCategoryMetadataPath(tenant, srs, category, true)
if !assert.NoError(t, err, "bad metadata path", clues.ToCore(err)) { if !assert.NoError(t, err, "bad metadata path", clues.ToCore(err)) {
continue continue
} }

View File

@ -213,7 +213,13 @@ func runDriveIncrementalTest(
} }
) )
rrPfx, err := path.ServicePrefix(atid, roidn.ID(), service, category) rrPfx, err := path.BuildPrefix(
atid,
[]path.ServiceResource{{
Service: service,
ProtectedResource: roidn.ID(),
}},
category)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
// strip the category from the prefix; we primarily want the tenant and resource owner. // strip the category from the prefix; we primarily want the tenant and resource owner.

View File

@ -198,8 +198,13 @@ func collect(
service path.ServiceType, service path.ServiceType,
col Collectable, col Collectable,
) (data.BackupCollection, error) { ) (data.BackupCollection, error) {
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: col.purpose,
}}
// construct the path of the container // construct the path of the container
p, err := path.Builder{}.ToStreamStorePath(tenantID, col.purpose, service, false) p, err := path.Builder{}.ToStreamStorePath(tenantID, srs, false)
if err != nil { if err != nil {
return nil, clues.Stack(err).WithClues(ctx) return nil, clues.Stack(err).WithClues(ctx)
} }
@ -257,10 +262,15 @@ func read(
rer inject.RestoreProducer, rer inject.RestoreProducer,
errs *fault.Bus, errs *fault.Bus,
) error { ) error {
srs := []path.ServiceResource{{
Service: service,
ProtectedResource: col.purpose,
}}
// construct the path of the container // construct the path of the container
p, err := path.Builder{}. p, err := path.Builder{}.
Append(col.itemName). Append(col.itemName).
ToStreamStorePath(tenantID, col.purpose, service, true) ToStreamStorePath(tenantID, srs, true)
if err != nil { if err != nil {
return clues.Stack(err).WithClues(ctx) return clues.Stack(err).WithClues(ctx)
} }

View File

@ -1122,8 +1122,10 @@ func makeItemPath(
p, err := path.Build( p, err := path.Build(
tenant, tenant,
resourceOwner, []path.ServiceResource{{
service, Service: service,
ProtectedResource: resourceOwner,
}},
category, category,
true, true,
elems...) elems...)

View File

@ -5,9 +5,20 @@ import "github.com/alcionai/corso/src/pkg/path"
// Reasoner describes the parts of the backup that make up its // Reasoner describes the parts of the backup that make up its
// data identity: the tenant, protected resources, services, and // data identity: the tenant, protected resources, services, and
// categories which are held within the backup. // categories which are held within the backup.
//
// Reasoner only recognizes the "primary" protected resource and
// service. IE: subservice resources and services are not recognized
// as part of the backup Reason.
type Reasoner interface { type Reasoner interface {
Tenant() string Tenant() string
// ProtectedResource represents the Primary protected resource.
// IE: if a path or backup supports subservices, this value
// should only provide the first service's resource, and not the
// resource for any subservice.
ProtectedResource() string ProtectedResource() string
// Service represents the Primary service.
// IE: if a path or backup supports subservices, this value
// should only provide the first service; not a subservice.
Service() path.ServiceType Service() path.ServiceType
Category() path.CategoryType Category() path.CategoryType
// SubtreePath returns the path prefix for data in existing backups that have // SubtreePath returns the path prefix for data in existing backups that have

View File

@ -0,0 +1,47 @@
package mock
import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/path"
)
type Reason struct {
TenantID string
Cat path.CategoryType
Svc path.ServiceType
Resource string
SubtreeErr error
}
func (r Reason) Tenant() string {
return r.TenantID
}
func (r Reason) Category() path.CategoryType {
return r.Cat
}
func (r Reason) Service() path.ServiceType {
return r.Svc
}
func (r Reason) ProtectedResource() string {
return r.Resource
}
func (r Reason) SubtreePath() (path.Path, error) {
if r.SubtreeErr != nil {
return nil, r.SubtreeErr
}
p, err := path.BuildPrefix(
r.Tenant(),
[]path.ServiceResource{{
ProtectedResource: r.Resource,
Service: r.Svc,
}},
r.Category())
return p, clues.Wrap(err, "building path").OrNil()
}

View File

@ -104,7 +104,17 @@ func (suite *RestoreUnitSuite) TestEnsureRestoreConfigDefaults() {
} }
func (suite *RestoreUnitSuite) TestRestoreConfig_piiHandling() { func (suite *RestoreUnitSuite) TestRestoreConfig_piiHandling() {
p, err := path.Build("tid", "ro", path.ExchangeService, path.EmailCategory, true, "foo", "bar", "baz") p, err := path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "ro",
}},
path.EmailCategory,
true,
"foo",
"bar",
"baz")
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
cdrc := control.DefaultRestoreConfig(dttm.HumanReadable) cdrc := control.DefaultRestoreConfig(dttm.HumanReadable)

View File

@ -19,7 +19,15 @@ const itemID = "item_id"
var ( var (
err error err error
itemPath, _ = path.Build("tid", "own", path.ExchangeService, path.ContactsCategory, false, "foo") itemPath, _ = path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "own",
}},
path.ContactsCategory,
false,
"foo")
) )
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

346
src/pkg/path/builder.go Normal file
View File

@ -0,0 +1,346 @@
package path
import (
"bytes"
"crypto/sha256"
"fmt"
"github.com/alcionai/clues"
)
// interface compliance required for handling PII
var (
_ clues.Concealer = &Builder{}
_ fmt.Stringer = &Builder{}
)
// Builder is a simple path representation that only tracks path elements. It
// can join, escape, and unescape elements. Higher-level packages are expected
// to wrap this struct to build resource-specific contexts (e.x. an
// ExchangeMailPath).
// Resource-specific paths allow access to more information like segments in the
// path. Builders that are turned into resource paths later on do not need to
// manually add prefixes for items that normally appear in the data layer (ex.
// tenant ID, service, user ID, etc).
type Builder struct {
// Unescaped version of elements.
elements Elements
}
// Append creates a copy of this Builder and adds the given elements them to the
// end of the new Builder. Elements are added in the order they are passed.
func (pb Builder) Append(elements ...string) *Builder {
res := &Builder{elements: make([]string, len(pb.elements))}
copy(res.elements, pb.elements)
// Unescaped elements can't fail validation.
//nolint:errcheck
res.appendElements(false, elements)
return res
}
func (pb *Builder) appendElements(escaped bool, elements []string) error {
for _, e := range elements {
if len(e) == 0 {
continue
}
tmp := e
if escaped {
tmp = TrimTrailingSlash(tmp)
// If tmp was just the path separator then it will be empty now.
if len(tmp) == 0 {
continue
}
if err := validateEscapedElement(tmp); err != nil {
return err
}
tmp = unescape(tmp)
}
pb.elements = append(pb.elements, tmp)
}
return nil
}
// UnescapeAndAppend creates a copy of this Builder and adds one or more already
// escaped path elements to the end of the new Builder. Elements are added in
// the order they are passed.
func (pb Builder) UnescapeAndAppend(elements ...string) (*Builder, error) {
res := &Builder{elements: make([]string, 0, len(pb.elements))}
copy(res.elements, pb.elements)
if err := res.appendElements(true, elements); err != nil {
return nil, err
}
return res, nil
}
// SplitUnescapeAppend takes in an escaped string representing a directory
// path, splits the string, and appends it to the current builder.
func (pb Builder) SplitUnescapeAppend(s string) (*Builder, error) {
elems := Split(TrimTrailingSlash(s))
return pb.UnescapeAndAppend(elems...)
}
func (pb Builder) PopFront() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
elements := make([]string, len(pb.elements)-1)
copy(elements, pb.elements[1:])
return &Builder{
elements: elements,
}
}
// Dir removes the last element from the builder.
func (pb Builder) Dir() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
return &Builder{
// Safe to use the same elements because Builders are immutable.
elements: pb.elements[:len(pb.elements)-1],
}
}
// HeadElem returns the first element in the Builder.
func (pb Builder) HeadElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[0]
}
// LastElem returns the last element in the Builder.
func (pb Builder) LastElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[len(pb.elements)-1]
}
// UpdateParent updates leading elements matching prev to be cur and returns
// true if it was updated. If prev is not a prefix of this Builder changes
// nothing and returns false. If either prev or cur is nil does nothing and
// returns false.
func (pb *Builder) UpdateParent(prev, cur *Builder) bool {
if prev == cur || prev == nil || cur == nil || len(prev.Elements()) > len(pb.Elements()) {
return false
}
parent := true
for i, e := range prev.Elements() {
if pb.elements[i] != e {
parent = false
break
}
}
if !parent {
return false
}
pb.elements = append(cur.Elements(), pb.elements[len(prev.Elements()):]...)
return true
}
// ShortRef produces a truncated hash of the builder that
// acts as a unique identifier.
func (pb Builder) ShortRef() string {
if len(pb.elements) == 0 {
return ""
}
data := bytes.Buffer{}
for _, element := range pb.elements {
data.WriteString(element)
}
sum := sha256.Sum256(data.Bytes())
// Some conversions to get the right number of characters in the output. This
// outputs hex, so we need to take the target number of characters and do the
// equivalent of (shortRefCharacters * 4) / 8. This is
// <number of bits represented> / <bits per byte> which gets us how many bytes
// to give to our format command.
numBytes := shortRefCharacters / 2
return fmt.Sprintf("%x", sum[:numBytes])
}
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
func (pb Builder) Elements() Elements {
return append(Elements{}, pb.elements...)
}
// withPrefix creates a Builder prefixed with the parameter values, and
// concatenated with the current builder elements.
func (pb Builder) withPrefix(elements ...string) *Builder {
res := Builder{}.Append(elements...)
res.elements = append(res.elements, pb.elements...)
return res
}
// verifyPrefix ensures that the tenant and resourceOwner are valid
// values, and that the builder has some directory structure.
// func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
// if err := verifyPrefixValues(tenant, resourceOwner); err != nil {
// return err
// }
// if len(pb.elements) == 0 {
// return clues.New("missing path beyond prefix")
// }
// return nil
// }
// ---------------------------------------------------------------------------
// Data Layer Path Transformers
// ---------------------------------------------------------------------------
func (pb Builder) ToStreamStorePath(
tenant string,
srs []ServiceResource,
isItem bool,
) (Path, error) {
cat := DetailsCategory
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
dlrp := newDataLayerResourcePath(pb, tenant, toMetadataServices(srs), cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToServiceCategoryMetadataPath(
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
dlrp := newDataLayerResourcePath(pb, tenant, toMetadataServices(srs), cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToDataLayerPath(
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
dlrp := newDataLayerResourcePath(pb, tenant, srs, cat, isItem)
return &dlrp, nil
}
func (pb Builder) ToDataLayerExchangePathForCategory(
tenant, mailboxID string,
category CategoryType,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(ExchangeService, mailboxID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, category, isItem)
}
func (pb Builder) ToDataLayerOneDrivePath(
tenant, userID string,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(OneDriveService, userID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, FilesCategory, isItem)
}
func (pb Builder) ToDataLayerSharePointPath(
tenant, siteID string,
category CategoryType,
isItem bool,
) (Path, error) {
srs, err := NewServiceResources(SharePointService, siteID)
if err != nil {
return nil, err
}
return pb.ToDataLayerPath(tenant, srs, category, isItem)
}
// TODO: ToDataLayerGroupsPath()
// ---------------------------------------------------------------------------
// Stringers and PII Concealer Compliance
// ---------------------------------------------------------------------------
// Conceal produces a concealed representation of the builder, suitable for
// logging, storing in errors, and other output.
func (pb Builder) Conceal() string {
return pb.elements.Conceal()
}
// Format produces a concealed representation of the builder, even when
// used within a PrintF, suitable for logging, storing in errors,
// and other output.
func (pb Builder) Format(fs fmt.State, _ rune) {
fmt.Fprint(fs, pb.Conceal())
}
// String returns a string that contains all path elements joined together.
// Elements of the path that need escaping are escaped.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) String() string {
return pb.elements.String()
}
// PlainString returns an unescaped, unmodified string of the builder.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) PlainString() string {
return pb.elements.PlainString()
}

View File

@ -0,0 +1,377 @@
package path
import (
"fmt"
"strings"
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type BuilderUnitSuite struct {
tester.Suite
}
func TestBuilderUnitSuite(t *testing.T) {
suite.Run(t, &BuilderUnitSuite{Suite: tester.NewUnitSuite(t)})
}
// set the clues hashing to mask for the span of this suite
func (suite *BuilderUnitSuite) SetupSuite() {
clues.SetHasher(clues.HashCfg{HashAlg: clues.Flatmask})
}
// revert clues hashing to plaintext for all other tests
func (suite *BuilderUnitSuite) TeardownSuite() {
clues.SetHasher(clues.NoHash())
}
func (suite *BuilderUnitSuite) TestAppend() {
table := append(append([]testData{}, genericCases...), basicUnescapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := Builder{}.Append(test.input...)
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *BuilderUnitSuite) TestAppendItem() {
t := suite.T()
srs, err := NewServiceResources(ExchangeService, "ro")
require.NoError(t, err, clues.ToCore(err))
p, err := Build("t", srs, EmailCategory, false, "foo", "bar")
require.NoError(t, err, clues.ToCore(err))
pb := p.ToBuilder()
assert.Equal(t, pb.String(), p.String())
pb = pb.Append("qux")
p, err = p.AppendItem("qux")
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, pb.String(), p.String())
_, err = p.AppendItem("fnords")
require.Error(t, err, clues.ToCore(err))
}
func (suite *BuilderUnitSuite) TestUnescapeAndAppend() {
table := append(append([]testData{}, genericCases...), basicEscapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := Builder{}.UnescapeAndAppend(test.input...)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *BuilderUnitSuite) TestEscapedFailure() {
target := "i_s"
for c := range charactersToEscape {
suite.Run(fmt.Sprintf("Unescaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(suite.T(), err, "path with unescaped %s did not error", string(c))
})
}
}
func (suite *BuilderUnitSuite) TestBadEscapeSequenceErrors() {
target := `i\_s/a`
notEscapes := []rune{'a', 'b', '#', '%'}
for _, c := range notEscapes {
suite.Run(fmt.Sprintf("Escaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(
suite.T(),
err,
"path with bad escape sequence %c%c did not error",
escapeCharacter,
c)
})
}
}
func (suite *BuilderUnitSuite) TestTrailingEscapeChar() {
base := []string{"this", "is", "a", "path"}
for i := 0; i < len(base); i++ {
suite.Run(fmt.Sprintf("Element%v", i), func() {
path := make([]string, len(base))
copy(path, base)
path[i] = path[i] + string(escapeCharacter)
_, err := Builder{}.UnescapeAndAppend(path...)
assert.Error(
suite.T(),
err,
"path with trailing escape character did not error")
})
}
}
func (suite *BuilderUnitSuite) TestElements() {
table := []struct {
name string
input []string
output []string
pathFunc func(elements []string) (*Builder, error)
}{
{
name: "SimpleEscapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
{
name: "SimpleUnescapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.Append(elements...), nil
},
},
{
name: "EscapedPath",
input: []string{"this", `is\/`, "a", "path"},
output: []string{"this", "is/", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := test.pathFunc(test.input)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, Elements(test.output), p.Elements())
})
}
}
func (suite *BuilderUnitSuite) TestPopFront() {
table := []struct {
name string
base *Builder
expectedString string
}{
{
name: "Empty",
base: &Builder{},
expectedString: "",
},
{
name: "OneElement",
base: Builder{}.Append("something"),
expectedString: "",
},
{
name: "TwoElements",
base: Builder{}.Append("something", "else"),
expectedString: "else",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expectedString, test.base.PopFront().String())
})
}
}
func (suite *BuilderUnitSuite) TestShortRef() {
table := []struct {
name string
inputElements []string
expectedLen int
}{
{
name: "PopulatedPath",
inputElements: []string{"this", "is", "a", "path"},
expectedLen: shortRefCharacters,
},
{
name: "EmptyPath",
inputElements: nil,
expectedLen: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
pb := Builder{}.Append(test.inputElements...)
ref := pb.ShortRef()
assert.Len(suite.T(), ref, test.expectedLen)
})
}
}
func (suite *BuilderUnitSuite) TestShortRefIsStable() {
t := suite.T()
pb := Builder{}.Append("this", "is", "a", "path")
prevRef := pb.ShortRef()
assert.Len(t, prevRef, shortRefCharacters)
for i := 0; i < 5; i++ {
ref := pb.ShortRef()
assert.Len(t, ref, shortRefCharacters)
assert.Equal(t, prevRef, ref, "ShortRef changed between calls")
prevRef = ref
}
}
func (suite *BuilderUnitSuite) TestShortRefIsUnique() {
pb1 := Builder{}.Append("this", "is", "a", "path")
pb2 := pb1.Append("also")
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
// TestShortRefUniqueWithEscaping tests that two paths that output the same
// unescaped string but different escaped strings have different shortrefs. This
// situation can occur when one path has embedded path separators while the
// other does not but contains the same characters.
func (suite *BuilderUnitSuite) TestShortRefUniqueWithEscaping() {
pb1 := Builder{}.Append(`this`, `is`, `a`, `path`)
pb2 := Builder{}.Append(`this`, `is/a`, `path`)
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
func (suite *BuilderUnitSuite) TestFolder() {
table := []struct {
name string
p func(t *testing.T) Path
escape bool
expectFolder string
expectSplit []string
}{
{
name: "clean path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "clean path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a//b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a\\//b/c",
expectSplit: []string{"a\\/", "b", "c"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := test.p(t)
result := p.Folder(test.escape)
assert.Equal(t, test.expectFolder, result)
assert.Equal(t, test.expectSplit, Split(result))
})
}
}
func (suite *BuilderUnitSuite) TestPIIHandling() {
t := suite.T()
srs, err := NewServiceResources(ExchangeService, "ro")
require.NoError(t, err, clues.ToCore(err))
p, err := Build("t", srs, EventsCategory, true, "dir", "item")
require.NoError(t, err)
table := []struct {
name string
p Path
expect string
expectPlain string
}{
{
name: "standard path",
p: p,
expect: "***/exchange/***/events/***/***",
expectPlain: "t/exchange/ro/events/dir/item",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expect, test.p.Conceal(), "conceal")
assert.Equal(t, test.expectPlain, test.p.String(), "string")
assert.Equal(t, test.expect, fmt.Sprintf("%s", test.p), "fmt %%s")
assert.Equal(t, test.expect, fmt.Sprintf("%+v", test.p), "fmt %%+v")
assert.Equal(t, test.expectPlain, test.p.PlainString(), "plain")
})
}
}

View File

@ -0,0 +1,90 @@
package path
import (
"fmt"
"strings"
"github.com/alcionai/clues"
)
var ErrorUnknownCategory = clues.New("unknown category string")
// CategoryType denotes what category of data the path corresponds to. The order
// of the enums below can be changed, but the string representation of each enum
// must remain the same or migration code needs to be added to handle changes to
// the string format.
type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
ListsCategory // lists
LibrariesCategory // libraries
PagesCategory // pages
DetailsCategory // details
)
func ToCategoryType(category string) CategoryType {
cat := strings.ToLower(category)
switch cat {
case strings.ToLower(EmailCategory.String()):
return EmailCategory
case strings.ToLower(ContactsCategory.String()):
return ContactsCategory
case strings.ToLower(EventsCategory.String()):
return EventsCategory
case strings.ToLower(FilesCategory.String()):
return FilesCategory
case strings.ToLower(LibrariesCategory.String()):
return LibrariesCategory
case strings.ToLower(ListsCategory.String()):
return ListsCategory
case strings.ToLower(PagesCategory.String()):
return PagesCategory
case strings.ToLower(DetailsCategory.String()):
return DetailsCategory
default:
return UnknownCategory
}
}
// ---------------------------------------------------------------------------
// Service-Category pairings
// ---------------------------------------------------------------------------
// serviceCategories is a mapping of all valid service/category pairs for
// non-metadata paths.
var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
ExchangeService: {
EmailCategory: {},
ContactsCategory: {},
EventsCategory: {},
},
OneDriveService: {
FilesCategory: {},
},
SharePointService: {
LibrariesCategory: {},
ListsCategory: {},
PagesCategory: {},
},
}
func ValidateServiceAndCategory(service ServiceType, category CategoryType) error {
cats, ok := serviceCategories[service]
if !ok {
return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))
}
if _, ok := cats[category]; !ok {
return clues.New("unknown service/category combination").
With("service", fmt.Sprintf("%q", service), "category", fmt.Sprintf("%q", category))
}
return nil
}

View File

@ -2,6 +2,7 @@ package path
import "github.com/alcionai/clues" import "github.com/alcionai/clues"
// TODO: Move this into m365/collection/drive
// drivePath is used to represent path components // drivePath is used to represent path components
// of an item within the drive i.e. // of an item within the drive i.e.
// Given `drives/b!X_8Z2zuXpkKkXZsr7gThk9oJpuj0yXVGnK5_VjRRPK-q725SX_8ZQJgFDK8PlFxA/root:/Folder1/Folder2/file` // Given `drives/b!X_8Z2zuXpkKkXZsr7gThk9oJpuj0yXVGnK5_VjRRPK-q725SX_8ZQJgFDK8PlFxA/root:/Folder1/Folder2/file`

View File

@ -59,7 +59,10 @@ func (suite *OneDrivePathSuite) Test_ToOneDrivePath() {
suite.Run(tt.name, func() { suite.Run(tt.name, func() {
t := suite.T() t := suite.T()
p, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, false, tt.pathElements...) srs, err := path.NewServiceResources(path.OneDriveService, "user")
require.NoError(t, err, clues.ToCore(err))
p, err := path.Build("tenant", srs, path.FilesCategory, false, tt.pathElements...)
require.NoError(suite.T(), err, clues.ToCore(err)) require.NoError(suite.T(), err, clues.ToCore(err))
got, err := path.ToDrivePath(p) got, err := path.ToDrivePath(p)

View File

@ -51,8 +51,6 @@
package path package path
import ( import (
"bytes"
"crypto/sha256"
"fmt" "fmt"
"strings" "strings"
@ -80,41 +78,58 @@ var (
// Resources that don't have the requested information should return an empty // Resources that don't have the requested information should return an empty
// string. // string.
type Path interface { type Path interface {
String() string // parts
Service() ServiceType
Category() CategoryType
Tenant() string Tenant() string
ResourceOwner() string // ServiceResources produces all of the services and subservices, along with
// the protected resource paired with the service, as contained in the path,
// in their order of appearance.
ServiceResources() []ServiceResource
// PrimaryService is the first service in ServiceResources()
PrimaryService() ServiceType
// PrimaryProtectedResource is the first ProtectedResource in ServiceResources()
PrimaryProtectedResource() string
Category() CategoryType
Folder(escaped bool) string Folder(escaped bool) string
Folders() Elements Folders() Elements
Item() string Item() string
// UpdateParent updates parent from old to new if the item/folder was
// parented by old path // type transformations
UpdateParent(prev, cur Path) bool
// PopFront returns a Builder object with the first element (left-side) // ToBuilder returns a Builder instance that represents the current Path.
// removed. As the resulting set of elements is no longer a valid resource ToBuilder() *Builder
// path a Builder is returned instead.
PopFront() *Builder
// Dir returns a Path object with the right-most element removed if possible.
// If removing the right-most element would discard one of the required prefix
// elements then an error is returned.
Dir() (Path, error)
// Elements returns all the elements in the path. This is a temporary function // Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text // and will likely be updated to handle encoded elements instead of clear-text
// elements in the future. // elements in the future.
Elements() Elements Elements() Elements
// Halves breaks the path into its prefix (tenant, services, resources, category)
// and suffix (all parts after the prefix). If either half is empty, that half
// returns an empty, non-nil, value.
Halves() (*Builder, Elements)
// ShortRef returns a short reference representing this path. The short
// reference is guaranteed to be unique. No guarantees are made about whether
// a short reference can be converted back into the Path that generated it.
ShortRef() string
// mutators
// Append returns a new Path object with the given element added to the end of // Append returns a new Path object with the given element added to the end of
// the old Path if possible. If the old Path is an item Path then Append // the old Path if possible. If the old Path is an item Path then Append
// returns an error. // returns an error.
Append(isItem bool, elems ...string) (Path, error) Append(isItem bool, elems ...string) (Path, error)
// AppendItem is a shorthand for Append(true, someItem) // AppendItem is a shorthand for Append(true, someItem)
AppendItem(item string) (Path, error) AppendItem(item string) (Path, error)
// ShortRef returns a short reference representing this path. The short // Dir returns a Path object with the right-most element removed if possible.
// reference is guaranteed to be unique. No guarantees are made about whether // If removing the right-most element would discard one of the required prefix
// a short reference can be converted back into the Path that generated it. // elements then an error is returned.
ShortRef() string Dir() (Path, error)
// ToBuilder returns a Builder instance that represents the current Path. // PopFront returns a Builder object with the first element (left-side)
ToBuilder() *Builder // removed. As the resulting set of elements is no longer a valid resource
// path a Builder is returned instead.
PopFront() *Builder
// UpdateParent updates parent from old to new if the item/folder was
// parented by old path
UpdateParent(prev, cur Path) bool
// Every path needs to comply with these funcs to ensure that PII // Every path needs to comply with these funcs to ensure that PII
// is appropriately hidden from logging, errors, and other outputs. // is appropriately hidden from logging, errors, and other outputs.
@ -122,12 +137,6 @@ type Path interface {
fmt.Stringer fmt.Stringer
} }
// interface compliance required for handling PII
var (
_ clues.Concealer = &Builder{}
_ fmt.Stringer = &Builder{}
)
// RestorePaths denotes the location to find an item in kopia and the path of // RestorePaths denotes the location to find an item in kopia and the path of
// the collection to place the item in for restore. // the collection to place the item in for restore.
type RestorePaths struct { type RestorePaths struct {
@ -135,396 +144,34 @@ type RestorePaths struct {
RestorePath Path RestorePath Path
} }
// Builder is a simple path representation that only tracks path elements. It
// can join, escape, and unescape elements. Higher-level packages are expected
// to wrap this struct to build resource-specific contexts (e.x. an
// ExchangeMailPath).
// Resource-specific paths allow access to more information like segments in the
// path. Builders that are turned into resource paths later on do not need to
// manually add prefixes for items that normally appear in the data layer (ex.
// tenant ID, service, user ID, etc).
type Builder struct {
// Unescaped version of elements.
elements Elements
}
// Append creates a copy of this Builder and adds the given elements them to the
// end of the new Builder. Elements are added in the order they are passed.
func (pb Builder) Append(elements ...string) *Builder {
res := &Builder{elements: make([]string, len(pb.elements))}
copy(res.elements, pb.elements)
// Unescaped elements can't fail validation.
//nolint:errcheck
res.appendElements(false, elements)
return res
}
func (pb *Builder) appendElements(escaped bool, elements []string) error {
for _, e := range elements {
if len(e) == 0 {
continue
}
tmp := e
if escaped {
tmp = TrimTrailingSlash(tmp)
// If tmp was just the path separator then it will be empty now.
if len(tmp) == 0 {
continue
}
if err := validateEscapedElement(tmp); err != nil {
return err
}
tmp = unescape(tmp)
}
pb.elements = append(pb.elements, tmp)
}
return nil
}
// UnescapeAndAppend creates a copy of this Builder and adds one or more already
// escaped path elements to the end of the new Builder. Elements are added in
// the order they are passed.
func (pb Builder) UnescapeAndAppend(elements ...string) (*Builder, error) {
res := &Builder{elements: make([]string, 0, len(pb.elements))}
copy(res.elements, pb.elements)
if err := res.appendElements(true, elements); err != nil {
return nil, err
}
return res, nil
}
// SplitUnescapeAppend takes in an escaped string representing a directory
// path, splits the string, and appends it to the current builder.
func (pb Builder) SplitUnescapeAppend(s string) (*Builder, error) {
elems := Split(TrimTrailingSlash(s))
return pb.UnescapeAndAppend(elems...)
}
func (pb Builder) PopFront() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
elements := make([]string, len(pb.elements)-1)
copy(elements, pb.elements[1:])
return &Builder{
elements: elements,
}
}
// Dir removes the last element from the builder.
func (pb Builder) Dir() *Builder {
if len(pb.elements) <= 1 {
return &Builder{}
}
return &Builder{
// Safe to use the same elements because Builders are immutable.
elements: pb.elements[:len(pb.elements)-1],
}
}
// HeadElem returns the first element in the Builder.
func (pb Builder) HeadElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[0]
}
// LastElem returns the last element in the Builder.
func (pb Builder) LastElem() string {
if len(pb.elements) == 0 {
return ""
}
return pb.elements[len(pb.elements)-1]
}
// UpdateParent updates leading elements matching prev to be cur and returns
// true if it was updated. If prev is not a prefix of this Builder changes
// nothing and returns false. If either prev or cur is nil does nothing and
// returns false.
func (pb *Builder) UpdateParent(prev, cur *Builder) bool {
if prev == cur || prev == nil || cur == nil || len(prev.Elements()) > len(pb.Elements()) {
return false
}
parent := true
for i, e := range prev.Elements() {
if pb.elements[i] != e {
parent = false
break
}
}
if !parent {
return false
}
pb.elements = append(cur.Elements(), pb.elements[len(prev.Elements()):]...)
return true
}
// ShortRef produces a truncated hash of the builder that
// acts as a unique identifier.
func (pb Builder) ShortRef() string {
if len(pb.elements) == 0 {
return ""
}
data := bytes.Buffer{}
for _, element := range pb.elements {
data.WriteString(element)
}
sum := sha256.Sum256(data.Bytes())
// Some conversions to get the right number of characters in the output. This
// outputs hex, so we need to take the target number of characters and do the
// equivalent of (shortRefCharacters * 4) / 8. This is
// <number of bits represented> / <bits per byte> which gets us how many bytes
// to give to our format command.
numBytes := shortRefCharacters / 2
return fmt.Sprintf("%x", sum[:numBytes])
}
// Elements returns all the elements in the path. This is a temporary function
// and will likely be updated to handle encoded elements instead of clear-text
// elements in the future.
func (pb Builder) Elements() Elements {
return append(Elements{}, pb.elements...)
}
func ServicePrefix(
tenant, resourceOwner string,
s ServiceType,
c CategoryType,
) (Path, error) {
pb := Builder{}
if err := ValidateServiceAndCategory(s, c); err != nil {
return nil, err
}
if err := verifyInputValues(tenant, resourceOwner); err != nil {
return nil, err
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(tenant, s.String(), resourceOwner, c.String()),
service: s,
category: c,
hasItem: false,
}, nil
}
// withPrefix creates a Builder prefixed with the parameter values, and
// concatenated with the current builder elements.
func (pb Builder) withPrefix(elements ...string) *Builder {
res := Builder{}.Append(elements...)
res.elements = append(res.elements, pb.elements...)
return res
}
// ---------------------------------------------------------------------------
// Data Layer Path Transformers
// ---------------------------------------------------------------------------
func (pb Builder) ToStreamStorePath(
tenant, purpose string,
service ServiceType,
isItem bool,
) (Path, error) {
if err := verifyInputValues(tenant, purpose); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
metadataService := UnknownService
switch service {
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
metadataService.String(),
purpose,
DetailsCategory.String()),
service: metadataService,
category: DetailsCategory,
hasItem: isItem,
}, nil
}
func (pb Builder) ToServiceCategoryMetadataPath(
tenant, user string,
service ServiceType,
category CategoryType,
isItem bool,
) (Path, error) {
if err := ValidateServiceAndCategory(service, category); err != nil {
return nil, err
}
if err := verifyInputValues(tenant, user); err != nil {
return nil, err
}
if isItem && len(pb.elements) == 0 {
return nil, clues.New("missing path beyond prefix")
}
metadataService := UnknownService
switch service {
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
metadataService.String(),
user,
category.String(),
),
service: metadataService,
category: category,
hasItem: isItem,
}, nil
}
func (pb Builder) ToDataLayerPath(
tenant, user string,
service ServiceType,
category CategoryType,
isItem bool,
) (Path, error) {
if err := ValidateServiceAndCategory(service, category); err != nil {
return nil, err
}
if err := pb.verifyPrefix(tenant, user); err != nil {
return nil, err
}
return &dataLayerResourcePath{
Builder: *pb.withPrefix(
tenant,
service.String(),
user,
category.String()),
service: service,
category: category,
hasItem: isItem,
}, nil
}
func (pb Builder) ToDataLayerExchangePathForCategory(
tenant, user string,
category CategoryType,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, user, ExchangeService, category, isItem)
}
func (pb Builder) ToDataLayerOneDrivePath(
tenant, user string,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, user, OneDriveService, FilesCategory, isItem)
}
func (pb Builder) ToDataLayerSharePointPath(
tenant, site string,
category CategoryType,
isItem bool,
) (Path, error) {
return pb.ToDataLayerPath(tenant, site, SharePointService, category, isItem)
}
// ---------------------------------------------------------------------------
// Stringers and PII Concealer Compliance
// ---------------------------------------------------------------------------
// Conceal produces a concealed representation of the builder, suitable for
// logging, storing in errors, and other output.
func (pb Builder) Conceal() string {
return pb.elements.Conceal()
}
// Format produces a concealed representation of the builder, even when
// used within a PrintF, suitable for logging, storing in errors,
// and other output.
func (pb Builder) Format(fs fmt.State, _ rune) {
fmt.Fprint(fs, pb.Conceal())
}
// String returns a string that contains all path elements joined together.
// Elements of the path that need escaping are escaped.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) String() string {
return pb.elements.String()
}
// PlainString returns an unescaped, unmodified string of the builder.
// The result is not concealed, and is not suitable for logging or structured
// errors.
func (pb Builder) PlainString() string {
return pb.elements.PlainString()
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Exported Helpers // Exported Helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func Build( func Build(
tenant, resourceOwner string, tenant string,
service ServiceType, srs []ServiceResource,
category CategoryType, category CategoryType,
hasItem bool, hasItem bool,
elements ...string, elements ...string,
) (Path, error) { ) (Path, error) {
b := Builder{}.Append(elements...) return Builder{}.
Append(elements...).
ToDataLayerPath(tenant, srs, category, hasItem)
}
return b.ToDataLayerPath( func BuildPrefix(
tenant, resourceOwner, tenant string,
service, category, srs []ServiceResource,
hasItem) cat CategoryType,
) (Path, error) {
if err := verifyPrefixValues(tenant, srs, cat); err != nil {
return nil, err
}
dlrp := newDataLayerResourcePath(Builder{}, tenant, srs, cat, false)
return &dlrp, nil
} }
// FromDataLayerPath parses the escaped path p, validates the elements in p // FromDataLayerPath parses the escaped path p, validates the elements in p
@ -544,24 +191,37 @@ func FromDataLayerPath(p string, isItem bool) (Path, error) {
return nil, clues.Stack(errParsingPath, err).With("path_string", p) return nil, clues.Stack(errParsingPath, err).With("path_string", p)
} }
// initial check for minimum required elements:
// tenant, service, resource, category, container/item
if len(pb.elements) < 5 { if len(pb.elements) < 5 {
return nil, clues.New("path has too few segments").With("path_string", p) return nil, clues.New("path has too few segments").With("path_string", p)
} }
service, category, err := validateServiceAndCategoryStrings( srs, catIdx, err := elementsToServiceResources(pb.elements[1:])
pb.elements[1],
pb.elements[3],
)
if err != nil { if err != nil {
return nil, clues.Stack(err)
}
// follow-up check: if more than one service exists, revisit the len check.
if len(srs) > 1 && len(pb.elements) < 3+(2*len(srs)) {
return nil, clues.New("path has too few segments").With("path_string", p)
}
// +1 to account for slicing the tenant when calling the transformer func.
category := ToCategoryType(pb.elements[catIdx+1])
if err := verifyPrefixValues(pb.elements[0], srs, category); err != nil {
return nil, clues.Stack(errParsingPath, err).With("path_string", p) return nil, clues.Stack(errParsingPath, err).With("path_string", p)
} }
return &dataLayerResourcePath{ dlrp := dataLayerResourcePath{
Builder: *pb, Builder: *pb,
service: service, serviceResources: srs,
category: category, category: category,
hasItem: isItem, hasItem: isItem,
}, nil }
return &dlrp, nil
} }
// TrimTrailingSlash takes an escaped path element and returns an escaped path // TrimTrailingSlash takes an escaped path element and returns an escaped path
@ -648,16 +308,21 @@ func Split(segment string) []string {
// Unexported Helpers // Unexported Helpers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func verifyInputValues(tenant, resourceOwner string) error { func verifyPrefixValues(
tenant string,
srs []ServiceResource,
cat CategoryType,
) error {
if len(tenant) == 0 { if len(tenant) == 0 {
return clues.Stack(errMissingSegment, clues.New("tenant")) return clues.Stack(errMissingSegment, clues.New("tenant"))
} }
if len(resourceOwner) == 0 { if err := validateServiceResources(srs); err != nil {
return clues.Stack(errMissingSegment, clues.New("resourceOwner")) return err
} }
return nil // only the final service is checked for its category validity
return ValidateServiceAndCategory(srs[len(srs)-1].Service, cat)
} }
// escapeElement takes a single path element and escapes all characters that // escapeElement takes a single path element and escapes all characters that
@ -762,17 +427,3 @@ func join(elements []string) string {
// '\' according to the escaping rules. // '\' according to the escaping rules.
return strings.Join(elements, string(PathSeparator)) return strings.Join(elements, string(PathSeparator))
} }
// verifyPrefix ensures that the tenant and resourceOwner are valid
// values, and that the builder has some directory structure.
func (pb Builder) verifyPrefix(tenant, resourceOwner string) error {
if err := verifyInputValues(tenant, resourceOwner); err != nil {
return err
}
if len(pb.elements) == 0 {
return clues.New("missing path beyond prefix")
}
return nil
}

View File

@ -2,7 +2,6 @@ package path
import ( import (
"fmt" "fmt"
"strings"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -233,239 +232,7 @@ func (suite *PathUnitSuite) TeardownSuite() {
clues.SetHasher(clues.NoHash()) clues.SetHasher(clues.NoHash())
} }
func (suite *PathUnitSuite) TestAppend() { func (suite *PathUnitSuite) TestFromDataLayerPathErrors() {
table := append(append([]testData{}, genericCases...), basicUnescapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := Builder{}.Append(test.input...)
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *PathUnitSuite) TestAppendItem() {
t := suite.T()
p, err := Build("t", "ro", ExchangeService, EmailCategory, false, "foo", "bar")
require.NoError(t, err, clues.ToCore(err))
pb := p.ToBuilder()
assert.Equal(t, pb.String(), p.String())
pb = pb.Append("qux")
p, err = p.AppendItem("qux")
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, pb.String(), p.String())
_, err = p.AppendItem("fnords")
require.Error(t, err, clues.ToCore(err))
}
func (suite *PathUnitSuite) TestUnescapeAndAppend() {
table := append(append([]testData{}, genericCases...), basicEscapedInputs...)
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := Builder{}.UnescapeAndAppend(test.input...)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, test.expectedString, p.String())
})
}
}
func (suite *PathUnitSuite) TestEscapedFailure() {
target := "i_s"
for c := range charactersToEscape {
suite.Run(fmt.Sprintf("Unescaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(suite.T(), err, "path with unescaped %s did not error", string(c))
})
}
}
func (suite *PathUnitSuite) TestBadEscapeSequenceErrors() {
target := `i\_s/a`
notEscapes := []rune{'a', 'b', '#', '%'}
for _, c := range notEscapes {
suite.Run(fmt.Sprintf("Escaped-%c", c), func() {
tmp := strings.ReplaceAll(target, "_", string(c))
_, err := Builder{}.UnescapeAndAppend("this", tmp, "path")
assert.Errorf(
suite.T(),
err,
"path with bad escape sequence %c%c did not error",
escapeCharacter,
c)
})
}
}
func (suite *PathUnitSuite) TestTrailingEscapeChar() {
base := []string{"this", "is", "a", "path"}
for i := 0; i < len(base); i++ {
suite.Run(fmt.Sprintf("Element%v", i), func() {
path := make([]string, len(base))
copy(path, base)
path[i] = path[i] + string(escapeCharacter)
_, err := Builder{}.UnescapeAndAppend(path...)
assert.Error(
suite.T(),
err,
"path with trailing escape character did not error")
})
}
}
func (suite *PathUnitSuite) TestElements() {
table := []struct {
name string
input []string
output []string
pathFunc func(elements []string) (*Builder, error)
}{
{
name: "SimpleEscapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
{
name: "SimpleUnescapedPath",
input: []string{"this", "is", "a", "path"},
output: []string{"this", "is", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.Append(elements...), nil
},
},
{
name: "EscapedPath",
input: []string{"this", `is\/`, "a", "path"},
output: []string{"this", "is/", "a", "path"},
pathFunc: func(elements []string) (*Builder, error) {
return Builder{}.UnescapeAndAppend(elements...)
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p, err := test.pathFunc(test.input)
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, Elements(test.output), p.Elements())
})
}
}
func (suite *PathUnitSuite) TestPopFront() {
table := []struct {
name string
base *Builder
expectedString string
}{
{
name: "Empty",
base: &Builder{},
expectedString: "",
},
{
name: "OneElement",
base: Builder{}.Append("something"),
expectedString: "",
},
{
name: "TwoElements",
base: Builder{}.Append("something", "else"),
expectedString: "else",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expectedString, test.base.PopFront().String())
})
}
}
func (suite *PathUnitSuite) TestShortRef() {
table := []struct {
name string
inputElements []string
expectedLen int
}{
{
name: "PopulatedPath",
inputElements: []string{"this", "is", "a", "path"},
expectedLen: shortRefCharacters,
},
{
name: "EmptyPath",
inputElements: nil,
expectedLen: 0,
},
}
for _, test := range table {
suite.Run(test.name, func() {
pb := Builder{}.Append(test.inputElements...)
ref := pb.ShortRef()
assert.Len(suite.T(), ref, test.expectedLen)
})
}
}
func (suite *PathUnitSuite) TestShortRefIsStable() {
t := suite.T()
pb := Builder{}.Append("this", "is", "a", "path")
prevRef := pb.ShortRef()
assert.Len(t, prevRef, shortRefCharacters)
for i := 0; i < 5; i++ {
ref := pb.ShortRef()
assert.Len(t, ref, shortRefCharacters)
assert.Equal(t, prevRef, ref, "ShortRef changed between calls")
prevRef = ref
}
}
func (suite *PathUnitSuite) TestShortRefIsUnique() {
pb1 := Builder{}.Append("this", "is", "a", "path")
pb2 := pb1.Append("also")
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
// TestShortRefUniqueWithEscaping tests that two paths that output the same
// unescaped string but different escaped strings have different shortrefs. This
// situation can occur when one path has embedded path separators while the
// other does not but contains the same characters.
func (suite *PathUnitSuite) TestShortRefUniqueWithEscaping() {
pb1 := Builder{}.Append(`this`, `is`, `a`, `path`)
pb2 := Builder{}.Append(`this`, `is/a`, `path`)
require.NotEqual(suite.T(), pb1, pb2)
assert.NotEqual(suite.T(), pb1.ShortRef(), pb2.ShortRef())
}
func (suite *PathUnitSuite) TestFromStringErrors() {
table := []struct { table := []struct {
name string name string
escapedPath string escapedPath string
@ -521,82 +288,7 @@ func (suite *PathUnitSuite) TestFromStringErrors() {
} }
} }
func (suite *PathUnitSuite) TestFolder() { func (suite *PathUnitSuite) TestFromDataLayerPath() {
table := []struct {
name string
p func(t *testing.T) Path
escape bool
expectFolder string
expectSplit []string
}{
{
name: "clean path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "clean path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a/b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
expectFolder: "a//b/c",
expectSplit: []string{"a", "b", "c"},
},
{
name: "escapable path escaped",
p: func(t *testing.T) Path {
p, err := Builder{}.
Append("a/", "b", "c").
ToDataLayerExchangePathForCategory("t", "u", EmailCategory, false)
require.NoError(t, err, clues.ToCore(err))
return p
},
escape: true,
expectFolder: "a\\//b/c",
expectSplit: []string{"a\\/", "b", "c"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
p := test.p(t)
result := p.Folder(test.escape)
assert.Equal(t, test.expectFolder, result)
assert.Equal(t, test.expectSplit, Split(result))
})
}
}
func (suite *PathUnitSuite) TestFromString() {
const ( const (
testTenant = "tenant" testTenant = "tenant"
testUser = "user" testUser = "user"
@ -642,14 +334,12 @@ func (suite *PathUnitSuite) TestFromString() {
testUser, testUser,
testElement1, testElement1,
testElement2, testElement2,
testElement3, testElement3),
),
expectedFolder: fmt.Sprintf( expectedFolder: fmt.Sprintf(
"%s/%s/%s", "%s/%s/%s",
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
testElement3, testElement3),
),
expectedSplit: []string{ expectedSplit: []string{
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
@ -659,8 +349,7 @@ func (suite *PathUnitSuite) TestFromString() {
expectedItemFolder: fmt.Sprintf( expectedItemFolder: fmt.Sprintf(
"%s/%s", "%s/%s",
testElementTrimmed, testElementTrimmed,
testElement2, testElement2),
),
expectedItemSplit: []string{ expectedItemSplit: []string{
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
@ -674,14 +363,12 @@ func (suite *PathUnitSuite) TestFromString() {
testUser, testUser,
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
testElement3, testElement3),
),
expectedFolder: fmt.Sprintf( expectedFolder: fmt.Sprintf(
"%s/%s/%s", "%s/%s/%s",
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
testElement3, testElement3),
),
expectedSplit: []string{ expectedSplit: []string{
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
@ -691,8 +378,7 @@ func (suite *PathUnitSuite) TestFromString() {
expectedItemFolder: fmt.Sprintf( expectedItemFolder: fmt.Sprintf(
"%s/%s", "%s/%s",
testElementTrimmed, testElementTrimmed,
testElement2, testElement2),
),
expectedItemSplit: []string{ expectedItemSplit: []string{
testElementTrimmed, testElementTrimmed,
testElement2, testElement2,
@ -706,16 +392,19 @@ func (suite *PathUnitSuite) TestFromString() {
suite.Run(fmt.Sprintf("%s-%s-%s", service, cat, item.name), func() { suite.Run(fmt.Sprintf("%s-%s-%s", service, cat, item.name), func() {
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() var (
testPath := fmt.Sprintf(test.unescapedPath, service, cat) t = suite.T()
testPath = fmt.Sprintf(test.unescapedPath, service, cat)
sr = ServiceResource{service, testUser}
)
p, err := FromDataLayerPath(testPath, item.isItem) p, err := FromDataLayerPath(testPath, item.isItem)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, service, p.Service(), "service") assert.Len(t, p.ServiceResources(), 1, "service resources")
assert.Equal(t, sr, p.ServiceResources()[0], "service resource")
assert.Equal(t, cat, p.Category(), "category") assert.Equal(t, cat, p.Category(), "category")
assert.Equal(t, testTenant, p.Tenant(), "tenant") assert.Equal(t, testTenant, p.Tenant(), "tenant")
assert.Equal(t, testUser, p.ResourceOwner(), "resource owner")
fld := p.Folder(false) fld := p.Folder(false)
escfld := p.Folder(true) escfld := p.Folder(true)
@ -740,77 +429,78 @@ func (suite *PathUnitSuite) TestFromString() {
} }
} }
func (suite *PathUnitSuite) TestPath_piiHandling() { func (suite *PathUnitSuite) TestBuildPrefix() {
p, err := Build("t", "ro", ExchangeService, EventsCategory, true, "dir", "item")
require.NoError(suite.T(), err)
table := []struct {
name string
p Path
expect string
expectPlain string
}{
{
name: "standard path",
p: p,
expect: "***/exchange/***/events/***/***",
expectPlain: "t/exchange/ro/events/dir/item",
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
assert.Equal(t, test.expect, test.p.Conceal(), "conceal")
assert.Equal(t, test.expectPlain, test.p.String(), "string")
assert.Equal(t, test.expect, fmt.Sprintf("%s", test.p), "fmt %%s")
assert.Equal(t, test.expect, fmt.Sprintf("%+v", test.p), "fmt %%+v")
assert.Equal(t, test.expectPlain, test.p.PlainString(), "plain")
})
}
}
func (suite *PathUnitSuite) TestToServicePrefix() {
table := []struct { table := []struct {
name string name string
service ServiceType
category CategoryType
tenant string tenant string
owner string srs []ServiceResource
category CategoryType
expect string expect string
expectErr require.ErrorAssertionFunc expectErr require.ErrorAssertionFunc
}{ }{
{ {
name: "ok", name: "ok",
service: ExchangeService,
category: ContactsCategory,
tenant: "t", tenant: "t",
owner: "ro", srs: []ServiceResource{{ExchangeService, "roo"}},
expect: join([]string{"t", ExchangeService.String(), "ro", ContactsCategory.String()}), category: ContactsCategory,
expect: join([]string{"t", ExchangeService.String(), "roo", ContactsCategory.String()}),
expectErr: require.NoError,
},
{
name: "ok with subservice",
tenant: "t",
srs: []ServiceResource{
{GroupsService, "roo"},
{SharePointService, "oor"},
},
category: LibrariesCategory,
expect: join([]string{
"t",
GroupsService.String(), "roo",
SharePointService.String(), "oor",
LibrariesCategory.String(),
}),
expectErr: require.NoError, expectErr: require.NoError,
}, },
{ {
name: "bad category", name: "bad category",
service: ExchangeService, srs: []ServiceResource{{ExchangeService, "roo"}},
category: FilesCategory, category: FilesCategory,
tenant: "t", tenant: "t",
owner: "ro",
expectErr: require.Error, expectErr: require.Error,
}, },
{ {
name: "bad tenant", name: "bad tenant",
service: ExchangeService,
category: ContactsCategory,
tenant: "", tenant: "",
owner: "ro", srs: []ServiceResource{{ExchangeService, "roo"}},
category: ContactsCategory,
expectErr: require.Error, expectErr: require.Error,
}, },
{ {
name: "bad owner", name: "bad resource",
service: ExchangeService,
category: ContactsCategory,
tenant: "t", tenant: "t",
owner: "", srs: []ServiceResource{{ExchangeService, ""}},
category: ContactsCategory,
expectErr: require.Error,
},
{
name: "bad subservice",
tenant: "t",
srs: []ServiceResource{
{ExchangeService, "roo"},
{OneDriveService, "oor"},
},
category: FilesCategory,
expectErr: require.Error,
},
{
name: "bad subservice resource",
tenant: "t",
srs: []ServiceResource{
{GroupsService, "roo"},
{SharePointService, ""},
},
category: LibrariesCategory,
expectErr: require.Error, expectErr: require.Error,
}, },
} }
@ -818,7 +508,7 @@ func (suite *PathUnitSuite) TestToServicePrefix() {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
r, err := ServicePrefix(test.tenant, test.owner, test.service, test.category) r, err := BuildPrefix(test.tenant, test.srs, test.category)
test.expectErr(t, err, clues.ToCore(err)) test.expectErr(t, err, clues.ToCore(err))
if r == nil { if r == nil {

View File

@ -1,154 +1,9 @@
package path package path
import ( import (
"fmt"
"strings"
"github.com/alcionai/clues" "github.com/alcionai/clues"
) )
var ErrorUnknownService = clues.New("unknown service string")
// ServiceType denotes what service the path corresponds to. Metadata services
// are also included though they are only used for paths that house metadata for
// Corso backups.
//
// Metadata services are not considered valid service types for resource paths
// though they can be used for metadata paths.
//
// The order of the enums below can be changed, but the string representation of
// each enum must remain the same or migration code needs to be added to handle
// changes to the string format.
type ServiceType int
//go:generate stringer -type=ServiceType -linecomment
const (
UnknownService ServiceType = iota
ExchangeService // exchange
OneDriveService // onedrive
SharePointService // sharepoint
ExchangeMetadataService // exchangeMetadata
OneDriveMetadataService // onedriveMetadata
SharePointMetadataService // sharepointMetadata
)
func toServiceType(service string) ServiceType {
s := strings.ToLower(service)
switch s {
case strings.ToLower(ExchangeService.String()):
return ExchangeService
case strings.ToLower(OneDriveService.String()):
return OneDriveService
case strings.ToLower(SharePointService.String()):
return SharePointService
case strings.ToLower(ExchangeMetadataService.String()):
return ExchangeMetadataService
case strings.ToLower(OneDriveMetadataService.String()):
return OneDriveMetadataService
case strings.ToLower(SharePointMetadataService.String()):
return SharePointMetadataService
default:
return UnknownService
}
}
var ErrorUnknownCategory = clues.New("unknown category string")
// CategoryType denotes what category of data the path corresponds to. The order
// of the enums below can be changed, but the string representation of each enum
// must remain the same or migration code needs to be added to handle changes to
// the string format.
type CategoryType int
//go:generate stringer -type=CategoryType -linecomment
const (
UnknownCategory CategoryType = iota
EmailCategory // email
ContactsCategory // contacts
EventsCategory // events
FilesCategory // files
ListsCategory // lists
LibrariesCategory // libraries
PagesCategory // pages
DetailsCategory // details
)
func ToCategoryType(category string) CategoryType {
cat := strings.ToLower(category)
switch cat {
case strings.ToLower(EmailCategory.String()):
return EmailCategory
case strings.ToLower(ContactsCategory.String()):
return ContactsCategory
case strings.ToLower(EventsCategory.String()):
return EventsCategory
case strings.ToLower(FilesCategory.String()):
return FilesCategory
case strings.ToLower(LibrariesCategory.String()):
return LibrariesCategory
case strings.ToLower(ListsCategory.String()):
return ListsCategory
case strings.ToLower(PagesCategory.String()):
return PagesCategory
case strings.ToLower(DetailsCategory.String()):
return DetailsCategory
default:
return UnknownCategory
}
}
// serviceCategories is a mapping of all valid service/category pairs for
// non-metadata paths.
var serviceCategories = map[ServiceType]map[CategoryType]struct{}{
ExchangeService: {
EmailCategory: {},
ContactsCategory: {},
EventsCategory: {},
},
OneDriveService: {
FilesCategory: {},
},
SharePointService: {
LibrariesCategory: {},
ListsCategory: {},
PagesCategory: {},
},
}
func validateServiceAndCategoryStrings(s, c string) (ServiceType, CategoryType, error) {
service := toServiceType(s)
if service == UnknownService {
return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("service", fmt.Sprintf("%q", s))
}
category := ToCategoryType(c)
if category == UnknownCategory {
return UnknownService, UnknownCategory, clues.Stack(ErrorUnknownService).With("category", fmt.Sprintf("%q", c))
}
if err := ValidateServiceAndCategory(service, category); err != nil {
return UnknownService, UnknownCategory, err
}
return service, category, nil
}
func ValidateServiceAndCategory(service ServiceType, category CategoryType) error {
cats, ok := serviceCategories[service]
if !ok {
return clues.New("unsupported service").With("service", fmt.Sprintf("%q", service))
}
if _, ok := cats[category]; !ok {
return clues.New("unknown service/category combination").
With("service", fmt.Sprintf("%q", service), "category", fmt.Sprintf("%q", category))
}
return nil
}
// dataLayerResourcePath allows callers to extract information from a // dataLayerResourcePath allows callers to extract information from a
// resource-specific path. This struct is unexported so that callers are // resource-specific path. This struct is unexported so that callers are
// forced to use the pre-defined constructors, making it impossible to create a // forced to use the pre-defined constructors, making it impossible to create a
@ -163,9 +18,28 @@ func ValidateServiceAndCategory(service ServiceType, category CategoryType) erro
// element after the prefix. // element after the prefix.
type dataLayerResourcePath struct { type dataLayerResourcePath struct {
Builder Builder
category CategoryType category CategoryType
service ServiceType serviceResources []ServiceResource
hasItem bool hasItem bool
}
// performs no validation, assumes the caller has validated the inputs.
func newDataLayerResourcePath(
pb Builder,
tenant string,
srs []ServiceResource,
cat CategoryType,
isItem bool,
) dataLayerResourcePath {
pfx := append([]string{tenant}, ServiceResourcesToElements(srs)...)
pfx = append(pfx, cat.String())
return dataLayerResourcePath{
Builder: *pb.withPrefix(pfx...),
serviceResources: srs,
category: cat,
hasItem: isItem,
}
} }
// Tenant returns the tenant ID embedded in the dataLayerResourcePath. // Tenant returns the tenant ID embedded in the dataLayerResourcePath.
@ -173,9 +47,28 @@ func (rp dataLayerResourcePath) Tenant() string {
return rp.Builder.elements[0] return rp.Builder.elements[0]
} }
// Service returns the ServiceType embedded in the dataLayerResourcePath. func (rp dataLayerResourcePath) ServiceResources() []ServiceResource {
func (rp dataLayerResourcePath) Service() ServiceType { return rp.serviceResources
return rp.service }
func (rp dataLayerResourcePath) PrimaryService() ServiceType {
srs := rp.serviceResources
if len(srs) == 0 {
return UnknownService
}
return srs[0].Service
}
func (rp dataLayerResourcePath) PrimaryProtectedResource() string {
srs := rp.serviceResources
if len(srs) == 0 {
return ""
}
return srs[0].ProtectedResource
} }
// Category returns the CategoryType embedded in the dataLayerResourcePath. // Category returns the CategoryType embedded in the dataLayerResourcePath.
@ -199,10 +92,16 @@ func (rp dataLayerResourcePath) lastFolderIdx() int {
return endIdx return endIdx
} }
func (rp dataLayerResourcePath) prefixLen() int {
return 2 + 2*len(rp.serviceResources)
}
// Folder returns the folder segment embedded in the dataLayerResourcePath. // Folder returns the folder segment embedded in the dataLayerResourcePath.
func (rp dataLayerResourcePath) Folder(escape bool) string { func (rp dataLayerResourcePath) Folder(escape bool) string {
endIdx := rp.lastFolderIdx() endIdx := rp.lastFolderIdx()
if endIdx == 4 { pfxLen := rp.prefixLen()
if endIdx == pfxLen {
return "" return ""
} }
@ -220,11 +119,14 @@ func (rp dataLayerResourcePath) Folder(escape bool) string {
// dataLayerResourcePath. // dataLayerResourcePath.
func (rp dataLayerResourcePath) Folders() Elements { func (rp dataLayerResourcePath) Folders() Elements {
endIdx := rp.lastFolderIdx() endIdx := rp.lastFolderIdx()
if endIdx == 4 { pfxLen := rp.prefixLen()
// if endIdx == prefix length, there are no folders
if endIdx == pfxLen {
return nil return nil
} }
return append([]string{}, rp.elements[4:endIdx]...) return append([]string{}, rp.elements[pfxLen:endIdx]...)
} }
// Item returns the item embedded in the dataLayerResourcePath if the path // Item returns the item embedded in the dataLayerResourcePath if the path
@ -240,15 +142,18 @@ func (rp dataLayerResourcePath) Item() string {
// Dir removes the last element from the path. If this would remove a // Dir removes the last element from the path. If this would remove a
// value that is part of the standard prefix structure, an error is returned. // value that is part of the standard prefix structure, an error is returned.
func (rp dataLayerResourcePath) Dir() (Path, error) { func (rp dataLayerResourcePath) Dir() (Path, error) {
if len(rp.elements) <= 4 { // Dir is not allowed to slice off any prefix values.
// The prefix len is determined by the length of the number of
// service+resource tuples, plus 2 (tenant and category).
if len(rp.elements) <= 2+(2*len(rp.serviceResources)) {
return nil, clues.New("unable to shorten path").With("path", rp) return nil, clues.New("unable to shorten path").With("path", rp)
} }
return &dataLayerResourcePath{ return &dataLayerResourcePath{
Builder: *rp.Builder.Dir(), Builder: *rp.Builder.Dir(),
service: rp.service, serviceResources: rp.serviceResources,
category: rp.category, category: rp.category,
hasItem: false, hasItem: false,
}, nil }, nil
} }
@ -261,10 +166,10 @@ func (rp dataLayerResourcePath) Append(
} }
return &dataLayerResourcePath{ return &dataLayerResourcePath{
Builder: *rp.Builder.Append(elems...), Builder: *rp.Builder.Append(elems...),
service: rp.service, serviceResources: rp.serviceResources,
category: rp.category, category: rp.category,
hasItem: isItem, hasItem: isItem,
}, nil }, nil
} }
@ -280,3 +185,17 @@ func (rp dataLayerResourcePath) ToBuilder() *Builder {
func (rp *dataLayerResourcePath) UpdateParent(prev, cur Path) bool { func (rp *dataLayerResourcePath) UpdateParent(prev, cur Path) bool {
return rp.Builder.UpdateParent(prev.ToBuilder(), cur.ToBuilder()) return rp.Builder.UpdateParent(prev.ToBuilder(), cur.ToBuilder())
} }
func (rp *dataLayerResourcePath) Halves() (*Builder, Elements) {
pfx, sfx := &Builder{}, Elements{}
b := rp.Builder
if len(b.elements) > 0 {
lenPfx := 2 + (len(rp.serviceResources) * 2)
pfx = &Builder{elements: append(Elements{}, b.elements[:lenPfx]...)}
sfx = append(Elements{}, b.elements[lenPfx-1:]...)
}
return pfx, sfx
}

View File

@ -15,14 +15,34 @@ import (
) )
const ( const (
testTenant = "aTenant" testTenant = "aTenant"
testUser = "aUser" testProtectedResource = "aProtectedResource"
) )
func elemsWithWithoutItem(elems path.Elements) func(isItem bool) path.Elements {
return func(isItem bool) path.Elements {
if isItem {
return elems[:len(elems)-1]
}
return elems
}
}
func itemWithWithoutItem(elems path.Elements) func(isItem bool) string {
return func(isItem bool) string {
if isItem {
return elems[len(elems)-1]
}
return ""
}
}
var ( var (
// Purposely doesn't have characters that need escaping so it can be easily // Purposely doesn't have characters that need escaping so it can be easily
// computed using strings.Join(). // computed using strings.Join().
rest = []string{"some", "folder", "path", "with", "possible", "item"} rest = path.Elements{"some", "folder", "path", "with", "possible", "item"}
missingInfo = []struct { missingInfo = []struct {
name string name string
@ -33,7 +53,7 @@ var (
{ {
name: "NoTenant", name: "NoTenant",
tenant: "", tenant: "",
user: testUser, user: testProtectedResource,
rest: rest, rest: rest,
}, },
{ {
@ -45,7 +65,7 @@ var (
{ {
name: "NoFolderOrItem", name: "NoFolderOrItem",
tenant: testTenant, tenant: testTenant,
user: testUser, user: testProtectedResource,
rest: nil, rest: nil,
}, },
} }
@ -70,60 +90,193 @@ var (
}, },
} }
// Set of acceptable service/category mixtures. // Set of acceptable service[/subservice]/category mixtures.
serviceCategories = []struct { serviceCategories = []struct {
service path.ServiceType name string
category path.CategoryType primaryService path.ServiceType
pathFunc func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) category path.CategoryType
pathFunc func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error)
expectFolders func(expect path.Elements) func(isItem bool) path.Elements
expectItem func(expect path.Elements) func(isItem bool) string
}{ }{
{ {
service: path.ExchangeService, name: path.ExchangeService.String() + path.EmailCategory.String(),
category: path.EmailCategory, primaryService: path.ExchangeService,
pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { category: path.EmailCategory,
return pb.ToDataLayerExchangePathForCategory(tenant, user, path.EmailCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.ExchangeService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.ExchangeService, name: path.ExchangeService.String() + path.ContactsCategory.String(),
category: path.ContactsCategory, primaryService: path.ExchangeService,
pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { category: path.ContactsCategory,
return pb.ToDataLayerExchangePathForCategory(tenant, user, path.ContactsCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.ExchangeService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.ExchangeService, name: path.ExchangeService.String() + path.EventsCategory.String(),
category: path.EventsCategory, primaryService: path.ExchangeService,
pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { category: path.EventsCategory,
return pb.ToDataLayerExchangePathForCategory(tenant, user, path.EventsCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.ExchangeService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.OneDriveService, name: path.OneDriveService.String() + path.FilesCategory.String(),
category: path.FilesCategory, primaryService: path.OneDriveService,
pathFunc: func(pb *path.Builder, tenant, user string, isItem bool) (path.Path, error) { category: path.FilesCategory,
return pb.ToDataLayerOneDrivePath(tenant, user, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.OneDriveService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.SharePointService, name: path.SharePointService.String() + path.LibrariesCategory.String(),
category: path.LibrariesCategory, primaryService: path.SharePointService,
pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) { category: path.LibrariesCategory,
return pb.ToDataLayerSharePointPath(tenant, site, path.LibrariesCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.SharePointService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.SharePointService, name: path.SharePointService.String() + path.ListsCategory.String(),
category: path.ListsCategory, primaryService: path.SharePointService,
pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) { category: path.ListsCategory,
return pb.ToDataLayerSharePointPath(tenant, site, path.ListsCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.SharePointService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
{ {
service: path.SharePointService, name: path.SharePointService.String() + path.PagesCategory.String(),
category: path.PagesCategory, primaryService: path.SharePointService,
pathFunc: func(pb *path.Builder, tenant, site string, isItem bool) (path.Path, error) { category: path.PagesCategory,
return pb.ToDataLayerSharePointPath(tenant, site, path.PagesCategory, isItem) pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.SharePointService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
}, },
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
},
{
name: path.GroupsService.String() + path.UnknownCategory.String(),
primaryService: path.GroupsService,
category: path.UnknownCategory,
pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(path.GroupsService, primaryResource)
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
},
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
},
{
name: path.GroupsService.String() + path.SharePointService.String() + path.UnknownCategory.String(),
primaryService: path.GroupsService,
category: path.LibrariesCategory,
pathFunc: func(
tenant, primaryResource string,
isItem bool,
suffix path.Elements,
) (path.Path, error) {
srs, err := path.NewServiceResources(
path.GroupsService,
primaryResource,
path.SharePointService,
"secondaryProtectedResource")
if err != nil {
return nil, err
}
return path.Build(tenant, srs, path.PagesCategory, isItem, suffix...)
},
expectFolders: elemsWithWithoutItem,
expectItem: itemWithWithoutItem,
}, },
} }
) )
@ -142,22 +295,13 @@ func (suite *DataLayerResourcePath) SetupSuite() {
func (suite *DataLayerResourcePath) TestMissingInfoErrors() { func (suite *DataLayerResourcePath) TestMissingInfoErrors() {
for _, types := range serviceCategories { for _, types := range serviceCategories {
suite.Run(types.service.String()+types.category.String(), func() { suite.Run(types.primaryService.String()+types.category.String(), func() {
for _, m := range modes { for _, m := range modes {
suite.Run(m.name, func() { suite.Run(m.name, func() {
for _, test := range missingInfo { for _, test := range missingInfo {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() _, err := types.pathFunc(test.tenant, test.user, m.isItem, rest)
assert.Error(suite.T(), err, clues.ToCore(err))
b := path.Builder{}.Append(test.rest...)
_, err := types.pathFunc(
b,
test.tenant,
test.user,
m.isItem,
)
assert.Error(t, err)
}) })
} }
}) })
@ -167,31 +311,23 @@ func (suite *DataLayerResourcePath) TestMissingInfoErrors() {
} }
func (suite *DataLayerResourcePath) TestMailItemNoFolder() { func (suite *DataLayerResourcePath) TestMailItemNoFolder() {
item := "item" for _, test := range serviceCategories {
b := path.Builder{}.Append(item) suite.Run(test.name, func() {
for _, types := range serviceCategories {
suite.Run(types.service.String()+types.category.String(), func() {
t := suite.T() t := suite.T()
p, err := types.pathFunc( p, err := test.pathFunc(testTenant, testProtectedResource, true, path.Elements{"item"})
b,
testTenant,
testUser,
true,
)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
assert.Empty(t, p.Folder(false)) assert.Empty(t, p.Folder(false))
assert.Empty(t, p.Folders()) assert.Empty(t, p.Folders())
assert.Equal(t, item, p.Item()) assert.Equal(t, test.expectItem(path.Elements{"item"})(true), p.Item())
}) })
} }
} }
func (suite *DataLayerResourcePath) TestPopFront() { func (suite *DataLayerResourcePath) TestPopFront() {
expected := path.Builder{}.Append(append( expected := path.Builder{}.Append(append(
[]string{path.ExchangeService.String(), testUser, path.EmailCategory.String()}, []string{path.ExchangeService.String(), testProtectedResource, path.EmailCategory.String()},
rest..., rest...,
)...) )...)
@ -202,7 +338,7 @@ func (suite *DataLayerResourcePath) TestPopFront() {
pb := path.Builder{}.Append(rest...) pb := path.Builder{}.Append(rest...)
p, err := pb.ToDataLayerExchangePathForCategory( p, err := pb.ToDataLayerExchangePathForCategory(
testTenant, testTenant,
testUser, testProtectedResource,
path.EmailCategory, path.EmailCategory,
m.isItem, m.isItem,
) )
@ -218,7 +354,7 @@ func (suite *DataLayerResourcePath) TestDir() {
elements := []string{ elements := []string{
testTenant, testTenant,
path.ExchangeService.String(), path.ExchangeService.String(),
testUser, testProtectedResource,
path.EmailCategory.String(), path.EmailCategory.String(),
} }
@ -227,7 +363,7 @@ func (suite *DataLayerResourcePath) TestDir() {
pb := path.Builder{}.Append(rest...) pb := path.Builder{}.Append(rest...)
p, err := pb.ToDataLayerExchangePathForCategory( p, err := pb.ToDataLayerExchangePathForCategory(
testTenant, testTenant,
testUser, testProtectedResource,
path.EmailCategory, path.EmailCategory,
m.isItem, m.isItem,
) )
@ -256,10 +392,10 @@ func (suite *DataLayerResourcePath) TestDir() {
func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() { func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
tenant := "a-tenant" tenant := "a-tenant"
user := "a-user" resource := "a-resource"
table := []struct { table := []struct {
name string name string
service path.ServiceType srs []path.ServiceResource
category path.CategoryType category path.CategoryType
postfix []string postfix []string
expectedService path.ServiceType expectedService path.ServiceType
@ -267,14 +403,14 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
}{ }{
{ {
name: "NoPostfixPasses", name: "NoPostfixPasses",
service: path.ExchangeService, srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EmailCategory, category: path.EmailCategory,
expectedService: path.ExchangeMetadataService, expectedService: path.ExchangeMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "PostfixPasses", name: "PostfixPasses",
service: path.ExchangeService, srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EmailCategory, category: path.EmailCategory,
postfix: []string{"a", "b"}, postfix: []string{"a", "b"},
expectedService: path.ExchangeMetadataService, expectedService: path.ExchangeMetadataService,
@ -282,48 +418,48 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
}, },
{ {
name: "Fails", name: "Fails",
service: path.ExchangeService, srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.FilesCategory, category: path.FilesCategory,
check: assert.Error, check: assert.Error,
}, },
{ {
name: "Passes", name: "Passes",
service: path.ExchangeService, srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.ContactsCategory, category: path.ContactsCategory,
expectedService: path.ExchangeMetadataService, expectedService: path.ExchangeMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "Passes", name: "Passes",
service: path.ExchangeService, srs: []path.ServiceResource{{path.ExchangeService, resource}},
category: path.EventsCategory, category: path.EventsCategory,
expectedService: path.ExchangeMetadataService, expectedService: path.ExchangeMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "Passes", name: "Passes",
service: path.OneDriveService, srs: []path.ServiceResource{{path.OneDriveService, resource}},
category: path.FilesCategory, category: path.FilesCategory,
expectedService: path.OneDriveMetadataService, expectedService: path.OneDriveMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "Passes", name: "Passes",
service: path.SharePointService, srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.LibrariesCategory, category: path.LibrariesCategory,
expectedService: path.SharePointMetadataService, expectedService: path.SharePointMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "Passes", name: "Passes",
service: path.SharePointService, srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.ListsCategory, category: path.ListsCategory,
expectedService: path.SharePointMetadataService, expectedService: path.SharePointMetadataService,
check: assert.NoError, check: assert.NoError,
}, },
{ {
name: "Passes", name: "Passes",
service: path.SharePointService, srs: []path.ServiceResource{{path.SharePointService, resource}},
category: path.PagesCategory, category: path.PagesCategory,
expectedService: path.SharePointMetadataService, expectedService: path.SharePointMetadataService,
check: assert.NoError, check: assert.NoError,
@ -331,27 +467,26 @@ func (suite *DataLayerResourcePath) TestToServiceCategoryMetadataPath() {
} }
for _, test := range table { for _, test := range table {
suite.Run(strings.Join([]string{ name := strings.Join([]string{
test.name, test.name,
test.service.String(), test.srs[0].Service.String(),
test.category.String(), test.category.String(),
}, "_"), func() { }, "_")
suite.Run(name, func() {
t := suite.T() t := suite.T()
pb := path.Builder{}.Append(test.postfix...) pb := path.Builder{}.Append(test.postfix...)
p, err := pb.ToServiceCategoryMetadataPath( p, err := pb.ToServiceCategoryMetadataPath(
tenant, tenant,
user, test.srs,
test.service,
test.category, test.category,
false) false)
test.check(t, err, clues.ToCore(err)) test.check(t, err, clues.ToCore(err))
if err != nil { if err == nil {
return assert.Equal(t, test.expectedService, p.ServiceResources()[0])
} }
assert.Equal(t, test.expectedService, p.Service())
}) })
} }
} }
@ -392,7 +527,7 @@ func (suite *DataLayerResourcePath) TestToExchangePathForCategory() {
p, err := b.ToDataLayerExchangePathForCategory( p, err := b.ToDataLayerExchangePathForCategory(
testTenant, testTenant,
testUser, testProtectedResource,
test.category, test.category,
m.isItem) m.isItem)
test.check(t, err, clues.ToCore(err)) test.check(t, err, clues.ToCore(err))
@ -402,9 +537,9 @@ func (suite *DataLayerResourcePath) TestToExchangePathForCategory() {
} }
assert.Equal(t, testTenant, p.Tenant()) assert.Equal(t, testTenant, p.Tenant())
assert.Equal(t, path.ExchangeService, p.Service()) assert.Equal(t, path.ExchangeService, p.PrimaryService())
assert.Equal(t, test.category, p.Category()) assert.Equal(t, test.category, p.Category())
assert.Equal(t, testUser, p.ResourceOwner()) assert.Equal(t, testProtectedResource, p.PrimaryProtectedResource())
assert.Equal(t, strings.Join(m.expectedFolders, "/"), p.Folder(false)) assert.Equal(t, strings.Join(m.expectedFolders, "/"), p.Folder(false))
assert.Equal(t, path.Elements(m.expectedFolders), p.Folders()) assert.Equal(t, path.Elements(m.expectedFolders), p.Folders())
assert.Equal(t, m.expectedItem, p.Item()) assert.Equal(t, m.expectedItem, p.Item())
@ -417,7 +552,8 @@ func (suite *DataLayerResourcePath) TestToExchangePathForCategory() {
type PopulatedDataLayerResourcePath struct { type PopulatedDataLayerResourcePath struct {
tester.Suite tester.Suite
// Bool value is whether the path is an item path or a folder path. // Bool value is whether the path is an item path or a folder path.
paths map[bool]path.Path serviceCategoriesToIsItemToPath map[string]map[bool]path.Path
isItemToPath map[bool]path.Path
} }
func TestPopulatedDataLayerResourcePath(t *testing.T) { func TestPopulatedDataLayerResourcePath(t *testing.T) {
@ -425,92 +561,109 @@ func TestPopulatedDataLayerResourcePath(t *testing.T) {
} }
func (suite *PopulatedDataLayerResourcePath) SetupSuite() { func (suite *PopulatedDataLayerResourcePath) SetupSuite() {
suite.paths = make(map[bool]path.Path, 2) suite.serviceCategoriesToIsItemToPath = map[string]map[bool]path.Path{}
base := path.Builder{}.Append(rest...)
for _, t := range []bool{true, false} { for _, sc := range serviceCategories {
p, err := base.ToDataLayerExchangePathForCategory( m := make(map[bool]path.Path, 2)
testTenant, suite.serviceCategoriesToIsItemToPath[sc.name] = m
testUser,
path.EmailCategory,
t,
)
require.NoError(suite.T(), err, clues.ToCore(err))
suite.paths[t] = p for _, is := range []bool{true, false} {
p, err := sc.pathFunc(testTenant, testProtectedResource, is, rest)
require.NoError(suite.T(), err, clues.ToCore(err))
suite.serviceCategoriesToIsItemToPath[sc.name][is] = p
suite.isItemToPath[is] = p
}
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestTenant() { func (suite *PopulatedDataLayerResourcePath) TestTenant() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, testTenant, suite.paths[m.isItem].Tenant()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), testTenant, p.Tenant())
})
}
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestService() { func (suite *PopulatedDataLayerResourcePath) TestPrimaryService() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, path.ExchangeService, suite.paths[m.isItem].Service()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), test.primaryService, p.PrimaryService())
})
}
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestCategory() { func (suite *PopulatedDataLayerResourcePath) TestCategory() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, path.EmailCategory, suite.paths[m.isItem].Category()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), test.category, p.Category())
})
}
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestResourceOwner() { func (suite *PopulatedDataLayerResourcePath) TestPrimaryProtectedResource() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, testUser, suite.paths[m.isItem].ResourceOwner()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), testProtectedResource, p.PrimaryProtectedResource())
})
}
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestFolder() { func (suite *PopulatedDataLayerResourcePath) TestFolder() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal( p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
t, assert.Equal(suite.T(), test.expectFolders(rest)(m.isItem).String(), p.Folder(true))
strings.Join(m.expectedFolders, "/"), })
suite.paths[m.isItem].Folder(false), }
)
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestFolders() { func (suite *PopulatedDataLayerResourcePath) TestFolders() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, path.Elements(m.expectedFolders), suite.paths[m.isItem].Folders()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), test.expectFolders(rest)(m.isItem), p.Folders())
})
}
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestItem() { func (suite *PopulatedDataLayerResourcePath) TestItem() {
for _, m := range modes { for _, test := range serviceCategories {
suite.Run(m.name, func() { suite.Run(test.name, func() {
t := suite.T() for _, m := range modes {
suite.Run(m.name, func() {
assert.Equal(t, m.expectedItem, suite.paths[m.isItem].Item()) p := suite.serviceCategoriesToIsItemToPath[test.name][m.isItem]
assert.Equal(suite.T(), test.expectItem(rest)(m.isItem), p.Item())
})
}
}) })
} }
} }
@ -535,8 +688,7 @@ func (suite *PopulatedDataLayerResourcePath) TestAppend() {
hasItem: false, hasItem: false,
expectedFolder: strings.Join( expectedFolder: strings.Join(
append(append([]string{}, rest...), newElement), append(append([]string{}, rest...), newElement),
"/", "/"),
),
expectedItem: "", expectedItem: "",
}, },
} }
@ -547,7 +699,7 @@ func (suite *PopulatedDataLayerResourcePath) TestAppend() {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
newPath, err := suite.paths[m.isItem].Append(test.hasItem, newElement) newPath, err := suite.isItemToPath[m.isItem].Append(test.hasItem, newElement)
// Items don't allow appending. // Items don't allow appending.
if m.isItem { if m.isItem {
@ -673,3 +825,57 @@ func (suite *PopulatedDataLayerResourcePath) TestUpdateParent_NoopsNils() {
}) })
} }
} }
func (suite *PopulatedDataLayerResourcePath) TestHalves() {
t := suite.T()
onlyPrefix, err := path.BuildPrefix(
"titd",
[]path.ServiceResource{{path.ExchangeService, "pr"}},
path.ContactsCategory)
require.NoError(t, err, clues.ToCore(err))
fullPath, err := path.Build(
"tid",
[]path.ServiceResource{{path.ExchangeService, "pr"}},
path.ContactsCategory,
true,
"fld", "item")
require.NoError(t, err, clues.ToCore(err))
table := []struct {
name string
dlrp path.Path
expectPfx *path.Builder
expectSfx path.Elements
}{
{
name: "only prefix",
dlrp: onlyPrefix,
expectPfx: path.Builder{}.Append(
"tid",
path.ExchangeService.String(),
"pr",
path.ContactsCategory.String()),
expectSfx: path.Elements{},
},
{
name: "full path",
dlrp: fullPath,
expectPfx: path.Builder{}.Append(
"tid",
path.ExchangeService.String(),
"pr",
path.ContactsCategory.String()),
expectSfx: path.Elements{"foo", "bar"},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
pfx, sfx := test.dlrp.Halves()
assert.Equal(t, test.expectPfx, pfx, "prefix")
assert.Equal(t, test.expectSfx, sfx, "suffix")
})
}
}

View File

@ -20,118 +20,76 @@ func TestServiceCategoryUnitSuite(t *testing.T) {
suite.Run(t, s) suite.Run(t, s)
} }
func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategoryBadStringErrors() { func (suite *ServiceCategoryUnitSuite) TestVerifyPrefixValues() {
table := []struct { table := []struct {
name string name string
service string service ServiceType
category string category CategoryType
}{ check assert.ErrorAssertionFunc
{
name: "Service",
service: "foo",
category: EmailCategory.String(),
},
{
name: "Category",
service: ExchangeService.String(),
category: "foo",
},
}
for _, test := range table {
suite.Run(test.name, func() {
_, _, err := validateServiceAndCategoryStrings(test.service, test.category)
assert.Error(suite.T(), err)
})
}
}
func (suite *ServiceCategoryUnitSuite) TestValidateServiceAndCategory() {
table := []struct {
name string
service string
category string
expectedService ServiceType
expectedCategory CategoryType
check assert.ErrorAssertionFunc
}{ }{
{ {
name: "UnknownService", name: "UnknownService",
service: UnknownService.String(), service: UnknownService,
category: EmailCategory.String(), category: EmailCategory,
check: assert.Error, check: assert.Error,
}, },
{ {
name: "UnknownCategory", name: "UnknownCategory",
service: ExchangeService.String(), service: ExchangeService,
category: UnknownCategory.String(), category: UnknownCategory,
check: assert.Error, check: assert.Error,
}, },
{ {
name: "BadServiceString", name: "BadServiceType",
service: "foo", service: ServiceType(-1),
category: EmailCategory.String(), category: EmailCategory,
check: assert.Error, check: assert.Error,
}, },
{ {
name: "BadCategoryString", name: "BadCategoryType",
service: ExchangeService.String(), service: ExchangeService,
category: "foo", category: CategoryType(-1),
check: assert.Error, check: assert.Error,
}, },
{ {
name: "ExchangeEmail", name: "ExchangeEmail",
service: ExchangeService.String(), service: ExchangeService,
category: EmailCategory.String(), category: EmailCategory,
expectedService: ExchangeService, check: assert.NoError,
expectedCategory: EmailCategory,
check: assert.NoError,
}, },
{ {
name: "ExchangeContacts", name: "ExchangeContacts",
service: ExchangeService.String(), service: ExchangeService,
category: ContactsCategory.String(), category: ContactsCategory,
expectedService: ExchangeService, check: assert.NoError,
expectedCategory: ContactsCategory,
check: assert.NoError,
}, },
{ {
name: "ExchangeEvents", name: "ExchangeEvents",
service: ExchangeService.String(), service: ExchangeService,
category: EventsCategory.String(), category: EventsCategory,
expectedService: ExchangeService, check: assert.NoError,
expectedCategory: EventsCategory,
check: assert.NoError,
}, },
{ {
name: "OneDriveFiles", name: "OneDriveFiles",
service: OneDriveService.String(), service: OneDriveService,
category: FilesCategory.String(), category: FilesCategory,
expectedService: OneDriveService, check: assert.NoError,
expectedCategory: FilesCategory,
check: assert.NoError,
}, },
{ {
name: "SharePointLibraries", name: "SharePointLibraries",
service: SharePointService.String(), service: SharePointService,
category: LibrariesCategory.String(), category: LibrariesCategory,
expectedService: SharePointService, check: assert.NoError,
expectedCategory: LibrariesCategory,
check: assert.NoError,
}, },
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
s, c, err := validateServiceAndCategoryStrings(test.service, test.category) srs := []ServiceResource{{test.service, "resource"}}
err := verifyPrefixValues("tid", srs, test.category)
test.check(t, err, clues.ToCore(err)) test.check(t, err, clues.ToCore(err))
if err != nil {
return
}
assert.Equal(t, test.expectedService, s)
assert.Equal(t, test.expectedCategory, c)
}) })
} }
} }
@ -155,9 +113,10 @@ func (suite *ServiceCategoryUnitSuite) TestToServiceType() {
} }
for _, test := range table { for _, test := range table {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() assert.Equal(
suite.T(),
assert.Equal(t, test.expected, toServiceType(test.service)) test.expected,
ToServiceType(test.service))
}) })
} }
} }

View File

@ -0,0 +1,205 @@
package path
import (
"github.com/alcionai/clues"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/common/str"
"github.com/alcionai/corso/src/internal/common/tform"
)
// ---------------------------------------------------------------------------
// Tuple
// ---------------------------------------------------------------------------
// ServiceResource holds a service + resource tuple. The tuple implies
// that the resource owns some data in the given service.
type ServiceResource struct {
Service ServiceType
ProtectedResource string
}
func MakeServiceResource(
st ServiceType,
protectedResource string,
) ServiceResource {
return ServiceResource{
Service: st,
ProtectedResource: protectedResource,
}
}
func (sr ServiceResource) validate() error {
if len(sr.ProtectedResource) == 0 {
return clues.Stack(errMissingSegment, clues.New("protected resource"))
}
return nil
}
// ---------------------------------------------------------------------------
// Exported Helpers
// ---------------------------------------------------------------------------
// NewServiceResources is a lenient constructor for building a
// new []ServiceResource. It allows the caller to pass in any
// number of arbitrary values, but will require the following:
// 1. even values must be path.ServiceType typed
// 2. odd values must be string typed
// 3. a non-zero, even number of values must be provided
func NewServiceResources(elems ...any) ([]ServiceResource, error) {
if len(elems) == 0 {
return nil, clues.New("missing service resources")
}
if len(elems)%2 == 1 {
return nil, clues.New("odd number of service resources")
}
srs := make([]ServiceResource, 0, len(elems)/2)
for i, j := 0, 1; i < len(elems); i, j = i+2, j+2 {
srv, err := tform.AnyToT[ServiceType](elems[i])
if err != nil {
return nil, clues.Wrap(err, "service")
}
pr, err := str.AnyToString(elems[j])
if err != nil {
return nil, clues.Wrap(err, "protected resource")
}
srs = append(srs, MakeServiceResource(srv, pr))
}
return srs, nil
}
func ServiceResourcesToResources(srs []ServiceResource) []string {
prs := make([]string, len(srs))
for i := range srs {
prs[i] = srs[i].ProtectedResource
}
return prs
}
func ServiceResourcesToServices(srs []ServiceResource) []ServiceType {
sts := make([]ServiceType, len(srs))
for i := range srs {
sts[i] = srs[i].Service
}
return sts
}
func ServiceResourcesMatchServices(srs []ServiceResource, sts []ServiceType) bool {
return slices.EqualFunc(srs, sts, func(sr ServiceResource, st ServiceType) bool {
return sr.Service == st
})
}
func ServiceResourcesToElements(srs []ServiceResource) Elements {
es := make(Elements, 0, len(srs)*2)
for _, tuple := range srs {
es = append(es, tuple.Service.String())
es = append(es, tuple.ProtectedResource)
}
return es
}
// ---------------------------------------------------------------------------
// Unexported Helpers
// ---------------------------------------------------------------------------
// elementsToServiceResources turns as many pairs of elems as possible
// into ServiceResource tuples. Elems must begin with a service, but
// may contain more entries than there are service/resource pairs.
// This transformer will continue consuming elements until it finds an
// even-numbered index that cannot be cast to a ServiceType.
// Returns the serviceResource pairs, the first index containing element
// that is not part of a service/resource pair, and an error if elems is
// len==0 or contains no services.
func elementsToServiceResources(elems Elements) ([]ServiceResource, int, error) {
if len(elems) == 0 {
return nil, -1, clues.Wrap(errMissingSegment, "service")
}
var (
srs = make([]ServiceResource, 0)
i int
)
for j := 1; i < len(elems); i, j = i+2, j+2 {
service := ToServiceType(elems[i])
if service == UnknownService {
if i == 0 {
return nil, -1, clues.Wrap(errMissingSegment, "service")
}
break
}
srs = append(srs, ServiceResource{service, elems[j]})
}
return srs, i, nil
}
// checks for the following:
// 1. each ServiceResource is valid
// 2. if len(srs) > 1, srs[i], srs[i+1] pass subservice checks.
func validateServiceResources(srs []ServiceResource) error {
switch len(srs) {
case 0:
return clues.Stack(errMissingSegment, clues.New("service"))
case 1:
return srs[0].validate()
}
for i, tuple := range srs {
if err := tuple.validate(); err != nil {
return err
}
if i+1 >= len(srs) {
continue
}
if err := ValidateServiceAndSubService(tuple.Service, srs[i+1].Service); err != nil {
return err
}
}
return nil
}
// makes a copy of the slice with all of the Services swapped for their
// metadata service countterparts.
func toMetadataServices(srs []ServiceResource) []ServiceResource {
msrs := make([]ServiceResource, 0, len(srs))
for _, sr := range srs {
msr := sr
metadataService := UnknownService
switch sr.Service {
// TODO: add groups
case ExchangeService:
metadataService = ExchangeMetadataService
case OneDriveService:
metadataService = OneDriveMetadataService
case SharePointService:
metadataService = SharePointMetadataService
}
msr.Service = metadataService
msrs = append(msrs, msr)
}
return msrs
}

View File

@ -0,0 +1,243 @@
package path
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type ServiceResourceUnitSuite struct {
tester.Suite
}
func TestServiceResourceUnitSuite(t *testing.T) {
suite.Run(t, &ServiceResourceUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ServiceResourceUnitSuite) TestNewServiceResource() {
table := []struct {
name string
input []any
expectErr assert.ErrorAssertionFunc
expectResult []ServiceResource
}{
{
name: "empty",
input: []any{},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "odd elems: 1",
input: []any{ExchangeService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "odd elems: 3",
input: []any{ExchangeService, "mailbox", OneDriveService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "non-service even index",
input: []any{"foo", "bar"},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "non-string odd index",
input: []any{ExchangeService, OneDriveService},
expectErr: assert.Error,
expectResult: nil,
},
{
name: "valid single",
input: []any{ExchangeService, "mailbox"},
expectErr: assert.NoError,
expectResult: []ServiceResource{{ExchangeService, "mailbox"}},
},
{
name: "valid multiple",
input: []any{ExchangeService, "mailbox", OneDriveService, "user"},
expectErr: assert.NoError,
expectResult: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result, err := NewServiceResources(test.input...)
test.expectErr(t, err, clues.ToCore(err))
assert.Equal(t, test.expectResult, result)
})
}
}
func (suite *ServiceResourceUnitSuite) TestValidateServiceResources() {
table := []struct {
name string
srs []ServiceResource
expect assert.ErrorAssertionFunc
}{
{
name: "empty",
srs: []ServiceResource{},
expect: assert.Error,
},
{
name: "invalid resource",
srs: []ServiceResource{{ExchangeService, ""}},
expect: assert.Error,
},
{
name: "invalid subservice",
srs: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
expect: assert.Error,
},
{
name: "valid",
srs: []ServiceResource{
{GroupsService, "group"},
{SharePointService, "site"},
},
expect: assert.NoError,
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
err := validateServiceResources(test.srs)
test.expect(t, err, clues.ToCore(err))
})
}
}
func (suite *ServiceResourceUnitSuite) TestServiceResourceToElements() {
table := []struct {
name string
srs []ServiceResource
expect Elements
}{
{
name: "empty",
srs: []ServiceResource{},
expect: Elements{},
},
{
name: "single",
srs: []ServiceResource{{ExchangeService, "user"}},
expect: Elements{ExchangeService.String(), "user"},
},
{
name: "multiple",
srs: []ServiceResource{
{ExchangeService, "mailbox"},
{OneDriveService, "user"},
},
expect: Elements{
ExchangeService.String(), "mailbox",
OneDriveService.String(), "user",
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
result := ServiceResourcesToElements(test.srs)
// not ElementsMatch, order matters
assert.Equal(t, test.expect, result)
})
}
}
func (suite *ServiceResourceUnitSuite) TestElementsToServiceResource() {
table := []struct {
name string
elems Elements
expectErr assert.ErrorAssertionFunc
expectIdx int
expectSRS []ServiceResource
}{
{
name: "empty",
elems: Elements{},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "nil",
elems: nil,
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "non-service 0th elem",
elems: Elements{"fnords"},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "non-service 2nd elem",
elems: Elements{ExchangeService.String(), "fnords", "smarf"},
expectErr: assert.Error,
expectIdx: -1,
expectSRS: nil,
},
{
name: "single serviceResource",
elems: Elements{ExchangeService.String(), "fnords"},
expectErr: assert.NoError,
expectIdx: 2,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}},
},
{
name: "single serviceResource and extra value",
elems: Elements{ExchangeService.String(), "fnords", "smarf"},
expectErr: assert.NoError,
expectIdx: 2,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}},
},
{
name: "multiple serviceResource",
elems: Elements{ExchangeService.String(), "fnords", OneDriveService.String(), "smarf"},
expectErr: assert.NoError,
expectIdx: 4,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}, {OneDriveService, "smarf"}},
},
{
name: "multiple serviceResource and extra value",
elems: Elements{ExchangeService.String(), "fnords", OneDriveService.String(), "smarf", "flaboigans"},
expectErr: assert.NoError,
expectIdx: 4,
expectSRS: []ServiceResource{{ExchangeService, "fnords"}, {OneDriveService, "smarf"}},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
srs, idx, err := elementsToServiceResources(test.elems)
test.expectErr(t, err, clues.ToCore(err))
assert.Equal(t, test.expectIdx, idx)
assert.Equal(t, test.expectSRS, srs)
})
}
}

View File

@ -0,0 +1,88 @@
package path
import (
"strings"
"github.com/alcionai/clues"
)
var ErrorUnknownService = clues.New("unknown service string")
// ServiceType denotes what service the path corresponds to. Metadata services
// are also included though they are only used for paths that house metadata for
// Corso backups.
//
// Metadata services are not considered valid service types for resource paths
// though they can be used for metadata paths.
//
// The order of the enums below can be changed, but the string representation of
// each enum must remain the same or migration code needs to be added to handle
// changes to the string format.
type ServiceType int
//go:generate stringer -type=ServiceType -linecomment
const (
UnknownService ServiceType = iota
ExchangeService // exchange
OneDriveService // onedrive
SharePointService // sharepoint
ExchangeMetadataService // exchangeMetadata
OneDriveMetadataService // onedriveMetadata
SharePointMetadataService // sharepointMetadata
GroupsService // groups
GroupsMetadataService // groupsMetadata
)
func ToServiceType(service string) ServiceType {
s := strings.ToLower(service)
switch s {
case strings.ToLower(ExchangeService.String()):
return ExchangeService
case strings.ToLower(OneDriveService.String()):
return OneDriveService
case strings.ToLower(SharePointService.String()):
return SharePointService
case strings.ToLower(ExchangeMetadataService.String()):
return ExchangeMetadataService
case strings.ToLower(OneDriveMetadataService.String()):
return OneDriveMetadataService
case strings.ToLower(SharePointMetadataService.String()):
return SharePointMetadataService
default:
return UnknownService
}
}
// subServices is a mapping of all valid service/subService pairs.
// a subService pair occurs when one service contains a reference
// to a protected resource of another service type, and the resource
// for that second service is the identifier which is used to discover
// data. A subService relationship may imply that the subservice data
// is wholly replicated/owned by the primary service, or it may not,
// each case differs.
//
// Ex:
// - groups/<gID>/sharepoint/<siteID> => each team in groups contains a
// complete sharepoint site.
// - groups/<gID>/member/<userID> => each user in a team can own one or
// more Chats. But the group does not contain the complete user data.
var subServices = map[ServiceType]map[ServiceType]struct{}{
GroupsService: {
SharePointService: {},
},
}
func ValidateServiceAndSubService(service, subService ServiceType) error {
subs, ok := subServices[service]
if !ok {
return clues.New("unsupported service").With("service", service)
}
if _, ok := subs[subService]; !ok {
return clues.New("unknown service/subService combination").
With("service", service, "subService", subService)
}
return nil
}

View File

@ -0,0 +1,54 @@
package path
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
)
type ServiceTypeUnitSuite struct {
tester.Suite
}
func TestServiceTypeUnitSuite(t *testing.T) {
suite.Run(t, &ServiceTypeUnitSuite{Suite: tester.NewUnitSuite(t)})
}
var knownServices = []ServiceType{
UnknownService,
ExchangeService,
OneDriveService,
SharePointService,
ExchangeMetadataService,
OneDriveMetadataService,
SharePointMetadataService,
GroupsService,
GroupsMetadataService,
}
func (suite *ServiceTypeUnitSuite) TestValildateServiceAndSubService() {
table := map[ServiceType]map[ServiceType]assert.ErrorAssertionFunc{}
for _, si := range knownServices {
table[si] = map[ServiceType]assert.ErrorAssertionFunc{}
for _, sj := range knownServices {
table[si][sj] = assert.Error
}
}
// expected successful
table[GroupsService][SharePointService] = assert.NoError
for srv, ti := range table {
for sub, expect := range ti {
suite.Run(srv.String()+"-"+sub.String(), func() {
err := ValidateServiceAndSubService(srv, sub)
expect(suite.T(), err, clues.ToCore(err))
})
}
}
}

View File

@ -348,7 +348,7 @@ func ensureAllUsersInDetails(
continue continue
} }
ro := p.ResourceOwner() ro := p.ServiceResources()[0].ProtectedResource
if !assert.NotEmpty(t, ro, "resource owner in path: "+rr) { if !assert.NotEmpty(t, ro, "resource owner in path: "+rr) {
continue continue
} }

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -43,6 +44,7 @@ type (
var ( var (
_ Reducer = &ExchangeRestore{} _ Reducer = &ExchangeRestore{}
_ pathCategorier = &ExchangeRestore{} _ pathCategorier = &ExchangeRestore{}
_ reasoner = &ExchangeRestore{}
) )
// NewExchange produces a new Selector with the service set to ServiceExchange. // NewExchange produces a new Selector with the service set to ServiceExchange.
@ -122,6 +124,13 @@ func (s exchange) PathCategories() selectorPathCategories {
} }
} }
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s exchange) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Stringers and Concealers // Stringers and Concealers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -815,8 +815,12 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
} }
joinedFldrs := strings.Join(newElems, "/") joinedFldrs := strings.Join(newElems, "/")
return stubRepoRef(
return stubRepoRef(p.Service(), p.Category(), p.ResourceOwner(), joinedFldrs, p.Item()) suite.T(),
p.ServiceResources(),
p.Category(),
joinedFldrs,
p.Item())
} }
makeDeets := func(refs ...path.Path) *details.Details { makeDeets := func(refs ...path.Path) *details.Details {
@ -1058,12 +1062,36 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce_locationRef() { func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce_locationRef() {
var ( var (
contact = stubRepoRef(path.ExchangeService, path.ContactsCategory, "uid", "id5/id6", "cid")
contactLocation = "conts/my_cont" contactLocation = "conts/my_cont"
event = stubRepoRef(path.ExchangeService, path.EventsCategory, "uid", "id1/id2", "eid")
eventLocation = "cal/my_cal" eventLocation = "cal/my_cal"
mail = stubRepoRef(path.ExchangeService, path.EmailCategory, "uid", "id3/id4", "mid")
mailLocation = "inbx/my_mail" mailLocation = "inbx/my_mail"
contact = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.ContactsCategory,
"id5/id6",
"cid")
event = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.EventsCategory,
"id1/id2",
"eid")
mail = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: "uid",
}},
path.EmailCategory,
"id3/id4",
"mid")
) )
makeDeets := func(refs ...string) *details.Details { makeDeets := func(refs ...string) *details.Details {

View File

@ -7,6 +7,7 @@ import (
"github.com/alcionai/clues" "github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -40,6 +41,7 @@ type (
var ( var (
_ Reducer = &GroupsRestore{} _ Reducer = &GroupsRestore{}
_ pathCategorier = &GroupsRestore{} _ pathCategorier = &GroupsRestore{}
_ reasoner = &GroupsRestore{}
) )
// NewGroupsBackup produces a new Selector with the service set to ServiceGroups. // NewGroupsBackup produces a new Selector with the service set to ServiceGroups.
@ -119,6 +121,13 @@ func (s groups) PathCategories() selectorPathCategories {
} }
} }
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s groups) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Stringers and Concealers // Stringers and Concealers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -2,7 +2,6 @@ package selectors
import ( import (
"fmt" "fmt"
"strings"
"testing" "testing"
"github.com/alcionai/clues" "github.com/alcionai/clues"
@ -167,6 +166,8 @@ func (s mockScope) PlainString() string { return plainString(s) }
// selectors // selectors
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
var _ servicerCategorizerProvider = &mockSel{}
type mockSel struct { type mockSel struct {
Selector Selector
} }
@ -183,6 +184,10 @@ func stubSelector(resourceOwners []string) mockSel {
} }
} }
func (m mockSel) PathCategories() selectorPathCategories {
return m.PathCategories()
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// helper funcs // helper funcs
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -207,7 +212,15 @@ func scopeMustHave[T scopeT](t *testing.T, sc T, m map[categorizer][]string) {
// stubPath ensures test path production matches that of fullPath design, // stubPath ensures test path production matches that of fullPath design,
// stubbing out static values where necessary. // stubbing out static values where necessary.
func stubPath(t *testing.T, user string, s []string, cat path.CategoryType) path.Path { func stubPath(t *testing.T, user string, s []string, cat path.CategoryType) path.Path {
pth, err := path.Build("tid", user, path.ExchangeService, cat, true, s...) pth, err := path.Build(
"tid",
[]path.ServiceResource{{
Service: path.ExchangeService,
ProtectedResource: user,
}},
cat,
true,
s...)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
return pth return pth
@ -215,6 +228,22 @@ func stubPath(t *testing.T, user string, s []string, cat path.CategoryType) path
// stubRepoRef ensures test path production matches that of repoRef design, // stubRepoRef ensures test path production matches that of repoRef design,
// stubbing out static values where necessary. // stubbing out static values where necessary.
func stubRepoRef(service path.ServiceType, data path.CategoryType, resourceOwner, folders, item string) string { func stubRepoRef(
return strings.Join([]string{"tid", service.String(), resourceOwner, data.String(), folders, item}, "/") t *testing.T,
srs []path.ServiceResource,
cat path.CategoryType,
folders, item string,
) string {
fs := path.Split(folders)
fs = append(fs, item)
pb, err := path.Build(
"tid",
srs,
cat,
true,
fs...)
require.NoError(t, err, clues.ToCore(err))
return pb.String()
} }

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var ( var (
_ Reducer = &OneDriveRestore{} _ Reducer = &OneDriveRestore{}
_ pathCategorier = &OneDriveRestore{} _ pathCategorier = &OneDriveRestore{}
_ reasoner = &OneDriveRestore{}
) )
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive. // NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
@ -121,6 +123,13 @@ func (s oneDrive) PathCategories() selectorPathCategories {
} }
} }
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s oneDrive) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Stringers and Concealers // Stringers and Concealers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -161,23 +161,32 @@ func (suite *OneDriveSelectorSuite) TestToOneDriveRestore() {
func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() { func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
var ( var (
file = stubRepoRef( file = stubRepoRef(
path.OneDriveService, suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory, path.FilesCategory,
"uid",
"drive/driveID/root:/folderA.d/folderB.d", "drive/driveID/root:/folderA.d/folderB.d",
"file") "file")
fileParent = "folderA/folderB" fileParent = "folderA/folderB"
file2 = stubRepoRef( file2 = stubRepoRef(
path.OneDriveService, suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory, path.FilesCategory,
"uid",
"drive/driveID/root:/folderA.d/folderC.d", "drive/driveID/root:/folderA.d/folderC.d",
"file2") "file2")
fileParent2 = "folderA/folderC" fileParent2 = "folderA/folderC"
file3 = stubRepoRef( file3 = stubRepoRef(
path.OneDriveService, suite.T(),
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "uid",
}},
path.FilesCategory, path.FilesCategory,
"uid",
"drive/driveID/root:/folderD.d/folderE.d", "drive/driveID/root:/folderD.d/folderE.d",
"file3") "file3")
fileParent3 = "folderD/folderE" fileParent3 = "folderD/folderE"
@ -314,7 +323,15 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
shortRef := "short" shortRef := "short"
elems := []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "dir1.d", "dir2.d", fileID} elems := []string{odConsts.DrivesPathDir, "driveID", odConsts.RootPathDir, "dir1.d", "dir2.d", fileID}
filePath, err := path.Build("tenant", "user", path.OneDriveService, path.FilesCategory, true, elems...) filePath, err := path.Build(
"tenant",
[]path.ServiceResource{{
Service: path.OneDriveService,
ProtectedResource: "user",
}},
path.FilesCategory,
true,
elems...)
require.NoError(t, err, clues.ToCore(err)) require.NoError(t, err, clues.ToCore(err))
fileLoc := path.Builder{}.Append("dir1", "dir2") fileLoc := path.Builder{}.Append("dir1", "dir2")
@ -351,8 +368,10 @@ func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
itemPath, err := path.Build( itemPath, err := path.Build(
"tenant", "tenant",
"site", []path.ServiceResource{{
path.OneDriveService, Service: path.OneDriveService,
ProtectedResource: "site",
}},
path.FilesCategory, path.FilesCategory,
true, true,
test.pathElems...) test.pathElems...)

View File

@ -0,0 +1,102 @@
package selectors
import (
"golang.org/x/exp/maps"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// reasoner interface compliance
// ---------------------------------------------------------------------------
var _ identity.Reasoner = &backupReason{}
type backupReason struct {
category path.CategoryType
resource string
service path.ServiceType
tenant string
}
func (br backupReason) Tenant() string {
return br.tenant
}
func (br backupReason) ProtectedResource() string {
return br.resource
}
func (br backupReason) Service() path.ServiceType {
return br.service
}
func (br backupReason) Category() path.CategoryType {
return br.category
}
func (br backupReason) SubtreePath() (path.Path, error) {
srs, err := path.NewServiceResources(br.service, br.resource)
if err != nil {
return nil, clues.Wrap(err, "building path prefix services")
}
return path.BuildPrefix(br.tenant, srs, br.category)
}
func (br backupReason) key() string {
return br.category.String() + br.resource + br.service.String() + br.tenant
}
// ---------------------------------------------------------------------------
// common transformer
// ---------------------------------------------------------------------------
type servicerCategorizerProvider interface {
pathServicer
pathCategorier
idname.Provider
}
// produces the Reasoner basis described by the selector.
// In cases of reasons with subservices (ie, multiple
// services described by a backup or path), the selector
// will only ever generate a ServiceResource for the first
// service+resource pair in the set.
//
// TODO: it may be possible, if necessary, to add subservice
// recognition to the service via additional scopes.
func reasonsFor(
sel servicerCategorizerProvider,
tenantID string,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := map[string]identity.Reasoner{}
resource := sel.ID()
if useOwnerNameForID {
resource = sel.Name()
}
pc := sel.PathCategories()
for _, sl := range [][]path.CategoryType{pc.Includes, pc.Filters} {
for _, cat := range sl {
br := backupReason{
category: cat,
resource: resource,
service: service,
tenant: tenantID,
}
reasons[br.key()] = br
}
}
return maps.Values(reasons)
}

View File

@ -0,0 +1,406 @@
package selectors
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/path"
)
type ReasonsUnitSuite struct {
tester.Suite
}
func TestReasonsUnitSuite(t *testing.T) {
suite.Run(t, &ReasonsUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "use name",
sel: func() ExchangeRestore {
sel := NewExchangeRestore([]string{"timbo"})
sel.Include(sel.MailFolders(Any()))
plainSel := sel.SetDiscreteOwnerIDName("timbo", "timbubba")
sel, err := plainSel.ToExchangeRestore()
require.NoError(suite.T(), err, clues.ToCore(err))
return *sel
},
useName: true,
expect: []expect{
{
tenant: tenantID,
resource: "timbubba",
category: email,
service: exchange,
subtreePath: stpFor("timbubba", email, exchange),
},
},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}
func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}

View File

@ -544,8 +544,11 @@ func reduce[T scopeT, C categoryT](
continue continue
} }
// first check, every entry needs to match the selector's resource owners. // first check, every entry needs to have at least one protected resource
if !matchesResourceOwner.Compare(repoPath.ResourceOwner()) { // that matches the selector's protected resources.
if !matchesResourceOwner.CompareAny(
path.ServiceResourcesToResources(
repoPath.ServiceResources())...) {
continue continue
} }

View File

@ -22,15 +22,15 @@ import (
// tests // tests
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
type SelectorScopesSuite struct { type ScopesUnitSuite struct {
tester.Suite tester.Suite
} }
func TestSelectorScopesSuite(t *testing.T) { func TestScopesUnitSuite(t *testing.T) {
suite.Run(t, &SelectorScopesSuite{Suite: tester.NewUnitSuite(t)}) suite.Run(t, &ScopesUnitSuite{Suite: tester.NewUnitSuite(t)})
} }
func (suite *SelectorScopesSuite) TestContains() { func (suite *ScopesUnitSuite) TestContains() {
table := []struct { table := []struct {
name string name string
scope func() mockScope scope func() mockScope
@ -108,7 +108,7 @@ func (suite *SelectorScopesSuite) TestContains() {
} }
} }
func (suite *SelectorScopesSuite) TestGetCatValue() { func (suite *ScopesUnitSuite) TestGetCatValue() {
t := suite.T() t := suite.T()
stub := stubScope("") stub := stubScope("")
@ -122,7 +122,7 @@ func (suite *SelectorScopesSuite) TestGetCatValue() {
getCatValue(stub, mockCategorizer("foo"))) getCatValue(stub, mockCategorizer("foo")))
} }
func (suite *SelectorScopesSuite) TestIsAnyTarget() { func (suite *ScopesUnitSuite) TestIsAnyTarget() {
t := suite.T() t := suite.T()
stub := stubScope("") stub := stubScope("")
assert.True(t, isAnyTarget(stub, rootCatStub)) assert.True(t, isAnyTarget(stub, rootCatStub))
@ -253,16 +253,19 @@ var reduceTestTable = []struct {
}, },
} }
func (suite *SelectorScopesSuite) TestReduce() { func (suite *ScopesUnitSuite) TestReduce() {
deets := func() details.Details { deets := func() details.Details {
return details.Details{ return details.Details{
DetailsModel: details.DetailsModel{ DetailsModel: details.DetailsModel{
Entries: []details.Entry{ Entries: []details.Entry{
{ {
RepoRef: stubRepoRef( RepoRef: stubRepoRef(
pathServiceStub, suite.T(),
[]path.ServiceResource{{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
}},
pathCatStub, pathCatStub,
rootCatStub.String(),
"stub", "stub",
leafCatStub.String(), leafCatStub.String(),
), ),
@ -298,16 +301,90 @@ func (suite *SelectorScopesSuite) TestReduce() {
} }
} }
func (suite *SelectorScopesSuite) TestReduce_locationRef() { func (suite *ScopesUnitSuite) TestReduce_locationRef() {
deets := func() details.Details {
return details.Details{
DetailsModel: details.DetailsModel{
Entries: []details.Entry{{
RepoRef: stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
}},
pathCatStub,
"stub",
leafCatStub.String(),
),
LocationRef: "a/b/c//defg",
}},
},
}
}
dataCats := map[path.CategoryType]mockCategorizer{
pathCatStub: rootCatStub,
}
for _, test := range reduceTestTable {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
ds := deets()
result := reduce[mockScope](
ctx,
&ds,
test.sel().Selector,
dataCats,
fault.New(true))
require.NotNil(t, result)
assert.Len(t, result.Entries, test.expectLen)
})
}
}
func (suite *ScopesUnitSuite) TestReduce_multipleServiceResources() {
deets := func() details.Details { deets := func() details.Details {
return details.Details{ return details.Details{
DetailsModel: details.DetailsModel{ DetailsModel: details.DetailsModel{
Entries: []details.Entry{ Entries: []details.Entry{
// tid/serv/"matching-id"/subserv/"non-matching-id"/...
{ {
RepoRef: stubRepoRef( RepoRef: stubRepoRef(
pathServiceStub, suite.T(),
[]path.ServiceResource{
{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
},
{
Service: pathServiceStub,
ProtectedResource: "foo",
},
},
pathCatStub,
"stub",
leafCatStub.String(),
),
LocationRef: "a/b/c//defg",
},
// tid/serv/"non-matching-id"/subserv/"matching-id"/...
{
RepoRef: stubRepoRef(
suite.T(),
[]path.ServiceResource{
{
Service: pathServiceStub,
ProtectedResource: "foo",
},
{
Service: pathServiceStub,
ProtectedResource: rootCatStub.String(),
},
},
pathCatStub, pathCatStub,
rootCatStub.String(),
"stub", "stub",
leafCatStub.String(), leafCatStub.String(),
), ),
@ -341,7 +418,7 @@ func (suite *SelectorScopesSuite) TestReduce_locationRef() {
} }
} }
func (suite *SelectorScopesSuite) TestScopesByCategory() { func (suite *ScopesUnitSuite) TestScopesByCategory() {
t := suite.T() t := suite.T()
s1 := stubScope("") s1 := stubScope("")
s2 := stubScope("") s2 := stubScope("")
@ -357,7 +434,7 @@ func (suite *SelectorScopesSuite) TestScopesByCategory() {
assert.Empty(t, result[leafCatStub]) assert.Empty(t, result[leafCatStub])
} }
func (suite *SelectorScopesSuite) TestPasses() { func (suite *ScopesUnitSuite) TestPasses() {
var ( var (
cat = rootCatStub cat = rootCatStub
pth = stubPath(suite.T(), "uid", []string{"fld"}, path.EventsCategory) pth = stubPath(suite.T(), "uid", []string{"fld"}, path.EventsCategory)
@ -401,7 +478,7 @@ func toMockScope(sc []scope) []mockScope {
return ms return ms
} }
func (suite *SelectorScopesSuite) TestMatchesPathValues() { func (suite *ScopesUnitSuite) TestMatchesPathValues() {
cat := rootCatStub cat := rootCatStub
short := "brunheelda" short := "brunheelda"
@ -460,7 +537,7 @@ func (suite *SelectorScopesSuite) TestMatchesPathValues() {
} }
} }
func (suite *SelectorScopesSuite) TestDefaultItemOptions() { func (suite *ScopesUnitSuite) TestDefaultItemOptions() {
table := []struct { table := []struct {
name string name string
cfg Config cfg Config
@ -521,7 +598,7 @@ func (suite *SelectorScopesSuite) TestDefaultItemOptions() {
} }
} }
func (suite *SelectorScopesSuite) TestClean() { func (suite *ScopesUnitSuite) TestClean() {
table := []struct { table := []struct {
name string name string
input []string input []string
@ -568,7 +645,7 @@ func (suite *SelectorScopesSuite) TestClean() {
} }
} }
func (suite *SelectorScopesSuite) TestScopeConfig() { func (suite *ScopesUnitSuite) TestScopeConfig() {
input := "input" input := "input"
table := []struct { table := []struct {
@ -608,7 +685,7 @@ func (ms mockFMTState) Width() (int, bool) { return 0, false }
func (ms mockFMTState) Precision() (int, bool) { return 0, false } func (ms mockFMTState) Precision() (int, bool) { return 0, false }
func (ms mockFMTState) Flag(int) bool { return false } func (ms mockFMTState) Flag(int) bool { return false }
func (suite *SelectorScopesSuite) TestScopesPII() { func (suite *ScopesUnitSuite) TestScopesPII() {
table := []struct { table := []struct {
name string name string
s mockScope s mockScope

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -87,6 +88,14 @@ type pathCategorier interface {
PathCategories() selectorPathCategories PathCategories() selectorPathCategories
} }
type pathServicer interface {
PathService() path.ServiceType
}
type reasoner interface {
Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Selector // Selector
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@ -273,7 +282,7 @@ func (s Selector) Reduce(
return r.Reduce(ctx, deets, errs), nil return r.Reduce(ctx, deets, errs), nil
} }
// returns the sets of path categories identified in each scope set. // PathCategories returns the sets of path categories identified in each scope set.
func (s Selector) PathCategories() (selectorPathCategories, error) { func (s Selector) PathCategories() (selectorPathCategories, error) {
ro, err := selectorAsIface[pathCategorier](s) ro, err := selectorAsIface[pathCategorier](s)
if err != nil { if err != nil {
@ -283,6 +292,18 @@ func (s Selector) PathCategories() (selectorPathCategories, error) {
return ro.PathCategories(), nil return ro.PathCategories(), nil
} }
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s Selector) Reasons(tenantID string, useOwnerNameForID bool) ([]identity.Reasoner, error) {
ro, err := selectorAsIface[reasoner](s)
if err != nil {
return nil, err
}
return ro.Reasons(tenantID, useOwnerNameForID), nil
}
// transformer for arbitrary selector interfaces // transformer for arbitrary selector interfaces
func selectorAsIface[T any](s Selector) (T, error) { func selectorAsIface[T any](s Selector) (T, error) {
var ( var (

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var ( var (
_ Reducer = &SharePointRestore{} _ Reducer = &SharePointRestore{}
_ pathCategorier = &SharePointRestore{} _ pathCategorier = &SharePointRestore{}
_ reasoner = &SharePointRestore{}
) )
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint. // NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
@ -121,6 +123,13 @@ func (s sharePoint) PathCategories() selectorPathCategories {
} }
} }
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s sharePoint) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Stringers and Concealers // Stringers and Concealers
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------

View File

@ -155,9 +155,12 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
} }
return stubRepoRef( return stubRepoRef(
path.SharePointService, suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: siteID,
}},
cat, cat,
siteID,
strings.Join(folderElems, "/"), strings.Join(folderElems, "/"),
item) item)
} }
@ -188,8 +191,24 @@ func (suite *SharePointSelectorSuite) TestSharePointRestore_Reduce() {
"sid", "sid",
append(slices.Clone(prefixElems), itemElems3...), append(slices.Clone(prefixElems), itemElems3...),
"item3") "item3")
item4 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item4") item4 = stubRepoRef(
item5 = stubRepoRef(path.SharePointService, path.PagesCategory, "sid", pairGH, "item5") suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: "sid",
}},
path.PagesCategory,
pairGH,
"item4")
item5 = stubRepoRef(
suite.T(),
[]path.ServiceResource{{
Service: path.SharePointService,
ProtectedResource: "sid",
}},
path.PagesCategory,
pairGH,
"item5")
) )
deets := &details.Details{ deets := &details.Details{
@ -417,10 +436,12 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
suite.Run(test.name, func() { suite.Run(test.name, func() {
t := suite.T() t := suite.T()
srs, err := path.NewServiceResources(path.SharePointService, "site")
require.NoError(t, err, clues.ToCore(err))
itemPath, err := path.Build( itemPath, err := path.Build(
"tenant", "tenant",
"site", srs,
path.SharePointService,
test.sc.PathType(), test.sc.PathType(),
true, true,
test.pathElems...) test.pathElems...)