larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use path filters in folder selectors (#1429)

Browse Source 
## Description

For folder-level scopes (ie, scopes that compare
folder-hierarchy path segments),  this change
replaces the standard "equals" and "prefix"
string comparators with the new PathContains
and PathPrefix comparators.

Next change is to interpret user inputs in the
cli to determine whether the comparator should
use contains or prefix behavior.

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1224

## Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2022-11-08 15:27:09 -07:00 committed by GitHub
parent 4909177822
commit b20e7af533
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 312 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/cli/backup/exchange_integration_test.go
View File

@ -291,13 +291,16 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
switch set {
case email:
scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder})
scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())
case contacts:
scopes = sel.ContactFolders([]string{suite.m365UserID}, []string{exchange.DefaultContactFolder})
scopes = sel.ContactFolders(
[]string{suite.m365UserID},
[]string{exchange.DefaultContactFolder},
selectors.PrefixMatch())
case events:
scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar})
scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())
}
sel.Include(scopes)
@ -515,7 +518,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
// some tests require an existing backup
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}))
sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))

9
src/cli/restore/exchange_integration_test.go
View File

@ -96,13 +96,16 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
switch set {
case email:
scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder})
scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())
case contacts:
scopes = sel.ContactFolders([]string{suite.m365UserID}, []string{exchange.DefaultContactFolder})
scopes = sel.ContactFolders(
[]string{suite.m365UserID},
[]string{exchange.DefaultContactFolder},
selectors.PrefixMatch())
case events:
scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar})
scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())
}
sel.Include(scopes)

6
src/internal/connector/exchange/iterators_test.go
View File

@ -87,9 +87,9 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() {
eb, err := sel.ToExchangeBackup()
require.NoError(suite.T(), err)
contactScope = sel.ContactFolders([]string{userID}, []string{DefaultContactFolder})
eventScope = sel.EventCalendars([]string{userID}, []string{DefaultCalendar})
mailScope = sel.MailFolders([]string{userID}, []string{DefaultMailFolder})
contactScope = sel.ContactFolders([]string{userID}, []string{DefaultContactFolder}, selectors.PrefixMatch())
eventScope = sel.EventCalendars([]string{userID}, []string{DefaultCalendar}, selectors.PrefixMatch())
mailScope = sel.MailFolders([]string{userID}, []string{DefaultMailFolder}, selectors.PrefixMatch())
eb.Include(contactScope, eventScope, mailScope)

26
src/internal/connector/exchange/service_functions_test.go
View File

@ -74,7 +74,9 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{DefaultCalendar})[0]
return selectors.
NewExchangeBackup().
EventCalendars([]string{userID}, []string{DefaultCalendar}, selectors.PrefixMatch())[0]
},
},
{
@ -83,7 +85,9 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
expectCount: assert.Greater,
expectErr: assert.NoError,
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{"Birthdays"})[0]
return selectors.
NewExchangeBackup().
EventCalendars([]string{userID}, []string{"Birthdays"}, selectors.PrefixMatch())[0]
},
},
{
@ -94,7 +98,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{invalidUser}, []string{DefaultContactFolder})[0]
EventCalendars([]string{invalidUser}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0]
},
},
{
@ -156,7 +160,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{DefaultContactFolder})[0]
ContactFolders([]string{user}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0]
},
},
{
@ -167,7 +171,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{user}, []string{"TrialFolder"})[0]
ContactFolders([]string{user}, []string{"TrialFolder"}, selectors.PrefixMatch())[0]
},
},
{
@ -178,7 +182,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
ContactFolders([]string{invalidUser}, []string{DefaultContactFolder})[0]
ContactFolders([]string{invalidUser}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0]
},
},
{
@ -240,7 +244,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{DefaultMailFolder})[0]
MailFolders([]string{userID}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0]
},
},
{
@ -251,7 +255,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{userID}, []string{"Drafts"})[0]
MailFolders([]string{userID}, []string{"Drafts"}, selectors.PrefixMatch())[0]
},
},
{
@ -262,7 +266,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() {
getScope: func(t *testing.T) selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{invalidUser}, []string{DefaultMailFolder})[0]
MailFolders([]string{invalidUser}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0]
},
},
{
@ -326,7 +330,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestCollectContainers() {
getScope: func() selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
EventCalendars([]string{user}, []string{DefaultCalendar})[0]
EventCalendars([]string{user}, []string{DefaultCalendar}, selectors.PrefixMatch())[0]
},
}, {
name: "Default Mail",
@ -335,7 +339,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestCollectContainers() {
getScope: func() selectors.ExchangeScope {
return selectors.
NewExchangeBackup().
MailFolders([]string{user}, []string{DefaultMailFolder})[0]
MailFolders([]string{user}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0]
},
},
}

24
src/internal/connector/graph_connector_helper_test.go
View File

@ -695,23 +695,21 @@ func makeExchangeBackupSel(
pth := mustParsePath(t, p, false)
require.Equal(t, path.ExchangeService.String(), pth.Service().String())
builder := sel.MailFolders
switch pth.Category() {
case path.EmailCategory:
toInclude = append(toInclude, sel.MailFolders(
[]string{pth.ResourceOwner()},
[]string{backupInputFromPath(pth).String()},
))
case path.ContactsCategory:
toInclude = append(toInclude, sel.ContactFolders(
[]string{pth.ResourceOwner()},
[]string{backupInputFromPath(pth).String()},
))
builder = sel.ContactFolders
case path.EventsCategory:
toInclude = append(toInclude, sel.EventCalendars(
[]string{pth.ResourceOwner()},
[]string{backupInputFromPath(pth).String()},
))
builder = sel.EventCalendars
case path.EmailCategory: // already set
}
toInclude = append(toInclude, builder(
[]string{pth.ResourceOwner()},
[]string{backupInputFromPath(pth).String()},
selectors.PrefixMatch(),
))
}
sel.Include(toInclude...)

26
src/internal/connector/graph_connector_test.go
View File

@ -137,7 +137,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
name: suite.user + " Email",
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}))
sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
return sel.Selector
},
@ -146,7 +146,10 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
name: suite.user + " Contacts",
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}))
sel.Include(sel.ContactFolders(
[]string{suite.user},
[]string{exchange.DefaultContactFolder},
selectors.PrefixMatch()))
return sel.Selector
},
@ -155,7 +158,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
name: suite.user + " Events",
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}))
sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
return sel.Selector
},
@ -190,7 +193,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() {
t := suite.T()
connector := loadConnector(ctx, t)
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}))
sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
collection, err := connector.createCollections(ctx, sel.Scopes()[0])
require.NoError(t, err)
@ -233,7 +236,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression(
getCollection: func(t *testing.T) []*exchange.Collection {
scope := selectors.
NewExchangeBackup().
ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder})[0]
ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch())[0]
collections, err := connector.createCollections(ctx, scope)
require.NoError(t, err)
@ -286,7 +289,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
expected: exchange.DefaultCalendar,
getCollection: func(t *testing.T) []*exchange.Collection {
sel := selectors.NewExchangeBackup()
sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}))
sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
collections, err := connector.createCollections(ctx, sel.Scopes()[0])
require.NoError(t, err)
@ -298,7 +301,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
expected: "Birthdays",
getCollection: func(t *testing.T) []*exchange.Collection {
sel := selectors.NewExchangeBackup()
sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}))
sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}, selectors.PrefixMatch()))
collections, err := connector.createCollections(ctx, sel.Scopes()[0])
require.NoError(t, err)
@ -345,7 +348,7 @@ func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() {
t := suite.T()
connector := loadConnector(ctx, t)
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}))
sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
scopes := sel.DiscreteScopes(connector.GetUsers())
for _, scope := range scopes {
@ -376,6 +379,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailFetch() {
scope: selectors.NewExchangeBackup().MailFolders(
[]string{userID},
[]string{exchange.DefaultMailFolder},
selectors.PrefixMatch(),
)[0],
folderNames: map[string]struct{}{
exchange.DefaultMailFolder: {},
@ -879,19 +883,19 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
t.Logf("Restore complete\n")
t.Log("Restore complete")
}
// Run a backup and compare its output with what we put in.
backupGC := loadConnector(ctx, t)
backupSel := backupSelectorForExpected(t, allExpectedData)
t.Logf("Selective backup of %s\n", backupSel)
t.Log("Selective backup of", backupSel)
dcs, err := backupGC.DataCollections(ctx, backupSel)
require.NoError(t, err)
t.Logf("Backup enumeration complete\n")
t.Log("Backup enumeration complete")
// Pull the data prior to waiting for the status as otherwise it will
// deadlock.

83
src/internal/connector/onedrive/collections_test.go
View File

@ -39,8 +39,14 @@ func TestOneDriveCollectionsSuite(t *testing.T) {
func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0]
tenant := "tenant"
user := "user"
const (
tenant = "tenant"
user = "user"
folder = "/folder"
folderSub = "/folder/subfolder"
pkg = "/package"
)
tests := []struct {
testCase string
@ -101,8 +107,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
driveItem("fileInRoot", testBaseDrivePath, true, false, false),
driveItem("folder", testBaseDrivePath, false, true, false),
driveItem("package", testBaseDrivePath, false, false, true),
driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false),
driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false),
driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: anyFolder,
expect: assert.NoError,
@ -111,32 +117,65 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
tenant,
user,
testBaseDrivePath,
testBaseDrivePath+"/folder",
testBaseDrivePath+"/package",
testBaseDrivePath+folder,
testBaseDrivePath+pkg,
),
expectedItemCount: 6,
expectedFileCount: 3,
expectedContainerCount: 3,
},
{
testCase: "match folder selector",
testCase: "contains folder selector",
items: []models.DriveItemable{
driveItem("fileInRoot", testBaseDrivePath, true, false, false),
driveItem("folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", testBaseDrivePath+"/folder", false, true, false),
driveItem("folder", testBaseDrivePath+"/folder/subfolder", false, true, false),
driveItem("subfolder", testBaseDrivePath+folder, false, true, false),
driveItem("folder", testBaseDrivePath+folderSub, false, true, false),
driveItem("package", testBaseDrivePath, false, false, true),
driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false),
driveItem("fileInFolder2", testBaseDrivePath+"/folder/subfolder/folder", true, false, false),
driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false),
driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: (&selectors.OneDriveBackup{}).Folders(selectors.Any(), []string{"folder"})[0],
expect: assert.NoError,
expectedCollectionPaths: append(
expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
),
expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+folderSub+folder,
)...,
),
expectedItemCount: 4,
expectedFileCount: 2,
expectedContainerCount: 2,
},
{
testCase: "prefix subfolder selector",
items: []models.DriveItemable{
driveItem("fileInRoot", testBaseDrivePath, true, false, false),
driveItem("folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", testBaseDrivePath+folder, false, true, false),
driveItem("folder", testBaseDrivePath+folderSub, false, true, false),
driveItem("package", testBaseDrivePath, false, false, true),
driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false),
driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: (&selectors.OneDriveBackup{}).
Folders(selectors.Any(), []string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
expect: assert.NoError,
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder",
testBaseDrivePath+folderSub+folder,
),
expectedItemCount: 2,
expectedFileCount: 1,
@ -147,11 +186,11 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
items: []models.DriveItemable{
driveItem("fileInRoot", testBaseDrivePath, true, false, false),
driveItem("folder", testBaseDrivePath, false, true, false),
driveItem("subfolder", testBaseDrivePath+"/folder", false, true, false),
driveItem("subfolder", testBaseDrivePath+folder, false, true, false),
driveItem("package", testBaseDrivePath, false, false, true),
driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false),
driveItem("fileInSubfolder", testBaseDrivePath+"/folder/subfolder", true, false, false),
driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false),
driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false),
driveItem("fileInSubfolder", testBaseDrivePath+folderSub, true, false, false),
driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false),
},
scope: (&selectors.OneDriveBackup{}).Folders(selectors.Any(), []string{"folder/subfolder"})[0],
expect: assert.NoError,
@ -159,7 +198,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
suite.T(),
tenant,
user,
testBaseDrivePath+"/folder/subfolder",
testBaseDrivePath+folderSub,
),
expectedItemCount: 2,
expectedFileCount: 1,
@ -175,10 +214,10 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil)
err := c.updateCollections(ctx, "driveID", tt.items)
tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap))
assert.Equal(t, tt.expectedItemCount, c.numItems)
assert.Equal(t, tt.expectedFileCount, c.numFiles)
assert.Equal(t, tt.expectedContainerCount, c.numContainers)
assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths")
assert.Equal(t, tt.expectedItemCount, c.numItems, "item count")
assert.Equal(t, tt.expectedFileCount, c.numFiles, "file count")
assert.Equal(t, tt.expectedContainerCount, c.numContainers, "container count")
for _, collPath := range tt.expectedCollectionPaths {
assert.Contains(t, c.collectionMap, collPath)
}

9
src/internal/operations/backup_test.go
View File

@ -189,7 +189,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
name: "Integration Exchange.Mail",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}))
sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
return &sel.Selector
},
},
@ -197,7 +197,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
name: "Integration Exchange.Contacts",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder}))
sel.Include(sel.ContactFolders(
[]string{m365UserID},
[]string{exchange.DefaultContactFolder},
selectors.PrefixMatch()))
return &sel.Selector
},
},
@ -205,7 +208,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() {
name: "Integration Exchange.Events",
selectFunc: func() *selectors.Selector {
sel := selectors.NewExchangeBackup()
sel.Include(sel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}))
sel.Include(sel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()))
return &sel.Selector
},
},

6
src/internal/operations/restore_test.go
View File

@ -180,9 +180,9 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() {
bsel := selectors.NewExchangeBackup()
bsel.Include(
bsel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}),
bsel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder}),
bsel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}),
bsel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()),
bsel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch()),
bsel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()),
)
bo, err := NewBackupOperation(

32
src/pkg/filters/filters.go
View File

@ -62,10 +62,11 @@ func normPathElem(s string) string {
// compare values against. Filter.Matches(v) returns
// true if Filter.Comparer(filter.target, v) is true.
type Filter struct {
Comparator comparator `json:"comparator"`
Target string `json:"target"` // the value to compare against
Targets []string `json:"targets"` // the set of values to compare against, always with "any-match" behavior
Negate bool `json:"negate"` // when true, negate the comparator result
Comparator comparator `json:"comparator"`
Target string `json:"target"` // the value to compare against
Targets []string `json:"targets"` // the set of values to compare
NormalizedTargets []string `json:"normalizedTargets"` // the set of comparable values post normalization
Negate bool `json:"negate"` // when true, negate the comparator result
}
// ----------------------------------------------------------------------------------------------------
@ -176,7 +177,7 @@ func PathPrefix(targets []string) Filter {
tgts[i] = normPathElem(targets[i])
}
return newSliceFilter(TargetPathPrefix, tgts, false)
return newSliceFilter(TargetPathPrefix, targets, tgts, false)
}
// NotPathPrefix creates a filter where Compare(v) is true if
@ -195,7 +196,7 @@ func NotPathPrefix(targets []string) Filter {
tgts[i] = normPathElem(targets[i])
}
return newSliceFilter(TargetPathPrefix, tgts, true)
return newSliceFilter(TargetPathPrefix, targets, tgts, true)
}
// PathContains creates a filter where Compare(v) is true if
@ -216,7 +217,7 @@ func PathContains(targets []string) Filter {
tgts[i] = normPathElem(targets[i])
}
return newSliceFilter(TargetPathContains, tgts, false)
return newSliceFilter(TargetPathContains, targets, tgts, false)
}
// NotPathContains creates a filter where Compare(v) is true if
@ -237,7 +238,7 @@ func NotPathContains(targets []string) Filter {
tgts[i] = normPathElem(targets[i])
}
return newSliceFilter(TargetPathContains, tgts, true)
return newSliceFilter(TargetPathContains, targets, tgts, true)
}
// newFilter is the standard filter constructor.
@ -250,11 +251,12 @@ func newFilter(c comparator, target string, negate bool) Filter {
}
// newSliceFilter constructs filters that contain multiple targets
func newSliceFilter(c comparator, targets []string, negate bool) Filter {
func newSliceFilter(c comparator, targets, normTargets []string, negate bool) Filter {
return Filter{
Comparator: c,
Targets: targets,
Negate: negate,
Comparator: c,
Targets: targets,
NormalizedTargets: normTargets,
Negate: negate,
}
}
@ -314,7 +316,7 @@ func (f Filter) Compare(input string) bool {
targets := []string{f.Target}
if hasSlice {
targets = f.Targets
targets = f.NormalizedTargets
}
for _, tgt := range targets {
@ -415,5 +417,9 @@ func (f Filter) String() string {
return "fail"
}
if len(f.Targets) > 0 {
return prefixString[f.Comparator] + strings.Join(f.Targets, ",")
}
return prefixString[f.Comparator] + f.Target
}

54
src/pkg/filters/filters_test.go
View File

@ -255,6 +255,33 @@ func (suite *FiltersSuite) TestPathPrefix() {
}
}
func (suite *FiltersSuite) TestPathPrefix_NormalizedTargets() {
table := []struct {
name string
targets []string
expect []string
}{
{"Single - no slash", []string{"fA"}, []string{"/fA/"}},
{"Single - pre slash", []string{"/fA"}, []string{"/fA/"}},
{"Single - suff slash", []string{"fA/"}, []string{"/fA/"}},
{"Single - both slashes", []string{"/fA/"}, []string{"/fA/"}},
{"Multipath - no slash", []string{"fA/fB"}, []string{"/fA/fB/"}},
{"Multipath - pre slash", []string{"/fA/fB"}, []string{"/fA/fB/"}},
{"Multipath - suff slash", []string{"fA/fB/"}, []string{"/fA/fB/"}},
{"Multipath - both slashes", []string{"/fA/fB/"}, []string{"/fA/fB/"}},
{"Multi input - no slash", []string{"fA", "fB"}, []string{"/fA/", "/fB/"}},
{"Multi input - pre slash", []string{"/fA", "/fB"}, []string{"/fA/", "/fB/"}},
{"Multi input - suff slash", []string{"fA/", "fB/"}, []string{"/fA/", "/fB/"}},
{"Multi input - both slashes", []string{"/fA/", "/fB/"}, []string{"/fA/", "/fB/"}},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
f := filters.PathPrefix(test.targets)
assert.Equal(t, test.expect, f.NormalizedTargets)
})
}
}
func (suite *FiltersSuite) TestPathContains() {
table := []struct {
name string
@ -306,3 +333,30 @@ func (suite *FiltersSuite) TestPathContains() {
})
}
}
func (suite *FiltersSuite) TestPathContains_NormalizedTargets() {
table := []struct {
name string
targets []string
expect []string
}{
{"Single - no slash", []string{"fA"}, []string{"/fA/"}},
{"Single - pre slash", []string{"/fA"}, []string{"/fA/"}},
{"Single - suff slash", []string{"fA/"}, []string{"/fA/"}},
{"Single - both slashes", []string{"/fA/"}, []string{"/fA/"}},
{"Multipath - no slash", []string{"fA/fB"}, []string{"/fA/fB/"}},
{"Multipath - pre slash", []string{"/fA/fB"}, []string{"/fA/fB/"}},
{"Multipath - suff slash", []string{"fA/fB/"}, []string{"/fA/fB/"}},
{"Multipath - both slashes", []string{"/fA/fB/"}, []string{"/fA/fB/"}},
{"Multi input - no slash", []string{"fA", "fB"}, []string{"/fA/", "/fB/"}},
{"Multi input - pre slash", []string{"/fA", "/fB"}, []string{"/fA/", "/fB/"}},
{"Multi input - suff slash", []string{"fA/", "fB/"}, []string{"/fA/", "/fB/"}},
{"Multi input - both slashes", []string{"/fA/", "/fB/"}, []string{"/fA/", "/fB/"}},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
f := filters.PathContains(test.targets)
assert.Equal(t, test.expect, f.NormalizedTargets)
})
}
}

30
src/pkg/selectors/exchange.go
View File

@ -190,11 +190,14 @@ func (s *exchange) Contacts(users, folders, contacts []string) []ExchangeScope {
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *exchange) ContactFolders(users, folders []string, opts ...option) []ExchangeScope {
scopes := []ExchangeScope{}
var (
scopes = []ExchangeScope{}
os = append([]option{pathType()}, opts...)
)
scopes = append(
scopes,
makeScope[ExchangeScope](ExchangeContactFolder, users, folders, opts...),
makeScope[ExchangeScope](ExchangeContactFolder, users, folders, os...),
)
return scopes
@ -222,11 +225,14 @@ func (s *exchange) Events(users, calendars, events []string) []ExchangeScope {
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *exchange) EventCalendars(users, events []string, opts ...option) []ExchangeScope {
scopes := []ExchangeScope{}
var (
scopes = []ExchangeScope{}
os = append([]option{pathType()}, opts...)
)
scopes = append(
scopes,
makeScope[ExchangeScope](ExchangeEventCalendar, users, events, opts...),
makeScope[ExchangeScope](ExchangeEventCalendar, users, events, os...),
)
return scopes
@ -253,11 +259,14 @@ func (s *exchange) Mails(users, folders, mails []string) []ExchangeScope {
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *exchange) MailFolders(users, folders []string, opts ...option) []ExchangeScope {
scopes := []ExchangeScope{}
var (
scopes = []ExchangeScope{}
os = append([]option{pathType()}, opts...)
)
scopes = append(
scopes,
makeScope[ExchangeScope](ExchangeMailFolder, users, folders, opts...),
makeScope[ExchangeScope](ExchangeMailFolder, users, folders, os...),
)
return scopes
@ -624,8 +633,13 @@ func (s ExchangeScope) Get(cat exchangeCategory) []string {
}
// sets a value by category to the scope. Only intended for internal use.
func (s ExchangeScope) set(cat exchangeCategory, v []string) ExchangeScope {
return set(s, cat, v)
func (s ExchangeScope) set(cat exchangeCategory, v []string, opts ...option) ExchangeScope {
os := []option{}
if cat == ExchangeContactFolder || cat == ExchangeEventCalendar || cat == ExchangeMailFolder {
os = append(os, pathType())
}
return set(s, cat, v, append(os, opts...)...)
}
// setDefaults ensures that contact folder, mail folder, and user category

36
src/pkg/selectors/exchange_test.go
View File

@ -773,12 +773,13 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesInfo() {
func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
const (
usr = "userID"
fld = "mailFolder"
fld1 = "mailFolder"
fld2 = "subFolder"
mail = "mailID"
)
var (
pth = stubPath(suite.T(), usr, []string{fld, mail}, path.EmailCategory)
pth = stubPath(suite.T(), usr, []string{fld1, fld2, mail}, path.EmailCategory)
short = "thisisahashofsomekind"
es = NewExchangeRestore()
)
@ -796,13 +797,14 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
{"one of multiple users", es.Users([]string{"smarf", usr}), "", assert.True},
{"all folders", es.MailFolders(Any(), Any()), "", assert.True},
{"no folders", es.MailFolders(Any(), None()), "", assert.False},
{"matching folder", es.MailFolders(Any(), []string{fld}), "", assert.True},
{"matching folder", es.MailFolders(Any(), []string{fld1}), "", assert.True},
{"incomplete matching folder", es.MailFolders(Any(), []string{"mail"}), "", assert.False},
{"non-matching folder", es.MailFolders(Any(), []string{"smarf"}), "", assert.False},
// This test validates that folders that match a substring of the scope are not included (bugfix 1)
{"non-matching folder substring", es.MailFolders(Any(), []string{fld + "_suffix"}), "", assert.False},
{"matching folder prefix", es.MailFolders(Any(), []string{"mailF"}), "", assert.True},
{"non-matching folder substring", es.MailFolders(Any(), []string{fld1 + "_suffix"}), "", assert.False},
{"matching folder prefix", es.MailFolders(Any(), []string{fld1}, PrefixMatch()), "", assert.True},
{"incomplete folder prefix", es.MailFolders(Any(), []string{"mail"}, PrefixMatch()), "", assert.False},
{"matching folder substring", es.MailFolders(Any(), []string{"Folder"}), "", assert.False},
{"one of multiple folders", es.MailFolders(Any(), []string{"smarf", fld}), "", assert.True},
{"one of multiple folders", es.MailFolders(Any(), []string{"smarf", fld2}), "", assert.True},
{"all mail", es.Mails(Any(), Any(), Any()), "", assert.True},
{"no mail", es.Mails(Any(), Any(), None()), "", assert.False},
{"matching mail", es.Mails(Any(), Any(), []string{mail}), "", assert.True},
@ -986,6 +988,26 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
},
arr(contactInSubFolder),
},
{
"only match contactInSubFolder by prefix",
makeDeets(contactInSubFolder, contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore()
er.Include(er.ContactFolders([]string{"uid"}, []string{"cfld1/cfld2"}, PrefixMatch()))
return er
},
arr(contactInSubFolder),
},
{
"only match contactInSubFolder by leaf folder",
makeDeets(contactInSubFolder, contact, event, mail),
func() *ExchangeRestore {
er := NewExchangeRestore()
er.Include(er.ContactFolders([]string{"uid"}, []string{"cfld2"}))
return er
},
arr(contactInSubFolder),
},
{
"only match event",
makeDeets(contact, event, mail),

4
src/pkg/selectors/helpers_test.go
View File

@ -171,11 +171,11 @@ func setScopesToDefault[T scopeT](ts []T) []T {
return ts
}
// calls assert.Equal(t, getCatValue(sc, k)[0], v) on each k:v pair in the map
// calls assert.Equal(t, v, getCatValue(sc, k)[0]) on each k:v pair in the map
func scopeMustHave[T scopeT](t *testing.T, sc T, m map[categorizer]string) {
for k, v := range m {
t.Run(k.String(), func(t *testing.T) {
assert.Equal(t, getCatValue(sc, k), split(v), "Key: %s", k)
assert.Equal(t, split(v), getCatValue(sc, k), "Key: %s", k)
})
}
}

16
src/pkg/selectors/onedrive.go
View File

@ -180,11 +180,14 @@ func (s *oneDrive) Users(users []string) []OneDriveScope {
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *oneDrive) Folders(users, folders []string, opts ...option) []OneDriveScope {
scopes := []OneDriveScope{}
var (
scopes = []OneDriveScope{}
os = append([]option{pathType()}, opts...)
)
scopes = append(
scopes,
makeScope[OneDriveScope](OneDriveFolder, users, folders, opts...),
makeScope[OneDriveScope](OneDriveFolder, users, folders, os...),
)
return scopes
@ -420,8 +423,13 @@ func (s OneDriveScope) Get(cat oneDriveCategory) []string {
}
// sets a value by category to the scope. Only intended for internal use.
func (s OneDriveScope) set(cat oneDriveCategory, v []string) OneDriveScope {
return set(s, cat, v)
func (s OneDriveScope) set(cat oneDriveCategory, v []string, opts ...option) OneDriveScope {
os := []option{}
if cat == OneDriveFolder {
os = append(os, pathType())
}
return set(s, cat, v, append(os, opts...)...)
}
// setDefaults ensures that user scopes express `AnyTgt` for their child category types.

53
src/pkg/selectors/scopes.go
View File

@ -158,16 +158,13 @@ func makeScope[T scopeT](
resources, vs []string,
opts ...option,
) T {
sc := scopeConfig{}
for _, opt := range opts {
opt(&sc)
}
sc := &scopeConfig{}
sc.populate(opts...)
s := T{
scopeKeyCategory: filters.Identity(cat.String()),
scopeKeyDataType: filters.Identity(cat.leafCat().String()),
cat.String(): filterize(sc, vs...),
cat.String(): filterize(*sc, vs...),
cat.rootCat().String(): filterize(scopeConfig{}, resources...),
}
@ -194,17 +191,18 @@ func makeFilterScope[T scopeT](
// ---------------------------------------------------------------------------
// matches returns true if the category is included in the scope's
// data type, and the target string is included in the scope.
func matches[T scopeT, C categoryT](s T, cat C, target string) bool {
// data type, and the input string passes the scope's filter for
// that category.
func matches[T scopeT, C categoryT](s T, cat C, inpt string) bool {
if !typeAndCategoryMatches(cat, s.categorizer()) {
return false
}
if len(target) == 0 {
if len(inpt) == 0 {
return false
}
return s[cat.String()].Compare(target)
return s[cat.String()].Compare(inpt)
}
// getCategory returns the scope's category value.
@ -222,18 +220,26 @@ func getFilterCategory[T scopeT](s T) string {
// delimiter, and returns the slice. If s[cat] is nil, returns
// None().
func getCatValue[T scopeT](s T, cat categorizer) []string {
v, ok := s[cat.String()]
filt, ok := s[cat.String()]
if !ok {
return None()
}
return split(v.Target)
if len(filt.Targets) > 0 {
return filt.Targets
}
return split(filt.Target)
}
// set sets a value by category to the scope. Only intended for internal
// use, not for exporting to callers.
func set[T scopeT](s T, cat categorizer, v []string) T {
s[cat.String()] = filterize(scopeConfig{}, v...)
func set[T scopeT](s T, cat categorizer, v []string, opts ...option) T {
sc := &scopeConfig{}
sc.populate(opts...)
s[cat.String()] = filterize(*sc, v...)
return s
}
@ -244,7 +250,7 @@ func isNoneTarget[T scopeT, C categoryT](s T, cat C) bool {
return false
}
return s[cat.String()].Target == NoneTgt
return s[cat.String()].Comparator == filters.Fails
}
// returns true if the category is included in the scope's category type,
@ -254,7 +260,7 @@ func isAnyTarget[T scopeT, C categoryT](s T, cat C) bool {
return false
}
return s[cat.String()].Target == AnyTgt
return s[cat.String()].Comparator == filters.Passes
}
// reduce filters the entries in the details to only those that match the
@ -436,7 +442,6 @@ func matchesPathValues[T scopeT, C categoryT](
var (
match bool
isLeaf = c.isLeaf()
isRoot = c == c.rootCat()
)
switch {
@ -445,19 +450,7 @@ func matchesPathValues[T scopeT, C categoryT](
case isLeaf && len(shortRef) > 0:
match = matches(sc, cc, pathVal) || matches(sc, cc, shortRef)
// Folder category - checks if any target folder is a prefix of the path folders.
// Assumes (correctly) that we need to split the targets and re-compose them into
// individual prefix matchers.
// TODO: assumes all folders require prefix matchers. Users can now specify whether
// the folder filter is a prefix match or not. We should respect that configuration.
case !isLeaf && !isRoot:
for _, tgt := range getCatValue(sc, c) {
if filters.Prefix(tgt).Compare(pathVal) {
match = true
break
}
}
// all other categories (root, folder, etc) just need to pass the filter
default:
match = matches(sc, cc, pathVal)
}

24
src/pkg/selectors/selectors.go
View File

@ -338,11 +338,18 @@ func addToSet(set []string, v []string) []string {
// ---------------------------------------------------------------------------
type scopeConfig struct {
usePathFilter bool
usePrefixFilter bool
}
type option func(*scopeConfig)
func (sc *scopeConfig) populate(opts ...option) {
for _, opt := range opts {
opt(sc)
}
}
// PrefixMatch ensures the selector uses a Prefix comparator, instead
// of contains or equals. Will not override a default Any() or None()
// comparator.
@ -352,6 +359,15 @@ func PrefixMatch() option {
}
}
// pathType is an internal-facing option. It is assumed that scope
// constructors will provide the pathType option whenever a folder-
// level scope (ie, a scope that compares path hierarchies) is created.
func pathType() option {
return func(sc *scopeConfig) {
sc.usePathFilter = true
}
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
@ -408,6 +424,14 @@ func filterize(sc scopeConfig, s ...string) filters.Filter {
return passAny
}
if sc.usePathFilter {
if sc.usePrefixFilter {
return filters.PathPrefix(s)
}
return filters.PathContains(s)
}
if sc.usePrefixFilter {
return filters.Prefix(join(s...))
}