diff --git a/src/cli/backup/exchange_integration_test.go b/src/cli/backup/exchange_integration_test.go index 1f500187e..d4d301375 100644 --- a/src/cli/backup/exchange_integration_test.go +++ b/src/cli/backup/exchange_integration_test.go @@ -291,13 +291,16 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() { switch set { case email: - scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}) + scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()) case contacts: - scopes = sel.ContactFolders([]string{suite.m365UserID}, []string{exchange.DefaultContactFolder}) + scopes = sel.ContactFolders( + []string{suite.m365UserID}, + []string{exchange.DefaultContactFolder}, + selectors.PrefixMatch()) case events: - scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}) + scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()) } sel.Include(scopes) @@ -515,7 +518,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() { // some tests require an existing backup sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder})) + sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector) require.NoError(t, suite.backupOp.Run(ctx)) diff --git a/src/cli/restore/exchange_integration_test.go b/src/cli/restore/exchange_integration_test.go index 8065b73c2..6cf7db6e7 100644 --- a/src/cli/restore/exchange_integration_test.go +++ b/src/cli/restore/exchange_integration_test.go @@ -96,13 +96,16 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() { switch set { case email: - scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}) + scopes = sel.MailFolders([]string{suite.m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()) case contacts: - scopes = sel.ContactFolders([]string{suite.m365UserID}, []string{exchange.DefaultContactFolder}) + scopes = sel.ContactFolders( + []string{suite.m365UserID}, + []string{exchange.DefaultContactFolder}, + selectors.PrefixMatch()) case events: - scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}) + scopes = sel.EventCalendars([]string{suite.m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()) } sel.Include(scopes) diff --git a/src/internal/connector/exchange/iterators_test.go b/src/internal/connector/exchange/iterators_test.go index ab8cf0719..795b302aa 100644 --- a/src/internal/connector/exchange/iterators_test.go +++ b/src/internal/connector/exchange/iterators_test.go @@ -87,9 +87,9 @@ func (suite *ExchangeIteratorSuite) TestCollectionFunctions() { eb, err := sel.ToExchangeBackup() require.NoError(suite.T(), err) - contactScope = sel.ContactFolders([]string{userID}, []string{DefaultContactFolder}) - eventScope = sel.EventCalendars([]string{userID}, []string{DefaultCalendar}) - mailScope = sel.MailFolders([]string{userID}, []string{DefaultMailFolder}) + contactScope = sel.ContactFolders([]string{userID}, []string{DefaultContactFolder}, selectors.PrefixMatch()) + eventScope = sel.EventCalendars([]string{userID}, []string{DefaultCalendar}, selectors.PrefixMatch()) + mailScope = sel.MailFolders([]string{userID}, []string{DefaultMailFolder}, selectors.PrefixMatch()) eb.Include(contactScope, eventScope, mailScope) diff --git a/src/internal/connector/exchange/service_functions_test.go b/src/internal/connector/exchange/service_functions_test.go index eb9ce4172..ee9d5fc5d 100644 --- a/src/internal/connector/exchange/service_functions_test.go +++ b/src/internal/connector/exchange/service_functions_test.go @@ -74,7 +74,9 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { expectCount: assert.Greater, expectErr: assert.NoError, getScope: func(t *testing.T) selectors.ExchangeScope { - return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{DefaultCalendar})[0] + return selectors. + NewExchangeBackup(). + EventCalendars([]string{userID}, []string{DefaultCalendar}, selectors.PrefixMatch())[0] }, }, { @@ -83,7 +85,9 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { expectCount: assert.Greater, expectErr: assert.NoError, getScope: func(t *testing.T) selectors.ExchangeScope { - return selectors.NewExchangeBackup().EventCalendars([]string{userID}, []string{"Birthdays"})[0] + return selectors. + NewExchangeBackup(). + EventCalendars([]string{userID}, []string{"Birthdays"}, selectors.PrefixMatch())[0] }, }, { @@ -94,7 +98,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllCalendars() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - EventCalendars([]string{invalidUser}, []string{DefaultContactFolder})[0] + EventCalendars([]string{invalidUser}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0] }, }, { @@ -156,7 +160,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - ContactFolders([]string{user}, []string{DefaultContactFolder})[0] + ContactFolders([]string{user}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0] }, }, { @@ -167,7 +171,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - ContactFolders([]string{user}, []string{"TrialFolder"})[0] + ContactFolders([]string{user}, []string{"TrialFolder"}, selectors.PrefixMatch())[0] }, }, { @@ -178,7 +182,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllContactFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - ContactFolders([]string{invalidUser}, []string{DefaultContactFolder})[0] + ContactFolders([]string{invalidUser}, []string{DefaultContactFolder}, selectors.PrefixMatch())[0] }, }, { @@ -240,7 +244,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - MailFolders([]string{userID}, []string{DefaultMailFolder})[0] + MailFolders([]string{userID}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0] }, }, { @@ -251,7 +255,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - MailFolders([]string{userID}, []string{"Drafts"})[0] + MailFolders([]string{userID}, []string{"Drafts"}, selectors.PrefixMatch())[0] }, }, { @@ -262,7 +266,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestGetAllMailFolders() { getScope: func(t *testing.T) selectors.ExchangeScope { return selectors. NewExchangeBackup(). - MailFolders([]string{invalidUser}, []string{DefaultMailFolder})[0] + MailFolders([]string{invalidUser}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0] }, }, { @@ -326,7 +330,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestCollectContainers() { getScope: func() selectors.ExchangeScope { return selectors. NewExchangeBackup(). - EventCalendars([]string{user}, []string{DefaultCalendar})[0] + EventCalendars([]string{user}, []string{DefaultCalendar}, selectors.PrefixMatch())[0] }, }, { name: "Default Mail", @@ -335,7 +339,7 @@ func (suite *ServiceFunctionsIntegrationSuite) TestCollectContainers() { getScope: func() selectors.ExchangeScope { return selectors. NewExchangeBackup(). - MailFolders([]string{user}, []string{DefaultMailFolder})[0] + MailFolders([]string{user}, []string{DefaultMailFolder}, selectors.PrefixMatch())[0] }, }, } diff --git a/src/internal/connector/graph_connector_helper_test.go b/src/internal/connector/graph_connector_helper_test.go index ae06014c7..bafecbd64 100644 --- a/src/internal/connector/graph_connector_helper_test.go +++ b/src/internal/connector/graph_connector_helper_test.go @@ -695,23 +695,21 @@ func makeExchangeBackupSel( pth := mustParsePath(t, p, false) require.Equal(t, path.ExchangeService.String(), pth.Service().String()) + builder := sel.MailFolders + switch pth.Category() { - case path.EmailCategory: - toInclude = append(toInclude, sel.MailFolders( - []string{pth.ResourceOwner()}, - []string{backupInputFromPath(pth).String()}, - )) case path.ContactsCategory: - toInclude = append(toInclude, sel.ContactFolders( - []string{pth.ResourceOwner()}, - []string{backupInputFromPath(pth).String()}, - )) + builder = sel.ContactFolders case path.EventsCategory: - toInclude = append(toInclude, sel.EventCalendars( - []string{pth.ResourceOwner()}, - []string{backupInputFromPath(pth).String()}, - )) + builder = sel.EventCalendars + case path.EmailCategory: // already set } + + toInclude = append(toInclude, builder( + []string{pth.ResourceOwner()}, + []string{backupInputFromPath(pth).String()}, + selectors.PrefixMatch(), + )) } sel.Include(toInclude...) diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 112f4cfa1..7e874a5e9 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -137,7 +137,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { name: suite.user + " Email", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder})) + sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) return sel.Selector }, @@ -146,7 +146,10 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { name: suite.user + " Contacts", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder})) + sel.Include(sel.ContactFolders( + []string{suite.user}, + []string{exchange.DefaultContactFolder}, + selectors.PrefixMatch())) return sel.Selector }, @@ -155,7 +158,7 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() { name: suite.user + " Events", getSelector: func(t *testing.T) selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar})) + sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) return sel.Selector }, @@ -190,7 +193,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailSerializationRegression() { t := suite.T() connector := loadConnector(ctx, t) sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder})) + sel.Include(sel.MailFolders([]string{suite.user}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) collection, err := connector.createCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) @@ -233,7 +236,7 @@ func (suite *GraphConnectorIntegrationSuite) TestContactSerializationRegression( getCollection: func(t *testing.T) []*exchange.Collection { scope := selectors. NewExchangeBackup(). - ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder})[0] + ContactFolders([]string{suite.user}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch())[0] collections, err := connector.createCollections(ctx, scope) require.NoError(t, err) @@ -286,7 +289,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() expected: exchange.DefaultCalendar, getCollection: func(t *testing.T) []*exchange.Collection { sel := selectors.NewExchangeBackup() - sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar})) + sel.Include(sel.EventCalendars([]string{suite.user}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) collections, err := connector.createCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) @@ -298,7 +301,7 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression() expected: "Birthdays", getCollection: func(t *testing.T) []*exchange.Collection { sel := selectors.NewExchangeBackup() - sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"})) + sel.Include(sel.EventCalendars([]string{suite.user}, []string{"Birthdays"}, selectors.PrefixMatch())) collections, err := connector.createCollections(ctx, sel.Scopes()[0]) require.NoError(t, err) @@ -345,7 +348,7 @@ func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() { t := suite.T() connector := loadConnector(ctx, t) sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder})) + sel.Include(sel.MailFolders(selectors.Any(), []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) scopes := sel.DiscreteScopes(connector.GetUsers()) for _, scope := range scopes { @@ -376,6 +379,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMailFetch() { scope: selectors.NewExchangeBackup().MailFolders( []string{userID}, []string{exchange.DefaultMailFolder}, + selectors.PrefixMatch(), )[0], folderNames: map[string]struct{}{ exchange.DefaultMailFolder: {}, @@ -879,19 +883,19 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames t, totalItems, len(deets.Entries), "details entries contains same item count as total successful items restored") - t.Logf("Restore complete\n") + t.Log("Restore complete") } // Run a backup and compare its output with what we put in. backupGC := loadConnector(ctx, t) backupSel := backupSelectorForExpected(t, allExpectedData) - t.Logf("Selective backup of %s\n", backupSel) + t.Log("Selective backup of", backupSel) dcs, err := backupGC.DataCollections(ctx, backupSel) require.NoError(t, err) - t.Logf("Backup enumeration complete\n") + t.Log("Backup enumeration complete") // Pull the data prior to waiting for the status as otherwise it will // deadlock. diff --git a/src/internal/connector/onedrive/collections_test.go b/src/internal/connector/onedrive/collections_test.go index 01512d89e..206f1b0e1 100644 --- a/src/internal/connector/onedrive/collections_test.go +++ b/src/internal/connector/onedrive/collections_test.go @@ -39,8 +39,14 @@ func TestOneDriveCollectionsSuite(t *testing.T) { func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { anyFolder := (&selectors.OneDriveBackup{}).Folders(selectors.Any(), selectors.Any())[0] - tenant := "tenant" - user := "user" + + const ( + tenant = "tenant" + user = "user" + folder = "/folder" + folderSub = "/folder/subfolder" + pkg = "/package" + ) tests := []struct { testCase string @@ -101,8 +107,8 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { driveItem("fileInRoot", testBaseDrivePath, true, false, false), driveItem("folder", testBaseDrivePath, false, true, false), driveItem("package", testBaseDrivePath, false, false, true), - driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false), - driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false), + driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false), + driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false), }, scope: anyFolder, expect: assert.NoError, @@ -111,32 +117,65 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { tenant, user, testBaseDrivePath, - testBaseDrivePath+"/folder", - testBaseDrivePath+"/package", + testBaseDrivePath+folder, + testBaseDrivePath+pkg, ), expectedItemCount: 6, expectedFileCount: 3, expectedContainerCount: 3, }, { - testCase: "match folder selector", + testCase: "contains folder selector", items: []models.DriveItemable{ driveItem("fileInRoot", testBaseDrivePath, true, false, false), driveItem("folder", testBaseDrivePath, false, true, false), - driveItem("subfolder", testBaseDrivePath+"/folder", false, true, false), - driveItem("folder", testBaseDrivePath+"/folder/subfolder", false, true, false), + driveItem("subfolder", testBaseDrivePath+folder, false, true, false), + driveItem("folder", testBaseDrivePath+folderSub, false, true, false), driveItem("package", testBaseDrivePath, false, false, true), - driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false), - driveItem("fileInFolder2", testBaseDrivePath+"/folder/subfolder/folder", true, false, false), - driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false), + driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false), + driveItem("fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), + driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false), }, scope: (&selectors.OneDriveBackup{}).Folders(selectors.Any(), []string{"folder"})[0], expect: assert.NoError, + expectedCollectionPaths: append( + expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+"/folder", + ), + expectedPathAsSlice( + suite.T(), + tenant, + user, + testBaseDrivePath+folderSub+folder, + )..., + ), + expectedItemCount: 4, + expectedFileCount: 2, + expectedContainerCount: 2, + }, + { + testCase: "prefix subfolder selector", + items: []models.DriveItemable{ + driveItem("fileInRoot", testBaseDrivePath, true, false, false), + driveItem("folder", testBaseDrivePath, false, true, false), + driveItem("subfolder", testBaseDrivePath+folder, false, true, false), + driveItem("folder", testBaseDrivePath+folderSub, false, true, false), + driveItem("package", testBaseDrivePath, false, false, true), + driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false), + driveItem("fileInFolder2", testBaseDrivePath+folderSub+folder, true, false, false), + driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false), + }, + scope: (&selectors.OneDriveBackup{}). + Folders(selectors.Any(), []string{"/folder/subfolder"}, selectors.PrefixMatch())[0], + expect: assert.NoError, expectedCollectionPaths: expectedPathAsSlice( suite.T(), tenant, user, - testBaseDrivePath+"/folder", + testBaseDrivePath+folderSub+folder, ), expectedItemCount: 2, expectedFileCount: 1, @@ -147,11 +186,11 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { items: []models.DriveItemable{ driveItem("fileInRoot", testBaseDrivePath, true, false, false), driveItem("folder", testBaseDrivePath, false, true, false), - driveItem("subfolder", testBaseDrivePath+"/folder", false, true, false), + driveItem("subfolder", testBaseDrivePath+folder, false, true, false), driveItem("package", testBaseDrivePath, false, false, true), - driveItem("fileInFolder", testBaseDrivePath+"/folder", true, false, false), - driveItem("fileInSubfolder", testBaseDrivePath+"/folder/subfolder", true, false, false), - driveItem("fileInPackage", testBaseDrivePath+"/package", true, false, false), + driveItem("fileInFolder", testBaseDrivePath+folder, true, false, false), + driveItem("fileInSubfolder", testBaseDrivePath+folderSub, true, false, false), + driveItem("fileInPackage", testBaseDrivePath+pkg, true, false, false), }, scope: (&selectors.OneDriveBackup{}).Folders(selectors.Any(), []string{"folder/subfolder"})[0], expect: assert.NoError, @@ -159,7 +198,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { suite.T(), tenant, user, - testBaseDrivePath+"/folder/subfolder", + testBaseDrivePath+folderSub, ), expectedItemCount: 2, expectedFileCount: 1, @@ -175,10 +214,10 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() { c := NewCollections(tenant, user, tt.scope, &MockGraphService{}, nil) err := c.updateCollections(ctx, "driveID", tt.items) tt.expect(t, err) - assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap)) - assert.Equal(t, tt.expectedItemCount, c.numItems) - assert.Equal(t, tt.expectedFileCount, c.numFiles) - assert.Equal(t, tt.expectedContainerCount, c.numContainers) + assert.Equal(t, len(tt.expectedCollectionPaths), len(c.collectionMap), "collection paths") + assert.Equal(t, tt.expectedItemCount, c.numItems, "item count") + assert.Equal(t, tt.expectedFileCount, c.numFiles, "file count") + assert.Equal(t, tt.expectedContainerCount, c.numContainers, "container count") for _, collPath := range tt.expectedCollectionPaths { assert.Contains(t, c.collectionMap, collPath) } diff --git a/src/internal/operations/backup_test.go b/src/internal/operations/backup_test.go index 52607c25f..8f26f9cd0 100644 --- a/src/internal/operations/backup_test.go +++ b/src/internal/operations/backup_test.go @@ -189,7 +189,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { name: "Integration Exchange.Mail", selectFunc: func() *selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder})) + sel.Include(sel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch())) return &sel.Selector }, }, @@ -197,7 +197,10 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { name: "Integration Exchange.Contacts", selectFunc: func() *selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder})) + sel.Include(sel.ContactFolders( + []string{m365UserID}, + []string{exchange.DefaultContactFolder}, + selectors.PrefixMatch())) return &sel.Selector }, }, @@ -205,7 +208,7 @@ func (suite *BackupOpIntegrationSuite) TestBackup_Run() { name: "Integration Exchange.Events", selectFunc: func() *selectors.Selector { sel := selectors.NewExchangeBackup() - sel.Include(sel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar})) + sel.Include(sel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch())) return &sel.Selector }, }, diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 3f190f2a8..e604ef505 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -180,9 +180,9 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { bsel := selectors.NewExchangeBackup() bsel.Include( - bsel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}), - bsel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder}), - bsel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}), + bsel.MailFolders([]string{m365UserID}, []string{exchange.DefaultMailFolder}, selectors.PrefixMatch()), + bsel.ContactFolders([]string{m365UserID}, []string{exchange.DefaultContactFolder}, selectors.PrefixMatch()), + bsel.EventCalendars([]string{m365UserID}, []string{exchange.DefaultCalendar}, selectors.PrefixMatch()), ) bo, err := NewBackupOperation( diff --git a/src/pkg/filters/filters.go b/src/pkg/filters/filters.go index 8a58021dc..0e6697d2e 100644 --- a/src/pkg/filters/filters.go +++ b/src/pkg/filters/filters.go @@ -62,10 +62,11 @@ func normPathElem(s string) string { // compare values against. Filter.Matches(v) returns // true if Filter.Comparer(filter.target, v) is true. type Filter struct { - Comparator comparator `json:"comparator"` - Target string `json:"target"` // the value to compare against - Targets []string `json:"targets"` // the set of values to compare against, always with "any-match" behavior - Negate bool `json:"negate"` // when true, negate the comparator result + Comparator comparator `json:"comparator"` + Target string `json:"target"` // the value to compare against + Targets []string `json:"targets"` // the set of values to compare + NormalizedTargets []string `json:"normalizedTargets"` // the set of comparable values post normalization + Negate bool `json:"negate"` // when true, negate the comparator result } // ---------------------------------------------------------------------------------------------------- @@ -176,7 +177,7 @@ func PathPrefix(targets []string) Filter { tgts[i] = normPathElem(targets[i]) } - return newSliceFilter(TargetPathPrefix, tgts, false) + return newSliceFilter(TargetPathPrefix, targets, tgts, false) } // NotPathPrefix creates a filter where Compare(v) is true if @@ -195,7 +196,7 @@ func NotPathPrefix(targets []string) Filter { tgts[i] = normPathElem(targets[i]) } - return newSliceFilter(TargetPathPrefix, tgts, true) + return newSliceFilter(TargetPathPrefix, targets, tgts, true) } // PathContains creates a filter where Compare(v) is true if @@ -216,7 +217,7 @@ func PathContains(targets []string) Filter { tgts[i] = normPathElem(targets[i]) } - return newSliceFilter(TargetPathContains, tgts, false) + return newSliceFilter(TargetPathContains, targets, tgts, false) } // NotPathContains creates a filter where Compare(v) is true if @@ -237,7 +238,7 @@ func NotPathContains(targets []string) Filter { tgts[i] = normPathElem(targets[i]) } - return newSliceFilter(TargetPathContains, tgts, true) + return newSliceFilter(TargetPathContains, targets, tgts, true) } // newFilter is the standard filter constructor. @@ -250,11 +251,12 @@ func newFilter(c comparator, target string, negate bool) Filter { } // newSliceFilter constructs filters that contain multiple targets -func newSliceFilter(c comparator, targets []string, negate bool) Filter { +func newSliceFilter(c comparator, targets, normTargets []string, negate bool) Filter { return Filter{ - Comparator: c, - Targets: targets, - Negate: negate, + Comparator: c, + Targets: targets, + NormalizedTargets: normTargets, + Negate: negate, } } @@ -314,7 +316,7 @@ func (f Filter) Compare(input string) bool { targets := []string{f.Target} if hasSlice { - targets = f.Targets + targets = f.NormalizedTargets } for _, tgt := range targets { @@ -415,5 +417,9 @@ func (f Filter) String() string { return "fail" } + if len(f.Targets) > 0 { + return prefixString[f.Comparator] + strings.Join(f.Targets, ",") + } + return prefixString[f.Comparator] + f.Target } diff --git a/src/pkg/filters/filters_test.go b/src/pkg/filters/filters_test.go index 8c9e5c82f..99b94467c 100644 --- a/src/pkg/filters/filters_test.go +++ b/src/pkg/filters/filters_test.go @@ -255,6 +255,33 @@ func (suite *FiltersSuite) TestPathPrefix() { } } +func (suite *FiltersSuite) TestPathPrefix_NormalizedTargets() { + table := []struct { + name string + targets []string + expect []string + }{ + {"Single - no slash", []string{"fA"}, []string{"/fA/"}}, + {"Single - pre slash", []string{"/fA"}, []string{"/fA/"}}, + {"Single - suff slash", []string{"fA/"}, []string{"/fA/"}}, + {"Single - both slashes", []string{"/fA/"}, []string{"/fA/"}}, + {"Multipath - no slash", []string{"fA/fB"}, []string{"/fA/fB/"}}, + {"Multipath - pre slash", []string{"/fA/fB"}, []string{"/fA/fB/"}}, + {"Multipath - suff slash", []string{"fA/fB/"}, []string{"/fA/fB/"}}, + {"Multipath - both slashes", []string{"/fA/fB/"}, []string{"/fA/fB/"}}, + {"Multi input - no slash", []string{"fA", "fB"}, []string{"/fA/", "/fB/"}}, + {"Multi input - pre slash", []string{"/fA", "/fB"}, []string{"/fA/", "/fB/"}}, + {"Multi input - suff slash", []string{"fA/", "fB/"}, []string{"/fA/", "/fB/"}}, + {"Multi input - both slashes", []string{"/fA/", "/fB/"}, []string{"/fA/", "/fB/"}}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + f := filters.PathPrefix(test.targets) + assert.Equal(t, test.expect, f.NormalizedTargets) + }) + } +} + func (suite *FiltersSuite) TestPathContains() { table := []struct { name string @@ -306,3 +333,30 @@ func (suite *FiltersSuite) TestPathContains() { }) } } + +func (suite *FiltersSuite) TestPathContains_NormalizedTargets() { + table := []struct { + name string + targets []string + expect []string + }{ + {"Single - no slash", []string{"fA"}, []string{"/fA/"}}, + {"Single - pre slash", []string{"/fA"}, []string{"/fA/"}}, + {"Single - suff slash", []string{"fA/"}, []string{"/fA/"}}, + {"Single - both slashes", []string{"/fA/"}, []string{"/fA/"}}, + {"Multipath - no slash", []string{"fA/fB"}, []string{"/fA/fB/"}}, + {"Multipath - pre slash", []string{"/fA/fB"}, []string{"/fA/fB/"}}, + {"Multipath - suff slash", []string{"fA/fB/"}, []string{"/fA/fB/"}}, + {"Multipath - both slashes", []string{"/fA/fB/"}, []string{"/fA/fB/"}}, + {"Multi input - no slash", []string{"fA", "fB"}, []string{"/fA/", "/fB/"}}, + {"Multi input - pre slash", []string{"/fA", "/fB"}, []string{"/fA/", "/fB/"}}, + {"Multi input - suff slash", []string{"fA/", "fB/"}, []string{"/fA/", "/fB/"}}, + {"Multi input - both slashes", []string{"/fA/", "/fB/"}, []string{"/fA/", "/fB/"}}, + } + for _, test := range table { + suite.T().Run(test.name, func(t *testing.T) { + f := filters.PathContains(test.targets) + assert.Equal(t, test.expect, f.NormalizedTargets) + }) + } +} diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index e6e339807..09609627c 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -190,11 +190,14 @@ func (s *exchange) Contacts(users, folders, contacts []string) []ExchangeScope { // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) ContactFolders(users, folders []string, opts ...option) []ExchangeScope { - scopes := []ExchangeScope{} + var ( + scopes = []ExchangeScope{} + os = append([]option{pathType()}, opts...) + ) scopes = append( scopes, - makeScope[ExchangeScope](ExchangeContactFolder, users, folders, opts...), + makeScope[ExchangeScope](ExchangeContactFolder, users, folders, os...), ) return scopes @@ -222,11 +225,14 @@ func (s *exchange) Events(users, calendars, events []string) []ExchangeScope { // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) EventCalendars(users, events []string, opts ...option) []ExchangeScope { - scopes := []ExchangeScope{} + var ( + scopes = []ExchangeScope{} + os = append([]option{pathType()}, opts...) + ) scopes = append( scopes, - makeScope[ExchangeScope](ExchangeEventCalendar, users, events, opts...), + makeScope[ExchangeScope](ExchangeEventCalendar, users, events, os...), ) return scopes @@ -253,11 +259,14 @@ func (s *exchange) Mails(users, folders, mails []string) []ExchangeScope { // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *exchange) MailFolders(users, folders []string, opts ...option) []ExchangeScope { - scopes := []ExchangeScope{} + var ( + scopes = []ExchangeScope{} + os = append([]option{pathType()}, opts...) + ) scopes = append( scopes, - makeScope[ExchangeScope](ExchangeMailFolder, users, folders, opts...), + makeScope[ExchangeScope](ExchangeMailFolder, users, folders, os...), ) return scopes @@ -624,8 +633,13 @@ func (s ExchangeScope) Get(cat exchangeCategory) []string { } // sets a value by category to the scope. Only intended for internal use. -func (s ExchangeScope) set(cat exchangeCategory, v []string) ExchangeScope { - return set(s, cat, v) +func (s ExchangeScope) set(cat exchangeCategory, v []string, opts ...option) ExchangeScope { + os := []option{} + if cat == ExchangeContactFolder || cat == ExchangeEventCalendar || cat == ExchangeMailFolder { + os = append(os, pathType()) + } + + return set(s, cat, v, append(os, opts...)...) } // setDefaults ensures that contact folder, mail folder, and user category diff --git a/src/pkg/selectors/exchange_test.go b/src/pkg/selectors/exchange_test.go index cafeec17c..1b89b02ba 100644 --- a/src/pkg/selectors/exchange_test.go +++ b/src/pkg/selectors/exchange_test.go @@ -773,12 +773,13 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesInfo() { func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() { const ( usr = "userID" - fld = "mailFolder" + fld1 = "mailFolder" + fld2 = "subFolder" mail = "mailID" ) var ( - pth = stubPath(suite.T(), usr, []string{fld, mail}, path.EmailCategory) + pth = stubPath(suite.T(), usr, []string{fld1, fld2, mail}, path.EmailCategory) short = "thisisahashofsomekind" es = NewExchangeRestore() ) @@ -796,13 +797,14 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() { {"one of multiple users", es.Users([]string{"smarf", usr}), "", assert.True}, {"all folders", es.MailFolders(Any(), Any()), "", assert.True}, {"no folders", es.MailFolders(Any(), None()), "", assert.False}, - {"matching folder", es.MailFolders(Any(), []string{fld}), "", assert.True}, + {"matching folder", es.MailFolders(Any(), []string{fld1}), "", assert.True}, + {"incomplete matching folder", es.MailFolders(Any(), []string{"mail"}), "", assert.False}, {"non-matching folder", es.MailFolders(Any(), []string{"smarf"}), "", assert.False}, - // This test validates that folders that match a substring of the scope are not included (bugfix 1) - {"non-matching folder substring", es.MailFolders(Any(), []string{fld + "_suffix"}), "", assert.False}, - {"matching folder prefix", es.MailFolders(Any(), []string{"mailF"}), "", assert.True}, + {"non-matching folder substring", es.MailFolders(Any(), []string{fld1 + "_suffix"}), "", assert.False}, + {"matching folder prefix", es.MailFolders(Any(), []string{fld1}, PrefixMatch()), "", assert.True}, + {"incomplete folder prefix", es.MailFolders(Any(), []string{"mail"}, PrefixMatch()), "", assert.False}, {"matching folder substring", es.MailFolders(Any(), []string{"Folder"}), "", assert.False}, - {"one of multiple folders", es.MailFolders(Any(), []string{"smarf", fld}), "", assert.True}, + {"one of multiple folders", es.MailFolders(Any(), []string{"smarf", fld2}), "", assert.True}, {"all mail", es.Mails(Any(), Any(), Any()), "", assert.True}, {"no mail", es.Mails(Any(), Any(), None()), "", assert.False}, {"matching mail", es.Mails(Any(), Any(), []string{mail}), "", assert.True}, @@ -986,6 +988,26 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() { }, arr(contactInSubFolder), }, + { + "only match contactInSubFolder by prefix", + makeDeets(contactInSubFolder, contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore() + er.Include(er.ContactFolders([]string{"uid"}, []string{"cfld1/cfld2"}, PrefixMatch())) + return er + }, + arr(contactInSubFolder), + }, + { + "only match contactInSubFolder by leaf folder", + makeDeets(contactInSubFolder, contact, event, mail), + func() *ExchangeRestore { + er := NewExchangeRestore() + er.Include(er.ContactFolders([]string{"uid"}, []string{"cfld2"})) + return er + }, + arr(contactInSubFolder), + }, { "only match event", makeDeets(contact, event, mail), diff --git a/src/pkg/selectors/helpers_test.go b/src/pkg/selectors/helpers_test.go index 93732749e..138549ce5 100644 --- a/src/pkg/selectors/helpers_test.go +++ b/src/pkg/selectors/helpers_test.go @@ -171,11 +171,11 @@ func setScopesToDefault[T scopeT](ts []T) []T { return ts } -// calls assert.Equal(t, getCatValue(sc, k)[0], v) on each k:v pair in the map +// calls assert.Equal(t, v, getCatValue(sc, k)[0]) on each k:v pair in the map func scopeMustHave[T scopeT](t *testing.T, sc T, m map[categorizer]string) { for k, v := range m { t.Run(k.String(), func(t *testing.T) { - assert.Equal(t, getCatValue(sc, k), split(v), "Key: %s", k) + assert.Equal(t, split(v), getCatValue(sc, k), "Key: %s", k) }) } } diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index b75b551e0..c868bad27 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -180,11 +180,14 @@ func (s *oneDrive) Users(users []string) []OneDriveScope { // If any slice contains selectors.None, that slice is reduced to [selectors.None] // If any slice is empty, it defaults to [selectors.None] func (s *oneDrive) Folders(users, folders []string, opts ...option) []OneDriveScope { - scopes := []OneDriveScope{} + var ( + scopes = []OneDriveScope{} + os = append([]option{pathType()}, opts...) + ) scopes = append( scopes, - makeScope[OneDriveScope](OneDriveFolder, users, folders, opts...), + makeScope[OneDriveScope](OneDriveFolder, users, folders, os...), ) return scopes @@ -420,8 +423,13 @@ func (s OneDriveScope) Get(cat oneDriveCategory) []string { } // sets a value by category to the scope. Only intended for internal use. -func (s OneDriveScope) set(cat oneDriveCategory, v []string) OneDriveScope { - return set(s, cat, v) +func (s OneDriveScope) set(cat oneDriveCategory, v []string, opts ...option) OneDriveScope { + os := []option{} + if cat == OneDriveFolder { + os = append(os, pathType()) + } + + return set(s, cat, v, append(os, opts...)...) } // setDefaults ensures that user scopes express `AnyTgt` for their child category types. diff --git a/src/pkg/selectors/scopes.go b/src/pkg/selectors/scopes.go index 2c0931219..291f2657d 100644 --- a/src/pkg/selectors/scopes.go +++ b/src/pkg/selectors/scopes.go @@ -158,16 +158,13 @@ func makeScope[T scopeT]( resources, vs []string, opts ...option, ) T { - sc := scopeConfig{} - - for _, opt := range opts { - opt(&sc) - } + sc := &scopeConfig{} + sc.populate(opts...) s := T{ scopeKeyCategory: filters.Identity(cat.String()), scopeKeyDataType: filters.Identity(cat.leafCat().String()), - cat.String(): filterize(sc, vs...), + cat.String(): filterize(*sc, vs...), cat.rootCat().String(): filterize(scopeConfig{}, resources...), } @@ -194,17 +191,18 @@ func makeFilterScope[T scopeT]( // --------------------------------------------------------------------------- // matches returns true if the category is included in the scope's -// data type, and the target string is included in the scope. -func matches[T scopeT, C categoryT](s T, cat C, target string) bool { +// data type, and the input string passes the scope's filter for +// that category. +func matches[T scopeT, C categoryT](s T, cat C, inpt string) bool { if !typeAndCategoryMatches(cat, s.categorizer()) { return false } - if len(target) == 0 { + if len(inpt) == 0 { return false } - return s[cat.String()].Compare(target) + return s[cat.String()].Compare(inpt) } // getCategory returns the scope's category value. @@ -222,18 +220,26 @@ func getFilterCategory[T scopeT](s T) string { // delimiter, and returns the slice. If s[cat] is nil, returns // None(). func getCatValue[T scopeT](s T, cat categorizer) []string { - v, ok := s[cat.String()] + filt, ok := s[cat.String()] if !ok { return None() } - return split(v.Target) + if len(filt.Targets) > 0 { + return filt.Targets + } + + return split(filt.Target) } // set sets a value by category to the scope. Only intended for internal // use, not for exporting to callers. -func set[T scopeT](s T, cat categorizer, v []string) T { - s[cat.String()] = filterize(scopeConfig{}, v...) +func set[T scopeT](s T, cat categorizer, v []string, opts ...option) T { + sc := &scopeConfig{} + sc.populate(opts...) + + s[cat.String()] = filterize(*sc, v...) + return s } @@ -244,7 +250,7 @@ func isNoneTarget[T scopeT, C categoryT](s T, cat C) bool { return false } - return s[cat.String()].Target == NoneTgt + return s[cat.String()].Comparator == filters.Fails } // returns true if the category is included in the scope's category type, @@ -254,7 +260,7 @@ func isAnyTarget[T scopeT, C categoryT](s T, cat C) bool { return false } - return s[cat.String()].Target == AnyTgt + return s[cat.String()].Comparator == filters.Passes } // reduce filters the entries in the details to only those that match the @@ -436,7 +442,6 @@ func matchesPathValues[T scopeT, C categoryT]( var ( match bool isLeaf = c.isLeaf() - isRoot = c == c.rootCat() ) switch { @@ -445,19 +450,7 @@ func matchesPathValues[T scopeT, C categoryT]( case isLeaf && len(shortRef) > 0: match = matches(sc, cc, pathVal) || matches(sc, cc, shortRef) - // Folder category - checks if any target folder is a prefix of the path folders. - // Assumes (correctly) that we need to split the targets and re-compose them into - // individual prefix matchers. - // TODO: assumes all folders require prefix matchers. Users can now specify whether - // the folder filter is a prefix match or not. We should respect that configuration. - case !isLeaf && !isRoot: - for _, tgt := range getCatValue(sc, c) { - if filters.Prefix(tgt).Compare(pathVal) { - match = true - break - } - } - + // all other categories (root, folder, etc) just need to pass the filter default: match = matches(sc, cc, pathVal) } diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 50d81241d..9ed0dcff8 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -338,11 +338,18 @@ func addToSet(set []string, v []string) []string { // --------------------------------------------------------------------------- type scopeConfig struct { + usePathFilter bool usePrefixFilter bool } type option func(*scopeConfig) +func (sc *scopeConfig) populate(opts ...option) { + for _, opt := range opts { + opt(sc) + } +} + // PrefixMatch ensures the selector uses a Prefix comparator, instead // of contains or equals. Will not override a default Any() or None() // comparator. @@ -352,6 +359,15 @@ func PrefixMatch() option { } } +// pathType is an internal-facing option. It is assumed that scope +// constructors will provide the pathType option whenever a folder- +// level scope (ie, a scope that compares path hierarchies) is created. +func pathType() option { + return func(sc *scopeConfig) { + sc.usePathFilter = true + } +} + // --------------------------------------------------------------------------- // helpers // --------------------------------------------------------------------------- @@ -408,6 +424,14 @@ func filterize(sc scopeConfig, s ...string) filters.Filter { return passAny } + if sc.usePathFilter { + if sc.usePrefixFilter { + return filters.PathPrefix(s) + } + + return filters.PathContains(s) + } + if sc.usePrefixFilter { return filters.Prefix(join(s...)) }