move selectorsToReason into selectors (#4006)

#### Does this PR need a docs update or release note?

- [x]  No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #3993

#### Test Plan

- [x]  Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-08-11 14:15:23 -06:00 committed by GitHub
parent 0a947386e6
commit 19bf0fdf7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 580 additions and 38 deletions

View File

@ -326,11 +326,17 @@ func (op *BackupOperation) do(
detailsStore streamstore.Streamer,
backupID model.StableID,
) (*details.Builder, error) {
var (
reasons = selectorToReasons(op.account.ID(), op.Selectors, false)
fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors)
lastBackupVersion = version.NoBackup
)
lastBackupVersion := version.NoBackup
reasons, err := op.Selectors.Reasons(op.account.ID(), false)
if err != nil {
return nil, clues.Wrap(err, "getting reasons")
}
fallbackReasons, err := makeFallbackReasons(op.account.ID(), op.Selectors)
if err != nil {
return nil, clues.Wrap(err, "getting fallback reasons")
}
logger.Ctx(ctx).With(
"control_options", op.Options,
@ -424,13 +430,14 @@ func (op *BackupOperation) do(
return deets, nil
}
func makeFallbackReasons(tenant string, sel selectors.Selector) []identity.Reasoner {
func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) {
if sel.PathService() != path.SharePointService &&
sel.DiscreteOwner != sel.DiscreteOwnerName {
return selectorToReasons(tenant, sel, true)
return sel.Reasons(tenant, true)
}
return nil
// return nil for fallback reasons since a nil value will no-op.
return nil, nil
}
// checker to see if conditions are correct for incremental backup behavior such as
@ -472,35 +479,6 @@ func produceBackupDataCollections(
// Consumer funcs
// ---------------------------------------------------------------------------
func selectorToReasons(
tenant string,
sel selectors.Selector,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := []identity.Reasoner{}
pcs, err := sel.PathCategories()
if err != nil {
// This is technically safe, it's just that the resulting backup won't be
// usable as a base for future incremental backups.
return nil
}
owner := sel.DiscreteOwner
if useOwnerNameForID {
owner = sel.DiscreteOwnerName
}
for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} {
for _, cat := range sl {
reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat))
}
}
return reasons
}
// calls kopia to backup the collections of data
func consumeBackupCollections(
ctx context.Context,

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -43,6 +44,7 @@ type (
var (
_ Reducer = &ExchangeRestore{}
_ pathCategorier = &ExchangeRestore{}
_ reasoner = &ExchangeRestore{}
)
// NewExchange produces a new Selector with the service set to ServiceExchange.
@ -122,6 +124,13 @@ func (s exchange) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s exchange) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -7,6 +7,7 @@ import (
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -40,6 +41,7 @@ type (
var (
_ Reducer = &GroupsRestore{}
_ pathCategorier = &GroupsRestore{}
_ reasoner = &GroupsRestore{}
)
// NewGroupsBackup produces a new Selector with the service set to ServiceGroups.
@ -119,6 +121,13 @@ func (s groups) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s groups) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -167,6 +167,8 @@ func (s mockScope) PlainString() string { return plainString(s) }
// selectors
// ---------------------------------------------------------------------------
var _ servicerCategorizerProvider = &mockSel{}
type mockSel struct {
Selector
}
@ -183,6 +185,14 @@ func stubSelector(resourceOwners []string) mockSel {
}
}
func (m mockSel) PathCategories() selectorPathCategories {
return selectorPathCategories{
Includes: []path.CategoryType{pathCatStub},
Excludes: []path.CategoryType{pathCatStub},
Filters: []path.CategoryType{pathCatStub},
}
}
// ---------------------------------------------------------------------------
// helper funcs
// ---------------------------------------------------------------------------

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var (
_ Reducer = &OneDriveRestore{}
_ pathCategorier = &OneDriveRestore{}
_ reasoner = &OneDriveRestore{}
)
// NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive.
@ -121,6 +123,13 @@ func (s oneDrive) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s oneDrive) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------

View File

@ -0,0 +1,91 @@
package selectors
import (
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/path"
)
// ---------------------------------------------------------------------------
// reasoner interface compliance
// ---------------------------------------------------------------------------
var _ identity.Reasoner = &backupReason{}
type backupReason struct {
category path.CategoryType
resource string
service path.ServiceType
tenant string
}
func (br backupReason) Tenant() string {
return br.tenant
}
func (br backupReason) ProtectedResource() string {
return br.resource
}
func (br backupReason) Service() path.ServiceType {
return br.service
}
func (br backupReason) Category() path.CategoryType {
return br.category
}
func (br backupReason) SubtreePath() (path.Path, error) {
return path.ServicePrefix(
br.tenant,
br.resource,
br.service,
br.category)
}
func (br backupReason) key() string {
return br.category.String() + br.resource + br.service.String() + br.tenant
}
// ---------------------------------------------------------------------------
// common transformer
// ---------------------------------------------------------------------------
type servicerCategorizerProvider interface {
pathServicer
pathCategorier
idname.Provider
}
func reasonsFor(
sel servicerCategorizerProvider,
tenantID string,
useOwnerNameForID bool,
) []identity.Reasoner {
service := sel.PathService()
reasons := map[string]identity.Reasoner{}
resource := sel.ID()
if useOwnerNameForID {
resource = sel.Name()
}
pc := sel.PathCategories()
for _, sl := range [][]path.CategoryType{pc.Includes, pc.Filters} {
for _, cat := range sl {
br := backupReason{
category: cat,
resource: resource,
service: service,
tenant: tenantID,
}
reasons[br.key()] = br
}
}
return maps.Values(reasons)
}

View File

@ -0,0 +1,406 @@
package selectors
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/path"
)
type ReasonsUnitSuite struct {
tester.Suite
}
func TestReasonsUnitSuite(t *testing.T) {
suite.Run(t, &ReasonsUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "use name",
sel: func() ExchangeRestore {
sel := NewExchangeRestore([]string{"timbo"})
sel.Include(sel.MailFolders(Any()))
plainSel := sel.SetDiscreteOwnerIDName("timbo", "timbubba")
sel, err := plainSel.ToExchangeRestore()
require.NoError(suite.T(), err, clues.ToCore(err))
return *sel
},
useName: true,
expect: []expect{
{
tenant: tenantID,
resource: "timbubba",
category: email,
service: exchange,
subtreePath: stpFor("timbubba", email, exchange),
},
},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}
func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() {
var (
tenantID = "tid"
exchange = path.ExchangeService.String()
email = path.EmailCategory.String()
contacts = path.ContactsCategory.String()
)
type expect struct {
tenant string
resource string
category string
service string
subtreePath string
subtreePathHadErr bool
}
stpFor := func(resource, category, service string) string {
return path.Builder{}.Append(tenantID, service, resource, category).String()
}
table := []struct {
name string
sel func() ExchangeRestore
useName bool
expect []expect
}{
{
name: "no scopes",
sel: func() ExchangeRestore {
return *NewExchangeRestore([]string{"timbo"})
},
expect: []expect{},
},
{
name: "only includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"bubba"})
sel.Include(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "bubba",
category: email,
service: exchange,
subtreePath: stpFor("bubba", email, exchange),
},
},
},
{
name: "only filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"tachoma dhaume"})
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "tachoma dhaume",
category: email,
service: exchange,
subtreePath: stpFor("tachoma dhaume", email, exchange),
},
},
},
{
name: "duplicate includes and filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"vyng vang zoombah"})
sel.Include(sel.MailFolders(Any()))
sel.Filter(sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "vyng vang zoombah",
category: email,
service: exchange,
subtreePath: stpFor("vyng vang zoombah", email, exchange),
},
},
},
{
name: "duplicate includes",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"fat billie"})
sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "fat billie",
category: email,
service: exchange,
subtreePath: stpFor("fat billie", email, exchange),
},
},
},
{
name: "duplicate filters",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"seathane"})
sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "seathane",
category: email,
service: exchange,
subtreePath: stpFor("seathane", email, exchange),
},
},
},
{
name: "no duplicates",
sel: func() ExchangeRestore {
sel := *NewExchangeRestore([]string{"perell"})
sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any()))
return sel
},
expect: []expect{
{
tenant: tenantID,
resource: "perell",
category: email,
service: exchange,
subtreePath: stpFor("perell", email, exchange),
},
{
tenant: tenantID,
resource: "perell",
category: contacts,
service: exchange,
subtreePath: stpFor("perell", contacts, exchange),
},
},
},
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
results := []expect{}
rs := reasonsFor(test.sel(), tenantID, test.useName)
for _, r := range rs {
stp, err := r.SubtreePath()
t.Log("stp err", err)
stpStr := ""
if stp != nil {
stpStr = stp.String()
}
results = append(results, expect{
tenant: r.Tenant(),
resource: r.ProtectedResource(),
service: r.Service().String(),
category: r.Category().String(),
subtreePath: stpStr,
subtreePathHadErr: err != nil,
})
}
assert.ElementsMatch(t, test.expect, results)
})
}
}

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -87,6 +88,14 @@ type pathCategorier interface {
PathCategories() selectorPathCategories
}
type pathServicer interface {
PathService() path.ServiceType
}
type reasoner interface {
Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner
}
// ---------------------------------------------------------------------------
// Selector
// ---------------------------------------------------------------------------
@ -273,7 +282,7 @@ func (s Selector) Reduce(
return r.Reduce(ctx, deets, errs), nil
}
// returns the sets of path categories identified in each scope set.
// PathCategories returns the sets of path categories identified in each scope set.
func (s Selector) PathCategories() (selectorPathCategories, error) {
ro, err := selectorAsIface[pathCategorier](s)
if err != nil {
@ -283,6 +292,18 @@ func (s Selector) PathCategories() (selectorPathCategories, error) {
return ro.PathCategories(), nil
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s Selector) Reasons(tenantID string, useOwnerNameForID bool) ([]identity.Reasoner, error) {
ro, err := selectorAsIface[reasoner](s)
if err != nil {
return nil, err
}
return ro.Reasons(tenantID, useOwnerNameForID), nil
}
// transformer for arbitrary selector interfaces
func selectorAsIface[T any](s Selector) (T, error) {
var (

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/backup/identity"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
@ -42,6 +43,7 @@ type (
var (
_ Reducer = &SharePointRestore{}
_ pathCategorier = &SharePointRestore{}
_ reasoner = &SharePointRestore{}
)
// NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint.
@ -121,6 +123,13 @@ func (s sharePoint) PathCategories() selectorPathCategories {
}
}
// Reasons returns a deduplicated set of the backup reasons produced
// using the selector's discrete owner and each scopes' service and
// category types.
func (s sharePoint) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner {
return reasonsFor(s, tenantID, useOwnerNameForID)
}
// ---------------------------------------------------------------------------
// Stringers and Concealers
// ---------------------------------------------------------------------------