diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 0c9a468a9..d3d4f0a9f 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -326,11 +326,17 @@ func (op *BackupOperation) do( detailsStore streamstore.Streamer, backupID model.StableID, ) (*details.Builder, error) { - var ( - reasons = selectorToReasons(op.account.ID(), op.Selectors, false) - fallbackReasons = makeFallbackReasons(op.account.ID(), op.Selectors) - lastBackupVersion = version.NoBackup - ) + lastBackupVersion := version.NoBackup + + reasons, err := op.Selectors.Reasons(op.account.ID(), false) + if err != nil { + return nil, clues.Wrap(err, "getting reasons") + } + + fallbackReasons, err := makeFallbackReasons(op.account.ID(), op.Selectors) + if err != nil { + return nil, clues.Wrap(err, "getting fallback reasons") + } logger.Ctx(ctx).With( "control_options", op.Options, @@ -424,13 +430,14 @@ func (op *BackupOperation) do( return deets, nil } -func makeFallbackReasons(tenant string, sel selectors.Selector) []identity.Reasoner { +func makeFallbackReasons(tenant string, sel selectors.Selector) ([]identity.Reasoner, error) { if sel.PathService() != path.SharePointService && sel.DiscreteOwner != sel.DiscreteOwnerName { - return selectorToReasons(tenant, sel, true) + return sel.Reasons(tenant, true) } - return nil + // return nil for fallback reasons since a nil value will no-op. + return nil, nil } // checker to see if conditions are correct for incremental backup behavior such as @@ -472,35 +479,6 @@ func produceBackupDataCollections( // Consumer funcs // --------------------------------------------------------------------------- -func selectorToReasons( - tenant string, - sel selectors.Selector, - useOwnerNameForID bool, -) []identity.Reasoner { - service := sel.PathService() - reasons := []identity.Reasoner{} - - pcs, err := sel.PathCategories() - if err != nil { - // This is technically safe, it's just that the resulting backup won't be - // usable as a base for future incremental backups. - return nil - } - - owner := sel.DiscreteOwner - if useOwnerNameForID { - owner = sel.DiscreteOwnerName - } - - for _, sl := range [][]path.CategoryType{pcs.Includes, pcs.Filters} { - for _, cat := range sl { - reasons = append(reasons, kopia.NewReason(tenant, owner, service, cat)) - } - } - - return reasons -} - // calls kopia to backup the collections of data func consumeBackupCollections( ctx context.Context, diff --git a/src/pkg/selectors/exchange.go b/src/pkg/selectors/exchange.go index 245909161..9a6b87638 100644 --- a/src/pkg/selectors/exchange.go +++ b/src/pkg/selectors/exchange.go @@ -9,6 +9,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" @@ -43,6 +44,7 @@ type ( var ( _ Reducer = &ExchangeRestore{} _ pathCategorier = &ExchangeRestore{} + _ reasoner = &ExchangeRestore{} ) // NewExchange produces a new Selector with the service set to ServiceExchange. @@ -122,6 +124,13 @@ func (s exchange) PathCategories() selectorPathCategories { } } +// Reasons returns a deduplicated set of the backup reasons produced +// using the selector's discrete owner and each scopes' service and +// category types. +func (s exchange) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner { + return reasonsFor(s, tenantID, useOwnerNameForID) +} + // --------------------------------------------------------------------------- // Stringers and Concealers // --------------------------------------------------------------------------- diff --git a/src/pkg/selectors/groups.go b/src/pkg/selectors/groups.go index cc4a7ebfd..7adf5398c 100644 --- a/src/pkg/selectors/groups.go +++ b/src/pkg/selectors/groups.go @@ -7,6 +7,7 @@ import ( "github.com/alcionai/clues" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" ) @@ -40,6 +41,7 @@ type ( var ( _ Reducer = &GroupsRestore{} _ pathCategorier = &GroupsRestore{} + _ reasoner = &GroupsRestore{} ) // NewGroupsBackup produces a new Selector with the service set to ServiceGroups. @@ -119,6 +121,13 @@ func (s groups) PathCategories() selectorPathCategories { } } +// Reasons returns a deduplicated set of the backup reasons produced +// using the selector's discrete owner and each scopes' service and +// category types. +func (s groups) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner { + return reasonsFor(s, tenantID, useOwnerNameForID) +} + // --------------------------------------------------------------------------- // Stringers and Concealers // --------------------------------------------------------------------------- diff --git a/src/pkg/selectors/helpers_test.go b/src/pkg/selectors/helpers_test.go index 82e68791e..618c38ecc 100644 --- a/src/pkg/selectors/helpers_test.go +++ b/src/pkg/selectors/helpers_test.go @@ -167,6 +167,8 @@ func (s mockScope) PlainString() string { return plainString(s) } // selectors // --------------------------------------------------------------------------- +var _ servicerCategorizerProvider = &mockSel{} + type mockSel struct { Selector } @@ -183,6 +185,14 @@ func stubSelector(resourceOwners []string) mockSel { } } +func (m mockSel) PathCategories() selectorPathCategories { + return selectorPathCategories{ + Includes: []path.CategoryType{pathCatStub}, + Excludes: []path.CategoryType{pathCatStub}, + Filters: []path.CategoryType{pathCatStub}, + } +} + // --------------------------------------------------------------------------- // helper funcs // --------------------------------------------------------------------------- diff --git a/src/pkg/selectors/onedrive.go b/src/pkg/selectors/onedrive.go index 057634215..5d1538a89 100644 --- a/src/pkg/selectors/onedrive.go +++ b/src/pkg/selectors/onedrive.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" @@ -42,6 +43,7 @@ type ( var ( _ Reducer = &OneDriveRestore{} _ pathCategorier = &OneDriveRestore{} + _ reasoner = &OneDriveRestore{} ) // NewOneDriveBackup produces a new Selector with the service set to ServiceOneDrive. @@ -121,6 +123,13 @@ func (s oneDrive) PathCategories() selectorPathCategories { } } +// Reasons returns a deduplicated set of the backup reasons produced +// using the selector's discrete owner and each scopes' service and +// category types. +func (s oneDrive) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner { + return reasonsFor(s, tenantID, useOwnerNameForID) +} + // --------------------------------------------------------------------------- // Stringers and Concealers // --------------------------------------------------------------------------- diff --git a/src/pkg/selectors/reasons.go b/src/pkg/selectors/reasons.go new file mode 100644 index 000000000..a44cf5c49 --- /dev/null +++ b/src/pkg/selectors/reasons.go @@ -0,0 +1,91 @@ +package selectors + +import ( + "golang.org/x/exp/maps" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/pkg/backup/identity" + "github.com/alcionai/corso/src/pkg/path" +) + +// --------------------------------------------------------------------------- +// reasoner interface compliance +// --------------------------------------------------------------------------- + +var _ identity.Reasoner = &backupReason{} + +type backupReason struct { + category path.CategoryType + resource string + service path.ServiceType + tenant string +} + +func (br backupReason) Tenant() string { + return br.tenant +} + +func (br backupReason) ProtectedResource() string { + return br.resource +} + +func (br backupReason) Service() path.ServiceType { + return br.service +} + +func (br backupReason) Category() path.CategoryType { + return br.category +} + +func (br backupReason) SubtreePath() (path.Path, error) { + return path.ServicePrefix( + br.tenant, + br.resource, + br.service, + br.category) +} + +func (br backupReason) key() string { + return br.category.String() + br.resource + br.service.String() + br.tenant +} + +// --------------------------------------------------------------------------- +// common transformer +// --------------------------------------------------------------------------- + +type servicerCategorizerProvider interface { + pathServicer + pathCategorier + idname.Provider +} + +func reasonsFor( + sel servicerCategorizerProvider, + tenantID string, + useOwnerNameForID bool, +) []identity.Reasoner { + service := sel.PathService() + reasons := map[string]identity.Reasoner{} + + resource := sel.ID() + if useOwnerNameForID { + resource = sel.Name() + } + + pc := sel.PathCategories() + + for _, sl := range [][]path.CategoryType{pc.Includes, pc.Filters} { + for _, cat := range sl { + br := backupReason{ + category: cat, + resource: resource, + service: service, + tenant: tenantID, + } + + reasons[br.key()] = br + } + } + + return maps.Values(reasons) +} diff --git a/src/pkg/selectors/reasons_test.go b/src/pkg/selectors/reasons_test.go new file mode 100644 index 000000000..92ff3b807 --- /dev/null +++ b/src/pkg/selectors/reasons_test.go @@ -0,0 +1,406 @@ +package selectors + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/path" +) + +type ReasonsUnitSuite struct { + tester.Suite +} + +func TestReasonsUnitSuite(t *testing.T) { + suite.Run(t, &ReasonsUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *ReasonsUnitSuite) TestReasonsFor_thorough() { + var ( + tenantID = "tid" + exchange = path.ExchangeService.String() + email = path.EmailCategory.String() + contacts = path.ContactsCategory.String() + ) + + type expect struct { + tenant string + resource string + category string + service string + subtreePath string + subtreePathHadErr bool + } + + stpFor := func(resource, category, service string) string { + return path.Builder{}.Append(tenantID, service, resource, category).String() + } + + table := []struct { + name string + sel func() ExchangeRestore + useName bool + expect []expect + }{ + { + name: "no scopes", + sel: func() ExchangeRestore { + return *NewExchangeRestore([]string{"timbo"}) + }, + expect: []expect{}, + }, + { + name: "use name", + sel: func() ExchangeRestore { + sel := NewExchangeRestore([]string{"timbo"}) + sel.Include(sel.MailFolders(Any())) + plainSel := sel.SetDiscreteOwnerIDName("timbo", "timbubba") + + sel, err := plainSel.ToExchangeRestore() + require.NoError(suite.T(), err, clues.ToCore(err)) + + return *sel + }, + useName: true, + expect: []expect{ + { + tenant: tenantID, + resource: "timbubba", + category: email, + service: exchange, + subtreePath: stpFor("timbubba", email, exchange), + }, + }, + }, + { + name: "only includes", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"bubba"}) + sel.Include(sel.MailFolders(Any())) + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "bubba", + category: email, + service: exchange, + subtreePath: stpFor("bubba", email, exchange), + }, + }, + }, + { + name: "only filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"tachoma dhaume"}) + sel.Filter(sel.MailFolders(Any())) + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "tachoma dhaume", + category: email, + service: exchange, + subtreePath: stpFor("tachoma dhaume", email, exchange), + }, + }, + }, + { + name: "duplicate includes and filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"vyng vang zoombah"}) + sel.Include(sel.MailFolders(Any())) + sel.Filter(sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "vyng vang zoombah", + category: email, + service: exchange, + subtreePath: stpFor("vyng vang zoombah", email, exchange), + }, + }, + }, + { + name: "duplicate includes", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"fat billie"}) + sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "fat billie", + category: email, + service: exchange, + subtreePath: stpFor("fat billie", email, exchange), + }, + }, + }, + { + name: "duplicate filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"seathane"}) + sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "seathane", + category: email, + service: exchange, + subtreePath: stpFor("seathane", email, exchange), + }, + }, + }, + { + name: "no duplicates", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"perell"}) + sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "perell", + category: email, + service: exchange, + subtreePath: stpFor("perell", email, exchange), + }, + { + tenant: tenantID, + resource: "perell", + category: contacts, + service: exchange, + subtreePath: stpFor("perell", contacts, exchange), + }, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + results := []expect{} + rs := reasonsFor(test.sel(), tenantID, test.useName) + + for _, r := range rs { + stp, err := r.SubtreePath() + + t.Log("stp err", err) + + stpStr := "" + if stp != nil { + stpStr = stp.String() + } + + results = append(results, expect{ + tenant: r.Tenant(), + resource: r.ProtectedResource(), + service: r.Service().String(), + category: r.Category().String(), + subtreePath: stpStr, + subtreePathHadErr: err != nil, + }) + } + + assert.ElementsMatch(t, test.expect, results) + }) + } +} + +func (suite *ReasonsUnitSuite) TestReasonsFor_serviceChecks() { + var ( + tenantID = "tid" + exchange = path.ExchangeService.String() + email = path.EmailCategory.String() + contacts = path.ContactsCategory.String() + ) + + type expect struct { + tenant string + resource string + category string + service string + subtreePath string + subtreePathHadErr bool + } + + stpFor := func(resource, category, service string) string { + return path.Builder{}.Append(tenantID, service, resource, category).String() + } + + table := []struct { + name string + sel func() ExchangeRestore + useName bool + expect []expect + }{ + { + name: "no scopes", + sel: func() ExchangeRestore { + return *NewExchangeRestore([]string{"timbo"}) + }, + expect: []expect{}, + }, + { + name: "only includes", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"bubba"}) + sel.Include(sel.MailFolders(Any())) + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "bubba", + category: email, + service: exchange, + subtreePath: stpFor("bubba", email, exchange), + }, + }, + }, + { + name: "only filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"tachoma dhaume"}) + sel.Filter(sel.MailFolders(Any())) + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "tachoma dhaume", + category: email, + service: exchange, + subtreePath: stpFor("tachoma dhaume", email, exchange), + }, + }, + }, + { + name: "duplicate includes and filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"vyng vang zoombah"}) + sel.Include(sel.MailFolders(Any())) + sel.Filter(sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "vyng vang zoombah", + category: email, + service: exchange, + subtreePath: stpFor("vyng vang zoombah", email, exchange), + }, + }, + }, + { + name: "duplicate includes", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"fat billie"}) + sel.Include(sel.MailFolders(Any()), sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "fat billie", + category: email, + service: exchange, + subtreePath: stpFor("fat billie", email, exchange), + }, + }, + }, + { + name: "duplicate filters", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"seathane"}) + sel.Filter(sel.MailFolders(Any()), sel.MailFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "seathane", + category: email, + service: exchange, + subtreePath: stpFor("seathane", email, exchange), + }, + }, + }, + { + name: "no duplicates", + sel: func() ExchangeRestore { + sel := *NewExchangeRestore([]string{"perell"}) + sel.Include(sel.MailFolders(Any()), sel.ContactFolders(Any())) + + return sel + }, + expect: []expect{ + { + tenant: tenantID, + resource: "perell", + category: email, + service: exchange, + subtreePath: stpFor("perell", email, exchange), + }, + { + tenant: tenantID, + resource: "perell", + category: contacts, + service: exchange, + subtreePath: stpFor("perell", contacts, exchange), + }, + }, + }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + results := []expect{} + rs := reasonsFor(test.sel(), tenantID, test.useName) + + for _, r := range rs { + stp, err := r.SubtreePath() + + t.Log("stp err", err) + + stpStr := "" + if stp != nil { + stpStr = stp.String() + } + + results = append(results, expect{ + tenant: r.Tenant(), + resource: r.ProtectedResource(), + service: r.Service().String(), + category: r.Category().String(), + subtreePath: stpStr, + subtreePathHadErr: err != nil, + }) + } + + assert.ElementsMatch(t, test.expect, results) + }) + } +} diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 474ab60f5..ac85f75c3 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" @@ -87,6 +88,14 @@ type pathCategorier interface { PathCategories() selectorPathCategories } +type pathServicer interface { + PathService() path.ServiceType +} + +type reasoner interface { + Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner +} + // --------------------------------------------------------------------------- // Selector // --------------------------------------------------------------------------- @@ -273,7 +282,7 @@ func (s Selector) Reduce( return r.Reduce(ctx, deets, errs), nil } -// returns the sets of path categories identified in each scope set. +// PathCategories returns the sets of path categories identified in each scope set. func (s Selector) PathCategories() (selectorPathCategories, error) { ro, err := selectorAsIface[pathCategorier](s) if err != nil { @@ -283,6 +292,18 @@ func (s Selector) PathCategories() (selectorPathCategories, error) { return ro.PathCategories(), nil } +// Reasons returns a deduplicated set of the backup reasons produced +// using the selector's discrete owner and each scopes' service and +// category types. +func (s Selector) Reasons(tenantID string, useOwnerNameForID bool) ([]identity.Reasoner, error) { + ro, err := selectorAsIface[reasoner](s) + if err != nil { + return nil, err + } + + return ro.Reasons(tenantID, useOwnerNameForID), nil +} + // transformer for arbitrary selector interfaces func selectorAsIface[T any](s Selector) (T, error) { var ( diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 31ad200c0..bf4f46671 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/common/dttm" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/backup/identity" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" @@ -42,6 +43,7 @@ type ( var ( _ Reducer = &SharePointRestore{} _ pathCategorier = &SharePointRestore{} + _ reasoner = &SharePointRestore{} ) // NewSharePointBackup produces a new Selector with the service set to ServiceSharePoint. @@ -121,6 +123,13 @@ func (s sharePoint) PathCategories() selectorPathCategories { } } +// Reasons returns a deduplicated set of the backup reasons produced +// using the selector's discrete owner and each scopes' service and +// category types. +func (s sharePoint) Reasons(tenantID string, useOwnerNameForID bool) []identity.Reasoner { + return reasonsFor(s, tenantID, useOwnerNameForID) +} + // --------------------------------------------------------------------------- // Stringers and Concealers // ---------------------------------------------------------------------------