From 286a74e819f47930c33bb283988723a97b4dc8a8 Mon Sep 17 00:00:00 2001 From: Keepers Date: Thu, 20 Oct 2022 19:04:54 -0600 Subject: [PATCH] persist service tag in backup model, list by service (#1233) ## Description Adds a tag describing the service within the backup model and the backup details model. Adds a filter builder pattern to the store wrapper for filtering results according to certain tags. Finally, adds a filter builder to specify the service type when listing backups. ## Type of change - [x] :bug: Bugfix ## Issue(s) * #1226 ## Test Plan - [x] :muscle: Manual - [x] :zap: Unit test --- src/cli/backup/exchange.go | 4 +++- src/cli/backup/onedrive.go | 4 +++- src/cli/utils/testdata/opts.go | 3 ++- src/internal/kopia/wrapper.go | 6 +++++ src/internal/kopia/wrapper_test.go | 23 ++++++++++++------- src/internal/model/model.go | 5 ++++ src/internal/operations/backup.go | 4 ++-- src/pkg/backup/backup.go | 3 +++ src/pkg/backup/backup_test.go | 3 +++ src/pkg/repository/repository.go | 6 ++--- src/pkg/selectors/selectors.go | 12 ++++++++++ src/pkg/store/backup.go | 37 ++++++++++++++++++++++++++++-- 12 files changed, 92 insertions(+), 18 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 11f29a3e3..bd7462747 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -16,8 +16,10 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" ) // ------------------------------------------------------------------------------------------------ @@ -371,7 +373,7 @@ func listExchangeCmd(cmd *cobra.Command, args []string) error { return nil } - bs, err := r.Backups(ctx) + bs, err := r.Backups(ctx, store.Service(path.ExchangeService)) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to list backups in the repository")) } diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index a050d47ce..cb9ee95d9 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -15,8 +15,10 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" ) // ------------------------------------------------------------------------------------------------ @@ -270,7 +272,7 @@ func listOneDriveCmd(cmd *cobra.Command, args []string) error { return nil } - bs, err := r.Backups(ctx) + bs, err := r.Backups(ctx, store.Service(path.OneDriveService)) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to list backups in the repository")) } diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index c145a829c..ce1c39ee5 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -12,6 +12,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/selectors/testdata" + "github.com/alcionai/corso/src/pkg/store" ) type ExchangeOptionsTest struct { @@ -407,7 +408,7 @@ func (MockBackupGetter) Backup( return nil, errors.New("unexpected call to mock") } -func (MockBackupGetter) Backups(context.Context) ([]backup.Backup, error) { +func (MockBackupGetter) Backups(context.Context, ...store.FilterOption) ([]backup.Backup, error) { return nil, errors.New("unexpected call to mock") } diff --git a/src/internal/kopia/wrapper.go b/src/internal/kopia/wrapper.go index 7c9acabfe..388f298a2 100644 --- a/src/internal/kopia/wrapper.go +++ b/src/internal/kopia/wrapper.go @@ -20,6 +20,7 @@ import ( "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/stats" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" @@ -472,6 +473,7 @@ func inflateDirTree( func (w Wrapper) BackupCollections( ctx context.Context, collections []data.Collection, + service path.ServiceType, ) (*BackupStats, *details.Details, error) { if w.c == nil { return nil, nil, errNotConnected @@ -488,6 +490,10 @@ func (w Wrapper) BackupCollections( deets: &details.Details{}, } + progress.deets.Tags = map[string]string{ + model.ServiceTag: service.String(), + } + dirTree, err := inflateDirTree(ctx, collections, progress) if err != nil { return nil, nil, errors.Wrap(err, "building kopia directories") diff --git a/src/internal/kopia/wrapper_test.go b/src/internal/kopia/wrapper_test.go index ff3f79806..7e213e2b1 100644 --- a/src/internal/kopia/wrapper_test.go +++ b/src/internal/kopia/wrapper_test.go @@ -20,6 +20,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" @@ -810,15 +811,16 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() { ), } - stats, rp, err := suite.w.BackupCollections(suite.ctx, collections) + stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService) assert.NoError(t, err) assert.Equal(t, stats.TotalFileCount, 47) assert.Equal(t, stats.TotalDirectoryCount, 6) assert.Equal(t, stats.IgnoredErrorCount, 0) assert.Equal(t, stats.ErrorCount, 0) assert.False(t, stats.Incomplete) + assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag]) // 47 file and 6 folder entries. - assert.Len(t, rp.Entries, 47+6) + assert.Len(t, deets.Entries, 47+6) } func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { @@ -843,8 +845,9 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() { fp2, err := suite.testPath2.Append(dc2.Names[0], true) require.NoError(t, err) - stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}) + stats, deets, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}, path.ExchangeService) require.NoError(t, err) + assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag]) require.NoError(t, k.Compression(ctx, "gzip")) @@ -908,7 +911,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { }, } - stats, rp, err := suite.w.BackupCollections(suite.ctx, collections) + stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService) require.NoError(t, err) assert.Equal(t, 0, stats.ErrorCount) @@ -916,8 +919,9 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() { assert.Equal(t, 6, stats.TotalDirectoryCount) assert.Equal(t, 1, stats.IgnoredErrorCount) assert.False(t, stats.Incomplete) + assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag]) // 5 file and 6 folder entries. - assert.Len(t, rp.Entries, 5+6) + assert.Len(t, deets.Entries, 5+6) } type backedupFile struct { @@ -946,11 +950,13 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections() ctx, flush := tester.NewContext() defer flush() - s, d, err := suite.w.BackupCollections(ctx, test.collections) + s, d, err := suite.w.BackupCollections(ctx, test.collections, path.UnknownService) require.NoError(t, err) assert.Equal(t, BackupStats{}, *s) assert.Empty(t, d.Entries) + // unknownService resolves to an empty string here. + assert.Equal(t, "", d.Tags[model.ServiceTag]) }) } } @@ -1090,15 +1096,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() { collections = append(collections, collection) } - stats, rp, err := suite.w.BackupCollections(suite.ctx, collections) + stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService) require.NoError(t, err) require.Equal(t, stats.ErrorCount, 0) require.Equal(t, stats.TotalFileCount, expectedFiles) require.Equal(t, stats.TotalDirectoryCount, expectedDirs) require.Equal(t, stats.IgnoredErrorCount, 0) require.False(t, stats.Incomplete) + assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag]) // 6 file and 6 folder entries. - assert.Len(t, rp.Entries, expectedFiles+expectedDirs) + assert.Len(t, deets.Entries, expectedFiles+expectedDirs) suite.snapshotID = manifest.ID(stats.SnapshotID) } diff --git a/src/internal/model/model.go b/src/internal/model/model.go index 5ac1b6042..5542ff9d8 100644 --- a/src/internal/model/model.go +++ b/src/internal/model/model.go @@ -20,6 +20,11 @@ const ( BackupDetailsSchema ) +// common tags for filtering +const ( + ServiceTag = "service" +) + // Valid returns true if the ModelType value fits within the iota range. func (mt Schema) Valid() bool { return mt > 0 && mt < BackupDetailsSchema+1 diff --git a/src/internal/operations/backup.go b/src/internal/operations/backup.go index 6e7bb05ae..bfef149e0 100644 --- a/src/internal/operations/backup.go +++ b/src/internal/operations/backup.go @@ -142,7 +142,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) { opStats.resourceCount = len(data.ResourceOwnerSet(cs)) // hand the results to the consumer - opStats.k, backupDetails, err = op.kopia.BackupCollections(ctx, cs) + opStats.k, backupDetails, err = op.kopia.BackupCollections(ctx, cs, op.Selectors.PathService()) if err != nil { err = errors.Wrap(err, "backing up service data") opStats.writeErr = err @@ -228,7 +228,7 @@ func (op *BackupOperation) createBackupModels( events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt), events.EndTime: op.Results.CompletedAt, events.Resources: op.Results.ResourceOwners, - events.Service: op.Selectors.Service.String(), + events.Service: op.Selectors.PathService().String(), events.StartTime: op.Results.StartedAt, events.Status: op.Status, }, diff --git a/src/pkg/backup/backup.go b/src/pkg/backup/backup.go index b75d7e1a2..97b02a373 100644 --- a/src/pkg/backup/backup.go +++ b/src/pkg/backup/backup.go @@ -50,6 +50,9 @@ func New( return &Backup{ BaseModel: model.BaseModel{ ID: id, + Tags: map[string]string{ + model.ServiceTag: selector.PathService().String(), + }, }, CreationTime: time.Now(), SnapshotID: snapshotID, diff --git a/src/pkg/backup/backup_test.go b/src/pkg/backup/backup_test.go index 2edd0cb4d..4fe2fa09f 100644 --- a/src/pkg/backup/backup_test.go +++ b/src/pkg/backup/backup_test.go @@ -31,6 +31,9 @@ func stubBackup(t time.Time) backup.Backup { return backup.Backup{ BaseModel: model.BaseModel{ ID: model.StableID("id"), + Tags: map[string]string{ + model.ServiceTag: sel.PathService().String(), + }, }, CreationTime: t, SnapshotID: "snapshot", diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index b65d3b41a..065d0d31a 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -24,7 +24,7 @@ import ( // repository. type BackupGetter interface { Backup(ctx context.Context, id model.StableID) (*backup.Backup, error) - Backups(ctx context.Context) ([]backup.Backup, error) + Backups(ctx context.Context, fs ...store.FilterOption) ([]backup.Backup, error) BackupDetails( ctx context.Context, backupID string, @@ -214,9 +214,9 @@ func (r repository) Backup(ctx context.Context, id model.StableID) (*backup.Back } // backups lists backups in a repository -func (r repository) Backups(ctx context.Context) ([]backup.Backup, error) { +func (r repository) Backups(ctx context.Context, fs ...store.FilterOption) ([]backup.Backup, error) { sw := store.NewKopiaStore(r.modelStore) - return sw.GetBackups(ctx) + return sw.GetBackups(ctx, fs...) } // BackupDetails returns the specified backup details object diff --git a/src/pkg/selectors/selectors.go b/src/pkg/selectors/selectors.go index 5729d7539..755421dd5 100644 --- a/src/pkg/selectors/selectors.go +++ b/src/pkg/selectors/selectors.go @@ -10,6 +10,7 @@ import ( "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/filters" + "github.com/alcionai/corso/src/pkg/path" ) type service int @@ -21,6 +22,12 @@ const ( ServiceOneDrive // OneDrive ) +var serviceToPathType = map[service]path.ServiceType{ + ServiceUnknown: path.UnknownService, + ServiceExchange: path.ExchangeService, + ServiceOneDrive: path.OneDriveService, +} + var ( ErrorBadSelectorCast = errors.New("wrong selector service type") ErrorNoMatchingItems = errors.New("no items match the specified selectors") @@ -178,6 +185,11 @@ func discreteScopes[T scopeT, C categoryT]( return sl } +// Returns the path.ServiceType matching the selector service. +func (s Selector) PathService() path.ServiceType { + return serviceToPathType[s.Service] +} + // --------------------------------------------------------------------------- // Printing Selectors for Human Reading // --------------------------------------------------------------------------- diff --git a/src/pkg/store/backup.go b/src/pkg/store/backup.go index b6a586b74..6fcf07fdc 100644 --- a/src/pkg/store/backup.go +++ b/src/pkg/store/backup.go @@ -9,8 +9,35 @@ import ( "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/path" ) +type queryFilters struct { + tags map[string]string +} + +type FilterOption func(*queryFilters) + +func (q *queryFilters) populate(qf ...FilterOption) { + if len(qf) == 0 { + return + } + + q.tags = map[string]string{} + + for _, fn := range qf { + fn(q) + } +} + +// Service ensures the retrieved backups only match +// the specified service. +func Service(pst path.ServiceType) FilterOption { + return func(qf *queryFilters) { + qf.tags[model.ServiceTag] = pst.String() + } +} + // GetBackup gets a single backup by id. func (w Wrapper) GetBackup(ctx context.Context, backupID model.StableID) (*backup.Backup, error) { b := backup.Backup{} @@ -24,8 +51,14 @@ func (w Wrapper) GetBackup(ctx context.Context, backupID model.StableID) (*backu } // GetDetailsFromBackupID retrieves all backups in the model store. -func (w Wrapper) GetBackups(ctx context.Context) ([]backup.Backup, error) { - bms, err := w.GetIDsForType(ctx, model.BackupSchema, nil) +func (w Wrapper) GetBackups( + ctx context.Context, + filters ...FilterOption, +) ([]backup.Backup, error) { + q := &queryFilters{} + q.populate(filters...) + + bms, err := w.GetIDsForType(ctx, model.BackupSchema, q.tags) if err != nil { return nil, err }