larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

persist service tag in backup model, list by service (#1233)

Browse Source 
## Description

Adds a tag describing the service within the backup model and the backup details model.  Adds a filter builder pattern to the store wrapper for filtering results according to certain tags.  Finally, adds a filter builder to specify the service type when listing backups.

## Type of change

- [x] 🐛 Bugfix

## Issue(s)

* #1226

## Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test
This commit is contained in:
Keepers 2022-10-20 19:04:54 -06:00 committed by GitHub
parent e82cdadc62
commit 286a74e819
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 92 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/cli/backup/exchange.go
View File

@ -16,8 +16,10 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/store"
)
// ------------------------------------------------------------------------------------------------
@ -371,7 +373,7 @@ func listExchangeCmd(cmd *cobra.Command, args []string) error {
return nil
}
bs, err := r.Backups(ctx)
bs, err := r.Backups(ctx, store.Service(path.ExchangeService))
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to list backups in the repository"))
}

4
src/cli/backup/onedrive.go
View File

@ -15,8 +15,10 @@ import (
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/store"
)
// ------------------------------------------------------------------------------------------------
@ -270,7 +272,7 @@ func listOneDriveCmd(cmd *cobra.Command, args []string) error {
return nil
}
bs, err := r.Backups(ctx)
bs, err := r.Backups(ctx, store.Service(path.OneDriveService))
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to list backups in the repository"))
}

3
src/cli/utils/testdata/opts.go vendored
View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/selectors/testdata"
"github.com/alcionai/corso/src/pkg/store"
)
type ExchangeOptionsTest struct {
@ -407,7 +408,7 @@ func (MockBackupGetter) Backup(
return nil, errors.New("unexpected call to mock")
}
func (MockBackupGetter) Backups(context.Context) ([]backup.Backup, error) {
func (MockBackupGetter) Backups(context.Context, ...store.FilterOption) ([]backup.Backup, error) {
return nil, errors.New("unexpected call to mock")
}

6
src/internal/kopia/wrapper.go
View File

@ -20,6 +20,7 @@ import (
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/stats"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
@ -472,6 +473,7 @@ func inflateDirTree(
func (w Wrapper) BackupCollections(
ctx context.Context,
collections []data.Collection,
service path.ServiceType,
) (*BackupStats, *details.Details, error) {
if w.c == nil {
return nil, nil, errNotConnected
@ -488,6 +490,10 @@ func (w Wrapper) BackupCollections(
deets: &details.Details{},
}
progress.deets.Tags = map[string]string{
model.ServiceTag: service.String(),
}
dirTree, err := inflateDirTree(ctx, collections, progress)
if err != nil {
return nil, nil, errors.Wrap(err, "building kopia directories")

23
src/internal/kopia/wrapper_test.go
View File

@ -20,6 +20,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
@ -810,15 +811,16 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections() {
),
}
stats, rp, err := suite.w.BackupCollections(suite.ctx, collections)
stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService)
assert.NoError(t, err)
assert.Equal(t, stats.TotalFileCount, 47)
assert.Equal(t, stats.TotalDirectoryCount, 6)
assert.Equal(t, stats.IgnoredErrorCount, 0)
assert.Equal(t, stats.ErrorCount, 0)
assert.False(t, stats.Incomplete)
assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag])
// 47 file and 6 folder entries.
assert.Len(t, rp.Entries, 47+6)
assert.Len(t, deets.Entries, 47+6)
}
func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
@ -843,8 +845,9 @@ func (suite *KopiaIntegrationSuite) TestRestoreAfterCompressionChange() {
fp2, err := suite.testPath2.Append(dc2.Names[0], true)
require.NoError(t, err)
stats, _, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2})
stats, deets, err := w.BackupCollections(ctx, []data.Collection{dc1, dc2}, path.ExchangeService)
require.NoError(t, err)
assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag])
require.NoError(t, k.Compression(ctx, "gzip"))
@ -908,7 +911,7 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
},
}
stats, rp, err := suite.w.BackupCollections(suite.ctx, collections)
stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService)
require.NoError(t, err)
assert.Equal(t, 0, stats.ErrorCount)
@ -916,8 +919,9 @@ func (suite *KopiaIntegrationSuite) TestBackupCollections_ReaderError() {
assert.Equal(t, 6, stats.TotalDirectoryCount)
assert.Equal(t, 1, stats.IgnoredErrorCount)
assert.False(t, stats.Incomplete)
assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag])
// 5 file and 6 folder entries.
assert.Len(t, rp.Entries, 5+6)
assert.Len(t, deets.Entries, 5+6)
}
type backedupFile struct {
@ -946,11 +950,13 @@ func (suite *KopiaIntegrationSuite) TestBackupCollectionsHandlesNoCollections()
ctx, flush := tester.NewContext()
defer flush()
s, d, err := suite.w.BackupCollections(ctx, test.collections)
s, d, err := suite.w.BackupCollections(ctx, test.collections, path.UnknownService)
require.NoError(t, err)
assert.Equal(t, BackupStats{}, *s)
assert.Empty(t, d.Entries)
// unknownService resolves to an empty string here.
assert.Equal(t, "", d.Tags[model.ServiceTag])
})
}
}
@ -1090,15 +1096,16 @@ func (suite *KopiaSimpleRepoIntegrationSuite) SetupTest() {
collections = append(collections, collection)
}
stats, rp, err := suite.w.BackupCollections(suite.ctx, collections)
stats, deets, err := suite.w.BackupCollections(suite.ctx, collections, path.ExchangeService)
require.NoError(t, err)
require.Equal(t, stats.ErrorCount, 0)
require.Equal(t, stats.TotalFileCount, expectedFiles)
require.Equal(t, stats.TotalDirectoryCount, expectedDirs)
require.Equal(t, stats.IgnoredErrorCount, 0)
require.False(t, stats.Incomplete)
assert.Equal(t, path.ExchangeService.String(), deets.Tags[model.ServiceTag])
// 6 file and 6 folder entries.
assert.Len(t, rp.Entries, expectedFiles+expectedDirs)
assert.Len(t, deets.Entries, expectedFiles+expectedDirs)
suite.snapshotID = manifest.ID(stats.SnapshotID)
}

5
src/internal/model/model.go
View File

@ -20,6 +20,11 @@ const (
BackupDetailsSchema
)
// common tags for filtering
const (
ServiceTag = "service"
)
// Valid returns true if the ModelType value fits within the iota range.
func (mt Schema) Valid() bool {
return mt > 0 && mt < BackupDetailsSchema+1

4
src/internal/operations/backup.go
View File

@ -142,7 +142,7 @@ func (op *BackupOperation) Run(ctx context.Context) (err error) {
opStats.resourceCount = len(data.ResourceOwnerSet(cs))
// hand the results to the consumer
opStats.k, backupDetails, err = op.kopia.BackupCollections(ctx, cs)
opStats.k, backupDetails, err = op.kopia.BackupCollections(ctx, cs, op.Selectors.PathService())
if err != nil {
err = errors.Wrap(err, "backing up service data")
opStats.writeErr = err
@ -228,7 +228,7 @@ func (op *BackupOperation) createBackupModels(
events.Duration: op.Results.CompletedAt.Sub(op.Results.StartedAt),
events.EndTime: op.Results.CompletedAt,
events.Resources: op.Results.ResourceOwners,
events.Service: op.Selectors.Service.String(),
events.Service: op.Selectors.PathService().String(),
events.StartTime: op.Results.StartedAt,
events.Status: op.Status,
},

3
src/pkg/backup/backup.go
View File

@ -50,6 +50,9 @@ func New(
return &Backup{
BaseModel: model.BaseModel{
ID: id,
Tags: map[string]string{
model.ServiceTag: selector.PathService().String(),
},
},
CreationTime: time.Now(),
SnapshotID: snapshotID,

3
src/pkg/backup/backup_test.go
View File

@ -31,6 +31,9 @@ func stubBackup(t time.Time) backup.Backup {
return backup.Backup{
BaseModel: model.BaseModel{
ID: model.StableID("id"),
Tags: map[string]string{
model.ServiceTag: sel.PathService().String(),
},
},
CreationTime: t,
SnapshotID: "snapshot",

6
src/pkg/repository/repository.go
View File

@ -24,7 +24,7 @@ import (
// repository.
type BackupGetter interface {
Backup(ctx context.Context, id model.StableID) (*backup.Backup, error)
Backups(ctx context.Context) ([]backup.Backup, error)
Backups(ctx context.Context, fs ...store.FilterOption) ([]backup.Backup, error)
BackupDetails(
ctx context.Context,
backupID string,
@ -214,9 +214,9 @@ func (r repository) Backup(ctx context.Context, id model.StableID) (*backup.Back
}
// backups lists backups in a repository
func (r repository) Backups(ctx context.Context) ([]backup.Backup, error) {
func (r repository) Backups(ctx context.Context, fs ...store.FilterOption) ([]backup.Backup, error) {
sw := store.NewKopiaStore(r.modelStore)
return sw.GetBackups(ctx)
return sw.GetBackups(ctx, fs...)
}
// BackupDetails returns the specified backup details object

12
src/pkg/selectors/selectors.go
View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
)
type service int
@ -21,6 +22,12 @@ const (
ServiceOneDrive // OneDrive
)
var serviceToPathType = map[service]path.ServiceType{
ServiceUnknown: path.UnknownService,
ServiceExchange: path.ExchangeService,
ServiceOneDrive: path.OneDriveService,
}
var (
ErrorBadSelectorCast = errors.New("wrong selector service type")
ErrorNoMatchingItems = errors.New("no items match the specified selectors")
@ -178,6 +185,11 @@ func discreteScopes[T scopeT, C categoryT](
return sl
}
// Returns the path.ServiceType matching the selector service.
func (s Selector) PathService() path.ServiceType {
return serviceToPathType[s.Service]
}
// ---------------------------------------------------------------------------
// Printing Selectors for Human Reading
// ---------------------------------------------------------------------------

37
src/pkg/store/backup.go
View File

@ -9,8 +9,35 @@ import (
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path"
)
type queryFilters struct {
tags map[string]string
}
type FilterOption func(*queryFilters)
func (q *queryFilters) populate(qf ...FilterOption) {
if len(qf) == 0 {
return
}
q.tags = map[string]string{}
for _, fn := range qf {
fn(q)
}
}
// Service ensures the retrieved backups only match
// the specified service.
func Service(pst path.ServiceType) FilterOption {
return func(qf *queryFilters) {
qf.tags[model.ServiceTag] = pst.String()
}
}
// GetBackup gets a single backup by id.
func (w Wrapper) GetBackup(ctx context.Context, backupID model.StableID) (*backup.Backup, error) {
b := backup.Backup{}
@ -24,8 +51,14 @@ func (w Wrapper) GetBackup(ctx context.Context, backupID model.StableID) (*backu
}
// GetDetailsFromBackupID retrieves all backups in the model store.
func (w Wrapper) GetBackups(ctx context.Context) ([]backup.Backup, error) {
bms, err := w.GetIDsForType(ctx, model.BackupSchema, nil)
func (w Wrapper) GetBackups(
ctx context.Context,
filters ...FilterOption,
) ([]backup.Backup, error) {
q := &queryFilters{}
q.populate(filters...)
bms, err := w.GetIDsForType(ctx, model.BackupSchema, q.tags)
if err != nil {
return nil, err
}