larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use selectors in OneDrive CLI (#996)

Browse Source 
## Description

Adds the following selectors to OneDrive details/restore :
- `file-name`, `folder`, `file-created-after`, `file-created-before`, `file-modified-after`, `file-modified-before`

Also includes a change where we remove the `drive/<driveID>/root:` prefix from parent path entries in details. This
is to improve readability. We will add drive back as a separate item in details if needed later.

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* #627 

## Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Vaibhav Kamra 2022-10-03 00:23:30 -07:00 committed by GitHub
parent 88af7f9b7c
commit 03bb63f52d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 400 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/cli/backup/onedrive.go
View File

@ -1,6 +1,8 @@
package backup
import (
"context"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -11,6 +13,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -21,6 +24,16 @@ import (
const oneDriveServiceCommand = "onedrive"
var (
folderPaths []string
fileNames []string
fileCreatedAfter string
fileCreatedBefore string
fileModifiedAfter string
fileModifiedBefore string
)
// called by backup.go to map parent subcommands to provider-specific handling.
func addOneDriveCommands(parent *cobra.Command) *cobra.Command {
var (
@ -44,6 +57,38 @@ func addOneDriveCommands(parent *cobra.Command) *cobra.Command {
fs.StringVar(&backupID, "backup", "", "ID of the backup containing the details to be shown")
cobra.CheckErr(c.MarkFlagRequired("backup"))
// onedrive hierarchy flags
fs.StringSliceVar(
&folderPaths,
"folder", nil,
"Select backup details by OneDrive folder; defaults to root")
fs.StringSliceVar(
&fileNames,
"file-name", nil,
"Select backup details by OneDrive file name")
// onedrive info flags
fs.StringVar(
&fileCreatedAfter,
"file-created-after", "",
"Select files created after this datetime")
fs.StringVar(
&fileCreatedBefore,
"file-created-before", "",
"Select files created before this datetime")
fs.StringVar(
&fileModifiedAfter,
"file-modified-after", "",
"Select files modified after this datetime")
fs.StringVar(
&fileModifiedBefore,
"file-modified-before", "",
"Select files modified before this datetime")
case deleteCommand:
c, fs = utils.AddCommand(parent, oneDriveDeleteCmd())
fs.StringVar(&backupID, "backup", "", "ID of the backup containing the details to be shown")
@ -202,18 +247,56 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
ds, _, err := r.BackupDetails(ctx, backupID)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to get backup details in the repository"))
opts := utils.OneDriveOpts{
Users: user,
Paths: folderPaths,
Names: fileNames,
CreatedAfter: fileCreatedAfter,
CreatedBefore: fileCreatedBefore,
ModifiedAfter: fileModifiedAfter,
ModifiedBefore: fileModifiedBefore,
}
// TODO: Support selectors and filters
ds, err := runDetailsOneDriveCmd(ctx, r, backupID, opts)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsOneDriveCmd actually performs the lookup in backup details. Assumes
// len(backupID) > 0.
func runDetailsOneDriveCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.OneDriveOpts,
) (*details.Details, error) {
d, _, err := r.BackupDetails(ctx, backupID)
if err != nil {
return nil, errors.Wrap(err, "Failed to get backup details in the repository")
}
sel := selectors.NewOneDriveRestore()
utils.IncludeOneDriveRestoreDataSelectors(sel, opts)
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
// if no selector flags were specified, get all data in the service.
if len(sel.Scopes()) == 0 {
sel.Include(sel.Users(selectors.Any()))
}
return sel.Reduce(ctx, d), nil
}
// `corso backup delete onedrive [<flag>...]`
func oneDriveDeleteCmd() *cobra.Command {
return &cobra.Command{

57
src/cli/restore/onedrive.go
View File

@ -15,6 +15,16 @@ import (
"github.com/alcionai/corso/src/pkg/selectors"
)
var (
folderPaths []string
fileNames []string
fileCreatedAfter string
fileCreatedBefore string
fileModifiedAfter string
fileModifiedBefore string
)
// called by restore.go to map parent subcommands to provider-specific handling.
func addOneDriveCommands(parent *cobra.Command) *cobra.Command {
var (
@ -37,6 +47,38 @@ func addOneDriveCommands(parent *cobra.Command) *cobra.Command {
"user", nil,
"Restore all data by user ID; accepts "+utils.Wildcard+" to select all users")
// onedrive hierarchy (path/name) flags
fs.StringSliceVar(
&folderPaths,
"folder", nil,
"Restore items by OneDrive folder; defaults to root")
fs.StringSliceVar(
&fileNames,
"file-name", nil,
"Restore items by OneDrive file name")
// onedrive info flags
fs.StringVar(
&fileCreatedAfter,
"file-created-after", "",
"Restore files created after this datetime")
fs.StringVar(
&fileCreatedBefore,
"file-created-before", "",
"Restore files created before this datetime")
fs.StringVar(
&fileModifiedAfter,
"file-modified-after", "",
"Restore files modified after this datetime")
fs.StringVar(
&fileModifiedBefore,
"file-modified-before", "",
"Restore files modified before this datetime")
// others
options.AddOperationFlags(c)
}
@ -80,11 +122,20 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
sel := selectors.NewOneDriveRestore()
if user != nil {
sel.Include(sel.Users(user))
opts := utils.OneDriveOpts{
Users: user,
Paths: folderPaths,
Names: fileNames,
CreatedAfter: fileCreatedAfter,
CreatedBefore: fileCreatedBefore,
ModifiedAfter: fileModifiedAfter,
ModifiedBefore: fileModifiedBefore,
}
sel := selectors.NewOneDriveRestore()
utils.IncludeOneDriveRestoreDataSelectors(sel, opts)
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
// if no selector flags were specified, get all data in the service.
if len(sel.Scopes()) == 0 {
sel.Include(sel.Users(selectors.Any()))

67
src/cli/utils/onedrive.go
View File

@ -2,8 +2,20 @@ package utils
import (
"errors"
"github.com/alcionai/corso/src/pkg/selectors"
)
type OneDriveOpts struct {
Users []string
Names []string
Paths []string
CreatedAfter string
CreatedBefore string
ModifiedAfter string
ModifiedBefore string
}
// ValidateOneDriveRestoreFlags checks common flags for correctness and interdependencies
func ValidateOneDriveRestoreFlags(backupID string) error {
if len(backupID) == 0 {
@ -12,3 +24,58 @@ func ValidateOneDriveRestoreFlags(backupID string) error {
return nil
}
// AddOneDriveFilter adds the scope of the provided values to the selector's
// filter set
func AddOneDriveFilter(
sel *selectors.OneDriveRestore,
v string,
f func(string) []selectors.OneDriveScope,
) {
if len(v) == 0 {
return
}
sel.Filter(f(v))
}
// IncludeOneDriveRestoreDataSelectors builds the common data-selector
// inclusions for OneDrive commands.
func IncludeOneDriveRestoreDataSelectors(
sel *selectors.OneDriveRestore,
opts OneDriveOpts,
) {
if len(opts.Users) == 0 {
opts.Users = selectors.Any()
}
lp, ln := len(opts.Paths), len(opts.Names)
// either scope the request to a set of users
if lp+ln == 0 {
sel.Include(sel.Users(opts.Users))
return
}
if lp == 0 {
opts.Paths = selectors.Any()
}
if ln == 0 {
opts.Names = selectors.Any()
}
sel.Include(sel.Items(opts.Users, opts.Paths, opts.Names))
}
// FilterOneDriveRestoreInfoSelectors builds the common info-selector filters.
func FilterOneDriveRestoreInfoSelectors(
sel *selectors.OneDriveRestore,
opts OneDriveOpts,
) {
AddOneDriveFilter(sel, opts.CreatedAfter, sel.CreatedAfter)
AddOneDriveFilter(sel, opts.CreatedBefore, sel.CreatedBefore)
AddOneDriveFilter(sel, opts.ModifiedAfter, sel.ModifiedAfter)
AddOneDriveFilter(sel, opts.ModifiedBefore, sel.ModifiedBefore)
}

14
src/internal/connector/onedrive/collection.go
View File

@ -111,6 +111,14 @@ func (oc *Collection) populateItems(ctx context.Context) {
itemsRead = 0
)
// Retrieve the OneDrive folder path to set later in
// `details.OneDriveInfo`
parentPathString, err := getDriveFolderPath(oc.folderPath)
if err != nil {
oc.reportAsCompleted(ctx, 0, err)
return
}
for _, itemID := range oc.driveItemIDs {
// Read the item
itemInfo, itemData, err := oc.itemReader(ctx, oc.service, oc.driveID, itemID)
@ -126,7 +134,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
// Item read successfully, add to collection
itemsRead++
itemInfo.ParentPath = oc.folderPath.String()
itemInfo.ParentPath = parentPathString
oc.data <- &Item{
id: itemInfo.ItemName,
@ -135,6 +143,10 @@ func (oc *Collection) populateItems(ctx context.Context) {
}
}
oc.reportAsCompleted(ctx, itemsRead, errs)
}
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsRead int, errs error) {
close(oc.data)
status := support.CreateStatus(ctx, support.Backup,

6
src/internal/connector/onedrive/collection_test.go
View File

@ -60,7 +60,9 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
wg := sync.WaitGroup{}
collStatus := support.ConnectorOperationStatus{}
folderPath, err := getCanonicalPath("dir1/dir2/dir3", "a-tenant", "a-user")
folderPath, err := getCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "a-tenant", "a-user")
require.NoError(t, err)
driveFolderPath, err := getDriveFolderPath(folderPath)
require.NoError(t, err)
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
@ -106,7 +108,7 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
require.NotNil(t, readItemInfo.Info())
require.NotNil(t, readItemInfo.Info().OneDrive)
assert.Equal(t, testItemName, readItemInfo.Info().OneDrive.ItemName)
assert.Equal(t, folderPath.String(), readItemInfo.Info().OneDrive.ParentPath)
assert.Equal(t, driveFolderPath, readItemInfo.Info().OneDrive.ParentPath)
}
func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {

10
src/internal/connector/onedrive/collections.go
View File

@ -82,6 +82,16 @@ func getCanonicalPath(p, tenant, user string) (path.Path, error) {
return res, nil
}
// Returns the path to the folder within the drive (i.e. under `root:`)
func getDriveFolderPath(p path.Path) (string, error) {
drivePath, err := toOneDrivePath(p)
if err != nil {
return "", err
}
return path.Builder{}.Append(drivePath.folders...).String(), nil
}
// updateCollections initializes and adds the provided OneDrive items to Collections
// A new collection is created for every OneDrive folder (or package)
func (c *Collections) updateCollections(ctx context.Context, driveID string, items []models.DriveItemable) error {

16
src/internal/operations/restore.go
View File

@ -149,12 +149,24 @@ func (op *RestoreOperation) Run(ctx context.Context) (err error) {
}
case selectors.ServiceOneDrive:
// TODO: Reduce `details` here when we add support for OneDrive restore filters
fds = d
odr, err := op.Selectors.ToOneDriveRestore()
if err != nil {
opStats.readErr = err
return err
}
// format the details and retrieve the items from kopia
fds = odr.Reduce(ctx, d)
if len(fds.Entries) == 0 {
return selectors.ErrorNoMatchingItems
}
default:
return errors.Errorf("Service %s not supported", op.Selectors.Service)
}
logger.Ctx(ctx).Infof("Discovered %d items in backup %s to restore", len(fds.Entries), op.BackupID)
fdsPaths := fds.Paths()
paths := make([]path.Path, len(fdsPaths))

105
src/pkg/selectors/onedrive.go
View File

@ -3,7 +3,9 @@ package selectors
import (
"context"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
)
@ -202,6 +204,65 @@ func (s *oneDrive) Items(users, folders, items []string) []OneDriveScope {
return scopes
}
// -------------------
// Filter Factories
// CreatedAfter produces a OneDrive item created-after filter scope.
// Matches any item where the created time is after the timestring.
// If the input equals selectors.Any, the scope will match all times.
// If the input is empty or selectors.None, the scope will always fail comparisons.
func (s *oneDrive) CreatedAfter(timeStrings string) []OneDriveScope {
return []OneDriveScope{
makeFilterScope[OneDriveScope](
OneDriveItem,
FileFilterCreatedAfter,
[]string{timeStrings},
wrapFilter(filters.Less)),
}
}
// CreatedBefore produces a OneDrive item created-before filter scope.
// Matches any item where the created time is before the timestring.
// If the input equals selectors.Any, the scope will match all times.
// If the input is empty or selectors.None, the scope will always fail comparisons.
func (s *oneDrive) CreatedBefore(timeStrings string) []OneDriveScope {
return []OneDriveScope{
makeFilterScope[OneDriveScope](
OneDriveItem,
FileFilterCreatedBefore,
[]string{timeStrings},
wrapFilter(filters.Greater)),
}
}
// ModifiedAfter produces a OneDrive item modified-after filter scope.
// Matches any item where the modified time is after the timestring.
// If the input equals selectors.Any, the scope will match all times.
// If the input is empty or selectors.None, the scope will always fail comparisons.
func (s *oneDrive) ModifiedAfter(timeStrings string) []OneDriveScope {
return []OneDriveScope{
makeFilterScope[OneDriveScope](
OneDriveItem,
FileFilterModifiedAfter,
[]string{timeStrings},
wrapFilter(filters.Less)),
}
}
// ModifiedBefore produces a OneDrive item modified-before filter scope.
// Matches any item where the modified time is before the timestring.
// If the input equals selectors.Any, the scope will match all times.
// If the input is empty or selectors.None, the scope will always fail comparisons.
func (s *oneDrive) ModifiedBefore(timeStrings string) []OneDriveScope {
return []OneDriveScope{
makeFilterScope[OneDriveScope](
OneDriveItem,
FileFilterModifiedBefore,
[]string{timeStrings},
wrapFilter(filters.Greater)),
}
}
// ---------------------------------------------------------------------------
// Categories
// ---------------------------------------------------------------------------
@ -219,6 +280,12 @@ const (
OneDriveUser oneDriveCategory = "OneDriveUser"
OneDriveItem oneDriveCategory = "OneDriveItem"
OneDriveFolder oneDriveCategory = "OneDriveFolder"
// filterable topics identified by OneDrive
FileFilterCreatedAfter oneDriveCategory = "FileFilterCreatedAfter"
FileFilterCreatedBefore oneDriveCategory = "FileFilterCreatedBefore"
FileFilterModifiedAfter oneDriveCategory = "FileFilterModifiedAfter"
FileFilterModifiedBefore oneDriveCategory = "FileFilterModifiedBefore"
)
// oneDrivePathSet describes the category type keys used in OneDrive paths.
@ -240,7 +307,9 @@ func (c oneDriveCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser
func (c oneDriveCategory) leafCat() categorizer {
switch c {
case OneDriveFolder, OneDriveItem:
case OneDriveFolder, OneDriveItem,
FileFilterCreatedAfter, FileFilterCreatedBefore,
FileFilterModifiedAfter, FileFilterModifiedBefore:
return OneDriveItem
}
@ -269,9 +338,12 @@ func (c oneDriveCategory) isLeaf() bool {
// [tenantID, service, userPN, category, folder, fileID]
// => {odUser: userPN, odFolder: folder, odFileID: fileID}
func (c oneDriveCategory) pathValues(p path.Path) map[categorizer]string {
// Ignore `drives/<driveID>/root:` for folder comparison
folder := path.Builder{}.Append(p.Folders()...).PopFront().PopFront().PopFront().String()
return map[categorizer]string{
OneDriveUser: p.ResourceOwner(),
OneDriveFolder: p.Folder(), // TODO: Should we filter out the DriveID here?
OneDriveFolder: folder,
OneDriveItem: p.Item(),
}
}
@ -360,31 +432,18 @@ func (s OneDriveScope) matchesInfo(dii details.ItemInfo) bool {
return false
}
// the scope must define targets to match on
filterCat := s.FilterCategory()
targets := s.Get(filterCat)
if len(targets) == 0 {
return false
i := ""
switch filterCat {
case FileFilterCreatedAfter, FileFilterCreatedBefore:
i = common.FormatTime(info.Created)
case FileFilterModifiedAfter, FileFilterModifiedBefore:
i = common.FormatTime(info.LastModified)
}
if targets[0] == AnyTgt {
return true
}
if targets[0] == NoneTgt {
return false
}
// any of the targets for a given info filter may succeed.
for _, target := range targets {
switch filterCat {
// TODO: populate oneDrive filter checks
default:
return target != NoneTgt
}
}
return false
return s.Matches(filterCat, i)
}
// ---------------------------------------------------------------------------

72
src/pkg/selectors/onedrive_test.go
View File

@ -3,11 +3,13 @@ package selectors
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/path"
)
@ -181,9 +183,9 @@ func (suite *OneDriveSelectorSuite) TestToOneDriveRestore() {
func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
var (
file = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "folderA/folderB", "file")
file2 = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "folderA/folderC", "file2")
file3 = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "folderD/folderE", "file3")
file = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "drive/driveID/root:/folderA/folderB", "file")
file2 = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "drive/driveID/root:/folderA/folderC", "file2")
file3 = stubRepoRef(path.OneDriveService, path.FilesCategory, "uid", "drive/driveID/root:/folderD/folderE", "file3")
)
deets := &details.Details{
@ -267,3 +269,67 @@ func (suite *OneDriveSelectorSuite) TestOneDriveRestore_Reduce() {
})
}
}
func (suite *OneDriveSelectorSuite) TestOneDriveCategory_PathValues() {
t := suite.T()
pathBuilder := path.Builder{}.Append("drive", "driveID", "root:", "dir1", "dir2", "file")
filePath, err := pathBuilder.ToDataLayerOneDrivePath("tenant", "user", true)
require.NoError(t, err)
expected := map[categorizer]string{
OneDriveUser: "user",
OneDriveFolder: "dir1/dir2",
OneDriveItem: "file",
}
assert.Equal(t, expected, OneDriveItem.pathValues(filePath))
}
func (suite *OneDriveSelectorSuite) TestOneDriveScope_MatchesInfo() {
ods := NewOneDriveRestore()
var (
epoch = time.Time{}
now = time.Now()
future = now.Add(1 * time.Minute)
)
itemInfo := details.ItemInfo{
OneDrive: &details.OneDriveInfo{
ItemType: details.OneDriveItem,
ParentPath: "folder1/folder2",
ItemName: "file1",
Size: 10,
Created: now,
LastModified: now,
},
}
table := []struct {
name string
scope []OneDriveScope
expect assert.BoolAssertionFunc
}{
{"file create after the epoch", ods.CreatedAfter(common.FormatTime(epoch)), assert.True},
{"file create after now", ods.CreatedAfter(common.FormatTime(now)), assert.False},
{"file create after later", ods.CreatedAfter(common.FormatTime(future)), assert.False},
{"file create before future", ods.CreatedBefore(common.FormatTime(future)), assert.True},
{"file create before now", ods.CreatedBefore(common.FormatTime(now)), assert.False},
{"file create before epoch", ods.CreatedBefore(common.FormatTime(now)), assert.False},
{"file modified after the epoch", ods.ModifiedAfter(common.FormatTime(epoch)), assert.True},
{"file modified after now", ods.ModifiedAfter(common.FormatTime(now)), assert.False},
{"file modified after later", ods.ModifiedAfter(common.FormatTime(future)), assert.False},
{"file modified before future", ods.ModifiedBefore(common.FormatTime(future)), assert.True},
{"file modified before now", ods.ModifiedBefore(common.FormatTime(now)), assert.False},
{"file modified before epoch", ods.ModifiedBefore(common.FormatTime(now)), assert.False},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
scopes := setScopesToDefault(test.scope)
for _, scope := range scopes {
test.expect(t, scope.matchesInfo(itemInfo))
}
})
}
}