larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CLI: SharePoint.Libraries selector expansion (#2679)

Browse Source 
<!-- Insert PR description-->
Selector expansion for filters includes the following:
- time flags for details filtering
- testing
---

#### Does this PR need a docs update or release note?

- [x] ✅ Yes, it's included

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* related to #2602<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] ⚡ Unit test
This commit is contained in:
Danny 2023-03-06 14:34:50 -08:00 committed by GitHub
parent d1404626f1
commit 9cfaf3c140
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 197 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/cli/backup/onedrive.go
View File

@ -63,11 +63,6 @@ corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \
var (
folderPaths []string
fileNames []string
fileCreatedAfter string
fileCreatedBefore string
fileModifiedAfter string
fileModifiedBefore string
)
// called by backup.go to map subcommands to provider-specific handling.
@ -125,20 +120,20 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
// onedrive info flags
fs.StringVar(
&fileCreatedAfter,
&utils.FileCreatedAfter,
utils.FileCreatedAfterFN, "",
"Select backup details for files created after this datetime.")
fs.StringVar(
&fileCreatedBefore,
&utils.FileCreatedBefore,
utils.FileCreatedBeforeFN, "",
"Select backup details for files created before this datetime.")
fs.StringVar(
&fileModifiedAfter,
&utils.FileModifiedAfter,
utils.FileModifiedAfterFN, "",
"Select backup details for files modified after this datetime.")
fs.StringVar(
&fileModifiedBefore,
&utils.FileModifiedBefore,
utils.FileModifiedBeforeFN, "",
"Select backup details for files modified before this datetime.")
@ -361,12 +356,12 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
opts := utils.OneDriveOpts{
Users: user,
Paths: folderPaths,
Names: fileNames,
FileCreatedAfter: fileCreatedAfter,
FileCreatedBefore: fileCreatedBefore,
FileModifiedAfter: fileModifiedAfter,
FileModifiedBefore: fileModifiedBefore,
Paths: folderPaths,
FileCreatedAfter: utils.FileCreatedAfter,
FileCreatedBefore: utils.FileCreatedBefore,
FileModifiedAfter: utils.FileModifiedAfter,
FileModifiedBefore: utils.FileModifiedBefore,
Populated: utils.GetPopulatedFlags(cmd),
}

45
src/cli/backup/sharepoint.go
View File

@ -70,7 +70,11 @@ corso backup delete sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd`
sharePointServiceCommandDetailsExamples = `# Explore <site>'s files from backup 1234abcd-12ab-cd34-56de-1234abcd
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --site <site_id>`
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --site <site_id>
# Find all site files that were created before a certain date.
corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--web-url https://example.com --file-created-before 2015-01-01T00:00:00
`
)
// called by backup.go to map subcommands to provider-specific handling.
@ -152,12 +156,26 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
"Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.",
)
// info flags
// sharepoint info flags
// fs.StringVar(
// &fileCreatedAfter,
// utils.FileCreatedAfterFN, "",
// "Select backup details for items created after this datetime.")
fs.StringVar(
&utils.FileCreatedAfter,
utils.FileCreatedAfterFN, "",
"Select backup details for items created after this datetime.")
fs.StringVar(
&utils.FileCreatedBefore,
utils.FileCreatedBeforeFN, "",
"Select backup details for files created before this datetime.")
fs.StringVar(
&utils.FileModifiedAfter,
utils.FileModifiedAfterFN, "",
"Select backup details for files modified after this datetime.")
fs.StringVar(
&utils.FileModifiedBefore,
utils.FileModifiedBeforeFN, "",
"Select backup details for files modified before this datetime.")
case deleteCommand:
c, fs = utils.AddCommand(cmd, sharePointDeleteCmd(), utils.MarkPreReleaseCommand())
@ -491,12 +509,15 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
defer utils.CloseRepo(ctx, r)
opts := utils.SharePointOpts{
LibraryItems: libraryItems,
LibraryPaths: libraryPaths,
Sites: site,
WebURLs: weburl,
Populated: utils.GetPopulatedFlags(cmd),
LibraryItems: libraryItems,
LibraryPaths: libraryPaths,
Sites: site,
WebURLs: weburl,
FileCreatedAfter: utils.FileCreatedAfter,
FileCreatedBefore: utils.FileCreatedBefore,
FileModifiedAfter: utils.FileModifiedAfter,
FileModifiedBefore: utils.FileModifiedBefore,
Populated: utils.GetPopulatedFlags(cmd),
}
ds, err := runDetailsSharePointCmd(ctx, r, backupID, opts, ctrlOpts.SkipReduce)

10
src/cli/restore/onedrive.go
View File

@ -133,12 +133,12 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error {
opts := utils.OneDriveOpts{
Users: user,
Paths: folderPaths,
Names: fileNames,
FileCreatedAfter: fileCreatedAfter,
FileCreatedBefore: fileCreatedBefore,
FileModifiedAfter: fileModifiedAfter,
FileModifiedBefore: fileModifiedBefore,
Paths: folderPaths,
FileCreatedAfter: utils.FileCreatedAfter,
FileCreatedBefore: utils.FileCreatedBefore,
FileModifiedAfter: utils.FileModifiedAfter,
FileModifiedBefore: utils.FileModifiedBefore,
Populated: utils.GetPopulatedFlags(cmd),
}

24
src/cli/restore/sharepoint.go
View File

@ -141,17 +141,19 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error {
}
opts := utils.SharePointOpts{
ListItems: listItems,
ListPaths: listPaths,
LibraryItems: libraryItems,
LibraryPaths: libraryPaths,
PageFolders: pageFolders,
Pages: pages,
Sites: site,
WebURLs: weburl,
// FileCreatedAfter: fileCreatedAfter,
Populated: utils.GetPopulatedFlags(cmd),
LibraryItems: libraryItems,
LibraryPaths: libraryPaths,
ListItems: listItems,
ListPaths: listPaths,
PageFolders: pageFolders,
Pages: pages,
Sites: site,
WebURLs: weburl,
FileCreatedAfter: utils.FileCreatedAfter,
FileCreatedBefore: utils.FileCreatedBefore,
FileModifiedAfter: utils.FileModifiedAfter,
FileModifiedBefore: utils.FileModifiedBefore,
Populated: utils.GetPopulatedFlags(cmd),
}
if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil {

14
src/cli/utils/flags.go
View File

@ -10,6 +10,20 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
// ==============================================
// Folder Object flags
// These options are flags for indicating
// that a time-based filter should be used for
// within returning objects for details.
// Used by: OneDrive, SharePoint
// ================================================
var (
FileCreatedAfter string
FileCreatedBefore string
FileModifiedAfter string
FileModifiedBefore string
)
type PopulatedFlags map[string]struct{}
func (fs PopulatedFlags) populate(pf *pflag.Flag) {

12
src/cli/utils/onedrive.go
View File

@ -7,14 +7,10 @@ import (
)
const (
FileFN = "file"
FolderFN = "folder"
NamesFN = "name"
PathsFN = "path"
FileCreatedAfterFN = "file-created-after"
FileCreatedBeforeFN = "file-created-before"
FileModifiedAfterFN = "file-modified-after"
FileModifiedBeforeFN = "file-modified-before"
FileFN = "file"
FolderFN = "folder"
NamesFN = "name"
PathsFN = "path"
)
type OneDriveOpts struct {

45
src/cli/utils/sharepoint.go
View File

@ -2,6 +2,7 @@ package utils
import (
"errors"
"fmt"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -17,14 +18,18 @@ const (
)
type SharePointOpts struct {
LibraryItems []string
LibraryPaths []string
ListItems []string
ListPaths []string
PageFolders []string
Pages []string
Sites []string
WebURLs []string
LibraryItems []string
LibraryPaths []string
ListItems []string
ListPaths []string
PageFolders []string
Pages []string
Sites []string
WebURLs []string
FileCreatedAfter string
FileCreatedBefore string
FileModifiedAfter string
FileModifiedBefore string
Populated PopulatedFlags
}
@ -35,9 +40,22 @@ func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error
return errors.New("a backup ID is required")
}
// if _, ok := opts.Populated[FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
// return errors.New("invalid time format for created-after")
// }
if _, ok := opts.Populated[FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
fmt.Printf("What was I sent: %v\n", opts.FileCreatedAfter)
return errors.New("invalid time format for " + FileCreatedAfterFN)
}
if _, ok := opts.Populated[FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
return errors.New("invalid time format for " + FileCreatedBeforeFN)
}
if _, ok := opts.Populated[FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
return errors.New("invalid time format for " + FileModifiedAfterFN)
}
if _, ok := opts.Populated[FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
return errors.New("invalid time format for " + FileModifiedBeforeFN)
}
return nil
}
@ -149,5 +167,8 @@ func FilterSharePointRestoreInfoSelectors(
sel *selectors.SharePointRestore,
opts SharePointOpts,
) {
// AddSharePointFilter(sel, opts.FileCreatedAfter, sel.CreatedAfter)
AddSharePointFilter(sel, opts.FileCreatedAfter, sel.CreatedAfter)
AddSharePointFilter(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddSharePointFilter(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddSharePointFilter(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
}

12
src/cli/utils/utils.go
View File

@ -19,10 +19,14 @@ import (
// common flag names
const (
BackupFN = "backup"
DataFN = "data"
SiteFN = "site"
UserFN = "user"
BackupFN = "backup"
DataFN = "data"
SiteFN = "site"
UserFN = "user"
FileCreatedAfterFN = "file-created-after"
FileCreatedBeforeFN = "file-created-before"
FileModifiedAfterFN = "file-modified-after"
FileModifiedBeforeFN = "file-modified-before"
)
const (

54
src/pkg/selectors/sharepoint.go
View File

@ -3,8 +3,10 @@ package selectors
import (
"context"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
)
@ -333,6 +335,46 @@ func (s *sharePoint) PageItems(pages, items []string, opts ...option) []SharePoi
// -------------------
// Filter Factories
func (s *sharePoint) CreatedAfter(timeStrings string) []SharePointScope {
return []SharePointScope{
makeFilterScope[SharePointScope](
SharePointLibraryItem,
FileFilterCreatedAfter,
[]string{timeStrings},
wrapFilter(filters.Less)),
}
}
func (s *sharePoint) CreatedBefore(timeStrings string) []SharePointScope {
return []SharePointScope{
makeFilterScope[SharePointScope](
SharePointLibraryItem,
FileFilterCreatedBefore,
[]string{timeStrings},
wrapFilter(filters.Greater)),
}
}
func (s *sharePoint) ModifiedAfter(timeStrings string) []SharePointScope {
return []SharePointScope{
makeFilterScope[SharePointScope](
SharePointLibraryItem,
FileFilterModifiedAfter,
[]string{timeStrings},
wrapFilter(filters.Less)),
}
}
func (s *sharePoint) ModifiedBefore(timeStrings string) []SharePointScope {
return []SharePointScope{
makeFilterScope[SharePointScope](
SharePointLibraryItem,
FileFilterModifiedBefore,
[]string{timeStrings},
wrapFilter(filters.Greater)),
}
}
// ---------------------------------------------------------------------------
// Categories
// ---------------------------------------------------------------------------
@ -358,6 +400,10 @@ const (
SharePointPage sharePointCategory = "SharePointPage"
// filterable topics identified by SharePoint
SiteFilterCreatedAfter sharePointCategory = "FileFilterCreatedAfter"
SiteFilterCreatedBefore sharePointCategory = "FileFilterCreatedBefore"
SiteFilterModifiedAfter sharePointCategory = "FileFilterModifiedAfter"
SiteFilterModifiedBefore sharePointCategory = "FileFilterModifiedBefore"
)
// sharePointLeafProperties describes common metadata of the leaf categories
@ -391,7 +437,9 @@ func (c sharePointCategory) String() string {
// Ex: ServiceUser.leafCat() => ServiceUser
func (c sharePointCategory) leafCat() categorizer {
switch c {
case SharePointLibrary, SharePointLibraryItem:
case SharePointLibrary, SharePointLibraryItem,
SiteFilterCreatedAfter, SiteFilterCreatedBefore,
SiteFilterModifiedAfter, SiteFilterModifiedBefore:
return SharePointLibraryItem
case SharePointList, SharePointListItem:
return SharePointListItem
@ -600,6 +648,10 @@ func (s SharePointScope) matchesInfo(dii details.ItemInfo) bool {
switch filterCat {
case SharePointWebURL:
i = info.WebURL
case SiteFilterCreatedAfter, SiteFilterCreatedBefore:
i = common.FormatTime(info.Created)
case SiteFilterModifiedAfter, SiteFilterModifiedBefore:
i = common.FormatTime(info.Modified)
}
return s.Matches(filterCat, i)

29
src/pkg/selectors/sharepoint_test.go
View File

@ -2,11 +2,13 @@ package selectors
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
@ -363,10 +365,14 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() {
func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
var (
ods = NewSharePointRestore(nil)
host = "www.website.com"
pth = "/foo"
url = host + pth
ods = NewSharePointRestore(Any())
host = "www.website.com"
pth = "/foo"
url = host + pth
epoch = time.Time{}
now = time.Now()
modification = now.Add(15 * time.Minute)
future = now.Add(45 * time.Minute)
)
table := []struct {
@ -385,6 +391,19 @@ func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
{"host does not contain substring", host, ods.WebURL([]string{"website"}), assert.False},
{"url does not suffix substring", url, ods.WebURL([]string{"oo"}), assert.False},
{"host mismatch", host, ods.WebURL([]string{"www.google.com"}), assert.False},
{"file create after the epoch", host, ods.CreatedAfter(common.FormatTime(epoch)), assert.True},
{"file create after now", host, ods.CreatedAfter(common.FormatTime(now)), assert.False},
{"file create after later", url, ods.CreatedAfter(common.FormatTime(future)), assert.False},
{"file create before future", host, ods.CreatedBefore(common.FormatTime(future)), assert.True},
{"file create before now", host, ods.CreatedBefore(common.FormatTime(now)), assert.False},
{"file create before modification", host, ods.CreatedBefore(common.FormatTime(modification)), assert.True},
{"file create before epoch", host, ods.CreatedBefore(common.FormatTime(now)), assert.False},
{"file modified after the epoch", host, ods.ModifiedAfter(common.FormatTime(epoch)), assert.True},
{"file modified after now", host, ods.ModifiedAfter(common.FormatTime(now)), assert.True},
{"file modified after later", host, ods.ModifiedAfter(common.FormatTime(future)), assert.False},
{"file modified before future", host, ods.ModifiedBefore(common.FormatTime(future)), assert.True},
{"file modified before now", host, ods.ModifiedBefore(common.FormatTime(now)), assert.False},
{"file modified before epoch", host, ods.ModifiedBefore(common.FormatTime(now)), assert.False},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -394,6 +413,8 @@ func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() {
SharePoint: &details.SharePointInfo{
ItemType: details.SharePointItem,
WebURL: test.infoURL,
Created: now,
Modified: modification,
},
}