From 9cfaf3c1404816a07fe257f785b493fc451f1f3c Mon Sep 17 00:00:00 2001 From: Danny Date: Mon, 6 Mar 2023 14:34:50 -0800 Subject: [PATCH] CLI: `SharePoint.Libraries` selector expansion (#2679) Selector expansion for filters includes the following: - time flags for details filtering - testing --- #### Does this PR need a docs update or release note? - [x] :white_check_mark: Yes, it's included #### Type of change - [x] :sunflower: Feature #### Issue(s) * related to #2602 #### Test Plan - [x] :zap: Unit test --- src/cli/backup/onedrive.go | 23 +++++------- src/cli/backup/sharepoint.go | 45 ++++++++++++++++------- src/cli/restore/onedrive.go | 10 +++--- src/cli/restore/sharepoint.go | 24 +++++++------ src/cli/utils/flags.go | 14 ++++++++ src/cli/utils/onedrive.go | 12 +++---- src/cli/utils/sharepoint.go | 45 ++++++++++++++++------- src/cli/utils/utils.go | 12 ++++--- src/pkg/selectors/sharepoint.go | 54 +++++++++++++++++++++++++++- src/pkg/selectors/sharepoint_test.go | 29 ++++++++++++--- 10 files changed, 197 insertions(+), 71 deletions(-) diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index e74d719be..c79fc32a7 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -63,11 +63,6 @@ corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \ var ( folderPaths []string fileNames []string - - fileCreatedAfter string - fileCreatedBefore string - fileModifiedAfter string - fileModifiedBefore string ) // called by backup.go to map subcommands to provider-specific handling. @@ -125,20 +120,20 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { // onedrive info flags fs.StringVar( - &fileCreatedAfter, + &utils.FileCreatedAfter, utils.FileCreatedAfterFN, "", "Select backup details for files created after this datetime.") fs.StringVar( - &fileCreatedBefore, + &utils.FileCreatedBefore, utils.FileCreatedBeforeFN, "", "Select backup details for files created before this datetime.") fs.StringVar( - &fileModifiedAfter, + &utils.FileModifiedAfter, utils.FileModifiedAfterFN, "", "Select backup details for files modified after this datetime.") fs.StringVar( - &fileModifiedBefore, + &utils.FileModifiedBefore, utils.FileModifiedBeforeFN, "", "Select backup details for files modified before this datetime.") @@ -361,12 +356,12 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error { opts := utils.OneDriveOpts{ Users: user, - Paths: folderPaths, Names: fileNames, - FileCreatedAfter: fileCreatedAfter, - FileCreatedBefore: fileCreatedBefore, - FileModifiedAfter: fileModifiedAfter, - FileModifiedBefore: fileModifiedBefore, + Paths: folderPaths, + FileCreatedAfter: utils.FileCreatedAfter, + FileCreatedBefore: utils.FileCreatedBefore, + FileModifiedAfter: utils.FileModifiedAfter, + FileModifiedBefore: utils.FileModifiedBefore, Populated: utils.GetPopulatedFlags(cmd), } diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index d6811b2e8..f5ed1bd87 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -70,7 +70,11 @@ corso backup delete sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd` sharePointServiceCommandDetailsExamples = `# Explore 's files from backup 1234abcd-12ab-cd34-56de-1234abcd -corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --site ` +corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --site +# Find all site files that were created before a certain date. +corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --web-url https://example.com --file-created-before 2015-01-01T00:00:00 +` ) // called by backup.go to map subcommands to provider-specific handling. @@ -152,12 +156,26 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { "Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.", ) - // info flags + // sharepoint info flags - // fs.StringVar( - // &fileCreatedAfter, - // utils.FileCreatedAfterFN, "", - // "Select backup details for items created after this datetime.") + fs.StringVar( + &utils.FileCreatedAfter, + utils.FileCreatedAfterFN, "", + "Select backup details for items created after this datetime.") + + fs.StringVar( + &utils.FileCreatedBefore, + utils.FileCreatedBeforeFN, "", + "Select backup details for files created before this datetime.") + + fs.StringVar( + &utils.FileModifiedAfter, + utils.FileModifiedAfterFN, "", + "Select backup details for files modified after this datetime.") + fs.StringVar( + &utils.FileModifiedBefore, + utils.FileModifiedBeforeFN, "", + "Select backup details for files modified before this datetime.") case deleteCommand: c, fs = utils.AddCommand(cmd, sharePointDeleteCmd(), utils.MarkPreReleaseCommand()) @@ -491,12 +509,15 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error { defer utils.CloseRepo(ctx, r) opts := utils.SharePointOpts{ - LibraryItems: libraryItems, - LibraryPaths: libraryPaths, - Sites: site, - WebURLs: weburl, - - Populated: utils.GetPopulatedFlags(cmd), + LibraryItems: libraryItems, + LibraryPaths: libraryPaths, + Sites: site, + WebURLs: weburl, + FileCreatedAfter: utils.FileCreatedAfter, + FileCreatedBefore: utils.FileCreatedBefore, + FileModifiedAfter: utils.FileModifiedAfter, + FileModifiedBefore: utils.FileModifiedBefore, + Populated: utils.GetPopulatedFlags(cmd), } ds, err := runDetailsSharePointCmd(ctx, r, backupID, opts, ctrlOpts.SkipReduce) diff --git a/src/cli/restore/onedrive.go b/src/cli/restore/onedrive.go index 56139d83d..048e3838b 100644 --- a/src/cli/restore/onedrive.go +++ b/src/cli/restore/onedrive.go @@ -133,12 +133,12 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { opts := utils.OneDriveOpts{ Users: user, - Paths: folderPaths, Names: fileNames, - FileCreatedAfter: fileCreatedAfter, - FileCreatedBefore: fileCreatedBefore, - FileModifiedAfter: fileModifiedAfter, - FileModifiedBefore: fileModifiedBefore, + Paths: folderPaths, + FileCreatedAfter: utils.FileCreatedAfter, + FileCreatedBefore: utils.FileCreatedBefore, + FileModifiedAfter: utils.FileModifiedAfter, + FileModifiedBefore: utils.FileModifiedBefore, Populated: utils.GetPopulatedFlags(cmd), } diff --git a/src/cli/restore/sharepoint.go b/src/cli/restore/sharepoint.go index a18052b37..4a78b45ed 100644 --- a/src/cli/restore/sharepoint.go +++ b/src/cli/restore/sharepoint.go @@ -141,17 +141,19 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error { } opts := utils.SharePointOpts{ - ListItems: listItems, - ListPaths: listPaths, - LibraryItems: libraryItems, - LibraryPaths: libraryPaths, - PageFolders: pageFolders, - Pages: pages, - Sites: site, - WebURLs: weburl, - // FileCreatedAfter: fileCreatedAfter, - - Populated: utils.GetPopulatedFlags(cmd), + LibraryItems: libraryItems, + LibraryPaths: libraryPaths, + ListItems: listItems, + ListPaths: listPaths, + PageFolders: pageFolders, + Pages: pages, + Sites: site, + WebURLs: weburl, + FileCreatedAfter: utils.FileCreatedAfter, + FileCreatedBefore: utils.FileCreatedBefore, + FileModifiedAfter: utils.FileModifiedAfter, + FileModifiedBefore: utils.FileModifiedBefore, + Populated: utils.GetPopulatedFlags(cmd), } if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil { diff --git a/src/cli/utils/flags.go b/src/cli/utils/flags.go index c4380e13f..ed446914d 100644 --- a/src/cli/utils/flags.go +++ b/src/cli/utils/flags.go @@ -10,6 +10,20 @@ import ( "github.com/alcionai/corso/src/pkg/path" ) +// ============================================== +// Folder Object flags +// These options are flags for indicating +// that a time-based filter should be used for +// within returning objects for details. +// Used by: OneDrive, SharePoint +// ================================================ +var ( + FileCreatedAfter string + FileCreatedBefore string + FileModifiedAfter string + FileModifiedBefore string +) + type PopulatedFlags map[string]struct{} func (fs PopulatedFlags) populate(pf *pflag.Flag) { diff --git a/src/cli/utils/onedrive.go b/src/cli/utils/onedrive.go index 4951a7b63..17d71fbe2 100644 --- a/src/cli/utils/onedrive.go +++ b/src/cli/utils/onedrive.go @@ -7,14 +7,10 @@ import ( ) const ( - FileFN = "file" - FolderFN = "folder" - NamesFN = "name" - PathsFN = "path" - FileCreatedAfterFN = "file-created-after" - FileCreatedBeforeFN = "file-created-before" - FileModifiedAfterFN = "file-modified-after" - FileModifiedBeforeFN = "file-modified-before" + FileFN = "file" + FolderFN = "folder" + NamesFN = "name" + PathsFN = "path" ) type OneDriveOpts struct { diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index fb9d39533..6724c81a3 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -2,6 +2,7 @@ package utils import ( "errors" + "fmt" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -17,14 +18,18 @@ const ( ) type SharePointOpts struct { - LibraryItems []string - LibraryPaths []string - ListItems []string - ListPaths []string - PageFolders []string - Pages []string - Sites []string - WebURLs []string + LibraryItems []string + LibraryPaths []string + ListItems []string + ListPaths []string + PageFolders []string + Pages []string + Sites []string + WebURLs []string + FileCreatedAfter string + FileCreatedBefore string + FileModifiedAfter string + FileModifiedBefore string Populated PopulatedFlags } @@ -35,9 +40,22 @@ func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error return errors.New("a backup ID is required") } - // if _, ok := opts.Populated[FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) { - // return errors.New("invalid time format for created-after") - // } + if _, ok := opts.Populated[FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) { + fmt.Printf("What was I sent: %v\n", opts.FileCreatedAfter) + return errors.New("invalid time format for " + FileCreatedAfterFN) + } + + if _, ok := opts.Populated[FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) { + return errors.New("invalid time format for " + FileCreatedBeforeFN) + } + + if _, ok := opts.Populated[FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) { + return errors.New("invalid time format for " + FileModifiedAfterFN) + } + + if _, ok := opts.Populated[FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) { + return errors.New("invalid time format for " + FileModifiedBeforeFN) + } return nil } @@ -149,5 +167,8 @@ func FilterSharePointRestoreInfoSelectors( sel *selectors.SharePointRestore, opts SharePointOpts, ) { - // AddSharePointFilter(sel, opts.FileCreatedAfter, sel.CreatedAfter) + AddSharePointFilter(sel, opts.FileCreatedAfter, sel.CreatedAfter) + AddSharePointFilter(sel, opts.FileCreatedBefore, sel.CreatedBefore) + AddSharePointFilter(sel, opts.FileModifiedAfter, sel.ModifiedAfter) + AddSharePointFilter(sel, opts.FileModifiedBefore, sel.ModifiedBefore) } diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 3f2b58675..f9921dd58 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -19,10 +19,14 @@ import ( // common flag names const ( - BackupFN = "backup" - DataFN = "data" - SiteFN = "site" - UserFN = "user" + BackupFN = "backup" + DataFN = "data" + SiteFN = "site" + UserFN = "user" + FileCreatedAfterFN = "file-created-after" + FileCreatedBeforeFN = "file-created-before" + FileModifiedAfterFN = "file-modified-after" + FileModifiedBeforeFN = "file-modified-before" ) const ( diff --git a/src/pkg/selectors/sharepoint.go b/src/pkg/selectors/sharepoint.go index 4da71852f..c0407f75a 100644 --- a/src/pkg/selectors/sharepoint.go +++ b/src/pkg/selectors/sharepoint.go @@ -3,8 +3,10 @@ package selectors import ( "context" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" ) @@ -333,6 +335,46 @@ func (s *sharePoint) PageItems(pages, items []string, opts ...option) []SharePoi // ------------------- // Filter Factories +func (s *sharePoint) CreatedAfter(timeStrings string) []SharePointScope { + return []SharePointScope{ + makeFilterScope[SharePointScope]( + SharePointLibraryItem, + FileFilterCreatedAfter, + []string{timeStrings}, + wrapFilter(filters.Less)), + } +} + +func (s *sharePoint) CreatedBefore(timeStrings string) []SharePointScope { + return []SharePointScope{ + makeFilterScope[SharePointScope]( + SharePointLibraryItem, + FileFilterCreatedBefore, + []string{timeStrings}, + wrapFilter(filters.Greater)), + } +} + +func (s *sharePoint) ModifiedAfter(timeStrings string) []SharePointScope { + return []SharePointScope{ + makeFilterScope[SharePointScope]( + SharePointLibraryItem, + FileFilterModifiedAfter, + []string{timeStrings}, + wrapFilter(filters.Less)), + } +} + +func (s *sharePoint) ModifiedBefore(timeStrings string) []SharePointScope { + return []SharePointScope{ + makeFilterScope[SharePointScope]( + SharePointLibraryItem, + FileFilterModifiedBefore, + []string{timeStrings}, + wrapFilter(filters.Greater)), + } +} + // --------------------------------------------------------------------------- // Categories // --------------------------------------------------------------------------- @@ -358,6 +400,10 @@ const ( SharePointPage sharePointCategory = "SharePointPage" // filterable topics identified by SharePoint + SiteFilterCreatedAfter sharePointCategory = "FileFilterCreatedAfter" + SiteFilterCreatedBefore sharePointCategory = "FileFilterCreatedBefore" + SiteFilterModifiedAfter sharePointCategory = "FileFilterModifiedAfter" + SiteFilterModifiedBefore sharePointCategory = "FileFilterModifiedBefore" ) // sharePointLeafProperties describes common metadata of the leaf categories @@ -391,7 +437,9 @@ func (c sharePointCategory) String() string { // Ex: ServiceUser.leafCat() => ServiceUser func (c sharePointCategory) leafCat() categorizer { switch c { - case SharePointLibrary, SharePointLibraryItem: + case SharePointLibrary, SharePointLibraryItem, + SiteFilterCreatedAfter, SiteFilterCreatedBefore, + SiteFilterModifiedAfter, SiteFilterModifiedBefore: return SharePointLibraryItem case SharePointList, SharePointListItem: return SharePointListItem @@ -600,6 +648,10 @@ func (s SharePointScope) matchesInfo(dii details.ItemInfo) bool { switch filterCat { case SharePointWebURL: i = info.WebURL + case SiteFilterCreatedAfter, SiteFilterCreatedBefore: + i = common.FormatTime(info.Created) + case SiteFilterModifiedAfter, SiteFilterModifiedBefore: + i = common.FormatTime(info.Modified) } return s.Matches(filterCat, i) diff --git a/src/pkg/selectors/sharepoint_test.go b/src/pkg/selectors/sharepoint_test.go index f1db389d7..a7b3aa701 100644 --- a/src/pkg/selectors/sharepoint_test.go +++ b/src/pkg/selectors/sharepoint_test.go @@ -2,11 +2,13 @@ package selectors import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/tester" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" @@ -363,10 +365,14 @@ func (suite *SharePointSelectorSuite) TestSharePointCategory_PathValues() { func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() { var ( - ods = NewSharePointRestore(nil) - host = "www.website.com" - pth = "/foo" - url = host + pth + ods = NewSharePointRestore(Any()) + host = "www.website.com" + pth = "/foo" + url = host + pth + epoch = time.Time{} + now = time.Now() + modification = now.Add(15 * time.Minute) + future = now.Add(45 * time.Minute) ) table := []struct { @@ -385,6 +391,19 @@ func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() { {"host does not contain substring", host, ods.WebURL([]string{"website"}), assert.False}, {"url does not suffix substring", url, ods.WebURL([]string{"oo"}), assert.False}, {"host mismatch", host, ods.WebURL([]string{"www.google.com"}), assert.False}, + {"file create after the epoch", host, ods.CreatedAfter(common.FormatTime(epoch)), assert.True}, + {"file create after now", host, ods.CreatedAfter(common.FormatTime(now)), assert.False}, + {"file create after later", url, ods.CreatedAfter(common.FormatTime(future)), assert.False}, + {"file create before future", host, ods.CreatedBefore(common.FormatTime(future)), assert.True}, + {"file create before now", host, ods.CreatedBefore(common.FormatTime(now)), assert.False}, + {"file create before modification", host, ods.CreatedBefore(common.FormatTime(modification)), assert.True}, + {"file create before epoch", host, ods.CreatedBefore(common.FormatTime(now)), assert.False}, + {"file modified after the epoch", host, ods.ModifiedAfter(common.FormatTime(epoch)), assert.True}, + {"file modified after now", host, ods.ModifiedAfter(common.FormatTime(now)), assert.True}, + {"file modified after later", host, ods.ModifiedAfter(common.FormatTime(future)), assert.False}, + {"file modified before future", host, ods.ModifiedBefore(common.FormatTime(future)), assert.True}, + {"file modified before now", host, ods.ModifiedBefore(common.FormatTime(now)), assert.False}, + {"file modified before epoch", host, ods.ModifiedBefore(common.FormatTime(now)), assert.False}, } for _, test := range table { suite.Run(test.name, func() { @@ -394,6 +413,8 @@ func (suite *SharePointSelectorSuite) TestSharePointScope_MatchesInfo() { SharePoint: &details.SharePointInfo{ ItemType: details.SharePointItem, WebURL: test.infoURL, + Created: now, + Modified: modification, }, }