larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CLI and selector changes for groups restore (#4218)

Browse Source 
<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #<issue>

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2023-09-14 14:59:01 +05:30 committed by GitHub
parent 80f11d9876
commit edf753382e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 426 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/cli/flags/groups.go
View File

@ -11,7 +11,7 @@ const GroupFN = "group"
var GroupFV []string
func AddGroupDetailsAndRestoreFlags(cmd *cobra.Command) {
// TODO: implement flags
// TODO: implement groups specific flags
}
// AddGroupFlag adds the --group flag, which accepts id or name values.

2
src/cli/restore/groups.go
View File

@ -28,6 +28,8 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
flags.AddBackupIDFlag(c, true)
flags.AddRestorePermissionsFlag(c)
flags.AddSharePointDetailsAndRestoreFlags(c) // for sp restores
flags.AddSiteIDFlag(c)
flags.AddRestoreConfigFlags(c)
flags.AddFailFastFlag(c)
flags.AddCorsoPassphaseFlags(c)

20
src/cli/restore/groups_test.go
View File

@ -62,6 +62,18 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
"--" + flags.RunModeFN, flags.RunModeFlagTest,
"--" + flags.BackupFN, testdata.BackupInput,
"--" + flags.LibraryFN, testdata.LibraryInput,
"--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput),
"--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput),
"--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput,
"--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput,
"--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput,
"--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput,
"--" + flags.ListItemFN, testdata.FlgInputs(testdata.ListItemInput),
"--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput),
"--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput),
"--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput),
"--" + flags.CollisionsFN, testdata.Collisions,
"--" + flags.DestinationFN, testdata.Destination,
"--" + flags.ToResourceFN, testdata.ToResource,
@ -88,6 +100,14 @@ func (suite *GroupsUnitSuite) TestAddGroupsCommands() {
opts := utils.MakeGroupsOpts(cmd)
assert.Equal(t, testdata.BackupInput, flags.BackupIDFV)
assert.Equal(t, testdata.LibraryInput, opts.Library)
assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName)
assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath)
assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter)
assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore)
assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter)
assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore)
assert.Equal(t, testdata.Collisions, opts.RestoreCfg.Collisions)
assert.Equal(t, testdata.Destination, opts.RestoreCfg.Destination)
assert.Equal(t, testdata.ToResource, opts.RestoreCfg.ProtectedResource)

122
src/cli/utils/groups.go
View File

@ -13,6 +13,21 @@ import (
type GroupsOpts struct {
Groups []string
SiteID []string
Library string
FileName []string // for libraries, to duplicate onedrive interface
FolderPath []string // for libraries, to duplicate onedrive interface
FileCreatedAfter string
FileCreatedBefore string
FileModifiedAfter string
FileModifiedBefore string
ListFolder []string
ListItem []string
PageFolder []string
Page []string
RestoreCfg RestoreCfgOpts
ExportCfg ExportCfgOpts
@ -45,7 +60,23 @@ func AddGroupsCategories(sel *selectors.GroupsBackup, cats []string) *selectors.
func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
return GroupsOpts{
Groups: flags.UserFV,
Groups: flags.GroupFV,
SiteID: flags.SiteIDFV,
Library: flags.LibraryFV,
FileName: flags.FileNameFV,
FolderPath: flags.FolderPathFV,
FileCreatedAfter: flags.FileCreatedAfterFV,
FileCreatedBefore: flags.FileCreatedBeforeFV,
FileModifiedAfter: flags.FileModifiedAfterFV,
FileModifiedBefore: flags.FileModifiedBeforeFV,
ListFolder: flags.ListFolderFV,
ListItem: flags.ListItemFV,
Page: flags.PageFV,
PageFolder: flags.PageFolderFV,
RestoreCfg: makeRestoreCfgOpts(cmd),
ExportCfg: makeExportCfgOpts(cmd),
@ -63,7 +94,21 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts) error {
return clues.New("a backup ID is required")
}
// TODO(meain): selectors (refer sharepoint)
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
}
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
}
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
}
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
}
return validateRestoreConfigFlags(flags.CollisionsFV, opts.RestoreCfg)
}
@ -87,16 +132,76 @@ func AddGroupInfo(
func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *selectors.GroupsRestore {
groups := opts.Groups
ls := len(opts.Groups)
lg := len(opts.Groups)
if ls == 0 {
// TODO(meain): handle sites once we add non-root site backup
// ls := len(opts.SiteID)
lfp, lfn := len(opts.FolderPath), len(opts.FileName)
slp, sli := len(opts.ListFolder), len(opts.ListItem)
pf, pi := len(opts.PageFolder), len(opts.Page)
if lg == 0 {
groups = selectors.Any()
}
sel := selectors.NewGroupsRestore(groups)
// TODO(meain): add selectors
if lfp+lfn+slp+sli+pf+pi == 0 {
sel.Include(sel.AllData())
return sel
}
if lfp+lfn > 0 {
if lfn == 0 {
opts.FileName = selectors.Any()
}
opts.FolderPath = trimFolderSlash(opts.FolderPath)
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.FolderPath)
if len(containsFolders) > 0 {
sel.Include(sel.LibraryItems(containsFolders, opts.FileName))
}
if len(prefixFolders) > 0 {
sel.Include(sel.LibraryItems(prefixFolders, opts.FileName, selectors.PrefixMatch()))
}
}
if slp+sli > 0 {
if sli == 0 {
opts.ListItem = selectors.Any()
}
opts.ListFolder = trimFolderSlash(opts.ListFolder)
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListFolder)
if len(containsFolders) > 0 {
sel.Include(sel.ListItems(containsFolders, opts.ListItem))
}
if len(prefixFolders) > 0 {
sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch()))
}
}
if pf+pi > 0 {
if pi == 0 {
opts.Page = selectors.Any()
}
opts.PageFolder = trimFolderSlash(opts.PageFolder)
containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.PageFolder)
if len(containsFolders) > 0 {
sel.Include(sel.PageItems(containsFolders, opts.Page))
}
if len(prefixFolders) > 0 {
sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch()))
}
}
return sel
}
@ -106,6 +211,9 @@ func FilterGroupsRestoreInfoSelectors(
sel *selectors.GroupsRestore,
opts GroupsOpts,
) {
// TODO(meain)
// AddGroupInfo(sel, opts.GroupID, sel.Library)
AddGroupInfo(sel, opts.Library, sel.Library)
AddGroupInfo(sel, opts.FileCreatedAfter, sel.CreatedAfter)
AddGroupInfo(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddGroupInfo(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddGroupInfo(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
}

227
src/cli/utils/groups_test.go
View File

@ -8,7 +8,9 @@ import (
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/dttm"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -27,6 +29,10 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
empty = []string{}
single = []string{"single"}
multi = []string{"more", "than", "one"}
containsOnly = []string{"contains"}
prefixOnly = []string{"/prefix"}
containsAndPrefix = []string{"contains", "/prefix"}
onlySlash = []string{string(path.PathSeparator)}
)
table := []struct {
@ -60,8 +66,105 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
},
expectIncludeLen: 2,
},
// TODO Add library specific tests once we have filters based
// on library folders
{
name: "library folder contains",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: containsOnly,
SiteID: empty,
},
expectIncludeLen: 1,
},
{
name: "library folder prefixes",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: prefixOnly,
SiteID: empty,
},
expectIncludeLen: 1,
},
{
name: "library folder prefixes and contains",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: containsAndPrefix,
SiteID: empty,
},
expectIncludeLen: 2,
},
{
name: "list contains",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: empty,
ListItem: empty,
ListFolder: containsOnly,
SiteID: empty,
},
expectIncludeLen: 1,
},
{
name: "list prefixes",
opts: utils.GroupsOpts{
ListFolder: prefixOnly,
},
expectIncludeLen: 1,
},
{
name: "list prefixes and contains",
opts: utils.GroupsOpts{
ListFolder: containsAndPrefix,
},
expectIncludeLen: 2,
},
{
name: "library folder suffixes",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: empty,
// SiteID: empty, // TODO(meain): Update once we support multiple sites
},
expectIncludeLen: 2,
},
{
name: "library folder suffixes and contains",
opts: utils.GroupsOpts{
FileName: empty,
FolderPath: empty,
// SiteID: empty, // TODO(meain): update once we support multiple sites
},
expectIncludeLen: 2,
},
{
name: "Page Folder",
opts: utils.GroupsOpts{
PageFolder: single,
},
expectIncludeLen: 1,
},
{
name: "Site Page ",
opts: utils.GroupsOpts{
Page: single,
},
expectIncludeLen: 1,
},
{
name: "Page & library Files",
opts: utils.GroupsOpts{
PageFolder: single,
FileName: multi,
},
expectIncludeLen: 2,
},
{
name: "folder with just /",
opts: utils.GroupsOpts{
FolderPath: onlySlash,
},
expectIncludeLen: 1,
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -95,64 +198,68 @@ func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() {
opts: utils.GroupsOpts{},
expect: assert.Error,
},
// TODO: Add tests for selectors once we have them
// {
// name: "all valid",
// backupID: "id",
// opts: utils.GroupsOpts{
// Populated: flags.PopulatedFlags{
// flags.FileCreatedAfterFN: struct{}{},
// flags.FileCreatedBeforeFN: struct{}{},
// flags.FileModifiedAfterFN: struct{}{},
// flags.FileModifiedBeforeFN: struct{}{},
// },
// },
// expect: assert.NoError,
// },
// {
// name: "invalid file created after",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileCreatedAfter: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileCreatedAfterFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file created before",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileCreatedBefore: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileCreatedBeforeFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file modified after",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileModifiedAfter: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileModifiedAfterFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file modified before",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileModifiedBefore: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileModifiedBeforeFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
{
name: "all valid",
backupID: "id",
opts: utils.GroupsOpts{
FileCreatedAfter: dttm.Now(),
FileCreatedBefore: dttm.Now(),
FileModifiedAfter: dttm.Now(),
FileModifiedBefore: dttm.Now(),
Populated: flags.PopulatedFlags{
flags.SiteFN: struct{}{},
flags.FileCreatedAfterFN: struct{}{},
flags.FileCreatedBeforeFN: struct{}{},
flags.FileModifiedAfterFN: struct{}{},
flags.FileModifiedBeforeFN: struct{}{},
},
},
expect: assert.NoError,
},
{
name: "invalid file created after",
backupID: "id",
opts: utils.GroupsOpts{
FileCreatedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.FileCreatedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid file created before",
backupID: "id",
opts: utils.GroupsOpts{
FileCreatedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.FileCreatedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid file modified after",
backupID: "id",
opts: utils.GroupsOpts{
FileModifiedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.FileModifiedAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid file modified before",
backupID: "id",
opts: utils.GroupsOpts{
FileModifiedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.FileModifiedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
}
for _, test := range table {
suite.Run(test.name, func() {

1
src/internal/m365/backup_test.go
View File

@ -518,7 +518,6 @@ func (suite *GroupsCollectionIntgSuite) TestCreateGroupsCollection_SharePoint()
require.NoError(t, err, clues.ToCore(err))
sel := selectors.NewGroupsBackup(groupIDs)
// TODO(meain): make use of selectors
sel.Include(sel.LibraryFolders([]string{"test"}, selectors.PrefixMatch()))
sel.SetDiscreteOwnerIDName(id, name)

1
src/internal/m365/collection/drive/library_handler.go
View File

@ -152,7 +152,6 @@ func (h libraryBackupHandler) NewLocationIDer(
driveID string,
elems ...string,
) details.LocationIDer {
// TODO(meain): path related changes for groups
return details.NewSharePointLocationIDer(driveID, elems...)
}

118
src/pkg/selectors/groups.go
View File

@ -310,9 +310,111 @@ func (s *groups) LibraryItems(libraries, items []string, opts ...option) []Group
return scopes
}
// Lists produces one or more Groups list scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// Any empty slice defaults to [selectors.None]
func (s *groups) Lists(lists []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(scopes, makeScope[GroupsScope](GroupsList, lists, os...))
return scopes
}
// ListItems produces one or more Groups list item scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
// options are only applied to the list scopes.
func (s *groups) ListItems(lists, items []string, opts ...option) []GroupsScope {
scopes := []GroupsScope{}
scopes = append(
scopes,
makeScope[GroupsScope](GroupsListItem, items, defaultItemOptions(s.Cfg)...).
set(GroupsList, lists, opts...))
return scopes
}
// Pages produces one or more Groups page scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
func (s *groups) Pages(pages []string, opts ...option) []GroupsScope {
var (
scopes = []GroupsScope{}
os = append([]option{pathComparator()}, opts...)
)
scopes = append(scopes, makeScope[GroupsScope](GroupsPageFolder, pages, os...))
return scopes
}
// PageItems produces one or more Groups page item scopes.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
// If any slice contains selectors.None, that slice is reduced to [selectors.None]
// If any slice is empty, it defaults to [selectors.None]
// options are only applied to the page scopes.
func (s *groups) PageItems(pages, items []string, opts ...option) []GroupsScope {
scopes := []GroupsScope{}
scopes = append(
scopes,
makeScope[GroupsScope](GroupsPage, items).
set(GroupsPageFolder, pages, opts...))
return scopes
}
// -------------------
// ItemInfo Factories
func (s *groups) CreatedAfter(timeStrings string) []GroupsScope {
return []GroupsScope{
makeInfoScope[GroupsScope](
GroupsLibraryItem,
GroupsInfoLibraryItemCreatedAfter,
[]string{timeStrings},
filters.Less),
}
}
func (s *groups) CreatedBefore(timeStrings string) []GroupsScope {
return []GroupsScope{
makeInfoScope[GroupsScope](
GroupsLibraryItem,
GroupsInfoLibraryItemCreatedBefore,
[]string{timeStrings},
filters.Greater),
}
}
func (s *groups) ModifiedAfter(timeStrings string) []GroupsScope {
return []GroupsScope{
makeInfoScope[GroupsScope](
GroupsLibraryItem,
GroupsInfoLibraryItemModifiedAfter,
[]string{timeStrings},
filters.Less),
}
}
func (s *groups) ModifiedBefore(timeStrings string) []GroupsScope {
return []GroupsScope{
makeInfoScope[GroupsScope](
GroupsLibraryItem,
GroupsInfoLibraryItemModifiedBefore,
[]string{timeStrings},
filters.Greater),
}
}
// MessageCreator produces one or more groups channelMessage info scopes.
// Matches any channel message created by the specified user.
// If any slice contains selectors.Any, that slice is reduced to [selectors.Any]
@ -404,8 +506,16 @@ const (
GroupsChannelMessage groupsCategory = "GroupsChannelMessage"
GroupsLibraryFolder groupsCategory = "GroupsLibraryFolder"
GroupsLibraryItem groupsCategory = "GroupsLibraryItem"
GroupsList groupsCategory = "GroupsList"
GroupsListItem groupsCategory = "GroupsListItem"
GroupsPageFolder groupsCategory = "GroupsPageFolder"
GroupsPage groupsCategory = "GroupsPage"
// details.itemInfo comparables
GroupsInfoLibraryItemCreatedAfter groupsCategory = "GroupsInfoLibraryItemCreatedAfter"
GroupsInfoLibraryItemCreatedBefore groupsCategory = "GroupsInfoLibraryItemCreatedBefore"
GroupsInfoLibraryItemModifiedAfter groupsCategory = "GroupsInfoLibraryItemModifiedAfter"
GroupsInfoLibraryItemModifiedBefore groupsCategory = "GroupsInfoLibraryItemModifiedBefore"
// channel and drive selection
GroupsInfoSiteLibraryDrive groupsCategory = "GroupsInfoSiteLibraryDrive"
@ -451,7 +561,9 @@ func (c groupsCategory) leafCat() categorizer {
GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore, GroupsInfoChannelMessageCreator,
GroupsInfoChannelMessageLastReplyAfter, GroupsInfoChannelMessageLastReplyBefore:
return GroupsChannelMessage
case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSiteLibraryDrive:
case GroupsLibraryFolder, GroupsLibraryItem, GroupsInfoSiteLibraryDrive,
GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore,
GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore:
return GroupsLibraryItem
}
@ -671,6 +783,10 @@ func (s GroupsScope) matchesInfo(dii details.ItemInfo) bool {
}
return matchesAny(s, GroupsInfoSiteLibraryDrive, ds)
case GroupsInfoLibraryItemCreatedAfter, GroupsInfoLibraryItemCreatedBefore:
i = dttm.Format(info.Created)
case GroupsInfoLibraryItemModifiedAfter, GroupsInfoLibraryItemModifiedBefore:
i = dttm.Format(info.Modified)
case GroupsInfoChannelMessageCreator:
i = info.MessageCreator
case GroupsInfoChannelMessageCreatedAfter, GroupsInfoChannelMessageCreatedBefore: