larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add groups cli selectors (#4231)

Browse Source 
populates all selectors in the groups cli.  adds
both info-based filters (such as message creation
time) and also basic channel and mesage selection.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🌻 Feature

#### Issue(s)

* #3989

#### Test Plan

- [x] ⚡ Unit test
- [x] 💚 E2E
This commit is contained in:
Keepers 2023-09-14 11:53:20 -06:00 committed by GitHub
parent cb319bb2ae
commit d11eea5f9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 306 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/cli/backup/groups.go
View File

@ -37,26 +37,25 @@ const (
groupsServiceCommandDetailsUseSuffix = "--backup <backupId>"
)
// TODO: correct examples
const (
groupsServiceCommandCreateExamples = `# Backup all Groups data for Alice
corso backup create groups --group alice@example.com
groupsServiceCommandCreateExamples = `# Backup all Groups and Teams data for the Marketing group
corso backup create groups --group Marketing
# Backup only Groups contacts for Alice and Bob
corso backup create groups --group engineering,sales --data contacts
# Backup only Teams conversations messages
corso backup create groups --group Marketing --data messages
# Backup all Groups data for all M365 users
# Backup all Groups and Teams data for all groups
corso backup create groups --group '*'`
groupsServiceCommandDeleteExamples = `# Delete Groups backup with ID 1234abcd-12ab-cd34-56de-1234abcd
corso backup delete groups --backup 1234abcd-12ab-cd34-56de-1234abcd`
groupsServiceCommandDetailsExamples = `# Explore items in Alice's latest backup (1234abcd...)
groupsServiceCommandDetailsExamples = `# Explore items in Marketing's latest backup (1234abcd...)
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd
# Explore calendar events occurring after start of 2022
# Explore Marketing messages posted after the start of 2022
corso backup details groups --backup 1234abcd-12ab-cd34-56de-1234abcd \
--event-starts-after 2022-01-01T00:00:00`
--last-message-reply-after 2022-01-01T00:00:00`
)
// called by backup.go to map subcommands to provider-specific handling.
@ -107,6 +106,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
flags.AddBackupIDFlag(c, true)
flags.AddGroupDetailsAndRestoreFlags(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)

14
src/cli/export/groups.go
View File

@ -27,6 +27,7 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false
flags.AddBackupIDFlag(c, true)
flags.AddGroupDetailsAndRestoreFlags(c)
flags.AddExportConfigFlags(c)
flags.AddFailFastFlag(c)
flags.AddCorsoPassphaseFlags(c)
@ -36,23 +37,22 @@ func addGroupsCommands(cmd *cobra.Command) *cobra.Command {
return c
}
// TODO: correct examples
const (
groupsServiceCommand = "groups"
teamsServiceCommand = "teams"
groupsServiceCommandUseSuffix = "<destination> --backup <backupId>"
//nolint:lll
groupsServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's last backup (1234abcd...) to my-exports directory
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef
groupsServiceCommandExportExamples = `# Export a message in Marketing's last backup (1234abcd...) to my-exports directory
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd --message 98765abcdef
# Export files named "FY2021 Planning.xlsx" in "Documents/Finance Reports" to current directory
# Export all messages named in channel "Finance Reports" to the current directory
corso export groups . --backup 1234abcd-12ab-cd34-56de-1234abcd \
--file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports"
--message '*' --channel "Finance Reports"
# Export all files and folders in folder "Documents/Finance Reports" that were created before 2020 to my-exports
# Export all messages in channel "Finance Reports" that were created before 2020 to my-exports
corso export groups my-exports --backup 1234abcd-12ab-cd34-56de-1234abcd
--folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00`
--channel "Finance Reports" --message-created-before 2020-01-01T00:00:00`
)
// `corso export groups [<flag>...] <destination>`

54
src/cli/flags/groups.go
View File

@ -6,12 +6,60 @@ import (
const DataMessages = "messages"
const GroupFN = "group"
const (
ChannelFN = "channel"
GroupFN = "group"
MessageFN = "message"
var GroupFV []string
MessageCreatedAfterFN = "message-created-after"
MessageCreatedBeforeFN = "message-created-before"
MessageLastReplyAfterFN = "message-last-reply-after"
MessageLastReplyBeforeFN = "message-last-reply-before"
)
var (
ChannelFV []string
GroupFV []string
MessageFV []string
MessageCreatedAfterFV string
MessageCreatedBeforeFV string
MessageLastReplyAfterFV string
MessageLastReplyBeforeFV string
)
func AddGroupDetailsAndRestoreFlags(cmd *cobra.Command) {
// TODO: implement groups specific flags
fs := cmd.Flags()
fs.StringSliceVar(
&ChannelFV,
ChannelFN, nil,
"Select data within a Team's Channel.")
fs.StringSliceVar(
&MessageFV,
MessageFN, nil,
"Select messages by reference.")
fs.StringVar(
&MessageCreatedAfterFV,
MessageCreatedAfterFN, "",
"Select messages created after this datetime.")
fs.StringVar(
&MessageCreatedBeforeFV,
MessageCreatedBeforeFN, "",
"Select messages created before this datetime.")
fs.StringVar(
&MessageLastReplyAfterFV,
MessageLastReplyAfterFN, "",
"Select messages with replies after this datetime.")
fs.StringVar(
&MessageLastReplyBeforeFV,
MessageLastReplyBeforeFN, "",
"Select messages with replies before this datetime.")
}
// AddGroupFlag adds the --group flag, which accepts id or name values.

1
src/cli/flags/onedrive.go
View File

@ -43,6 +43,7 @@ func AddOneDriveDetailsAndRestoreFlags(cmd *cobra.Command) {
&FileCreatedAfterFV,
FileCreatedAfterFN, "",
"Select files created after this datetime.")
fs.StringVar(
&FileCreatedBeforeFV,
FileCreatedBeforeFN, "",

96
src/cli/utils/groups.go
View File

@ -12,6 +12,13 @@ import (
type GroupsOpts struct {
Groups []string
Channels []string
Messages []string
MessageCreatedAfter string
MessageCreatedBefore string
MessageLastReplyAfter string
MessageLastReplyBefore string
SiteID []string
Library string
@ -61,7 +68,8 @@ func AddGroupsCategories(sel *selectors.GroupsBackup, cats []string) *selectors.
func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
return GroupsOpts{
Groups: flags.GroupFV,
Channels: flags.ChannelFV,
Messages: flags.MessageFV,
SiteID: flags.SiteIDFV,
Library: flags.LibraryFV,
@ -71,6 +79,10 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
FileCreatedBefore: flags.FileCreatedBeforeFV,
FileModifiedAfter: flags.FileModifiedAfterFV,
FileModifiedBefore: flags.FileModifiedBeforeFV,
MessageCreatedAfter: flags.MessageCreatedAfterFV,
MessageCreatedBefore: flags.MessageCreatedBeforeFV,
MessageLastReplyAfter: flags.MessageLastReplyAfterFV,
MessageLastReplyBefore: flags.MessageLastReplyBeforeFV,
ListFolder: flags.ListFolderFV,
ListItem: flags.ListItemFV,
@ -110,12 +122,30 @@ func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts) error {
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
}
if _, ok := opts.Populated[flags.MessageCreatedAfterFN]; ok && !IsValidTimeFormat(opts.MessageCreatedAfter) {
return clues.New("invalid time format for " + flags.MessageCreatedAfterFN)
}
if _, ok := opts.Populated[flags.MessageCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.MessageCreatedBefore) {
return clues.New("invalid time format for " + flags.MessageCreatedBeforeFN)
}
if _, ok := opts.Populated[flags.MessageLastReplyAfterFN]; ok && !IsValidTimeFormat(opts.MessageLastReplyAfter) {
return clues.New("invalid time format for " + flags.MessageLastReplyAfterFN)
}
if _, ok := opts.Populated[flags.MessageLastReplyBeforeFN]; ok && !IsValidTimeFormat(opts.MessageLastReplyBefore) {
return clues.New("invalid time format for " + flags.MessageLastReplyBeforeFN)
}
// TODO(meain): selectors (refer sharepoint)
return validateRestoreConfigFlags(flags.CollisionsFV, opts.RestoreCfg)
}
// AddGroupInfo adds the scope of the provided values to the selector's
// AddGroupsFilter adds the scope of the provided values to the selector's
// filter set
func AddGroupInfo(
func AddGroupsFilter(
sel *selectors.GroupsRestore,
v string,
f func(string) []selectors.GroupsScope,
@ -130,16 +160,15 @@ func AddGroupInfo(
// IncludeGroupsRestoreDataSelectors builds the common data-selector
// inclusions for Group commands.
func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *selectors.GroupsRestore {
groups := opts.Groups
lg := len(opts.Groups)
var (
groups = opts.Groups
lfp, lfn = len(opts.FolderPath), len(opts.FileName)
llf, lli = len(opts.ListFolder), len(opts.ListItem)
lpf, lpi = len(opts.PageFolder), len(opts.Page)
lg, lch, lm = len(opts.Groups), len(opts.Channels), len(opts.Messages)
// TODO(meain): handle sites once we add non-root site backup
// ls := len(opts.SiteID)
lfp, lfn := len(opts.FolderPath), len(opts.FileName)
slp, sli := len(opts.ListFolder), len(opts.ListItem)
pf, pi := len(opts.PageFolder), len(opts.Page)
)
if lg == 0 {
groups = selectors.Any()
@ -147,11 +176,14 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
sel := selectors.NewGroupsRestore(groups)
if lfp+lfn+slp+sli+pf+pi == 0 {
if lfp+lfn+llf+lli+lpf+lpi+lch+lm == 0 {
sel.Include(sel.AllData())
return sel
}
// sharepoint site selectors
if lfp+lfn+llf+lli+lpf+lpi > 0 {
if lfp+lfn > 0 {
if lfn == 0 {
opts.FileName = selectors.Any()
@ -169,8 +201,8 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
}
}
if slp+sli > 0 {
if sli == 0 {
if llf+lli > 0 {
if lli == 0 {
opts.ListItem = selectors.Any()
}
@ -186,8 +218,8 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
}
}
if pf+pi > 0 {
if pi == 0 {
if lpf+lpi > 0 {
if lpi == 0 {
opts.Page = selectors.Any()
}
@ -202,6 +234,24 @@ func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *se
sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch()))
}
}
}
// channel and message selectors
if lch+lm > 0 {
// if no channel is specified, include all channels
if lch == 0 {
opts.Channels = selectors.Any()
}
// if no message is specified, only select channels
// otherwise, look for channel/message pairs
if lm == 0 {
sel.Include(sel.Channels(opts.Channels))
} else {
sel.Include(sel.ChannelMessages(opts.Channels, opts.Messages))
}
}
return sel
}
@ -211,9 +261,13 @@ func FilterGroupsRestoreInfoSelectors(
sel *selectors.GroupsRestore,
opts GroupsOpts,
) {
AddGroupInfo(sel, opts.Library, sel.Library)
AddGroupInfo(sel, opts.FileCreatedAfter, sel.CreatedAfter)
AddGroupInfo(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddGroupInfo(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddGroupInfo(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
AddGroupsFilter(sel, opts.Library, sel.Library)
AddGroupsFilter(sel, opts.FileCreatedAfter, sel.CreatedAfter)
AddGroupsFilter(sel, opts.FileCreatedBefore, sel.CreatedBefore)
AddGroupsFilter(sel, opts.FileModifiedAfter, sel.ModifiedAfter)
AddGroupsFilter(sel, opts.FileModifiedBefore, sel.ModifiedBefore)
AddGroupsFilter(sel, opts.MessageCreatedAfter, sel.MessageCreatedAfter)
AddGroupsFilter(sel, opts.MessageCreatedBefore, sel.MessageCreatedBefore)
AddGroupsFilter(sel, opts.MessageLastReplyAfter, sel.MessageLastReplyAfter)
AddGroupsFilter(sel, opts.MessageLastReplyBefore, sel.MessageLastReplyBefore)
}

110
src/cli/utils/groups_test.go
View File

@ -22,8 +22,6 @@ func TestGroupsUtilsSuite(t *testing.T) {
suite.Run(t, &GroupsUtilsSuite{Suite: tester.NewUnitSuite(t)})
}
// Tests selector build for Groups properly
// differentiates between the 3 categories: Pages, Libraries and Lists CLI
func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
var (
empty = []string{}
@ -40,6 +38,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
opts utils.GroupsOpts
expectIncludeLen int
}{
// resource
{
name: "no inputs",
opts: utils.GroupsOpts{},
@ -66,6 +65,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
},
expectIncludeLen: 2,
},
// sharepoint
{
name: "library folder contains",
opts: utils.GroupsOpts{
@ -165,6 +165,50 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
},
expectIncludeLen: 1,
},
// channels
{
name: "multiple channel multiple message",
opts: utils.GroupsOpts{
Groups: single,
Channels: multi,
Messages: multi,
},
expectIncludeLen: 1,
},
{
name: "single channel multiple message",
opts: utils.GroupsOpts{
Groups: single,
Channels: single,
Messages: multi,
},
expectIncludeLen: 1,
},
{
name: "single channel and message",
opts: utils.GroupsOpts{
Groups: single,
Channels: single,
Messages: single,
},
expectIncludeLen: 1,
},
{
name: "multiple channel only",
opts: utils.GroupsOpts{
Groups: single,
Channels: multi,
},
expectIncludeLen: 1,
},
{
name: "single channel only",
opts: utils.GroupsOpts{
Groups: single,
Channels: single,
},
expectIncludeLen: 1,
},
}
for _, test := range table {
suite.Run(test.name, func() {
@ -174,7 +218,7 @@ func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
defer flush()
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, test.opts)
assert.Len(suite.T(), sel.Includes, test.expectIncludeLen)
assert.Len(t, sel.Includes, test.expectIncludeLen)
})
}
}
@ -206,16 +250,25 @@ func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() {
FileCreatedBefore: dttm.Now(),
FileModifiedAfter: dttm.Now(),
FileModifiedBefore: dttm.Now(),
MessageCreatedAfter: dttm.Now(),
MessageCreatedBefore: dttm.Now(),
MessageLastReplyAfter: dttm.Now(),
MessageLastReplyBefore: dttm.Now(),
Populated: flags.PopulatedFlags{
flags.SiteFN: struct{}{},
flags.FileCreatedAfterFN: struct{}{},
flags.FileCreatedBeforeFN: struct{}{},
flags.FileModifiedAfterFN: struct{}{},
flags.FileModifiedBeforeFN: struct{}{},
flags.MessageCreatedAfterFN: struct{}{},
flags.MessageCreatedBeforeFN: struct{}{},
flags.MessageLastReplyAfterFN: struct{}{},
flags.MessageLastReplyBeforeFN: struct{}{},
},
},
expect: assert.NoError,
},
// sharepoint
{
name: "invalid file created after",
backupID: "id",
@ -238,6 +291,17 @@ func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() {
},
expect: assert.Error,
},
{
name: "invalid file modified before",
backupID: "id",
opts: utils.GroupsOpts{
FileModifiedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.FileModifiedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid file modified after",
backupID: "id",
@ -249,13 +313,47 @@ func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() {
},
expect: assert.Error,
},
// channels
{
name: "invalid file modified before",
name: "invalid message last reply before",
backupID: "id",
opts: utils.GroupsOpts{
FileModifiedBefore: "1235",
MessageLastReplyBefore: "1235",
Populated: flags.PopulatedFlags{
flags.FileModifiedBeforeFN: struct{}{},
flags.MessageLastReplyBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid message last reply after",
backupID: "id",
opts: utils.GroupsOpts{
MessageLastReplyAfter: "1235",
Populated: flags.PopulatedFlags{
flags.MessageLastReplyAfterFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid message created before",
backupID: "id",
opts: utils.GroupsOpts{
MessageCreatedBefore: "1235",
Populated: flags.PopulatedFlags{
flags.MessageCreatedBeforeFN: struct{}{},
},
},
expect: assert.Error,
},
{
name: "invalid message created after",
backupID: "id",
opts: utils.GroupsOpts{
MessageCreatedAfter: "1235",
Populated: flags.PopulatedFlags{
flags.MessageCreatedAfterFN: struct{}{},
},
},
expect: assert.Error,

8
src/cli/utils/onedrive.go
View File

@ -52,19 +52,19 @@ func ValidateOneDriveRestoreFlags(backupID string, opts OneDriveOpts) error {
}
if _, ok := opts.Populated[flags.FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) {
return clues.New("invalid time format for created-after")
return clues.New("invalid time format for " + flags.FileCreatedAfterFN)
}
if _, ok := opts.Populated[flags.FileCreatedBeforeFN]; ok && !IsValidTimeFormat(opts.FileCreatedBefore) {
return clues.New("invalid time format for created-before")
return clues.New("invalid time format for " + flags.FileCreatedBeforeFN)
}
if _, ok := opts.Populated[flags.FileModifiedAfterFN]; ok && !IsValidTimeFormat(opts.FileModifiedAfter) {
return clues.New("invalid time format for modified-after")
return clues.New("invalid time format for " + flags.FileModifiedAfterFN)
}
if _, ok := opts.Populated[flags.FileModifiedBeforeFN]; ok && !IsValidTimeFormat(opts.FileModifiedBefore) {
return clues.New("invalid time format for modified-before")
return clues.New("invalid time format for " + flags.FileModifiedBeforeFN)
}
return validateRestoreConfigFlags(flags.CollisionsFV, opts.RestoreCfg)