larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Group CLI (#4043)

Browse Source 
CLI changes for groups.

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/issues/3990
* 
#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
Abin Simon 2023-08-23 13:34:32 +05:30 committed by GitHub
parent 74b92adbc3
commit 7b6c6026ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 486 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/cli/backup/backup.go
View File

@ -39,6 +39,7 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{
addExchangeCommands,
addOneDriveCommands,
addSharePointCommands,
addGroupsCommands,
addTeamsCommands,
}

147
src/cli/backup/groups.go
View File

@ -1,14 +1,27 @@
package backup
import (
"context"
"errors"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
)
// ------------------------------------------------------------------------------------------------
@ -134,7 +147,38 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error {
return nil
}
return Only(ctx, utils.ErrNotYetImplemented)
if err := validateGroupsBackupCreateFlags(flags.GroupFV, flags.CategoryDataFV); err != nil {
return err
}
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
// TODO: log/print recoverable errors
errs := fault.New(false)
ins, err := m365.GroupsMap(ctx, *acct, errs)
if err != nil {
return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 groups"))
}
sel := groupsBackupCreateSelectors(ctx, ins, flags.GroupFV, flags.CategoryDataFV)
selectorSet := []selectors.Selector{}
for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) {
selectorSet = append(selectorSet, discSel.Selector)
}
return runBackups(
ctx,
r,
"Group", "group",
selectorSet,
ins)
}
// ------------------------------------------------------------------------------------------------
@ -172,17 +216,71 @@ func groupsDetailsCmd() *cobra.Command {
// processes a groups service backup.
func detailsGroupsCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
if utils.HasNoFlagsAndShownHelp(cmd) {
return nil
}
if err := validateGroupBackupCreateFlags(flags.GroupFV); err != nil {
ctx := cmd.Context()
opts := utils.MakeGroupsOpts(cmd)
r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.GroupsService, repo.S3Overrides(cmd))
if err != nil {
return Only(ctx, err)
}
return Only(ctx, utils.ErrNotYetImplemented)
defer utils.CloseRepo(ctx, r)
ds, err := runDetailsGroupsCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsGroupsCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Failure() == nil.
func runDetailsGroupsCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.GroupsOpts,
skipReduce bool,
) (*details.Details, error) {
if err := utils.ValidateGroupsRestoreFlags(backupID, opts); err != nil {
return nil, err
}
ctx = clues.Add(ctx, "backup_id", backupID)
d, _, errs := r.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("no backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
ctx = clues.Add(ctx, "details_entries", len(d.Entries))
if !skipReduce {
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterGroupsRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}
return d, nil
}
// ------------------------------------------------------------------------------------------------
@ -208,7 +306,7 @@ func deleteGroupsCmd(cmd *cobra.Command, args []string) error {
// helpers
// ---------------------------------------------------------------------------
func validateGroupBackupCreateFlags(groups []string) error {
func validateGroupsBackupCreateFlags(groups, cats []string) error {
if len(groups) == 0 {
return clues.New(
"requires one or more --" +
@ -228,3 +326,40 @@ func validateGroupBackupCreateFlags(groups []string) error {
return nil
}
// TODO: users might specify a data type, this only supports AllData().
func groupsBackupCreateSelectors(
ctx context.Context,
ins idname.Cacher,
group, cats []string,
) *selectors.GroupsBackup {
if filters.PathContains(group).Compare(flags.Wildcard) {
return includeAllGroupWithCategories(ins, cats)
}
sel := selectors.NewGroupsBackup(slices.Clone(group))
return addGroupsCategories(sel, cats)
}
func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup {
return addGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories)
}
func addGroupsCategories(sel *selectors.GroupsBackup, cats []string) *selectors.GroupsBackup {
if len(cats) == 0 {
sel.Include(sel.AllData())
}
// TODO(meain): handle filtering
// for _, d := range cats {
// switch d {
// case dataLibraries:
// sel.Include(sel.LibraryFolders(selectors.Any()))
// case dataPages:
// sel.Include(sel.Pages(selectors.Any()))
// }
// }
return sel
}

57
src/cli/utils/groups.go
View File

@ -1,9 +1,13 @@
package utils
import (
"context"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/flags"
"github.com/alcionai/corso/src/pkg/selectors"
)
type GroupsOpts struct {
@ -28,3 +32,56 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts {
Populated: flags.GetPopulatedFlags(cmd),
}
}
// ValidateGroupsRestoreFlags checks common flags for correctness and interdependencies
func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts) error {
if len(backupID) == 0 {
return clues.New("a backup ID is required")
}
// TODO(meain): selectors (refer sharepoint)
return validateRestoreConfigFlags(flags.CollisionsFV, opts.RestoreCfg)
}
// AddGroupInfo adds the scope of the provided values to the selector's
// filter set
func AddGroupInfo(
sel *selectors.GroupsRestore,
v string,
f func(string) []selectors.GroupsScope,
) {
if len(v) == 0 {
return
}
sel.Filter(f(v))
}
// IncludeGroupsRestoreDataSelectors builds the common data-selector
// inclusions for Group commands.
func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *selectors.GroupsRestore {
groups := opts.Groups
ls := len(opts.Groups)
if ls == 0 {
groups = selectors.Any()
}
sel := selectors.NewGroupsRestore(groups)
// TODO(meain): add selectors
sel.Include(sel.AllData())
return sel
}
// FilterGroupsRestoreInfoSelectors builds the common info-selector filters.
func FilterGroupsRestoreInfoSelectors(
sel *selectors.GroupsRestore,
opts GroupsOpts,
) {
// TODO(meain)
// AddGroupInfo(sel, opts.GroupID, sel.Library)
}

161
src/cli/utils/groups_test.go Normal file
View File

@ -0,0 +1,161 @@
package utils_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/tester"
)
type GroupsUtilsSuite struct {
tester.Suite
}
func TestGroupsUtilsSuite(t *testing.T) {
suite.Run(t, &GroupsUtilsSuite{Suite: tester.NewUnitSuite(t)})
}
// Tests selector build for Groups properly
// differentiates between the 3 categories: Pages, Libraries and Lists CLI
func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() {
var (
empty = []string{}
single = []string{"single"}
multi = []string{"more", "than", "one"}
)
table := []struct {
name string
opts utils.GroupsOpts
expectIncludeLen int
}{
{
name: "no inputs",
opts: utils.GroupsOpts{},
expectIncludeLen: 2,
},
{
name: "empty",
opts: utils.GroupsOpts{
Groups: empty,
},
expectIncludeLen: 2,
},
{
name: "single inputs",
opts: utils.GroupsOpts{
Groups: single,
},
expectIncludeLen: 2,
},
{
name: "multi inputs",
opts: utils.GroupsOpts{
Groups: multi,
},
expectIncludeLen: 2,
},
// TODO Add library specific tests once we have filters based
// on library folders
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, test.opts)
assert.Len(suite.T(), sel.Includes, test.expectIncludeLen)
})
}
}
func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() {
table := []struct {
name string
backupID string
opts utils.GroupsOpts
expect assert.ErrorAssertionFunc
}{
{
name: "no opts",
backupID: "id",
opts: utils.GroupsOpts{},
expect: assert.NoError,
},
{
name: "no backupID",
backupID: "",
opts: utils.GroupsOpts{},
expect: assert.Error,
},
// TODO: Add tests for selectors once we have them
// {
// name: "all valid",
// backupID: "id",
// opts: utils.GroupsOpts{
// Populated: flags.PopulatedFlags{
// flags.FileCreatedAfterFN: struct{}{},
// flags.FileCreatedBeforeFN: struct{}{},
// flags.FileModifiedAfterFN: struct{}{},
// flags.FileModifiedBeforeFN: struct{}{},
// },
// },
// expect: assert.NoError,
// },
// {
// name: "invalid file created after",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileCreatedAfter: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileCreatedAfterFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file created before",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileCreatedBefore: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileCreatedBeforeFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file modified after",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileModifiedAfter: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileModifiedAfterFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
// {
// name: "invalid file modified before",
// backupID: "id",
// opts: utils.GroupsOpts{
// FileModifiedBefore: "1235",
// Populated: flags.PopulatedFlags{
// flags.FileModifiedBeforeFN: struct{}{},
// },
// },
// expect: assert.Error,
// },
}
for _, test := range table {
suite.Run(test.name, func() {
t := suite.T()
test.expect(t, utils.ValidateGroupsRestoreFlags(test.backupID, test.opts))
})
}
}

24
src/internal/m365/backup.go
View File

@ -156,6 +156,17 @@ func (ctrl *Controller) IsBackupRunnable(
service path.ServiceType,
resourceOwner string,
) (bool, error) {
if service == path.GroupsService {
_, err := ctrl.AC.Groups().GetByID(ctx, resourceOwner)
if err != nil {
// TODO(meain): check for error message in case groups are
// not enabled at all similar to sharepoint
return false, err
}
return true, nil
}
if service == path.SharePointService {
_, err := ctrl.AC.Sites().GetRoot(ctx)
if err != nil {
@ -181,7 +192,7 @@ func (ctrl *Controller) IsBackupRunnable(
return true, nil
}
func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error {
func verifyBackupInputs(sels selectors.Selector, cachedIDs []string) error {
var ids []string
switch sels.Service {
@ -189,16 +200,13 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error {
// Exchange and OneDrive user existence now checked in checkServiceEnabled.
return nil
case selectors.ServiceGroups:
// TODO(meain): check for group existence.
return nil
case selectors.ServiceSharePoint:
ids = siteIDs
case selectors.ServiceSharePoint, selectors.ServiceGroups:
ids = cachedIDs
}
if !filters.Contains(ids).Compare(sels.ID()) {
return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner)
return clues.Stack(graph.ErrResourceOwnerNotFound).
With("selector_protected_resource", sels.DiscreteOwner)
}
return nil

23
src/internal/m365/collection/drive/group_handler.go
View File

@ -100,6 +100,7 @@ func (h groupBackupHandler) NewLocationIDer(
driveID string,
elems ...string,
) details.LocationIDer {
// TODO(meain): path fixes
return details.NewSharePointLocationIDer(driveID, elems...)
}
@ -124,7 +125,6 @@ func (h groupBackupHandler) IsAllPass() bool {
func (h groupBackupHandler) IncludesDir(dir string) bool {
// TODO(meain)
// return h.scope.Matches(selectors.SharePointGroupFolder, dir)
return true
}
@ -138,7 +138,7 @@ func augmentGroupItemInfo(
size int64,
parentPath *path.Builder,
) details.ItemInfo {
var driveName, driveID, creatorEmail string
var driveName, driveID, creatorEmail, siteID, weburl string
// TODO: we rely on this info for details/restore lookups,
// so if it's nil we have an issue, and will need an alternative
@ -159,15 +159,15 @@ func augmentGroupItemInfo(
}
}
// gsi := item.GetSharepointIds()
// if gsi != nil {
// siteID = ptr.Val(gsi.GetSiteId())
// weburl = ptr.Val(gsi.GetSiteUrl())
gsi := item.GetSharepointIds()
if gsi != nil {
siteID = ptr.Val(gsi.GetSiteId())
weburl = ptr.Val(gsi.GetSiteUrl())
// if len(weburl) == 0 {
// weburl = constructWebURL(item.GetAdditionalData())
// }
// }
if len(weburl) == 0 {
weburl = constructWebURL(item.GetAdditionalData())
}
}
if item.GetParentReference() != nil {
driveID = ptr.Val(item.GetParentReference().GetDriveId())
@ -179,6 +179,7 @@ func augmentGroupItemInfo(
pps = parentPath.String()
}
// TODO: Add channel name and ID
dii.Groups = &details.GroupsInfo{
Created: ptr.Val(item.GetCreatedDateTime()),
DriveID: driveID,
@ -189,6 +190,8 @@ func augmentGroupItemInfo(
Owner: creatorEmail,
ParentPath: pps,
Size: size,
SiteID: siteID,
WebURL: weburl,
}
dii.Extension = &details.ExtensionData{}

56
src/pkg/backup/details/groups.go
View File

@ -11,24 +11,48 @@ import (
// NewGroupsLocationIDer builds a LocationIDer for the groups.
func NewGroupsLocationIDer(
category path.CategoryType,
driveID string,
escapedFolders ...string,
) uniqueLoc {
// TODO: implement
return uniqueLoc{}
) (uniqueLoc, error) {
// TODO(meain): path fixes
if err := path.ValidateServiceAndCategory(path.GroupsService, category); err != nil {
return uniqueLoc{}, clues.Wrap(err, "making groups LocationIDer")
}
pb := path.Builder{}.Append(category.String())
prefixElems := 1
if driveID != "" { // non sp paths don't have driveID
pb.Append(driveID)
prefixElems = 2
}
pb.Append(escapedFolders...)
return uniqueLoc{pb, prefixElems}, nil
}
// GroupsInfo describes a groups item
type GroupsInfo struct {
Created time.Time `json:"created,omitempty"`
DriveName string `json:"driveName,omitempty"`
DriveID string `json:"driveID,omitempty"`
ItemName string `json:"itemName,omitempty"`
ItemType ItemType `json:"itemType,omitempty"`
Modified time.Time `json:"modified,omitempty"`
Owner string `json:"owner,omitempty"`
ParentPath string `json:"parentPath,omitempty"`
Size int64 `json:"size,omitempty"`
// Channels Specific
ChannelName string `json:"channelName,omitempty"`
ChannelID string `json:"channelID,omitempty"`
// SharePoint specific
DriveName string `json:"driveName,omitempty"`
DriveID string `json:"driveID,omitempty"`
SiteID string `json:"siteID,omitempty"`
WebURL string `json:"webURL,omitempty"`
}
// Headers returns the human-readable names of properties in a SharePointInfo
@ -51,9 +75,27 @@ func (i *GroupsInfo) UpdateParentPath(newLocPath *path.Builder) {
}
func (i *GroupsInfo) uniqueLocation(baseLoc *path.Builder) (*uniqueLoc, error) {
return nil, clues.New("not yet implemented")
var category path.CategoryType
switch i.ItemType {
case SharePointLibrary:
category = path.LibrariesCategory
if len(i.DriveID) == 0 {
return nil, clues.New("empty drive ID")
}
}
loc, err := NewGroupsLocationIDer(category, i.DriveID, baseLoc.Elements()...)
return &loc, err
}
func (i *GroupsInfo) updateFolder(f *FolderInfo) error {
return clues.New("not yet implemented")
// TODO(meain): path updates if any
if i.ItemType == SharePointLibrary {
return updateFolderWithinDrive(SharePointLibrary, i.DriveName, i.DriveID, f)
}
return clues.New("unsupported ItemType for GroupsInfo").With("item_type", i.ItemType)
}

2
src/pkg/backup/details/iteminfo.go
View File

@ -28,7 +28,7 @@ const (
ExchangeMail ItemType = 3
// SharePoint (10x)
SharePointLibrary ItemType = 101
SharePointLibrary ItemType = 101 // also used for groups
SharePointList ItemType = 102
SharePointPage ItemType = 103

1
src/pkg/selectors/groups.go
View File

@ -425,7 +425,6 @@ func (c groupsCategory) pathValues(
folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem
rFld = ent.Groups.ParentPath
default:
return nil, clues.New("unrecognized groupsCategory").With("category", c)
}

23
src/pkg/services/m365/groups.go
View File

@ -6,6 +6,7 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
@ -80,7 +81,7 @@ func getAllGroups(
// helpers
// ---------------------------------------------------------------------------
// parseUser extracts information from `models.Groupable` we care about
// parseGroup extracts information from `models.Groupable` we care about
func parseGroup(ctx context.Context, mg models.Groupable) (*Group, error) {
if mg.GetDisplayName() == nil {
return nil, clues.New("group missing display name").
@ -95,3 +96,23 @@ func parseGroup(ctx context.Context, mg models.Groupable) (*Group, error) {
return u, nil
}
// GroupsMap retrieves an id-name cache of all groups in the tenant.
func GroupsMap(
ctx context.Context,
acct account.Account,
errs *fault.Bus,
) (idname.Cacher, error) {
groups, err := Groups(ctx, acct, errs)
if err != nil {
return idname.NewCache(nil), err
}
itn := make(map[string]string, len(groups))
for _, s := range groups {
itn[s.ID] = s.DisplayName
}
return idname.NewCache(itn), nil
}

25
src/pkg/services/m365/groups_test.go
View File

@ -68,6 +68,31 @@ func (suite *GroupsIntgSuite) TestGroups() {
}
}
func (suite *GroupsIntgSuite) TestGroupsMap() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
graph.InitializeConcurrencyLimiter(ctx, true, 4)
gm, err := m365.GroupsMap(ctx, suite.acct, fault.New(true))
assert.NoError(t, err, clues.ToCore(err))
assert.NotEmpty(t, gm)
for _, gid := range gm.IDs() {
suite.Run("group_"+gid, func() {
t := suite.T()
assert.NotEmpty(t, gid)
name, ok := gm.NameOf(gid)
assert.True(t, ok)
assert.NotEmpty(t, name)
})
}
}
func (suite *GroupsIntgSuite) TestGroups_InvalidCredentials() {
table := []struct {
name string