diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 56b5c5ef4..c21f5cbb3 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -39,6 +39,7 @@ var serviceCommands = []func(cmd *cobra.Command) *cobra.Command{ addExchangeCommands, addOneDriveCommands, addSharePointCommands, + addGroupsCommands, addTeamsCommands, } diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go index 1dc490ae7..f4cc101f0 100644 --- a/src/cli/backup/groups.go +++ b/src/cli/backup/groups.go @@ -1,14 +1,27 @@ package backup import ( + "context" + "errors" + "github.com/alcionai/clues" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/exp/slices" "github.com/alcionai/corso/src/cli/flags" . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/repository" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/services/m365" ) // ------------------------------------------------------------------------------------------------ @@ -134,7 +147,38 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error { return nil } - return Only(ctx, utils.ErrNotYetImplemented) + if err := validateGroupsBackupCreateFlags(flags.GroupFV, flags.CategoryDataFV); err != nil { + return err + } + + r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, path.GroupsService, repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + // TODO: log/print recoverable errors + errs := fault.New(false) + + ins, err := m365.GroupsMap(ctx, *acct, errs) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to retrieve M365 groups")) + } + + sel := groupsBackupCreateSelectors(ctx, ins, flags.GroupFV, flags.CategoryDataFV) + selectorSet := []selectors.Selector{} + + for _, discSel := range sel.SplitByResourceOwner(ins.IDs()) { + selectorSet = append(selectorSet, discSel.Selector) + } + + return runBackups( + ctx, + r, + "Group", "group", + selectorSet, + ins) } // ------------------------------------------------------------------------------------------------ @@ -172,17 +216,71 @@ func groupsDetailsCmd() *cobra.Command { // processes a groups service backup. func detailsGroupsCmd(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - if utils.HasNoFlagsAndShownHelp(cmd) { return nil } - if err := validateGroupBackupCreateFlags(flags.GroupFV); err != nil { + ctx := cmd.Context() + opts := utils.MakeGroupsOpts(cmd) + + r, _, _, ctrlOpts, err := utils.GetAccountAndConnect(ctx, path.GroupsService, repo.S3Overrides(cmd)) + if err != nil { return Only(ctx, err) } - return Only(ctx, utils.ErrNotYetImplemented) + defer utils.CloseRepo(ctx, r) + + ds, err := runDetailsGroupsCmd(ctx, r, flags.BackupIDFV, opts, ctrlOpts.SkipReduce) + if err != nil { + return Only(ctx, err) + } + + if len(ds.Entries) == 0 { + Info(ctx, selectors.ErrorNoMatchingItems) + return nil + } + + ds.PrintEntries(ctx) + + return nil +} + +// runDetailsGroupsCmd actually performs the lookup in backup details. +// the fault.Errors return is always non-nil. Callers should check if +// errs.Failure() == nil. +func runDetailsGroupsCmd( + ctx context.Context, + r repository.BackupGetter, + backupID string, + opts utils.GroupsOpts, + skipReduce bool, +) (*details.Details, error) { + if err := utils.ValidateGroupsRestoreFlags(backupID, opts); err != nil { + return nil, err + } + + ctx = clues.Add(ctx, "backup_id", backupID) + + d, _, errs := r.GetBackupDetails(ctx, backupID) + // TODO: log/track recoverable errors + if errs.Failure() != nil { + if errors.Is(errs.Failure(), data.ErrNotFound) { + return nil, clues.New("no backup exists with the id " + backupID) + } + + return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") + } + + ctx = clues.Add(ctx, "details_entries", len(d.Entries)) + + if !skipReduce { + sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterGroupsRestoreInfoSelectors(sel, opts) + d = sel.Reduce(ctx, d, errs) + } + + return d, nil } // ------------------------------------------------------------------------------------------------ @@ -208,7 +306,7 @@ func deleteGroupsCmd(cmd *cobra.Command, args []string) error { // helpers // --------------------------------------------------------------------------- -func validateGroupBackupCreateFlags(groups []string) error { +func validateGroupsBackupCreateFlags(groups, cats []string) error { if len(groups) == 0 { return clues.New( "requires one or more --" + @@ -228,3 +326,40 @@ func validateGroupBackupCreateFlags(groups []string) error { return nil } + +// TODO: users might specify a data type, this only supports AllData(). +func groupsBackupCreateSelectors( + ctx context.Context, + ins idname.Cacher, + group, cats []string, +) *selectors.GroupsBackup { + if filters.PathContains(group).Compare(flags.Wildcard) { + return includeAllGroupWithCategories(ins, cats) + } + + sel := selectors.NewGroupsBackup(slices.Clone(group)) + + return addGroupsCategories(sel, cats) +} + +func includeAllGroupWithCategories(ins idname.Cacher, categories []string) *selectors.GroupsBackup { + return addGroupsCategories(selectors.NewGroupsBackup(ins.IDs()), categories) +} + +func addGroupsCategories(sel *selectors.GroupsBackup, cats []string) *selectors.GroupsBackup { + if len(cats) == 0 { + sel.Include(sel.AllData()) + } + + // TODO(meain): handle filtering + // for _, d := range cats { + // switch d { + // case dataLibraries: + // sel.Include(sel.LibraryFolders(selectors.Any())) + // case dataPages: + // sel.Include(sel.Pages(selectors.Any())) + // } + // } + + return sel +} diff --git a/src/cli/utils/groups.go b/src/cli/utils/groups.go index 9b0827d46..cabc9f3c6 100644 --- a/src/cli/utils/groups.go +++ b/src/cli/utils/groups.go @@ -1,9 +1,13 @@ package utils import ( + "context" + + "github.com/alcionai/clues" "github.com/spf13/cobra" "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/pkg/selectors" ) type GroupsOpts struct { @@ -28,3 +32,56 @@ func MakeGroupsOpts(cmd *cobra.Command) GroupsOpts { Populated: flags.GetPopulatedFlags(cmd), } } + +// ValidateGroupsRestoreFlags checks common flags for correctness and interdependencies +func ValidateGroupsRestoreFlags(backupID string, opts GroupsOpts) error { + if len(backupID) == 0 { + return clues.New("a backup ID is required") + } + + // TODO(meain): selectors (refer sharepoint) + + return validateRestoreConfigFlags(flags.CollisionsFV, opts.RestoreCfg) +} + +// AddGroupInfo adds the scope of the provided values to the selector's +// filter set +func AddGroupInfo( + sel *selectors.GroupsRestore, + v string, + f func(string) []selectors.GroupsScope, +) { + if len(v) == 0 { + return + } + + sel.Filter(f(v)) +} + +// IncludeGroupsRestoreDataSelectors builds the common data-selector +// inclusions for Group commands. +func IncludeGroupsRestoreDataSelectors(ctx context.Context, opts GroupsOpts) *selectors.GroupsRestore { + groups := opts.Groups + + ls := len(opts.Groups) + + if ls == 0 { + groups = selectors.Any() + } + + sel := selectors.NewGroupsRestore(groups) + + // TODO(meain): add selectors + sel.Include(sel.AllData()) + + return sel +} + +// FilterGroupsRestoreInfoSelectors builds the common info-selector filters. +func FilterGroupsRestoreInfoSelectors( + sel *selectors.GroupsRestore, + opts GroupsOpts, +) { + // TODO(meain) + // AddGroupInfo(sel, opts.GroupID, sel.Library) +} diff --git a/src/cli/utils/groups_test.go b/src/cli/utils/groups_test.go new file mode 100644 index 000000000..e2a48faf0 --- /dev/null +++ b/src/cli/utils/groups_test.go @@ -0,0 +1,161 @@ +package utils_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/tester" +) + +type GroupsUtilsSuite struct { + tester.Suite +} + +func TestGroupsUtilsSuite(t *testing.T) { + suite.Run(t, &GroupsUtilsSuite{Suite: tester.NewUnitSuite(t)}) +} + +// Tests selector build for Groups properly +// differentiates between the 3 categories: Pages, Libraries and Lists CLI +func (suite *GroupsUtilsSuite) TestIncludeGroupsRestoreDataSelectors() { + var ( + empty = []string{} + single = []string{"single"} + multi = []string{"more", "than", "one"} + ) + + table := []struct { + name string + opts utils.GroupsOpts + expectIncludeLen int + }{ + { + name: "no inputs", + opts: utils.GroupsOpts{}, + expectIncludeLen: 2, + }, + { + name: "empty", + opts: utils.GroupsOpts{ + Groups: empty, + }, + expectIncludeLen: 2, + }, + { + name: "single inputs", + opts: utils.GroupsOpts{ + Groups: single, + }, + expectIncludeLen: 2, + }, + { + name: "multi inputs", + opts: utils.GroupsOpts{ + Groups: multi, + }, + expectIncludeLen: 2, + }, + // TODO Add library specific tests once we have filters based + // on library folders + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + sel := utils.IncludeGroupsRestoreDataSelectors(ctx, test.opts) + assert.Len(suite.T(), sel.Includes, test.expectIncludeLen) + }) + } +} + +func (suite *GroupsUtilsSuite) TestValidateGroupsRestoreFlags() { + table := []struct { + name string + backupID string + opts utils.GroupsOpts + expect assert.ErrorAssertionFunc + }{ + { + name: "no opts", + backupID: "id", + opts: utils.GroupsOpts{}, + expect: assert.NoError, + }, + { + name: "no backupID", + backupID: "", + opts: utils.GroupsOpts{}, + expect: assert.Error, + }, + // TODO: Add tests for selectors once we have them + // { + // name: "all valid", + // backupID: "id", + // opts: utils.GroupsOpts{ + // Populated: flags.PopulatedFlags{ + // flags.FileCreatedAfterFN: struct{}{}, + // flags.FileCreatedBeforeFN: struct{}{}, + // flags.FileModifiedAfterFN: struct{}{}, + // flags.FileModifiedBeforeFN: struct{}{}, + // }, + // }, + // expect: assert.NoError, + // }, + // { + // name: "invalid file created after", + // backupID: "id", + // opts: utils.GroupsOpts{ + // FileCreatedAfter: "1235", + // Populated: flags.PopulatedFlags{ + // flags.FileCreatedAfterFN: struct{}{}, + // }, + // }, + // expect: assert.Error, + // }, + // { + // name: "invalid file created before", + // backupID: "id", + // opts: utils.GroupsOpts{ + // FileCreatedBefore: "1235", + // Populated: flags.PopulatedFlags{ + // flags.FileCreatedBeforeFN: struct{}{}, + // }, + // }, + // expect: assert.Error, + // }, + // { + // name: "invalid file modified after", + // backupID: "id", + // opts: utils.GroupsOpts{ + // FileModifiedAfter: "1235", + // Populated: flags.PopulatedFlags{ + // flags.FileModifiedAfterFN: struct{}{}, + // }, + // }, + // expect: assert.Error, + // }, + // { + // name: "invalid file modified before", + // backupID: "id", + // opts: utils.GroupsOpts{ + // FileModifiedBefore: "1235", + // Populated: flags.PopulatedFlags{ + // flags.FileModifiedBeforeFN: struct{}{}, + // }, + // }, + // expect: assert.Error, + // }, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + test.expect(t, utils.ValidateGroupsRestoreFlags(test.backupID, test.opts)) + }) + } +} diff --git a/src/internal/m365/backup.go b/src/internal/m365/backup.go index 805dcebd1..55c4c7fdb 100644 --- a/src/internal/m365/backup.go +++ b/src/internal/m365/backup.go @@ -156,6 +156,17 @@ func (ctrl *Controller) IsBackupRunnable( service path.ServiceType, resourceOwner string, ) (bool, error) { + if service == path.GroupsService { + _, err := ctrl.AC.Groups().GetByID(ctx, resourceOwner) + if err != nil { + // TODO(meain): check for error message in case groups are + // not enabled at all similar to sharepoint + return false, err + } + + return true, nil + } + if service == path.SharePointService { _, err := ctrl.AC.Sites().GetRoot(ctx) if err != nil { @@ -181,7 +192,7 @@ func (ctrl *Controller) IsBackupRunnable( return true, nil } -func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error { +func verifyBackupInputs(sels selectors.Selector, cachedIDs []string) error { var ids []string switch sels.Service { @@ -189,16 +200,13 @@ func verifyBackupInputs(sels selectors.Selector, siteIDs []string) error { // Exchange and OneDrive user existence now checked in checkServiceEnabled. return nil - case selectors.ServiceGroups: - // TODO(meain): check for group existence. - return nil - - case selectors.ServiceSharePoint: - ids = siteIDs + case selectors.ServiceSharePoint, selectors.ServiceGroups: + ids = cachedIDs } if !filters.Contains(ids).Compare(sels.ID()) { - return clues.Stack(graph.ErrResourceOwnerNotFound).With("missing_protected_resource", sels.DiscreteOwner) + return clues.Stack(graph.ErrResourceOwnerNotFound). + With("selector_protected_resource", sels.DiscreteOwner) } return nil diff --git a/src/internal/m365/collection/drive/group_handler.go b/src/internal/m365/collection/drive/group_handler.go index 81bbf36af..136d61b2d 100644 --- a/src/internal/m365/collection/drive/group_handler.go +++ b/src/internal/m365/collection/drive/group_handler.go @@ -100,6 +100,7 @@ func (h groupBackupHandler) NewLocationIDer( driveID string, elems ...string, ) details.LocationIDer { + // TODO(meain): path fixes return details.NewSharePointLocationIDer(driveID, elems...) } @@ -124,7 +125,6 @@ func (h groupBackupHandler) IsAllPass() bool { func (h groupBackupHandler) IncludesDir(dir string) bool { // TODO(meain) - // return h.scope.Matches(selectors.SharePointGroupFolder, dir) return true } @@ -138,7 +138,7 @@ func augmentGroupItemInfo( size int64, parentPath *path.Builder, ) details.ItemInfo { - var driveName, driveID, creatorEmail string + var driveName, driveID, creatorEmail, siteID, weburl string // TODO: we rely on this info for details/restore lookups, // so if it's nil we have an issue, and will need an alternative @@ -159,15 +159,15 @@ func augmentGroupItemInfo( } } - // gsi := item.GetSharepointIds() - // if gsi != nil { - // siteID = ptr.Val(gsi.GetSiteId()) - // weburl = ptr.Val(gsi.GetSiteUrl()) + gsi := item.GetSharepointIds() + if gsi != nil { + siteID = ptr.Val(gsi.GetSiteId()) + weburl = ptr.Val(gsi.GetSiteUrl()) - // if len(weburl) == 0 { - // weburl = constructWebURL(item.GetAdditionalData()) - // } - // } + if len(weburl) == 0 { + weburl = constructWebURL(item.GetAdditionalData()) + } + } if item.GetParentReference() != nil { driveID = ptr.Val(item.GetParentReference().GetDriveId()) @@ -179,6 +179,7 @@ func augmentGroupItemInfo( pps = parentPath.String() } + // TODO: Add channel name and ID dii.Groups = &details.GroupsInfo{ Created: ptr.Val(item.GetCreatedDateTime()), DriveID: driveID, @@ -189,6 +190,8 @@ func augmentGroupItemInfo( Owner: creatorEmail, ParentPath: pps, Size: size, + SiteID: siteID, + WebURL: weburl, } dii.Extension = &details.ExtensionData{} diff --git a/src/pkg/backup/details/groups.go b/src/pkg/backup/details/groups.go index 398d8f529..9065d6bbb 100644 --- a/src/pkg/backup/details/groups.go +++ b/src/pkg/backup/details/groups.go @@ -11,24 +11,48 @@ import ( // NewGroupsLocationIDer builds a LocationIDer for the groups. func NewGroupsLocationIDer( + category path.CategoryType, driveID string, escapedFolders ...string, -) uniqueLoc { - // TODO: implement - return uniqueLoc{} +) (uniqueLoc, error) { + // TODO(meain): path fixes + if err := path.ValidateServiceAndCategory(path.GroupsService, category); err != nil { + return uniqueLoc{}, clues.Wrap(err, "making groups LocationIDer") + } + + pb := path.Builder{}.Append(category.String()) + prefixElems := 1 + + if driveID != "" { // non sp paths don't have driveID + pb.Append(driveID) + + prefixElems = 2 + } + + pb.Append(escapedFolders...) + + return uniqueLoc{pb, prefixElems}, nil } // GroupsInfo describes a groups item type GroupsInfo struct { Created time.Time `json:"created,omitempty"` - DriveName string `json:"driveName,omitempty"` - DriveID string `json:"driveID,omitempty"` ItemName string `json:"itemName,omitempty"` ItemType ItemType `json:"itemType,omitempty"` Modified time.Time `json:"modified,omitempty"` Owner string `json:"owner,omitempty"` ParentPath string `json:"parentPath,omitempty"` Size int64 `json:"size,omitempty"` + + // Channels Specific + ChannelName string `json:"channelName,omitempty"` + ChannelID string `json:"channelID,omitempty"` + + // SharePoint specific + DriveName string `json:"driveName,omitempty"` + DriveID string `json:"driveID,omitempty"` + SiteID string `json:"siteID,omitempty"` + WebURL string `json:"webURL,omitempty"` } // Headers returns the human-readable names of properties in a SharePointInfo @@ -51,9 +75,27 @@ func (i *GroupsInfo) UpdateParentPath(newLocPath *path.Builder) { } func (i *GroupsInfo) uniqueLocation(baseLoc *path.Builder) (*uniqueLoc, error) { - return nil, clues.New("not yet implemented") + var category path.CategoryType + + switch i.ItemType { + case SharePointLibrary: + category = path.LibrariesCategory + + if len(i.DriveID) == 0 { + return nil, clues.New("empty drive ID") + } + } + + loc, err := NewGroupsLocationIDer(category, i.DriveID, baseLoc.Elements()...) + + return &loc, err } func (i *GroupsInfo) updateFolder(f *FolderInfo) error { - return clues.New("not yet implemented") + // TODO(meain): path updates if any + if i.ItemType == SharePointLibrary { + return updateFolderWithinDrive(SharePointLibrary, i.DriveName, i.DriveID, f) + } + + return clues.New("unsupported ItemType for GroupsInfo").With("item_type", i.ItemType) } diff --git a/src/pkg/backup/details/iteminfo.go b/src/pkg/backup/details/iteminfo.go index fbd6a92cd..a8ba23100 100644 --- a/src/pkg/backup/details/iteminfo.go +++ b/src/pkg/backup/details/iteminfo.go @@ -28,7 +28,7 @@ const ( ExchangeMail ItemType = 3 // SharePoint (10x) - SharePointLibrary ItemType = 101 + SharePointLibrary ItemType = 101 // also used for groups SharePointList ItemType = 102 SharePointPage ItemType = 103 diff --git a/src/pkg/selectors/groups.go b/src/pkg/selectors/groups.go index 50aa3db74..6f1bd1d74 100644 --- a/src/pkg/selectors/groups.go +++ b/src/pkg/selectors/groups.go @@ -425,7 +425,6 @@ func (c groupsCategory) pathValues( folderCat, itemCat = GroupsLibraryFolder, GroupsLibraryItem rFld = ent.Groups.ParentPath - default: return nil, clues.New("unrecognized groupsCategory").With("category", c) } diff --git a/src/pkg/services/m365/groups.go b/src/pkg/services/m365/groups.go index f4924be22..a32195c1c 100644 --- a/src/pkg/services/m365/groups.go +++ b/src/pkg/services/m365/groups.go @@ -6,6 +6,7 @@ import ( "github.com/alcionai/clues" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/fault" @@ -80,7 +81,7 @@ func getAllGroups( // helpers // --------------------------------------------------------------------------- -// parseUser extracts information from `models.Groupable` we care about +// parseGroup extracts information from `models.Groupable` we care about func parseGroup(ctx context.Context, mg models.Groupable) (*Group, error) { if mg.GetDisplayName() == nil { return nil, clues.New("group missing display name"). @@ -95,3 +96,23 @@ func parseGroup(ctx context.Context, mg models.Groupable) (*Group, error) { return u, nil } + +// GroupsMap retrieves an id-name cache of all groups in the tenant. +func GroupsMap( + ctx context.Context, + acct account.Account, + errs *fault.Bus, +) (idname.Cacher, error) { + groups, err := Groups(ctx, acct, errs) + if err != nil { + return idname.NewCache(nil), err + } + + itn := make(map[string]string, len(groups)) + + for _, s := range groups { + itn[s.ID] = s.DisplayName + } + + return idname.NewCache(itn), nil +} diff --git a/src/pkg/services/m365/groups_test.go b/src/pkg/services/m365/groups_test.go index 8fa650a98..9219209f0 100644 --- a/src/pkg/services/m365/groups_test.go +++ b/src/pkg/services/m365/groups_test.go @@ -68,6 +68,31 @@ func (suite *GroupsIntgSuite) TestGroups() { } } +func (suite *GroupsIntgSuite) TestGroupsMap() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + graph.InitializeConcurrencyLimiter(ctx, true, 4) + + gm, err := m365.GroupsMap(ctx, suite.acct, fault.New(true)) + assert.NoError(t, err, clues.ToCore(err)) + assert.NotEmpty(t, gm) + + for _, gid := range gm.IDs() { + suite.Run("group_"+gid, func() { + t := suite.T() + + assert.NotEmpty(t, gid) + + name, ok := gm.NameOf(gid) + assert.True(t, ok) + assert.NotEmpty(t, name) + }) + } +} + func (suite *GroupsIntgSuite) TestGroups_InvalidCredentials() { table := []struct { name string