Compare commits

...

3 Commits
main ... 3135-4

Author SHA1 Message Date
ryanfkeepers
dea84eeb37 add back link to sp perms docs 2023-05-05 10:07:51 -06:00
ryanfkeepers
39de7d03d8 fixup bad rebase state 2023-05-03 15:20:32 -06:00
ryanfkeepers
5a7e02ea31 permission basics
this doesn't solve all permission handling in sharepoint, but
it sets a lot of groundwork that needs to be done anyway.
2023-05-03 15:20:32 -06:00
14 changed files with 170 additions and 108 deletions

View File

@ -49,10 +49,10 @@ const (
oneDriveServiceCommandRestoreExamples = `# Restore file with ID 98765abcdef
corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef
# Restore file with ID 98765abcdef along with its associated permissions
# Restore the file with ID 98765abcdef along with its associated permissions
corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef --restore-permissions
# Restore Alice's file named "FY2021 Planning.xlsx in "Documents/Finance Reports" from a specific backup
# Restore Alice's file named "FY2021 Planning.xlsx" in "Documents/Finance Reports" from a specific backup
corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \
--user alice@example.com --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports"

View File

@ -33,6 +33,8 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command {
utils.AddBackupIDFlag(c, true)
utils.AddSharePointDetailsAndRestoreFlags(c)
options.AddRestorePermissionsFlag(c)
options.AddFailFastFlag(c)
}
@ -47,7 +49,11 @@ const (
sharePointServiceCommandRestoreExamples = `# Restore file with ID 98765abcdef
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef
# Restore a file named "ServerRenderTemplate.xsl in "Display Templates/Style Sheets".
# Restore the file with ID 98765abcdef along with its associated permissions
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--file 98765abcdef --restore-permissions
# Restore a file named "ServerRenderTemplate.xsl" in "Display Templates/Style Sheets".
corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \
--file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets"

View File

@ -43,4 +43,6 @@ var (
PageFolderInput = []string{"pageFolder1", "pageFolder2"}
PageInput = []string{"page1", "page2"}
RestorePermissions = true
)

View File

@ -228,10 +228,6 @@ func (m *SiteItemRequestBuilder) Patch(
return res.(msmodel.Siteable), nil
}
// Permissions provides operations to manage the permissions property of the microsoft.graph.site entity.
// PermissionsById provides operations to manage the permissions property of the microsoft.graph.site entity.
// Sites provides operations to manage the sites property of the microsoft.graph.site entity.
// func (m *SiteItemRequestBuilder) Sites()
// SitesById provides operations to manage the sites property of the microsoft.graph.site entity.
//
//nolint:revive,wsl

View File

@ -431,12 +431,6 @@ func (si suiteInfoImpl) Resource() Resource {
// SharePoint shares most of its libraries implementation with OneDrive so we
// only test simple things here and leave the more extensive testing to
// OneDrive.
//
// TODO(ashmrtn): SharePoint doesn't have permissions backup/restore enabled
// right now. Adjust the tests here when that is enabled so we have at least
// basic assurances that it's doing the right thing. We can leave the more
// extensive permissions tests to OneDrive as well.
type GraphConnectorSharePointIntegrationSuite struct {
tester.Suite
suiteInfo
@ -486,6 +480,18 @@ func (suite *GraphConnectorSharePointIntegrationSuite) TestRestoreAndBackup_Mult
testRestoreAndBackupMultipleFilesAndFoldersNoPermissions(suite, version.Backup)
}
func (suite *GraphConnectorSharePointIntegrationSuite) TestPermissionsRestoreAndBackup() {
testPermissionsRestoreAndBackup(suite, version.Backup)
}
func (suite *GraphConnectorSharePointIntegrationSuite) TestPermissionsBackupAndNoRestore() {
testPermissionsBackupAndNoRestore(suite, version.Backup)
}
func (suite *GraphConnectorSharePointIntegrationSuite) TestPermissionsInheritanceRestoreAndBackup() {
testPermissionsInheritanceRestoreAndBackup(suite, version.Backup)
}
// ---------------------------------------------------------------------------
// OneDrive most recent backup version
// ---------------------------------------------------------------------------

View File

@ -18,7 +18,9 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/api/mock"
"github.com/alcionai/corso/src/internal/connector/onedrive/metadata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
@ -281,6 +283,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
type OneDriveSuite struct {
tester.Suite
userID string
creds account.M365Config
}
func TestOneDriveSuite(t *testing.T) {
@ -292,7 +295,15 @@ func TestOneDriveSuite(t *testing.T) {
}
func (suite *OneDriveSuite) SetupSuite() {
suite.userID = tester.SecondaryM365UserID(suite.T())
t := suite.T()
suite.userID = tester.SecondaryM365UserID(t)
acct := tester.NewM365Account(t)
creds, err := acct.M365Config()
require.NoError(t, err)
suite.creds = creds
}
func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
@ -333,17 +344,46 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
rootFolder, err := api.GetDriveRoot(ctx, gs, driveID)
require.NoError(t, err, clues.ToCore(err))
restoreFolders := path.Builder{}.Append(folderElements...)
restoreDir := path.Builder{}.Append(folderElements...)
drivePath := path.DrivePath{
DriveID: driveID,
Root: "root:",
Folders: folderElements,
}
folderID, err := CreateRestoreFolders(ctx, gs, driveID, ptr.Val(rootFolder.GetId()), restoreFolders, NewFolderCache())
folderID, err := CreateRestoreFolders(
ctx,
suite.creds,
gs,
&drivePath,
ptr.Val(rootFolder.GetId()),
restoreDir,
nil, // only needed for permissions
metadata.Metadata{},
map[string]metadata.Metadata{},
NewFolderCache(),
map[string]string{},
false)
require.NoError(t, err, clues.ToCore(err))
folderIDs = append(folderIDs, folderID)
folderName2 := "Corso_Folder_Test_" + dttm.FormatNow(dttm.SafeForTesting)
restoreFolders = restoreFolders.Append(folderName2)
restoreDir = restoreDir.Append(folderName2)
folderID, err = CreateRestoreFolders(ctx, gs, driveID, ptr.Val(rootFolder.GetId()), restoreFolders, NewFolderCache())
folderID, err = CreateRestoreFolders(
ctx,
suite.creds,
gs,
&drivePath,
ptr.Val(rootFolder.GetId()),
restoreDir,
nil, // only needed for permissions
metadata.Metadata{},
map[string]metadata.Metadata{},
NewFolderCache(),
map[string]string{},
false)
require.NoError(t, err, clues.ToCore(err))
folderIDs = append(folderIDs, folderID)

View File

@ -63,7 +63,6 @@ func sharePointItemMetaReader(
driveID string,
item models.DriveItemable,
) (io.ReadCloser, int, error) {
// TODO: include permissions
return baseItemMetaReader(ctx, service, driveID, item)
}
@ -242,6 +241,9 @@ func filterUserPermissions(ctx context.Context, perms []models.Permissionable) [
// write - Design | Edit | Contribute (no difference in /permissions api)
// read - Read
// empty - Restricted View
//
// helpful docs:
// https://devblogs.microsoft.com/microsoft365dev/controlling-app-access-on-specific-sharepoint-site-collections/
roles := p.GetRoles()
entityID := ""
@ -250,15 +252,16 @@ func filterUserPermissions(ctx context.Context, perms []models.Permissionable) [
} else if gv2.GetGroup() != nil {
entityID = ptr.Val(gv2.GetGroup().GetId())
} else {
// TODO Add application permissions when adding permissions for SharePoint
// https://devblogs.microsoft.com/microsoft365dev/controlling-app-access-on-specific-sharepoint-site-collections/
logm := logger.Ctx(ctx)
if gv2.GetApplication() != nil {
logm.With("application_id", ptr.Val(gv2.GetApplication().GetId()))
entityID = ptr.Val(gv2.GetApplication().GetId())
}
if gv2.GetDevice() != nil {
logm.With("device_id", ptr.Val(gv2.GetDevice().GetId()))
}
logm.Info("untracked permission")
}

View File

@ -89,8 +89,9 @@ func getCollectionMetadata(
// permissions. folderMetas is expected to have all the parent
// directory metas for this to work.
func computeParentPermissions(
itemPath path.Path,
folderMetas map[string]metadata.Metadata,
originDir path.Path,
// map parent dir -> parent's metadata
parentMetas map[string]metadata.Metadata,
) (metadata.Metadata, error) {
var (
parent path.Path
@ -100,7 +101,7 @@ func computeParentPermissions(
ok bool
)
parent = itemPath
parent = originDir
for {
parent, err = parent.Dir()
@ -117,7 +118,7 @@ func computeParentPermissions(
return metadata.Metadata{}, nil
}
meta, ok = folderMetas[parent.String()]
meta, ok = parentMetas[parent.String()]
if !ok {
return metadata.Metadata{}, clues.New("no parent meta")
}
@ -137,7 +138,7 @@ func UpdatePermissions(
driveID string,
itemID string,
permAdded, permRemoved []metadata.Permission,
permissionIDMappings map[string]string,
oldPermIDToNewID map[string]string,
) error {
// The ordering of the operations is important here. We first
// remove all the removed permissions and then add the added ones.
@ -151,7 +152,7 @@ func UpdatePermissions(
return graph.Wrap(ctx, err, "creating delete client")
}
pid, ok := permissionIDMappings[p.ID]
pid, ok := oldPermIDToNewID[p.ID]
if !ok {
return clues.New("no new permission id").WithClues(ctx)
}
@ -212,7 +213,7 @@ func UpdatePermissions(
return graph.Wrap(ctx, err, "setting permissions")
}
permissionIDMappings[p.ID] = ptr.Val(np.GetValue()[0].GetId())
oldPermIDToNewID[p.ID] = ptr.Val(np.GetValue()[0].GetId())
}
return nil
@ -228,23 +229,24 @@ func RestorePermissions(
service graph.Servicer,
driveID string,
itemID string,
itemPath path.Path,
meta metadata.Metadata,
folderMetas map[string]metadata.Metadata,
permissionIDMappings map[string]string,
originDir path.Path,
current metadata.Metadata,
// map parent dir -> parent's metadata
parentMetas map[string]metadata.Metadata,
oldPermIDToNewID map[string]string,
) error {
if meta.SharingMode == metadata.SharingModeInherited {
if current.SharingMode == metadata.SharingModeInherited {
return nil
}
ctx = clues.Add(ctx, "permission_item_id", itemID)
parentPermissions, err := computeParentPermissions(itemPath, folderMetas)
parentPermissions, err := computeParentPermissions(originDir, parentMetas)
if err != nil {
return clues.Wrap(err, "parent permissions").WithClues(ctx)
}
permAdded, permRemoved := metadata.DiffPermissions(parentPermissions.Permissions, meta.Permissions)
permAdded, permRemoved := metadata.DiffPermissions(parentPermissions.Permissions, current.Permissions)
return UpdatePermissions(ctx, creds, service, driveID, itemID, permAdded, permRemoved, permissionIDMappings)
return UpdatePermissions(ctx, creds, service, driveID, itemID, permAdded, permRemoved, oldPermIDToNewID)
}

View File

@ -22,10 +22,14 @@ func TestPermissionsUnitTestSuite(t *testing.T) {
suite.Run(t, &PermissionsUnitTestSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *PermissionsUnitTestSuite) TestComputeParentPermissions() {
func (suite *PermissionsUnitTestSuite) TestComputeParentPermissions_oneDrive() {
runComputeParentPermissionsTest(suite, path.OneDriveService, path.FilesCategory, "user")
}
func (suite *PermissionsUnitTestSuite) TestComputeParentPermissions_sharePoint() {
runComputeParentPermissionsTest(suite, path.SharePointService, path.LibrariesCategory, "site")
}
func runComputeParentPermissionsTest(
suite *PermissionsUnitTestSuite,
service path.ServiceType,

View File

@ -46,15 +46,15 @@ func RestoreCollections(
errs *fault.Bus,
) (*support.ConnectorOperationStatus, error) {
var (
restoreMetrics support.CollectionMetrics
metrics support.CollectionMetrics
folderMetas = map[string]metadata.Metadata{}
restoreMetrics support.CollectionMetrics
metrics support.CollectionMetrics
parentDirToMeta = map[string]metadata.Metadata{}
// permissionIDMappings is used to map between old and new id
// oldPermIDToNewID is used to map between old and new id
// of permissions as we restore them
permissionIDMappings = map[string]string{}
fc = NewFolderCache()
rootIDCache = map[string]string{}
oldPermIDToNewID = map[string]string{}
fc = NewFolderCache()
rootFolderIDCache = map[string]string{}
)
ctx = clues.Add(
@ -91,10 +91,10 @@ func RestoreCollections(
backupVersion,
service,
dc,
folderMetas,
permissionIDMappings,
parentDirToMeta,
oldPermIDToNewID,
fc,
rootIDCache,
rootFolderIDCache,
OneDriveSource,
dest.ContainerName,
deets,
@ -132,10 +132,12 @@ func RestoreCollection(
backupVersion int,
service graph.Servicer,
dc data.RestoreCollection,
folderMetas map[string]metadata.Metadata,
permissionIDMappings map[string]string,
// cache of parent dir -> parent's metadata,
// mutated during this call
parentDirToMeta map[string]metadata.Metadata,
oldPermIDToNewID map[string]string,
fc *folderCache,
rootIDCache map[string]string, // map of drive id -> root folder ID
rootFolderIDCache map[string]string, // map of drive id -> root folder ID
source driveSource,
restoreContainerName string,
deets *details.Builder,
@ -157,29 +159,29 @@ func RestoreCollection(
return metrics, clues.Wrap(err, "creating drive path").WithClues(ctx)
}
if rootIDCache == nil {
rootIDCache = map[string]string{}
if rootFolderIDCache == nil {
rootFolderIDCache = map[string]string{}
}
if _, ok := rootIDCache[drivePath.DriveID]; !ok {
if _, ok := rootFolderIDCache[drivePath.DriveID]; !ok {
root, err := api.GetDriveRoot(ctx, service, drivePath.DriveID)
if err != nil {
return metrics, clues.Wrap(err, "getting drive root id")
}
rootIDCache[drivePath.DriveID] = ptr.Val(root.GetId())
rootFolderIDCache[drivePath.DriveID] = ptr.Val(root.GetId())
}
// Assemble folder hierarchy we're going to restore into (we recreate the folder hierarchy
// from the backup under this the restore folder instead of root)
// i.e. Restore into `<restoreContainerName>/<original folder path>`
// the drive into which this folder gets restored is tracked separately in drivePath.
restoreFolderElements := path.Builder{}.Append(restoreContainerName).Append(drivePath.Folders...)
restoreDir := path.Builder{}.Append(restoreContainerName).Append(drivePath.Folders...)
ctx = clues.Add(
ctx,
"directory", dc.FullPath().Folder(false),
"destination_elements", restoreFolderElements,
"destination_dir", restoreDir,
"drive_id", drivePath.DriveID)
trace.Log(ctx, "gc:oneDrive:restoreCollection", directory.String())
@ -189,7 +191,7 @@ func RestoreCollection(
ctx,
drivePath,
dc,
folderMetas,
parentDirToMeta,
backupVersion,
restorePerms)
if err != nil {
@ -197,24 +199,24 @@ func RestoreCollection(
}
// Create restore folders and get the folder ID of the folder the data stream will be restored in
restoreFolderID, err := createRestoreFoldersWithPermissions(
restoreFolderID, err := CreateRestoreFolders(
ctx,
creds,
service,
drivePath,
rootIDCache[drivePath.DriveID],
restoreFolderElements,
rootFolderIDCache[drivePath.DriveID],
restoreDir,
dc.FullPath(),
colMeta,
folderMetas,
parentDirToMeta,
fc,
permissionIDMappings,
oldPermIDToNewID,
restorePerms)
if err != nil {
return metrics, clues.Wrap(err, "creating folders for restore")
}
folderMetas[dc.FullPath().String()] = colMeta
parentDirToMeta[dc.FullPath().String()] = colMeta
items := dc.Items(ctx, errs)
for {
@ -247,8 +249,8 @@ func RestoreCollection(
drivePath,
restoreFolderID,
copyBuffer,
folderMetas,
permissionIDMappings,
parentDirToMeta,
oldPermIDToNewID,
restorePerms,
itemData,
itemPath)
@ -298,8 +300,8 @@ func restoreItem(
drivePath *path.DrivePath,
restoreFolderID string,
copyBuffer []byte,
folderMetas map[string]metadata.Metadata,
permissionIDMappings map[string]string,
parentDirToMeta map[string]metadata.Metadata,
oldPermIDToNewID map[string]string,
restorePerms bool,
itemData data.Stream,
itemPath path.Path,
@ -348,7 +350,7 @@ func restoreItem(
}
trimmedPath := strings.TrimSuffix(itemPath.String(), metadata.DirMetaFileSuffix)
folderMetas[trimmedPath] = meta
parentDirToMeta[trimmedPath] = meta
return details.ItemInfo{}, true, nil
}
@ -366,8 +368,8 @@ func restoreItem(
restoreFolderID,
copyBuffer,
restorePerms,
folderMetas,
permissionIDMappings,
parentDirToMeta,
oldPermIDToNewID,
itemPath,
itemData)
if err != nil {
@ -389,8 +391,8 @@ func restoreItem(
restoreFolderID,
copyBuffer,
restorePerms,
folderMetas,
permissionIDMappings,
parentDirToMeta,
oldPermIDToNewID,
itemPath,
itemData)
if err != nil {
@ -439,8 +441,8 @@ func restoreV1File(
restoreFolderID string,
copyBuffer []byte,
restorePerms bool,
folderMetas map[string]metadata.Metadata,
permissionIDMappings map[string]string,
parentDirToMeta map[string]metadata.Metadata,
oldPermIDToNewID map[string]string,
itemPath path.Path,
itemData data.Stream,
) (details.ItemInfo, error) {
@ -481,8 +483,8 @@ func restoreV1File(
itemID,
itemPath,
meta,
folderMetas,
permissionIDMappings)
parentDirToMeta,
oldPermIDToNewID)
if err != nil {
return details.ItemInfo{}, clues.Wrap(err, "restoring item permissions")
}
@ -500,8 +502,8 @@ func restoreV6File(
restoreFolderID string,
copyBuffer []byte,
restorePerms bool,
folderMetas map[string]metadata.Metadata,
permissionIDMappings map[string]string,
parentDirToMeta map[string]metadata.Metadata,
oldPermIDToNewID map[string]string,
itemPath path.Path,
itemData data.Stream,
) (details.ItemInfo, error) {
@ -553,8 +555,8 @@ func restoreV6File(
itemID,
itemPath,
meta,
folderMetas,
permissionIDMappings)
parentDirToMeta,
oldPermIDToNewID)
if err != nil {
return details.ItemInfo{}, clues.Wrap(err, "restoring item permissions")
}
@ -562,31 +564,31 @@ func restoreV6File(
return itemInfo, nil
}
// createRestoreFoldersWithPermissions creates the restore folder hierarchy in
// CreateRestoreFolders creates the restore folder hierarchy in
// the specified drive and returns the folder ID of the last folder entry in the
// hierarchy. Permissions are only applied to the last folder in the hierarchy.
// Passing nil for the permissions results in just creating the folder(s).
// folderCache is mutated, as a side effect of populating the items.
func createRestoreFoldersWithPermissions(
func CreateRestoreFolders(
ctx context.Context,
creds account.M365Config,
service graph.Servicer,
drivePath *path.DrivePath,
driveRootID string,
restoreFolders *path.Builder,
folderPath path.Path,
folderMetadata metadata.Metadata,
folderMetas map[string]metadata.Metadata,
driveRootFolderID string,
restoreDir *path.Builder,
originDir path.Path,
colMeta metadata.Metadata,
parentDirToMeta map[string]metadata.Metadata,
fc *folderCache,
permissionIDMappings map[string]string,
oldPermIDToNewID map[string]string,
restorePerms bool,
) (string, error) {
id, err := CreateRestoreFolders(
id, err := createRestoreFolders(
ctx,
service,
drivePath.DriveID,
driveRootID,
restoreFolders,
driveRootFolderID,
restoreDir,
fc)
if err != nil {
return "", err
@ -607,28 +609,28 @@ func createRestoreFoldersWithPermissions(
service,
drivePath.DriveID,
id,
folderPath,
folderMetadata,
folderMetas,
permissionIDMappings)
originDir,
colMeta,
parentDirToMeta,
oldPermIDToNewID)
return id, err
}
// CreateRestoreFolders creates the restore folder hierarchy in the specified
// createRestoreFolders creates the restore folder hierarchy in the specified
// drive and returns the folder ID of the last folder entry in the hierarchy.
// folderCache is mutated, as a side effect of populating the items.
func CreateRestoreFolders(
func createRestoreFolders(
ctx context.Context,
service graph.Servicer,
driveID, driveRootID string,
restoreFolders *path.Builder,
restoreDir *path.Builder,
fc *folderCache,
) (string, error) {
var (
location = &path.Builder{}
parentFolderID = driveRootID
folders = restoreFolders.Elements()
folders = restoreDir.Elements()
)
for _, folder := range folders {

View File

@ -23,6 +23,8 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter {
return nil
}
var _ graph.Servicer = &oneDriveService{}
// TODO(ashmrtn): Merge with similar structs in graph and exchange packages.
type oneDriveService struct {
client msgraphsdk.GraphServiceClient

View File

@ -86,9 +86,7 @@ func DataCollections(
}
case path.LibrariesCategory:
var excludes map[string]map[string]struct{}
spcs, excludes, err = collectLibraries(
spcs, excluded, err = collectLibraries(
ctx,
itemClient,
serv,
@ -104,7 +102,7 @@ func DataCollections(
continue
}
for prefix, excludes := range excludes {
for prefix, excludes := range excluded {
if _, ok := excluded[prefix]; !ok {
excluded[prefix] = map[string]struct{}{}
}

View File

@ -386,13 +386,16 @@ func generateContainerOfItems(
dest,
collections)
opts := control.Defaults()
opts.RestorePermissions = true
deets, err := gc.ConsumeRestoreCollections(
ctx,
backupVersion,
acct,
sel,
dest,
control.Options{RestorePermissions: true},
opts,
dataColls,
fault.New(true))
require.NoError(t, err, clues.ToCore(err))

View File

@ -226,11 +226,9 @@ func (dm DetailsModel) FilterMetaFiles() DetailsModel {
}
// Check if a file is a metadata file. These are used to store
// additional data like permissions in case of OneDrive and are not to
// additional data like permissions (in case of drive items) and are not to
// be treated as regular files.
func (de Entry) isMetaFile() bool {
// TODO: Add meta file filtering to SharePoint as well once we add
// meta files for SharePoint.
return de.ItemInfo.OneDrive != nil && de.ItemInfo.OneDrive.IsMeta
}