Onedrive clean up (#2519)

## Description
Creates two files. One for Permissions and another for onedrive/data_collections.
1. Is a file used to hold all the permission specific functions for `OneDrive`. These may need to be leveraged in the future for SharePoint. 
2. Previously, the backup function for `OneDrive` was housed within the graph connector data collections file. All other applications have their code housed within their respective directories. Moved the increment graph connector messages outside of the function to match the other applications.  
<!-- Insert PR description-->

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

<!--- Please check the type of change your PR introduces: --->

- [x] 🧹 Tech Debt/Cleanup


## Test Plan
- [x] ⚡ Unit test
- [x] 💚 E2E

src/internal/connector/data_collections.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	"golang.org/x/exp/maps"
 
 	"github.com/alcionai/clues"
 	"github.com/alcionai/corso/src/internal/connector/discovery"
@@ -20,7 +19,6 @@ import (
 	"github.com/alcionai/corso/src/pkg/backup/details"
 	"github.com/alcionai/corso/src/pkg/control"
 	"github.com/alcionai/corso/src/pkg/fault"
-	"github.com/alcionai/corso/src/pkg/logger"
 	"github.com/alcionai/corso/src/pkg/path"
 	"github.com/alcionai/corso/src/pkg/selectors"
 )
@@ -91,7 +89,27 @@ func (gc *GraphConnector) DataCollections(
 		return colls, excludes, nil
 
 	case selectors.ServiceOneDrive:
-		return gc.OneDriveDataCollections(ctx, sels, metadata, ctrlOpts)
+		colls, excludes, err := onedrive.DataCollections(
+			ctx,
+			sels, metadata,
+			gc.credentials.AzureTenantID,
+			gc.itemClient,
+			gc.Service,
+			gc.UpdateStatus,
+			ctrlOpts,
+		)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, c := range colls {
+			// kopia doesn't stream Items() from deleted collections.
+			if c.State() != data.DeletedState {
+				gc.incrementAwaitingMessages()
+			}
+		}
+
+		return colls, excludes, nil
 
 	case selectors.ServiceSharePoint:
 		colls, excludes, err := sharepoint.DataCollections(
@@ -107,9 +125,7 @@ func (gc *GraphConnector) DataCollections(
 			return nil, nil, err
 		}
 
-		for range colls {
-			gc.incrementAwaitingMessages()
-		}
+		gc.incrementMessagesBy(len(colls))
 
 		return colls, excludes, nil
 
@@ -171,74 +187,6 @@ func checkServiceEnabled(
 	return true, nil
 }
 
-// ---------------------------------------------------------------------------
-// OneDrive
-// ---------------------------------------------------------------------------
-
-type odFolderMatcher struct {
-	scope selectors.OneDriveScope
-}
-
-func (fm odFolderMatcher) IsAny() bool {
-	return fm.scope.IsAny(selectors.OneDriveFolder)
-}
-
-func (fm odFolderMatcher) Matches(dir string) bool {
-	return fm.scope.Matches(selectors.OneDriveFolder, dir)
-}
-
-// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
-// for the specified user
-func (gc *GraphConnector) OneDriveDataCollections(
-	ctx context.Context,
-	selector selectors.Selector,
-	metadata []data.RestoreCollection,
-	ctrlOpts control.Options,
-) ([]data.BackupCollection, map[string]struct{}, error) {
-	odb, err := selector.ToOneDriveBackup()
-	if err != nil {
-		return nil, nil, clues.Wrap(err, "parsing selector").WithClues(ctx)
-	}
-
-	var (
-		user        = selector.DiscreteOwner
-		collections = []data.BackupCollection{}
-		allExcludes = map[string]struct{}{}
-	)
-
-	// for each scope that includes oneDrive items, get all
-	for _, scope := range odb.Scopes() {
-		logger.Ctx(ctx).Debug("creating OneDrive collections")
-
-		odcs, excludes, err := onedrive.NewCollections(
-			gc.itemClient,
-			gc.credentials.AzureTenantID,
-			user,
-			onedrive.OneDriveSource,
-			odFolderMatcher{scope},
-			gc.Service,
-			gc.UpdateStatus,
-			ctrlOpts,
-		).Get(ctx, metadata)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		collections = append(collections, odcs...)
-
-		maps.Copy(allExcludes, excludes)
-	}
-
-	for _, c := range collections {
-		if c.State() != data.DeletedState {
-			// kopia doesn't stream Items() from deleted collections
-			gc.incrementAwaitingMessages()
-		}
-	}
-
-	return collections, allExcludes, nil
-}
-
 // RestoreDataCollections restores data from the specified collections
 // into M365 using the GraphAPI.
 // SideEffect: gc.status is updated at the completion of operation

src/internal/connector/graph_connector.go

@@ -293,6 +293,10 @@ func (gc *GraphConnector) incrementAwaitingMessages() {
 	gc.wg.Add(1)
 }
 
+func (gc *GraphConnector) incrementMessagesBy(num int) {
+	gc.wg.Add(num)
+}
+
 // ---------------------------------------------------------------------------
 // Helper Funcs
 // ---------------------------------------------------------------------------

src/internal/connector/onedrive/data_collections.go

@@ -0,0 +1,80 @@
+package onedrive
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/alcionai/clues"
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/internal/connector/support"
+	"github.com/alcionai/corso/src/internal/data"
+	"github.com/alcionai/corso/src/pkg/control"
+	"github.com/alcionai/corso/src/pkg/logger"
+	"github.com/alcionai/corso/src/pkg/selectors"
+	"golang.org/x/exp/maps"
+)
+
+// ---------------------------------------------------------------------------
+// OneDrive
+// ---------------------------------------------------------------------------
+
+type odFolderMatcher struct {
+	scope selectors.OneDriveScope
+}
+
+func (fm odFolderMatcher) IsAny() bool {
+	return fm.scope.IsAny(selectors.OneDriveFolder)
+}
+
+func (fm odFolderMatcher) Matches(dir string) bool {
+	return fm.scope.Matches(selectors.OneDriveFolder, dir)
+}
+
+// OneDriveDataCollections returns a set of DataCollection which represents the OneDrive data
+// for the specified user
+func DataCollections(
+	ctx context.Context,
+	selector selectors.Selector,
+	metadata []data.RestoreCollection,
+	tenant string,
+	itemClient *http.Client,
+	service graph.Servicer,
+	su support.StatusUpdater,
+	ctrlOpts control.Options,
+) ([]data.BackupCollection, map[string]struct{}, error) {
+	odb, err := selector.ToOneDriveBackup()
+	if err != nil {
+		return nil, nil, clues.Wrap(err, "parsing selector").WithClues(ctx)
+	}
+
+	var (
+		user        = selector.DiscreteOwner
+		collections = []data.BackupCollection{}
+		allExcludes = map[string]struct{}{}
+	)
+
+	// for each scope that includes oneDrive items, get all
+	for _, scope := range odb.Scopes() {
+		logger.Ctx(ctx).With("user", user).Debug("Creating OneDrive collections")
+
+		odcs, excludes, err := NewCollections(
+			itemClient,
+			tenant,
+			user,
+			OneDriveSource,
+			odFolderMatcher{scope},
+			service,
+			su,
+			ctrlOpts,
+		).Get(ctx, metadata)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		collections = append(collections, odcs...)
+
+		maps.Copy(allExcludes, excludes)
+	}
+
+	return collections, allExcludes, nil
+}

src/internal/connector/onedrive/permission.go

@@ -0,0 +1,198 @@
+package onedrive
+
+import (
+	"context"
+
+	"github.com/alcionai/clues"
+	"github.com/alcionai/corso/src/internal/connector/graph"
+	"github.com/alcionai/corso/src/pkg/path"
+	msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+	"github.com/pkg/errors"
+)
+
+func getParentPermissions(
+	parentPath path.Path,
+	parentPermissions map[string][]UserPermission,
+) ([]UserPermission, error) {
+	parentPerms, ok := parentPermissions[parentPath.String()]
+	if !ok {
+		onedrivePath, err := path.ToOneDrivePath(parentPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "invalid restore path")
+		}
+
+		if len(onedrivePath.Folders) != 0 {
+			return nil, errors.Wrap(err, "computing item permissions")
+		}
+
+		parentPerms = []UserPermission{}
+	}
+
+	return parentPerms, nil
+}
+
+func getParentAndCollectionPermissions(
+	drivePath *path.DrivePath,
+	collectionPath path.Path,
+	permissions map[string][]UserPermission,
+	restorePerms bool,
+) ([]UserPermission, []UserPermission, error) {
+	if !restorePerms {
+		return nil, nil, nil
+	}
+
+	var parentPerms []UserPermission
+
+	// Only get parent permissions if we're not restoring the root.
+	if len(drivePath.Folders) > 0 {
+		parentPath, err := collectionPath.Dir()
+		if err != nil {
+			return nil, nil, clues.Wrap(err, "getting parent path")
+		}
+
+		parentPerms, err = getParentPermissions(parentPath, permissions)
+		if err != nil {
+			return nil, nil, clues.Wrap(err, "getting parent permissions")
+		}
+	}
+
+	// TODO(ashmrtn): For versions after this pull the permissions from the
+	// current collection with Fetch().
+	colPerms, err := getParentPermissions(collectionPath, permissions)
+	if err != nil {
+		return nil, nil, clues.Wrap(err, "getting collection permissions")
+	}
+
+	return parentPerms, colPerms, nil
+}
+
+// createRestoreFoldersWithPermissions creates the restore folder hierarchy in
+// the specified drive and returns the folder ID of the last folder entry in the
+// hierarchy. Permissions are only applied to the last folder in the hierarchy.
+// Passing nil for the permissions results in just creating the folder(s).
+func createRestoreFoldersWithPermissions(
+	ctx context.Context,
+	service graph.Servicer,
+	driveID string,
+	restoreFolders []string,
+	parentPermissions []UserPermission,
+	folderPermissions []UserPermission,
+	permissionIDMappings map[string]string,
+) (string, error) {
+	id, err := CreateRestoreFolders(ctx, service, driveID, restoreFolders)
+	if err != nil {
+		return "", err
+	}
+
+	err = restorePermissions(
+		ctx,
+		service,
+		driveID,
+		id,
+		parentPermissions,
+		folderPermissions,
+		permissionIDMappings)
+
+	return id, err
+}
+
+// getChildPermissions is to filter out permissions present in the
+// parent from the ones that are available for child. This is
+// necessary as we store the nested permissions in the child. We
+// cannot avoid storing the nested permissions as it is possible that
+// a file in a folder can remove the nested permission that is present
+// on itself.
+func getChildPermissions(childPermissions, parentPermissions []UserPermission) ([]UserPermission, []UserPermission) {
+	addedPermissions := []UserPermission{}
+	removedPermissions := []UserPermission{}
+
+	for _, cp := range childPermissions {
+		found := false
+
+		for _, pp := range parentPermissions {
+			if cp.ID == pp.ID {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			addedPermissions = append(addedPermissions, cp)
+		}
+	}
+
+	for _, pp := range parentPermissions {
+		found := false
+
+		for _, cp := range childPermissions {
+			if pp.ID == cp.ID {
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			removedPermissions = append(removedPermissions, pp)
+		}
+	}
+
+	return addedPermissions, removedPermissions
+}
+
+// restorePermissions takes in the permissions that were added and the
+// removed(ones present in parent but not in child) and adds/removes
+// the necessary permissions on onedrive objects.
+func restorePermissions(
+	ctx context.Context,
+	service graph.Servicer,
+	driveID string,
+	itemID string,
+	parentPerms []UserPermission,
+	childPerms []UserPermission,
+	permissionIDMappings map[string]string,
+) error {
+	permAdded, permRemoved := getChildPermissions(childPerms, parentPerms)
+
+	ctx = clues.Add(ctx, "permission_item_id", itemID)
+
+	for _, p := range permRemoved {
+		err := service.Client().
+			DrivesById(driveID).
+			ItemsById(itemID).
+			PermissionsById(permissionIDMappings[p.ID]).
+			Delete(ctx, nil)
+		if err != nil {
+			return clues.Wrap(err, "removing permissions").WithClues(ctx).With(graph.ErrData(err)...)
+		}
+	}
+
+	for _, p := range permAdded {
+		pbody := msdrive.NewItemsItemInvitePostRequestBody()
+		pbody.SetRoles(p.Roles)
+
+		if p.Expiration != nil {
+			expiry := p.Expiration.String()
+			pbody.SetExpirationDateTime(&expiry)
+		}
+
+		si := false
+		pbody.SetSendInvitation(&si)
+
+		rs := true
+		pbody.SetRequireSignIn(&rs)
+
+		rec := models.NewDriveRecipient()
+		rec.SetEmail(&p.Email)
+		pbody.SetRecipients([]models.DriveRecipientable{rec})
+
+		np, err := service.Client().DrivesById(driveID).ItemsById(itemID).Invite().Post(ctx, pbody, nil)
+		if err != nil {
+			return clues.Wrap(err, "setting permissions").WithClues(ctx).With(graph.ErrData(err)...)
+		}
+
+		permissionIDMappings[p.ID] = *np.GetValue()[0].GetId()
+	}
+
+	return nil
+}

src/internal/connector/onedrive/restore.go

@@ -9,8 +9,6 @@ import (
 	"strings"
 
 	"github.com/alcionai/clues"
-	msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive"
-	"github.com/microsoftgraph/msgraph-sdk-go/models"
 	"github.com/pkg/errors"
 
 	"github.com/alcionai/corso/src/internal/common/ptr"
@@ -32,62 +30,6 @@ import (
 // https://docs.microsoft.com/en-us/graph/api/driveitem-createuploadsession?view=graph-rest-1.0#best-practices
 const copyBufferSize = 5 * 1024 * 1024
 
-func getParentPermissions(
-	parentPath path.Path,
-	parentPermissions map[string][]UserPermission,
-) ([]UserPermission, error) {
-	parentPerms, ok := parentPermissions[parentPath.String()]
-	if !ok {
-		onedrivePath, err := path.ToOneDrivePath(parentPath)
-		if err != nil {
-			return nil, errors.Wrap(err, "invalid restore path")
-		}
-
-		if len(onedrivePath.Folders) != 0 {
-			return nil, errors.Wrap(err, "computing item permissions")
-		}
-
-		parentPerms = []UserPermission{}
-	}
-
-	return parentPerms, nil
-}
-
-func getParentAndCollectionPermissions(
-	drivePath *path.DrivePath,
-	collectionPath path.Path,
-	permissions map[string][]UserPermission,
-	restorePerms bool,
-) ([]UserPermission, []UserPermission, error) {
-	if !restorePerms {
-		return nil, nil, nil
-	}
-
-	var parentPerms []UserPermission
-
-	// Only get parent permissions if we're not restoring the root.
-	if len(drivePath.Folders) > 0 {
-		parentPath, err := collectionPath.Dir()
-		if err != nil {
-			return nil, nil, clues.Wrap(err, "getting parent path")
-		}
-
-		parentPerms, err = getParentPermissions(parentPath, permissions)
-		if err != nil {
-			return nil, nil, clues.Wrap(err, "getting parent permissions")
-		}
-	}
-
-	// TODO(ashmrtn): For versions after this pull the permissions from the
-	// current collection with Fetch().
-	colPerms, err := getParentPermissions(collectionPath, permissions)
-	if err != nil {
-		return nil, nil, clues.Wrap(err, "getting collection permissions")
-	}
-
-	return parentPerms, colPerms, nil
-}
-
 // RestoreCollections will restore the specified data collections into OneDrive
 func RestoreCollections(
 	ctx context.Context,
@@ -518,36 +460,6 @@ func restoreV2File(
 	return itemInfo, nil
 }
 
-// createRestoreFoldersWithPermissions creates the restore folder hierarchy in
-// the specified drive and returns the folder ID of the last folder entry in the
-// hierarchy. Permissions are only applied to the last folder in the hierarchy.
-// Passing nil for the permissions results in just creating the folder(s).
-func createRestoreFoldersWithPermissions(
-	ctx context.Context,
-	service graph.Servicer,
-	driveID string,
-	restoreFolders []string,
-	parentPermissions []UserPermission,
-	folderPermissions []UserPermission,
-	permissionIDMappings map[string]string,
-) (string, error) {
-	id, err := CreateRestoreFolders(ctx, service, driveID, restoreFolders)
-	if err != nil {
-		return "", err
-	}
-
-	err = restorePermissions(
-		ctx,
-		service,
-		driveID,
-		id,
-		parentPermissions,
-		folderPermissions,
-		permissionIDMappings)
-
-	return id, err
-}
-
 // CreateRestoreFolders creates the restore folder hierarchy in the specified
 // drive and returns the folder ID of the last folder entry in the hierarchy.
 func CreateRestoreFolders(
@@ -690,103 +602,3 @@ func getMetadata(metar io.ReadCloser) (Metadata, error) {
 
 	return meta, nil
 }
-
-// getChildPermissions is to filter out permissions present in the
-// parent from the ones that are available for child. This is
-// necessary as we store the nested permissions in the child. We
-// cannot avoid storing the nested permissions as it is possible that
-// a file in a folder can remove the nested permission that is present
-// on itself.
-func getChildPermissions(childPermissions, parentPermissions []UserPermission) ([]UserPermission, []UserPermission) {
-	addedPermissions := []UserPermission{}
-	removedPermissions := []UserPermission{}
-
-	for _, cp := range childPermissions {
-		found := false
-
-		for _, pp := range parentPermissions {
-			if cp.ID == pp.ID {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			addedPermissions = append(addedPermissions, cp)
-		}
-	}
-
-	for _, pp := range parentPermissions {
-		found := false
-
-		for _, cp := range childPermissions {
-			if pp.ID == cp.ID {
-				found = true
-				break
-			}
-		}
-
-		if !found {
-			removedPermissions = append(removedPermissions, pp)
-		}
-	}
-
-	return addedPermissions, removedPermissions
-}
-
-// restorePermissions takes in the permissions that were added and the
-// removed(ones present in parent but not in child) and adds/removes
-// the necessary permissions on onedrive objects.
-func restorePermissions(
-	ctx context.Context,
-	service graph.Servicer,
-	driveID string,
-	itemID string,
-	parentPerms []UserPermission,
-	childPerms []UserPermission,
-	permissionIDMappings map[string]string,
-) error {
-	permAdded, permRemoved := getChildPermissions(childPerms, parentPerms)
-
-	ctx = clues.Add(ctx, "permission_item_id", itemID)
-
-	for _, p := range permRemoved {
-		err := service.Client().
-			DrivesById(driveID).
-			ItemsById(itemID).
-			PermissionsById(permissionIDMappings[p.ID]).
-			Delete(ctx, nil)
-		if err != nil {
-			return clues.Wrap(err, "removing permissions").WithClues(ctx).With(graph.ErrData(err)...)
-		}
-	}
-
-	for _, p := range permAdded {
-		pbody := msdrive.NewItemsItemInvitePostRequestBody()
-		pbody.SetRoles(p.Roles)
-
-		if p.Expiration != nil {
-			expiry := p.Expiration.String()
-			pbody.SetExpirationDateTime(&expiry)
-		}
-
-		si := false
-		pbody.SetSendInvitation(&si)
-
-		rs := true
-		pbody.SetRequireSignIn(&rs)
-
-		rec := models.NewDriveRecipient()
-		rec.SetEmail(&p.Email)
-		pbody.SetRecipients([]models.DriveRecipientable{rec})
-
-		np, err := service.Client().DrivesById(driveID).ItemsById(itemID).Invite().Post(ctx, pbody, nil)
-		if err != nil {
-			return clues.Wrap(err, "setting permissions").WithClues(ctx).With(graph.ErrData(err)...)
-		}
-
-		permissionIDMappings[p.ID] = *np.GetValue()[0].GetId()
-	}
-
-	return nil
-}