larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split backups by (resource owner, service) at CLI layer (#1609)

Browse Source 
## Description

Sketch of how to split backups by (resource owner, service) for exchange. Needs more support from selectors package to be complete.

## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [ ] 💻 CI/Deployment
- [ ] 🐹 Trivial/Minor

## Issue(s)

* closes #1505

## Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2022-11-30 13:50:49 -08:00 committed by GitHub
parent c4333ad531
commit 27d0980526
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 155 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/cli/backup/exchange.go
View File

@ -3,6 +3,7 @@ package backup
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -19,6 +20,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
"github.com/alcionai/corso/src/pkg/store"
)
@ -270,27 +272,62 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
sel := exchangeBackupCreateSelectors(user, exchangeData)
bo, err := r.NewBackup(ctx, sel)
users, err := m365.UserIDs(ctx, acct)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange backup"))
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users"))
}
err = bo.Run(ctx)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to run Exchange backup"))
var (
errs *multierror.Error
bIDs []model.StableID
)
for _, scope := range sel.DiscreteScopes(users) {
for _, selUser := range scope.Get(selectors.ExchangeUser) {
opSel := selectors.NewExchangeBackup()
opSel.Include([]selectors.ExchangeScope{scope.DiscreteCopy(selUser)})
bo, err := r.NewBackup(ctx, opSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to initialize Exchange backup for user %s",
scope.Get(selectors.ExchangeUser),
))
continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run Exchange backup for user %s",
scope.Get(selectors.ExchangeUser),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
}
}
bu, err := r.Backup(ctx, bo.Results.BackupID)
bups, err := r.Backups(ctx, bIDs)
if err != nil {
return Only(ctx, errors.Wrap(err, "Unable to retrieve backup results from storage"))
}
bu.Print(ctx)
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}
return nil
}
func exchangeBackupCreateSelectors(userIDs, data []string) selectors.Selector {
func exchangeBackupCreateSelectors(userIDs, data []string) *selectors.ExchangeBackup {
sel := selectors.NewExchangeBackup()
if len(data) == 0 {
@ -310,7 +347,7 @@ func exchangeBackupCreateSelectors(userIDs, data []string) selectors.Selector {
}
}
return sel.Selector
return sel
}
func validateExchangeBackupCreateFlags(userIDs, data []string) error {

57
src/cli/backup/onedrive.go
View File

@ -3,6 +3,7 @@ package backup
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -18,6 +19,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
"github.com/alcionai/corso/src/pkg/store"
)
@ -192,22 +194,57 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
sel := oneDriveBackupCreateSelectors(user)
bo, err := r.NewBackup(ctx, sel)
users, err := m365.UserIDs(ctx, acct)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive backup"))
return Only(ctx, errors.Wrap(err, "Failed to retrieve M365 users"))
}
err = bo.Run(ctx)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to run OneDrive backup"))
var (
errs *multierror.Error
bIDs []model.StableID
)
for _, scope := range sel.DiscreteScopes(users) {
for _, selUser := range scope.Get(selectors.OneDriveUser) {
opSel := selectors.NewOneDriveBackup()
opSel.Include([]selectors.OneDriveScope{scope.DiscreteCopy(selUser)})
bo, err := r.NewBackup(ctx, opSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to initialize OneDrive backup for user %s",
scope.Get(selectors.OneDriveUser),
))
continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run OneDrive backup for user %s",
scope.Get(selectors.OneDriveUser),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
}
}
bu, err := r.Backup(ctx, bo.Results.BackupID)
bups, err := r.Backups(ctx, bIDs)
if err != nil {
return errors.Wrap(err, "Unable to retrieve backup results from storage")
return Only(ctx, errors.Wrap(err, "Unable to retrieve backup results from storage"))
}
bu.Print(ctx)
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}
return nil
}
@ -220,11 +257,11 @@ func validateOneDriveBackupCreateFlags(users []string) error {
return nil
}
func oneDriveBackupCreateSelectors(users []string) selectors.Selector {
func oneDriveBackupCreateSelectors(users []string) *selectors.OneDriveBackup {
sel := selectors.NewOneDriveBackup()
sel.Include(sel.Users(users))
return sel.Selector
return sel
}
// ------------------------------------------------------------------------------------------------

57
src/cli/backup/sharepoint.go
View File

@ -3,6 +3,7 @@ package backup
import (
"context"
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@ -18,6 +19,7 @@ import (
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
"github.com/alcionai/corso/src/pkg/store"
)
@ -181,22 +183,57 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
sel := sharePointBackupCreateSelectors(site)
bo, err := r.NewBackup(ctx, sel)
sites, err := m365.Sites(ctx, acct)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to initialize SharePoint backup"))
return Only(ctx, errors.Wrap(err, "Failed to retrieve SharePoint sites"))
}
err = bo.Run(ctx)
if err != nil {
return Only(ctx, errors.Wrap(err, "Failed to run SharePoint backup"))
var (
errs *multierror.Error
bIDs []model.StableID
)
for _, scope := range sel.DiscreteScopes(sites) {
for _, selSite := range scope.Get(selectors.SharePointSite) {
opSel := selectors.NewSharePointBackup()
opSel.Include([]selectors.SharePointScope{scope.DiscreteCopy(selSite)})
bo, err := r.NewBackup(ctx, opSel.Selector)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to initialize SharePoint backup for site %s",
scope.Get(selectors.SharePointSite),
))
continue
}
err = bo.Run(ctx)
if err != nil {
errs = multierror.Append(errs, errors.Wrapf(
err,
"Failed to run SharePoint backup for site %s",
scope.Get(selectors.SharePointSite),
))
continue
}
bIDs = append(bIDs, bo.Results.BackupID)
}
}
bu, err := r.Backup(ctx, bo.Results.BackupID)
bups, err := r.Backups(ctx, bIDs)
if err != nil {
return errors.Wrap(err, "Unable to retrieve backup results from storage")
return Only(ctx, errors.Wrap(err, "Unable to retrieve backup results from storage"))
}
bu.Print(ctx)
backup.PrintAll(ctx, bups)
if e := errs.ErrorOrNil(); e != nil {
return Only(ctx, e)
}
return nil
}
@ -209,11 +246,11 @@ func validateSharePointBackupCreateFlags(sites []string) error {
return nil
}
func sharePointBackupCreateSelectors(sites []string) selectors.Selector {
func sharePointBackupCreateSelectors(sites []string) *selectors.SharePointBackup {
sel := selectors.NewSharePointBackup()
sel.Include(sel.Sites(sites))
return sel.Selector
return sel
}
// ------------------------------------------------------------------------------------------------

4
src/internal/connector/graph_connector.go
View File

@ -104,6 +104,10 @@ func NewGraphConnector(ctx context.Context, acct account.Account, r resource) (*
gc.graphService = *aService
// TODO(ashmrtn): When selectors only encapsulate a single resource owner that
// is not a wildcard don't populate users or sites when making the connector.
// For now this keeps things functioning if callers do pass in a selector like
// "*" instead of.
if r == AllResources || r == Users {
if err = gc.setTenantUsers(ctx); err != nil {
return nil, errors.Wrap(err, "retrieving tenant user list")

11
src/pkg/services/m365/m365.go
View File

@ -39,3 +39,14 @@ func GetEmailAndUserID(ctx context.Context, m365Account account.Account) (map[st
return gc.Users, nil
}
// Sites returns a list of SharePoint sites in the specified M365 tenant
// TODO: Implement paging support
func Sites(ctx context.Context, m365Account account.Account) ([]string, error) {
gc, err := connector.NewGraphConnector(ctx, m365Account, connector.Sites)
if err != nil {
return nil, errors.Wrap(err, "could not initialize M365 graph connection")
}
return gc.GetSites(), nil
}