diff --git a/CHANGELOG.md b/CHANGELOG.md index 5170dc50f..906267535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Restore requires the protected resource to have access to the service being restored. ### Added -- Added option to export data from OneDrive backups as individual files or as a single zip file. +- Added option to export data from OneDrive and SharePoint backups as individual files or as a single zip file. ## [v0.11.1] (beta) - 2023-07-20 diff --git a/src/cli/export/export.go b/src/cli/export/export.go index d3ec09a8a..e0deed014 100644 --- a/src/cli/export/export.go +++ b/src/cli/export/export.go @@ -1,11 +1,26 @@ package export import ( + "context" + "errors" + + "github.com/alcionai/clues" "github.com/spf13/cobra" + + . "github.com/alcionai/corso/src/cli/print" + "github.com/alcionai/corso/src/cli/repo" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/internal/common/dttm" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/selectors" ) var exportCommands = []func(cmd *cobra.Command) *cobra.Command{ addOneDriveCommands, + addSharePointCommands, } // AddCommands attaches all `corso export * *` commands to the parent. @@ -37,3 +52,57 @@ func exportCmd() *cobra.Command { func handleExportCmd(cmd *cobra.Command, args []string) error { return cmd.Help() } + +func runExport( + ctx context.Context, + cmd *cobra.Command, + args []string, + ueco utils.ExportCfgOpts, + sel selectors.Selector, + backupID, serviceName string, +) error { + r, _, _, _, err := utils.GetAccountAndConnect(ctx, sel.PathService(), repo.S3Overrides(cmd)) + if err != nil { + return Only(ctx, err) + } + + defer utils.CloseRepo(ctx, r) + + exportLocation := args[0] + if len(exportLocation) == 0 { + // This should not be possible, but adding it just in case. + exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) + } + + Infof(ctx, "Exporting to folder %s", exportLocation) + + eo, err := r.NewExport( + ctx, + backupID, + sel, + utils.MakeExportConfig(ctx, ueco)) + if err != nil { + return Only(ctx, clues.Wrap(err, "Failed to initialize "+serviceName+" export")) + } + + expColl, err := eo.Run(ctx) + if err != nil { + if errors.Is(err, data.ErrNotFound) { + return Only(ctx, clues.New("Backup or backup details missing for id "+backupID)) + } + + return Only(ctx, clues.Wrap(err, "Failed to run "+serviceName+" export")) + } + + // It would be better to give a progressbar than a spinner, but we + // have any way of knowing how many files are available as of now. + diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") + defer close(diskWriteComplete) + + err = export.ConsumeExportCollections(ctx, exportLocation, expColl, eo.Errors) + if err != nil { + return Only(ctx, err) + } + + return nil +} diff --git a/src/cli/export/onedrive.go b/src/cli/export/onedrive.go index 6e715153d..593149bd9 100644 --- a/src/cli/export/onedrive.go +++ b/src/cli/export/onedrive.go @@ -1,26 +1,12 @@ package export import ( - "context" - "io" - "os" - ospath "path" - - "github.com/alcionai/clues" "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/alcionai/corso/src/cli/flags" - . "github.com/alcionai/corso/src/cli/print" - "github.com/alcionai/corso/src/cli/repo" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/common/dttm" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/internal/observe" - "github.com/alcionai/corso/src/pkg/control" - "github.com/alcionai/corso/src/pkg/export" - "github.com/alcionai/corso/src/pkg/path" ) // called by export.go to map subcommands to provider-specific handling. @@ -103,113 +89,8 @@ func exportOneDriveCmd(cmd *cobra.Command, args []string) error { return err } - r, _, _, _, err := utils.GetAccountAndConnect(ctx, path.OneDriveService, repo.S3Overrides(cmd)) - if err != nil { - return Only(ctx, err) - } - - defer utils.CloseRepo(ctx, r) - - exportLocation := args[0] - if exportLocation == "" { - // This is unlikely, but adding it just in case. - exportLocation = control.DefaultRestoreLocation + dttm.FormatNow(dttm.HumanReadableDriveItem) - } - - Infof(ctx, "Exporting to folder %s", exportLocation) - sel := utils.IncludeOneDriveRestoreDataSelectors(opts) utils.FilterOneDriveRestoreInfoSelectors(sel, opts) - eo, err := r.NewExport( - ctx, - flags.BackupIDFV, - sel.Selector, - utils.MakeExportConfig(ctx, opts.ExportCfg), - ) - if err != nil { - return Only(ctx, clues.Wrap(err, "Failed to initialize OneDrive export")) - } - - expColl, err := eo.Run(ctx) - if err != nil { - if errors.Is(err, data.ErrNotFound) { - return Only(ctx, clues.New("Backup or backup details missing for id "+flags.BackupIDFV)) - } - - return Only(ctx, clues.Wrap(err, "Failed to run OneDrive export")) - } - - // It would be better to give a progressbar than a spinner, but we - // have know way of knowing how many files are available as of now. - diskWriteComplete := observe.MessageWithCompletion(ctx, "Writing data to disk") - defer func() { - diskWriteComplete <- struct{}{} - close(diskWriteComplete) - }() - - err = writeExportCollections(ctx, exportLocation, expColl) - if err != nil { - return err - } - - return nil -} - -func writeExportCollections( - ctx context.Context, - exportLocation string, - expColl []export.Collection, -) error { - for _, col := range expColl { - folder := ospath.Join(exportLocation, col.BasePath()) - - for item := range col.Items(ctx) { - err := item.Error - if err != nil { - return Only(ctx, clues.Wrap(err, "getting item").With("dir_name", folder)) - } - - err = writeExportItem(ctx, item, folder) - if err != nil { - return err - } - } - } - - return nil -} - -// writeExportItem writes an ExportItem to disk in the specified folder. -func writeExportItem(ctx context.Context, item export.Item, folder string) error { - name := item.Data.Name - fpath := ospath.Join(folder, name) - - progReader, pclose := observe.ItemSpinner( - ctx, - item.Data.Body, - observe.ItemExportMsg, - clues.Hide(name)) - - defer item.Data.Body.Close() - defer pclose() - - err := os.MkdirAll(folder, os.ModePerm) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating directory").With("dir_name", folder)) - } - - // In case the user tries to restore to a non-clean - // directory, we might run into collisions an fail. - f, err := os.Create(fpath) - if err != nil { - return Only(ctx, clues.Wrap(err, "creating file").With("file_name", name, "file_dir", folder)) - } - - _, err = io.Copy(f, progReader) - if err != nil { - return Only(ctx, clues.Wrap(err, "writing file").With("file_name", name, "file_dir", folder)) - } - - return nil + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "OneDrive") } diff --git a/src/cli/export/onedrive_test.go b/src/cli/export/onedrive_test.go index 775dd4a70..59ab966e8 100644 --- a/src/cli/export/onedrive_test.go +++ b/src/cli/export/onedrive_test.go @@ -59,6 +59,7 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { cmd.SetArgs([]string{ "onedrive", + testdata.RestoreDestination, "--" + flags.RunModeFN, flags.RunModeFlagTest, "--" + flags.BackupFN, testdata.BackupInput, "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), @@ -68,15 +69,14 @@ func (suite *OneDriveUnitSuite) TestAddOneDriveCommands() { "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, - "--" + flags.ArchiveFN, - "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, - testdata.RestoreDestination, + // bool flags + "--" + flags.ArchiveFN, }) cmd.SetOut(new(bytes.Buffer)) // drop output diff --git a/src/cli/export/sharepoint.go b/src/cli/export/sharepoint.go new file mode 100644 index 000000000..ec71a5f2b --- /dev/null +++ b/src/cli/export/sharepoint.go @@ -0,0 +1,100 @@ +package export + +import ( + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" +) + +// called by export.go to map subcommands to provider-specific handling. +func addSharePointCommands(cmd *cobra.Command) *cobra.Command { + var ( + c *cobra.Command + fs *pflag.FlagSet + ) + + switch cmd.Use { + case exportCommand: + c, fs = utils.AddCommand(cmd, sharePointExportCmd()) + + c.Use = c.Use + " " + sharePointServiceCommandUseSuffix + + // Flags addition ordering should follow the order we want them to appear in help and docs: + // More generic (ex: --user) and more frequently used flags take precedence. + fs.SortFlags = false + + flags.AddBackupIDFlag(c, true) + flags.AddSharePointDetailsAndRestoreFlags(c) + flags.AddExportConfigFlags(c) + flags.AddFailFastFlag(c) + flags.AddCorsoPassphaseFlags(c) + flags.AddAWSCredsFlags(c) + } + + return c +} + +const ( + sharePointServiceCommand = "sharepoint" + sharePointServiceCommandUseSuffix = "--backup " + + //nolint:lll + sharePointServiceCommandExportExamples = `# Export file with ID 98765abcdef in Bob's latest backup (1234abcd...) to my-exports directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef my-exports + +# Export files named "ServerRenderTemplate.xsl" in the folder "Display Templates/Style Sheets". as archive to current directory +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" --archive . + +# Export all files in the folder "Display Templates/Style Sheets" that were created before 2020 to my-exports directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" my-exports + +# Export all files in the "Documents" library to current directory. +corso export sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --library Documents --folder "Display Templates/Style Sheets" .` +) + +// `corso export sharepoint [...] ` +func sharePointExportCmd() *cobra.Command { + return &cobra.Command{ + Use: sharePointServiceCommand, + Short: "Export M365 SharePoint service data", + RunE: exportSharePointCmd, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return errors.New("missing restore destination") + } + + return nil + }, + Example: sharePointServiceCommandExportExamples, + } +} + +// processes an sharepoint service export. +func exportSharePointCmd(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + if utils.HasNoFlagsAndShownHelp(cmd) { + return nil + } + + opts := utils.MakeSharePointOpts(cmd) + + if flags.RunModeFV == flags.RunModeFlagTest { + return nil + } + + if err := utils.ValidateSharePointRestoreFlags(flags.BackupIDFV, opts); err != nil { + return err + } + + sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + utils.FilterSharePointRestoreInfoSelectors(sel, opts) + + return runExport(ctx, cmd, args, opts.ExportCfg, sel.Selector, flags.BackupIDFV, "SharePoint") +} diff --git a/src/cli/export/sharepoint_test.go b/src/cli/export/sharepoint_test.go new file mode 100644 index 000000000..48ce28f5c --- /dev/null +++ b/src/cli/export/sharepoint_test.go @@ -0,0 +1,118 @@ +package export + +import ( + "bytes" + "testing" + + "github.com/alcionai/clues" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/flags" + "github.com/alcionai/corso/src/cli/utils" + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" +) + +type SharePointUnitSuite struct { + tester.Suite +} + +func TestSharePointUnitSuite(t *testing.T) { + suite.Run(t, &SharePointUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *SharePointUnitSuite) TestAddSharePointCommands() { + expectUse := sharePointServiceCommand + " " + sharePointServiceCommandUseSuffix + + table := []struct { + name string + use string + expectUse string + expectShort string + expectRunE func(*cobra.Command, []string) error + }{ + {"export sharepoint", exportCommand, expectUse, sharePointExportCmd().Short, exportSharePointCmd}, + } + for _, test := range table { + suite.Run(test.name, func() { + t := suite.T() + + cmd := &cobra.Command{Use: test.use} + + // normally a persistent flag from the root. + // required to ensure a dry run. + flags.AddRunModeFlag(cmd, true) + + c := addSharePointCommands(cmd) + require.NotNil(t, c) + + cmds := cmd.Commands() + require.Len(t, cmds, 1) + + child := cmds[0] + assert.Equal(t, test.expectUse, child.Use) + assert.Equal(t, test.expectShort, child.Short) + tester.AreSameFunc(t, test.expectRunE, child.RunE) + + cmd.SetArgs([]string{ + "sharepoint", + testdata.RestoreDestination, + "--" + flags.RunModeFN, flags.RunModeFlagTest, + "--" + flags.BackupFN, testdata.BackupInput, + "--" + flags.LibraryFN, testdata.LibraryInput, + "--" + flags.FileFN, testdata.FlgInputs(testdata.FileNameInput), + "--" + flags.FolderFN, testdata.FlgInputs(testdata.FolderPathInput), + "--" + flags.FileCreatedAfterFN, testdata.FileCreatedAfterInput, + "--" + flags.FileCreatedBeforeFN, testdata.FileCreatedBeforeInput, + "--" + flags.FileModifiedAfterFN, testdata.FileModifiedAfterInput, + "--" + flags.FileModifiedBeforeFN, testdata.FileModifiedBeforeInput, + "--" + flags.ListItemFN, testdata.FlgInputs(testdata.ListItemInput), + "--" + flags.ListFolderFN, testdata.FlgInputs(testdata.ListFolderInput), + "--" + flags.PageFN, testdata.FlgInputs(testdata.PageInput), + "--" + flags.PageFolderFN, testdata.FlgInputs(testdata.PageFolderInput), + + "--" + flags.AWSAccessKeyFN, testdata.AWSAccessKeyID, + "--" + flags.AWSSecretAccessKeyFN, testdata.AWSSecretAccessKey, + "--" + flags.AWSSessionTokenFN, testdata.AWSSessionToken, + + "--" + flags.CorsoPassphraseFN, testdata.CorsoPassphrase, + + // bool flags + "--" + flags.ArchiveFN, + }) + + cmd.SetOut(new(bytes.Buffer)) // drop output + cmd.SetErr(new(bytes.Buffer)) // drop output + err := cmd.Execute() + assert.NoError(t, err, clues.ToCore(err)) + + opts := utils.MakeSharePointOpts(cmd) + assert.Equal(t, testdata.BackupInput, flags.BackupIDFV) + + assert.Equal(t, testdata.LibraryInput, opts.Library) + assert.ElementsMatch(t, testdata.FileNameInput, opts.FileName) + assert.ElementsMatch(t, testdata.FolderPathInput, opts.FolderPath) + assert.Equal(t, testdata.FileCreatedAfterInput, opts.FileCreatedAfter) + assert.Equal(t, testdata.FileCreatedBeforeInput, opts.FileCreatedBefore) + assert.Equal(t, testdata.FileModifiedAfterInput, opts.FileModifiedAfter) + assert.Equal(t, testdata.FileModifiedBeforeInput, opts.FileModifiedBefore) + + assert.ElementsMatch(t, testdata.ListItemInput, opts.ListItem) + assert.ElementsMatch(t, testdata.ListFolderInput, opts.ListFolder) + + assert.ElementsMatch(t, testdata.PageInput, opts.Page) + assert.ElementsMatch(t, testdata.PageFolderInput, opts.PageFolder) + + assert.Equal(t, testdata.Archive, opts.ExportCfg.Archive) + + assert.Equal(t, testdata.AWSAccessKeyID, flags.AWSAccessKeyFV) + assert.Equal(t, testdata.AWSSecretAccessKey, flags.AWSSecretAccessKeyFV) + assert.Equal(t, testdata.AWSSessionToken, flags.AWSSessionTokenFV) + + assert.Equal(t, testdata.CorsoPassphrase, flags.CorsoPassphraseFV) + }) + } +} diff --git a/src/cli/restore/sharepoint_test.go b/src/cli/restore/sharepoint_test.go index 6a8de8e57..c9bc8277f 100644 --- a/src/cli/restore/sharepoint_test.go +++ b/src/cli/restore/sharepoint_test.go @@ -34,7 +34,7 @@ func (suite *SharePointUnitSuite) TestAddSharePointCommands() { expectShort string expectRunE func(*cobra.Command, []string) error }{ - {"restore onedrive", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, + {"restore sharepoint", restoreCommand, expectUse, sharePointRestoreCmd().Short, restoreSharePointCmd}, } for _, test := range table { suite.Run(test.name, func() { diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index 6672beda1..2ab43d90c 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -32,6 +32,7 @@ type SharePointOpts struct { Page []string RestoreCfg RestoreCfgOpts + ExportCfg ExportCfgOpts Populated flags.PopulatedFlags } @@ -56,6 +57,7 @@ func MakeSharePointOpts(cmd *cobra.Command) SharePointOpts { PageFolder: flags.PageFolderFV, RestoreCfg: makeRestoreCfgOpts(cmd), + ExportCfg: makeExportCfgOpts(cmd), // populated contains the list of flags that appear in the // command, according to pflags. Use this to differentiate diff --git a/src/internal/m365/export.go b/src/internal/m365/export.go index 085881803..4da037e26 100644 --- a/src/internal/m365/export.go +++ b/src/internal/m365/export.go @@ -41,7 +41,8 @@ func (ctrl *Controller) ProduceExportCollections( ) switch sels.Service { - case selectors.ServiceOneDrive: + case selectors.ServiceOneDrive, selectors.ServiceSharePoint: + // OneDrive and SharePoint can share the code to create collections expCollections, err = onedrive.ProduceExportCollections( ctx, backupVersion, diff --git a/src/pkg/export/consume.go b/src/pkg/export/consume.go new file mode 100644 index 000000000..899f9c3ba --- /dev/null +++ b/src/pkg/export/consume.go @@ -0,0 +1,79 @@ +package export + +import ( + "context" + "io" + "os" + "path/filepath" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/fault" +) + +func ConsumeExportCollections( + ctx context.Context, + exportLocation string, + expColl []Collection, + errs *fault.Bus, +) error { + el := errs.Local() + + for _, col := range expColl { + if el.Failure() != nil { + break + } + + folder := filepath.Join(exportLocation, col.BasePath()) + ictx := clues.Add(ctx, "dir_name", folder) + + for item := range col.Items(ctx) { + if item.Error != nil { + el.AddRecoverable(ictx, clues.Wrap(item.Error, "getting item").WithClues(ctx)) + } + + if err := writeItem(ictx, item, folder); err != nil { + el.AddRecoverable( + ictx, + clues.Wrap(err, "writing item").With("file_name", item.Data.Name).WithClues(ctx)) + } + } + } + + return el.Failure() +} + +// writeItem writes an ExportItem to disk in the specified folder. +func writeItem(ctx context.Context, item Item, folder string) error { + name := item.Data.Name + fpath := filepath.Join(folder, name) + + progReader, pclose := observe.ItemSpinner( + ctx, + item.Data.Body, + observe.ItemExportMsg, + clues.Hide(name)) + + defer item.Data.Body.Close() + defer pclose() + + err := os.MkdirAll(folder, os.ModePerm) + if err != nil { + return clues.Wrap(err, "creating directory") + } + + // In case the user tries to restore to a non-clean + // directory, we might run into collisions an fail. + f, err := os.Create(fpath) + if err != nil { + return clues.Wrap(err, "creating file") + } + + _, err = io.Copy(f, progReader) + if err != nil { + return clues.Wrap(err, "writing data") + } + + return nil +} diff --git a/src/cli/export/export_test.go b/src/pkg/export/consume_test.go similarity index 86% rename from src/cli/export/export_test.go rename to src/pkg/export/consume_test.go index f3df68177..7d22dc237 100644 --- a/src/cli/export/export_test.go +++ b/src/pkg/export/consume_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/pkg/export" + "github.com/alcionai/corso/src/pkg/fault" ) type ExportE2ESuite struct { @@ -31,12 +31,12 @@ func (suite *ExportE2ESuite) SetupSuite() { type mockExportCollection struct { path string - items []export.Item + items []Item } func (mec mockExportCollection) BasePath() string { return mec.path } -func (mec mockExportCollection) Items(context.Context) <-chan export.Item { - ch := make(chan export.Item) +func (mec mockExportCollection) Items(context.Context) <-chan Item { + ch := make(chan Item) go func() { defer close(ch) @@ -49,7 +49,7 @@ func (mec mockExportCollection) Items(context.Context) <-chan export.Item { return ch } -func (suite *ExportE2ESuite) TestWriteExportCollection() { +func (suite *ExportE2ESuite) TestConsumeExportCollection() { type ei struct { name string body string @@ -132,12 +132,12 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { ctx, flush := tester.NewContext(t) defer flush() - ecs := []export.Collection{} + ecs := []Collection{} for _, col := range test.cols { - items := []export.Item{} + items := []Item{} for _, item := range col.items { - items = append(items, export.Item{ - Data: export.ItemData{ + items = append(items, Item{ + Data: ItemData{ Name: item.name, Body: io.NopCloser((bytes.NewBufferString(item.body))), }, @@ -154,7 +154,7 @@ func (suite *ExportE2ESuite) TestWriteExportCollection() { require.NoError(t, err) defer os.RemoveAll(dir) - err = writeExportCollections(ctx, dir, ecs) + err = ConsumeExportCollections(ctx, dir, ecs, fault.New(true)) require.NoError(t, err, "writing data") for _, col := range test.cols { diff --git a/src/pkg/export/export.go b/src/pkg/export/export.go index 76a6b6d8b..73c173e04 100644 --- a/src/pkg/export/export.go +++ b/src/pkg/export/export.go @@ -7,7 +7,9 @@ import ( // Collection is the interface that is returned to the SDK consumer type Collection interface { - // BasePath gets the base path of the collection + // BasePath gets the base path of the collection. This is derived + // from FullPath, but trim out thing like drive id or any other part + // that is not needed to show the path to the collection. BasePath() string // Items gets the items within the collection(folder)