From 285f8f01efbe8d002d0c40a8991cc0baffb8a015 Mon Sep 17 00:00:00 2001 From: Keepers Date: Tue, 21 Mar 2023 11:33:24 -0600 Subject: [PATCH] update sharepoint cli examples (#2822) Updates the sharepoint cli examples to better reflect the current support and utilities. --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :sunflower: Feature #### Issue(s) * #2786 #### Test Plan - [x] :muscle: Manual - [x] :green_heart: E2E --- src/cli/backup/exchange.go | 175 ++++-------------- src/cli/backup/onedrive.go | 80 ++------ src/cli/backup/sharepoint.go | 141 ++++---------- src/cli/restore/exchange.go | 91 +-------- src/cli/restore/onedrive.go | 69 +------ src/cli/restore/sharepoint.go | 113 +++-------- src/cli/utils/exchange.go | 121 +++++++++++- src/cli/utils/flags.go | 57 +++++- src/cli/utils/onedrive.go | 36 ++++ src/cli/utils/sharepoint.go | 131 +++++++++---- src/cli/utils/sharepoint_test.go | 100 +++++----- src/cli/utils/testdata/opts.go | 16 +- .../connector/data_collections_test.go | 3 +- src/internal/connector/graph_connector.go | 3 +- src/internal/connector/onedrive/collection.go | 65 ++++--- .../connector/onedrive/collection_test.go | 12 +- .../connector/onedrive/collections.go | 6 +- src/internal/connector/onedrive/item.go | 71 +++---- src/internal/connector/onedrive/permission.go | 2 +- src/internal/connector/onedrive/restore.go | 2 +- src/internal/observe/observe.go | 18 +- src/internal/operations/restore_test.go | 56 +++--- src/pkg/backup/details/details.go | 4 +- src/pkg/backup/details/details_test.go | 5 +- 24 files changed, 606 insertions(+), 771 deletions(-) diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index 2d91e06b8..cc2a36d64 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -26,28 +26,7 @@ import ( // exchange bucket info from flags var ( - backupID string exchangeData []string - user []string - - contact []string - contactFolder []string - contactName string - - email []string - emailFolder []string - emailReceivedAfter string - emailReceivedBefore string - emailSender string - emailSubject string - - event []string - eventCalendar []string - eventOrganizer string - eventRecurs string - eventStartsAfter string - eventStartsBefore string - eventSubject string ) const ( @@ -81,15 +60,15 @@ corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd --user a # Explore Alice's emails with subject containing "Hello world" in folder "Inbox" from a specific backup corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com --email-subject "Hello world" --email-folder Inbox + --user alice@example.com --email-subject "Hello world" --email-folder Inbox # Explore Bobs's events occurring after start of 2022 from a specific backup corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user bob@example.com --event-starts-after 2022-01-01T00:00:00 + --user bob@example.com --event-starts-after 2022-01-01T00:00:00 # Explore Alice's contacts with name containing Andy from a specific backup corso backup details exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com --contact-name Andy` + --user alice@example.com --contact-name Andy` ) // called by backup.go to map subcommands to provider-specific handling. @@ -109,10 +88,8 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { // Flags addition ordering should follow the order we want them to appear in help and docs: // More generic (ex: --user) and more frequently used flags take precedence. - fs.StringSliceVar( - &user, - utils.UserFN, nil, - "Backup Exchange data by a user's email; accepts '"+utils.Wildcard+"' to select all users") + utils.AddUserFlag(c) + fs.StringSliceVar( &exchangeData, utils.DataFN, nil, @@ -122,17 +99,15 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { case listCommand: c, fs = utils.AddCommand(cmd, exchangeListCmd()) + fs.SortFlags = false - fs.StringVar(&backupID, - "backup", "", - "Display a specific backup, including the items that failed or were skipped during processing.") - + utils.AddBackupIDFlag(c, false) addFailedItemsFN(c) addSkippedItemsFN(c) addRecoveredErrorsFN(c) case detailsCommand: - c, fs = utils.AddCommand(cmd, exchangeDetailsCmd()) + c, _ = utils.AddCommand(cmd, exchangeDetailsCmd()) c.Use = c.Use + " " + exchangeServiceCommandDetailsUseSuffix c.Example = exchangeServiceCommandDetailsExamples @@ -141,95 +116,16 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { // Flags addition ordering should follow the order we want them to appear in help and docs: // More generic (ex: --user) and more frequently used flags take precedence. - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to explore. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) - fs.StringSliceVar( - &user, - utils.UserFN, nil, - "Select backup details by user ID; accepts '"+utils.Wildcard+"' to select all users.") - - // email flags - fs.StringSliceVar( - &email, - utils.EmailFN, nil, - "Select backup details for emails by email ID; accepts '"+utils.Wildcard+"' to select all emails.") - fs.StringSliceVar( - &emailFolder, - utils.EmailFolderFN, nil, - "Select backup details for emails within a folder; accepts '"+utils.Wildcard+"' to select all email folders.") - fs.StringVar( - &emailSubject, - utils.EmailSubjectFN, "", - "Select backup details for emails with a subject containing this value.") - fs.StringVar( - &emailSender, - utils.EmailSenderFN, "", - "Select backup details for emails from a specific sender.") - fs.StringVar( - &emailReceivedAfter, - utils.EmailReceivedAfterFN, "", - "Select backup details for emails received after this datetime.") - fs.StringVar( - &emailReceivedBefore, - utils.EmailReceivedBeforeFN, "", - "Select backup details for emails received before this datetime.") - - // event flags - fs.StringSliceVar( - &event, - utils.EventFN, nil, - "Select backup details for events by event ID; accepts '"+utils.Wildcard+"' to select all events.") - fs.StringSliceVar( - &eventCalendar, - utils.EventCalendarFN, nil, - "Select backup details for events under a calendar; accepts '"+utils.Wildcard+"' to select all events.") - fs.StringVar( - &eventSubject, - utils.EventSubjectFN, "", - "Select backup details for events with a subject containing this value.") - fs.StringVar( - &eventOrganizer, - utils.EventOrganizerFN, "", - "Select backup details for events from a specific organizer.") - fs.StringVar( - &eventRecurs, - utils.EventRecursFN, "", - "Select backup details for recurring events. Use `--event-recurs false` to select non-recurring events.") - fs.StringVar( - &eventStartsAfter, - utils.EventStartsAfterFN, "", - "Select backup details for events starting after this datetime.") - fs.StringVar( - &eventStartsBefore, - utils.EventStartsBeforeFN, "", - "Select backup details for events starting before this datetime.") - - // contact flags - fs.StringSliceVar( - &contact, - utils.ContactFN, nil, - "Select backup details for contacts by contact ID; accepts '"+utils.Wildcard+"' to select all contacts.") - fs.StringSliceVar( - &contactFolder, - utils.ContactFolderFN, nil, - "Select backup details for contacts within a folder; accepts '"+utils.Wildcard+"' to select all contact folders.") - fs.StringVar( - &contactName, - utils.ContactNameFN, "", - "Select backup details for contacts whose contact name contains this value.") + utils.AddBackupIDFlag(c, true) + utils.AddExchangeDetailsAndRestoreFlags(c) case deleteCommand: - c, fs = utils.AddCommand(cmd, exchangeDeleteCmd()) + c, _ = utils.AddCommand(cmd, exchangeDeleteCmd()) c.Use = c.Use + " " + exchangeServiceCommandDeleteUseSuffix c.Example = exchangeServiceCommandDeleteExamples - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to delete. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) + utils.AddBackupIDFlag(c, true) } return c @@ -257,7 +153,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { return nil } - if err := validateExchangeBackupCreateFlags(user, exchangeData); err != nil { + if err := validateExchangeBackupCreateFlags(utils.User, exchangeData); err != nil { return err } @@ -268,7 +164,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { defer utils.CloseRepo(ctx, r) - sel := exchangeBackupCreateSelectors(user, exchangeData) + sel := exchangeBackupCreateSelectors(utils.User, exchangeData) // TODO: log/print recoverable errors errs := fault.New(false) @@ -346,7 +242,7 @@ func exchangeListCmd() *cobra.Command { // lists the history of backup operations func listExchangeCmd(cmd *cobra.Command, args []string) error { - return genericListCommand(cmd, backupID, path.ExchangeService, args) + return genericListCommand(cmd, utils.BackupID, path.ExchangeService, args) } // ------------------------------------------------------------------------------------------------ @@ -372,23 +268,26 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.ExchangeOpts{ - Contact: contact, - ContactFolder: contactFolder, - Email: email, - EmailFolder: emailFolder, - Event: event, - EventCalendar: eventCalendar, - Users: user, - ContactName: contactName, - EmailReceivedAfter: emailReceivedAfter, - EmailReceivedBefore: emailReceivedBefore, - EmailSender: emailSender, - EmailSubject: emailSubject, - EventOrganizer: eventOrganizer, - EventRecurs: eventRecurs, - EventStartsAfter: eventStartsAfter, - EventStartsBefore: eventStartsBefore, - EventSubject: eventSubject, + Users: utils.User, + + Contact: utils.Contact, + ContactFolder: utils.ContactFolder, + ContactName: utils.ContactName, + + Email: utils.Email, + EmailFolder: utils.EmailFolder, + EmailReceivedAfter: utils.EmailReceivedAfter, + EmailReceivedBefore: utils.EmailReceivedBefore, + EmailSender: utils.EmailSender, + EmailSubject: utils.EmailSubject, + EventOrganizer: utils.EventOrganizer, + + Event: utils.Event, + EventCalendar: utils.EventCalendar, + EventRecurs: utils.EventRecurs, + EventStartsAfter: utils.EventStartsAfter, + EventStartsBefore: utils.EventStartsBefore, + EventSubject: utils.EventSubject, Populated: utils.GetPopulatedFlags(cmd), } @@ -402,7 +301,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error { ctrlOpts := options.Control() - ds, err := runDetailsExchangeCmd(ctx, r, backupID, opts, ctrlOpts.SkipReduce) + ds, err := runDetailsExchangeCmd(ctx, r, utils.BackupID, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) } @@ -468,5 +367,5 @@ func exchangeDeleteCmd() *cobra.Command { // deletes an exchange service backup. func deleteExchangeCmd(cmd *cobra.Command, args []string) error { - return genericDeleteCommand(cmd, backupID, "Exchange", args) + return genericDeleteCommand(cmd, utils.BackupID, "Exchange", args) } diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index 4a623e22a..490697e54 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -49,11 +49,11 @@ corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd --user a # Explore Alice or Bob's files with name containing "Fiscal 22" in folder "Reports" corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com,bob@example.com --file-name "Fiscal 22" --folder "Reports" + --user alice@example.com,bob@example.com --file-name "Fiscal 22" --folder "Reports" # Explore Alice's files created before end of 2015 from a specific backup corso backup details onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com --file-created-before 2015-01-01T00:00:00` + --user alice@example.com --file-created-before 2015-01-01T00:00:00` ) // called by backup.go to map subcommands to provider-specific handling. @@ -65,83 +65,41 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { switch cmd.Use { case createCommand: - c, fs = utils.AddCommand(cmd, oneDriveCreateCmd()) + c, _ = utils.AddCommand(cmd, oneDriveCreateCmd()) options.AddFeatureToggle(cmd, options.EnablePermissionsBackup()) c.Use = c.Use + " " + oneDriveServiceCommandCreateUseSuffix c.Example = oneDriveServiceCommandCreateExamples - fs.StringSliceVar(&user, - utils.UserFN, nil, - "Backup OneDrive data by user's email address; accepts '"+utils.Wildcard+"' to select all users. (required)") + utils.AddUserFlag(c) options.AddOperationFlags(c) case listCommand: c, fs = utils.AddCommand(cmd, oneDriveListCmd()) + fs.SortFlags = false - fs.StringVar(&backupID, - "backup", "", - "Display a specific backup, including the items that failed or were skipped during processing.") - + utils.AddBackupIDFlag(c, false) addFailedItemsFN(c) addSkippedItemsFN(c) addRecoveredErrorsFN(c) case detailsCommand: - c, fs = utils.AddCommand(cmd, oneDriveDetailsCmd()) + c, _ = utils.AddCommand(cmd, oneDriveDetailsCmd()) c.Use = c.Use + " " + oneDriveServiceCommandDetailsUseSuffix c.Example = oneDriveServiceCommandDetailsExamples options.AddSkipReduceFlag(c) - - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to explore. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) - - // onedrive hierarchy flags - - fs.StringSliceVar( - &utils.FolderPaths, - utils.FolderFN, nil, - "Select backup details by OneDrive folder; defaults to root.") - - fs.StringSliceVar( - &utils.FileNames, - utils.FileFN, nil, - "Select backup details by file name.") - - // onedrive info flags - - fs.StringVar( - &utils.FileCreatedAfter, - utils.FileCreatedAfterFN, "", - "Select backup details for files created after this datetime.") - fs.StringVar( - &utils.FileCreatedBefore, - utils.FileCreatedBeforeFN, "", - "Select backup details for files created before this datetime.") - - fs.StringVar( - &utils.FileModifiedAfter, - utils.FileModifiedAfterFN, "", - "Select backup details for files modified after this datetime.") - fs.StringVar( - &utils.FileModifiedBefore, - utils.FileModifiedBeforeFN, "", - "Select backup details for files modified before this datetime.") + utils.AddBackupIDFlag(c, true) + utils.AddOneDriveDetailsAndRestoreFlags(c) case deleteCommand: - c, fs = utils.AddCommand(cmd, oneDriveDeleteCmd()) + c, _ = utils.AddCommand(cmd, oneDriveDeleteCmd()) c.Use = c.Use + " " + oneDriveServiceCommandDeleteUseSuffix c.Example = oneDriveServiceCommandDeleteExamples - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to delete. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) + utils.AddBackupIDFlag(c, true) } return c @@ -170,7 +128,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error { return nil } - if err := validateOneDriveBackupCreateFlags(user); err != nil { + if err := validateOneDriveBackupCreateFlags(utils.User); err != nil { return err } @@ -181,7 +139,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error { defer utils.CloseRepo(ctx, r) - sel := oneDriveBackupCreateSelectors(user) + sel := oneDriveBackupCreateSelectors(utils.User) // TODO: log/print recoverable errors errs := fault.New(false) @@ -236,7 +194,7 @@ func oneDriveListCmd() *cobra.Command { // lists the history of backup operations func listOneDriveCmd(cmd *cobra.Command, args []string) error { - return genericListCommand(cmd, backupID, path.OneDriveService, args) + return genericListCommand(cmd, utils.BackupID, path.OneDriveService, args) } // ------------------------------------------------------------------------------------------------ @@ -262,9 +220,9 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.OneDriveOpts{ - Users: user, - FileNames: utils.FileNames, - FolderPaths: utils.FolderPaths, + Users: utils.User, + FileNames: utils.FileName, + FolderPaths: utils.FolderPath, FileCreatedAfter: utils.FileCreatedAfter, FileCreatedBefore: utils.FileCreatedBefore, FileModifiedAfter: utils.FileModifiedAfter, @@ -282,7 +240,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error { ctrlOpts := options.Control() - ds, err := runDetailsOneDriveCmd(ctx, r, backupID, opts, ctrlOpts.SkipReduce) + ds, err := runDetailsOneDriveCmd(ctx, r, utils.BackupID, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) } @@ -345,5 +303,5 @@ func oneDriveDeleteCmd() *cobra.Command { // deletes a oneDrive service backup. func deleteOneDriveCmd(cmd *cobra.Command, args []string) error { - return genericDeleteCommand(cmd, backupID, "OneDrive", args) + return genericDeleteCommand(cmd, utils.BackupID, "OneDrive", args) } diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index 44b69c389..19a2f0e81 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -26,12 +26,7 @@ import ( // ------------------------------------------------------------------------------------------------ // sharePoint bucket info from flags -var ( - pageFolders []string - page []string - - sharepointData []string -) +var sharepointData []string const ( dataLibraries = "libraries" @@ -40,32 +35,34 @@ const ( const ( sharePointServiceCommand = "sharepoint" - sharePointServiceCommandCreateUseSuffix = "--web-url | '" + utils.Wildcard + "'" + sharePointServiceCommandCreateUseSuffix = "--site | '" + utils.Wildcard + "'" sharePointServiceCommandDeleteUseSuffix = "--backup " sharePointServiceCommandDetailsUseSuffix = "--backup " ) const ( sharePointServiceCommandCreateExamples = `# Backup SharePoint data for a Site -corso backup create sharepoint --web-url +corso backup create sharepoint --site # Backup SharePoint for two sites: HR and Team corso backup create sharepoint --site https://example.com/hr,https://example.com/team # Backup all SharePoint data for all Sites -corso backup create sharepoint --web-url '*'` +corso backup create sharepoint --site '*'` sharePointServiceCommandDeleteExamples = `# Delete SharePoint backup with ID 1234abcd-12ab-cd34-56de-1234abcd corso backup delete sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd` sharePointServiceCommandDetailsExamples = `# Explore a site's files from backup 1234abcd-12ab-cd34-56de-1234abcd +corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd -corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --web-url https://example.com - -# Find all site files that were created before a certain date. - +# Find all files that were created before a certain date. corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --web-url https://example.com --file-created-before 2015-01-01T00:00:00 + --file-created-before 2015-01-01T00:00:00 --folder "Display Templates/Style Sheets" + +# Find all files within a specific library. +corso backup details sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ + --library documents --folder "Display Templates/Style Sheets" ` ) @@ -83,15 +80,8 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { c.Use = c.Use + " " + sharePointServiceCommandCreateUseSuffix c.Example = sharePointServiceCommandCreateExamples - fs.StringArrayVar( - &utils.Site, - utils.SiteFN, nil, - "Backup SharePoint data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") - - fs.StringSliceVar( - &utils.WebURL, - utils.WebURLFN, nil, - "Restore data by site web URL; accepts '"+utils.Wildcard+"' to select all sites.") + utils.AddSiteFlag(c) + utils.AddSiteIDFlag(c) fs.StringSliceVar( &sharepointData, @@ -103,98 +93,30 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { case listCommand: c, fs = utils.AddCommand(cmd, sharePointListCmd()) + fs.SortFlags = false - fs.StringVar(&backupID, - "backup", "", - "Display a specific backup, including the items that failed or were skipped during processing.") - + utils.AddBackupIDFlag(c, false) addFailedItemsFN(c) addSkippedItemsFN(c) addRecoveredErrorsFN(c) case detailsCommand: - c, fs = utils.AddCommand(cmd, sharePointDetailsCmd()) + c, _ = utils.AddCommand(cmd, sharePointDetailsCmd()) c.Use = c.Use + " " + sharePointServiceCommandDetailsUseSuffix c.Example = sharePointServiceCommandDetailsExamples options.AddSkipReduceFlag(c) - - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to retrieve.") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) - - // sharepoint hierarchy flags - - fs.StringVar( - &utils.Library, - utils.LibraryFN, "", - "Select backup details within a library. Defaults includes all libraries.") - - fs.StringSliceVar( - &utils.FolderPaths, - utils.FolderFN, nil, - "Select backup details by folder; defaults to root.") - - fs.StringSliceVar( - &utils.FileNames, - utils.FileFN, nil, - "Select backup details by file name.") - - fs.StringArrayVar( - &utils.Site, - utils.SiteFN, nil, - "Select backup details by site ID; accepts '"+utils.Wildcard+"' to select all sites.") - - fs.StringSliceVar( - &utils.WebURL, - utils.WebURLFN, nil, - "Select backup data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") - - fs.StringSliceVar( - &pageFolders, - utils.PageFolderFN, nil, - "Select backup data by folder name; accepts '"+utils.Wildcard+"' to select all folders.") - cobra.CheckErr(fs.MarkHidden(utils.PageFolderFN)) - - fs.StringSliceVar( - &page, - utils.PagesFN, nil, - "Select backup data by file name; accepts '"+utils.Wildcard+"' to select all pages within the site.") - cobra.CheckErr(fs.MarkHidden(utils.PagesFN)) - - // sharepoint info flags - - fs.StringVar( - &utils.FileCreatedAfter, - utils.FileCreatedAfterFN, "", - "Select backup details created after this datetime.") - - fs.StringVar( - &utils.FileCreatedBefore, - utils.FileCreatedBeforeFN, "", - "Select backup details created before this datetime.") - - fs.StringVar( - &utils.FileModifiedAfter, - utils.FileModifiedAfterFN, "", - "Select backup details modified after this datetime.") - fs.StringVar( - &utils.FileModifiedBefore, - utils.FileModifiedBeforeFN, "", - "Select backup details modified before this datetime.") + utils.AddBackupIDFlag(c, true) + utils.AddSharePointDetailsAndRestoreFlags(c) case deleteCommand: - c, fs = utils.AddCommand(cmd, sharePointDeleteCmd()) + c, _ = utils.AddCommand(cmd, sharePointDeleteCmd()) c.Use = c.Use + " " + sharePointServiceCommandDeleteUseSuffix c.Example = sharePointServiceCommandDeleteExamples - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to delete. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) + utils.AddBackupIDFlag(c, true) } return c @@ -223,7 +145,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return nil } - if err := validateSharePointBackupCreateFlags(utils.Site, utils.WebURL, sharepointData); err != nil { + if err := validateSharePointBackupCreateFlags(utils.SiteID, utils.WebURL, sharepointData); err != nil { return err } @@ -242,9 +164,9 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { return Only(ctx, errors.Wrap(err, "Failed to connect to Microsoft APIs")) } - sel, err := sharePointBackupCreateSelectors(ctx, utils.Site, utils.WebURL, sharepointData, gc) + sel, err := sharePointBackupCreateSelectors(ctx, utils.SiteID, utils.WebURL, sharepointData, gc) if err != nil { - return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and Web URL")) + return Only(ctx, errors.Wrap(err, "Retrieving up sharepoint sites by ID and URL")) } selectorSet := []selectors.Selector{} @@ -265,8 +187,7 @@ func validateSharePointBackupCreateFlags(sites, weburls, cats []string) error { if len(sites) == 0 && len(weburls) == 0 { return errors.New( "requires one or more --" + - utils.SiteFN + " ids, --" + - utils.WebURLFN + " urls, or the wildcard --" + + utils.SiteFN + " urls, or the wildcard --" + utils.SiteFN + " *", ) } @@ -359,7 +280,7 @@ func sharePointListCmd() *cobra.Command { // lists the history of backup operations func listSharePointCmd(cmd *cobra.Command, args []string) error { - return genericListCommand(cmd, backupID, path.SharePointService, args) + return genericListCommand(cmd, utils.BackupID, path.SharePointService, args) } // ------------------------------------------------------------------------------------------------ @@ -379,7 +300,7 @@ func sharePointDeleteCmd() *cobra.Command { // deletes a sharePoint service backup. func deleteSharePointCmd(cmd *cobra.Command, args []string) error { - return genericDeleteCommand(cmd, backupID, "SharePoint", args) + return genericDeleteCommand(cmd, utils.BackupID, "SharePoint", args) } // ------------------------------------------------------------------------------------------------ @@ -405,11 +326,11 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error { ctx := cmd.Context() opts := utils.SharePointOpts{ - FolderPaths: utils.FolderPaths, - FileNames: utils.FileNames, + FolderPath: utils.FolderPath, + FileName: utils.FileName, Library: utils.Library, - Sites: utils.Site, - WebURLs: utils.WebURL, + SiteID: utils.SiteID, + WebURL: utils.WebURL, FileCreatedAfter: fileCreatedAfter, FileCreatedBefore: fileCreatedBefore, FileModifiedAfter: fileModifiedAfter, @@ -427,7 +348,7 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error { ctrlOpts := options.Control() - ds, err := runDetailsSharePointCmd(ctx, r, backupID, opts, ctrlOpts.SkipReduce) + ds, err := runDetailsSharePointCmd(ctx, r, utils.BackupID, opts, ctrlOpts.SkipReduce) if err != nil { return Only(ctx, err) } diff --git a/src/cli/restore/exchange.go b/src/cli/restore/exchange.go index da295533f..02f1916b5 100644 --- a/src/cli/restore/exchange.go +++ b/src/cli/restore/exchange.go @@ -17,8 +17,7 @@ import ( // exchange bucket info from flags var ( - backupID string - user []string + user []string contact []string contactFolder []string @@ -56,82 +55,10 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command { // Flags addition ordering should follow the order we want them to appear in help and docs: // More generic (ex: --user) and more frequently used flags take precedence. // general flags - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to restore. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) + fs.SortFlags = false - fs.StringSliceVar(&user, - utils.UserFN, nil, - "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") - - // email flags - fs.StringSliceVar(&email, - utils.EmailFN, nil, - "Restore emails by ID; accepts '"+utils.Wildcard+"' to select all emails.") - fs.StringSliceVar( - &emailFolder, - utils.EmailFolderFN, nil, - "Restore emails within a folder; accepts '"+utils.Wildcard+"' to select all email folders.") - fs.StringVar( - &emailSubject, - utils.EmailSubjectFN, "", - "Restore emails with a subject containing this value.") - fs.StringVar( - &emailSender, - utils.EmailSenderFN, "", - "Restore emails from a specific sender.") - fs.StringVar( - &emailReceivedAfter, - utils.EmailReceivedAfterFN, "", - "Restore emails received after this datetime.") - fs.StringVar( - &emailReceivedBefore, - utils.EmailReceivedBeforeFN, "", - "Restore emails received before this datetime.") - - // event flags - fs.StringSliceVar(&event, - utils.EventFN, nil, - "Restore events by event ID; accepts '"+utils.Wildcard+"' to select all events.") - fs.StringSliceVar( - &eventCalendar, - utils.EventCalendarFN, nil, - "Restore events under a calendar; accepts '"+utils.Wildcard+"' to select all event calendars.") - fs.StringVar( - &eventSubject, - utils.EventSubjectFN, "", - "Restore events with a subject containing this value.") - fs.StringVar( - &eventOrganizer, - utils.EventOrganizerFN, "", - "Restore events from a specific organizer.") - fs.StringVar( - &eventRecurs, - utils.EventRecursFN, "", - "Restore recurring events. Use `--event-recurs false` to restore non-recurring events.") - fs.StringVar( - &eventStartsAfter, - utils.EventStartsAfterFN, "", - "Restore events starting after this datetime.") - fs.StringVar( - &eventStartsBefore, - utils.EventStartsBeforeFN, "", - "Restore events starting before this datetime.") - - // contacts flags - fs.StringSliceVar( - &contact, - utils.ContactFN, nil, - "Restore contacts by contact ID; accepts '"+utils.Wildcard+"' to select all contacts.") - fs.StringSliceVar( - &contactFolder, - utils.ContactFolderFN, nil, - "Restore contacts within a folder; accepts '"+utils.Wildcard+"' to select all contact folders.") - fs.StringVar( - &contactName, - utils.ContactNameFN, "", - "Restore contacts whose contact name contains this value.") + utils.AddBackupIDFlag(c, true) + utils.AddExchangeDetailsAndRestoreFlags(c) // others options.AddOperationFlags(c) @@ -149,11 +76,11 @@ corso restore exchange --backup 1234abcd-12ab-cd34-56de-1234abcd --email 98765ab # Restore Alice's emails with subject containing "Hello world" in "Inbox" from a specific backup corso restore exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com --email-subject "Hello world" --email-folder Inbox + --user alice@example.com --email-subject "Hello world" --email-folder Inbox # Restore Bobs's entire calendar from a specific backup corso restore exchange --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user bob@example.com --event-calendar Calendar + --user bob@example.com --event-calendar Calendar # Restore contact with ID abdef0101 from a specific backup corso restore exchange --backup 1234abcd-12ab-cd34-56de-1234abcd --contact abdef0101` @@ -200,7 +127,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error { Populated: utils.GetPopulatedFlags(cmd), } - if err := utils.ValidateExchangeRestoreFlags(backupID, opts); err != nil { + if err := utils.ValidateExchangeRestoreFlags(utils.BackupID, opts); err != nil { return err } @@ -222,7 +149,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error { sel := utils.IncludeExchangeRestoreDataSelectors(opts) utils.FilterExchangeRestoreInfoSelectors(sel, opts) - ro, err := r.NewRestore(ctx, backupID, sel.Selector, dest) + ro, err := r.NewRestore(ctx, utils.BackupID, sel.Selector, dest) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to initialize Exchange restore")) } @@ -230,7 +157,7 @@ func restoreExchangeCmd(cmd *cobra.Command, args []string) error { ds, err := ro.Run(ctx) if err != nil { if errors.Is(err, data.ErrNotFound) { - return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", backupID)) + return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", utils.BackupID)) } return Only(ctx, errors.Wrap(err, "Failed to run Exchange restore")) diff --git a/src/cli/restore/onedrive.go b/src/cli/restore/onedrive.go index 9beeaf3a9..b0aaca1eb 100644 --- a/src/cli/restore/onedrive.go +++ b/src/cli/restore/onedrive.go @@ -15,16 +15,6 @@ import ( "github.com/alcionai/corso/src/pkg/repository" ) -var ( - folderPaths []string - fileNames []string - - fileCreatedAfter string - fileCreatedBefore string - fileModifiedAfter string - fileModifiedBefore string -) - // called by restore.go to map subcommands to provider-specific handling. func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { var ( @@ -42,49 +32,8 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command { // More generic (ex: --user) and more frequently used flags take precedence. fs.SortFlags = false - fs.StringVar(&backupID, - utils.BackupFN, "", - "ID of the backup to restore. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) - - fs.StringSliceVar(&user, - utils.UserFN, nil, - "Restore data by user's email address; accepts '"+utils.Wildcard+"' to select all users.") - - // onedrive hierarchy (path/name) flags - - fs.StringSliceVar( - &folderPaths, - utils.FolderFN, nil, - "Restore items by OneDrive folder; defaults to root") - - fs.StringSliceVar( - &fileNames, - utils.FileFN, nil, - "Restore items by file name or ID") - - // permissions restore flag - options.AddRestorePermissionsFlag(c) - - // onedrive info flags - - fs.StringVar( - &fileCreatedAfter, - utils.FileCreatedAfterFN, "", - "Restore files created after this datetime") - fs.StringVar( - &fileCreatedBefore, - utils.FileCreatedBeforeFN, "", - "Restore files created before this datetime") - - fs.StringVar( - &fileModifiedAfter, - utils.FileModifiedAfterFN, "", - "Restore files modified after this datetime") - fs.StringVar( - &fileModifiedBefore, - utils.FileModifiedBeforeFN, "", - "Restore files modified before this datetime") + utils.AddBackupIDFlag(c, true) + utils.AddOneDriveDetailsAndRestoreFlags(c) // others options.AddOperationFlags(c) @@ -105,11 +54,11 @@ corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abc # Restore Alice's file named "FY2021 Planning.xlsx in "Documents/Finance Reports" from a specific backup corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --user alice@example.com --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" + --user alice@example.com --file "FY2021 Planning.xlsx" --folder "Documents/Finance Reports" # Restore all files from Bob's folder that were created before 2020 when captured in a specific backup corso restore onedrive --backup 1234abcd-12ab-cd34-56de-1234abcd - --user bob@example.com --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` + --user bob@example.com --folder "Documents/Finance Reports" --file-created-before 2020-01-01T00:00:00` ) // `corso restore onedrive [...]` @@ -133,8 +82,8 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { opts := utils.OneDriveOpts{ Users: user, - FileNames: fileNames, - FolderPaths: folderPaths, + FileNames: utils.FileName, + FolderPaths: utils.FolderPath, FileCreatedAfter: utils.FileCreatedAfter, FileCreatedBefore: utils.FileCreatedBefore, FileModifiedAfter: utils.FileModifiedAfter, @@ -143,7 +92,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { Populated: utils.GetPopulatedFlags(cmd), } - if err := utils.ValidateOneDriveRestoreFlags(backupID, opts); err != nil { + if err := utils.ValidateOneDriveRestoreFlags(utils.BackupID, opts); err != nil { return err } @@ -165,7 +114,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { sel := utils.IncludeOneDriveRestoreDataSelectors(opts) utils.FilterOneDriveRestoreInfoSelectors(sel, opts) - ro, err := r.NewRestore(ctx, backupID, sel.Selector, dest) + ro, err := r.NewRestore(ctx, utils.BackupID, sel.Selector, dest) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to initialize OneDrive restore")) } @@ -173,7 +122,7 @@ func restoreOneDriveCmd(cmd *cobra.Command, args []string) error { ds, err := ro.Run(ctx) if err != nil { if errors.Is(err, data.ErrNotFound) { - return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", backupID)) + return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", utils.BackupID)) } return Only(ctx, errors.Wrap(err, "Failed to run OneDrive restore")) diff --git a/src/cli/restore/sharepoint.go b/src/cli/restore/sharepoint.go index 8f002711b..410e87c3f 100644 --- a/src/cli/restore/sharepoint.go +++ b/src/cli/restore/sharepoint.go @@ -39,83 +39,8 @@ func addSharePointCommands(cmd *cobra.Command) *cobra.Command { // More generic (ex: --site) and more frequently used flags take precedence. fs.SortFlags = false - fs.StringVar( - &backupID, - utils.BackupFN, "", - "ID of the backup to restore. (required)") - cobra.CheckErr(c.MarkFlagRequired(utils.BackupFN)) - - fs.StringSliceVar( - &utils.Site, - utils.SiteFN, nil, - "Restore data by site ID; accepts '"+utils.Wildcard+"' to select all sites.") - - fs.StringSliceVar( - &utils.WebURL, - utils.WebURLFN, nil, - "Restore data by site webURL; accepts '"+utils.Wildcard+"' to select all sites.") - - // sharepoint hierarchy (path/name) flags - - fs.StringVar( - &utils.Library, - utils.LibraryFN, "", - "Restore files within a library. Default includes all libraries.") - - fs.StringSliceVar( - &utils.FolderPaths, - utils.FolderFN, nil, - "Restore files by folder; defaults to root.") - - fs.StringSliceVar( - &utils.FileNames, - utils.FileFN, nil, - "Restore files by name.") - - fs.StringSliceVar( - &listPaths, - utils.ListFN, nil, - "Restore list items by SharePoint list ID") - cobra.CheckErr(fs.MarkHidden(utils.ListFN)) - - fs.StringSliceVar( - &listItems, - utils.ListItemFN, nil, - "Restore list items by ID") - cobra.CheckErr(fs.MarkHidden(utils.ListItemFN)) - - fs.StringSliceVar( - &pageFolders, - utils.PageFolderFN, nil, - "Restore Site pages by page folder name") - cobra.CheckErr(fs.MarkHidden(utils.PageFolderFN)) - - fs.StringSliceVar( - &pages, - utils.PagesFN, nil, - "Restore site pages by file name(s)") - cobra.CheckErr(fs.MarkHidden(utils.PagesFN)) - - // sharepoint info flags - - fs.StringVar( - &utils.FileCreatedAfter, - utils.FileCreatedAfterFN, "", - "Restore files created after this datetime.") - - fs.StringVar( - &utils.FileCreatedBefore, - utils.FileCreatedBeforeFN, "", - "Restore files created before this datetime.") - - fs.StringVar( - &utils.FileModifiedAfter, - utils.FileModifiedAfterFN, "", - "Restore files modified after this datetime.") - fs.StringVar( - &utils.FileModifiedBefore, - utils.FileModifiedBeforeFN, "", - "Restore files modified before this datetime.") + utils.AddBackupIDFlag(c, true) + utils.AddSharePointDetailsAndRestoreFlags(c) // others options.AddOperationFlags(c) @@ -132,13 +57,17 @@ const ( sharePointServiceCommandRestoreExamples = `# Restore file with ID 98765abcdef corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd --file 98765abcdef -# Restore a Site's file named "ServerRenderTemplate.xsl in "Display Templates/Style Sheets" from a specific backup +# Restore a file named "ServerRenderTemplate.xsl in "Display Templates/Style Sheets". corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd \ - --web-url https://example.com --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" + --file "ServerRenderTemplate.xsl" --folder "Display Templates/Style Sheets" -# Restore all files from a Site that were created before 2020 when captured in a specific backup +# Restore all files that were created before 2020. corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd - --web-url https://example.com --folder "Display Templates/Style Sheets" --file-created-before 2020-01-01T00:00:00` + --file-created-before 2020-01-01T00:00:00 --folder "Display Templates/Style Sheets" + +# Restore all files in a certain library. +corso restore sharepoint --backup 1234abcd-12ab-cd34-56de-1234abcd + --library documents --folder "Display Templates/Style Sheets" ` ) // `corso restore sharepoint [...]` @@ -161,15 +90,15 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error { } opts := utils.SharePointOpts{ - FileNames: utils.FileNames, - FolderPaths: utils.FolderPaths, + FileName: utils.FileName, + FolderPath: utils.FolderPath, Library: utils.Library, - ListItems: listItems, - ListPaths: listPaths, - PageFolders: pageFolders, - Pages: pages, - Sites: utils.Site, - WebURLs: utils.WebURL, + ListItem: listItems, + ListPath: listPaths, + PageFolder: pageFolders, + Page: pages, + SiteID: utils.SiteID, + WebURL: utils.WebURL, FileCreatedAfter: utils.FileCreatedAfter, FileCreatedBefore: utils.FileCreatedBefore, FileModifiedAfter: utils.FileModifiedAfter, @@ -177,7 +106,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error { Populated: utils.GetPopulatedFlags(cmd), } - if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil { + if err := utils.ValidateSharePointRestoreFlags(utils.BackupID, opts); err != nil { return err } @@ -199,7 +128,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error { sel := utils.IncludeSharePointRestoreDataSelectors(opts) utils.FilterSharePointRestoreInfoSelectors(sel, opts) - ro, err := r.NewRestore(ctx, backupID, sel.Selector, dest) + ro, err := r.NewRestore(ctx, utils.BackupID, sel.Selector, dest) if err != nil { return Only(ctx, errors.Wrap(err, "Failed to initialize SharePoint restore")) } @@ -207,7 +136,7 @@ func restoreSharePointCmd(cmd *cobra.Command, args []string) error { ds, err := ro.Run(ctx) if err != nil { if errors.Is(err, data.ErrNotFound) { - return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", backupID)) + return Only(ctx, errors.Errorf("Backup or backup details missing for id %s", utils.BackupID)) } return Only(ctx, errors.Wrap(err, "Failed to run SharePoint restore")) diff --git a/src/cli/utils/exchange.go b/src/cli/utils/exchange.go index f209a719b..53b845d43 100644 --- a/src/cli/utils/exchange.go +++ b/src/cli/utils/exchange.go @@ -4,26 +4,51 @@ import ( "errors" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/spf13/cobra" ) // flag names const ( - ContactFN = "contact" - ContactFolderFN = "contact-folder" + ContactFN = "contact" + ContactFolderFN = "contact-folder" + ContactNameFN = "contact-name" + EmailFN = "email" EmailFolderFN = "email-folder" - EventFN = "event" - EventCalendarFN = "event-calendar" - ContactNameFN = "contact-name" EmailReceivedAfterFN = "email-received-after" EmailReceivedBeforeFN = "email-received-before" EmailSenderFN = "email-sender" EmailSubjectFN = "email-subject" - EventOrganizerFN = "event-organizer" - EventRecursFN = "event-recurs" - EventStartsAfterFN = "event-starts-after" - EventStartsBeforeFN = "event-starts-before" - EventSubjectFN = "event-subject" + + EventFN = "event" + EventCalendarFN = "event-calendar" + EventOrganizerFN = "event-organizer" + EventRecursFN = "event-recurs" + EventStartsAfterFN = "event-starts-after" + EventStartsBeforeFN = "event-starts-before" + EventSubjectFN = "event-subject" +) + +// flag population values +var ( + Contact []string + ContactFolder []string + ContactName string + + Email []string + EmailFolder []string + EmailReceivedAfter string + EmailReceivedBefore string + EmailSender string + EmailSubject string + + Event []string + EventCalendar []string + EventOrganizer string + EventRecurs string + EventStartsAfter string + EventStartsBefore string + EventSubject string ) type ExchangeOpts struct { @@ -48,6 +73,82 @@ type ExchangeOpts struct { Populated PopulatedFlags } +// AddExchangeDetailsAndRestoreFlags adds flags that are common to both the +// details and restore commands. +func AddExchangeDetailsAndRestoreFlags(cmd *cobra.Command) { + fs := cmd.Flags() + + // email flags + fs.StringSliceVar( + &Email, + EmailFN, nil, + "Select emails by email ID; accepts '"+Wildcard+"' to select all emails.") + fs.StringSliceVar( + &EmailFolder, + EmailFolderFN, nil, + "Select emails within a folder; accepts '"+Wildcard+"' to select all email folders.") + fs.StringVar( + &EmailSubject, + EmailSubjectFN, "", + "Select emails with a subject containing this value.") + fs.StringVar( + &EmailSender, + EmailSenderFN, "", + "Select emails from a specific sender.") + fs.StringVar( + &EmailReceivedAfter, + EmailReceivedAfterFN, "", + "Select emails received after this datetime.") + fs.StringVar( + &EmailReceivedBefore, + EmailReceivedBeforeFN, "", + "Select emails received before this datetime.") + + // event flags + fs.StringSliceVar( + &Event, + EventFN, nil, + "Select events by event ID; accepts '"+Wildcard+"' to select all events.") + fs.StringSliceVar( + &EventCalendar, + EventCalendarFN, nil, + "Select events under a calendar; accepts '"+Wildcard+"' to select all events.") + fs.StringVar( + &EventSubject, + EventSubjectFN, "", + "Select events with a subject containing this value.") + fs.StringVar( + &EventOrganizer, + EventOrganizerFN, "", + "Select events from a specific organizer.") + fs.StringVar( + &EventRecurs, + EventRecursFN, "", + "Select recurring events. Use `--event-recurs false` to select non-recurring events.") + fs.StringVar( + &EventStartsAfter, + EventStartsAfterFN, "", + "Select events starting after this datetime.") + fs.StringVar( + &EventStartsBefore, + EventStartsBeforeFN, "", + "Select events starting before this datetime.") + + // contact flags + fs.StringSliceVar( + &Contact, + ContactFN, nil, + "Select contacts by contact ID; accepts '"+Wildcard+"' to select all contacts.") + fs.StringSliceVar( + &ContactFolder, + ContactFolderFN, nil, + "Select contacts within a folder; accepts '"+Wildcard+"' to select all contact folders.") + fs.StringVar( + &ContactName, + ContactNameFN, "", + "Select contacts whose contact name contains this value.") +} + // AddExchangeInclude adds the scope of the provided values to the selector's // inclusion set. Any unpopulated slice will be replaced with selectors.Any() // to act as a wildcard. diff --git a/src/cli/utils/flags.go b/src/cli/utils/flags.go index 1dbd00968..1fb7ad5cf 100644 --- a/src/cli/utils/flags.go +++ b/src/cli/utils/flags.go @@ -12,8 +12,10 @@ import ( // common flag vars var ( - FolderPaths []string - FileNames []string + BackupID string + + FolderPath []string + FileName []string FileCreatedAfter string FileCreatedBefore string @@ -21,8 +23,10 @@ var ( FileModifiedBefore string Library string - Site []string + SiteID []string WebURL []string + + User []string ) // common flag names @@ -30,9 +34,9 @@ const ( BackupFN = "backup" DataFN = "data" LibraryFN = "library" - SiteFN = "site" + SiteFN = "site" // site only accepts WebURL values + SiteIDFN = "site-id" // site-id accepts actual site ids UserFN = "user" - WebURLFN = "web-url" FileFN = "file" FolderFN = "folder" @@ -43,6 +47,49 @@ const ( FileModifiedBeforeFN = "file-modified-before" ) +// AddBackupIDFlag adds the --backup flag. +func AddBackupIDFlag(cmd *cobra.Command, require bool) { + cmd.Flags().StringVar(&BackupID, BackupFN, "", "ID of the backup to retrieve.") + + if require { + cobra.CheckErr(cmd.MarkFlagRequired(BackupFN)) + } +} + +// AddUserFlag adds the --user flag. +func AddUserFlag(cmd *cobra.Command) { + cmd.Flags().StringSliceVar( + &User, + UserFN, nil, + "Backup a specific user's data; accepts '"+Wildcard+"' to select all users.") + cobra.CheckErr(cmd.MarkFlagRequired(UserFN)) +} + +// AddSiteIDFlag adds the --site-id flag, which accepts site ID values. +// This flag is hidden, since we expect users to prefer the --site url +// and do not want to encourage confusion. +func AddSiteIDFlag(cmd *cobra.Command) { + fs := cmd.Flags() + + // note string ARRAY var. IDs naturally contain commas, so we cannot accept + // duplicate values within a flag declaration. ie: --site-id a,b,c does not + // work. Users must call --site-id a --site-id b --site-id c. + fs.StringArrayVar( + &SiteID, + SiteIDFN, nil, + //nolint:lll + "Backup data by site ID; accepts '"+Wildcard+"' to select all sites. Args cannot be comma-delimited and must use multiple flags.") + cobra.CheckErr(fs.MarkHidden(SiteIDFN)) +} + +// AddSiteFlag adds the --site flag, which accepts webURL values. +func AddSiteFlag(cmd *cobra.Command) { + cmd.Flags().StringSliceVar( + &WebURL, + SiteFN, nil, + "Backup data by site URL; accepts '"+Wildcard+"' to select all sites.") +} + type PopulatedFlags map[string]struct{} func (fs PopulatedFlags) populate(pf *pflag.Flag) { diff --git a/src/cli/utils/onedrive.go b/src/cli/utils/onedrive.go index 04a317ff4..06631c4dc 100644 --- a/src/cli/utils/onedrive.go +++ b/src/cli/utils/onedrive.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/spf13/cobra" ) type OneDriveOpts struct { @@ -18,6 +19,41 @@ type OneDriveOpts struct { Populated PopulatedFlags } +// AddOneDriveDetailsAndRestoreFlags adds flags that are common to both the +// details and restore commands. +func AddOneDriveDetailsAndRestoreFlags(cmd *cobra.Command) { + fs := cmd.Flags() + + fs.StringSliceVar( + &FolderPath, + FolderFN, nil, + "Select files by OneDrive folder; defaults to root.") + + fs.StringSliceVar( + &FileName, + FileFN, nil, + "Select files by name.") + + fs.StringVar( + &FileCreatedAfter, + FileCreatedAfterFN, "", + "Select files created after this datetime.") + fs.StringVar( + &FileCreatedBefore, + FileCreatedBeforeFN, "", + "Select files created before this datetime.") + + fs.StringVar( + &FileModifiedAfter, + FileModifiedAfterFN, "", + "Select files modified after this datetime.") + + fs.StringVar( + &FileModifiedBefore, + FileModifiedBeforeFN, "", + "Select files modified before this datetime.") +} + // ValidateOneDriveRestoreFlags checks common flags for correctness and interdependencies func ValidateOneDriveRestoreFlags(backupID string, opts OneDriveOpts) error { if len(backupID) == 0 { diff --git a/src/cli/utils/sharepoint.go b/src/cli/utils/sharepoint.go index f31cff7e9..8a99537b9 100644 --- a/src/cli/utils/sharepoint.go +++ b/src/cli/utils/sharepoint.go @@ -2,28 +2,38 @@ package utils import ( "errors" - "fmt" "github.com/alcionai/corso/src/pkg/selectors" + "github.com/spf13/cobra" ) const ( ListItemFN = "list-item" ListFN = "list" - PageFolderFN = "page-folders" - PagesFN = "pages" + PageFolderFN = "page-folder" + PagesFN = "page" +) + +// flag population variables +var ( + PageFolder []string + Page []string ) type SharePointOpts struct { - FileNames []string // for libraries, to duplicate onedrive interface - FolderPaths []string // for libraries, to duplicate onedrive interface - Library string - ListItems []string - ListPaths []string - PageFolders []string - Pages []string - Sites []string - WebURLs []string + Library string + FileName []string // for libraries, to duplicate onedrive interface + FolderPath []string // for libraries, to duplicate onedrive interface + + ListItem []string + ListPath []string + + PageFolder []string + Page []string + + SiteID []string + WebURL []string + FileCreatedAfter string FileCreatedBefore string FileModifiedAfter string @@ -32,6 +42,58 @@ type SharePointOpts struct { Populated PopulatedFlags } +// AddSharePointDetailsAndRestoreFlags adds flags that are common to both the +// details and restore commands. +func AddSharePointDetailsAndRestoreFlags(cmd *cobra.Command) { + fs := cmd.Flags() + + fs.StringVar( + &Library, + LibraryFN, "", + "Select only this library; defaults to all libraries.") + + fs.StringSliceVar( + &FolderPath, + FolderFN, nil, + "Select by folder; defaults to root.") + + fs.StringSliceVar( + &FileName, + FileFN, nil, + "Select by file name.") + + fs.StringSliceVar( + &PageFolder, + PageFolderFN, nil, + "Select pages by folder name; accepts '"+Wildcard+"' to select all folders.") + cobra.CheckErr(fs.MarkHidden(PageFolderFN)) + + fs.StringSliceVar( + &Page, + PagesFN, nil, + "Select pages by item name; accepts '"+Wildcard+"' to select all pages within the site.") + cobra.CheckErr(fs.MarkHidden(PagesFN)) + + fs.StringVar( + &FileCreatedAfter, + FileCreatedAfterFN, "", + "Select files created after this datetime.") + + fs.StringVar( + &FileCreatedBefore, + FileCreatedBeforeFN, "", + "Select files created before this datetime.") + + fs.StringVar( + &FileModifiedAfter, + FileModifiedAfterFN, "", + "Select files modified after this datetime.") + fs.StringVar( + &FileModifiedBefore, + FileModifiedBeforeFN, "", + "Select files modified before this datetime.") +} + // ValidateSharePointRestoreFlags checks common flags for correctness and interdependencies func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error { if len(backupID) == 0 { @@ -39,7 +101,6 @@ func ValidateSharePointRestoreFlags(backupID string, opts SharePointOpts) error } if _, ok := opts.Populated[FileCreatedAfterFN]; ok && !IsValidTimeFormat(opts.FileCreatedAfter) { - fmt.Printf("What was I sent: %v\n", opts.FileCreatedAfter) return errors.New("invalid time format for " + FileCreatedAfterFN) } @@ -75,12 +136,12 @@ func AddSharePointInfo( // IncludeSharePointRestoreDataSelectors builds the common data-selector // inclusions for SharePoint commands. func IncludeSharePointRestoreDataSelectors(opts SharePointOpts) *selectors.SharePointRestore { - sites := opts.Sites + sites := opts.SiteID - lfp, lfn := len(opts.FolderPaths), len(opts.FileNames) - ls, lwu := len(opts.Sites), len(opts.WebURLs) - slp, sli := len(opts.ListPaths), len(opts.ListItems) - pf, pi := len(opts.PageFolders), len(opts.Pages) + lfp, lfn := len(opts.FolderPath), len(opts.FileName) + ls, lwu := len(opts.SiteID), len(opts.WebURL) + slp, sli := len(opts.ListPath), len(opts.ListItem) + pf, pi := len(opts.PageFolder), len(opts.Page) if ls == 0 { sites = selectors.Any() @@ -95,58 +156,58 @@ func IncludeSharePointRestoreDataSelectors(opts SharePointOpts) *selectors.Share if lfp+lfn > 0 { if lfn == 0 { - opts.FileNames = selectors.Any() + opts.FileName = selectors.Any() } - opts.FolderPaths = trimFolderSlash(opts.FolderPaths) - containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.FolderPaths) + opts.FolderPath = trimFolderSlash(opts.FolderPath) + containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.FolderPath) if len(containsFolders) > 0 { - sel.Include(sel.LibraryItems(containsFolders, opts.FileNames)) + sel.Include(sel.LibraryItems(containsFolders, opts.FileName)) } if len(prefixFolders) > 0 { - sel.Include(sel.LibraryItems(prefixFolders, opts.FileNames, selectors.PrefixMatch())) + sel.Include(sel.LibraryItems(prefixFolders, opts.FileName, selectors.PrefixMatch())) } } if slp+sli > 0 { if sli == 0 { - opts.ListItems = selectors.Any() + opts.ListItem = selectors.Any() } - opts.ListPaths = trimFolderSlash(opts.ListPaths) - containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListPaths) + opts.ListPath = trimFolderSlash(opts.ListPath) + containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.ListPath) if len(containsFolders) > 0 { - sel.Include(sel.ListItems(containsFolders, opts.ListItems)) + sel.Include(sel.ListItems(containsFolders, opts.ListItem)) } if len(prefixFolders) > 0 { - sel.Include(sel.ListItems(prefixFolders, opts.ListItems, selectors.PrefixMatch())) + sel.Include(sel.ListItems(prefixFolders, opts.ListItem, selectors.PrefixMatch())) } } if pf+pi > 0 { if pi == 0 { - opts.Pages = selectors.Any() + opts.Page = selectors.Any() } - opts.PageFolders = trimFolderSlash(opts.PageFolders) - containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.PageFolders) + opts.PageFolder = trimFolderSlash(opts.PageFolder) + containsFolders, prefixFolders := splitFoldersIntoContainsAndPrefix(opts.PageFolder) if len(containsFolders) > 0 { - sel.Include(sel.PageItems(containsFolders, opts.Pages)) + sel.Include(sel.PageItems(containsFolders, opts.Page)) } if len(prefixFolders) > 0 { - sel.Include(sel.PageItems(prefixFolders, opts.Pages, selectors.PrefixMatch())) + sel.Include(sel.PageItems(prefixFolders, opts.Page, selectors.PrefixMatch())) } } if lwu > 0 { - opts.WebURLs = trimFolderSlash(opts.WebURLs) - containsURLs, suffixURLs := splitFoldersIntoContainsAndPrefix(opts.WebURLs) + opts.WebURL = trimFolderSlash(opts.WebURL) + containsURLs, suffixURLs := splitFoldersIntoContainsAndPrefix(opts.WebURL) if len(containsURLs) > 0 { sel.Include(sel.WebURL(containsURLs)) diff --git a/src/cli/utils/sharepoint_test.go b/src/cli/utils/sharepoint_test.go index 1c467381d..a622fb4e1 100644 --- a/src/cli/utils/sharepoint_test.go +++ b/src/cli/utils/sharepoint_test.go @@ -43,140 +43,140 @@ func (suite *SharePointUtilsSuite) TestIncludeSharePointRestoreDataSelectors() { { name: "single inputs", opts: utils.SharePointOpts{ - FileNames: single, - FolderPaths: single, - Sites: single, - WebURLs: single, + FileName: single, + FolderPath: single, + SiteID: single, + WebURL: single, }, expectIncludeLen: 4, }, { name: "single extended", opts: utils.SharePointOpts{ - FileNames: single, - FolderPaths: single, - ListItems: single, - ListPaths: single, - Sites: single, - WebURLs: single, + FileName: single, + FolderPath: single, + ListItem: single, + ListPath: single, + SiteID: single, + WebURL: single, }, expectIncludeLen: 5, }, { name: "multi inputs", opts: utils.SharePointOpts{ - FileNames: multi, - FolderPaths: multi, - Sites: multi, - WebURLs: multi, + FileName: multi, + FolderPath: multi, + SiteID: multi, + WebURL: multi, }, expectIncludeLen: 4, }, { name: "library folder contains", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: containsOnly, - Sites: empty, - WebURLs: empty, + FileName: empty, + FolderPath: containsOnly, + SiteID: empty, + WebURL: empty, }, expectIncludeLen: 1, }, { name: "library folder prefixes", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: prefixOnly, - Sites: empty, - WebURLs: empty, + FileName: empty, + FolderPath: prefixOnly, + SiteID: empty, + WebURL: empty, }, expectIncludeLen: 1, }, { name: "library folder prefixes and contains", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: containsAndPrefix, - Sites: empty, - WebURLs: empty, + FileName: empty, + FolderPath: containsAndPrefix, + SiteID: empty, + WebURL: empty, }, expectIncludeLen: 2, }, { name: "list contains", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: empty, - ListItems: empty, - ListPaths: containsOnly, - Sites: empty, - WebURLs: empty, + FileName: empty, + FolderPath: empty, + ListItem: empty, + ListPath: containsOnly, + SiteID: empty, + WebURL: empty, }, expectIncludeLen: 1, }, { name: "list prefixes", opts: utils.SharePointOpts{ - ListPaths: prefixOnly, + ListPath: prefixOnly, }, expectIncludeLen: 1, }, { name: "list prefixes and contains", opts: utils.SharePointOpts{ - ListPaths: containsAndPrefix, + ListPath: containsAndPrefix, }, expectIncludeLen: 2, }, { name: "weburl contains", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: empty, - Sites: empty, - WebURLs: containsOnly, + FileName: empty, + FolderPath: empty, + SiteID: empty, + WebURL: containsOnly, }, expectIncludeLen: 3, }, { name: "library folder suffixes", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: empty, - Sites: empty, - WebURLs: prefixOnly, // prefix pattern matches suffix pattern + FileName: empty, + FolderPath: empty, + SiteID: empty, + WebURL: prefixOnly, // prefix pattern matches suffix pattern }, expectIncludeLen: 3, }, { name: "library folder suffixes and contains", opts: utils.SharePointOpts{ - FileNames: empty, - FolderPaths: empty, - Sites: empty, - WebURLs: containsAndPrefix, // prefix pattern matches suffix pattern + FileName: empty, + FolderPath: empty, + SiteID: empty, + WebURL: containsAndPrefix, // prefix pattern matches suffix pattern }, expectIncludeLen: 6, }, { name: "Page Folder", opts: utils.SharePointOpts{ - PageFolders: single, + PageFolder: single, }, expectIncludeLen: 1, }, { name: "Site Page ", opts: utils.SharePointOpts{ - Pages: single, + Page: single, }, expectIncludeLen: 1, }, { name: "Page & library Files", opts: utils.SharePointOpts{ - PageFolders: single, - FileNames: multi, + PageFolder: single, + FileName: multi, }, expectIncludeLen: 2, }, diff --git a/src/cli/utils/testdata/opts.go b/src/cli/utils/testdata/opts.go index d01016402..3d54bb07e 100644 --- a/src/cli/utils/testdata/opts.go +++ b/src/cli/utils/testdata/opts.go @@ -470,28 +470,28 @@ var ( Name: "AllLibraryItems", Expected: testdata.SharePointLibraryItems, Opts: utils.SharePointOpts{ - FolderPaths: selectors.Any(), + FolderPath: selectors.Any(), }, }, { Name: "FolderPrefixMatch", Expected: testdata.SharePointLibraryItems, Opts: utils.SharePointOpts{ - FolderPaths: []string{testdata.SharePointLibraryFolder}, + FolderPath: []string{testdata.SharePointLibraryFolder}, }, }, { Name: "FolderPrefixMatchTrailingSlash", Expected: testdata.SharePointLibraryItems, Opts: utils.SharePointOpts{ - FolderPaths: []string{testdata.SharePointLibraryFolder + "/"}, + FolderPath: []string{testdata.SharePointLibraryFolder + "/"}, }, }, { Name: "FolderPrefixMatchTrailingSlash", Expected: testdata.SharePointLibraryItems, Opts: utils.SharePointOpts{ - FolderPaths: []string{testdata.SharePointLibraryFolder + "/"}, + FolderPath: []string{testdata.SharePointLibraryFolder + "/"}, }, }, { @@ -501,7 +501,7 @@ var ( testdata.SharePointLibraryItems[1], }, Opts: utils.SharePointOpts{ - FileNames: []string{ + FileName: []string{ testdata.SharePointLibraryItems[0].ShortRef, testdata.SharePointLibraryItems[1].ShortRef, }, @@ -511,7 +511,7 @@ var ( Name: "SingleItem", Expected: []details.DetailsEntry{testdata.SharePointLibraryItems[0]}, Opts: utils.SharePointOpts{ - FileNames: []string{ + FileName: []string{ testdata.SharePointLibraryItems[0].SharePoint.ItemName, }, }, @@ -523,7 +523,7 @@ var ( testdata.SharePointLibraryItems[1], }, Opts: utils.SharePointOpts{ - FileNames: []string{ + FileName: []string{ testdata.SharePointLibraryItems[0].SharePoint.ItemName, testdata.SharePointLibraryItems[1].SharePoint.ItemName, }, @@ -533,7 +533,7 @@ var ( Name: "NoSelectRepoItemName", Expected: []details.DetailsEntry{}, Opts: utils.SharePointOpts{ - FileNames: []string{ + FileName: []string{ testdata.SharePointLibraryItemPath1.Item(), }, }, diff --git a/src/internal/connector/data_collections_test.go b/src/internal/connector/data_collections_test.go index 6da05e319..87b28e2ae 100644 --- a/src/internal/connector/data_collections_test.go +++ b/src/internal/connector/data_collections_test.go @@ -255,11 +255,12 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti for _, test := range tests { suite.Run(test.name, func() { t := suite.T() + sel := test.getSelector() collections, excludes, err := sharepoint.DataCollections( ctx, graph.HTTPClient(graph.NoTimeout()), - test.getSelector(), + sel, connector.credentials, connector.Service, connector, diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index 2f726dad3..c1b3f6529 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -41,7 +41,7 @@ type GraphConnector struct { itemClient *http.Client // configured to handle large item downloads tenant string - Sites map[string]string // key value + Sites map[string]string // webURL -> siteID and siteID -> webURL credentials account.M365Config // wg is used to track completion of GC tasks @@ -310,6 +310,7 @@ func getResources( } resources[k] = v + resources[v] = k return true } diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index 33a8ac5c2..8d5ce97e4 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -157,6 +157,7 @@ func NewCollection( case SharePointSource: c.itemGetter = getDriveItem c.itemReader = sharePointItemReader + c.itemMetaReader = sharePointItemMetaReader default: c.itemGetter = getDriveItem c.itemReader = oneDriveItemReader @@ -383,10 +384,15 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { return } + queuedPath := "/" + parentPathString + if oc.source == SharePointSource && len(oc.driveName) > 0 { + queuedPath = "/" + oc.driveName + queuedPath + } + folderProgress, colCloser := observe.ProgressWithCount( ctx, observe.ItemQueueMsg, - observe.PII("/"+parentPathString), + observe.PII(queuedPath), int64(len(oc.driveItems))) defer colCloser() defer close(folderProgress) @@ -420,11 +426,11 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { err error ) - ctx = clues.Add(ctx, + ctx = clues.Add( + ctx, "backup_item_id", itemID, "backup_item_name", itemName, - "backup_item_size", itemSize, - ) + "backup_item_size", itemSize) item.SetParentReference(setName(item.GetParentReference(), oc.driveName)) @@ -442,19 +448,17 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { metaSuffix = DirMetaFileSuffix } - if oc.source == OneDriveSource { - // Fetch metadata for the file - itemMeta, itemMetaSize, err = oc.itemMetaReader( - ctx, - oc.service, - oc.driveID, - item, - oc.ctrl.ToggleFeatures.EnablePermissionsBackup) + // Fetch metadata for the file + itemMeta, itemMetaSize, err = oc.itemMetaReader( + ctx, + oc.service, + oc.driveID, + item, + oc.ctrl.ToggleFeatures.EnablePermissionsBackup) - if err != nil { - el.AddRecoverable(clues.Wrap(err, "getting item metadata").Label(fault.LabelForceNoBackupCreation)) - return - } + if err != nil { + el.AddRecoverable(clues.Wrap(err, "getting item metadata").Label(fault.LabelForceNoBackupCreation)) + return } switch oc.source { @@ -469,10 +473,7 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { ctx = clues.Add(ctx, "backup_item_info", itemInfo) if isFile { - dataSuffix := "" - if oc.source == OneDriveSource { - dataSuffix = DataFileSuffix - } + dataSuffix := DataFileSuffix // Construct a new lazy readCloser to feed to the collection consumer. // This ensures that downloads won't be attempted unless that consumer @@ -503,20 +504,18 @@ func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) { } } - if oc.source == OneDriveSource { - metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) { - progReader, closer := observe.ItemProgress( - ctx, itemMeta, observe.ItemBackupMsg, - observe.PII(metaFileName+metaSuffix), int64(itemMetaSize)) - go closer() - return progReader, nil - }) + metaReader := lazy.NewLazyReadCloser(func() (io.ReadCloser, error) { + progReader, closer := observe.ItemProgress( + ctx, itemMeta, observe.ItemBackupMsg, + observe.PII(metaFileName+metaSuffix), int64(itemMetaSize)) + go closer() + return progReader, nil + }) - oc.data <- &metadataItem{ - id: metaFileName + metaSuffix, - data: metaReader, - modTime: time.Now(), - } + oc.data <- &metadataItem{ + id: metaFileName + metaSuffix, + data: metaReader, + modTime: time.Now(), } // Item read successfully, add to collection diff --git a/src/internal/connector/onedrive/collection_test.go b/src/internal/connector/onedrive/collection_test.go index a45d212b6..1d4fe5dd1 100644 --- a/src/internal/connector/onedrive/collection_test.go +++ b/src/internal/connector/onedrive/collection_test.go @@ -255,11 +255,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { wg.Wait() - if test.source == OneDriveSource { - require.Len(t, readItems, 2) // .data and .meta - } else { - require.Len(t, readItems, 1) - } + require.Len(t, readItems, 2) // .data and .meta // Expect only 1 item require.Equal(t, 1, collStatus.Metrics.Objects) @@ -269,11 +265,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() { readItem := readItems[0] readItemInfo := readItem.(data.StreamInfo) - if test.source == OneDriveSource { - assert.Equal(t, testItemID+DataFileSuffix, readItem.UUID()) - } else { - assert.Equal(t, testItemID, readItem.UUID()) - } + assert.Equal(t, testItemID+DataFileSuffix, readItem.UUID()) require.Implements(t, (*data.StreamModTime)(nil), readItem) mt := readItem.(data.StreamModTime) diff --git a/src/internal/connector/onedrive/collections.go b/src/internal/connector/onedrive/collections.go index e3cc6a8d8..dc39a9469 100644 --- a/src/internal/connector/onedrive/collections.go +++ b/src/internal/connector/onedrive/collections.go @@ -273,9 +273,7 @@ func (c *Collections) Get( return nil, nil, graph.Stack(ctx, err) } - retry := c.source == OneDriveSource - - drives, err := drives(ctx, pager, retry) + drives, err := drives(ctx, pager, true) if err != nil { return nil, nil, err } @@ -729,7 +727,7 @@ func (c *Collections) UpdateCollections( c.CollectionMap[driveID][itemID] = col c.NumContainers++ - if c.source != OneDriveSource || item.GetRoot() != nil { + if item.GetRoot() != nil { continue } diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index 2f17f9e81..59f10c3f8 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -69,9 +69,32 @@ func oneDriveItemMetaReader( item models.DriveItemable, fetchPermissions bool, ) (io.ReadCloser, int, error) { - meta := Metadata{ - FileName: ptr.Val(item.GetName()), - } + return baseItemMetaReader(ctx, service, driveID, item, fetchPermissions) +} + +func sharePointItemMetaReader( + ctx context.Context, + service graph.Servicer, + driveID string, + item models.DriveItemable, + fetchPermissions bool, +) (io.ReadCloser, int, error) { + // TODO: include permissions + return baseItemMetaReader(ctx, service, driveID, item, false) +} + +func baseItemMetaReader( + ctx context.Context, + service graph.Servicer, + driveID string, + item models.DriveItemable, + fetchPermissions bool, +) (io.ReadCloser, int, error) { + var ( + perms []UserPermission + err error + meta = Metadata{FileName: ptr.Val(item.GetName())} + ) if item.GetShared() == nil { meta.SharingMode = SharingModeInherited @@ -79,41 +102,21 @@ func oneDriveItemMetaReader( meta.SharingMode = SharingModeCustom } - var ( - perms []UserPermission - err error - ) - if meta.SharingMode == SharingModeCustom && fetchPermissions { - perms, err = oneDriveItemPermissionInfo(ctx, service, driveID, ptr.Val(item.GetId())) + perms, err = driveItemPermissionInfo(ctx, service, driveID, ptr.Val(item.GetId())) if err != nil { - // Keep this in an if-block because if it's not then we have a weird issue - // of having no value in error but golang thinking it's non nil because of - // the way interfaces work. - err = clues.Wrap(err, "fetching item permissions") - } else { - meta.Permissions = perms - } - } - - metaJSON, serializeErr := json.Marshal(meta) - if serializeErr != nil { - serializeErr = clues.Wrap(serializeErr, "serializing item metadata") - - // Need to check if err was already non-nil since it doesn't filter nil - // values out in calls to Stack(). - if err != nil { - err = clues.Stack(err, serializeErr) - } else { - err = serializeErr + return nil, 0, err } - return nil, 0, err + meta.Permissions = perms } - r := io.NopCloser(bytes.NewReader(metaJSON)) + metaJSON, err := json.Marshal(meta) + if err != nil { + return nil, 0, clues.Wrap(err, "serializing item metadata").WithClues(ctx) + } - return r, len(metaJSON), err + return io.NopCloser(bytes.NewReader(metaJSON)), len(metaJSON), nil } // oneDriveItemReader will return a io.ReadCloser for the specified item @@ -222,9 +225,9 @@ func oneDriveItemInfo(di models.DriveItemable, itemSize int64) *details.OneDrive } } -// OneDriveItemPermissionInfo will fetch the permission information +// driveItemPermissionInfo will fetch the permission information // for a drive item given a drive and item id. -func oneDriveItemPermissionInfo( +func driveItemPermissionInfo( ctx context.Context, service graph.Servicer, driveID string, @@ -237,7 +240,7 @@ func oneDriveItemPermissionInfo( Permissions(). Get(ctx, nil) if err != nil { - return nil, graph.Wrap(ctx, err, "getting item metadata").With("item_id", itemID) + return nil, graph.Wrap(ctx, err, "fetching item permissions").With("item_id", itemID) } uperms := filterUserPermissions(ctx, perm.GetValue()) diff --git a/src/internal/connector/onedrive/permission.go b/src/internal/connector/onedrive/permission.go index 6157b03a0..b618d59b4 100644 --- a/src/internal/connector/onedrive/permission.go +++ b/src/internal/connector/onedrive/permission.go @@ -187,7 +187,7 @@ func restorePermissions( ctx = clues.Add(ctx, "permission_item_id", itemID) // TODO(meain): Compute this from the data that we have instead of fetching from graph - currentPermissions, err := oneDriveItemPermissionInfo(ctx, service, driveID, itemID) + currentPermissions, err := driveItemPermissionInfo(ctx, service, driveID, itemID) if err != nil { return graph.Wrap(ctx, err, "fetching current permissions") } diff --git a/src/internal/connector/onedrive/restore.go b/src/internal/connector/onedrive/restore.go index cf7c8127e..933709f8e 100644 --- a/src/internal/connector/onedrive/restore.go +++ b/src/internal/connector/onedrive/restore.go @@ -217,7 +217,7 @@ func RestoreCollection( continue } - if source == OneDriveSource && backupVersion >= version.OneDrive1DataAndMetaFiles { + if backupVersion >= version.OneDrive1DataAndMetaFiles { name := itemData.UUID() if strings.HasSuffix(name, DataFileSuffix) { diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go index d49713e12..3476fa94c 100644 --- a/src/internal/observe/observe.go +++ b/src/internal/observe/observe.go @@ -167,9 +167,12 @@ func Message(ctx context.Context, msgs ...cleanable) { bar := progress.New( -1, mpb.NopStyle(), - mpb.PrependDecorators( - decor.Name(message, decor.WC{W: len(message) + 1, C: decor.DidentRight}), - ), + mpb.PrependDecorators(decor.Name( + message, + decor.WC{ + W: len(message) + 1, + C: decor.DidentRight, + })), ) // Complete the bar immediately @@ -206,8 +209,7 @@ func MessageWithCompletion( mpb.SpinnerStyle(frames...).PositionLeft(), mpb.PrependDecorators( decor.Name(message+":"), - decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 8}), - ), + decor.Elapsed(decor.ET_STYLE_GO, decor.WC{W: 8})), mpb.BarFillerOnComplete("done"), ) @@ -261,8 +263,7 @@ func ItemProgress( decor.Name(header, decor.WCSyncSpaceR), decor.Name(iname.String(), decor.WCSyncSpaceR), decor.CountersKibiByte(" %.1f/%.1f ", decor.WC{W: 8}), - decor.NewPercentage("%d ", decor.WC{W: 4}), - ), + decor.NewPercentage("%d ", decor.WC{W: 4})), } if !cfg.keepBarsAfterComplete { @@ -308,8 +309,7 @@ func ProgressWithCount( mpb.PrependDecorators( decor.Name(header, decor.WCSyncSpaceR), decor.Name(message.String()), - decor.Counters(0, " %d/%d "), - ), + decor.Counters(0, " %d/%d ")), } if !cfg.keepBarsAfterComplete { diff --git a/src/internal/operations/restore_test.go b/src/internal/operations/restore_test.go index 97572436e..27533fe78 100644 --- a/src/internal/operations/restore_test.go +++ b/src/internal/operations/restore_test.go @@ -136,17 +136,21 @@ func (suite *RestoreOpSuite) TestRestoreOperation_PersistResults() { // integration // --------------------------------------------------------------------------- +type bupResults struct { + backupID model.StableID + items int +} + type RestoreOpIntegrationSuite struct { tester.Suite - backupID model.StableID - sharepointID model.StableID - shareItems int - numItems int - kopiaCloser func(ctx context.Context) - kw *kopia.Wrapper - sw *store.Wrapper - ms *kopia.ModelStore + exchange bupResults + sharepoint bupResults + + kopiaCloser func(ctx context.Context) + kw *kopia.Wrapper + sw *store.Wrapper + ms *kopia.ModelStore } func TestRestoreOpIntegrationSuite(t *testing.T) { @@ -212,10 +216,12 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { require.NoError(t, err, clues.ToCore(err)) require.NotEmpty(t, bo.Results.BackupID) - suite.backupID = bo.Results.BackupID - // Discount metadata files (3 paths, 3 deltas) as - // they are not part of the data restored. - suite.numItems = bo.Results.ItemsWritten - 6 + suite.exchange = bupResults{ + backupID: bo.Results.BackupID, + // Discount metadata files (3 paths, 3 deltas) as + // they are not part of the data restored. + items: bo.Results.ItemsWritten - 6, + } siteID := tester.M365SiteID(t) sites := []string{siteID} @@ -230,16 +236,19 @@ func (suite *RestoreOpIntegrationSuite) SetupSuite() { sw, acct, csel.Selector, - evmock.NewBus(), - ) + evmock.NewBus()) require.NoError(t, err, clues.ToCore(err)) err = bo.Run(ctx) require.NoError(t, err, clues.ToCore(err)) require.NotEmpty(t, bo.Results.BackupID) - suite.sharepointID = bo.Results.BackupID - // Discount MetaData files (1 path, 1 delta) - suite.shareItems = bo.Results.ItemsWritten - 2 + + suite.sharepoint = bupResults{ + backupID: bo.Results.BackupID, + // Discount metadata files (2 paths, 2 deltas) as + // they are not part of the data restored. + items: bo.Results.ItemsWritten - 4, + } } func (suite *RestoreOpIntegrationSuite) TearDownSuite() { @@ -313,8 +322,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { }{ { name: "Exchange_Restore", - bID: suite.backupID, - expectedItems: suite.numItems, + bID: suite.exchange.backupID, + expectedItems: suite.exchange.items, dest: tester.DefaultTestRestoreDestination(), getSelector: func(t *testing.T) selectors.Selector { users := []string{tester.M365UserID(t)} @@ -326,8 +335,8 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { }, { name: "SharePoint_Restore", - bID: suite.sharepointID, - expectedItems: suite.shareItems, + bID: suite.sharepoint.backupID, + expectedItems: suite.sharepoint.items, dest: control.DefaultRestoreDestination(common.SimpleDateTimeOneDrive), getSelector: func(t *testing.T) selectors.Selector { bsel := selectors.NewSharePointRestore([]string{tester.M365SiteID(t)}) @@ -337,13 +346,16 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run() { }, cleanup: func(t *testing.T, dest string) { act := tester.NewM365Account(t) + m365, err := act.M365Config() require.NoError(t, err, clues.ToCore(err)) adpt, err := graph.CreateAdapter(m365.AzureTenantID, m365.AzureClientID, m365.AzureClientSecret) require.NoError(t, err, clues.ToCore(err)) + service := graph.NewService(adpt) pager := api.NewSiteDrivePager(service, tester.M365SiteID(t), []string{"id", "name"}) + driveID, err := pager.GetDriveIDByName(ctx, "Documents") require.NoError(t, err, clues.ToCore(err)) require.NotEmpty(t, driveID) @@ -415,7 +427,7 @@ func (suite *RestoreOpIntegrationSuite) TestRestore_Run_ErrorNoResults() { suite.kw, suite.sw, tester.NewM365Account(t), - suite.backupID, + suite.exchange.backupID, rsel.Selector, dest, mb) diff --git a/src/pkg/backup/details/details.go b/src/pkg/backup/details/details.go index fff392f7f..95577bb20 100644 --- a/src/pkg/backup/details/details.go +++ b/src/pkg/backup/details/details.go @@ -652,7 +652,7 @@ type SharePointInfo struct { // Headers returns the human-readable names of properties in a SharePointInfo // for printing out to a terminal in a columnar display. func (i SharePointInfo) Headers() []string { - return []string{"ItemName", "Library", "ParentPath", "Size", "WebURL", "Created", "Modified"} + return []string{"ItemName", "Library", "ParentPath", "Size", "Owner", "Created", "Modified"} } // Values returns the values matching the Headers list for printing @@ -663,7 +663,7 @@ func (i SharePointInfo) Values() []string { i.DriveName, i.ParentPath, humanize.Bytes(uint64(i.Size)), - i.WebURL, + i.Owner, common.FormatTabularDisplayTime(i.Created), common.FormatTabularDisplayTime(i.Modified), } diff --git a/src/pkg/backup/details/details_test.go b/src/pkg/backup/details/details_test.go index 3c2fd8db6..27ed09de6 100644 --- a/src/pkg/backup/details/details_test.go +++ b/src/pkg/backup/details/details_test.go @@ -119,19 +119,20 @@ func (suite *DetailsUnitSuite) TestDetailsEntry_HeadersValues() { Size: 1000, WebURL: "https://not.a.real/url", DriveName: "aLibrary", + Owner: "user@email.com", Created: now, Modified: now, }, }, }, - expectHs: []string{"ID", "ItemName", "Library", "ParentPath", "Size", "WebURL", "Created", "Modified"}, + expectHs: []string{"ID", "ItemName", "Library", "ParentPath", "Size", "Owner", "Created", "Modified"}, expectVs: []string{ "deadbeef", "itemName", "aLibrary", "parentPath", "1.0 kB", - "https://not.a.real/url", + "user@email.com", nowStr, nowStr, },