From b15f8a6fcde07f984d4ca094fc7e551daa30cfe5 Mon Sep 17 00:00:00 2001 From: Keepers Date: Sat, 30 Sep 2023 10:56:13 -0600 Subject: [PATCH] add generic details command (#4352) centralizes details command processing in the cli --- #### Does this PR need a docs update or release note? - [x] :no_entry: No #### Type of change - [x] :broom: Tech Debt/Cleanup #### Issue(s) * #2025 --- src/cli/backup/backup.go | 63 ++- src/cli/backup/backup_test.go | 68 +++ src/cli/backup/exchange.go | 75 +-- src/cli/backup/exchange_test.go | 52 -- src/cli/backup/groups.go | 73 +-- src/cli/backup/helpers_test.go | 3 +- src/cli/backup/onedrive.go | 75 +-- src/cli/backup/onedrive_test.go | 52 -- src/cli/backup/sharepoint.go | 75 +-- src/cli/backup/sharepoint_test.go | 52 -- src/cli/repo/filesystem.go | 8 +- src/cli/repo/filesystem_e2e_test.go | 3 +- src/cli/repo/s3.go | 6 +- src/cli/repo/s3_e2e_test.go | 3 +- src/cli/restore/exchange_e2e_test.go | 3 +- src/cli/utils/utils.go | 8 +- src/cmd/longevity_test/longevity.go | 6 +- src/internal/m365/controller.go | 39 +- src/pkg/path/service_type.go | 6 +- src/pkg/repository/backups.go | 359 ++++++++++++ src/pkg/repository/data_providers.go | 88 +++ src/pkg/repository/exports.go | 40 ++ .../loadtest/repository_load_test.go | 3 +- src/pkg/repository/repository.go | 515 ++---------------- src/pkg/repository/repository_test.go | 52 +- src/pkg/repository/restores.go | 42 ++ src/pkg/services/m365/api/access.go | 68 +++ src/pkg/services/m365/api/access_test.go | 122 +++++ src/pkg/services/m365/api/client.go | 11 + 29 files changed, 1043 insertions(+), 927 deletions(-) create mode 100644 src/cli/backup/backup_test.go create mode 100644 src/pkg/repository/backups.go create mode 100644 src/pkg/repository/data_providers.go create mode 100644 src/pkg/repository/exports.go create mode 100644 src/pkg/repository/restores.go create mode 100644 src/pkg/services/m365/api/access.go create mode 100644 src/pkg/services/m365/api/access_test.go diff --git a/src/cli/backup/backup.go b/src/cli/backup/backup.go index 8b6808a01..5d885e059 100644 --- a/src/cli/backup/backup.go +++ b/src/cli/backup/backup.go @@ -16,6 +16,8 @@ import ( "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/m365/graph" "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" @@ -163,7 +165,7 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error { // standard set of selector behavior that we want used in the cli var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true} -func runBackups( +func genericCreateCommand( ctx context.Context, r repository.Repositoryer, serviceName string, @@ -332,6 +334,65 @@ func genericListCommand( return nil } +func genericDetailsCommand( + cmd *cobra.Command, + backupID string, + sel selectors.Selector, +) (*details.Details, error) { + ctx := cmd.Context() + + r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService) + if err != nil { + return nil, clues.Stack(err) + } + + defer utils.CloseRepo(ctx, r) + + return genericDetailsCore( + ctx, + r, + backupID, + sel, + rdao.Opts) +} + +func genericDetailsCore( + ctx context.Context, + bg repository.BackupGetter, + backupID string, + sel selectors.Selector, + opts control.Options, +) (*details.Details, error) { + ctx = clues.Add(ctx, "backup_id", backupID) + + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + + d, _, errs := bg.GetBackupDetails(ctx, backupID) + // TODO: log/track recoverable errors + if errs.Failure() != nil { + if errors.Is(errs.Failure(), data.ErrNotFound) { + return nil, clues.New("no backup exists with the id " + backupID) + } + + return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") + } + + if opts.SkipReduce { + return d, nil + } + + d, err := sel.Reduce(ctx, d, errs) + if err != nil { + return nil, clues.Wrap(err, "filtering backup details to selection") + } + + return d, nil +} + +// --------------------------------------------------------------------------- +// helper funcs +// --------------------------------------------------------------------------- + func ifShow(flag string) bool { return strings.ToLower(strings.TrimSpace(flag)) == "show" } diff --git a/src/cli/backup/backup_test.go b/src/cli/backup/backup_test.go new file mode 100644 index 000000000..4d70702ae --- /dev/null +++ b/src/cli/backup/backup_test.go @@ -0,0 +1,68 @@ +package backup + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/cli/utils/testdata" + "github.com/alcionai/corso/src/internal/tester" + dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/path" + "github.com/alcionai/corso/src/pkg/selectors" +) + +type BackupUnitSuite struct { + tester.Suite +} + +func TestBackupUnitSuite(t *testing.T) { + suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)}) +} + +func (suite *BackupUnitSuite) TestGenericDetailsCore() { + t := suite.T() + + expected := append( + append( + dtd.GetItemsForVersion( + t, + path.ExchangeService, + path.EmailCategory, + 0, + -1), + dtd.GetItemsForVersion( + t, + path.ExchangeService, + path.EventsCategory, + 0, + -1)...), + dtd.GetItemsForVersion( + t, + path.ExchangeService, + path.ContactsCategory, + 0, + -1)...) + + ctx, flush := tester.NewContext(t) + defer flush() + + bg := testdata.VersionedBackupGetter{ + Details: dtd.GetDetailsSetForVersion(t, 0), + } + + sel := selectors.NewExchangeBackup([]string{"user-id"}) + sel.Include(sel.AllData()) + + output, err := genericDetailsCore( + ctx, + bg, + "backup-ID", + sel.Selector, + control.DefaultOptions()) + assert.NoError(t, err, clues.ToCore(err)) + assert.ElementsMatch(t, expected, output.Entries) +} diff --git a/src/cli/backup/exchange.go b/src/cli/backup/exchange.go index d4f0d9534..d25cefff0 100644 --- a/src/cli/backup/exchange.go +++ b/src/cli/backup/exchange.go @@ -1,21 +1,15 @@ package backup import ( - "context" - "github.com/alcionai/clues" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/alcionai/corso/src/cli/flags" . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -182,7 +176,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error { selectorSet = append(selectorSet, discSel.Selector) } - return runBackups( + return genericCreateCommand( ctx, r, "Exchange", @@ -272,74 +266,31 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error { return nil } + return runDetailsExchangeCmd(cmd) +} + +func runDetailsExchangeCmd(cmd *cobra.Command) error { ctx := cmd.Context() opts := utils.MakeExchangeOpts(cmd) - r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.ExchangeService) + sel := utils.IncludeExchangeRestoreDataSelectors(opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterExchangeRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) if err != nil { return Only(ctx, err) } - defer utils.CloseRepo(ctx, r) - - ds, err := runDetailsExchangeCmd( - ctx, - r, - flags.BackupIDFV, - opts, - rdao.Opts.SkipReduce) - if err != nil { - return Only(ctx, err) - } - - if len(ds.Entries) == 0 { + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { Info(ctx, selectors.ErrorNoMatchingItems) - return nil } - ds.PrintEntries(ctx) - return nil } -// runDetailsExchangeCmd actually performs the lookup in backup details. -// the fault.Errors return is always non-nil. Callers should check if -// errs.Failure() == nil. -func runDetailsExchangeCmd( - ctx context.Context, - r repository.BackupGetter, - backupID string, - opts utils.ExchangeOpts, - skipReduce bool, -) (*details.Details, error) { - if err := utils.ValidateExchangeRestoreFlags(backupID, opts); err != nil { - return nil, err - } - - ctx = clues.Add(ctx, "backup_id", backupID) - - d, _, errs := r.GetBackupDetails(ctx, backupID) - // TODO: log/track recoverable errors - if errs.Failure() != nil { - if errors.Is(errs.Failure(), data.ErrNotFound) { - return nil, clues.New("No backup exists with the id " + backupID) - } - - return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") - } - - ctx = clues.Add(ctx, "details_entries", len(d.Entries)) - - if !skipReduce { - sel := utils.IncludeExchangeRestoreDataSelectors(opts) - sel.Configure(selectors.Config{OnlyMatchItemNames: true}) - utils.FilterExchangeRestoreInfoSelectors(sel, opts) - d = sel.Reduce(ctx, d, errs) - } - - return d, nil -} - // ------------------------------------------------------------------------------------------------ // backup delete // ------------------------------------------------------------------------------------------------ diff --git a/src/cli/backup/exchange_test.go b/src/cli/backup/exchange_test.go index 87b6f49c8..1ed8f718e 100644 --- a/src/cli/backup/exchange_test.go +++ b/src/cli/backup/exchange_test.go @@ -1,7 +1,6 @@ package backup import ( - "fmt" "strconv" "testing" @@ -15,10 +14,7 @@ import ( flagsTD "github.com/alcionai/corso/src/cli/flags/testdata" cliTD "github.com/alcionai/corso/src/cli/testdata" "github.com/alcionai/corso/src/cli/utils" - utilsTD "github.com/alcionai/corso/src/cli/utils/testdata" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/internal/version" - dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" ) @@ -368,51 +364,3 @@ func (suite *ExchangeUnitSuite) TestExchangeBackupCreateSelectors() { }) } } - -func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectors() { - for v := 0; v <= version.Backup; v++ { - suite.Run(fmt.Sprintf("version%d", v), func() { - for _, test := range utilsTD.ExchangeOptionDetailLookups { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - bg := utilsTD.VersionedBackupGetter{ - Details: dtd.GetDetailsSetForVersion(t, v), - } - - output, err := runDetailsExchangeCmd( - ctx, - bg, - "backup-ID", - test.Opts(t, v), - false) - assert.NoError(t, err, clues.ToCore(err)) - assert.ElementsMatch(t, test.Expected(t, v), output.Entries) - }) - } - }) - } -} - -func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectorsBadFormats() { - for _, test := range utilsTD.BadExchangeOptionsFormats { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - output, err := runDetailsExchangeCmd( - ctx, - test.BackupGetter, - "backup-ID", - test.Opts(t, version.Backup), - false) - assert.Error(t, err, clues.ToCore(err)) - assert.Empty(t, output) - }) - } -} diff --git a/src/cli/backup/groups.go b/src/cli/backup/groups.go index c8be220f3..d834e5f29 100644 --- a/src/cli/backup/groups.go +++ b/src/cli/backup/groups.go @@ -2,7 +2,6 @@ package backup import ( "context" - "errors" "fmt" "github.com/alcionai/clues" @@ -14,12 +13,9 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common/idname" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365" ) @@ -174,7 +170,7 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error { selectorSet = append(selectorSet, discSel.Selector) } - return runBackups( + return genericCreateCommand( ctx, r, "Group", @@ -225,74 +221,31 @@ func detailsGroupsCmd(cmd *cobra.Command, args []string) error { return nil } + return runDetailsGroupsCmd(cmd) +} + +func runDetailsGroupsCmd(cmd *cobra.Command) error { ctx := cmd.Context() opts := utils.MakeGroupsOpts(cmd) - r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.GroupsService) + sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterGroupsRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) if err != nil { return Only(ctx, err) } - defer utils.CloseRepo(ctx, r) - - ds, err := runDetailsGroupsCmd( - ctx, - r, - flags.BackupIDFV, - opts, - rdao.Opts.SkipReduce) - if err != nil { - return Only(ctx, err) - } - - if len(ds.Entries) == 0 { + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { Info(ctx, selectors.ErrorNoMatchingItems) - return nil } - ds.PrintEntries(ctx) - return nil } -// runDetailsGroupsCmd actually performs the lookup in backup details. -// the fault.Errors return is always non-nil. Callers should check if -// errs.Failure() == nil. -func runDetailsGroupsCmd( - ctx context.Context, - r repository.BackupGetter, - backupID string, - opts utils.GroupsOpts, - skipReduce bool, -) (*details.Details, error) { - if err := utils.ValidateGroupsRestoreFlags(backupID, opts); err != nil { - return nil, err - } - - ctx = clues.Add(ctx, "backup_id", backupID) - - d, _, errs := r.GetBackupDetails(ctx, backupID) - // TODO: log/track recoverable errors - if errs.Failure() != nil { - if errors.Is(errs.Failure(), data.ErrNotFound) { - return nil, clues.New("no backup exists with the id " + backupID) - } - - return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") - } - - ctx = clues.Add(ctx, "details_entries", len(d.Entries)) - - if !skipReduce { - sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts) - sel.Configure(selectors.Config{OnlyMatchItemNames: true}) - utils.FilterGroupsRestoreInfoSelectors(sel, opts) - d = sel.Reduce(ctx, d, errs) - } - - return d, nil -} - // ------------------------------------------------------------------------------------------------ // backup delete // ------------------------------------------------------------------------------------------------ diff --git a/src/cli/backup/helpers_test.go b/src/cli/backup/helpers_test.go index 8589d70d0..14486f703 100644 --- a/src/cli/backup/helpers_test.go +++ b/src/cli/backup/helpers_test.go @@ -21,7 +21,6 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" - ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/services/m365/api/mock" @@ -160,7 +159,7 @@ func prepM365Test( repository.NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = repo.Initialize(ctx, ctrlRepo.Retention{}) + err = repo.Initialize(ctx, repository.InitConfig{}) require.NoError(t, err, clues.ToCore(err)) return dependencies{ diff --git a/src/cli/backup/onedrive.go b/src/cli/backup/onedrive.go index fa8170f64..54d479b7c 100644 --- a/src/cli/backup/onedrive.go +++ b/src/cli/backup/onedrive.go @@ -1,21 +1,15 @@ package backup import ( - "context" - "github.com/alcionai/clues" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/alcionai/corso/src/cli/flags" . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -162,7 +156,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error { selectorSet = append(selectorSet, discSel.Selector) } - return runBackups( + return genericCreateCommand( ctx, r, "OneDrive", @@ -229,74 +223,31 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error { return nil } + return runDetailsOneDriveCmd(cmd) +} + +func runDetailsOneDriveCmd(cmd *cobra.Command) error { ctx := cmd.Context() opts := utils.MakeOneDriveOpts(cmd) - r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService) + sel := utils.IncludeOneDriveRestoreDataSelectors(opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterOneDriveRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) if err != nil { return Only(ctx, err) } - defer utils.CloseRepo(ctx, r) - - ds, err := runDetailsOneDriveCmd( - ctx, - r, - flags.BackupIDFV, - opts, - rdao.Opts.SkipReduce) - if err != nil { - return Only(ctx, err) - } - - if len(ds.Entries) == 0 { + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { Info(ctx, selectors.ErrorNoMatchingItems) - return nil } - ds.PrintEntries(ctx) - return nil } -// runDetailsOneDriveCmd actually performs the lookup in backup details. -// the fault.Errors return is always non-nil. Callers should check if -// errs.Failure() == nil. -func runDetailsOneDriveCmd( - ctx context.Context, - r repository.BackupGetter, - backupID string, - opts utils.OneDriveOpts, - skipReduce bool, -) (*details.Details, error) { - if err := utils.ValidateOneDriveRestoreFlags(backupID, opts); err != nil { - return nil, err - } - - ctx = clues.Add(ctx, "backup_id", backupID) - - d, _, errs := r.GetBackupDetails(ctx, backupID) - // TODO: log/track recoverable errors - if errs.Failure() != nil { - if errors.Is(errs.Failure(), data.ErrNotFound) { - return nil, clues.New("no backup exists with the id " + backupID) - } - - return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") - } - - ctx = clues.Add(ctx, "details_entries", len(d.Entries)) - - if !skipReduce { - sel := utils.IncludeOneDriveRestoreDataSelectors(opts) - sel.Configure(selectors.Config{OnlyMatchItemNames: true}) - utils.FilterOneDriveRestoreInfoSelectors(sel, opts) - d = sel.Reduce(ctx, d, errs) - } - - return d, nil -} - // `corso backup delete onedrive [...]` func oneDriveDeleteCmd() *cobra.Command { return &cobra.Command{ diff --git a/src/cli/backup/onedrive_test.go b/src/cli/backup/onedrive_test.go index 6d0e0b202..8c1bb583f 100644 --- a/src/cli/backup/onedrive_test.go +++ b/src/cli/backup/onedrive_test.go @@ -1,7 +1,6 @@ package backup import ( - "fmt" "testing" "github.com/alcionai/clues" @@ -14,10 +13,7 @@ import ( flagsTD "github.com/alcionai/corso/src/cli/flags/testdata" cliTD "github.com/alcionai/corso/src/cli/testdata" "github.com/alcionai/corso/src/cli/utils" - utilsTD "github.com/alcionai/corso/src/cli/utils/testdata" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/internal/version" - dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" ) @@ -227,51 +223,3 @@ func (suite *OneDriveUnitSuite) TestValidateOneDriveBackupCreateFlags() { }) } } - -func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectors() { - for v := 0; v <= version.Backup; v++ { - suite.Run(fmt.Sprintf("version%d", v), func() { - for _, test := range utilsTD.OneDriveOptionDetailLookups { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - bg := utilsTD.VersionedBackupGetter{ - Details: dtd.GetDetailsSetForVersion(t, v), - } - - output, err := runDetailsOneDriveCmd( - ctx, - bg, - "backup-ID", - test.Opts(t, v), - false) - assert.NoError(t, err, clues.ToCore(err)) - assert.ElementsMatch(t, test.Expected(t, v), output.Entries) - }) - } - }) - } -} - -func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectorsBadFormats() { - for _, test := range utilsTD.BadOneDriveOptionsFormats { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - output, err := runDetailsOneDriveCmd( - ctx, - test.BackupGetter, - "backup-ID", - test.Opts(t, version.Backup), - false) - assert.Error(t, err, clues.ToCore(err)) - assert.Empty(t, output) - }) - } -} diff --git a/src/cli/backup/sharepoint.go b/src/cli/backup/sharepoint.go index 507a4a6d2..bfeefaa54 100644 --- a/src/cli/backup/sharepoint.go +++ b/src/cli/backup/sharepoint.go @@ -4,7 +4,6 @@ import ( "context" "github.com/alcionai/clues" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/exp/slices" @@ -13,12 +12,9 @@ import ( . "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/utils" "github.com/alcionai/corso/src/internal/common/idname" - "github.com/alcionai/corso/src/internal/data" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/filters" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365" ) @@ -179,7 +175,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error { selectorSet = append(selectorSet, discSel.Selector) } - return runBackups( + return genericCreateCommand( ctx, r, "SharePoint", @@ -303,7 +299,7 @@ func deleteSharePointCmd(cmd *cobra.Command, args []string) error { // backup details // ------------------------------------------------------------------------------------------------ -// `corso backup details onedrive [...]` +// `corso backup details SharePoint [...]` func sharePointDetailsCmd() *cobra.Command { return &cobra.Command{ Use: sharePointServiceCommand, @@ -324,70 +320,27 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error { return nil } + return runDetailsSharePointCmd(cmd) +} + +func runDetailsSharePointCmd(cmd *cobra.Command) error { ctx := cmd.Context() opts := utils.MakeSharePointOpts(cmd) - r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.SharePointService) + sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) + sel.Configure(selectors.Config{OnlyMatchItemNames: true}) + utils.FilterSharePointRestoreInfoSelectors(sel, opts) + + ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector) if err != nil { return Only(ctx, err) } - defer utils.CloseRepo(ctx, r) - - ds, err := runDetailsSharePointCmd( - ctx, - r, - flags.BackupIDFV, - opts, - rdao.Opts.SkipReduce) - if err != nil { - return Only(ctx, err) - } - - if len(ds.Entries) == 0 { + if len(ds.Entries) > 0 { + ds.PrintEntries(ctx) + } else { Info(ctx, selectors.ErrorNoMatchingItems) - return nil } - ds.PrintEntries(ctx) - return nil } - -// runDetailsSharePointCmd actually performs the lookup in backup details. -// the fault.Errors return is always non-nil. Callers should check if -// errs.Failure() == nil. -func runDetailsSharePointCmd( - ctx context.Context, - r repository.BackupGetter, - backupID string, - opts utils.SharePointOpts, - skipReduce bool, -) (*details.Details, error) { - if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil { - return nil, err - } - - ctx = clues.Add(ctx, "backup_id", backupID) - - d, _, errs := r.GetBackupDetails(ctx, backupID) - // TODO: log/track recoverable errors - if errs.Failure() != nil { - if errors.Is(errs.Failure(), data.ErrNotFound) { - return nil, clues.New("no backup exists with the id " + backupID) - } - - return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository") - } - - ctx = clues.Add(ctx, "details_entries", len(d.Entries)) - - if !skipReduce { - sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts) - sel.Configure(selectors.Config{OnlyMatchItemNames: true}) - utils.FilterSharePointRestoreInfoSelectors(sel, opts) - d = sel.Reduce(ctx, d, errs) - } - - return d, nil -} diff --git a/src/cli/backup/sharepoint_test.go b/src/cli/backup/sharepoint_test.go index f09bbe878..f018a7ba2 100644 --- a/src/cli/backup/sharepoint_test.go +++ b/src/cli/backup/sharepoint_test.go @@ -1,7 +1,6 @@ package backup import ( - "fmt" "strings" "testing" @@ -15,11 +14,8 @@ import ( flagsTD "github.com/alcionai/corso/src/cli/flags/testdata" cliTD "github.com/alcionai/corso/src/cli/testdata" "github.com/alcionai/corso/src/cli/utils" - utilsTD "github.com/alcionai/corso/src/cli/utils/testdata" "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/tester" - "github.com/alcionai/corso/src/internal/version" - dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -339,51 +335,3 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() { }) } } - -func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectors() { - for v := 0; v <= version.Backup; v++ { - suite.Run(fmt.Sprintf("version%d", v), func() { - for _, test := range utilsTD.SharePointOptionDetailLookups { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - bg := utilsTD.VersionedBackupGetter{ - Details: dtd.GetDetailsSetForVersion(t, v), - } - - output, err := runDetailsSharePointCmd( - ctx, - bg, - "backup-ID", - test.Opts(t, v), - false) - assert.NoError(t, err, clues.ToCore(err)) - assert.ElementsMatch(t, test.Expected(t, v), output.Entries) - }) - } - }) - } -} - -func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectorsBadFormats() { - for _, test := range utilsTD.BadSharePointOptionsFormats { - suite.Run(test.Name, func() { - t := suite.T() - - ctx, flush := tester.NewContext(t) - defer flush() - - output, err := runDetailsSharePointCmd( - ctx, - test.BackupGetter, - "backup-ID", - test.Opts(t, version.Backup), - false) - assert.Error(t, err, clues.ToCore(err)) - assert.Empty(t, output) - }) - } -} diff --git a/src/cli/repo/filesystem.go b/src/cli/repo/filesystem.go index 40e8b05a5..f6a495f21 100644 --- a/src/cli/repo/filesystem.go +++ b/src/cli/repo/filesystem.go @@ -85,7 +85,7 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error { opt := utils.ControlWithConfig(cfg) // Retention is not supported for filesystem repos. - retention := ctrlRepo.Retention{} + retentionOpts := ctrlRepo.Retention{} // SendStartCorsoEvent uses distict ID as tenant ID because repoID is still not generated utils.SendStartCorsoEvent( @@ -116,7 +116,9 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller")) } - if err = r.Initialize(ctx, retention); err != nil { + ric := repository.InitConfig{RetentionOpts: retentionOpts} + + if err = r.Initialize(ctx, ric); err != nil { if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil } @@ -207,7 +209,7 @@ func connectFilesystemCmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to create a repository controller")) } - if err := r.Connect(ctx); err != nil { + if err := r.Connect(ctx, repository.ConnConfig{}); err != nil { return Only(ctx, clues.Stack(ErrConnectingRepo, err)) } diff --git a/src/cli/repo/filesystem_e2e_test.go b/src/cli/repo/filesystem_e2e_test.go index d7a28047c..6a76e3fa8 100644 --- a/src/cli/repo/filesystem_e2e_test.go +++ b/src/cli/repo/filesystem_e2e_test.go @@ -16,7 +16,6 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" - ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -138,7 +137,7 @@ func (suite *FilesystemE2ESuite) TestConnectFilesystemCmd() { repository.NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, repository.InitConfig{}) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/repo/s3.go b/src/cli/repo/s3.go index 253be0dfe..3fb0833e6 100644 --- a/src/cli/repo/s3.go +++ b/src/cli/repo/s3.go @@ -138,7 +138,9 @@ func initS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller")) } - if err = r.Initialize(ctx, retentionOpts); err != nil { + ric := repository.InitConfig{RetentionOpts: retentionOpts} + + if err = r.Initialize(ctx, ric); err != nil { if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) { return nil } @@ -221,7 +223,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error { return Only(ctx, clues.Wrap(err, "Failed to create a repository controller")) } - if err := r.Connect(ctx); err != nil { + if err := r.Connect(ctx, repository.ConnConfig{}); err != nil { return Only(ctx, clues.Stack(ErrConnectingRepo, err)) } diff --git a/src/cli/repo/s3_e2e_test.go b/src/cli/repo/s3_e2e_test.go index a9f50e277..e1d65c4f3 100644 --- a/src/cli/repo/s3_e2e_test.go +++ b/src/cli/repo/s3_e2e_test.go @@ -18,7 +18,6 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" - ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/storage" storeTD "github.com/alcionai/corso/src/pkg/storage/testdata" @@ -214,7 +213,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() { repository.NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, repository.InitConfig{}) require.NoError(t, err, clues.ToCore(err)) // then test it diff --git a/src/cli/restore/exchange_e2e_test.go b/src/cli/restore/exchange_e2e_test.go index 36c6b8973..67896831b 100644 --- a/src/cli/restore/exchange_e2e_test.go +++ b/src/cli/restore/exchange_e2e_test.go @@ -20,7 +20,6 @@ import ( "github.com/alcionai/corso/src/internal/tester/tconfig" "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" - ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/repository" "github.com/alcionai/corso/src/pkg/selectors" @@ -92,7 +91,7 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() { repository.NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = suite.repo.Initialize(ctx, ctrlRepo.Retention{}) + err = suite.repo.Initialize(ctx, repository.InitConfig{Service: path.ExchangeService}) require.NoError(t, err, clues.ToCore(err)) suite.backupOps = make(map[path.CategoryType]operations.BackupOperation) diff --git a/src/cli/utils/utils.go b/src/cli/utils/utils.go index 2a4e3de34..2ee9ac090 100644 --- a/src/cli/utils/utils.go +++ b/src/cli/utils/utils.go @@ -78,16 +78,10 @@ func GetAccountAndConnectWithOverrides( return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "creating a repository controller") } - if err := r.Connect(ctx); err != nil { + if err := r.Connect(ctx, repository.ConnConfig{Service: pst}); err != nil { return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository") } - // this initializes our graph api client configurations, - // including control options such as concurency limitations. - if _, err := r.ConnectToM365(ctx, pst); err != nil { - return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "connecting to m365") - } - rdao := RepoDetailsAndOpts{ Repo: cfg, Opts: opts, diff --git a/src/cmd/longevity_test/longevity.go b/src/cmd/longevity_test/longevity.go index ec7862191..c8e9f29cf 100644 --- a/src/cmd/longevity_test/longevity.go +++ b/src/cmd/longevity_test/longevity.go @@ -72,7 +72,7 @@ func deleteBackups( // Only supported for S3 repos currently. func pitrListBackups( ctx context.Context, - service path.ServiceType, + pst path.ServiceType, pitr time.Time, backupIDs []string, ) error { @@ -113,14 +113,14 @@ func pitrListBackups( return clues.Wrap(err, "creating a repo") } - err = r.Connect(ctx) + err = r.Connect(ctx, repository.ConnConfig{Service: pst}) if err != nil { return clues.Wrap(err, "connecting to the repository") } defer r.Close(ctx) - backups, err := r.BackupsByTag(ctx, store.Service(service)) + backups, err := r.BackupsByTag(ctx, store.Service(pst)) if err != nil { return clues.Wrap(err, "listing backups").WithClues(ctx) } diff --git a/src/internal/m365/controller.go b/src/internal/m365/controller.go index 0bd15ee17..3e0b3af93 100644 --- a/src/internal/m365/controller.go +++ b/src/internal/m365/controller.go @@ -79,20 +79,29 @@ func NewController( return nil, clues.Wrap(err, "creating api client").WithClues(ctx) } - rc := resource.UnknownResource + var rCli *resourceClient - switch pst { - case path.ExchangeService, path.OneDriveService: - rc = resource.Users - case path.GroupsService: - rc = resource.Groups - case path.SharePointService: - rc = resource.Sites - } + // no failure for unknown service. + // In that case we create a controller that doesn't attempt to look up any resource + // data. This case helps avoid unnecessary service calls when the end user is running + // repo init and connect commands via the CLI. All other callers should be expected + // to pass in a known service, or else expect downstream failures. + if pst != path.UnknownService { + rc := resource.UnknownResource - rCli, err := getResourceClient(rc, ac) - if err != nil { - return nil, clues.Wrap(err, "creating resource client").WithClues(ctx) + switch pst { + case path.ExchangeService, path.OneDriveService: + rc = resource.Users + case path.GroupsService: + rc = resource.Groups + case path.SharePointService: + rc = resource.Sites + } + + rCli, err = getResourceClient(rc, ac) + if err != nil { + return nil, clues.Wrap(err, "creating resource client").WithClues(ctx) + } } ctrl := Controller{ @@ -110,6 +119,10 @@ func NewController( return &ctrl, nil } +func (ctrl *Controller) VerifyAccess(ctx context.Context) error { + return ctrl.AC.Access().GetToken(ctx) +} + // --------------------------------------------------------------------------- // Processing Status // --------------------------------------------------------------------------- @@ -195,7 +208,7 @@ func getResourceClient(rc resource.Category, ac api.Client) (*resourceClient, er case resource.Groups: return &resourceClient{enum: rc, getter: ac.Groups()}, nil default: - return nil, clues.New("unrecognized owner resource enum").With("resource_enum", rc) + return nil, clues.New("unrecognized owner resource type").With("resource_enum", rc) } } diff --git a/src/pkg/path/service_type.go b/src/pkg/path/service_type.go index a4a99ec6c..14847ce35 100644 --- a/src/pkg/path/service_type.go +++ b/src/pkg/path/service_type.go @@ -15,9 +15,9 @@ var ErrorUnknownService = clues.New("unknown service string") // Metadata services are not considered valid service types for resource paths // though they can be used for metadata paths. // -// The order of the enums below can be changed, but the string representation of -// each enum must remain the same or migration code needs to be added to handle -// changes to the string format. +// The string representaton of each enum _must remain the same_. In case of +// changes to those values, we'll need migration code to handle transitions +// across states else we'll get marshalling/unmarshalling errors. type ServiceType int //go:generate stringer -type=ServiceType -linecomment diff --git a/src/pkg/repository/backups.go b/src/pkg/repository/backups.go new file mode 100644 index 000000000..a4314eb01 --- /dev/null +++ b/src/pkg/repository/backups.go @@ -0,0 +1,359 @@ +package repository + +import ( + "context" + + "github.com/alcionai/clues" + "github.com/kopia/kopia/repo/manifest" + "github.com/pkg/errors" + + "github.com/alcionai/corso/src/internal/common/idname" + "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/kopia" + "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/internal/streamstore" + "github.com/alcionai/corso/src/internal/version" + "github.com/alcionai/corso/src/pkg/backup" + "github.com/alcionai/corso/src/pkg/backup/details" + "github.com/alcionai/corso/src/pkg/fault" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +// BackupGetter deals with retrieving metadata about backups from the +// repository. +type BackupGetter interface { + Backup(ctx context.Context, id string) (*backup.Backup, error) + Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) + BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) + GetBackupDetails( + ctx context.Context, + backupID string, + ) (*details.Details, *backup.Backup, *fault.Bus) + GetBackupErrors( + ctx context.Context, + backupID string, + ) (*fault.Errors, *backup.Backup, *fault.Bus) +} + +type Backuper interface { + NewBackup( + ctx context.Context, + self selectors.Selector, + ) (operations.BackupOperation, error) + NewBackupWithLookup( + ctx context.Context, + self selectors.Selector, + ins idname.Cacher, + ) (operations.BackupOperation, error) + DeleteBackups( + ctx context.Context, + failOnMissing bool, + ids ...string, + ) error +} + +// NewBackup generates a BackupOperation runner. +func (r repository) NewBackup( + ctx context.Context, + sel selectors.Selector, +) (operations.BackupOperation, error) { + return r.NewBackupWithLookup(ctx, sel, nil) +} + +// NewBackupWithLookup generates a BackupOperation runner. +// ownerIDToName and ownerNameToID are optional populations, in case the caller has +// already generated those values. +func (r repository) NewBackupWithLookup( + ctx context.Context, + sel selectors.Selector, + ins idname.Cacher, +) (operations.BackupOperation, error) { + err := r.ConnectDataProvider(ctx, sel.PathService()) + if err != nil { + return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") + } + + ownerID, ownerName, err := r.Provider.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) + if err != nil { + return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") + } + + // TODO: retrieve display name from gc + sel = sel.SetDiscreteOwnerIDName(ownerID, ownerName) + + return operations.NewBackupOperation( + ctx, + r.Opts, + r.dataLayer, + store.NewWrapper(r.modelStore), + r.Provider, + r.Account, + sel, + sel, // the selector acts as an IDNamer for its discrete resource owner. + r.Bus) +} + +// Backup retrieves a backup by id. +func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) { + return getBackup(ctx, id, store.NewWrapper(r.modelStore)) +} + +// getBackup handles the processing for Backup. +func getBackup( + ctx context.Context, + id string, + sw store.BackupGetter, +) (*backup.Backup, error) { + b, err := sw.GetBackup(ctx, model.StableID(id)) + if err != nil { + return nil, errWrapper(err) + } + + return b, nil +} + +// Backups lists backups by ID. Returns as many backups as possible with +// errors for the backups it was unable to retrieve. +func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) { + var ( + bups []*backup.Backup + errs = fault.New(false) + sw = store.NewWrapper(r.modelStore) + ) + + for _, id := range ids { + ictx := clues.Add(ctx, "backup_id", id) + + b, err := sw.GetBackup(ictx, model.StableID(id)) + if err != nil { + errs.AddRecoverable(ctx, errWrapper(err)) + } + + bups = append(bups, b) + } + + return bups, errs +} + +// BackupsByTag lists all backups in a repository that contain all the tags +// specified. +func (r repository) BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) { + sw := store.NewWrapper(r.modelStore) + return backupsByTag(ctx, sw, fs) +} + +// backupsByTag returns all backups matching all provided tags. +// +// TODO(ashmrtn): This exists mostly for testing, but we could restructure the +// code in this file so there's a more elegant mocking solution. +func backupsByTag( + ctx context.Context, + sw store.BackupWrapper, + fs []store.FilterOption, +) ([]*backup.Backup, error) { + bs, err := sw.GetBackups(ctx, fs...) + if err != nil { + return nil, clues.Stack(err) + } + + // Filter out assist backup bases as they're considered incomplete and we + // haven't been displaying them before now. + res := make([]*backup.Backup, 0, len(bs)) + + for _, b := range bs { + if t := b.Tags[model.BackupTypeTag]; t != model.AssistBackup { + res = append(res, b) + } + } + + return res, nil +} + +// BackupDetails returns the specified backup.Details +func (r repository) GetBackupDetails( + ctx context.Context, + backupID string, +) (*details.Details, *backup.Backup, *fault.Bus) { + errs := fault.New(false) + + deets, bup, err := getBackupDetails( + ctx, + backupID, + r.Account.ID(), + r.dataLayer, + store.NewWrapper(r.modelStore), + errs) + + return deets, bup, errs.Fail(err) +} + +// getBackupDetails handles the processing for GetBackupDetails. +func getBackupDetails( + ctx context.Context, + backupID, tenantID string, + kw *kopia.Wrapper, + sw store.BackupGetter, + errs *fault.Bus, +) (*details.Details, *backup.Backup, error) { + b, err := sw.GetBackup(ctx, model.StableID(backupID)) + if err != nil { + return nil, nil, errWrapper(err) + } + + ssid := b.StreamStoreID + if len(ssid) == 0 { + ssid = b.DetailsID + } + + if len(ssid) == 0 { + return nil, b, clues.New("no streamstore id in backup").WithClues(ctx) + } + + var ( + sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService()) + deets details.Details + ) + + err = sstore.Read( + ctx, + ssid, + streamstore.DetailsReader(details.UnmarshalTo(&deets)), + errs) + if err != nil { + return nil, nil, err + } + + // Retroactively fill in isMeta information for items in older + // backup versions without that info + // version.Restore2 introduces the IsMeta flag, so only v1 needs a check. + if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker { + for _, d := range deets.Entries { + if d.OneDrive != nil { + d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef) + } + } + } + + deets.DetailsModel = deets.FilterMetaFiles() + + return &deets, b, nil +} + +// BackupErrors returns the specified backup's fault.Errors +func (r repository) GetBackupErrors( + ctx context.Context, + backupID string, +) (*fault.Errors, *backup.Backup, *fault.Bus) { + errs := fault.New(false) + + fe, bup, err := getBackupErrors( + ctx, + backupID, + r.Account.ID(), + r.dataLayer, + store.NewWrapper(r.modelStore), + errs) + + return fe, bup, errs.Fail(err) +} + +// getBackupErrors handles the processing for GetBackupErrors. +func getBackupErrors( + ctx context.Context, + backupID, tenantID string, + kw *kopia.Wrapper, + sw store.BackupGetter, + errs *fault.Bus, +) (*fault.Errors, *backup.Backup, error) { + b, err := sw.GetBackup(ctx, model.StableID(backupID)) + if err != nil { + return nil, nil, errWrapper(err) + } + + ssid := b.StreamStoreID + if len(ssid) == 0 { + return nil, b, clues.New("missing streamstore id in backup").WithClues(ctx) + } + + var ( + sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService()) + fe fault.Errors + ) + + err = sstore.Read( + ctx, + ssid, + streamstore.FaultErrorsReader(fault.UnmarshalErrorsTo(&fe)), + errs) + if err != nil { + return nil, nil, err + } + + return &fe, b, nil +} + +// DeleteBackups removes the backups from both the model store and the backup +// storage. +// +// If failOnMissing is true then returns an error if a backup model can't be +// found. Otherwise ignores missing backup models. +// +// Missing models or snapshots during the actual deletion do not cause errors. +// +// All backups are delete as an atomic unit so any failures will result in no +// deletions. +func (r repository) DeleteBackups( + ctx context.Context, + failOnMissing bool, + ids ...string, +) error { + return deleteBackups(ctx, store.NewWrapper(r.modelStore), failOnMissing, ids...) +} + +// deleteBackup handles the processing for backup deletion. +func deleteBackups( + ctx context.Context, + sw store.BackupGetterModelDeleter, + failOnMissing bool, + ids ...string, +) error { + // Although we haven't explicitly stated it, snapshots are technically + // manifests in kopia. This means we can use the same delete API to remove + // them and backup models. Deleting all of them together gives us both + // atomicity guarantees (around when data will be flushed) and helps reduce + // the number of manifest blobs that kopia will create. + var toDelete []manifest.ID + + for _, id := range ids { + b, err := sw.GetBackup(ctx, model.StableID(id)) + if err != nil { + if !failOnMissing && errors.Is(err, data.ErrNotFound) { + continue + } + + return clues.Stack(errWrapper(err)). + WithClues(ctx). + With("delete_backup_id", id) + } + + toDelete = append(toDelete, b.ModelStoreID) + + if len(b.SnapshotID) > 0 { + toDelete = append(toDelete, manifest.ID(b.SnapshotID)) + } + + ssid := b.StreamStoreID + if len(ssid) == 0 { + ssid = b.DetailsID + } + + if len(ssid) > 0 { + toDelete = append(toDelete, manifest.ID(ssid)) + } + } + + return sw.DeleteWithModelStoreIDs(ctx, toDelete...) +} diff --git a/src/pkg/repository/data_providers.go b/src/pkg/repository/data_providers.go new file mode 100644 index 000000000..f95f85b56 --- /dev/null +++ b/src/pkg/repository/data_providers.go @@ -0,0 +1,88 @@ +package repository + +import ( + "context" + "fmt" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/m365" + "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/internal/operations/inject" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/path" +) + +type DataProvider interface { + inject.BackupProducer + inject.ExportConsumer + inject.RestoreConsumer + + VerifyAccess(ctx context.Context) error +} + +type DataProviderConnector interface { + // ConnectDataProvider initializes configurations + // and establishes the client connection with the + // data provider for this operation. + ConnectDataProvider( + ctx context.Context, + pst path.ServiceType, + ) error +} + +func (r *repository) ConnectDataProvider( + ctx context.Context, + pst path.ServiceType, +) error { + var ( + provider DataProvider + err error + ) + + switch r.Account.Provider { + case account.ProviderM365: + provider, err = connectToM365(ctx, *r, pst) + default: + err = clues.New("unrecognized provider").WithClues(ctx) + } + + if err != nil { + return clues.Wrap(err, "connecting data provider") + } + + if err := provider.VerifyAccess(ctx); err != nil { + return clues.Wrap(err, fmt.Sprintf("verifying %s account connection", r.Account.Provider)) + } + + r.Provider = provider + + return nil +} + +func connectToM365( + ctx context.Context, + r repository, + pst path.ServiceType, +) (*m365.Controller, error) { + if r.Provider != nil { + ctrl, ok := r.Provider.(*m365.Controller) + if !ok { + // if the provider is initialized to a non-m365 controller, we should not + // attempt to connnect to m365 afterward. + return nil, clues.New("Attempted to connect to multiple data providers") + } + + return ctrl, nil + } + + progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365") + defer close(progressBar) + + ctrl, err := m365.NewController(ctx, r.Account, pst, r.Opts) + if err != nil { + return nil, clues.Wrap(err, "creating m365 client controller") + } + + return ctrl, nil +} diff --git a/src/pkg/repository/exports.go b/src/pkg/repository/exports.go new file mode 100644 index 000000000..2aadd2bfb --- /dev/null +++ b/src/pkg/repository/exports.go @@ -0,0 +1,40 @@ +package repository + +import ( + "context" + + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +type Exporter interface { + NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, + ) (operations.ExportOperation, error) +} + +// NewExport generates a exportOperation runner. +func (r repository) NewExport( + ctx context.Context, + backupID string, + sel selectors.Selector, + exportCfg control.ExportConfig, +) (operations.ExportOperation, error) { + return operations.NewExportOperation( + ctx, + r.Opts, + r.dataLayer, + store.NewWrapper(r.modelStore), + r.Provider, + r.Account, + model.StableID(backupID), + sel, + exportCfg, + r.Bus) +} diff --git a/src/pkg/repository/loadtest/repository_load_test.go b/src/pkg/repository/loadtest/repository_load_test.go index 9cfc38ffc..d65cb21e1 100644 --- a/src/pkg/repository/loadtest/repository_load_test.go +++ b/src/pkg/repository/loadtest/repository_load_test.go @@ -21,7 +21,6 @@ import ( "github.com/alcionai/corso/src/pkg/backup" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" - ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/path" @@ -111,7 +110,7 @@ func initM365Repo(t *testing.T) ( repository.NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, repository.InitConfig{}) require.NoError(t, err, clues.ToCore(err)) return ctx, r, ac, st diff --git a/src/pkg/repository/repository.go b/src/pkg/repository/repository.go index 277eb1bba..539c3c3b1 100644 --- a/src/pkg/repository/repository.go +++ b/src/pkg/repository/repository.go @@ -6,31 +6,20 @@ import ( "github.com/alcionai/clues" "github.com/google/uuid" - "github.com/kopia/kopia/repo/manifest" "github.com/pkg/errors" "github.com/alcionai/corso/src/internal/common/crash" - "github.com/alcionai/corso/src/internal/common/idname" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/events" "github.com/alcionai/corso/src/internal/kopia" - "github.com/alcionai/corso/src/internal/m365" - "github.com/alcionai/corso/src/internal/m365/collection/drive/metadata" "github.com/alcionai/corso/src/internal/model" "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/internal/operations" - "github.com/alcionai/corso/src/internal/streamstore" - "github.com/alcionai/corso/src/internal/version" "github.com/alcionai/corso/src/pkg/account" - "github.com/alcionai/corso/src/pkg/backup" - "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/control" ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" - "github.com/alcionai/corso/src/pkg/count" - "github.com/alcionai/corso/src/pkg/fault" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" - "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/storage" "github.com/alcionai/corso/src/pkg/store" ) @@ -42,48 +31,24 @@ var ( ErrorBackupNotFound = clues.New("no backup exists with that id") ) -// BackupGetter deals with retrieving metadata about backups from the -// repository. -type BackupGetter interface { - Backup(ctx context.Context, id string) (*backup.Backup, error) - Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) - BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) - GetBackupDetails( - ctx context.Context, - backupID string, - ) (*details.Details, *backup.Backup, *fault.Bus) - GetBackupErrors( - ctx context.Context, - backupID string, - ) (*fault.Errors, *backup.Backup, *fault.Bus) -} - type Repositoryer interface { - Initialize(ctx context.Context, retentionOpts ctrlRepo.Retention) error - Connect(ctx context.Context) error + Backuper + BackupGetter + Restorer + Exporter + DataProviderConnector + + Initialize( + ctx context.Context, + cfg InitConfig, + ) error + Connect( + ctx context.Context, + cfg ConnConfig, + ) error GetID() string Close(context.Context) error - NewBackup( - ctx context.Context, - self selectors.Selector, - ) (operations.BackupOperation, error) - NewBackupWithLookup( - ctx context.Context, - self selectors.Selector, - ins idname.Cacher, - ) (operations.BackupOperation, error) - NewRestore( - ctx context.Context, - backupID string, - sel selectors.Selector, - restoreCfg control.RestoreConfig, - ) (operations.RestoreOperation, error) - NewExport( - ctx context.Context, - backupID string, - sel selectors.Selector, - exportCfg control.ExportConfig, - ) (operations.ExportOperation, error) + NewMaintenance( ctx context.Context, mOpts ctrlRepo.Maintenance, @@ -92,14 +57,6 @@ type Repositoryer interface { ctx context.Context, rcOpts ctrlRepo.Retention, ) (operations.RetentionConfigOperation, error) - DeleteBackups(ctx context.Context, failOnMissing bool, ids ...string) error - BackupGetter - // ConnectToM365 establishes graph api connections - // and initializes api client configurations. - ConnectToM365( - ctx context.Context, - pst path.ServiceType, - ) (*m365.Controller, error) } // Repository contains storage provider information. @@ -108,9 +65,10 @@ type repository struct { CreatedAt time.Time Version string // in case of future breaking changes - Account account.Account // the user's m365 account connection details - Storage storage.Storage // the storage provider details and configuration - Opts control.Options + Account account.Account // the user's m365 account connection details + Storage storage.Storage // the storage provider details and configuration + Opts control.Options + Provider DataProvider // the client controller used for external user data CRUD Bus events.Eventer dataLayer *kopia.Wrapper @@ -125,7 +83,7 @@ func (r repository) GetID() string { func New( ctx context.Context, acct account.Account, - s storage.Storage, + st storage.Storage, opts control.Options, configFileRepoID string, ) (repo *repository, err error) { @@ -133,16 +91,16 @@ func New( ctx, "acct_provider", acct.Provider.String(), "acct_id", clues.Hide(acct.ID()), - "storage_provider", s.Provider.String()) + "storage_provider", st.Provider.String()) - bus, err := events.NewBus(ctx, s, acct.ID(), opts) + bus, err := events.NewBus(ctx, st, acct.ID(), opts) if err != nil { return nil, clues.Wrap(err, "constructing event bus").WithClues(ctx) } repoID := configFileRepoID if len(configFileRepoID) == 0 { - repoID = newRepoID(s) + repoID = newRepoID(st) } bus.SetRepoID(repoID) @@ -151,7 +109,7 @@ func New( ID: repoID, Version: "v1", Account: acct, - Storage: s, + Storage: st, Bus: bus, Opts: opts, } @@ -163,17 +121,22 @@ func New( return &r, nil } +type InitConfig struct { + // tells the data provider which service to + // use for its connection pattern. Optional. + Service path.ServiceType + RetentionOpts ctrlRepo.Retention +} + // Initialize will: -// - validate the m365 account & secrets // - connect to the m365 account to ensure communication capability -// - validate the provider config & secrets // - initialize the kopia repo with the provider and retention parameters // - update maintenance retention parameters as needed // - store the configuration details // - connect to the provider func (r *repository) Initialize( ctx context.Context, - retentionOpts ctrlRepo.Retention, + cfg InitConfig, ) (err error) { ctx = clues.Add( ctx, @@ -187,10 +150,14 @@ func (r *repository) Initialize( } }() + if err := r.ConnectDataProvider(ctx, cfg.Service); err != nil { + return clues.Stack(err) + } + observe.Message(ctx, "Initializing repository") kopiaRef := kopia.NewConn(r.Storage) - if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts); err != nil { + if err := kopiaRef.Initialize(ctx, r.Opts.Repo, cfg.RetentionOpts); err != nil { // replace common internal errors so that sdk users can check results with errors.Is() if errors.Is(err, kopia.ErrorRepoAlreadyExists) { return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx) @@ -221,12 +188,21 @@ func (r *repository) Initialize( return nil } +type ConnConfig struct { + // tells the data provider which service to + // use for its connection pattern. Leave empty + // to skip the provider connection. + Service path.ServiceType +} + // Connect will: -// - validate the m365 account details -// - connect to the m365 account to ensure communication capability +// - connect to the m365 account // - connect to the provider storage // - return the connected repository -func (r *repository) Connect(ctx context.Context) (err error) { +func (r *repository) Connect( + ctx context.Context, + cfg ConnConfig, +) (err error) { ctx = clues.Add( ctx, "acct_provider", r.Account.Provider.String(), @@ -239,6 +215,10 @@ func (r *repository) Connect(ctx context.Context) (err error) { } }() + if err := r.ConnectDataProvider(ctx, cfg.Service); err != nil { + return clues.Stack(err) + } + observe.Message(ctx, "Connecting to repository") kopiaRef := kopia.NewConn(r.Storage) @@ -297,98 +277,6 @@ func (r *repository) Close(ctx context.Context) error { return nil } -// NewBackup generates a BackupOperation runner. -func (r repository) NewBackup( - ctx context.Context, - sel selectors.Selector, -) (operations.BackupOperation, error) { - return r.NewBackupWithLookup(ctx, sel, nil) -} - -// NewBackupWithLookup generates a BackupOperation runner. -// ownerIDToName and ownerNameToID are optional populations, in case the caller has -// already generated those values. -func (r repository) NewBackupWithLookup( - ctx context.Context, - sel selectors.Selector, - ins idname.Cacher, -) (operations.BackupOperation, error) { - ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts) - if err != nil { - return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365") - } - - ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins) - if err != nil { - return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details") - } - - // TODO: retrieve display name from gc - sel = sel.SetDiscreteOwnerIDName(ownerID, ownerName) - - return operations.NewBackupOperation( - ctx, - r.Opts, - r.dataLayer, - store.NewWrapper(r.modelStore), - ctrl, - r.Account, - sel, - sel, // the selector acts as an IDNamer for its discrete resource owner. - r.Bus) -} - -// NewExport generates a exportOperation runner. -func (r repository) NewExport( - ctx context.Context, - backupID string, - sel selectors.Selector, - exportCfg control.ExportConfig, -) (operations.ExportOperation, error) { - ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts) - if err != nil { - return operations.ExportOperation{}, clues.Wrap(err, "connecting to m365") - } - - return operations.NewExportOperation( - ctx, - r.Opts, - r.dataLayer, - store.NewWrapper(r.modelStore), - ctrl, - r.Account, - model.StableID(backupID), - sel, - exportCfg, - r.Bus) -} - -// NewRestore generates a restoreOperation runner. -func (r repository) NewRestore( - ctx context.Context, - backupID string, - sel selectors.Selector, - restoreCfg control.RestoreConfig, -) (operations.RestoreOperation, error) { - ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts) - if err != nil { - return operations.RestoreOperation{}, clues.Wrap(err, "connecting to m365") - } - - return operations.NewRestoreOperation( - ctx, - r.Opts, - r.dataLayer, - store.NewWrapper(r.modelStore), - ctrl, - r.Account, - model.StableID(backupID), - sel, - restoreCfg, - r.Bus, - count.New()) -} - func (r repository) NewMaintenance( ctx context.Context, mOpts ctrlRepo.Maintenance, @@ -414,280 +302,6 @@ func (r repository) NewRetentionConfig( r.Bus) } -// Backup retrieves a backup by id. -func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) { - return getBackup(ctx, id, store.NewWrapper(r.modelStore)) -} - -// getBackup handles the processing for Backup. -func getBackup( - ctx context.Context, - id string, - sw store.BackupGetter, -) (*backup.Backup, error) { - b, err := sw.GetBackup(ctx, model.StableID(id)) - if err != nil { - return nil, errWrapper(err) - } - - return b, nil -} - -// Backups lists backups by ID. Returns as many backups as possible with -// errors for the backups it was unable to retrieve. -func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) { - var ( - bups []*backup.Backup - errs = fault.New(false) - sw = store.NewWrapper(r.modelStore) - ) - - for _, id := range ids { - ictx := clues.Add(ctx, "backup_id", id) - - b, err := sw.GetBackup(ictx, model.StableID(id)) - if err != nil { - errs.AddRecoverable(ctx, errWrapper(err)) - } - - bups = append(bups, b) - } - - return bups, errs -} - -// BackupsByTag lists all backups in a repository that contain all the tags -// specified. -func (r repository) BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) { - sw := store.NewWrapper(r.modelStore) - return backupsByTag(ctx, sw, fs) -} - -// backupsByTag returns all backups matching all provided tags. -// -// TODO(ashmrtn): This exists mostly for testing, but we could restructure the -// code in this file so there's a more elegant mocking solution. -func backupsByTag( - ctx context.Context, - sw store.BackupWrapper, - fs []store.FilterOption, -) ([]*backup.Backup, error) { - bs, err := sw.GetBackups(ctx, fs...) - if err != nil { - return nil, clues.Stack(err) - } - - // Filter out assist backup bases as they're considered incomplete and we - // haven't been displaying them before now. - res := make([]*backup.Backup, 0, len(bs)) - - for _, b := range bs { - if t := b.Tags[model.BackupTypeTag]; t != model.AssistBackup { - res = append(res, b) - } - } - - return res, nil -} - -// BackupDetails returns the specified backup.Details -func (r repository) GetBackupDetails( - ctx context.Context, - backupID string, -) (*details.Details, *backup.Backup, *fault.Bus) { - errs := fault.New(false) - - deets, bup, err := getBackupDetails( - ctx, - backupID, - r.Account.ID(), - r.dataLayer, - store.NewWrapper(r.modelStore), - errs) - - return deets, bup, errs.Fail(err) -} - -// getBackupDetails handles the processing for GetBackupDetails. -func getBackupDetails( - ctx context.Context, - backupID, tenantID string, - kw *kopia.Wrapper, - sw store.BackupGetter, - errs *fault.Bus, -) (*details.Details, *backup.Backup, error) { - b, err := sw.GetBackup(ctx, model.StableID(backupID)) - if err != nil { - return nil, nil, errWrapper(err) - } - - ssid := b.StreamStoreID - if len(ssid) == 0 { - ssid = b.DetailsID - } - - if len(ssid) == 0 { - return nil, b, clues.New("no streamstore id in backup").WithClues(ctx) - } - - var ( - sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService()) - deets details.Details - ) - - err = sstore.Read( - ctx, - ssid, - streamstore.DetailsReader(details.UnmarshalTo(&deets)), - errs) - if err != nil { - return nil, nil, err - } - - // Retroactively fill in isMeta information for items in older - // backup versions without that info - // version.Restore2 introduces the IsMeta flag, so only v1 needs a check. - if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker { - for _, d := range deets.Entries { - if d.OneDrive != nil { - d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef) - } - } - } - - deets.DetailsModel = deets.FilterMetaFiles() - - return &deets, b, nil -} - -// BackupErrors returns the specified backup's fault.Errors -func (r repository) GetBackupErrors( - ctx context.Context, - backupID string, -) (*fault.Errors, *backup.Backup, *fault.Bus) { - errs := fault.New(false) - - fe, bup, err := getBackupErrors( - ctx, - backupID, - r.Account.ID(), - r.dataLayer, - store.NewWrapper(r.modelStore), - errs) - - return fe, bup, errs.Fail(err) -} - -// getBackupErrors handles the processing for GetBackupErrors. -func getBackupErrors( - ctx context.Context, - backupID, tenantID string, - kw *kopia.Wrapper, - sw store.BackupGetter, - errs *fault.Bus, -) (*fault.Errors, *backup.Backup, error) { - b, err := sw.GetBackup(ctx, model.StableID(backupID)) - if err != nil { - return nil, nil, errWrapper(err) - } - - ssid := b.StreamStoreID - if len(ssid) == 0 { - return nil, b, clues.New("missing streamstore id in backup").WithClues(ctx) - } - - var ( - sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService()) - fe fault.Errors - ) - - err = sstore.Read( - ctx, - ssid, - streamstore.FaultErrorsReader(fault.UnmarshalErrorsTo(&fe)), - errs) - if err != nil { - return nil, nil, err - } - - return &fe, b, nil -} - -// DeleteBackups removes the backups from both the model store and the backup -// storage. -// -// If failOnMissing is true then returns an error if a backup model can't be -// found. Otherwise ignores missing backup models. -// -// Missing models or snapshots during the actual deletion do not cause errors. -// -// All backups are delete as an atomic unit so any failures will result in no -// deletions. -func (r repository) DeleteBackups( - ctx context.Context, - failOnMissing bool, - ids ...string, -) error { - return deleteBackups(ctx, store.NewWrapper(r.modelStore), failOnMissing, ids...) -} - -// deleteBackup handles the processing for backup deletion. -func deleteBackups( - ctx context.Context, - sw store.BackupGetterModelDeleter, - failOnMissing bool, - ids ...string, -) error { - // Although we haven't explicitly stated it, snapshots are technically - // manifests in kopia. This means we can use the same delete API to remove - // them and backup models. Deleting all of them together gives us both - // atomicity guarantees (around when data will be flushed) and helps reduce - // the number of manifest blobs that kopia will create. - var toDelete []manifest.ID - - for _, id := range ids { - b, err := sw.GetBackup(ctx, model.StableID(id)) - if err != nil { - if !failOnMissing && errors.Is(err, data.ErrNotFound) { - continue - } - - return clues.Stack(errWrapper(err)). - WithClues(ctx). - With("delete_backup_id", id) - } - - toDelete = append(toDelete, b.ModelStoreID) - - if len(b.SnapshotID) > 0 { - toDelete = append(toDelete, manifest.ID(b.SnapshotID)) - } - - ssid := b.StreamStoreID - if len(ssid) == 0 { - ssid = b.DetailsID - } - - if len(ssid) > 0 { - toDelete = append(toDelete, manifest.ID(ssid)) - } - } - - return sw.DeleteWithModelStoreIDs(ctx, toDelete...) -} - -func (r repository) ConnectToM365( - ctx context.Context, - pst path.ServiceType, -) (*m365.Controller, error) { - ctrl, err := connectToM365(ctx, pst, r.Account, r.Opts) - if err != nil { - return nil, clues.Wrap(err, "connecting to m365") - } - - return ctrl, nil -} - // --------------------------------------------------------------------------- // Repository ID Model // --------------------------------------------------------------------------- @@ -736,29 +350,6 @@ func newRepoID(s storage.Storage) string { // helpers // --------------------------------------------------------------------------- -var m365nonce bool - -func connectToM365( - ctx context.Context, - pst path.ServiceType, - acct account.Account, - co control.Options, -) (*m365.Controller, error) { - if !m365nonce { - m365nonce = true - - progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365") - defer close(progressBar) - } - - ctrl, err := m365.NewController(ctx, acct, pst, co) - if err != nil { - return nil, err - } - - return ctrl, nil -} - func errWrapper(err error) error { if errors.Is(err, data.ErrNotFound) { return clues.Stack(ErrorBackupNotFound, err) diff --git a/src/pkg/repository/repository_test.go b/src/pkg/repository/repository_test.go index c276f35f5..97456fe70 100644 --- a/src/pkg/repository/repository_test.go +++ b/src/pkg/repository/repository_test.go @@ -17,6 +17,7 @@ import ( ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository" "github.com/alcionai/corso/src/pkg/control/testdata" "github.com/alcionai/corso/src/pkg/extensions" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" "github.com/alcionai/corso/src/pkg/services/m365/api" "github.com/alcionai/corso/src/pkg/storage" @@ -69,7 +70,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -85,12 +86,12 @@ func (suite *RepositoryUnitSuite) TestConnect() { errCheck assert.ErrorAssertionFunc }{ { - storage.ProviderUnknown.String(), - func() (storage.Storage, error) { + name: storage.ProviderUnknown.String(), + storage: func() (storage.Storage, error) { return storage.NewStorage(storage.ProviderUnknown) }, - account.Account{}, - assert.Error, + account: account.Account{}, + errCheck: assert.Error, }, } for _, test := range table { @@ -111,7 +112,7 @@ func (suite *RepositoryUnitSuite) TestConnect() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Connect(ctx) + err = r.Connect(ctx, ConnConfig{}) test.errCheck(t, err, clues.ToCore(err)) }) } @@ -136,12 +137,13 @@ func TestRepositoryIntegrationSuite(t *testing.T) { func (suite *RepositoryIntegrationSuite) TestInitialize() { table := []struct { name string - account account.Account + account func(*testing.T) account.Account storage func(tester.TestT) storage.Storage errCheck assert.ErrorAssertionFunc }{ { name: "success", + account: tconfig.NewM365Account, storage: storeTD.NewPrefixedS3Storage, errCheck: assert.NoError, }, @@ -156,13 +158,13 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() { st := test.storage(t) r, err := New( ctx, - test.account, + test.account(t), st, control.DefaultOptions(), NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) if err == nil { defer func() { err := r.Close(ctx) @@ -204,7 +206,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err) defer func() { @@ -218,21 +220,23 @@ func (suite *RepositoryIntegrationSuite) TestConnect() { ctx, flush := tester.NewContext(t) defer flush() + acct := tconfig.NewM365Account(t) + // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) r, err := New( ctx, - account.Account{}, + acct, st, control.DefaultOptions(), NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err, clues.ToCore(err)) // now re-connect - err = r.Connect(ctx) + err = r.Connect(ctx, ConnConfig{}) assert.NoError(t, err, clues.ToCore(err)) } @@ -242,17 +246,19 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { ctx, flush := tester.NewContext(t) defer flush() + acct := tconfig.NewM365Account(t) + // need to initialize the repository before we can test connecting to it. st := storeTD.NewPrefixedS3Storage(t) r, err := New( ctx, - account.Account{}, + acct, st, control.DefaultOptions(), NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err, clues.ToCore(err)) oldID := r.GetID() @@ -261,7 +267,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() { require.NoError(t, err, clues.ToCore(err)) // now re-connect - err = r.Connect(ctx) + err = r.Connect(ctx, ConnConfig{}) require.NoError(t, err, clues.ToCore(err)) assert.Equal(t, oldID, r.GetID()) } @@ -284,7 +290,8 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + // service doesn't matter here, we just need a valid value. + err = r.Initialize(ctx, InitConfig{Service: path.ExchangeService}) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -313,7 +320,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() { "") require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err, clues.ToCore(err)) ro, err := r.NewRestore( @@ -343,7 +350,8 @@ func (suite *RepositoryIntegrationSuite) TestNewBackupAndDelete() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + // service doesn't matter here, we just need a valid value. + err = r.Initialize(ctx, InitConfig{Service: path.ExchangeService}) require.NoError(t, err, clues.ToCore(err)) userID := tconfig.M365UserID(t) @@ -396,7 +404,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err, clues.ToCore(err)) mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{}) @@ -465,11 +473,11 @@ func (suite *RepositoryIntegrationSuite) Test_Options() { NewRepoID) require.NoError(t, err, clues.ToCore(err)) - err = r.Initialize(ctx, ctrlRepo.Retention{}) + err = r.Initialize(ctx, InitConfig{}) require.NoError(t, err) assert.Equal(t, test.expectedLen, len(r.Opts.ItemExtensionFactory)) - err = r.Connect(ctx) + err = r.Connect(ctx, ConnConfig{}) assert.NoError(t, err) assert.Equal(t, test.expectedLen, len(r.Opts.ItemExtensionFactory)) }) diff --git a/src/pkg/repository/restores.go b/src/pkg/repository/restores.go new file mode 100644 index 000000000..6fe121e76 --- /dev/null +++ b/src/pkg/repository/restores.go @@ -0,0 +1,42 @@ +package repository + +import ( + "context" + + "github.com/alcionai/corso/src/internal/model" + "github.com/alcionai/corso/src/internal/operations" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/count" + "github.com/alcionai/corso/src/pkg/selectors" + "github.com/alcionai/corso/src/pkg/store" +) + +type Restorer interface { + NewRestore( + ctx context.Context, + backupID string, + sel selectors.Selector, + restoreCfg control.RestoreConfig, + ) (operations.RestoreOperation, error) +} + +// NewRestore generates a restoreOperation runner. +func (r repository) NewRestore( + ctx context.Context, + backupID string, + sel selectors.Selector, + restoreCfg control.RestoreConfig, +) (operations.RestoreOperation, error) { + return operations.NewRestoreOperation( + ctx, + r.Opts, + r.dataLayer, + store.NewWrapper(r.modelStore), + r.Provider, + r.Account, + model.StableID(backupID), + sel, + restoreCfg, + r.Bus, + count.New()) +} diff --git a/src/pkg/services/m365/api/access.go b/src/pkg/services/m365/api/access.go new file mode 100644 index 000000000..956f9db05 --- /dev/null +++ b/src/pkg/services/m365/api/access.go @@ -0,0 +1,68 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/alcionai/clues" + + "github.com/alcionai/corso/src/internal/m365/graph" +) + +// --------------------------------------------------------------------------- +// controller +// --------------------------------------------------------------------------- + +func (c Client) Access() Access { + return Access{c} +} + +// Access is an interface-compliant provider of the client. +type Access struct { + Client +} + +// GetToken retrieves a m365 application auth token using client id and secret credentials. +// This token is not normally needed in order for corso to function, and is implemented +// primarily as a way to exercise the validity of those credentials without need of specific +// permissions. +func (c Access) GetToken( + ctx context.Context, +) error { + var ( + //nolint:lll + // https://learn.microsoft.com/en-us/graph/connecting-external-content-connectors-api-postman#step-5-get-an-authentication-token + rawURL = fmt.Sprintf( + "https://login.microsoftonline.com/%s/oauth2/v2.0/token", + c.Credentials.AzureTenantID) + headers = map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + } + body = strings.NewReader(fmt.Sprintf( + "client_id=%s"+ + "&client_secret=%s"+ + "&scope=https://graph.microsoft.com/.default"+ + "&grant_type=client_credentials", + c.Credentials.AzureClientID, + c.Credentials.AzureClientSecret)) + ) + + resp, err := c.Post(ctx, rawURL, headers, body) + if err != nil { + return graph.Stack(ctx, err) + } + + if resp.StatusCode == http.StatusBadRequest { + return clues.New("incorrect tenant or application parameters") + } + + if resp.StatusCode/100 == 4 || resp.StatusCode/100 == 5 { + return clues.New("non-2xx response: " + resp.Status) + } + + defer resp.Body.Close() + + return nil +} diff --git a/src/pkg/services/m365/api/access_test.go b/src/pkg/services/m365/api/access_test.go new file mode 100644 index 000000000..c903fcde1 --- /dev/null +++ b/src/pkg/services/m365/api/access_test.go @@ -0,0 +1,122 @@ +package api_test + +import ( + "testing" + + "github.com/alcionai/clues" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/internal/tester/tconfig" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +type AccessAPIIntgSuite struct { + tester.Suite + its intgTesterSetup +} + +func TestAccessAPIIntgSuite(t *testing.T) { + suite.Run(t, &AccessAPIIntgSuite{ + Suite: tester.NewIntegrationSuite( + t, + [][]string{tconfig.M365AcctCredEnvs}), + }) +} + +func (suite *AccessAPIIntgSuite) SetupSuite() { + suite.its = newIntegrationTesterSetup(suite.T()) +} + +func (suite *AccessAPIIntgSuite) TestGetToken() { + tests := []struct { + name string + creds func() account.M365Config + expectErr require.ErrorAssertionFunc + }{ + { + name: "good", + creds: func() account.M365Config { return suite.its.ac.Credentials }, + expectErr: require.NoError, + }, + { + name: "bad tenant ID", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureTenantID = "ZIM" + + return creds + }, + expectErr: require.Error, + }, + { + name: "missing tenant ID", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureTenantID = "" + + return creds + }, + expectErr: require.Error, + }, + { + name: "bad client ID", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureClientID = "GIR" + + return creds + }, + expectErr: require.Error, + }, + { + name: "missing client ID", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureClientID = "" + + return creds + }, + expectErr: require.Error, + }, + { + name: "bad client secret", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureClientSecret = "MY TALLEST" + + return creds + }, + expectErr: require.Error, + }, + { + name: "missing client secret", + creds: func() account.M365Config { + creds := suite.its.ac.Credentials + creds.AzureClientSecret = "" + + return creds + }, + expectErr: require.Error, + }, + } + for _, test := range tests { + suite.Run(test.name, func() { + t := suite.T() + + ctx, flush := tester.NewContext(t) + defer flush() + + ac, err := api.NewClient(suite.its.ac.Credentials, control.DefaultOptions()) + require.NoError(t, err, clues.ToCore(err)) + + ac.Credentials = test.creds() + + err = ac.Access().GetToken(ctx) + test.expectErr(t, err, clues.ToCore(err)) + }) + } +} diff --git a/src/pkg/services/m365/api/client.go b/src/pkg/services/m365/api/client.go index a3f1fcee7..a0d90eb46 100644 --- a/src/pkg/services/m365/api/client.go +++ b/src/pkg/services/m365/api/client.go @@ -2,6 +2,7 @@ package api import ( "context" + "io" "net/http" "github.com/alcionai/clues" @@ -119,6 +120,16 @@ func (c Client) Get( return c.Requester.Request(ctx, http.MethodGet, url, nil, headers) } +// Get performs an ad-hoc get request using its graph.Requester +func (c Client) Post( + ctx context.Context, + url string, + headers map[string]string, + body io.Reader, +) (*http.Response, error) { + return c.Requester.Request(ctx, http.MethodGet, url, body, headers) +} + // --------------------------------------------------------------------------- // per-call config // ---------------------------------------------------------------------------