larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add generic details command (#4352)

Browse Source 
centralizes details command processing in the cli

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

* #2025
This commit is contained in:
Keepers 2023-09-30 10:56:13 -06:00 committed by GitHub
parent 5eaf95052d
commit b15f8a6fcd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1043 additions and 927 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/cli/backup/backup.go
View File

@ -16,6 +16,8 @@ import (
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
@ -163,7 +165,7 @@ func handleDeleteCmd(cmd *cobra.Command, args []string) error {
// standard set of selector behavior that we want used in the cli
var defaultSelectorConfig = selectors.Config{OnlyMatchItemNames: true}
func runBackups(
func genericCreateCommand(
ctx context.Context,
r repository.Repositoryer,
serviceName string,
@ -332,6 +334,65 @@ func genericListCommand(
return nil
}
func genericDetailsCommand(
cmd *cobra.Command,
backupID string,
sel selectors.Selector,
) (*details.Details, error) {
ctx := cmd.Context()
r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService)
if err != nil {
return nil, clues.Stack(err)
}
defer utils.CloseRepo(ctx, r)
return genericDetailsCore(
ctx,
r,
backupID,
sel,
rdao.Opts)
}
func genericDetailsCore(
ctx context.Context,
bg repository.BackupGetter,
backupID string,
sel selectors.Selector,
opts control.Options,
) (*details.Details, error) {
ctx = clues.Add(ctx, "backup_id", backupID)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
d, _, errs := bg.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("no backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
if opts.SkipReduce {
return d, nil
}
d, err := sel.Reduce(ctx, d, errs)
if err != nil {
return nil, clues.Wrap(err, "filtering backup details to selection")
}
return d, nil
}
// ---------------------------------------------------------------------------
// helper funcs
// ---------------------------------------------------------------------------
func ifShow(flag string) bool {
return strings.ToLower(strings.TrimSpace(flag)) == "show"
}

68
src/cli/backup/backup_test.go Normal file
View File

@ -0,0 +1,68 @@
package backup
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
type BackupUnitSuite struct {
tester.Suite
}
func TestBackupUnitSuite(t *testing.T) {
suite.Run(t, &BackupUnitSuite{Suite: tester.NewUnitSuite(t)})
}
func (suite *BackupUnitSuite) TestGenericDetailsCore() {
t := suite.T()
expected := append(
append(
dtd.GetItemsForVersion(
t,
path.ExchangeService,
path.EmailCategory,
0,
-1),
dtd.GetItemsForVersion(
t,
path.ExchangeService,
path.EventsCategory,
0,
-1)...),
dtd.GetItemsForVersion(
t,
path.ExchangeService,
path.ContactsCategory,
0,
-1)...)
ctx, flush := tester.NewContext(t)
defer flush()
bg := testdata.VersionedBackupGetter{
Details: dtd.GetDetailsSetForVersion(t, 0),
}
sel := selectors.NewExchangeBackup([]string{"user-id"})
sel.Include(sel.AllData())
output, err := genericDetailsCore(
ctx,
bg,
"backup-ID",
sel.Selector,
control.DefaultOptions())
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, expected, output.Entries)
}

75
src/cli/backup/exchange.go
View File

@ -1,21 +1,15 @@
package backup
import (
"context"
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -182,7 +176,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
selectorSet = append(selectorSet, discSel.Selector)
}
return runBackups(
return genericCreateCommand(
ctx,
r,
"Exchange",
@ -272,74 +266,31 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
return nil
}
return runDetailsExchangeCmd(cmd)
}
func runDetailsExchangeCmd(cmd *cobra.Command) error {
ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd)
r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.ExchangeService)
sel := utils.IncludeExchangeRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterExchangeRestoreInfoSelectors(sel, opts)
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ds, err := runDetailsExchangeCmd(
ctx,
r,
flags.BackupIDFV,
opts,
rdao.Opts.SkipReduce)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
if len(ds.Entries) > 0 {
ds.PrintEntries(ctx)
} else {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsExchangeCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Failure() == nil.
func runDetailsExchangeCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.ExchangeOpts,
skipReduce bool,
) (*details.Details, error) {
if err := utils.ValidateExchangeRestoreFlags(backupID, opts); err != nil {
return nil, err
}
ctx = clues.Add(ctx, "backup_id", backupID)
d, _, errs := r.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("No backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
ctx = clues.Add(ctx, "details_entries", len(d.Entries))
if !skipReduce {
sel := utils.IncludeExchangeRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterExchangeRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}
return d, nil
}
// ------------------------------------------------------------------------------------------------
// backup delete
// ------------------------------------------------------------------------------------------------

52
src/cli/backup/exchange_test.go
View File

@ -1,7 +1,6 @@
package backup
import (
"fmt"
"strconv"
"testing"
@ -15,10 +14,7 @@ import (
flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/cli/utils"
utilsTD "github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control"
)
@ -368,51 +364,3 @@ func (suite *ExchangeUnitSuite) TestExchangeBackupCreateSelectors() {
})
}
}
func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectors() {
for v := 0; v <= version.Backup; v++ {
suite.Run(fmt.Sprintf("version%d", v), func() {
for _, test := range utilsTD.ExchangeOptionDetailLookups {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
bg := utilsTD.VersionedBackupGetter{
Details: dtd.GetDetailsSetForVersion(t, v),
}
output, err := runDetailsExchangeCmd(
ctx,
bg,
"backup-ID",
test.Opts(t, v),
false)
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
})
}
})
}
}
func (suite *ExchangeUnitSuite) TestExchangeBackupDetailsSelectorsBadFormats() {
for _, test := range utilsTD.BadExchangeOptionsFormats {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
output, err := runDetailsExchangeCmd(
ctx,
test.BackupGetter,
"backup-ID",
test.Opts(t, version.Backup),
false)
assert.Error(t, err, clues.ToCore(err))
assert.Empty(t, output)
})
}
}

73
src/cli/backup/groups.go
View File

@ -2,7 +2,6 @@ package backup
import (
"context"
"errors"
"fmt"
"github.com/alcionai/clues"
@ -14,12 +13,9 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
)
@ -174,7 +170,7 @@ func createGroupsCmd(cmd *cobra.Command, args []string) error {
selectorSet = append(selectorSet, discSel.Selector)
}
return runBackups(
return genericCreateCommand(
ctx,
r,
"Group",
@ -225,74 +221,31 @@ func detailsGroupsCmd(cmd *cobra.Command, args []string) error {
return nil
}
return runDetailsGroupsCmd(cmd)
}
func runDetailsGroupsCmd(cmd *cobra.Command) error {
ctx := cmd.Context()
opts := utils.MakeGroupsOpts(cmd)
r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.GroupsService)
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterGroupsRestoreInfoSelectors(sel, opts)
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ds, err := runDetailsGroupsCmd(
ctx,
r,
flags.BackupIDFV,
opts,
rdao.Opts.SkipReduce)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
if len(ds.Entries) > 0 {
ds.PrintEntries(ctx)
} else {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsGroupsCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Failure() == nil.
func runDetailsGroupsCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.GroupsOpts,
skipReduce bool,
) (*details.Details, error) {
if err := utils.ValidateGroupsRestoreFlags(backupID, opts); err != nil {
return nil, err
}
ctx = clues.Add(ctx, "backup_id", backupID)
d, _, errs := r.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("no backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
ctx = clues.Add(ctx, "details_entries", len(d.Entries))
if !skipReduce {
sel := utils.IncludeGroupsRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterGroupsRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}
return d, nil
}
// ------------------------------------------------------------------------------------------------
// backup delete
// ------------------------------------------------------------------------------------------------

3
src/cli/backup/helpers_test.go
View File

@ -21,7 +21,6 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/services/m365/api/mock"
@ -160,7 +159,7 @@ func prepM365Test(
repository.NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = repo.Initialize(ctx, ctrlRepo.Retention{})
err = repo.Initialize(ctx, repository.InitConfig{})
require.NoError(t, err, clues.ToCore(err))
return dependencies{

75
src/cli/backup/onedrive.go
View File

@ -1,21 +1,15 @@
package backup
import (
"context"
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -162,7 +156,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
selectorSet = append(selectorSet, discSel.Selector)
}
return runBackups(
return genericCreateCommand(
ctx,
r,
"OneDrive",
@ -229,74 +223,31 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
return nil
}
return runDetailsOneDriveCmd(cmd)
}
func runDetailsOneDriveCmd(cmd *cobra.Command) error {
ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd)
r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.OneDriveService)
sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ds, err := runDetailsOneDriveCmd(
ctx,
r,
flags.BackupIDFV,
opts,
rdao.Opts.SkipReduce)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
if len(ds.Entries) > 0 {
ds.PrintEntries(ctx)
} else {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsOneDriveCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Failure() == nil.
func runDetailsOneDriveCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.OneDriveOpts,
skipReduce bool,
) (*details.Details, error) {
if err := utils.ValidateOneDriveRestoreFlags(backupID, opts); err != nil {
return nil, err
}
ctx = clues.Add(ctx, "backup_id", backupID)
d, _, errs := r.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("no backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
ctx = clues.Add(ctx, "details_entries", len(d.Entries))
if !skipReduce {
sel := utils.IncludeOneDriveRestoreDataSelectors(opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterOneDriveRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}
return d, nil
}
// `corso backup delete onedrive [<flag>...]`
func oneDriveDeleteCmd() *cobra.Command {
return &cobra.Command{

52
src/cli/backup/onedrive_test.go
View File

@ -1,7 +1,6 @@
package backup
import (
"fmt"
"testing"
"github.com/alcionai/clues"
@ -14,10 +13,7 @@ import (
flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/cli/utils"
utilsTD "github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control"
)
@ -227,51 +223,3 @@ func (suite *OneDriveUnitSuite) TestValidateOneDriveBackupCreateFlags() {
})
}
}
func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectors() {
for v := 0; v <= version.Backup; v++ {
suite.Run(fmt.Sprintf("version%d", v), func() {
for _, test := range utilsTD.OneDriveOptionDetailLookups {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
bg := utilsTD.VersionedBackupGetter{
Details: dtd.GetDetailsSetForVersion(t, v),
}
output, err := runDetailsOneDriveCmd(
ctx,
bg,
"backup-ID",
test.Opts(t, v),
false)
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
})
}
})
}
}
func (suite *OneDriveUnitSuite) TestOneDriveBackupDetailsSelectorsBadFormats() {
for _, test := range utilsTD.BadOneDriveOptionsFormats {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
output, err := runDetailsOneDriveCmd(
ctx,
test.BackupGetter,
"backup-ID",
test.Opts(t, version.Backup),
false)
assert.Error(t, err, clues.ToCore(err))
assert.Empty(t, output)
})
}
}

75
src/cli/backup/sharepoint.go
View File

@ -4,7 +4,6 @@ import (
"context"
"github.com/alcionai/clues"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/exp/slices"
@ -13,12 +12,9 @@ import (
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365"
)
@ -179,7 +175,7 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
selectorSet = append(selectorSet, discSel.Selector)
}
return runBackups(
return genericCreateCommand(
ctx,
r,
"SharePoint",
@ -303,7 +299,7 @@ func deleteSharePointCmd(cmd *cobra.Command, args []string) error {
// backup details
// ------------------------------------------------------------------------------------------------
// `corso backup details onedrive [<flag>...]`
// `corso backup details SharePoint [<flag>...]`
func sharePointDetailsCmd() *cobra.Command {
return &cobra.Command{
Use: sharePointServiceCommand,
@ -324,70 +320,27 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
return nil
}
return runDetailsSharePointCmd(cmd)
}
func runDetailsSharePointCmd(cmd *cobra.Command) error {
ctx := cmd.Context()
opts := utils.MakeSharePointOpts(cmd)
r, rdao, err := utils.GetAccountAndConnect(ctx, cmd, path.SharePointService)
sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterSharePointRestoreInfoSelectors(sel, opts)
ds, err := genericDetailsCommand(cmd, flags.BackupIDFV, sel.Selector)
if err != nil {
return Only(ctx, err)
}
defer utils.CloseRepo(ctx, r)
ds, err := runDetailsSharePointCmd(
ctx,
r,
flags.BackupIDFV,
opts,
rdao.Opts.SkipReduce)
if err != nil {
return Only(ctx, err)
}
if len(ds.Entries) == 0 {
if len(ds.Entries) > 0 {
ds.PrintEntries(ctx)
} else {
Info(ctx, selectors.ErrorNoMatchingItems)
return nil
}
ds.PrintEntries(ctx)
return nil
}
// runDetailsSharePointCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Failure() == nil.
func runDetailsSharePointCmd(
ctx context.Context,
r repository.BackupGetter,
backupID string,
opts utils.SharePointOpts,
skipReduce bool,
) (*details.Details, error) {
if err := utils.ValidateSharePointRestoreFlags(backupID, opts); err != nil {
return nil, err
}
ctx = clues.Add(ctx, "backup_id", backupID)
d, _, errs := r.GetBackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, clues.New("no backup exists with the id " + backupID)
}
return nil, clues.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
ctx = clues.Add(ctx, "details_entries", len(d.Entries))
if !skipReduce {
sel := utils.IncludeSharePointRestoreDataSelectors(ctx, opts)
sel.Configure(selectors.Config{OnlyMatchItemNames: true})
utils.FilterSharePointRestoreInfoSelectors(sel, opts)
d = sel.Reduce(ctx, d, errs)
}
return d, nil
}

52
src/cli/backup/sharepoint_test.go
View File

@ -1,7 +1,6 @@
package backup
import (
"fmt"
"strings"
"testing"
@ -15,11 +14,8 @@ import (
flagsTD "github.com/alcionai/corso/src/cli/flags/testdata"
cliTD "github.com/alcionai/corso/src/cli/testdata"
"github.com/alcionai/corso/src/cli/utils"
utilsTD "github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/version"
dtd "github.com/alcionai/corso/src/pkg/backup/details/testdata"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -339,51 +335,3 @@ func (suite *SharePointUnitSuite) TestSharePointBackupCreateSelectors() {
})
}
}
func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectors() {
for v := 0; v <= version.Backup; v++ {
suite.Run(fmt.Sprintf("version%d", v), func() {
for _, test := range utilsTD.SharePointOptionDetailLookups {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
bg := utilsTD.VersionedBackupGetter{
Details: dtd.GetDetailsSetForVersion(t, v),
}
output, err := runDetailsSharePointCmd(
ctx,
bg,
"backup-ID",
test.Opts(t, v),
false)
assert.NoError(t, err, clues.ToCore(err))
assert.ElementsMatch(t, test.Expected(t, v), output.Entries)
})
}
})
}
}
func (suite *SharePointUnitSuite) TestSharePointBackupDetailsSelectorsBadFormats() {
for _, test := range utilsTD.BadSharePointOptionsFormats {
suite.Run(test.Name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
output, err := runDetailsSharePointCmd(
ctx,
test.BackupGetter,
"backup-ID",
test.Opts(t, version.Backup),
false)
assert.Error(t, err, clues.ToCore(err))
assert.Empty(t, output)
})
}
}

8
src/cli/repo/filesystem.go
View File

@ -85,7 +85,7 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
opt := utils.ControlWithConfig(cfg)
// Retention is not supported for filesystem repos.
retention := ctrlRepo.Retention{}
retentionOpts := ctrlRepo.Retention{}
// SendStartCorsoEvent uses distict ID as tenant ID because repoID is still not generated
utils.SendStartCorsoEvent(
@ -116,7 +116,9 @@ func initFilesystemCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller"))
}
if err = r.Initialize(ctx, retention); err != nil {
ric := repository.InitConfig{RetentionOpts: retentionOpts}
if err = r.Initialize(ctx, ric); err != nil {
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
return nil
}
@ -207,7 +209,7 @@ func connectFilesystemCmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Wrap(err, "Failed to create a repository controller"))
}
if err := r.Connect(ctx); err != nil {
if err := r.Connect(ctx, repository.ConnConfig{}); err != nil {
return Only(ctx, clues.Stack(ErrConnectingRepo, err))
}

3
src/cli/repo/filesystem_e2e_test.go
View File

@ -16,7 +16,6 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
@ -138,7 +137,7 @@ func (suite *FilesystemE2ESuite) TestConnectFilesystemCmd() {
repository.NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, repository.InitConfig{})
require.NoError(t, err, clues.ToCore(err))
// then test it

6
src/cli/repo/s3.go
View File

@ -138,7 +138,9 @@ func initS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Wrap(err, "Failed to construct the repository controller"))
}
if err = r.Initialize(ctx, retentionOpts); err != nil {
ric := repository.InitConfig{RetentionOpts: retentionOpts}
if err = r.Initialize(ctx, ric); err != nil {
if flags.SucceedIfExistsFV && errors.Is(err, repository.ErrorRepoAlreadyExists) {
return nil
}
@ -221,7 +223,7 @@ func connectS3Cmd(cmd *cobra.Command, args []string) error {
return Only(ctx, clues.Wrap(err, "Failed to create a repository controller"))
}
if err := r.Connect(ctx); err != nil {
if err := r.Connect(ctx, repository.ConnConfig{}); err != nil {
return Only(ctx, clues.Stack(ErrConnectingRepo, err))
}

3
src/cli/repo/s3_e2e_test.go
View File

@ -18,7 +18,6 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/storage"
storeTD "github.com/alcionai/corso/src/pkg/storage/testdata"
@ -214,7 +213,7 @@ func (suite *S3E2ESuite) TestConnectS3Cmd() {
repository.NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, repository.InitConfig{})
require.NoError(t, err, clues.ToCore(err))
// then test it

3
src/cli/restore/exchange_e2e_test.go
View File

@ -20,7 +20,6 @@ import (
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/repository"
"github.com/alcionai/corso/src/pkg/selectors"
@ -92,7 +91,7 @@ func (suite *RestoreExchangeE2ESuite) SetupSuite() {
repository.NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = suite.repo.Initialize(ctx, ctrlRepo.Retention{})
err = suite.repo.Initialize(ctx, repository.InitConfig{Service: path.ExchangeService})
require.NoError(t, err, clues.ToCore(err))
suite.backupOps = make(map[path.CategoryType]operations.BackupOperation)

8
src/cli/utils/utils.go
View File

@ -78,16 +78,10 @@ func GetAccountAndConnectWithOverrides(
return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "creating a repository controller")
}
if err := r.Connect(ctx); err != nil {
if err := r.Connect(ctx, repository.ConnConfig{Service: pst}); err != nil {
return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "connecting to the "+cfg.Storage.Provider.String()+" repository")
}
// this initializes our graph api client configurations,
// including control options such as concurency limitations.
if _, err := r.ConnectToM365(ctx, pst); err != nil {
return nil, RepoDetailsAndOpts{}, clues.Wrap(err, "connecting to m365")
}
rdao := RepoDetailsAndOpts{
Repo: cfg,
Opts: opts,

6
src/cmd/longevity_test/longevity.go
View File

@ -72,7 +72,7 @@ func deleteBackups(
// Only supported for S3 repos currently.
func pitrListBackups(
ctx context.Context,
service path.ServiceType,
pst path.ServiceType,
pitr time.Time,
backupIDs []string,
) error {
@ -113,14 +113,14 @@ func pitrListBackups(
return clues.Wrap(err, "creating a repo")
}
err = r.Connect(ctx)
err = r.Connect(ctx, repository.ConnConfig{Service: pst})
if err != nil {
return clues.Wrap(err, "connecting to the repository")
}
defer r.Close(ctx)
backups, err := r.BackupsByTag(ctx, store.Service(service))
backups, err := r.BackupsByTag(ctx, store.Service(pst))
if err != nil {
return clues.Wrap(err, "listing backups").WithClues(ctx)
}

39
src/internal/m365/controller.go
View File

@ -79,20 +79,29 @@ func NewController(
return nil, clues.Wrap(err, "creating api client").WithClues(ctx)
}
rc := resource.UnknownResource
var rCli *resourceClient
switch pst {
case path.ExchangeService, path.OneDriveService:
rc = resource.Users
case path.GroupsService:
rc = resource.Groups
case path.SharePointService:
rc = resource.Sites
}
// no failure for unknown service.
// In that case we create a controller that doesn't attempt to look up any resource
// data. This case helps avoid unnecessary service calls when the end user is running
// repo init and connect commands via the CLI. All other callers should be expected
// to pass in a known service, or else expect downstream failures.
if pst != path.UnknownService {
rc := resource.UnknownResource
rCli, err := getResourceClient(rc, ac)
if err != nil {
return nil, clues.Wrap(err, "creating resource client").WithClues(ctx)
switch pst {
case path.ExchangeService, path.OneDriveService:
rc = resource.Users
case path.GroupsService:
rc = resource.Groups
case path.SharePointService:
rc = resource.Sites
}
rCli, err = getResourceClient(rc, ac)
if err != nil {
return nil, clues.Wrap(err, "creating resource client").WithClues(ctx)
}
}
ctrl := Controller{
@ -110,6 +119,10 @@ func NewController(
return &ctrl, nil
}
func (ctrl *Controller) VerifyAccess(ctx context.Context) error {
return ctrl.AC.Access().GetToken(ctx)
}
// ---------------------------------------------------------------------------
// Processing Status
// ---------------------------------------------------------------------------
@ -195,7 +208,7 @@ func getResourceClient(rc resource.Category, ac api.Client) (*resourceClient, er
case resource.Groups:
return &resourceClient{enum: rc, getter: ac.Groups()}, nil
default:
return nil, clues.New("unrecognized owner resource enum").With("resource_enum", rc)
return nil, clues.New("unrecognized owner resource type").With("resource_enum", rc)
}
}

6
src/pkg/path/service_type.go
View File

@ -15,9 +15,9 @@ var ErrorUnknownService = clues.New("unknown service string")
// Metadata services are not considered valid service types for resource paths
// though they can be used for metadata paths.
//
// The order of the enums below can be changed, but the string representation of
// each enum must remain the same or migration code needs to be added to handle
// changes to the string format.
// The string representaton of each enum _must remain the same_. In case of
// changes to those values, we'll need migration code to handle transitions
// across states else we'll get marshalling/unmarshalling errors.
type ServiceType int
//go:generate stringer -type=ServiceType -linecomment

359
src/pkg/repository/backups.go Normal file
View File

@ -0,0 +1,359 @@
package repository
import (
"context"
"github.com/alcionai/clues"
"github.com/kopia/kopia/repo/manifest"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/streamstore"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/store"
)
// BackupGetter deals with retrieving metadata about backups from the
// repository.
type BackupGetter interface {
Backup(ctx context.Context, id string) (*backup.Backup, error)
Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus)
BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error)
GetBackupDetails(
ctx context.Context,
backupID string,
) (*details.Details, *backup.Backup, *fault.Bus)
GetBackupErrors(
ctx context.Context,
backupID string,
) (*fault.Errors, *backup.Backup, *fault.Bus)
}
type Backuper interface {
NewBackup(
ctx context.Context,
self selectors.Selector,
) (operations.BackupOperation, error)
NewBackupWithLookup(
ctx context.Context,
self selectors.Selector,
ins idname.Cacher,
) (operations.BackupOperation, error)
DeleteBackups(
ctx context.Context,
failOnMissing bool,
ids ...string,
) error
}
// NewBackup generates a BackupOperation runner.
func (r repository) NewBackup(
ctx context.Context,
sel selectors.Selector,
) (operations.BackupOperation, error) {
return r.NewBackupWithLookup(ctx, sel, nil)
}
// NewBackupWithLookup generates a BackupOperation runner.
// ownerIDToName and ownerNameToID are optional populations, in case the caller has
// already generated those values.
func (r repository) NewBackupWithLookup(
ctx context.Context,
sel selectors.Selector,
ins idname.Cacher,
) (operations.BackupOperation, error) {
err := r.ConnectDataProvider(ctx, sel.PathService())
if err != nil {
return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365")
}
ownerID, ownerName, err := r.Provider.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins)
if err != nil {
return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details")
}
// TODO: retrieve display name from gc
sel = sel.SetDiscreteOwnerIDName(ownerID, ownerName)
return operations.NewBackupOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
r.Provider,
r.Account,
sel,
sel, // the selector acts as an IDNamer for its discrete resource owner.
r.Bus)
}
// Backup retrieves a backup by id.
func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) {
return getBackup(ctx, id, store.NewWrapper(r.modelStore))
}
// getBackup handles the processing for Backup.
func getBackup(
ctx context.Context,
id string,
sw store.BackupGetter,
) (*backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(id))
if err != nil {
return nil, errWrapper(err)
}
return b, nil
}
// Backups lists backups by ID. Returns as many backups as possible with
// errors for the backups it was unable to retrieve.
func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) {
var (
bups []*backup.Backup
errs = fault.New(false)
sw = store.NewWrapper(r.modelStore)
)
for _, id := range ids {
ictx := clues.Add(ctx, "backup_id", id)
b, err := sw.GetBackup(ictx, model.StableID(id))
if err != nil {
errs.AddRecoverable(ctx, errWrapper(err))
}
bups = append(bups, b)
}
return bups, errs
}
// BackupsByTag lists all backups in a repository that contain all the tags
// specified.
func (r repository) BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) {
sw := store.NewWrapper(r.modelStore)
return backupsByTag(ctx, sw, fs)
}
// backupsByTag returns all backups matching all provided tags.
//
// TODO(ashmrtn): This exists mostly for testing, but we could restructure the
// code in this file so there's a more elegant mocking solution.
func backupsByTag(
ctx context.Context,
sw store.BackupWrapper,
fs []store.FilterOption,
) ([]*backup.Backup, error) {
bs, err := sw.GetBackups(ctx, fs...)
if err != nil {
return nil, clues.Stack(err)
}
// Filter out assist backup bases as they're considered incomplete and we
// haven't been displaying them before now.
res := make([]*backup.Backup, 0, len(bs))
for _, b := range bs {
if t := b.Tags[model.BackupTypeTag]; t != model.AssistBackup {
res = append(res, b)
}
}
return res, nil
}
// BackupDetails returns the specified backup.Details
func (r repository) GetBackupDetails(
ctx context.Context,
backupID string,
) (*details.Details, *backup.Backup, *fault.Bus) {
errs := fault.New(false)
deets, bup, err := getBackupDetails(
ctx,
backupID,
r.Account.ID(),
r.dataLayer,
store.NewWrapper(r.modelStore),
errs)
return deets, bup, errs.Fail(err)
}
// getBackupDetails handles the processing for GetBackupDetails.
func getBackupDetails(
ctx context.Context,
backupID, tenantID string,
kw *kopia.Wrapper,
sw store.BackupGetter,
errs *fault.Bus,
) (*details.Details, *backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(backupID))
if err != nil {
return nil, nil, errWrapper(err)
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
ssid = b.DetailsID
}
if len(ssid) == 0 {
return nil, b, clues.New("no streamstore id in backup").WithClues(ctx)
}
var (
sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService())
deets details.Details
)
err = sstore.Read(
ctx,
ssid,
streamstore.DetailsReader(details.UnmarshalTo(&deets)),
errs)
if err != nil {
return nil, nil, err
}
// Retroactively fill in isMeta information for items in older
// backup versions without that info
// version.Restore2 introduces the IsMeta flag, so only v1 needs a check.
if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker {
for _, d := range deets.Entries {
if d.OneDrive != nil {
d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef)
}
}
}
deets.DetailsModel = deets.FilterMetaFiles()
return &deets, b, nil
}
// BackupErrors returns the specified backup's fault.Errors
func (r repository) GetBackupErrors(
ctx context.Context,
backupID string,
) (*fault.Errors, *backup.Backup, *fault.Bus) {
errs := fault.New(false)
fe, bup, err := getBackupErrors(
ctx,
backupID,
r.Account.ID(),
r.dataLayer,
store.NewWrapper(r.modelStore),
errs)
return fe, bup, errs.Fail(err)
}
// getBackupErrors handles the processing for GetBackupErrors.
func getBackupErrors(
ctx context.Context,
backupID, tenantID string,
kw *kopia.Wrapper,
sw store.BackupGetter,
errs *fault.Bus,
) (*fault.Errors, *backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(backupID))
if err != nil {
return nil, nil, errWrapper(err)
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
return nil, b, clues.New("missing streamstore id in backup").WithClues(ctx)
}
var (
sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService())
fe fault.Errors
)
err = sstore.Read(
ctx,
ssid,
streamstore.FaultErrorsReader(fault.UnmarshalErrorsTo(&fe)),
errs)
if err != nil {
return nil, nil, err
}
return &fe, b, nil
}
// DeleteBackups removes the backups from both the model store and the backup
// storage.
//
// If failOnMissing is true then returns an error if a backup model can't be
// found. Otherwise ignores missing backup models.
//
// Missing models or snapshots during the actual deletion do not cause errors.
//
// All backups are delete as an atomic unit so any failures will result in no
// deletions.
func (r repository) DeleteBackups(
ctx context.Context,
failOnMissing bool,
ids ...string,
) error {
return deleteBackups(ctx, store.NewWrapper(r.modelStore), failOnMissing, ids...)
}
// deleteBackup handles the processing for backup deletion.
func deleteBackups(
ctx context.Context,
sw store.BackupGetterModelDeleter,
failOnMissing bool,
ids ...string,
) error {
// Although we haven't explicitly stated it, snapshots are technically
// manifests in kopia. This means we can use the same delete API to remove
// them and backup models. Deleting all of them together gives us both
// atomicity guarantees (around when data will be flushed) and helps reduce
// the number of manifest blobs that kopia will create.
var toDelete []manifest.ID
for _, id := range ids {
b, err := sw.GetBackup(ctx, model.StableID(id))
if err != nil {
if !failOnMissing && errors.Is(err, data.ErrNotFound) {
continue
}
return clues.Stack(errWrapper(err)).
WithClues(ctx).
With("delete_backup_id", id)
}
toDelete = append(toDelete, b.ModelStoreID)
if len(b.SnapshotID) > 0 {
toDelete = append(toDelete, manifest.ID(b.SnapshotID))
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
ssid = b.DetailsID
}
if len(ssid) > 0 {
toDelete = append(toDelete, manifest.ID(ssid))
}
}
return sw.DeleteWithModelStoreIDs(ctx, toDelete...)
}

88
src/pkg/repository/data_providers.go Normal file
View File

@ -0,0 +1,88 @@
package repository
import (
"context"
"fmt"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/m365"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations/inject"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/path"
)
type DataProvider interface {
inject.BackupProducer
inject.ExportConsumer
inject.RestoreConsumer
VerifyAccess(ctx context.Context) error
}
type DataProviderConnector interface {
// ConnectDataProvider initializes configurations
// and establishes the client connection with the
// data provider for this operation.
ConnectDataProvider(
ctx context.Context,
pst path.ServiceType,
) error
}
func (r *repository) ConnectDataProvider(
ctx context.Context,
pst path.ServiceType,
) error {
var (
provider DataProvider
err error
)
switch r.Account.Provider {
case account.ProviderM365:
provider, err = connectToM365(ctx, *r, pst)
default:
err = clues.New("unrecognized provider").WithClues(ctx)
}
if err != nil {
return clues.Wrap(err, "connecting data provider")
}
if err := provider.VerifyAccess(ctx); err != nil {
return clues.Wrap(err, fmt.Sprintf("verifying %s account connection", r.Account.Provider))
}
r.Provider = provider
return nil
}
func connectToM365(
ctx context.Context,
r repository,
pst path.ServiceType,
) (*m365.Controller, error) {
if r.Provider != nil {
ctrl, ok := r.Provider.(*m365.Controller)
if !ok {
// if the provider is initialized to a non-m365 controller, we should not
// attempt to connnect to m365 afterward.
return nil, clues.New("Attempted to connect to multiple data providers")
}
return ctrl, nil
}
progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365")
defer close(progressBar)
ctrl, err := m365.NewController(ctx, r.Account, pst, r.Opts)
if err != nil {
return nil, clues.Wrap(err, "creating m365 client controller")
}
return ctrl, nil
}

40
src/pkg/repository/exports.go Normal file
View File

@ -0,0 +1,40 @@
package repository
import (
"context"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/store"
)
type Exporter interface {
NewExport(
ctx context.Context,
backupID string,
sel selectors.Selector,
exportCfg control.ExportConfig,
) (operations.ExportOperation, error)
}
// NewExport generates a exportOperation runner.
func (r repository) NewExport(
ctx context.Context,
backupID string,
sel selectors.Selector,
exportCfg control.ExportConfig,
) (operations.ExportOperation, error) {
return operations.NewExportOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
r.Provider,
r.Account,
model.StableID(backupID),
sel,
exportCfg,
r.Bus)
}

3
src/pkg/repository/loadtest/repository_load_test.go
View File

@ -21,7 +21,6 @@ import (
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
ctrlTD "github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -111,7 +110,7 @@ func initM365Repo(t *testing.T) (
repository.NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, repository.InitConfig{})
require.NoError(t, err, clues.ToCore(err))
return ctx, r, ac, st

515
src/pkg/repository/repository.go
View File

@ -6,31 +6,20 @@ import (
"github.com/alcionai/clues"
"github.com/google/uuid"
"github.com/kopia/kopia/repo/manifest"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/crash"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/events"
"github.com/alcionai/corso/src/internal/kopia"
"github.com/alcionai/corso/src/internal/m365"
"github.com/alcionai/corso/src/internal/m365/collection/drive/metadata"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/streamstore"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/storage"
"github.com/alcionai/corso/src/pkg/store"
)
@ -42,48 +31,24 @@ var (
ErrorBackupNotFound = clues.New("no backup exists with that id")
)
// BackupGetter deals with retrieving metadata about backups from the
// repository.
type BackupGetter interface {
Backup(ctx context.Context, id string) (*backup.Backup, error)
Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus)
BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error)
GetBackupDetails(
ctx context.Context,
backupID string,
) (*details.Details, *backup.Backup, *fault.Bus)
GetBackupErrors(
ctx context.Context,
backupID string,
) (*fault.Errors, *backup.Backup, *fault.Bus)
}
type Repositoryer interface {
Initialize(ctx context.Context, retentionOpts ctrlRepo.Retention) error
Connect(ctx context.Context) error
Backuper
BackupGetter
Restorer
Exporter
DataProviderConnector
Initialize(
ctx context.Context,
cfg InitConfig,
) error
Connect(
ctx context.Context,
cfg ConnConfig,
) error
GetID() string
Close(context.Context) error
NewBackup(
ctx context.Context,
self selectors.Selector,
) (operations.BackupOperation, error)
NewBackupWithLookup(
ctx context.Context,
self selectors.Selector,
ins idname.Cacher,
) (operations.BackupOperation, error)
NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error)
NewExport(
ctx context.Context,
backupID string,
sel selectors.Selector,
exportCfg control.ExportConfig,
) (operations.ExportOperation, error)
NewMaintenance(
ctx context.Context,
mOpts ctrlRepo.Maintenance,
@ -92,14 +57,6 @@ type Repositoryer interface {
ctx context.Context,
rcOpts ctrlRepo.Retention,
) (operations.RetentionConfigOperation, error)
DeleteBackups(ctx context.Context, failOnMissing bool, ids ...string) error
BackupGetter
// ConnectToM365 establishes graph api connections
// and initializes api client configurations.
ConnectToM365(
ctx context.Context,
pst path.ServiceType,
) (*m365.Controller, error)
}
// Repository contains storage provider information.
@ -108,9 +65,10 @@ type repository struct {
CreatedAt time.Time
Version string // in case of future breaking changes
Account account.Account // the user's m365 account connection details
Storage storage.Storage // the storage provider details and configuration
Opts control.Options
Account account.Account // the user's m365 account connection details
Storage storage.Storage // the storage provider details and configuration
Opts control.Options
Provider DataProvider // the client controller used for external user data CRUD
Bus events.Eventer
dataLayer *kopia.Wrapper
@ -125,7 +83,7 @@ func (r repository) GetID() string {
func New(
ctx context.Context,
acct account.Account,
s storage.Storage,
st storage.Storage,
opts control.Options,
configFileRepoID string,
) (repo *repository, err error) {
@ -133,16 +91,16 @@ func New(
ctx,
"acct_provider", acct.Provider.String(),
"acct_id", clues.Hide(acct.ID()),
"storage_provider", s.Provider.String())
"storage_provider", st.Provider.String())
bus, err := events.NewBus(ctx, s, acct.ID(), opts)
bus, err := events.NewBus(ctx, st, acct.ID(), opts)
if err != nil {
return nil, clues.Wrap(err, "constructing event bus").WithClues(ctx)
}
repoID := configFileRepoID
if len(configFileRepoID) == 0 {
repoID = newRepoID(s)
repoID = newRepoID(st)
}
bus.SetRepoID(repoID)
@ -151,7 +109,7 @@ func New(
ID: repoID,
Version: "v1",
Account: acct,
Storage: s,
Storage: st,
Bus: bus,
Opts: opts,
}
@ -163,17 +121,22 @@ func New(
return &r, nil
}
type InitConfig struct {
// tells the data provider which service to
// use for its connection pattern. Optional.
Service path.ServiceType
RetentionOpts ctrlRepo.Retention
}
// Initialize will:
// - validate the m365 account & secrets
// - connect to the m365 account to ensure communication capability
// - validate the provider config & secrets
// - initialize the kopia repo with the provider and retention parameters
// - update maintenance retention parameters as needed
// - store the configuration details
// - connect to the provider
func (r *repository) Initialize(
ctx context.Context,
retentionOpts ctrlRepo.Retention,
cfg InitConfig,
) (err error) {
ctx = clues.Add(
ctx,
@ -187,10 +150,14 @@ func (r *repository) Initialize(
}
}()
if err := r.ConnectDataProvider(ctx, cfg.Service); err != nil {
return clues.Stack(err)
}
observe.Message(ctx, "Initializing repository")
kopiaRef := kopia.NewConn(r.Storage)
if err := kopiaRef.Initialize(ctx, r.Opts.Repo, retentionOpts); err != nil {
if err := kopiaRef.Initialize(ctx, r.Opts.Repo, cfg.RetentionOpts); err != nil {
// replace common internal errors so that sdk users can check results with errors.Is()
if errors.Is(err, kopia.ErrorRepoAlreadyExists) {
return clues.Stack(ErrorRepoAlreadyExists, err).WithClues(ctx)
@ -221,12 +188,21 @@ func (r *repository) Initialize(
return nil
}
type ConnConfig struct {
// tells the data provider which service to
// use for its connection pattern. Leave empty
// to skip the provider connection.
Service path.ServiceType
}
// Connect will:
// - validate the m365 account details
// - connect to the m365 account to ensure communication capability
// - connect to the m365 account
// - connect to the provider storage
// - return the connected repository
func (r *repository) Connect(ctx context.Context) (err error) {
func (r *repository) Connect(
ctx context.Context,
cfg ConnConfig,
) (err error) {
ctx = clues.Add(
ctx,
"acct_provider", r.Account.Provider.String(),
@ -239,6 +215,10 @@ func (r *repository) Connect(ctx context.Context) (err error) {
}
}()
if err := r.ConnectDataProvider(ctx, cfg.Service); err != nil {
return clues.Stack(err)
}
observe.Message(ctx, "Connecting to repository")
kopiaRef := kopia.NewConn(r.Storage)
@ -297,98 +277,6 @@ func (r *repository) Close(ctx context.Context) error {
return nil
}
// NewBackup generates a BackupOperation runner.
func (r repository) NewBackup(
ctx context.Context,
sel selectors.Selector,
) (operations.BackupOperation, error) {
return r.NewBackupWithLookup(ctx, sel, nil)
}
// NewBackupWithLookup generates a BackupOperation runner.
// ownerIDToName and ownerNameToID are optional populations, in case the caller has
// already generated those values.
func (r repository) NewBackupWithLookup(
ctx context.Context,
sel selectors.Selector,
ins idname.Cacher,
) (operations.BackupOperation, error) {
ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts)
if err != nil {
return operations.BackupOperation{}, clues.Wrap(err, "connecting to m365")
}
ownerID, ownerName, err := ctrl.PopulateProtectedResourceIDAndName(ctx, sel.DiscreteOwner, ins)
if err != nil {
return operations.BackupOperation{}, clues.Wrap(err, "resolving resource owner details")
}
// TODO: retrieve display name from gc
sel = sel.SetDiscreteOwnerIDName(ownerID, ownerName)
return operations.NewBackupOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
ctrl,
r.Account,
sel,
sel, // the selector acts as an IDNamer for its discrete resource owner.
r.Bus)
}
// NewExport generates a exportOperation runner.
func (r repository) NewExport(
ctx context.Context,
backupID string,
sel selectors.Selector,
exportCfg control.ExportConfig,
) (operations.ExportOperation, error) {
ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts)
if err != nil {
return operations.ExportOperation{}, clues.Wrap(err, "connecting to m365")
}
return operations.NewExportOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
ctrl,
r.Account,
model.StableID(backupID),
sel,
exportCfg,
r.Bus)
}
// NewRestore generates a restoreOperation runner.
func (r repository) NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error) {
ctrl, err := connectToM365(ctx, sel.PathService(), r.Account, r.Opts)
if err != nil {
return operations.RestoreOperation{}, clues.Wrap(err, "connecting to m365")
}
return operations.NewRestoreOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
ctrl,
r.Account,
model.StableID(backupID),
sel,
restoreCfg,
r.Bus,
count.New())
}
func (r repository) NewMaintenance(
ctx context.Context,
mOpts ctrlRepo.Maintenance,
@ -414,280 +302,6 @@ func (r repository) NewRetentionConfig(
r.Bus)
}
// Backup retrieves a backup by id.
func (r repository) Backup(ctx context.Context, id string) (*backup.Backup, error) {
return getBackup(ctx, id, store.NewWrapper(r.modelStore))
}
// getBackup handles the processing for Backup.
func getBackup(
ctx context.Context,
id string,
sw store.BackupGetter,
) (*backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(id))
if err != nil {
return nil, errWrapper(err)
}
return b, nil
}
// Backups lists backups by ID. Returns as many backups as possible with
// errors for the backups it was unable to retrieve.
func (r repository) Backups(ctx context.Context, ids []string) ([]*backup.Backup, *fault.Bus) {
var (
bups []*backup.Backup
errs = fault.New(false)
sw = store.NewWrapper(r.modelStore)
)
for _, id := range ids {
ictx := clues.Add(ctx, "backup_id", id)
b, err := sw.GetBackup(ictx, model.StableID(id))
if err != nil {
errs.AddRecoverable(ctx, errWrapper(err))
}
bups = append(bups, b)
}
return bups, errs
}
// BackupsByTag lists all backups in a repository that contain all the tags
// specified.
func (r repository) BackupsByTag(ctx context.Context, fs ...store.FilterOption) ([]*backup.Backup, error) {
sw := store.NewWrapper(r.modelStore)
return backupsByTag(ctx, sw, fs)
}
// backupsByTag returns all backups matching all provided tags.
//
// TODO(ashmrtn): This exists mostly for testing, but we could restructure the
// code in this file so there's a more elegant mocking solution.
func backupsByTag(
ctx context.Context,
sw store.BackupWrapper,
fs []store.FilterOption,
) ([]*backup.Backup, error) {
bs, err := sw.GetBackups(ctx, fs...)
if err != nil {
return nil, clues.Stack(err)
}
// Filter out assist backup bases as they're considered incomplete and we
// haven't been displaying them before now.
res := make([]*backup.Backup, 0, len(bs))
for _, b := range bs {
if t := b.Tags[model.BackupTypeTag]; t != model.AssistBackup {
res = append(res, b)
}
}
return res, nil
}
// BackupDetails returns the specified backup.Details
func (r repository) GetBackupDetails(
ctx context.Context,
backupID string,
) (*details.Details, *backup.Backup, *fault.Bus) {
errs := fault.New(false)
deets, bup, err := getBackupDetails(
ctx,
backupID,
r.Account.ID(),
r.dataLayer,
store.NewWrapper(r.modelStore),
errs)
return deets, bup, errs.Fail(err)
}
// getBackupDetails handles the processing for GetBackupDetails.
func getBackupDetails(
ctx context.Context,
backupID, tenantID string,
kw *kopia.Wrapper,
sw store.BackupGetter,
errs *fault.Bus,
) (*details.Details, *backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(backupID))
if err != nil {
return nil, nil, errWrapper(err)
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
ssid = b.DetailsID
}
if len(ssid) == 0 {
return nil, b, clues.New("no streamstore id in backup").WithClues(ctx)
}
var (
sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService())
deets details.Details
)
err = sstore.Read(
ctx,
ssid,
streamstore.DetailsReader(details.UnmarshalTo(&deets)),
errs)
if err != nil {
return nil, nil, err
}
// Retroactively fill in isMeta information for items in older
// backup versions without that info
// version.Restore2 introduces the IsMeta flag, so only v1 needs a check.
if b.Version >= version.OneDrive1DataAndMetaFiles && b.Version < version.OneDrive3IsMetaMarker {
for _, d := range deets.Entries {
if d.OneDrive != nil {
d.OneDrive.IsMeta = metadata.HasMetaSuffix(d.RepoRef)
}
}
}
deets.DetailsModel = deets.FilterMetaFiles()
return &deets, b, nil
}
// BackupErrors returns the specified backup's fault.Errors
func (r repository) GetBackupErrors(
ctx context.Context,
backupID string,
) (*fault.Errors, *backup.Backup, *fault.Bus) {
errs := fault.New(false)
fe, bup, err := getBackupErrors(
ctx,
backupID,
r.Account.ID(),
r.dataLayer,
store.NewWrapper(r.modelStore),
errs)
return fe, bup, errs.Fail(err)
}
// getBackupErrors handles the processing for GetBackupErrors.
func getBackupErrors(
ctx context.Context,
backupID, tenantID string,
kw *kopia.Wrapper,
sw store.BackupGetter,
errs *fault.Bus,
) (*fault.Errors, *backup.Backup, error) {
b, err := sw.GetBackup(ctx, model.StableID(backupID))
if err != nil {
return nil, nil, errWrapper(err)
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
return nil, b, clues.New("missing streamstore id in backup").WithClues(ctx)
}
var (
sstore = streamstore.NewStreamer(kw, tenantID, b.Selector.PathService())
fe fault.Errors
)
err = sstore.Read(
ctx,
ssid,
streamstore.FaultErrorsReader(fault.UnmarshalErrorsTo(&fe)),
errs)
if err != nil {
return nil, nil, err
}
return &fe, b, nil
}
// DeleteBackups removes the backups from both the model store and the backup
// storage.
//
// If failOnMissing is true then returns an error if a backup model can't be
// found. Otherwise ignores missing backup models.
//
// Missing models or snapshots during the actual deletion do not cause errors.
//
// All backups are delete as an atomic unit so any failures will result in no
// deletions.
func (r repository) DeleteBackups(
ctx context.Context,
failOnMissing bool,
ids ...string,
) error {
return deleteBackups(ctx, store.NewWrapper(r.modelStore), failOnMissing, ids...)
}
// deleteBackup handles the processing for backup deletion.
func deleteBackups(
ctx context.Context,
sw store.BackupGetterModelDeleter,
failOnMissing bool,
ids ...string,
) error {
// Although we haven't explicitly stated it, snapshots are technically
// manifests in kopia. This means we can use the same delete API to remove
// them and backup models. Deleting all of them together gives us both
// atomicity guarantees (around when data will be flushed) and helps reduce
// the number of manifest blobs that kopia will create.
var toDelete []manifest.ID
for _, id := range ids {
b, err := sw.GetBackup(ctx, model.StableID(id))
if err != nil {
if !failOnMissing && errors.Is(err, data.ErrNotFound) {
continue
}
return clues.Stack(errWrapper(err)).
WithClues(ctx).
With("delete_backup_id", id)
}
toDelete = append(toDelete, b.ModelStoreID)
if len(b.SnapshotID) > 0 {
toDelete = append(toDelete, manifest.ID(b.SnapshotID))
}
ssid := b.StreamStoreID
if len(ssid) == 0 {
ssid = b.DetailsID
}
if len(ssid) > 0 {
toDelete = append(toDelete, manifest.ID(ssid))
}
}
return sw.DeleteWithModelStoreIDs(ctx, toDelete...)
}
func (r repository) ConnectToM365(
ctx context.Context,
pst path.ServiceType,
) (*m365.Controller, error) {
ctrl, err := connectToM365(ctx, pst, r.Account, r.Opts)
if err != nil {
return nil, clues.Wrap(err, "connecting to m365")
}
return ctrl, nil
}
// ---------------------------------------------------------------------------
// Repository ID Model
// ---------------------------------------------------------------------------
@ -736,29 +350,6 @@ func newRepoID(s storage.Storage) string {
// helpers
// ---------------------------------------------------------------------------
var m365nonce bool
func connectToM365(
ctx context.Context,
pst path.ServiceType,
acct account.Account,
co control.Options,
) (*m365.Controller, error) {
if !m365nonce {
m365nonce = true
progressBar := observe.MessageWithCompletion(ctx, "Connecting to M365")
defer close(progressBar)
}
ctrl, err := m365.NewController(ctx, acct, pst, co)
if err != nil {
return nil, err
}
return ctrl, nil
}
func errWrapper(err error) error {
if errors.Is(err, data.ErrNotFound) {
return clues.Stack(ErrorBackupNotFound, err)

52
src/pkg/repository/repository_test.go
View File

@ -17,6 +17,7 @@ import (
ctrlRepo "github.com/alcionai/corso/src/pkg/control/repository"
"github.com/alcionai/corso/src/pkg/control/testdata"
"github.com/alcionai/corso/src/pkg/extensions"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/services/m365/api"
"github.com/alcionai/corso/src/pkg/storage"
@ -69,7 +70,7 @@ func (suite *RepositoryUnitSuite) TestInitialize() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
test.errCheck(t, err, clues.ToCore(err))
})
}
@ -85,12 +86,12 @@ func (suite *RepositoryUnitSuite) TestConnect() {
errCheck assert.ErrorAssertionFunc
}{
{
storage.ProviderUnknown.String(),
func() (storage.Storage, error) {
name: storage.ProviderUnknown.String(),
storage: func() (storage.Storage, error) {
return storage.NewStorage(storage.ProviderUnknown)
},
account.Account{},
assert.Error,
account: account.Account{},
errCheck: assert.Error,
},
}
for _, test := range table {
@ -111,7 +112,7 @@ func (suite *RepositoryUnitSuite) TestConnect() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Connect(ctx)
err = r.Connect(ctx, ConnConfig{})
test.errCheck(t, err, clues.ToCore(err))
})
}
@ -136,12 +137,13 @@ func TestRepositoryIntegrationSuite(t *testing.T) {
func (suite *RepositoryIntegrationSuite) TestInitialize() {
table := []struct {
name string
account account.Account
account func(*testing.T) account.Account
storage func(tester.TestT) storage.Storage
errCheck assert.ErrorAssertionFunc
}{
{
name: "success",
account: tconfig.NewM365Account,
storage: storeTD.NewPrefixedS3Storage,
errCheck: assert.NoError,
},
@ -156,13 +158,13 @@ func (suite *RepositoryIntegrationSuite) TestInitialize() {
st := test.storage(t)
r, err := New(
ctx,
test.account,
test.account(t),
st,
control.DefaultOptions(),
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
if err == nil {
defer func() {
err := r.Close(ctx)
@ -204,7 +206,7 @@ func (suite *RepositoryIntegrationSuite) TestInitializeWithRole() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err)
defer func() {
@ -218,21 +220,23 @@ func (suite *RepositoryIntegrationSuite) TestConnect() {
ctx, flush := tester.NewContext(t)
defer flush()
acct := tconfig.NewM365Account(t)
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := New(
ctx,
account.Account{},
acct,
st,
control.DefaultOptions(),
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err, clues.ToCore(err))
// now re-connect
err = r.Connect(ctx)
err = r.Connect(ctx, ConnConfig{})
assert.NoError(t, err, clues.ToCore(err))
}
@ -242,17 +246,19 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
ctx, flush := tester.NewContext(t)
defer flush()
acct := tconfig.NewM365Account(t)
// need to initialize the repository before we can test connecting to it.
st := storeTD.NewPrefixedS3Storage(t)
r, err := New(
ctx,
account.Account{},
acct,
st,
control.DefaultOptions(),
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err, clues.ToCore(err))
oldID := r.GetID()
@ -261,7 +267,7 @@ func (suite *RepositoryIntegrationSuite) TestConnect_sameID() {
require.NoError(t, err, clues.ToCore(err))
// now re-connect
err = r.Connect(ctx)
err = r.Connect(ctx, ConnConfig{})
require.NoError(t, err, clues.ToCore(err))
assert.Equal(t, oldID, r.GetID())
}
@ -284,7 +290,8 @@ func (suite *RepositoryIntegrationSuite) TestNewBackup() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
// service doesn't matter here, we just need a valid value.
err = r.Initialize(ctx, InitConfig{Service: path.ExchangeService})
require.NoError(t, err, clues.ToCore(err))
userID := tconfig.M365UserID(t)
@ -313,7 +320,7 @@ func (suite *RepositoryIntegrationSuite) TestNewRestore() {
"")
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err, clues.ToCore(err))
ro, err := r.NewRestore(
@ -343,7 +350,8 @@ func (suite *RepositoryIntegrationSuite) TestNewBackupAndDelete() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
// service doesn't matter here, we just need a valid value.
err = r.Initialize(ctx, InitConfig{Service: path.ExchangeService})
require.NoError(t, err, clues.ToCore(err))
userID := tconfig.M365UserID(t)
@ -396,7 +404,7 @@ func (suite *RepositoryIntegrationSuite) TestNewMaintenance() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err, clues.ToCore(err))
mo, err := r.NewMaintenance(ctx, ctrlRepo.Maintenance{})
@ -465,11 +473,11 @@ func (suite *RepositoryIntegrationSuite) Test_Options() {
NewRepoID)
require.NoError(t, err, clues.ToCore(err))
err = r.Initialize(ctx, ctrlRepo.Retention{})
err = r.Initialize(ctx, InitConfig{})
require.NoError(t, err)
assert.Equal(t, test.expectedLen, len(r.Opts.ItemExtensionFactory))
err = r.Connect(ctx)
err = r.Connect(ctx, ConnConfig{})
assert.NoError(t, err)
assert.Equal(t, test.expectedLen, len(r.Opts.ItemExtensionFactory))
})

42
src/pkg/repository/restores.go Normal file
View File

@ -0,0 +1,42 @@
package repository
import (
"context"
"github.com/alcionai/corso/src/internal/model"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/count"
"github.com/alcionai/corso/src/pkg/selectors"
"github.com/alcionai/corso/src/pkg/store"
)
type Restorer interface {
NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error)
}
// NewRestore generates a restoreOperation runner.
func (r repository) NewRestore(
ctx context.Context,
backupID string,
sel selectors.Selector,
restoreCfg control.RestoreConfig,
) (operations.RestoreOperation, error) {
return operations.NewRestoreOperation(
ctx,
r.Opts,
r.dataLayer,
store.NewWrapper(r.modelStore),
r.Provider,
r.Account,
model.StableID(backupID),
sel,
restoreCfg,
r.Bus,
count.New())
}

68
src/pkg/services/m365/api/access.go Normal file
View File

@ -0,0 +1,68 @@
package api
import (
"context"
"fmt"
"net/http"
"strings"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/m365/graph"
)
// ---------------------------------------------------------------------------
// controller
// ---------------------------------------------------------------------------
func (c Client) Access() Access {
return Access{c}
}
// Access is an interface-compliant provider of the client.
type Access struct {
Client
}
// GetToken retrieves a m365 application auth token using client id and secret credentials.
// This token is not normally needed in order for corso to function, and is implemented
// primarily as a way to exercise the validity of those credentials without need of specific
// permissions.
func (c Access) GetToken(
ctx context.Context,
) error {
var (
//nolint:lll
// https://learn.microsoft.com/en-us/graph/connecting-external-content-connectors-api-postman#step-5-get-an-authentication-token
rawURL = fmt.Sprintf(
"https://login.microsoftonline.com/%s/oauth2/v2.0/token",
c.Credentials.AzureTenantID)
headers = map[string]string{
"Content-Type": "application/x-www-form-urlencoded",
}
body = strings.NewReader(fmt.Sprintf(
"client_id=%s"+
"&client_secret=%s"+
"&scope=https://graph.microsoft.com/.default"+
"&grant_type=client_credentials",
c.Credentials.AzureClientID,
c.Credentials.AzureClientSecret))
)
resp, err := c.Post(ctx, rawURL, headers, body)
if err != nil {
return graph.Stack(ctx, err)
}
if resp.StatusCode == http.StatusBadRequest {
return clues.New("incorrect tenant or application parameters")
}
if resp.StatusCode/100 == 4 || resp.StatusCode/100 == 5 {
return clues.New("non-2xx response: " + resp.Status)
}
defer resp.Body.Close()
return nil
}

122
src/pkg/services/m365/api/access_test.go Normal file
View File

@ -0,0 +1,122 @@
package api_test
import (
"testing"
"github.com/alcionai/clues"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type AccessAPIIntgSuite struct {
tester.Suite
its intgTesterSetup
}
func TestAccessAPIIntgSuite(t *testing.T) {
suite.Run(t, &AccessAPIIntgSuite{
Suite: tester.NewIntegrationSuite(
t,
[][]string{tconfig.M365AcctCredEnvs}),
})
}
func (suite *AccessAPIIntgSuite) SetupSuite() {
suite.its = newIntegrationTesterSetup(suite.T())
}
func (suite *AccessAPIIntgSuite) TestGetToken() {
tests := []struct {
name string
creds func() account.M365Config
expectErr require.ErrorAssertionFunc
}{
{
name: "good",
creds: func() account.M365Config { return suite.its.ac.Credentials },
expectErr: require.NoError,
},
{
name: "bad tenant ID",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureTenantID = "ZIM"
return creds
},
expectErr: require.Error,
},
{
name: "missing tenant ID",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureTenantID = ""
return creds
},
expectErr: require.Error,
},
{
name: "bad client ID",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureClientID = "GIR"
return creds
},
expectErr: require.Error,
},
{
name: "missing client ID",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureClientID = ""
return creds
},
expectErr: require.Error,
},
{
name: "bad client secret",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureClientSecret = "MY TALLEST"
return creds
},
expectErr: require.Error,
},
{
name: "missing client secret",
creds: func() account.M365Config {
creds := suite.its.ac.Credentials
creds.AzureClientSecret = ""
return creds
},
expectErr: require.Error,
},
}
for _, test := range tests {
suite.Run(test.name, func() {
t := suite.T()
ctx, flush := tester.NewContext(t)
defer flush()
ac, err := api.NewClient(suite.its.ac.Credentials, control.DefaultOptions())
require.NoError(t, err, clues.ToCore(err))
ac.Credentials = test.creds()
err = ac.Access().GetToken(ctx)
test.expectErr(t, err, clues.ToCore(err))
})
}
}

11
src/pkg/services/m365/api/client.go
View File

@ -2,6 +2,7 @@ package api
import (
"context"
"io"
"net/http"
"github.com/alcionai/clues"
@ -119,6 +120,16 @@ func (c Client) Get(
return c.Requester.Request(ctx, http.MethodGet, url, nil, headers)
}
// Get performs an ad-hoc get request using its graph.Requester
func (c Client) Post(
ctx context.Context,
url string,
headers map[string]string,
body io.Reader,
) (*http.Response, error) {
return c.Requester.Request(ctx, http.MethodGet, url, body, headers)
}
// ---------------------------------------------------------------------------
// per-call config
// ---------------------------------------------------------------------------