Compare commits

...

19 Commits

Author SHA1 Message Date
ryanfkeepers
2eaeaf1f04 Wrap all assert/require error assertion funcs
Since structured error data from clues is not added
to error message logging from assert and require,
we're slicing out data that is valuable to debugging.
This adds a package in tester which wraps the error
assertion funcs so that structured error data is
appended to all failure outputs.
2023-02-23 15:25:27 -07:00
ryanfkeepers
4a882dec3b add logger CtxErr() qol func 2023-02-21 13:00:34 -07:00
ryanfkeepers
294ee7173e remove common errors
Now that clues is in place, we don't need the
errors.As handling provided by common errors.
Current errors.As can be replaced with either
errors.Is checks, or HasLabel checks on the
status code.
2023-02-21 10:42:41 -07:00
ryanfkeepers
5cf020d5c8 remove stats.Errs
Now that fault errors is in place, we don't need
the operation stats errs.
2023-02-20 18:58:26 -07:00
ryanfkeepers
3de82f9a5b refactor garph.ErrData 2023-02-20 18:53:44 -07:00
ryanfkeepers
27e9661e63 remove wrap and append support 2023-02-20 18:09:38 -07:00
ryanfkeepers
aa90a36e76 amend kopia error clues 2023-02-20 17:56:21 -07:00
ryanfkeepers
e2530a0bd9 remove readErr and writeErr from operations
Now that fault is in place, we can remove the
readErrs and writeErrs from operation persistence.
2023-02-20 17:32:49 -07:00
ryanfkeepers
7b50d2401a remove errors from connector status
Now that fault is in place, we can remove the error
tracking functionality of graph status, and let that
focus purely on metrics.
2023-02-20 17:21:03 -07:00
ryanfkeepers
5593352226 fix unit tests 2023-02-20 15:28:23 -07:00
ryanfkeepers
f88cf05117 fault package funcs rename
Renaming the funcs in the fault
package to be more clear about
their purpose and behavior.  Largely
just find&replace changes, except
for fault.go and the fault examples.
2023-02-20 14:44:09 -07:00
ryanfkeepers
7b1682c68d last little onedrive completion 2023-02-19 09:05:17 -07:00
ryanfkeepers
88b5df728b add clues & fault to onedrive collections 2023-02-19 09:03:10 -07:00
ryanfkeepers
c62c246ee2 fix clues addall and withall 2023-02-19 08:31:41 -07:00
ryanfkeepers
c5b5a60d4e use tracker 2023-02-19 08:30:38 -07:00
ryanfkeepers
734e90c960 adding clues & fault to onedrive restore 2023-02-19 08:30:38 -07:00
ryanfkeepers
3edc74c170 linter fix 2023-02-19 08:26:37 -07:00
ryanfkeepers
66f734b4db watch for et errors 2023-02-19 08:26:09 -07:00
ryanfkeepers
b4a31c08dd add fault.tracker for error additions
Realized we had a race condition: in an async
runtime it's possible for an errs.Err() to be
returned by multiple functions, even though that
Err() was only sourced by one of them.  The
addition of a tracker contains the returned
error into the scope of that func so that only
the error produced in the current iteration is
returned.
2023-02-19 08:25:02 -07:00
170 changed files with 2721 additions and 3104 deletions

View File

@ -314,8 +314,8 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
bups, ferrs := r.Backups(ctx, bIDs)
// TODO: print/log recoverable errors
if ferrs.Err() != nil {
return Only(ctx, errors.Wrap(ferrs.Err(), "Unable to retrieve backup results from storage"))
if ferrs.Failure() != nil {
return Only(ctx, errors.Wrap(ferrs.Failure(), "Unable to retrieve backup results from storage"))
}
backup.PrintAll(ctx, bups)
@ -492,7 +492,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
// runDetailsExchangeCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Err() == nil.
// errs.Failure() == nil.
func runDetailsExchangeCmd(
ctx context.Context,
r repository.BackupGetter,
@ -505,12 +505,12 @@ func runDetailsExchangeCmd(
d, _, errs := r.BackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Err() != nil {
if errors.Is(errs.Err(), data.ErrNotFound) {
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, errors.Errorf("No backup exists with the id %s", backupID)
}
return nil, errors.Wrap(errs.Err(), "Failed to get backup details in the repository")
return nil, errors.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
sel := utils.IncludeExchangeRestoreDataSelectors(opts)

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
@ -73,7 +74,7 @@ func (suite *NoBackupExchangeIntegrationSuite) SetupSuite() {
suite.recorder = strings.Builder{}
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -88,7 +89,7 @@ func (suite *NoBackupExchangeIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
}
func (suite *NoBackupExchangeIntegrationSuite) TestExchangeBackupListCmd_empty() {
@ -110,7 +111,7 @@ func (suite *NoBackupExchangeIntegrationSuite) TestExchangeBackupListCmd_empty()
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
@ -155,7 +156,7 @@ func (suite *BackupExchangeIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -170,7 +171,7 @@ func (suite *BackupExchangeIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
}
func (suite *BackupExchangeIntegrationSuite) TestExchangeBackupCmd() {
@ -196,7 +197,7 @@ func (suite *BackupExchangeIntegrationSuite) TestExchangeBackupCmd() {
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := recorder.String()
t.Log("backup results", result)
@ -243,7 +244,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
suite.recorder = strings.Builder{}
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -261,7 +262,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.backupOps = make(map[path.CategoryType]string)
@ -287,18 +288,18 @@ func (suite *PreparedBackupExchangeIntegrationSuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, bop.Run(ctx))
require.NoError(t, err)
aw.MustNoErr(t, bop.Run(ctx))
aw.MustNoErr(t, err)
bIDs := string(bop.Results.BackupID)
// sanity check, ensure we can find the backup and its details immediately
b, err := suite.repo.Backup(ctx, bop.Results.BackupID)
require.NoError(t, err, "retrieving recent backup by ID")
aw.MustNoErr(t, err, "retrieving recent backup by ID")
require.Equal(t, bIDs, string(b.ID), "repo backup matches results id")
_, b, errs := suite.repo.BackupDetails(ctx, bIDs)
require.NoError(t, errs.Err(), "retrieving recent backup details by ID")
require.Empty(t, errs.Errs(), "retrieving recent backup details by ID")
aw.MustNoErr(t, errs.Failure(), "retrieving recent backup details by ID")
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
require.Equal(t, bIDs, string(b.ID), "repo details matches results id")
suite.backupOps[set] = string(b.ID)
@ -324,7 +325,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeListCmd() {
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
// compare the output
result := suite.recorder.String()
@ -355,7 +356,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeListCmd_singleI
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
// compare the output
result := suite.recorder.String()
@ -380,7 +381,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeListCmd_badID()
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
})
}
}
@ -398,8 +399,8 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeDetailsCmd() {
// fetch the details from the repo first
deets, _, errs := suite.repo.BackupDetails(ctx, string(bID))
require.NoError(t, errs.Err())
require.Empty(t, errs.Errs())
aw.MustNoErr(t, errs.Failure())
require.Empty(t, errs.Recovered())
cmd := tester.StubRootCmd(
"backup", "details", "exchange",
@ -412,7 +413,7 @@ func (suite *PreparedBackupExchangeIntegrationSuite) TestExchangeDetailsCmd() {
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
// compare the output
result := suite.recorder.String()
@ -473,7 +474,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -489,7 +490,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
m365UserID := tester.M365UserID(t)
users := []string{m365UserID}
@ -499,8 +500,8 @@ func (suite *BackupDeleteExchangeIntegrationSuite) SetupSuite() {
sel.Include(sel.MailFolders([]string{exchange.DefaultMailFolder}, selectors.PrefixMatch()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))
require.NoError(t, err)
aw.MustNoErr(t, suite.backupOp.Run(ctx))
aw.MustNoErr(t, err)
}
func (suite *BackupDeleteExchangeIntegrationSuite) TestExchangeBackupDeleteCmd() {
@ -517,7 +518,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) TestExchangeBackupDeleteCmd()
cli.BuildCommandTree(cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
// a follow-up details call should fail, due to the backup ID being deleted
cmd = tester.StubRootCmd(
@ -526,7 +527,7 @@ func (suite *BackupDeleteExchangeIntegrationSuite) TestExchangeBackupDeleteCmd()
"--backup", string(suite.backupOp.Results.BackupID))
cli.BuildCommandTree(cmd)
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}
func (suite *BackupDeleteExchangeIntegrationSuite) TestExchangeBackupDeleteCmd_UnknownID() {
@ -543,5 +544,5 @@ func (suite *BackupDeleteExchangeIntegrationSuite) TestExchangeBackupDeleteCmd_U
cli.BuildCommandTree(cmd)
// unknown backupIDs should error since the modelStore can't find the backup
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}

View File

@ -11,6 +11,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type ExchangeSuite struct {
@ -74,23 +75,23 @@ func (suite *ExchangeSuite) TestValidateBackupCreateFlags() {
}{
{
name: "no users or data",
expect: assert.Error,
expect: aw.Err,
},
{
name: "no users only data",
data: []string{dataEmail},
expect: assert.Error,
expect: aw.Err,
},
{
name: "unrecognized data category",
user: []string{"fnord"},
data: []string{"smurfs"},
expect: assert.Error,
expect: aw.Err,
},
{
name: "only users no data",
user: []string{"fnord"},
expect: assert.NoError,
expect: aw.NoErr,
},
}
for _, test := range table {
@ -224,7 +225,7 @@ func (suite *ExchangeSuite) TestExchangeBackupDetailsSelectors() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.NoError(t, err, "failure")
aw.NoErr(t, err, "failure")
assert.ElementsMatch(t, test.Expected, output.Entries)
})
}
@ -241,7 +242,7 @@ func (suite *ExchangeSuite) TestExchangeBackupDetailsSelectorsBadFormats() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.Error(t, err, "failure")
aw.Err(t, err, "failure")
assert.Empty(t, output)
})
}

View File

@ -237,8 +237,8 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
bups, ferrs := r.Backups(ctx, bIDs)
// TODO: print/log recoverable errors
if ferrs.Err() != nil {
return Only(ctx, errors.Wrap(ferrs.Err(), "Unable to retrieve backup results from storage"))
if ferrs.Failure() != nil {
return Only(ctx, errors.Wrap(ferrs.Failure(), "Unable to retrieve backup results from storage"))
}
backup.PrintAll(ctx, bups)
@ -384,7 +384,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
// runDetailsOneDriveCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Err() == nil.
// errs.Failure() == nil.
func runDetailsOneDriveCmd(
ctx context.Context,
r repository.BackupGetter,
@ -397,12 +397,12 @@ func runDetailsOneDriveCmd(
d, _, errs := r.BackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Err() != nil {
if errors.Is(errs.Err(), data.ErrNotFound) {
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, errors.Errorf("no backup exists with the id %s", backupID)
}
return nil, errors.Wrap(errs.Err(), "Failed to get backup details in the repository")
return nil, errors.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
sel := utils.IncludeOneDriveRestoreDataSelectors(opts)

View File

@ -8,7 +8,6 @@ import (
"github.com/google/uuid"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
@ -17,6 +16,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
@ -58,7 +58,7 @@ func (suite *NoBackupOneDriveIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -79,7 +79,7 @@ func (suite *NoBackupOneDriveIntegrationSuite) SetupSuite() {
control.Options{
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
})
require.NoError(t, err)
aw.MustNoErr(t, err)
}
func (suite *NoBackupOneDriveIntegrationSuite) TestOneDriveBackupListCmd_empty() {
@ -101,7 +101,7 @@ func (suite *NoBackupOneDriveIntegrationSuite) TestOneDriveBackupListCmd_empty()
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
@ -143,7 +143,7 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -165,7 +165,7 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) SetupSuite() {
control.Options{
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
})
require.NoError(t, err)
aw.MustNoErr(t, err)
m365UserID := tester.M365UserID(t)
users := []string{m365UserID}
@ -175,8 +175,8 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) SetupSuite() {
sel.Include(sel.Folders(selectors.Any()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))
require.NoError(t, err)
aw.MustNoErr(t, suite.backupOp.Run(ctx))
aw.MustNoErr(t, err)
}
func (suite *BackupDeleteOneDriveIntegrationSuite) TestOneDriveBackupDeleteCmd() {
@ -198,7 +198,7 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) TestOneDriveBackupDeleteCmd()
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
@ -211,7 +211,7 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) TestOneDriveBackupDeleteCmd()
"--backup", string(suite.backupOp.Results.BackupID))
cli.BuildCommandTree(cmd)
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}
func (suite *BackupDeleteOneDriveIntegrationSuite) TestOneDriveBackupDeleteCmd_unknownID() {
@ -228,5 +228,5 @@ func (suite *BackupDeleteOneDriveIntegrationSuite) TestOneDriveBackupDeleteCmd_u
cli.BuildCommandTree(cmd)
// unknown backupIDs should error since the modelStore can't find the backup
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type OneDriveSuite struct {
@ -73,12 +74,12 @@ func (suite *OneDriveSuite) TestValidateOneDriveBackupCreateFlags() {
}{
{
name: "no users",
expect: assert.Error,
expect: aw.Err,
},
{
name: "users",
user: []string{"fnord"},
expect: assert.NoError,
expect: aw.NoErr,
},
}
for _, test := range table {
@ -99,7 +100,7 @@ func (suite *OneDriveSuite) TestOneDriveBackupDetailsSelectors() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.NoError(t, err)
aw.NoErr(t, err)
assert.ElementsMatch(t, test.Expected, output.Entries)
})
}
@ -116,7 +117,7 @@ func (suite *OneDriveSuite) TestOneDriveBackupDetailsSelectorsBadFormats() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.Error(t, err)
aw.Err(t, err)
assert.Empty(t, output)
})
}

View File

@ -257,8 +257,8 @@ func createSharePointCmd(cmd *cobra.Command, args []string) error {
bups, ferrs := r.Backups(ctx, bIDs)
// TODO: print/log recoverable errors
if ferrs.Err() != nil {
return Only(ctx, errors.Wrap(ferrs.Err(), "Unable to retrieve backup results from storage"))
if ferrs.Failure() != nil {
return Only(ctx, errors.Wrap(ferrs.Failure(), "Unable to retrieve backup results from storage"))
}
backup.PrintAll(ctx, bups)
@ -506,7 +506,7 @@ func detailsSharePointCmd(cmd *cobra.Command, args []string) error {
// runDetailsSharePointCmd actually performs the lookup in backup details.
// the fault.Errors return is always non-nil. Callers should check if
// errs.Err() == nil.
// errs.Failure() == nil.
func runDetailsSharePointCmd(
ctx context.Context,
r repository.BackupGetter,
@ -519,12 +519,12 @@ func runDetailsSharePointCmd(
d, _, errs := r.BackupDetails(ctx, backupID)
// TODO: log/track recoverable errors
if errs.Err() != nil {
if errors.Is(errs.Err(), data.ErrNotFound) {
if errs.Failure() != nil {
if errors.Is(errs.Failure(), data.ErrNotFound) {
return nil, errors.Errorf("no backup exists with the id %s", backupID)
}
return nil, errors.Wrap(errs.Err(), "Failed to get backup details in the repository")
return nil, errors.Wrap(errs.Failure(), "Failed to get backup details in the repository")
}
sel := utils.IncludeSharePointRestoreDataSelectors(opts)

View File

@ -8,7 +8,6 @@ import (
"github.com/google/uuid"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
@ -17,6 +16,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
@ -58,7 +58,7 @@ func (suite *NoBackupSharePointIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -73,7 +73,7 @@ func (suite *NoBackupSharePointIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
}
func (suite *NoBackupSharePointIntegrationSuite) TestSharePointBackupListCmd_empty() {
@ -95,7 +95,7 @@ func (suite *NoBackupSharePointIntegrationSuite) TestSharePointBackupListCmd_emp
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
@ -133,7 +133,7 @@ func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -149,7 +149,7 @@ func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
m365SiteID := tester.M365SiteID(t)
sites := []string{m365SiteID}
@ -159,8 +159,8 @@ func (suite *BackupDeleteSharePointIntegrationSuite) SetupSuite() {
sel.Include(sel.Libraries(selectors.Any()))
suite.backupOp, err = suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, suite.backupOp.Run(ctx))
require.NoError(t, err)
aw.MustNoErr(t, suite.backupOp.Run(ctx))
aw.MustNoErr(t, err)
}
func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd() {
@ -182,7 +182,7 @@ func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteC
ctx = print.SetRootCmd(ctx, cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
result := suite.recorder.String()
@ -197,7 +197,7 @@ func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteC
// "--backup", string(suite.backupOp.Results.BackupID))
// cli.BuildCommandTree(cmd)
// require.Error(t, cmd.ExecuteContext(ctx))
// aw.MustErr(t, cmd.ExecuteContext(ctx))
func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteCmd_unknownID() {
t := suite.T()
@ -213,5 +213,5 @@ func (suite *BackupDeleteSharePointIntegrationSuite) TestSharePointBackupDeleteC
cli.BuildCommandTree(cmd)
// unknown backupIDs should error since the modelStore can't find the backup
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/cli/utils/testdata"
"github.com/alcionai/corso/src/internal/connector"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -77,23 +78,23 @@ func (suite *SharePointSuite) TestValidateSharePointBackupCreateFlags() {
}{
{
name: "no sites or urls",
expect: assert.Error,
expect: aw.Err,
},
{
name: "sites",
site: []string{"smarf"},
expect: assert.NoError,
expect: aw.NoErr,
},
{
name: "urls",
weburl: []string{"fnord"},
expect: assert.NoError,
expect: aw.NoErr,
},
{
name: "both",
site: []string{"smarf"},
weburl: []string{"fnord"},
expect: assert.NoError,
expect: aw.NoErr,
},
}
for _, test := range table {
@ -196,7 +197,7 @@ func (suite *SharePointSuite) TestSharePointBackupCreateSelectors() {
defer flush()
sel, err := sharePointBackupCreateSelectors(ctx, test.site, test.weburl, test.data, gc)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.ElementsMatch(t, test.expect, sel.DiscreteResourceOwners())
})
@ -214,7 +215,7 @@ func (suite *SharePointSuite) TestSharePointBackupDetailsSelectors() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.NoError(t, err)
aw.NoErr(t, err)
assert.ElementsMatch(t, test.Expected, output.Entries)
})
}
@ -231,7 +232,7 @@ func (suite *SharePointSuite) TestSharePointBackupDetailsSelectorsBadFormats() {
test.BackupGetter,
"backup-ID",
test.Opts)
assert.Error(t, err)
aw.Err(t, err)
assert.Empty(t, output)
})
}

View File

@ -6,7 +6,6 @@ import (
"regexp"
"strings"
"github.com/alcionai/clues"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
@ -130,9 +129,7 @@ func Handle() {
}()
if err := corsoCmd.ExecuteContext(ctx); err != nil {
logger.Ctx(ctx).
With("err", err).
Errorw("cli execution", clues.InErr(err).Slice()...)
logger.CtxErr(ctx, err).Error("cli execution")
os.Exit(1)
}
}

View File

@ -8,10 +8,10 @@ import (
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/storage"
@ -53,20 +53,20 @@ func (suite *ConfigSuite) TestReadRepoConfigBasic() {
testConfigData := fmt.Sprintf(configFileTemplate, b, tID)
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
err := os.WriteFile(testConfigFilePath, []byte(testConfigData), 0o700)
require.NoError(t, err)
aw.MustNoErr(t, err)
// Configure viper to read test config file
vpr.SetConfigFile(testConfigFilePath)
// Read and validate config
require.NoError(t, vpr.ReadInConfig(), "reading repo config")
aw.MustNoErr(t, vpr.ReadInConfig(), "reading repo config")
s3Cfg, err := s3ConfigsFromViper(vpr)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, b, s3Cfg.Bucket)
m365, err := m365ConfigsFromViper(vpr)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, tID, m365.AzureTenantID)
}
@ -83,22 +83,22 @@ func (suite *ConfigSuite) TestWriteReadConfig() {
// Configure viper to read test config file
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
aw.MustNoErr(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{Bucket: bkt, DoNotUseTLS: true, DoNotVerifyTLS: true}
m365 := account.M365Config{AzureTenantID: tid}
require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
require.NoError(t, vpr.ReadInConfig(), "reading repo config")
aw.MustNoErr(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
aw.MustNoErr(t, vpr.ReadInConfig(), "reading repo config")
readS3Cfg, err := s3ConfigsFromViper(vpr)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
assert.Equal(t, readS3Cfg.DoNotUseTLS, s3Cfg.DoNotUseTLS)
assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
readM365, err := m365ConfigsFromViper(vpr)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
}
@ -115,13 +115,13 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
// Configure viper to read test config file
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
aw.MustNoErr(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{Bucket: bkt}
m365 := account.M365Config{AzureTenantID: tid}
require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
require.NoError(t, vpr.ReadInConfig(), "reading repo config")
aw.MustNoErr(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
aw.MustNoErr(t, vpr.ReadInConfig(), "reading repo config")
table := []struct {
name string
@ -134,7 +134,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
storage.Bucket: bkt,
account.AzureTenantID: tid,
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "empty values",
@ -142,17 +142,17 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
storage.Bucket: "",
account.AzureTenantID: "",
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "no overrides",
input: map[string]string{},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "nil map",
input: nil,
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "no recognized keys",
@ -160,7 +160,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
"fnords": "smurfs",
"nonsense": "",
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "mismatch",
@ -168,7 +168,7 @@ func (suite *ConfigSuite) TestMustMatchConfig() {
storage.Bucket: tid,
account.AzureTenantID: bkt,
},
errCheck: assert.Error,
errCheck: aw.Err,
},
}
for _, test := range table {
@ -212,7 +212,7 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
// Configure viper to read test config file
testConfigFilePath := filepath.Join(t.TempDir(), "corso.toml")
require.NoError(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
aw.MustNoErr(t, initWithViper(vpr, testConfigFilePath), "initializing repo config")
s3Cfg := storage.S3Config{
Bucket: bkt,
@ -223,14 +223,14 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
}
m365 := account.M365Config{AzureTenantID: tid}
require.NoError(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
require.NoError(t, vpr.ReadInConfig(), "reading repo config")
aw.MustNoErr(t, writeRepoConfigWithViper(vpr, s3Cfg, m365), "writing repo config")
aw.MustNoErr(t, vpr.ReadInConfig(), "reading repo config")
st, ac, err := getStorageAndAccountWithViper(vpr, true, nil)
require.NoError(t, err, "getting storage and account from config")
aw.MustNoErr(t, err, "getting storage and account from config")
readS3Cfg, err := st.S3Config()
require.NoError(t, err, "reading s3 config from storage")
aw.MustNoErr(t, err, "reading s3 config from storage")
assert.Equal(t, readS3Cfg.Bucket, s3Cfg.Bucket)
assert.Equal(t, readS3Cfg.Endpoint, s3Cfg.Endpoint)
assert.Equal(t, readS3Cfg.Prefix, s3Cfg.Prefix)
@ -238,11 +238,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount() {
assert.Equal(t, readS3Cfg.DoNotVerifyTLS, s3Cfg.DoNotVerifyTLS)
common, err := st.CommonConfig()
require.NoError(t, err, "reading common config from storage")
aw.MustNoErr(t, err, "reading common config from storage")
assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase))
readM365, err := ac.M365Config()
require.NoError(t, err, "reading m365 config from account")
aw.MustNoErr(t, err, "reading m365 config from account")
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))
@ -273,10 +273,10 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
}
st, ac, err := getStorageAndAccountWithViper(vpr, false, overrides)
require.NoError(t, err, "getting storage and account from config")
aw.MustNoErr(t, err, "getting storage and account from config")
readS3Cfg, err := st.S3Config()
require.NoError(t, err, "reading s3 config from storage")
aw.MustNoErr(t, err, "reading s3 config from storage")
assert.Equal(t, readS3Cfg.Bucket, bkt)
assert.Equal(t, readS3Cfg.Endpoint, end)
assert.Equal(t, readS3Cfg.Prefix, pfx)
@ -284,11 +284,11 @@ func (suite *ConfigIntegrationSuite) TestGetStorageAndAccount_noFileOnlyOverride
assert.True(t, readS3Cfg.DoNotVerifyTLS)
common, err := st.CommonConfig()
require.NoError(t, err, "reading common config from storage")
aw.MustNoErr(t, err, "reading common config from storage")
assert.Equal(t, common.CorsoPassphrase, os.Getenv(credentials.CorsoPassphrase))
readM365, err := ac.M365Config()
require.NoError(t, err, "reading m365 config from account")
aw.MustNoErr(t, err, "reading m365 config from account")
assert.Equal(t, readM365.AzureTenantID, m365.AzureTenantID)
assert.Equal(t, readM365.AzureClientID, os.Getenv(credentials.AzureClientID))
assert.Equal(t, readM365.AzureClientSecret, os.Getenv(credentials.AzureClientSecret))

View File

@ -5,6 +5,7 @@ import (
"context"
"testing"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
@ -24,7 +25,7 @@ func (suite *PrintUnitSuite) TestOnly() {
// cannot use tester.NewContext() here: circular imports
//nolint:forbidigo
ctx := SetRootCmd(context.Background(), c)
assert.NoError(t, Only(ctx, nil))
aw.NoErr(t, Only(ctx, nil))
assert.True(t, c.SilenceUsage)
}

View File

@ -3,13 +3,12 @@ package repo_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/cli"
"github.com/alcionai/corso/src/cli/config"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/repository"
@ -55,7 +54,7 @@ func (suite *S3IntegrationSuite) TestInitS3Cmd() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
@ -69,12 +68,12 @@ func (suite *S3IntegrationSuite) TestInitS3Cmd() {
cli.BuildCommandTree(cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
// a second initialization should result in an error
err = cmd.ExecuteContext(ctx)
assert.Error(t, err)
assert.ErrorIs(t, err, repository.ErrorRepoAlreadyExists)
aw.Err(t, err)
aw.ErrIs(t, err, repository.ErrorRepoAlreadyExists)
})
}
}
@ -87,7 +86,7 @@ func (suite *S3IntegrationSuite) TestInitMultipleTimes() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
@ -104,7 +103,7 @@ func (suite *S3IntegrationSuite) TestInitMultipleTimes() {
cli.BuildCommandTree(cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
}
}
@ -116,7 +115,7 @@ func (suite *S3IntegrationSuite) TestInitS3Cmd_missingBucket() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
@ -129,7 +128,7 @@ func (suite *S3IntegrationSuite) TestInitS3Cmd_missingBucket() {
cli.BuildCommandTree(cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}
func (suite *S3IntegrationSuite) TestConnectS3Cmd() {
@ -154,7 +153,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -167,7 +166,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd() {
// init the repo first
_, err = repository.Initialize(ctx, account.Account{}, st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
// then test it
cmd := tester.StubRootCmd(
@ -179,7 +178,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd() {
cli.BuildCommandTree(cmd)
// run the command
assert.NoError(t, cmd.ExecuteContext(ctx))
aw.NoErr(t, cmd.ExecuteContext(ctx))
})
}
}
@ -192,7 +191,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd_BadBucket() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
@ -206,7 +205,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd_BadBucket() {
cli.BuildCommandTree(cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}
func (suite *S3IntegrationSuite) TestConnectS3Cmd_BadPrefix() {
@ -217,7 +216,7 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd_BadPrefix() {
st := tester.NewPrefixedS3Storage(t)
cfg, err := st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
vpr, configFP := tester.MakeTempTestConfigClone(t, nil)
@ -231,5 +230,5 @@ func (suite *S3IntegrationSuite) TestConnectS3Cmd_BadPrefix() {
cli.BuildCommandTree(cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
}

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/operations"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
@ -64,7 +65,7 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
suite.st = tester.NewPrefixedS3Storage(t)
cfg, err := suite.st.S3Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
force := map[string]string{
tester.TestCfgAccountProvider: "M365",
@ -78,7 +79,7 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
// init the repo first
suite.repo, err = repository.Initialize(ctx, suite.acct, suite.st, control.Options{})
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.backupOps = make(map[path.CategoryType]operations.BackupOperation)
@ -102,18 +103,18 @@ func (suite *RestoreExchangeIntegrationSuite) SetupSuite() {
sel.Include(scopes)
bop, err := suite.repo.NewBackup(ctx, sel.Selector)
require.NoError(t, bop.Run(ctx))
require.NoError(t, err)
aw.MustNoErr(t, bop.Run(ctx))
aw.MustNoErr(t, err)
suite.backupOps[set] = bop
// sanity check, ensure we can find the backup and its details immediately
_, err = suite.repo.Backup(ctx, bop.Results.BackupID)
require.NoError(t, err, "retrieving recent backup by ID")
aw.MustNoErr(t, err, "retrieving recent backup by ID")
_, _, errs := suite.repo.BackupDetails(ctx, string(bop.Results.BackupID))
require.NoError(t, errs.Err(), "retrieving recent backup details by ID")
require.Empty(t, errs.Errs(), "retrieving recent backup details by ID")
aw.MustNoErr(t, errs.Failure(), "retrieving recent backup details by ID")
require.Empty(t, errs.Recovered(), "retrieving recent backup details by ID")
}
}
@ -132,7 +133,7 @@ func (suite *RestoreExchangeIntegrationSuite) TestExchangeRestoreCmd() {
cli.BuildCommandTree(cmd)
// run the command
require.NoError(t, cmd.ExecuteContext(ctx))
aw.MustNoErr(t, cmd.ExecuteContext(ctx))
})
}
}
@ -165,7 +166,7 @@ func (suite *RestoreExchangeIntegrationSuite) TestExchangeRestoreCmd_badTimeFlag
cli.BuildCommandTree(cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
})
}
}
@ -196,7 +197,7 @@ func (suite *RestoreExchangeIntegrationSuite) TestExchangeRestoreCmd_badBoolFlag
cli.BuildCommandTree(cmd)
// run the command
require.Error(t, cmd.ExecuteContext(ctx))
aw.MustErr(t, cmd.ExecuteContext(ctx))
})
}
}

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -30,23 +31,23 @@ func (suite *ExchangeUtilsSuite) TestValidateRestoreFlags() {
name: "with backupid",
backupID: "bid",
opts: utils.ExchangeOpts{},
expect: assert.NoError,
expect: aw.NoErr,
},
{
name: "no backupid",
opts: utils.ExchangeOpts{},
expect: assert.Error,
expect: aw.Err,
},
{
name: "valid time",
backupID: "bid",
opts: utils.ExchangeOpts{EmailReceivedAfter: common.Now()},
expect: assert.NoError,
expect: aw.NoErr,
},
{
name: "invalid time",
opts: utils.ExchangeOpts{EmailReceivedAfter: "fnords"},
expect: assert.Error,
expect: aw.Err,
},
}
for _, test := range table {

View File

@ -501,7 +501,7 @@ func (MockBackupGetter) Backup(
func (MockBackupGetter) Backups(
context.Context,
[]model.StableID,
) ([]*backup.Backup, *fault.Errors) {
) ([]*backup.Backup, *fault.Bus) {
return nil, fault.New(false).Fail(errors.New("unexpected call to mock"))
}
@ -515,7 +515,7 @@ func (MockBackupGetter) BackupsByTag(
func (bg *MockBackupGetter) BackupDetails(
ctx context.Context,
backupID string,
) (*details.Details, *backup.Backup, *fault.Errors) {
) (*details.Details, *backup.Backup, *fault.Bus) {
if bg == nil {
return testdata.GetDetailsSet(), nil, fault.New(true)
}

View File

@ -6,6 +6,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -25,11 +26,11 @@ func (suite *CliUtilsSuite) TestRequireProps() {
}{
{
props: map[string]string{"exists": "I have seen the fnords!"},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
props: map[string]string{"not-exists": ""},
errCheck: assert.Error,
errCheck: aw.Err,
},
}
for _, test := range table {

View File

@ -53,7 +53,7 @@ func generateAndRestoreItems(
howMany int,
dbf dataBuilderFunc,
opts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) (*details.Details, error) {
items := make([]item, 0, howMany)

View File

@ -1,7 +1,6 @@
package impl
import (
"github.com/alcionai/clues"
"github.com/spf13/cobra"
. "github.com/alcionai/corso/src/cli/print"
@ -78,9 +77,8 @@ func handleExchangeEmailFactory(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
log := logger.Ctx(ctx)
for _, e := range errs.Errs() {
log.Errorw(e.Error(), clues.InErr(err).Slice()...)
for _, e := range errs.Recovered() {
logger.CtxErr(ctx, err).Error(e.Error())
}
deets.PrintEntries(ctx)
@ -125,9 +123,8 @@ func handleExchangeCalendarEventFactory(cmd *cobra.Command, args []string) error
return Only(ctx, err)
}
log := logger.Ctx(ctx)
for _, e := range errs.Errs() {
log.Errorw(e.Error(), clues.InErr(err).Slice()...)
for _, e := range errs.Recovered() {
logger.CtxErr(ctx, err).Error(e.Error())
}
deets.PrintEntries(ctx)
@ -177,9 +174,8 @@ func handleExchangeContactFactory(cmd *cobra.Command, args []string) error {
return Only(ctx, err)
}
log := logger.Ctx(ctx)
for _, e := range errs.Errs() {
log.Errorw(e.Error(), clues.InErr(err).Slice()...)
for _, e := range errs.Recovered() {
logger.CtxErr(ctx, err).Error(e.Error())
}
deets.PrintEntries(ctx)

View File

@ -93,7 +93,7 @@ func runDisplayM365JSON(
ctx context.Context,
creds account.M365Config,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) error {
var (
bs []byte
@ -143,7 +143,7 @@ type itemer interface {
GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error)
Serialize(
ctx context.Context,
@ -156,7 +156,7 @@ func getItem(
ctx context.Context,
itm itemer,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) ([]byte, error) {
sp, _, err := itm.GetItem(ctx, user, itemID, errs)
if err != nil {

View File

@ -157,7 +157,7 @@ func purgeOneDriveFolders(
return nil, err
}
cfs, err := onedrive.GetAllFolders(ctx, gs, pager, prefix)
cfs, err := onedrive.GetAllFolders(ctx, gs, pager, prefix, fault.New(true))
if err != nil {
return nil, err
}

View File

@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type CommonConfigsSuite struct {
@ -49,9 +50,9 @@ func (suite *CommonConfigsSuite) TestUnionConfigs_string() {
bc stringConfig2
errCheck assert.ErrorAssertionFunc
}{
{"no error", stringConfig{keyExpect, nil}, stringConfig2{keyExpect2, nil}, assert.NoError},
{"tc error", stringConfig{keyExpect, assert.AnError}, stringConfig2{keyExpect2, nil}, assert.Error},
{"fc error", stringConfig{keyExpect, nil}, stringConfig2{keyExpect2, assert.AnError}, assert.Error},
{"no error", stringConfig{keyExpect, nil}, stringConfig2{keyExpect2, nil}, aw.NoErr},
{"tc error", stringConfig{keyExpect, assert.AnError}, stringConfig2{keyExpect2, nil}, aw.Err},
{"fc error", stringConfig{keyExpect, nil}, stringConfig2{keyExpect2, assert.AnError}, aw.Err},
}
for _, test := range table {
suite.Run(test.name, func() {

View File

@ -1,62 +0,0 @@
package common
import (
"fmt"
"io"
)
// TODO: Remove in favor of clues.Stack()
// Err provides boiler-plate functions that other types of errors can use
// if they wish to be compared with `errors.As()`. This struct ensures that
// stack traces are printed when requested (if present) and that Err
// chains `errors.As()`, `errors.Is()`, and `errors.Cause()` calls properly.
//
// When using errors.As, note that the variable that is passed as the second
// parameter must be a pointer to a type that exactly matches the returned type of the error previously. For
// example, if a struct was returned, the second parameter should be a pointer
// to said struct. If a pointer to a struct was returned, then a pointer to a
// pointer of the struct should be passed.
type Err struct {
Err error
}
func EncapsulateError(e error) *Err {
return &Err{Err: e}
}
func (e Err) Error() string {
return e.Err.Error()
}
func (e Err) Cause() error {
return e.Err
}
func (e Err) Unwrap() error {
return e.Err
}
// Format complies with the Formatter interface and gives pretty printing when
// functions like `fmt.Printf("%+v")` are called. Implementing this allows Err
// to print stack traces from the encapsulated error.
func (e Err) Format(s fmt.State, verb rune) {
if f, ok := e.Err.(fmt.Formatter); ok {
f.Format(s, verb)
return
}
// Formatting magic courtesy of github.com/pkg/errors.
switch verb {
case 'v':
if s.Flag('+') {
fmt.Fprintf(s, "%+v\n", e.Cause())
return
}
fallthrough
case 's', 'q':
// nolint:errcheck
_, _ = io.WriteString(s, e.Error())
}
}

View File

@ -1,84 +0,0 @@
package common_test
import (
"fmt"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
)
type testErr struct {
common.Err
}
type testErr2 struct {
common.Err
}
type ErrorsUnitSuite struct {
tester.Suite
}
func TestErrorsUnitSuite(t *testing.T) {
s := &ErrorsUnitSuite{Suite: tester.NewUnitSuite(t)}
suite.Run(t, s)
}
func (suite *ErrorsUnitSuite) TestPropagatesCause() {
err := assert.AnError
te := testErr{*common.EncapsulateError(err)}
te2 := testErr2{*common.EncapsulateError(te)}
assert.Equal(suite.T(), assert.AnError, errors.Cause(te2))
}
func (suite *ErrorsUnitSuite) TestPropagatesIs() {
err := assert.AnError
te := testErr{*common.EncapsulateError(err)}
te2 := testErr2{*common.EncapsulateError(te)}
assert.True(suite.T(), errors.Is(te2, err))
}
func (suite *ErrorsUnitSuite) TestPropagatesAs() {
err := assert.AnError
te := testErr{*common.EncapsulateError(err)}
te2 := testErr2{*common.EncapsulateError(te)}
tmp := testErr{}
assert.True(suite.T(), errors.As(te2, &tmp))
}
func (suite *ErrorsUnitSuite) TestAs() {
err := assert.AnError
te := testErr{*common.EncapsulateError(err)}
te2 := testErr2{*common.EncapsulateError(te)}
tmp := testErr2{}
assert.True(suite.T(), errors.As(te2, &tmp))
}
func (suite *ErrorsUnitSuite) TestAsIsUnique() {
err := assert.AnError
te := testErr{*common.EncapsulateError(err)}
tmp := testErr2{}
assert.False(suite.T(), errors.As(te, &tmp))
}
func (suite *ErrorsUnitSuite) TestPrintsStack() {
err := assert.AnError
err = errors.Wrap(err, "wrapped error")
te := testErr{*common.EncapsulateError(err)}
te2 := testErr2{*common.EncapsulateError(te)}
out := fmt.Sprintf("%+v", te2)
// Stack trace should include a line noting that we're running testify.
assert.Contains(suite.T(), out, "testify")
}

View File

@ -5,11 +5,11 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type CommonTimeUnitSuite struct {
@ -48,14 +48,14 @@ func (suite *CommonTimeUnitSuite) TestParseTime() {
nowStr := now.Format(time.RFC3339Nano)
result, err := common.ParseTime(nowStr)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, now.UTC(), result)
_, err = common.ParseTime("")
require.Error(t, err)
aw.MustErr(t, err)
_, err = common.ParseTime("flablabls")
require.Error(t, err)
aw.MustErr(t, err)
}
func (suite *CommonTimeUnitSuite) TestExtractTime() {
@ -68,14 +68,14 @@ func (suite *CommonTimeUnitSuite) TestExtractTime() {
c, err := common.ParseTime(ts)
require.NoError(t, err)
aw.MustNoErr(t, err)
return c
}
parseT := func(v string) time.Time {
t, err := time.Parse(time.RFC3339, v)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
return t
}
@ -151,7 +151,7 @@ func (suite *CommonTimeUnitSuite) TestExtractTime() {
t := suite.T()
result, err := common.ExtractTime(test.input)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, test.expect, comparable(t, result, test.clippedFormat))
})
}

View File

@ -39,7 +39,7 @@ func (gc *GraphConnector) DataCollections(
sels selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
ctx, end := D.Span(ctx, "gc:dataCollections", D.Index("service", sels.Service.String()))
defer end()
@ -91,7 +91,7 @@ func (gc *GraphConnector) DataCollections(
return colls, excludes, nil
case selectors.ServiceOneDrive:
return gc.OneDriveDataCollections(ctx, sels, metadata, ctrlOpts)
return gc.OneDriveDataCollections(ctx, sels, metadata, ctrlOpts, errs)
case selectors.ServiceSharePoint:
colls, excludes, err := sharepoint.DataCollections(
@ -193,6 +193,7 @@ func (gc *GraphConnector) OneDriveDataCollections(
selector selectors.Selector,
metadata []data.RestoreCollection,
ctrlOpts control.Options,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
odb, err := selector.ToOneDriveBackup()
if err != nil {
@ -218,7 +219,7 @@ func (gc *GraphConnector) OneDriveDataCollections(
gc.Service,
gc.UpdateStatus,
ctrlOpts,
).Get(ctx, metadata)
).Get(ctx, metadata, errs)
if err != nil {
return nil, nil, err
}
@ -249,7 +250,7 @@ func (gc *GraphConnector) RestoreDataCollections(
dest control.RestoreDestination,
opts control.Options,
dcs []data.RestoreCollection,
errs *fault.Errors,
errs *fault.Bus,
) (*details.Details, error) {
ctx, end := D.Span(ctx, "connector:restore")
defer end()
@ -268,7 +269,7 @@ func (gc *GraphConnector) RestoreDataCollections(
case selectors.ServiceExchange:
status, err = exchange.RestoreExchangeDataCollections(ctx, creds, gc.Service, dest, dcs, deets, errs)
case selectors.ServiceOneDrive:
status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets)
status, err = onedrive.RestoreCollections(ctx, backupVersion, gc.Service, dest, opts, dcs, deets, errs)
case selectors.ServiceSharePoint:
status, err = sharepoint.RestoreCollections(ctx, backupVersion, creds, gc.Service, dest, dcs, deets, errs)
default:

View File

@ -6,13 +6,13 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -108,7 +108,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
connector.UpdateStatus,
control.Options{},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Empty(t, excludes)
for range collections {
@ -124,12 +124,12 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestExchangeDataCollection
for object := range col.Items(ctx, fault.New(true)) {
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(object.ToReader())
assert.NoError(t, err, "received a buf.Read error")
aw.NoErr(t, err, "received a buf.Read error")
}
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Successful)
assert.NotZero(t, status.Metrics.Successes)
t.Log(status.String())
})
}
@ -208,7 +208,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestDataCollections_invali
nil,
control.Options{},
fault.New(true))
assert.Error(t, err)
aw.Err(t, err)
assert.Empty(t, collections)
assert.Empty(t, excludes)
})
@ -260,7 +260,7 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
connector,
control.Options{},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
// Not expecting excludes as this isn't an incremental backup.
assert.Empty(t, excludes)
@ -276,12 +276,12 @@ func (suite *ConnectorDataCollectionIntegrationSuite) TestSharePointDataCollecti
for object := range coll.Items(ctx, fault.New(true)) {
buf := &bytes.Buffer{}
_, err := buf.ReadFrom(object.ToReader())
assert.NoError(t, err, "reading item")
aw.NoErr(t, err, "reading item")
}
}
status := connector.AwaitStatus()
assert.NotZero(t, status.Successful)
assert.NotZero(t, status.Metrics.Successes)
t.Log(status.String())
})
}
@ -338,7 +338,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
nil,
control.Options{},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Len(t, cols, 1)
// No excludes yet as this isn't an incremental backup.
assert.Empty(t, excludes)
@ -369,7 +369,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
nil,
control.Options{},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Less(t, 0, len(cols))
// No excludes yet as this isn't an incremental backup.
assert.Empty(t, excludes)
@ -381,7 +381,7 @@ func (suite *ConnectorCreateSharePointCollectionIntegrationSuite) TestCreateShar
t.Log("File: " + item.UUID())
bs, err := io.ReadAll(item.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
t.Log(string(bs))
}
}

View File

@ -10,6 +10,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type BetaUnitSuite struct {
@ -24,14 +25,14 @@ func (suite *BetaUnitSuite) TestBetaService_Adapter() {
t := suite.T()
a := tester.NewMockM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
adpt, err := graph.CreateAdapter(
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
service := NewBetaService(adpt)
require.NotNil(t, service)
@ -45,5 +46,5 @@ func (suite *BetaUnitSuite) TestBetaService_Adapter() {
byteArray, err := service.Serialize(testPage)
assert.NotEmpty(t, byteArray)
assert.NoError(t, err)
aw.NoErr(t, err)
}

View File

@ -86,7 +86,7 @@ func userOptions(fs *string) *users.UsersRequestBuilderGetRequestConfiguration {
}
// GetAll retrieves all users.
func (c Users) GetAll(ctx context.Context, errs *fault.Errors) ([]models.Userable, error) {
func (c Users) GetAll(ctx context.Context, errs *fault.Bus) ([]models.Userable, error) {
service, err := c.service()
if err != nil {
return nil, err
@ -97,7 +97,7 @@ func (c Users) GetAll(ctx context.Context, errs *fault.Errors) ([]models.Userabl
resp, err = service.Client().Users().Get(ctx, userOptions(&userFilterNoGuests))
if err != nil {
return nil, clues.Wrap(err, "getting all users").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting all users")
}
iter, err := msgraphgocore.NewPageIterator(
@ -105,19 +105,22 @@ func (c Users) GetAll(ctx context.Context, errs *fault.Errors) ([]models.Userabl
service.Adapter(),
models.CreateUserCollectionResponseFromDiscriminatorValue)
if err != nil {
return nil, clues.Wrap(err, "creating users iterator").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "creating users iterator")
}
us := make([]models.Userable, 0)
var (
us = make([]models.Userable, 0)
el = errs.Local()
)
iterator := func(item any) bool {
if errs.Err() != nil {
if el.Failure() != nil {
return false
}
u, err := validateUser(item)
if err != nil {
errs.Add(clues.Wrap(err, "validating user").WithClues(ctx).With(graph.ErrData(err)...))
el.AddRecoverable(graph.Wrap(ctx, err, "validating user"))
} else {
us = append(us, u)
}
@ -126,10 +129,10 @@ func (c Users) GetAll(ctx context.Context, errs *fault.Errors) ([]models.Userabl
}
if err := iter.Iterate(ctx, iterator); err != nil {
return nil, clues.Wrap(err, "iterating all users").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "iterating all users")
}
return us, errs.Err()
return us, el.Failure()
}
func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, error) {
@ -141,7 +144,7 @@ func (c Users) GetByID(ctx context.Context, userID string) (models.Userable, err
resp, err = c.stable.Client().UsersById(userID).Get(ctx, nil)
if err != nil {
return nil, clues.Wrap(err, "getting user").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting user")
}
return resp, err
@ -160,7 +163,7 @@ func (c Users) GetInfo(ctx context.Context, userID string) (*UserInfo, error) {
if err != nil {
if !graph.IsErrExchangeMailFolderNotFound(err) {
return nil, clues.Wrap(err, "getting user's mail folder").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting user's mail folder")
}
delete(userInfo.DiscoveredServices, path.ExchangeService)

View File

@ -15,7 +15,7 @@ import (
// ---------------------------------------------------------------------------
type getAller interface {
GetAll(context.Context, *fault.Errors) ([]models.Userable, error)
GetAll(context.Context, *fault.Bus) ([]models.Userable, error)
}
type getter interface {
@ -36,7 +36,7 @@ type getWithInfoer interface {
// ---------------------------------------------------------------------------
// Users fetches all users in the tenant.
func Users(ctx context.Context, ga getAller, errs *fault.Errors) ([]models.Userable, error) {
func Users(ctx context.Context, ga getAller, errs *fault.Bus) ([]models.Userable, error) {
return ga.GetAll(ctx, errs)
}

View File

@ -5,13 +5,13 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -37,7 +37,7 @@ func (suite *ExchangeServiceSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
@ -45,7 +45,7 @@ func (suite *ExchangeServiceSuite) SetupSuite() {
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.gs = graph.NewService(adpt)
}
@ -59,22 +59,22 @@ func (suite *ExchangeServiceSuite) TestOptionsForCalendars() {
{
name: "Empty Literal",
params: []string{},
checkError: assert.NoError,
checkError: aw.NoErr,
},
{
name: "Invalid Parameter",
params: []string{"status"},
checkError: assert.Error,
checkError: aw.Err,
},
{
name: "Invalid Parameters",
params: []string{"status", "height", "month"},
checkError: assert.Error,
checkError: aw.Err,
},
{
name: "Valid Parameters",
params: []string{"changeKey", "events", "owner"},
checkError: assert.NoError,
checkError: aw.NoErr,
},
}
for _, test := range tests {
@ -98,19 +98,19 @@ func (suite *ExchangeServiceSuite) TestOptionsForFolders() {
{
name: "Valid Folder Option",
params: []string{"parentFolderId"},
checkError: assert.NoError,
checkError: aw.NoErr,
expected: 2,
},
{
name: "Multiple Folder Options: Valid",
params: []string{"displayName", "isHidden"},
checkError: assert.NoError,
checkError: aw.NoErr,
expected: 3,
},
{
name: "Invalid Folder option param",
params: []string{"status"},
checkError: assert.Error,
checkError: aw.Err,
},
}
for _, test := range tests {
@ -135,19 +135,19 @@ func (suite *ExchangeServiceSuite) TestOptionsForContacts() {
{
name: "Valid Contact Option",
params: []string{"displayName"},
checkError: assert.NoError,
checkError: aw.NoErr,
expected: 2,
},
{
name: "Multiple Contact Options: Valid",
params: []string{"displayName", "parentFolderId"},
checkError: assert.NoError,
checkError: aw.NoErr,
expected: 3,
},
{
name: "Invalid Contact Option param",
params: []string{"status"},
checkError: assert.Error,
checkError: aw.Err,
},
}
for _, test := range tests {
@ -180,7 +180,7 @@ func (suite *ExchangeServiceSuite) TestHasAttachments() {
"This is testing",
)
message, err := support.CreateMessageFromBytes(byteArray)
require.NoError(t, err)
aw.MustNoErr(t, err)
return message.GetBody()
},
},
@ -190,7 +190,7 @@ func (suite *ExchangeServiceSuite) TestHasAttachments() {
getBodyable: func(t *testing.T) models.ItemBodyable {
byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy")
message, err := support.CreateMessageFromBytes(byteArray)
require.NoError(t, err)
aw.MustNoErr(t, err)
return message.GetBody()
},
},

View File

@ -49,7 +49,7 @@ func (c Contacts) CreateContactFolder(
mdl, err := c.stable.Client().UsersById(user).ContactFolders().Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Wrap(err, "creating contact folder").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "creating contact folder")
}
return mdl, nil
@ -62,7 +62,7 @@ func (c Contacts) DeleteContainer(
) error {
err := c.stable.Client().UsersById(user).ContactFoldersById(folderID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
return nil
@ -72,11 +72,11 @@ func (c Contacts) DeleteContainer(
func (c Contacts) GetItem(
ctx context.Context,
user, itemID string,
_ *fault.Errors, // no attachments to iterate over, so this goes unused
_ *fault.Bus, // no attachments to iterate over, so this goes unused
) (serialization.Parsable, *details.ExchangeInfo, error) {
cont, err := c.stable.Client().UsersById(user).ContactsById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Stack(ctx, err)
}
return cont, ContactInfo(cont), nil
@ -88,12 +88,12 @@ func (c Contacts) GetContainerByID(
) (graph.Container, error) {
ofcf, err := optionsForContactFolderByID([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, clues.Wrap(err, "setting contact folder options").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "setting contact folder options")
}
resp, err := c.stable.Client().UsersById(userID).ContactFoldersById(dirID).Get(ctx, ofcf)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return resp, nil
@ -101,29 +101,25 @@ func (c Contacts) GetContainerByID(
// EnumerateContainers iterates through all of the users current
// contacts folders, converting each to a graph.CacheFolder, and calling
// fn(cf) on each one. If fn(cf) errors, the error is aggregated
// into a multierror that gets returned to the caller.
// fn(cf) on each one.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
func (c Contacts) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
errs *fault.Bus,
) error {
service, err := c.service()
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
fields := []string{"displayName", "parentFolderId"}
ofcf, err := optionsForContactChildFolders(fields)
if err != nil {
return clues.Wrap(err, "setting contact child folder options").
WithClues(ctx).
With(graph.ErrData(err)...).
With("options_fields", fields)
return graph.Wrap(ctx, err, "setting contact child folder options")
}
builder := service.Client().
@ -134,16 +130,16 @@ func (c Contacts) EnumerateContainers(
for {
resp, err := builder.Get(ctx, ofcf)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
for _, fold := range resp.GetValue() {
if errs.Err() != nil {
return errs.Err()
if errs.Failure() != nil {
return errs.Failure()
}
if err := checkIDAndName(fold); err != nil {
errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...))
errs.AddRecoverable(graph.Stack(ctx, err))
continue
}
@ -154,7 +150,7 @@ func (c Contacts) EnumerateContainers(
temp := graph.NewCacheFolder(fold, nil, nil)
if err := fn(temp); err != nil {
errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
errs.AddRecoverable(graph.Stack(fctx, err))
continue
}
}
@ -167,7 +163,7 @@ func (c Contacts) EnumerateContainers(
builder = users.NewItemContactFoldersItemChildFoldersRequestBuilder(link, service.Adapter())
}
return errs.Err()
return errs.Failure()
}
// ---------------------------------------------------------------------------
@ -185,7 +181,7 @@ type contactPager struct {
func (p *contactPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return resp, nil
@ -205,7 +201,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
) ([]string, []string, DeltaUpdate, error) {
service, err := c.service()
if err != nil {
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, DeltaUpdate{}, graph.Stack(ctx, err)
}
var resetDelta bool
@ -220,7 +216,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
return nil,
nil,
DeltaUpdate{},
clues.Wrap(err, "setting contact folder options").WithClues(ctx).With(graph.ErrData(err)...)
graph.Wrap(ctx, err, "setting contact folder options")
}
if len(oldDelta) > 0 {
@ -238,7 +234,7 @@ func (c Contacts) GetAddedAndRemovedItemIDs(
// only return on error if it is NOT a delta issue.
// on bad deltas we retry the call with the regular builder
if !graph.IsErrInvalidDelta(err) {
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, DeltaUpdate{}, graph.Stack(ctx, err)
}
resetDelta = true
@ -294,12 +290,12 @@ func (c Contacts) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", contact); err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, clues.Wrap(err, "serializing contact").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "serializing contact")
}
return bs, nil

View File

@ -50,7 +50,7 @@ func (c Events) CreateCalendar(
mdl, err := c.stable.Client().UsersById(user).Calendars().Post(ctx, requestbody, nil)
if err != nil {
return nil, clues.Wrap(err, "creating calendar").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "creating calendar")
}
return mdl, nil
@ -64,7 +64,7 @@ func (c Events) DeleteContainer(
) error {
err := c.stable.Client().UsersById(user).CalendarsById(calendarID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
return nil
@ -76,17 +76,17 @@ func (c Events) GetContainerByID(
) (graph.Container, error) {
service, err := c.service()
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
ofc, err := optionsForCalendarsByID([]string{"name", "owner"})
if err != nil {
return nil, clues.Wrap(err, "setting event calendar options").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "setting event calendar options")
}
cal, err := service.Client().UsersById(userID).CalendarsById(containerID).Get(ctx, ofc)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err).WithClues(ctx)
}
return graph.CalendarDisplayable{Calendarable: cal}, nil
@ -96,7 +96,7 @@ func (c Events) GetContainerByID(
func (c Events) GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error) {
var (
err error
@ -105,7 +105,7 @@ func (c Events) GetItem(
event, err = c.stable.Client().UsersById(user).EventsById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Stack(ctx, err)
}
if *event.GetHasAttachments() || HasAttachments(event.GetBody()) {
@ -122,7 +122,7 @@ func (c Events) GetItem(
Attachments().
Get(ctx, options)
if err != nil {
return nil, nil, clues.Wrap(err, "event attachment download").WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Wrap(ctx, err, "event attachment download")
}
event.SetAttachments(attached.GetValue())
@ -133,24 +133,23 @@ func (c Events) GetItem(
// EnumerateContainers iterates through all of the users current
// calendars, converting each to a graph.CacheFolder, and
// calling fn(cf) on each one. If fn(cf) errors, the error is
// aggregated into a multierror that gets returned to the caller.
// calling fn(cf) on each one.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
func (c Events) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
errs *fault.Bus,
) error {
service, err := c.service()
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
ofc, err := optionsForCalendars([]string{"name"})
if err != nil {
return clues.Wrap(err, "setting calendar options").WithClues(ctx).With(graph.ErrData(err)...)
return graph.Wrap(ctx, err, "setting calendar options")
}
builder := service.Client().UsersById(userID).Calendars()
@ -158,13 +157,13 @@ func (c Events) EnumerateContainers(
for {
resp, err := builder.Get(ctx, ofc)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
for _, cal := range resp.GetValue() {
cd := CalendarDisplayable{Calendarable: cal}
if err := checkIDAndName(cd); err != nil {
errs.Add(clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...))
errs.AddRecoverable(graph.Stack(ctx, err))
continue
}
@ -178,7 +177,7 @@ func (c Events) EnumerateContainers(
path.Builder{}.Append(ptr.Val(cd.GetId())), // storage path
path.Builder{}.Append(ptr.Val(cd.GetDisplayName()))) // display location
if err := fn(temp); err != nil {
errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
errs.AddRecoverable(graph.Stack(fctx, err))
continue
}
}
@ -191,7 +190,7 @@ func (c Events) EnumerateContainers(
builder = users.NewItemCalendarsRequestBuilder(link, service.Adapter())
}
return errs.Err()
return errs.Failure()
}
// ---------------------------------------------------------------------------
@ -213,7 +212,7 @@ type eventPager struct {
func (p *eventPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
resp, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return resp, nil
@ -256,7 +255,7 @@ func (c Events) GetAddedAndRemovedItemIDs(
// only return on error if it is NOT a delta issue.
// on bad deltas we retry the call with the regular builder
if !graph.IsErrInvalidDelta(err) {
return nil, nil, DeltaUpdate{}, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, DeltaUpdate{}, graph.Stack(ctx, err)
}
resetDelta = true
@ -322,12 +321,12 @@ func (c Events) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", event); err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, clues.Wrap(err, "serializing event").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "serializing event")
}
return bs, nil

View File

@ -6,12 +6,12 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/backup/details"
)
@ -128,7 +128,7 @@ func (suite *EventsAPIUnitSuite) TestEventInfo() {
event, err = support.CreateEventFromBytes(bytes)
)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
return event, &details.ExchangeInfo{
ItemType: details.ExchangeEvent,

View File

@ -50,7 +50,7 @@ func (c Mail) CreateMailFolder(
mdl, err := c.stable.Client().UsersById(user).MailFolders().Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Wrap(err, "creating mail folder").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "creating mail folder")
}
return mdl, nil
@ -62,7 +62,7 @@ func (c Mail) CreateMailFolderWithParent(
) (models.MailFolderable, error) {
service, err := c.service()
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
isHidden := false
@ -77,7 +77,7 @@ func (c Mail) CreateMailFolderWithParent(
ChildFolders().
Post(ctx, requestBody, nil)
if err != nil {
return nil, clues.Wrap(err, "creating nested mail folder").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "creating nested mail folder")
}
return mdl, nil
@ -91,7 +91,7 @@ func (c Mail) DeleteContainer(
) error {
err := c.stable.Client().UsersById(user).MailFoldersById(folderID).Delete(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
return nil
@ -103,17 +103,17 @@ func (c Mail) GetContainerByID(
) (graph.Container, error) {
service, err := c.service()
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
ofmf, err := optionsForMailFoldersItem([]string{"displayName", "parentFolderId"})
if err != nil {
return nil, clues.Wrap(err, "setting mail folder options").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "setting mail folder options")
}
resp, err := service.Client().UsersById(userID).MailFoldersById(dirID).Get(ctx, ofmf)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return resp, nil
@ -124,11 +124,11 @@ func (c Mail) GetContainerByID(
func (c Mail) GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error) {
mail, err := c.stable.Client().UsersById(user).MessagesById(itemID).Get(ctx, nil)
if err != nil {
return nil, nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Stack(ctx, err)
}
if *mail.GetHasAttachments() || HasAttachments(mail.GetBody()) {
@ -145,7 +145,7 @@ func (c Mail) GetItem(
Attachments().
Get(ctx, options)
if err != nil {
return nil, nil, clues.Wrap(err, "mail attachment download").WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Wrap(ctx, err, "mail attachment download")
}
mail.SetAttachments(attached.GetValue())
@ -156,19 +156,18 @@ func (c Mail) GetItem(
// EnumerateContainers iterates through all of the users current
// mail folders, converting each to a graph.CacheFolder, and calling
// fn(cf) on each one. If fn(cf) errors, the error is aggregated
// into a multierror that gets returned to the caller.
// fn(cf) on each one.
// Folder hierarchy is represented in its current state, and does
// not contain historical data.
func (c Mail) EnumerateContainers(
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
errs *fault.Bus,
) error {
service, err := c.service()
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
builder := service.Client().
@ -179,7 +178,7 @@ func (c Mail) EnumerateContainers(
for {
resp, err := builder.Get(ctx, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
for _, v := range resp.GetValue() {
@ -190,7 +189,7 @@ func (c Mail) EnumerateContainers(
temp := graph.NewCacheFolder(v, nil, nil)
if err := fn(temp); err != nil {
errs.Add(clues.Stack(err).WithClues(fctx).With(graph.ErrData(err)...))
errs.AddRecoverable(graph.Stack(fctx, err))
continue
}
}
@ -203,7 +202,7 @@ func (c Mail) EnumerateContainers(
builder = users.NewItemMailFoldersDeltaRequestBuilder(link, service.Adapter())
}
return errs.Err()
return errs.Failure()
}
// ---------------------------------------------------------------------------
@ -221,7 +220,7 @@ type mailPager struct {
func (p *mailPager) getPage(ctx context.Context) (api.DeltaPageLinker, error) {
page, err := p.builder.Get(ctx, p.options)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return page, nil
@ -259,7 +258,7 @@ func (c Mail) GetAddedAndRemovedItemIDs(
return nil,
nil,
DeltaUpdate{},
clues.Wrap(err, "setting contact folder options").WithClues(ctx).With(graph.ErrData(err)...)
graph.Wrap(ctx, err, "setting contact folder options")
}
if len(oldDelta) > 0 {
@ -332,12 +331,12 @@ func (c Mail) Serialize(
defer writer.Close()
if err = writer.WriteObjectValue("", msg); err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
bs, err := writer.GetSerializedContent()
if err != nil {
return nil, clues.Wrap(err, "serializing email").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "serializing email")
}
return bs, nil

View File

@ -74,14 +74,14 @@ func getItemsAddedAndRemovedFromContainer(
// get the next page of data, check for standard errors
resp, err := pager.getPage(ctx)
if err != nil {
return nil, nil, deltaURL, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, deltaURL, graph.Stack(ctx, err)
}
// each category type responds with a different interface, but all
// of them comply with GetValue, which is where we'll get our item data.
items, err := pager.valuesIn(resp)
if err != nil {
return nil, nil, "", clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, "", graph.Stack(ctx, err)
}
itemCount += len(items)

View File

@ -69,9 +69,7 @@ func uploadAttachment(
if attachmentType == models.ITEM_ATTACHMENTTYPE {
a, err := support.ToItemAttachment(attachment)
if err != nil {
logger.Ctx(ctx).
With("err", err).
Infow("item attachment restore not supported for this type. skipping upload.", clues.InErr(err).Slice()...)
logger.CtxErr(ctx, err).Info("item attachment restore not supported for this type. skipping upload.")
return nil
}

View File

@ -3,7 +3,6 @@ package exchange
import (
"context"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
msusers "github.com/microsoftgraph/msgraph-sdk-go/users"
@ -44,7 +43,7 @@ func (mau *mailAttachmentUploader) uploadSmallAttachment(ctx context.Context, at
Attachments().
Post(ctx, attach, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
return nil
@ -68,7 +67,7 @@ func (mau *mailAttachmentUploader) uploadSession(
CreateUploadSession().
Post(ctx, session, nil)
if err != nil {
return nil, clues.Wrap(err, "uploading mail attachment").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "uploading mail attachment")
}
return r, nil
@ -94,7 +93,7 @@ func (eau *eventAttachmentUploader) uploadSmallAttachment(ctx context.Context, a
Attachments().
Post(ctx, attach, nil)
if err != nil {
return clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return graph.Stack(ctx, err)
}
return nil
@ -116,7 +115,7 @@ func (eau *eventAttachmentUploader) uploadSession(
CreateUploadSession().
Post(ctx, session, nil)
if err != nil {
return nil, clues.Wrap(err, "uploading event attachment").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "uploading event attachment")
}
return r, nil

View File

@ -47,7 +47,7 @@ func (cfc *contactFolderCache) populateContactRoot(
// as of (Oct-07-2022)
func (cfc *contactFolderCache) Populate(
ctx context.Context,
errs *fault.Errors,
errs *fault.Bus,
baseID string,
baseContainerPather ...string,
) error {

View File

@ -28,7 +28,7 @@ type containersEnumerator interface {
ctx context.Context,
userID, baseDirID string,
fn func(graph.CacheFolder) error,
errs *fault.Errors,
errs *fault.Bus,
) error
}

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -78,7 +79,7 @@ var (
displayName: &testName,
parentID: &testParentID,
},
check: assert.Error,
check: aw.Err,
},
{
name: "NilDisplayName",
@ -87,7 +88,7 @@ var (
displayName: nil,
parentID: &testParentID,
},
check: assert.Error,
check: aw.Err,
},
{
name: "EmptyID",
@ -96,7 +97,7 @@ var (
displayName: &testName,
parentID: &testParentID,
},
check: assert.NoError,
check: aw.NoErr,
},
{
name: "EmptyDisplayName",
@ -105,7 +106,7 @@ var (
displayName: &emptyString,
parentID: &testParentID,
},
check: assert.NoError,
check: aw.NoErr,
},
{
name: "AllValues",
@ -114,7 +115,7 @@ var (
displayName: &testName,
parentID: &testParentID,
},
check: assert.NoError,
check: aw.NoErr,
},
}
)
@ -136,7 +137,7 @@ func (suite *FolderCacheUnitSuite) TestCheckRequiredValues() {
displayName: &testName,
parentID: nil,
},
check: assert.Error,
check: aw.Err,
},
{
name: "EmptyParentFolderID",
@ -145,7 +146,7 @@ func (suite *FolderCacheUnitSuite) TestCheckRequiredValues() {
displayName: &testName,
parentID: &emptyString,
},
check: assert.NoError,
check: aw.NoErr,
},
}
@ -174,7 +175,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
},
nil,
nil),
check: assert.Error,
check: aw.Err,
},
{
name: "NoParentPath",
@ -186,7 +187,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
},
path.Builder{}.Append("foo"),
path.Builder{}.Append("loc")),
check: assert.NoError,
check: aw.NoErr,
},
{
name: "NoName",
@ -198,7 +199,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
},
path.Builder{}.Append("foo"),
path.Builder{}.Append("loc")),
check: assert.Error,
check: aw.Err,
},
{
name: "NoID",
@ -210,7 +211,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
},
path.Builder{}.Append("foo"),
path.Builder{}.Append("loc")),
check: assert.Error,
check: aw.Err,
},
{
name: "NoPath",
@ -222,7 +223,7 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
},
nil,
nil),
check: assert.NoError,
check: aw.NoErr,
},
}
@ -341,12 +342,12 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestDepthLimit() {
{
name: "AtLimit",
numContainers: maxIterations,
check: assert.NoError,
check: aw.NoErr,
},
{
name: "OverLimit",
numContainers: maxIterations + 1,
check: assert.Error,
check: aw.Err,
},
}
@ -365,7 +366,7 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestPopulatePaths() {
t := suite.T()
require.NoError(t, suite.fc.populatePaths(ctx, false))
aw.MustNoErr(t, suite.fc.populatePaths(ctx, false))
items := suite.fc.Items()
gotPaths := make([]string, 0, len(items))
@ -389,7 +390,7 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderNoPathsCached
for _, c := range suite.allContainers {
suite.T().Run(*c.GetDisplayName(), func(t *testing.T) {
p, l, err := suite.fc.IDToPath(ctx, c.id, false)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
})
@ -403,7 +404,7 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderNoPathsCached
for _, c := range suite.containersWithID {
suite.T().Run(*c.GetDisplayName(), func(t *testing.T) {
p, l, err := suite.fcWithID.IDToPath(ctx, c.id, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
})
@ -418,14 +419,14 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderCachesPaths()
c := suite.allContainers[len(suite.allContainers)-1]
p, l, err := suite.fc.IDToPath(ctx, c.id, false)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
c.parentID = "foo"
p, l, err = suite.fc.IDToPath(ctx, c.id, false)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
}
@ -438,14 +439,14 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderCachesPaths_u
c := suite.containersWithID[len(suite.containersWithID)-1]
p, l, err := suite.fcWithID.IDToPath(ctx, c.id, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
c.parentID = "foo"
p, l, err = suite.fcWithID.IDToPath(ctx, c.id, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, c.expectedPath, p.String())
assert.Equal(t, c.expectedLocation, l.String())
}
@ -461,7 +462,7 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderErrorsParentN
delete(suite.fc.cache, almostLast.id)
_, _, err := suite.fc.IDToPath(ctx, last.id, false)
assert.Error(t, err)
aw.Err(t, err)
}
func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderErrorsNotFound() {
@ -471,7 +472,7 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestLookupCachedFolderErrorsNotFoun
t := suite.T()
_, _, err := suite.fc.IDToPath(ctx, "foo", false)
assert.Error(t, err)
aw.Err(t, err)
}
func (suite *ConfiguredFolderCacheUnitSuite) TestAddToCache() {
@ -490,12 +491,12 @@ func (suite *ConfiguredFolderCacheUnitSuite) TestAddToCache() {
m.expectedLocation = stdpath.Join(last.expectedPath, m.displayName)
require.Empty(t, suite.fc.DestinationNameToID(dest), "destination not yet added to cache")
require.NoError(t, suite.fc.AddToCache(ctx, m, false))
aw.MustNoErr(t, suite.fc.AddToCache(ctx, m, false))
require.Empty(t, suite.fc.DestinationNameToID(dest),
"destination id from cache, still empty, because this is not a calendar")
p, l, err := suite.fc.IDToPath(ctx, m.id, false)
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, m.expectedPath, p.String())
assert.Equal(t, m.expectedLocation, l.String())
}
@ -525,7 +526,7 @@ func (suite *FolderCacheIntegrationSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
@ -533,11 +534,11 @@ func (suite *FolderCacheIntegrationSuite) SetupSuite() {
m365.AzureTenantID,
m365.AzureClientID,
m365.AzureClientSecret)
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
}
// Testing to ensure that cache system works for in multiple different environments
@ -547,7 +548,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
var (
user = tester.M365UserID(suite.T())
@ -573,7 +574,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.EmailCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return pth
},
@ -586,7 +587,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.EmailCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return pth
},
@ -602,7 +603,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.ContactsCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return aPath
},
@ -614,7 +615,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.ContactsCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return aPath
},
@ -631,7 +632,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.EventsCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return aPath
},
@ -643,7 +644,7 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
user,
path.EventsCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return aPath
},
@ -660,12 +661,12 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
folderName,
directoryCaches,
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
resolver := directoryCaches[test.category]
_, _, err = resolver.IDToPath(ctx, folderID, test.useIDForPath)
assert.NoError(t, err)
aw.NoErr(t, err)
parentContainer := folderName
if test.useIDForPath {
@ -679,10 +680,10 @@ func (suite *FolderCacheIntegrationSuite) TestCreateContainerDestination() {
parentContainer,
directoryCaches,
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
_, _, err = resolver.IDToPath(ctx, secondID, test.useIDForPath)
require.NoError(t, err)
aw.MustNoErr(t, err)
p := stdpath.Join(test.folderPrefix, parentContainer)
_, ok := resolver.PathInCache(p)

View File

@ -64,7 +64,7 @@ type DeltaPath struct {
func parseMetadataCollections(
ctx context.Context,
colls []data.RestoreCollection,
errs *fault.Errors,
errs *fault.Bus,
) (CatDeltaPaths, error) {
// cdp stores metadata
cdp := CatDeltaPaths{
@ -168,7 +168,7 @@ func DataCollections(
acct account.M365Config,
su support.StatusUpdater,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
eb, err := selector.ToExchangeBackup()
if err != nil {
@ -178,6 +178,7 @@ func DataCollections(
var (
user = selector.DiscreteOwner
collections = []data.BackupCollection{}
el = errs.Local()
)
cdps, err := parseMetadataCollections(ctx, metadata, errs)
@ -186,7 +187,7 @@ func DataCollections(
}
for _, scope := range eb.Scopes() {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -200,14 +201,14 @@ func DataCollections(
su,
errs)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
collections = append(collections, dcs...)
}
return collections, nil, errs.Err()
return collections, nil, el.Failure()
}
func getterByType(ac api.Client, category path.CategoryType) (addedAndRemovedItemIDsGetter, error) {
@ -234,7 +235,7 @@ func createCollections(
dps DeltaPaths,
ctrlOpts control.Options,
su support.StatusUpdater,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, error) {
var (
allCollections = make([]data.BackupCollection, 0)

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
@ -50,7 +51,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.DeltaURLsFileName, "delta-link"},
},
expect: map[string]DeltaPath{},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "multiple delta urls",
@ -58,7 +59,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.DeltaURLsFileName, "delta-link"},
{graph.DeltaURLsFileName, "delta-link-2"},
},
expectError: assert.Error,
expectError: aw.Err,
},
{
name: "previous path only",
@ -66,7 +67,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.PreviousPathFileName, "prev-path"},
},
expect: map[string]DeltaPath{},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "multiple previous paths",
@ -74,7 +75,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.PreviousPathFileName, "prev-path"},
{graph.PreviousPathFileName, "prev-path-2"},
},
expectError: assert.Error,
expectError: aw.Err,
},
{
name: "delta urls and previous paths",
@ -88,7 +89,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
path: "prev-path",
},
},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "delta urls and empty previous paths",
@ -97,7 +98,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.PreviousPathFileName, ""},
},
expect: map[string]DeltaPath{},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "empty delta urls and previous paths",
@ -106,7 +107,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
{graph.PreviousPathFileName, "prev-path"},
},
expect: map[string]DeltaPath{},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "delta urls with special chars",
@ -120,7 +121,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
path: "prev-path",
},
},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "delta urls with escaped chars",
@ -134,7 +135,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
path: "prev-path",
},
},
expectError: assert.NoError,
expectError: aw.NoErr,
},
{
name: "delta urls with newline char runes",
@ -151,7 +152,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
path: "prev-path",
},
},
expectError: assert.NoError,
expectError: aw.NoErr,
},
}
for _, test := range table {
@ -174,7 +175,7 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
entries,
func(cos *support.ConnectorOperationStatus) {},
)
require.NoError(t, err)
aw.MustNoErr(t, err)
cdps, err := parseMetadataCollections(ctx, []data.RestoreCollection{
data.NotFoundRestoreCollection{Collection: coll},
@ -200,7 +201,6 @@ func (suite *DataCollectionsUnitSuite) TestParseMetadataCollections() {
func newStatusUpdater(t *testing.T, wg *sync.WaitGroup) func(status *support.ConnectorOperationStatus) {
updater := func(status *support.ConnectorOperationStatus) {
defer wg.Done()
assert.Zero(t, status.ErrorCount)
}
return updater
@ -240,7 +240,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() {
acct, err = tester.NewM365Account(suite.T()).M365Config()
)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
tests := []struct {
name string
@ -270,7 +270,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailFetch() {
control.Options{},
func(status *support.ConnectorOperationStatus) {},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService {
@ -298,7 +298,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
acct, err = tester.NewM365Account(suite.T()).M365Config()
)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
tests := []struct {
name string
@ -338,7 +338,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
control.Options{},
func(status *support.ConnectorOperationStatus) {},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Less(t, 1, len(collections), "retrieved metadata and data collections")
var metadata data.BackupCollection
@ -354,7 +354,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
cdps, err := parseMetadataCollections(ctx, []data.RestoreCollection{
data.NotFoundRestoreCollection{Collection: metadata},
}, fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
dps := cdps[test.scope.Category().PathType()]
@ -369,7 +369,7 @@ func (suite *DataCollectionsIntegrationSuite) TestDelta() {
control.Options{},
func(status *support.ConnectorOperationStatus) {},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
// TODO(keepers): this isn't a very useful test at the moment. It needs to
// investigate the items in the original and delta collections to at least
@ -401,7 +401,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
)
acct, err := tester.NewM365Account(t).M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
sel := selectors.NewExchangeBackup(users)
sel.Include(sel.MailFolders([]string{DefaultMailFolder}, selectors.PrefixMatch()))
@ -415,7 +415,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
control.Options{},
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
wg.Add(len(collections))
@ -429,7 +429,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotZero(t, read)
if isMetadata {
@ -438,7 +438,7 @@ func (suite *DataCollectionsIntegrationSuite) TestMailSerializationRegression()
message, err := support.CreateMessageFromBytes(buf.Bytes())
assert.NotNil(t, message)
assert.NoError(t, err)
aw.NoErr(t, err)
}
})
}
@ -454,7 +454,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
defer flush()
acct, err := tester.NewM365Account(suite.T()).M365Config()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
users := []string{suite.user}
@ -483,7 +483,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
control.Options{},
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
wg.Add(len(edcs))
@ -497,7 +497,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
for stream := range edc.Items(ctx, fault.New(true)) {
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(stream.ToReader())
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotZero(t, read)
if isMetadata {
@ -506,7 +506,7 @@ func (suite *DataCollectionsIntegrationSuite) TestContactSerializationRegression
contact, err := support.CreateContactFromBytes(buf.Bytes())
assert.NotNil(t, contact)
assert.NoError(t, err, "error on converting contact bytes: "+buf.String())
aw.NoErr(t, err, "error on converting contact bytes: "+buf.String())
count++
}
@ -530,12 +530,12 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
defer flush()
acct, err := tester.NewM365Account(suite.T()).M365Config()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
users := []string{suite.user}
ac, err := api.NewClient(acct)
require.NoError(suite.T(), err, "creating client")
aw.MustNoErr(suite.T(), err, "creating client")
var (
calID string
@ -554,7 +554,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
return nil
}
require.NoError(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn, fault.New(true)))
aw.MustNoErr(suite.T(), ac.Events().EnumerateContainers(ctx, suite.user, DefaultCalendar, fn, fault.New(true)))
tests := []struct {
name, expected string
@ -591,7 +591,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
control.Options{},
newStatusUpdater(t, &wg),
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
require.Len(t, collections, 2)
wg.Add(len(collections))
@ -610,7 +610,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
buf := &bytes.Buffer{}
read, err := buf.ReadFrom(item.ToReader())
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotZero(t, read)
if isMetadata {
@ -619,7 +619,7 @@ func (suite *DataCollectionsIntegrationSuite) TestEventsSerializationRegression(
event, err := support.CreateEventFromBytes(buf.Bytes())
assert.NotNil(t, event)
assert.NoError(t, err, "creating event from bytes: "+buf.String())
aw.NoErr(t, err, "creating event from bytes: "+buf.String())
}
}

View File

@ -62,7 +62,7 @@ func (ecc *eventCalendarCache) populateEventRoot(ctx context.Context) error {
// @param baseID: ignored. Present to conform to interface
func (ecc *eventCalendarCache) Populate(
ctx context.Context,
errs *fault.Errors,
errs *fault.Bus,
baseID string,
baseContainerPath ...string,
) error {

View File

@ -45,7 +45,7 @@ type itemer interface {
GetItem(
ctx context.Context,
user, itemID string,
errs *fault.Errors,
errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error)
Serialize(
ctx context.Context,
@ -127,7 +127,7 @@ func NewCollection(
// Items utility function to asynchronously execute process to fill data channel with
// M365 exchange objects and returns the data channel
func (col *Collection) Items(ctx context.Context, errs *fault.Errors) <-chan data.Stream {
func (col *Collection) Items(ctx context.Context, errs *fault.Bus) <-chan data.Stream {
go col.streamItems(ctx, errs)
return col.data
}
@ -163,7 +163,7 @@ func (col Collection) DoNotMergeItems() bool {
// streamItems is a utility function that uses col.collectionType to be able to serialize
// all the M365IDs defined in the added field. data channel is closed by this function
func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
func (col *Collection) streamItems(ctx context.Context, errs *fault.Bus) {
var (
success int64
totalBytes int64
@ -177,7 +177,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
)
defer func() {
col.finishPopulation(ctx, int(success), totalBytes, errs.Err())
col.finishPopulation(ctx, int(success), totalBytes, errs.Failure())
}()
if len(col.added)+len(col.removed) > 0 {
@ -226,7 +226,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
// add any new items
for id := range col.added {
if errs.Err() != nil {
if errs.Failure() != nil {
break
}
@ -253,7 +253,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
atomic.AddInt64(&success, 1)
log.With("err", err).Infow("item not found", clues.InErr(err).Slice()...)
} else {
errs.Add(clues.Wrap(err, "fetching item"))
errs.AddRecoverable(clues.Wrap(err, "fetching item"))
}
return
@ -261,7 +261,7 @@ func (col *Collection) streamItems(ctx context.Context, errs *fault.Errors) {
data, err := col.items.Serialize(ctx, item, user, id)
if err != nil {
errs.Add(clues.Wrap(err, "serializing item"))
errs.AddRecoverable(clues.Wrap(err, "serializing item"))
return
}
@ -291,7 +291,7 @@ func getItemWithRetries(
ctx context.Context,
userID, itemID string,
items itemer,
errs *fault.Errors,
errs *fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error) {
item, info, err := items.GetItem(ctx, userID, itemID, errs)
if err != nil {
@ -316,11 +316,10 @@ func (col *Collection) finishPopulation(
support.Backup,
1,
support.CollectionMetrics{
Objects: attempted,
Successes: success,
TotalBytes: totalBytes,
Objects: attempted,
Successes: success,
Bytes: totalBytes,
},
err,
col.fullPath.Folder(false))
logger.Ctx(ctx).Debugw("done streaming items", "status", status.String())

View File

@ -7,13 +7,12 @@ import (
"github.com/microsoft/kiota-abstractions-go/serialization"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -30,7 +29,7 @@ type mockItemer struct {
func (mi *mockItemer) GetItem(
context.Context,
string, string,
*fault.Errors,
*fault.Bus,
) (serialization.Parsable, *details.ExchangeInfo, error) {
mi.getCount++
return nil, nil, mi.getErr
@ -92,7 +91,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeData_FullPath() {
path.EmailCategory,
false,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
edc := Collection{
user: user,
@ -114,7 +113,7 @@ func (suite *ExchangeDataCollectionSuite) TestExchangeDataCollection_NewExchange
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
edc := Collection{
user: name,
@ -128,15 +127,15 @@ func (suite *ExchangeDataCollectionSuite) TestNewCollection_state() {
fooP, err := path.Builder{}.
Append("foo").
ToDataLayerExchangePathForCategory("t", "u", path.EmailCategory, false)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
barP, err := path.Builder{}.
Append("bar").
ToDataLayerExchangePathForCategory("t", "u", path.EmailCategory, false)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
locP, err := path.Builder{}.
Append("human-readable").
ToDataLayerExchangePathForCategory("t", "u", path.EmailCategory, false)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
table := []struct {
name string
@ -199,7 +198,7 @@ func (suite *ExchangeDataCollectionSuite) TestGetItemWithRetries() {
name: "happy",
items: &mockItemer{},
expectErr: func(t *testing.T, err error) {
assert.NoError(t, err)
aw.NoErr(t, err)
},
expectGetCalls: 1,
},
@ -207,16 +206,14 @@ func (suite *ExchangeDataCollectionSuite) TestGetItemWithRetries() {
name: "an error",
items: &mockItemer{getErr: assert.AnError},
expectErr: func(t *testing.T, err error) {
assert.Error(t, err)
aw.Err(t, err)
},
expectGetCalls: 3,
},
{
name: "deleted in flight",
items: &mockItemer{
getErr: graph.ErrDeletedInFlight{
Err: *common.EncapsulateError(assert.AnError),
},
getErr: graph.ErrDeletedInFlight,
},
expectErr: func(t *testing.T, err error) {
assert.True(t, graph.IsErrDeletedInFlight(err), "is ErrDeletedInFlight")

View File

@ -4,12 +4,12 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -36,7 +36,7 @@ func (suite *CacheResolverSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
}
@ -46,10 +46,10 @@ func (suite *CacheResolverSuite) TestPopulate() {
defer flush()
ac, err := api.NewClient(suite.credentials)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
cal, err := ac.Events().GetContainerByID(ctx, tester.M365UserID(suite.T()), DefaultCalendar)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
eventFunc := func(t *testing.T) graph.ContainerResolver {
return &eventCalendarCache{
@ -120,7 +120,7 @@ func (suite *CacheResolverSuite) TestPopulate() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
resolver := test.resolverFunc(t)
require.NoError(t, resolver.Populate(ctx, fault.New(true), test.root, test.basePath))
aw.MustNoErr(t, resolver.Populate(ctx, fault.New(true), test.root, test.basePath))
_, isFound := resolver.PathInCache(test.folderInCache)
test.canFind(t, isFound)

View File

@ -4,13 +4,13 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type ExchangeIteratorSuite struct {
@ -31,7 +31,7 @@ func (suite *ExchangeIteratorSuite) TestDisplayable() {
t := suite.T()
bytes := mockconnector.GetMockContactBytes("Displayable")
contact, err := support.CreateContactFromBytes(bytes)
require.NoError(t, err)
aw.MustNoErr(t, err)
aDisplayable, ok := contact.(graph.Displayable)
assert.True(t, ok)
@ -43,7 +43,7 @@ func (suite *ExchangeIteratorSuite) TestDescendable() {
t := suite.T()
bytes := mockconnector.GetMockMessageBytes("Descendable")
message, err := support.CreateMessageFromBytes(bytes)
require.NoError(t, err)
aw.MustNoErr(t, err)
aDescendable, ok := message.(graph.Descendable)
assert.True(t, ok)

View File

@ -72,7 +72,7 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context) error {
// for the base container in the cache.
func (mc *mailFolderCache) Populate(
ctx context.Context,
errs *fault.Errors,
errs *fault.Bus,
baseID string,
baseContainerPath ...string,
) error {

View File

@ -5,11 +5,11 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -48,7 +48,7 @@ func (suite *MailFolderCacheIntegrationSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
}
@ -83,7 +83,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
ac, err := api.NewClient(suite.credentials)
require.NoError(t, err)
aw.MustNoErr(t, err)
acm := ac.Mail()
@ -93,10 +93,10 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
getter: acm,
}
require.NoError(t, mfc.Populate(ctx, fault.New(true), test.root, test.path...))
aw.MustNoErr(t, mfc.Populate(ctx, fault.New(true), test.root, test.path...))
p, l, err := mfc.IDToPath(ctx, testFolderID, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
t.Logf("Path: %s\n", p.String())
t.Logf("Location: %s\n", l.String())

View File

@ -6,15 +6,14 @@ import (
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -43,18 +42,18 @@ func (suite *ExchangeRestoreSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
suite.ac, err = api.NewClient(m365)
require.NoError(t, err)
aw.MustNoErr(t, err)
adpt, err := graph.CreateAdapter(m365.AzureTenantID, m365.AzureClientID, m365.AzureClientSecret)
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.gs = graph.NewService(adpt)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
}
// TestRestoreContact ensures contact object can be created, placed into
@ -71,14 +70,14 @@ func (suite *ExchangeRestoreSuite) TestRestoreContact() {
)
aFolder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
folderID := *aFolder.GetId()
defer func() {
// Remove the folder containing contact prior to exiting test
err = suite.ac.Contacts().DeleteContainer(ctx, userID, folderID)
assert.NoError(t, err)
aw.NoErr(t, err)
}()
info, err := RestoreExchangeContact(
@ -88,7 +87,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreContact() {
control.Copy,
folderID,
userID)
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
aw.NoErr(t, err)
assert.NotNil(t, info, "contact item info")
}
@ -105,14 +104,14 @@ func (suite *ExchangeRestoreSuite) TestRestoreEvent() {
)
calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, name)
require.NoError(t, err)
aw.MustNoErr(t, err)
calendarID := *calendar.GetId()
defer func() {
// Removes calendar containing events created during the test
err = suite.ac.Events().DeleteContainer(ctx, userID, calendarID)
assert.NoError(t, err)
aw.NoErr(t, err)
}()
info, err := RestoreExchangeEvent(ctx,
@ -122,7 +121,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreEvent() {
calendarID,
userID,
fault.New(true))
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
aw.NoErr(t, err)
assert.NotNil(t, info, "event item info")
}
@ -135,10 +134,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
t := suite.T()
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
service, err := createService(m365)
require.NoError(t, err)
aw.MustNoErr(t, err)
deleters := map[path.CategoryType]containerDeleter{
path.EmailCategory: suite.ac.Mail(),
@ -161,7 +160,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailObject: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -173,7 +172,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -185,7 +184,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreEventItemAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -197,7 +196,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailItemAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -212,7 +211,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailBasicItemAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -227,7 +226,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "ItemMailAttachmentwAttachment " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -242,7 +241,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "ItemMailAttachment_Contact " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -254,7 +253,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreNestedEventItemAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -266,7 +265,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithLargeAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -278,7 +277,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithAttachments: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -290,7 +289,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreMailwithReferenceAttachment: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Mail().CreateMailFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -303,7 +302,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
folderName := "TestRestoreContactObject: " + common.FormatSimpleDateTime(now)
folder, err := suite.ac.Contacts().CreateContactFolder(ctx, userID, folderName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *folder.GetId()
},
@ -315,7 +314,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject: " + common.FormatSimpleDateTime(now)
calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *calendar.GetId()
},
@ -327,7 +326,7 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination: func(t *testing.T, ctx context.Context) string {
calendarName := "TestRestoreEventObject_" + common.FormatSimpleDateTime(now)
calendar, err := suite.ac.Events().CreateCalendar(ctx, userID, calendarName)
require.NoError(t, err)
aw.MustNoErr(t, err)
return *calendar.GetId()
},
@ -349,10 +348,10 @@ func (suite *ExchangeRestoreSuite) TestRestoreExchangeObject() {
destination,
userID,
fault.New(true))
assert.NoError(t, err, support.ConnectorStackErrorTrace(err))
aw.NoErr(t, err)
assert.NotNil(t, info, "item info was not populated")
assert.NotNil(t, deleters)
assert.NoError(t, deleters[test.category].DeleteContainer(ctx, userID, destination))
aw.NoErr(t, deleters[test.category].DeleteContainer(ctx, userID, destination))
})
}
}

View File

@ -36,7 +36,7 @@ func createService(credentials account.M365Config) (*graph.Service, error) {
func PopulateExchangeContainerResolver(
ctx context.Context,
qp graph.QueryParams,
errs *fault.Errors,
errs *fault.Bus,
) (graph.ContainerResolver, error) {
var (
res graph.ContainerResolver

View File

@ -39,7 +39,7 @@ func filterContainersAndFillCollections(
scope selectors.ExchangeScope,
dps DeltaPaths,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) error {
var (
// folder ID -> delta url or folder path lookups
@ -68,9 +68,11 @@ func filterContainersAndFillCollections(
return err
}
el := errs.Local()
for _, c := range resolver.Items() {
if errs.Err() != nil {
return errs.Err()
if el.Failure() != nil {
return el.Failure()
}
cID := *c.GetId()
@ -91,7 +93,7 @@ func filterContainersAndFillCollections(
if len(prevPathStr) > 0 {
if prevPath, err = pathFromPrevString(prevPathStr); err != nil {
logger.Ctx(ctx).With("err", err).Errorw("parsing prev path", clues.InErr(err).Slice()...)
logger.CtxErr(ctx, err).Error("parsing prev path")
// if the previous path is unusable, then the delta must be, too.
prevDelta = ""
}
@ -100,7 +102,7 @@ func filterContainersAndFillCollections(
added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ctx, qp.ResourceOwner, cID, prevDelta)
if err != nil {
if !graph.IsErrDeletedInFlight(err) {
errs.Add(err)
el.AddRecoverable(err)
continue
}
@ -155,8 +157,12 @@ func filterContainersAndFillCollections(
// in the `previousPath` set, but does not exist in the current container
// resolver (which contains all the resource owners' current containers).
for id, p := range tombstones {
if el.Failure() != nil {
return el.Failure()
}
if collections[id] != nil {
errs.Add(clues.Wrap(err, "conflict: tombstone exists for a live collection").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "conflict: tombstone exists for a live collection").WithClues(ctx))
continue
}
@ -169,7 +175,7 @@ func filterContainersAndFillCollections(
prevPath, err := pathFromPrevString(p)
if err != nil {
// technically shouldn't ever happen. But just in case...
logger.Ctx(ctx).With("err", err).Errorw("parsing tombstone prev path", clues.InErr(err).Slice()...)
logger.CtxErr(ctx, err).Error("parsing tombstone prev path")
continue
}
@ -205,15 +211,14 @@ func filterContainersAndFillCollections(
path.ExchangeService,
qp.Category,
entries,
statusUpdater,
)
statusUpdater)
if err != nil {
return clues.Wrap(err, "making metadata collection")
}
collections["metadata"] = col
return errs.Err()
return el.Failure()
}
// produces a set of id:path pairs from the deltapaths map.

View File

@ -9,12 +9,12 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -91,8 +91,8 @@ func (m mockResolver) DestinationNameToID(dest string) string { return m.added[d
func (m mockResolver) IDToPath(context.Context, string, bool) (*path.Builder, *path.Builder, error) {
return nil, nil, nil
}
func (m mockResolver) PathInCache(string) (string, bool) { return "", false }
func (m mockResolver) Populate(context.Context, *fault.Errors, string, ...string) error { return nil }
func (m mockResolver) PathInCache(string) (string, bool) { return "", false }
func (m mockResolver) Populate(context.Context, *fault.Bus, string, ...string) error { return nil }
// ---------------------------------------------------------------------------
// tests
@ -110,7 +110,7 @@ func TestServiceIteratorsSuite(t *testing.T) {
func (suite *ServiceIteratorsSuite) SetupSuite() {
a := tester.NewMockM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
suite.creds = m365
}
@ -140,7 +140,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
added: []string{"a1", "a2", "a3"},
removed: []string{"r1", "r2", "r3"},
newDelta: api.DeltaUpdate{URL: "delta_url"},
err: graph.ErrDeletedInFlight{Err: *common.EncapsulateError(assert.AnError)},
err: graph.ErrDeletedInFlight,
}
container1 = mockContainer{
id: strPtr("1"),
@ -172,7 +172,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 1,
expectMetadataColls: 1,
},
@ -184,7 +184,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 2,
expectMetadataColls: 1,
},
@ -196,7 +196,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 2,
expectMetadataColls: 1,
},
@ -208,7 +208,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1, container2),
scope: selectors.NewExchangeBackup(nil).MailFolders(selectors.None())[0],
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 0,
expectMetadataColls: 1,
},
@ -219,7 +219,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 1,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
@ -231,7 +231,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 0,
expectMetadataColls: 1,
},
@ -243,7 +243,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 2,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
@ -256,7 +256,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 1,
expectMetadataColls: 1,
},
@ -269,7 +269,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
resolver: newMockResolver(container1, container2),
scope: allScope,
failFast: true,
expectErr: assert.NoError,
expectErr: aw.NoErr,
expectNewColls: 2,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
@ -283,7 +283,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
resolver: newMockResolver(container1, container2),
scope: allScope,
failFast: true,
expectErr: assert.Error,
expectErr: aw.Err,
expectNewColls: 0,
expectMetadataColls: 0,
},
@ -459,7 +459,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_repea
dps,
control.Options{FailFast: true},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
// collection assertions
@ -534,7 +534,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_incre
p, err := path.Builder{}.
Append(at...).
ToDataLayerExchangePathForCategory(tenantID, userID, cat, false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return p
}
@ -811,7 +811,7 @@ func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_incre
test.dps,
control.Options{},
fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
metadatas := 0
for _, c := range collections {

View File

@ -37,7 +37,7 @@ func RestoreExchangeObject(
policy control.CollisionPolicy,
service graph.Servicer,
destination, user string,
errs *fault.Errors,
errs *fault.Bus,
) (*details.ExchangeInfo, error) {
if policy != control.Copy {
return nil, clues.Wrap(clues.New(policy.String()), "policy not supported for Exchange restore").WithClues(ctx)
@ -70,14 +70,14 @@ func RestoreExchangeContact(
) (*details.ExchangeInfo, error) {
contact, err := support.CreateContactFromBytes(bits)
if err != nil {
return nil, clues.Wrap(err, "creating contact from bytes").WithClues(ctx)
return nil, graph.Wrap(ctx, err, "creating contact from bytes")
}
ctx = clues.Add(ctx, "item_id", ptr.Val(contact.GetId()))
response, err := service.Client().UsersById(user).ContactFoldersById(destination).Contacts().Post(ctx, contact, nil)
if err != nil {
return nil, clues.Wrap(err, "uploading Contact").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "uploading Contact")
}
if response == nil {
@ -102,7 +102,7 @@ func RestoreExchangeEvent(
service graph.Servicer,
cp control.CollisionPolicy,
destination, user string,
errs *fault.Errors,
errs *fault.Bus,
) (*details.ExchangeInfo, error) {
event, err := support.CreateEventFromBytes(bits)
if err != nil {
@ -112,6 +112,7 @@ func RestoreExchangeEvent(
ctx = clues.Add(ctx, "item_id", ptr.Val(event.GetId()))
var (
el = errs.Local()
transformedEvent = support.ToEventSimplified(event)
attached []models.Attachmentable
)
@ -124,7 +125,7 @@ func RestoreExchangeEvent(
response, err := service.Client().UsersById(user).CalendarsById(destination).Events().Post(ctx, transformedEvent, nil)
if err != nil {
return nil, clues.Wrap(err, "uploading event").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "uploading event")
}
if response == nil {
@ -139,19 +140,19 @@ func RestoreExchangeEvent(
}
for _, attach := range attached {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
if err := uploadAttachment(ctx, uploader, attach); err != nil {
errs.Add(err)
el.AddRecoverable(err)
}
}
info := api.EventInfo(event)
info.Size = int64(len(bits))
return info, errs.Err()
return info, el.Failure()
}
// RestoreMailMessage utility function to place an exchange.Mail
@ -166,7 +167,7 @@ func RestoreMailMessage(
service graph.Servicer,
cp control.CollisionPolicy,
destination, user string,
errs *fault.Errors,
errs *fault.Bus,
) (*details.ExchangeInfo, error) {
// Creates messageable object from original bytes
originalMessage, err := support.CreateMessageFromBytes(bits)
@ -239,7 +240,7 @@ func SendMailToBackStore(
service graph.Servicer,
user, destination string,
message models.Messageable,
errs *fault.Errors,
errs *fault.Bus,
) error {
attached := message.GetAttachments()
@ -248,24 +249,26 @@ func SendMailToBackStore(
response, err := service.Client().UsersById(user).MailFoldersById(destination).Messages().Post(ctx, message, nil)
if err != nil {
return clues.Wrap(err, "restoring mail").WithClues(ctx).With(graph.ErrData(err)...)
return graph.Wrap(ctx, err, "restoring mail")
}
if response == nil {
return clues.New("nil response from post").WithClues(ctx)
}
id := ptr.Val(response.GetId())
uploader := &mailAttachmentUploader{
userID: user,
folderID: destination,
itemID: id,
service: service,
}
var (
el = errs.Local()
id = ptr.Val(response.GetId())
uploader = &mailAttachmentUploader{
userID: user,
folderID: destination,
itemID: id,
service: service,
}
)
for _, attachment := range attached {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -273,20 +276,20 @@ func SendMailToBackStore(
if ptr.Val(attachment.GetOdataType()) == "#microsoft.graph.itemAttachment" {
name := ptr.Val(attachment.GetName())
logger.Ctx(ctx).
With("err", err, "attachment_name", name).
Infow("mail upload failed", clues.InErr(err).Slice()...)
logger.CtxErr(ctx, err).
With("attachment_name", name).
Info("mail upload failed")
continue
}
errs.Add(errors.Wrap(err, "uploading mail attachment"))
el.AddRecoverable(errors.Wrap(err, "uploading mail attachment"))
break
}
}
return errs.Err()
return el.Failure()
}
// RestoreExchangeDataCollections restores M365 objects in data.RestoreCollection to MSFT
@ -299,7 +302,7 @@ func RestoreExchangeDataCollections(
dest control.RestoreDestination,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Errors,
errs *fault.Bus,
) (*support.ConnectorOperationStatus, error) {
var (
directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerResolver)
@ -307,6 +310,7 @@ func RestoreExchangeDataCollections(
userID string
// TODO policy to be updated from external source after completion of refactoring
policy = control.Copy
el = errs.Local()
)
if len(dcs) > 0 {
@ -315,8 +319,8 @@ func RestoreExchangeDataCollections(
}
for _, dc := range dcs {
if errs.Err() != nil {
return nil, errs.Err()
if el.Failure() != nil {
break
}
userCaches := directoryCaches[userID]
@ -333,13 +337,13 @@ func RestoreExchangeDataCollections(
userCaches,
errs)
if err != nil {
errs.Add(clues.Wrap(err, "creating destination").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "creating destination").WithClues(ctx))
continue
}
temp, canceled := restoreCollection(ctx, gs, dc, containerID, policy, deets, errs)
metrics.Combine(temp)
metrics = support.CombineMetrics(metrics, temp)
if canceled {
break
@ -351,10 +355,9 @@ func RestoreExchangeDataCollections(
support.Restore,
len(dcs),
metrics,
errs.Err(),
dest.ContainerName)
return status, errs.Err()
return status, el.Failure()
}
// restoreCollection handles restoration of an individual collection.
@ -365,7 +368,7 @@ func restoreCollection(
folderID string,
policy control.CollisionPolicy,
deets *details.Builder,
errs *fault.Errors,
errs *fault.Bus,
) (support.CollectionMetrics, bool) {
ctx, end := D.Span(ctx, "gc:exchange:restoreCollection", D.Label("path", dc.FullPath()))
defer end()
@ -396,11 +399,11 @@ func restoreCollection(
for {
select {
case <-ctx.Done():
errs.Add(clues.Wrap(ctx.Err(), "context cancelled").WithClues(ctx))
errs.AddRecoverable(clues.Wrap(ctx.Err(), "context cancelled").WithClues(ctx))
return metrics, true
case itemData, ok := <-items:
if !ok || errs.Err() != nil {
if !ok || errs.Failure() != nil {
return metrics, false
}
@ -412,7 +415,7 @@ func restoreCollection(
_, err := buf.ReadFrom(itemData.ToReader())
if err != nil {
errs.Add(clues.Wrap(err, "reading item bytes").WithClues(ictx))
errs.AddRecoverable(clues.Wrap(err, "reading item bytes").WithClues(ictx))
continue
}
@ -428,16 +431,16 @@ func restoreCollection(
user,
errs)
if err != nil {
errs.Add(err)
errs.AddRecoverable(err)
continue
}
metrics.TotalBytes += int64(len(byteArray))
metrics.Bytes += int64(len(byteArray))
metrics.Successes++
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
errs.Add(clues.Wrap(err, "building full path with item").WithClues(ctx))
errs.AddRecoverable(clues.Wrap(err, "building full path with item").WithClues(ctx))
continue
}
@ -472,7 +475,7 @@ func CreateContainerDestination(
directory path.Path,
destination string,
caches map[path.CategoryType]graph.ContainerResolver,
errs *fault.Errors,
errs *fault.Bus,
) (string, error) {
var (
newCache = false
@ -585,7 +588,7 @@ func establishMailRestoreLocation(
mfc graph.ContainerResolver,
user string,
isNewCache bool,
errs *fault.Errors,
errs *fault.Bus,
) (string, error) {
// Process starts with the root folder in order to recreate
// the top-level folder with the same tactic
@ -606,7 +609,7 @@ func establishMailRestoreLocation(
temp, err := ac.Mail().CreateMailFolderWithParent(ctx, user, folder, folderID)
if err != nil {
// Should only error if cache malfunctions or incorrect parameters
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return "", err
}
folderID = *temp.GetId()
@ -644,7 +647,7 @@ func establishContactsRestoreLocation(
cfc graph.ContainerResolver,
user string,
isNewCache bool,
errs *fault.Errors,
errs *fault.Bus,
) (string, error) {
cached, ok := cfc.PathInCache(folders[0])
if ok {
@ -655,7 +658,7 @@ func establishContactsRestoreLocation(
temp, err := ac.Contacts().CreateContactFolder(ctx, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return "", err
}
folderID := *temp.GetId()
@ -680,7 +683,7 @@ func establishEventsRestoreLocation(
ecc graph.ContainerResolver, // eventCalendarCache
user string,
isNewCache bool,
errs *fault.Errors,
errs *fault.Bus,
) (string, error) {
// Need to prefix with the "Other Calendars" folder so lookup happens properly.
cached, ok := ecc.PathInCache(folders[0])
@ -692,7 +695,7 @@ func establishEventsRestoreLocation(
temp, err := ac.Events().CreateCalendar(ctx, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return "", err
}
folderID := *temp.GetId()

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -31,7 +32,7 @@ func (suite *BetaClientSuite) SetupSuite() {
t := suite.T()
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
}
@ -44,7 +45,7 @@ func (suite *BetaClientSuite) TestCreateBetaClient() {
suite.credentials.AzureClientSecret,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
client := NewBetaClient(adpt)
assert.NotNil(t, client)
@ -64,7 +65,7 @@ func (suite *BetaClientSuite) TestBasicClientGetFunctionality() {
suite.credentials.AzureClientSecret,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
client := NewBetaClient(adpt)
require.NotNil(t, client)
@ -73,7 +74,7 @@ func (suite *BetaClientSuite) TestBasicClientGetFunctionality() {
collection, err := client.SitesById(siteID).Pages().Get(ctx, nil)
// Ensures that the client is able to receive data from beta
// Not Registered Error: content type application/json does not have a factory registered to be parsed
require.NoError(t, err)
aw.MustNoErr(t, err)
for _, page := range collection.GetValue() {
assert.NotNil(t, page, "betasdk call for page does not return value.")

View File

@ -65,7 +65,7 @@ type ContainerResolver interface {
// @param ctx is necessary param for Graph API tracing
// @param baseFolderID represents the M365ID base that the resolver will
// conclude its search. Default input is "".
Populate(ctx context.Context, errs *fault.Errors, baseFolderID string, baseContainerPather ...string) error
Populate(ctx context.Context, errs *fault.Bus, baseFolderID string, baseContainerPather ...string) error
// PathInCache performs a look up of a path reprensentation
// and returns the m365ID of directory iff the pathString

View File

@ -6,12 +6,14 @@ import (
"net/http"
"net/url"
"os"
"strings"
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
"github.com/pkg/errors"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/common/ptr"
)
// ---------------------------------------------------------------------------
@ -29,26 +31,36 @@ const (
errCodeMailboxNotEnabledForRESTAPI = "MailboxNotEnabledForRESTAPI"
)
var (
Err401Unauthorized = errors.New("401 unauthorized")
// normally the graph client will catch this for us, but in case we
// run our own client Do(), we need to translate it to a timeout type
// failure locally.
Err429TooManyRequests = errors.New("429 too many requests")
Err503ServiceUnavailable = errors.New("503 Service Unavailable")
Err504GatewayTimeout = errors.New("504 Gateway Timeout")
Err500InternalServerError = errors.New("500 Internal Server Error")
const (
mysiteURLNotFound = "unable to retrieve user's mysite URL"
mysiteNotFound = "user's mysite not found"
)
// The folder or item was deleted between the time we identified
// it and when we tried to fetch data for it.
type ErrDeletedInFlight struct {
common.Err
var Labels = struct {
MysiteNotFound string
}{
MysiteNotFound: "mysite_not_found",
}
var (
// The folder or item was deleted between the time we identified
// it and when we tried to fetch data for it.
ErrDeletedInFlight = clues.New("deleted in flight")
// Delta tokens can be desycned or expired. In either case, the token
// becomes invalid, and cannot be used again.
// https://learn.microsoft.com/en-us/graph/errors#code-property
ErrInvalidDelta = clues.New("inalid delta token")
// Timeout errors are identified for tracking the need to retry calls.
// Other delay errors, like throttling, are already handled by the
// graph client's built-in retries.
// https://github.com/microsoftgraph/msgraph-sdk-go/issues/302
ErrTimeout = clues.New("communication timeout")
)
func IsErrDeletedInFlight(err error) bool {
e := ErrDeletedInFlight{}
if errors.As(err, &e) {
if errors.Is(err, ErrDeletedInFlight) {
return true
}
@ -59,103 +71,39 @@ func IsErrDeletedInFlight(err error) bool {
return false
}
// Delta tokens can be desycned or expired. In either case, the token
// becomes invalid, and cannot be used again.
// https://learn.microsoft.com/en-us/graph/errors#code-property
type ErrInvalidDelta struct {
common.Err
}
func IsErrInvalidDelta(err error) bool {
e := ErrInvalidDelta{}
if errors.As(err, &e) {
return true
}
if hasErrorCode(err, errCodeSyncStateNotFound, errCodeResyncRequired) {
return true
}
return false
return hasErrorCode(err, errCodeSyncStateNotFound, errCodeResyncRequired) ||
errors.Is(err, ErrInvalidDelta)
}
func IsErrExchangeMailFolderNotFound(err error) bool {
return hasErrorCode(err, errCodeResourceNotFound, errCodeMailboxNotEnabledForRESTAPI)
}
// Timeout errors are identified for tracking the need to retry calls.
// Other delay errors, like throttling, are already handled by the
// graph client's built-in retries.
// https://github.com/microsoftgraph/msgraph-sdk-go/issues/302
type ErrTimeout struct {
common.Err
}
func IsErrTimeout(err error) bool {
e := ErrTimeout{}
if errors.As(err, &e) {
return true
}
if errors.Is(err, context.DeadlineExceeded) || os.IsTimeout(err) || errors.Is(err, http.ErrHandlerTimeout) {
return true
}
switch err := err.(type) {
case *url.Error:
return err.Timeout()
default:
return false
}
}
type ErrThrottled struct {
common.Err
}
func IsErrThrottled(err error) bool {
if errors.Is(err, Err429TooManyRequests) {
return true
}
if hasErrorCode(err, errCodeActivityLimitReached) {
return true
}
e := ErrThrottled{}
return errors.As(err, &e)
}
type ErrUnauthorized struct {
common.Err
return errors.Is(err, ErrTimeout) ||
errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, http.ErrHandlerTimeout) ||
os.IsTimeout(err)
}
func IsErrUnauthorized(err error) bool {
// TODO: refine this investigation. We don't currently know if
// a specific item download url expired, or if the full connection
// auth expired.
if errors.Is(err, Err401Unauthorized) {
return true
}
e := ErrUnauthorized{}
return errors.As(err, &e)
return clues.HasLabel(err, LabelStatus(http.StatusUnauthorized))
}
type ErrInternalServerError struct {
common.Err
}
func IsInternalServerError(err error) bool {
if errors.Is(err, Err500InternalServerError) {
return true
}
e := ErrInternalServerError{}
return errors.As(err, &e)
// LabelStatus transforms the provided statusCode into
// a standard label that can be attached to a clues error
// and later reviewed when checking error statuses.
func LabelStatus(statusCode int) string {
return fmt.Sprintf("status_code_%d", statusCode)
}
// ---------------------------------------------------------------------------
@ -179,43 +127,74 @@ func hasErrorCode(err error, codes ...string) bool {
return slices.Contains(codes, *oDataError.GetError().GetCode())
}
// ErrData is a helper function that extracts ODataError metadata from
// the error. If the error is not an ODataError type, returns an empty
// slice. The returned value is guaranteed to be an even-length pairing
// of key, value tuples.
func ErrData(e error) []any {
result := make([]any, 0)
// Wrap is a helper function that extracts ODataError metadata from
// the error. If the error is not an ODataError type, returns the error.
func Wrap(ctx context.Context, e error, msg string) *clues.Err {
if e == nil {
return result
return nil
}
odErr, ok := e.(odataerrors.ODataErrorable)
if !ok {
return result
return clues.Wrap(e, msg).WithClues(ctx)
}
// Get MainError
mainErr := odErr.GetError()
data, innerMsg := ErrData(odErr)
result = appendIf(result, "odataerror_code", mainErr.GetCode())
result = appendIf(result, "odataerror_message", mainErr.GetMessage())
result = appendIf(result, "odataerror_target", mainErr.GetTarget())
return setLabels(clues.Wrap(e, msg).WithClues(ctx).With(data...), innerMsg)
}
// Stack is a helper function that extracts ODataError metadata from
// the error. If the error is not an ODataError type, returns the error.
func Stack(ctx context.Context, e error) *clues.Err {
if e == nil {
return nil
}
odErr, ok := e.(odataerrors.ODataErrorable)
if !ok {
return clues.Stack(e).WithClues(ctx)
}
data, innerMsg := ErrData(odErr)
return setLabels(clues.Stack(e).WithClues(ctx).With(data...), innerMsg)
}
func setLabels(err *clues.Err, msg string) *clues.Err {
if strings.Contains(msg, mysiteNotFound) || strings.Contains(msg, mysiteURLNotFound) {
err = err.Label(Labels.MysiteNotFound)
}
return err
}
func ErrData(err odataerrors.ODataErrorable) ([]any, string) {
data := make([]any, 0)
// Get MainError
mainErr := err.GetError()
data = appendIf(data, "odataerror_code", mainErr.GetCode())
data = appendIf(data, "odataerror_message", mainErr.GetMessage())
data = appendIf(data, "odataerror_target", mainErr.GetTarget())
msgConcat := ptr.Val(mainErr.GetMessage())
for i, d := range mainErr.GetDetails() {
pfx := fmt.Sprintf("odataerror_details_%d_", i)
result = appendIf(result, pfx+"code", d.GetCode())
result = appendIf(result, pfx+"message", d.GetMessage())
result = appendIf(result, pfx+"target", d.GetTarget())
data = appendIf(data, pfx+"code", d.GetCode())
data = appendIf(data, pfx+"message", d.GetMessage())
data = appendIf(data, pfx+"target", d.GetTarget())
msgConcat += ptr.Val(d.GetMessage())
}
inner := mainErr.GetInnererror()
if inner != nil {
result = appendIf(result, "odataerror_inner_cli_req_id", inner.GetClientRequestId())
result = appendIf(result, "odataerror_inner_req_id", inner.GetRequestId())
data = appendIf(data, "odataerror_inner_cli_req_id", inner.GetClientRequestId())
data = appendIf(data, "odataerror_inner_req_id", inner.GetRequestId())
}
return result
return data, strings.ToLower(msgConcat)
}
func appendIf(a []any, k string, v *string) []any {

View File

@ -7,8 +7,6 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
)
type GraphErrorsUnitSuite struct {
@ -46,7 +44,7 @@ func (suite *GraphErrorsUnitSuite) TestIsErrDeletedInFlight() {
},
{
name: "as",
err: ErrDeletedInFlight{Err: *common.EncapsulateError(assert.AnError)},
err: ErrDeletedInFlight,
expect: assert.True,
},
{
@ -90,7 +88,7 @@ func (suite *GraphErrorsUnitSuite) TestIsErrInvalidDelta() {
},
{
name: "as",
err: ErrInvalidDelta{Err: *common.EncapsulateError(assert.AnError)},
err: ErrInvalidDelta,
expect: assert.True,
},
{
@ -129,7 +127,7 @@ func (suite *GraphErrorsUnitSuite) TestIsErrTimeout() {
},
{
name: "as",
err: ErrTimeout{Err: *common.EncapsulateError(assert.AnError)},
err: ErrTimeout,
expect: assert.True,
},
{
@ -144,105 +142,3 @@ func (suite *GraphErrorsUnitSuite) TestIsErrTimeout() {
})
}
}
func (suite *GraphErrorsUnitSuite) TestIsErrThrottled() {
table := []struct {
name string
err error
expect assert.BoolAssertionFunc
}{
{
name: "nil",
err: nil,
expect: assert.False,
},
{
name: "non-matching",
err: assert.AnError,
expect: assert.False,
},
{
name: "as",
err: ErrThrottled{Err: *common.EncapsulateError(assert.AnError)},
expect: assert.True,
},
{
name: "is429",
err: Err429TooManyRequests,
expect: assert.True,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
test.expect(t, IsErrThrottled(test.err))
})
}
}
func (suite *GraphErrorsUnitSuite) TestIsErrUnauthorized() {
table := []struct {
name string
err error
expect assert.BoolAssertionFunc
}{
{
name: "nil",
err: nil,
expect: assert.False,
},
{
name: "non-matching",
err: assert.AnError,
expect: assert.False,
},
{
name: "as",
err: ErrUnauthorized{Err: *common.EncapsulateError(assert.AnError)},
expect: assert.True,
},
{
name: "is429",
err: Err401Unauthorized,
expect: assert.True,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
test.expect(t, IsErrUnauthorized(test.err))
})
}
}
func (suite *GraphErrorsUnitSuite) TestIsInternalServerError() {
table := []struct {
name string
err error
expect assert.BoolAssertionFunc
}{
{
name: "nil",
err: nil,
expect: assert.False,
},
{
name: "non-matching",
err: assert.AnError,
expect: assert.False,
},
{
name: "as",
err: ErrInternalServerError{Err: *common.EncapsulateError(assert.AnError)},
expect: assert.True,
},
{
name: "is429",
err: Err500InternalServerError,
expect: assert.True,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
test.expect(t, IsInternalServerError(test.err))
})
}
}

View File

@ -134,7 +134,7 @@ func (md MetadataCollection) DoNotMergeItems() bool {
func (md MetadataCollection) Items(
ctx context.Context,
errs *fault.Errors,
errs *fault.Bus,
) <-chan data.Stream {
res := make(chan data.Stream)
@ -150,11 +150,10 @@ func (md MetadataCollection) Items(
support.Backup,
1,
support.CollectionMetrics{
Objects: len(md.items),
Successes: len(md.items),
TotalBytes: totalBytes,
Objects: len(md.items),
Successes: len(md.items),
Bytes: totalBytes,
},
nil,
md.fullPath.Folder(false),
)

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -35,7 +36,7 @@ func (suite *MetadataCollectionUnitSuite) TestFullPath() {
path.EmailCategory,
false,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
c := NewMetadataCollection(p, nil, nil)
@ -78,14 +79,14 @@ func (suite *MetadataCollectionUnitSuite) TestItems() {
path.EmailCategory,
false,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
c := NewMetadataCollection(
p,
items,
func(c *support.ConnectorOperationStatus) {
assert.Equal(t, len(itemNames), c.ObjectCount)
assert.Equal(t, len(itemNames), c.Successful)
assert.Equal(t, len(itemNames), c.Metrics.Objects)
assert.Equal(t, len(itemNames), c.Metrics.Successes)
},
)
@ -96,7 +97,7 @@ func (suite *MetadataCollectionUnitSuite) TestItems() {
gotNames = append(gotNames, s.UUID())
buf, err := io.ReadAll(s.ToReader())
if !assert.NoError(t, err) {
if !aw.NoErr(t, err) {
continue
}
@ -125,7 +126,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() {
cat: path.EmailCategory,
metadata: NewMetadataEntry("", nil),
collectionCheck: assert.Nil,
errCheck: assert.Error,
errCheck: aw.Err,
},
{
name: "Tokens",
@ -138,7 +139,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() {
"hola": "mundo",
}),
collectionCheck: assert.NotNil,
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "BadCategory",
@ -151,7 +152,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() {
"hola": "mundo",
}),
collectionCheck: assert.Nil,
errCheck: assert.Error,
errCheck: aw.Err,
},
}
@ -187,7 +188,7 @@ func (suite *MetadataCollectionUnitSuite) TestMakeMetadataCollection() {
itemCount++
err := decoder.Decode(&gotMap)
if !assert.NoError(t, err) {
if !aw.NoErr(t, err) {
continue
}

View File

@ -6,7 +6,6 @@ import (
"strconv"
"time"
"github.com/alcionai/clues"
backoff "github.com/cenkalti/backoff/v4"
khttp "github.com/microsoft/kiota-http-go"
)
@ -50,10 +49,7 @@ func (middleware RetryHandler) retryRequest(
response, err := pipeline.Next(req, middlewareIndex)
if err != nil && !IsErrTimeout(err) {
return response, clues.Stack(err).
WithClues(ctx).
With("retry_count", executionCount).
With(ErrData(err)...)
return response, Stack(ctx, err).With("retry_count", executionCount)
}
return middleware.retryRequest(ctx,
@ -68,10 +64,7 @@ func (middleware RetryHandler) retryRequest(
}
if respErr != nil {
return nil, clues.Stack(respErr).
WithClues(ctx).
With("retry_count", executionCount).
With(ErrData(respErr)...)
return nil, Stack(ctx, respErr).With("retry_count", executionCount)
}
return resp, nil

View File

@ -8,7 +8,6 @@ import (
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/alcionai/clues"
backoff "github.com/cenkalti/backoff/v4"
"github.com/microsoft/kiota-abstractions-go/serialization"
ka "github.com/microsoft/kiota-authentication-azure-go"
@ -336,7 +335,7 @@ func (middleware RetryHandler) Intercept(
response, err := pipeline.Next(req, middlewareIndex)
if err != nil && !IsErrTimeout(err) {
return response, clues.Stack(err).WithClues(ctx).With(ErrData(err)...)
return response, Stack(ctx, err)
}
exponentialBackOff := backoff.NewExponentialBackOff()
@ -354,7 +353,7 @@ func (middleware RetryHandler) Intercept(
exponentialBackOff,
err)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(ErrData(err)...)
return nil, Stack(ctx, err)
}
return response, nil

View File

@ -7,10 +7,10 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -27,7 +27,7 @@ func (suite *GraphUnitSuite) SetupSuite() {
t := suite.T()
a := tester.NewMockM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.credentials = m365
}
@ -39,7 +39,7 @@ func (suite *GraphUnitSuite) TestCreateAdapter() {
suite.credentials.AzureClientID,
suite.credentials.AzureClientSecret)
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotNil(t, adpt)
}
@ -79,7 +79,7 @@ func (suite *GraphUnitSuite) TestSerializationEndPoint() {
suite.credentials.AzureTenantID,
suite.credentials.AzureClientID,
suite.credentials.AzureClientSecret)
require.NoError(t, err)
aw.MustNoErr(t, err)
serv := NewService(adpt)
email := models.NewMessage()
@ -87,7 +87,7 @@ func (suite *GraphUnitSuite) TestSerializationEndPoint() {
email.SetSubject(&subject)
byteArray, err := serv.Serialize(email)
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotNil(t, byteArray)
t.Log(string(byteArray))
}

View File

@ -68,7 +68,7 @@ func NewGraphConnector(
itemClient *http.Client,
acct account.Account,
r resource,
errs *fault.Errors,
errs *fault.Bus,
) (*GraphConnector, error) {
m365, err := acct.M365Config()
if err != nil {
@ -129,7 +129,7 @@ func (gc *GraphConnector) createService() (*graph.Service, error) {
// setTenantUsers queries the M365 to identify the users in the
// workspace. The users field is updated during this method
// iff the returned error is nil
func (gc *GraphConnector) setTenantUsers(ctx context.Context, errs *fault.Errors) error {
func (gc *GraphConnector) setTenantUsers(ctx context.Context, errs *fault.Bus) error {
ctx, end := D.Span(ctx, "gc:setTenantUsers")
defer end()
@ -160,7 +160,7 @@ func (gc *GraphConnector) GetUsersIds() []string {
// setTenantSites queries the M365 to identify the sites in the
// workspace. The sites field is updated during this method
// iff the returned error is nil.
func (gc *GraphConnector) setTenantSites(ctx context.Context, errs *fault.Errors) error {
func (gc *GraphConnector) setTenantSites(ctx context.Context, errs *fault.Bus) error {
gc.Sites = map[string]string{}
ctx, end := D.Span(ctx, "gc:setTenantSites")
@ -232,7 +232,7 @@ func (gc *GraphConnector) GetSiteIDs() []string {
func (gc *GraphConnector) UnionSiteIDsAndWebURLs(
ctx context.Context,
ids, urls []string,
errs *fault.Errors,
errs *fault.Bus,
) ([]string, error) {
if len(gc.Sites) == 0 {
if err := gc.setTenantSites(ctx, errs); err != nil {
@ -314,31 +314,31 @@ func getResources(
query func(context.Context, graph.Servicer) (serialization.Parsable, error),
parser func(parseNode serialization.ParseNode) (serialization.Parsable, error),
identify func(any) (string, string, error),
errs *fault.Errors,
errs *fault.Bus,
) (map[string]string, error) {
resources := map[string]string{}
response, err := query(ctx, gs)
if err != nil {
return nil, clues.Wrap(err, "retrieving tenant's resources").
WithClues(ctx).
With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "retrieving tenant's resources")
}
iter, err := msgraphgocore.NewPageIterator(response, gs.Adapter(), parser)
if err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
el := errs.Local()
callbackFunc := func(item any) bool {
if errs.Err() != nil {
if el.Failure() != nil {
return false
}
k, v, err := identify(item)
if err != nil {
if !errors.Is(err, errKnownSkippableCase) {
errs.Add(clues.Stack(err).
el.AddRecoverable(clues.Stack(err).
WithClues(ctx).
With("query_url", gs.Adapter().GetBaseUrl()))
}
@ -352,8 +352,8 @@ func getResources(
}
if err := iter.Iterate(ctx, callbackFunc); err != nil {
return nil, clues.Stack(err).WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Stack(ctx, err)
}
return resources, errs.Err()
return resources, el.Failure()
}

View File

@ -4,14 +4,13 @@ import (
"sync"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/fault"
@ -54,7 +53,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestBadConnection() {
AzureTenantID: "data",
},
)
require.NoError(t, err)
aw.MustNoErr(t, err)
return a
},
},
@ -92,17 +91,11 @@ func statusTestTask(gc *GraphConnector, objects, success, folder int) {
ctx,
support.Restore, folder,
support.CollectionMetrics{
Objects: objects,
Successes: success,
TotalBytes: 0,
Objects: objects,
Successes: success,
Bytes: 0,
},
support.WrapAndAppend(
"tres",
errors.New("three"),
support.WrapAndAppend("arc376", errors.New("one"), errors.New("two")),
),
"statusTestTask",
)
"statusTestTask")
gc.UpdateStatus(status)
}
@ -123,11 +116,11 @@ func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() {
assert.NotEmpty(t, gc.PrintableStatus())
// Expect 8 objects
assert.Equal(t, 8, gc.Status().ObjectCount)
assert.Equal(t, 8, gc.Status().Metrics.Objects)
// Expect 2 success
assert.Equal(t, 2, gc.Status().Successful)
assert.Equal(t, 2, gc.Status().Metrics.Successes)
// Expect 2 folders
assert.Equal(t, 2, gc.Status().FolderCount)
assert.Equal(t, 2, gc.Status().Folders)
}
func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
@ -146,14 +139,14 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
}{
{
name: "No scopes",
checkError: assert.Error,
checkError: aw.Err,
getSelector: func(t *testing.T) selectors.Selector {
return selectors.NewExchangeBackup(nil).Selector
},
},
{
name: "Valid Single User",
checkError: assert.NoError,
checkError: aw.NoErr,
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup([]string{"bobKelso@someHospital.org"})
sel.Include(sel.MailFolders(selectors.Any()))
@ -162,7 +155,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
},
{
name: "Partial invalid user",
checkError: assert.Error,
checkError: aw.Err,
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewExchangeBackup([]string{"bobkelso@someHospital.org", "janitor@someHospital.org"})
sel.Include(sel.MailFolders(selectors.Any()))
@ -172,7 +165,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs() {
},
{
name: "Invalid discrete owner",
checkError: assert.Error,
checkError: aw.Err,
getSelector: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"janitor@someHospital.org"})
sel.Include(sel.AllData())
@ -202,7 +195,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices
}{
{
name: "Valid User",
checkError: assert.NoError,
checkError: aw.NoErr,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"elliotReid@someHospital.org", "foo@SomeCompany.org"})
sel.Exclude(sel.Folders(selectors.Any()))
@ -224,7 +217,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices
},
{
name: "Invalid User",
checkError: assert.Error,
checkError: aw.Err,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewOneDriveBackup([]string{"foo@SomeCompany.org"})
sel.Exclude(sel.Folders(selectors.Any()))
@ -243,7 +236,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices
},
{
name: "valid sites",
checkError: assert.NoError,
checkError: aw.NoErr,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"abc.site.foo", "bar.site.baz"})
sel.DiscreteOwner = "abc.site.foo"
@ -265,7 +258,7 @@ func (suite *DisconnectedGraphConnectorSuite) TestVerifyBackupInputs_allServices
},
{
name: "invalid sites",
checkError: assert.Error,
checkError: aw.Err,
excludes: func(t *testing.T) selectors.Selector {
sel := selectors.NewSharePointBackup([]string{"fnords.smarfs.brawnhilda"})
sel.Exclude(sel.AllData())

View File

@ -22,6 +22,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -58,7 +59,7 @@ func mustToDataLayerPath(
err = errors.Errorf("bad service type %s", service.String())
}
require.NoError(t, err)
aw.MustNoErr(t, err)
return res
}
@ -600,12 +601,12 @@ func compareExchangeEmail(
item data.Stream,
) {
itemData, err := io.ReadAll(item.ToReader())
if !assert.NoError(t, err, "reading collection item: %s", item.UUID()) {
if !aw.NoErr(t, err, "reading collection item: %s", item.UUID()) {
return
}
itemMessage, err := support.CreateMessageFromBytes(itemData)
if !assert.NoError(t, err, "deserializing backed up message") {
if !aw.NoErr(t, err, "deserializing backed up message") {
return
}
@ -615,7 +616,7 @@ func compareExchangeEmail(
}
expectedMessage, err := support.CreateMessageFromBytes(expectedBytes)
assert.NoError(t, err, "deserializing source message")
aw.NoErr(t, err, "deserializing source message")
checkMessage(t, expectedMessage, itemMessage)
}
@ -626,12 +627,12 @@ func compareExchangeContact(
item data.Stream,
) {
itemData, err := io.ReadAll(item.ToReader())
if !assert.NoError(t, err, "reading collection item: %s", item.UUID()) {
if !aw.NoErr(t, err, "reading collection item: %s", item.UUID()) {
return
}
itemContact, err := support.CreateContactFromBytes(itemData)
if !assert.NoError(t, err, "deserializing backed up contact") {
if !aw.NoErr(t, err, "deserializing backed up contact") {
return
}
@ -641,7 +642,7 @@ func compareExchangeContact(
}
expectedContact, err := support.CreateContactFromBytes(expectedBytes)
assert.NoError(t, err, "deserializing source contact")
aw.NoErr(t, err, "deserializing source contact")
checkContact(t, expectedContact, itemContact)
}
@ -652,12 +653,12 @@ func compareExchangeEvent(
item data.Stream,
) {
itemData, err := io.ReadAll(item.ToReader())
if !assert.NoError(t, err, "reading collection item: %s", item.UUID()) {
if !aw.NoErr(t, err, "reading collection item: %s", item.UUID()) {
return
}
itemEvent, err := support.CreateEventFromBytes(itemData)
if !assert.NoError(t, err, "deserializing backed up contact") {
if !aw.NoErr(t, err, "deserializing backed up contact") {
return
}
@ -667,7 +668,7 @@ func compareExchangeEvent(
}
expectedEvent, err := support.CreateEventFromBytes(expectedBytes)
assert.NoError(t, err, "deserializing source contact")
aw.NoErr(t, err, "deserializing source contact")
checkEvent(t, expectedEvent, itemEvent)
}
@ -715,7 +716,7 @@ func compareOneDriveItem(
}
buf, err := io.ReadAll(item.ToReader())
if !assert.NoError(t, err) {
if !aw.NoErr(t, err) {
return
}
@ -1108,7 +1109,7 @@ func loadConnector(ctx context.Context, t *testing.T, itemClient *http.Client, r
a := tester.NewM365Account(t)
connector, err := NewGraphConnector(ctx, itemClient, a, r, fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
return connector
}

View File

@ -5,18 +5,18 @@ import (
"testing"
"time"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/backup"
"github.com/alcionai/corso/src/pkg/control"
@ -126,7 +126,7 @@ func (suite *GraphConnectorUnitSuite) TestUnionSiteIDsAndWebURLs() {
defer flush()
result, err := gc.UnionSiteIDsAndWebURLs(ctx, test.ids, test.urls, fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
assert.ElementsMatch(t, test.expect, result)
})
}
@ -181,7 +181,7 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
defer flush()
owners, err := api.NewClient(suite.connector.credentials)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
newConnector.Owners = owners
suite.Empty(len(newConnector.Users))
@ -208,13 +208,13 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantSites() {
t := suite.T()
service, err := newConnector.createService()
require.NoError(t, err)
aw.MustNoErr(t, err)
newConnector.Service = service
assert.Equal(t, 0, len(newConnector.Sites))
err = newConnector.setTenantSites(ctx, fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
assert.Less(t, 0, len(newConnector.Sites))
for _, site := range newConnector.Sites {
@ -247,13 +247,13 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreFailsBadService() {
},
nil,
fault.New(true))
assert.Error(t, err)
aw.Err(t, err)
assert.NotNil(t, deets)
status := suite.connector.AwaitStatus()
assert.Equal(t, 0, status.ObjectCount)
assert.Equal(t, 0, status.FolderCount)
assert.Equal(t, 0, status.Successful)
assert.Equal(t, 0, status.Metrics.Objects)
assert.Equal(t, 0, status.Folders)
assert.Equal(t, 0, status.Metrics.Successes)
}
func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
@ -324,13 +324,13 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
},
test.col,
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.NotNil(t, deets)
stats := suite.connector.AwaitStatus()
assert.Zero(t, stats.ObjectCount)
assert.Zero(t, stats.FolderCount)
assert.Zero(t, stats.Successful)
assert.Zero(t, stats.Metrics.Objects)
assert.Zero(t, stats.Folders)
assert.Zero(t, stats.Metrics.Successes)
})
}
}
@ -349,19 +349,15 @@ func mustGetDefaultDriveID(
//revive:enable:context-as-argument
d, err := service.Client().UsersById(userID).Drive().Get(ctx, nil)
if err != nil {
err = errors.Wrapf(
err,
"failed to retrieve default user drive. user: %s, details: %s",
userID,
support.ConnectorStackErrorTrace(err),
)
err = graph.Wrap(ctx, err, "retrieving drive")
}
require.NoError(t, err)
require.NotNil(t, d.GetId())
require.NotEmpty(t, *d.GetId())
aw.MustNoErr(t, err)
return *d.GetId()
id := ptr.Val(d.GetId())
require.NotEmpty(t, id)
return id
}
func getCollectionsAndExpected(
@ -426,16 +422,14 @@ func runRestore(
config.opts,
collections,
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.NotNil(t, deets)
status := restoreGC.AwaitStatus()
runTime := time.Since(start)
assert.NoError(t, status.Err, "restored status.Err")
assert.Zero(t, status.ErrorCount, "restored status.ErrorCount")
assert.Equal(t, numRestoreItems, status.ObjectCount, "restored status.ObjectCount")
assert.Equal(t, numRestoreItems, status.Successful, "restored status.Successful")
assert.Equal(t, numRestoreItems, status.Metrics.Objects, "restored status.Metrics.Objects")
assert.Equal(t, numRestoreItems, status.Metrics.Successes, "restored status.Metrics.Successes")
assert.Len(
t,
deets.Entries,
@ -485,7 +479,7 @@ func runBackupAndCompare(
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
// No excludes yet because this isn't an incremental backup.
assert.Empty(t, excludes)
@ -497,12 +491,10 @@ func runBackupAndCompare(
status := backupGC.AwaitStatus()
assert.NoError(t, status.Err, "backup status.Err")
assert.Zero(t, status.ErrorCount, "backup status.ErrorCount")
assert.Equalf(t, totalItems+skipped, status.ObjectCount,
"backup status.ObjectCount; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Successful,
"backup status.Successful; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Metrics.Objects,
"backup status.Metrics.Objects; wanted %d items + %d skipped", totalItems, skipped)
assert.Equalf(t, totalItems+skipped, status.Metrics.Successes,
"backup status.Metrics.Successes; wanted %d items + %d skipped", totalItems, skipped)
}
func runRestoreBackupTest(
@ -961,13 +953,13 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
},
collections,
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
require.NotNil(t, deets)
status := restoreGC.AwaitStatus()
// Always just 1 because it's just 1 collection.
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal(t, totalItems, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, totalItems, status.Metrics.Successes, "status.Metrics.Successes")
assert.Equal(
t, totalItems, len(deets.Entries),
"details entries contains same item count as total successful items restored")
@ -990,7 +982,7 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
ToggleFeatures: control.Toggles{EnablePermissionsBackup: true},
},
fault.New(true))
require.NoError(t, err)
aw.MustNoErr(t, err)
// No excludes yet because this isn't an incremental backup.
assert.Empty(t, excludes)
@ -1001,8 +993,8 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
skipped := checkCollections(t, ctx, allItems, allExpectedData, dcs, true)
status := backupGC.AwaitStatus()
assert.Equal(t, allItems+skipped, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, allItems+skipped, status.Successful, "status.Successful")
assert.Equal(t, allItems+skipped, status.Metrics.Objects, "status.Metrics.Objects")
assert.Equal(t, allItems+skipped, status.Metrics.Successes, "status.Metrics.Successes")
})
}
}

View File

@ -111,7 +111,7 @@ func (medc MockExchangeDataCollection) DoNotMergeItems() bool { return med
// channel is closed when there are no more items available.
func (medc *MockExchangeDataCollection) Items(
ctx context.Context,
_ *fault.Errors, // unused
_ *fault.Bus, // unused
) <-chan data.Stream {
res := make(chan data.Stream)

View File

@ -15,6 +15,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -35,7 +36,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection() {
for item := range mdc.Items(ctx, fault.New(true)) {
_, err := io.ReadAll(item.ToReader())
assert.NoError(suite.T(), err)
aw.NoErr(suite.T(), err)
messagesRead++
}
@ -52,7 +53,7 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollectionItemSize() {
for item := range mdc.Items(ctx, fault.New(true)) {
buf, err := io.ReadAll(item.ToReader())
assert.NoError(t, err)
aw.NoErr(t, err)
assert.Implements(t, (*data.StreamSize)(nil), item)
s := item.(data.StreamSize)
@ -72,11 +73,11 @@ func (suite *MockExchangeCollectionSuite) TestMockExchangeCollection_NewExchange
for stream := range mdc.Items(ctx, fault.New(true)) {
_, err := buf.ReadFrom(stream.ToReader())
assert.NoError(t, err)
aw.NoErr(t, err)
byteArray := buf.Bytes()
something, err := support.CreateFromBytes(byteArray, models.CreateMessageFromDiscriminatorValue)
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotNil(t, something)
}
}
@ -104,7 +105,7 @@ func (suite *MockExchangeDataSuite) TestMockExchangeData() {
ID: id,
Reader: io.NopCloser(bytes.NewReader(itemData)),
},
check: require.NoError,
check: aw.MustNoErr,
},
{
name: "Error",
@ -112,7 +113,7 @@ func (suite *MockExchangeDataSuite) TestMockExchangeData() {
ID: id,
ReadErr: assert.AnError,
},
check: require.Error,
check: aw.MustErr,
},
}
@ -192,10 +193,10 @@ func (suite *MockExchangeDataSuite) TestMockByteHydration() {
temp := mockconnector.GetMockList(subject, "Artist", emptyMap)
writer := kioser.NewJsonSerializationWriter()
err := writer.WriteObjectValue("", temp)
require.NoError(t, err)
aw.MustNoErr(t, err)
bytes, err := writer.GetSerializedContent()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
_, err = support.CreateListFromBytes(bytes)
@ -206,7 +207,7 @@ func (suite *MockExchangeDataSuite) TestMockByteHydration() {
name: "SharePoint: List 6 Items",
transformation: func(t *testing.T) error {
bytes, err := mockconnector.GetMockListBytes(subject)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
_, err = support.CreateListFromBytes(bytes)
return err
},
@ -225,7 +226,7 @@ func (suite *MockExchangeDataSuite) TestMockByteHydration() {
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
err := test.transformation(t)
assert.NoError(t, err)
aw.NoErr(t, err)
})
}
}

View File

@ -8,9 +8,9 @@ import (
kw "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
)
@ -48,7 +48,7 @@ func (mlc *MockListCollection) PreviousPath() path.Path {
func (mlc *MockListCollection) Items(
ctx context.Context,
_ *fault.Errors, // unused
_ *fault.Bus, // unused
) <-chan data.Stream {
res := make(chan data.Stream)
@ -159,7 +159,7 @@ func GetMockListBytes(title string) ([]byte, error) {
// of the Mocked SharePoint List
func GetMockListStream(t *testing.T, title string, numOfItems int) *MockListData {
byteArray, err := GetMockListBytes(title)
require.NoError(t, err)
aw.MustNoErr(t, err)
listData := &MockListData{
ID: title,

View File

@ -9,9 +9,9 @@ import (
js "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/tester/aw"
)
//nolint:lll
@ -693,10 +693,10 @@ func GetMockMessageWithNestedItemAttachmentEvent(subject string) []byte {
func GetMockMessageWithNestedItemAttachmentMail(t *testing.T, nested []byte, subject string) []byte {
base := GetMockMessageBytes(subject)
message, err := hydrateMessage(base)
require.NoError(t, err)
aw.MustNoErr(t, err)
nestedMessage, err := hydrateMessage(nested)
require.NoError(t, err)
aw.MustNoErr(t, err)
iaNode := models.NewItemAttachment()
attachmentSize := int32(len(nested))
@ -713,13 +713,13 @@ func GetMockMessageWithNestedItemAttachmentMail(t *testing.T, nested []byte, sub
func GetMockMessageWithNestedItemAttachmentContact(t *testing.T, nested []byte, subject string) []byte {
base := GetMockMessageBytes(subject)
message, err := hydrateMessage(base)
require.NoError(t, err)
aw.MustNoErr(t, err)
parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", nested)
require.NoError(t, err)
aw.MustNoErr(t, err)
anObject, err := parseNode.GetObjectValue(models.CreateContactFromDiscriminatorValue)
require.NoError(t, err)
aw.MustNoErr(t, err)
contact := anObject.(models.Contactable)
internalName := "Nested Contact"
@ -736,10 +736,10 @@ func GetMockMessageWithNestedItemAttachmentContact(t *testing.T, nested []byte,
func serialize(t *testing.T, item absser.Parsable) []byte {
wtr := js.NewJsonSerializationWriter()
err := wtr.WriteObjectValue("", item)
require.NoError(t, err)
aw.MustNoErr(t, err)
byteArray, err := wtr.GetSerializedContent()
require.NoError(t, err)
aw.MustNoErr(t, err)
return byteArray
}

View File

@ -2,12 +2,13 @@ package api
import (
"context"
"fmt"
"github.com/alcionai/clues"
msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models"
mssites "github.com/microsoftgraph/msgraph-sdk-go/sites"
msusers "github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
@ -16,10 +17,7 @@ import (
func getValues[T any](l api.PageLinker) ([]T, error) {
page, ok := l.(interface{ GetValue() []T })
if !ok {
return nil, errors.Errorf(
"response of type [%T] does not comply with GetValue() interface",
l,
)
return nil, clues.New("page does not comply with GetValue() interface").With("page_item_type", fmt.Sprintf("%T", l))
}
return page.GetValue(), nil
@ -69,8 +67,11 @@ func (p *driveItemPager) GetPage(ctx context.Context) (api.DeltaPageLinker, erro
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, err
return resp, nil
}
func (p *driveItemPager) SetNext(link string) {
@ -163,8 +164,11 @@ func (p *siteDrivePager) GetPage(ctx context.Context) (api.PageLinker, error) {
)
resp, err = p.builder.Get(ctx, p.options)
if err != nil {
return nil, graph.Stack(ctx, err)
}
return resp, err
return resp, nil
}
func (p *siteDrivePager) SetNext(link string) {

View File

@ -10,10 +10,12 @@ import (
"sync/atomic"
"time"
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/spatialcurrent/go-lazy/pkg/lazy"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -80,9 +82,10 @@ type Collection struct {
// itemReadFunc returns a reader for the specified item
type itemReaderFunc func(
ctx context.Context,
hc *http.Client,
item models.DriveItemable,
) (itemInfo details.ItemInfo, itemData io.ReadCloser, err error)
) (details.ItemInfo, io.ReadCloser, error)
// itemMetaReaderFunc returns a reader for the metadata of the
// specified item
@ -141,9 +144,9 @@ func (oc *Collection) Add(item models.DriveItemable) {
// Items() returns the channel containing M365 Exchange objects
func (oc *Collection) Items(
ctx context.Context,
errs *fault.Errors, // TODO: currently unused while onedrive isn't up to date with clues/fault
errs *fault.Bus, // TODO: currently unused while onedrive isn't up to date with clues/fault
) <-chan data.Stream {
go oc.populateItems(ctx)
go oc.populateItems(ctx, errs)
return oc.data
}
@ -216,23 +219,22 @@ func (od *Item) ModTime() time.Time {
// populateItems iterates through items added to the collection
// and uses the collection `itemReader` to read the item
func (oc *Collection) populateItems(ctx context.Context) {
func (oc *Collection) populateItems(ctx context.Context, errs *fault.Bus) {
var (
errs error
byteCount int64
itemsRead int64
dirsRead int64
itemsFound int64
dirsFound int64
wg sync.WaitGroup
m sync.Mutex
el = errs.Local()
)
// Retrieve the OneDrive folder path to set later in
// `details.OneDriveInfo`
parentPathString, err := path.GetDriveFolderPath(oc.folderPath)
if err != nil {
oc.reportAsCompleted(ctx, 0, 0, 0, err)
oc.reportAsCompleted(ctx, 0, 0, 0)
return
}
@ -247,14 +249,8 @@ func (oc *Collection) populateItems(ctx context.Context) {
semaphoreCh := make(chan struct{}, urlPrefetchChannelBufferSize)
defer close(semaphoreCh)
errUpdater := func(id string, err error) {
m.Lock()
errs = support.WrapAndAppend(id, err, errs)
m.Unlock()
}
for _, item := range oc.driveItems {
if oc.ctrl.FailFast && errs != nil {
if el.Failure() != nil {
break
}
@ -262,22 +258,27 @@ func (oc *Collection) populateItems(ctx context.Context) {
wg.Add(1)
go func(item models.DriveItemable) {
go func(ctx context.Context, item models.DriveItemable) {
defer wg.Done()
defer func() { <-semaphoreCh }()
// Read the item
var (
itemID = *item.GetId()
itemName = *item.GetName()
itemSize = *item.GetSize()
itemID = ptr.Val(item.GetId())
itemName = ptr.Val(item.GetName())
itemSize = ptr.Val(item.GetSize())
itemInfo details.ItemInfo
itemMeta io.ReadCloser
itemMetaSize int
metaSuffix string
err error
)
ctx = clues.Add(ctx,
"restore_item_id", itemID,
"restore_item_name", itemName,
"restore_item_size", itemSize,
"restore_item_info", itemInfo)
isFile := item.GetFile() != nil
if isFile {
@ -301,9 +302,8 @@ func (oc *Collection) populateItems(ctx context.Context) {
itemMetaSize = 2
} else {
itemMeta, itemMetaSize, err = oc.itemMetaReader(ctx, oc.service, oc.driveID, item)
if err != nil {
errUpdater(*item.GetId(), errors.Wrap(err, "failed to get item permissions"))
el.AddRecoverable(clues.Wrap(err, "getting item permissions"))
return
}
}
@ -335,7 +335,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
err error
)
_, itemData, err = oc.itemReader(oc.itemClient, item)
_, itemData, err = oc.itemReader(ctx, oc.itemClient, item)
if err != nil && graph.IsErrUnauthorized(err) {
// assume unauthorized requests are a sign of an expired
@ -351,7 +351,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
// check for errors following retries
if err != nil {
errUpdater(itemID, err)
el.AddRecoverable(clues.Stack(err).WithClues(ctx))
return nil, err
}
@ -361,8 +361,7 @@ func (oc *Collection) populateItems(ctx context.Context) {
itemData,
observe.ItemBackupMsg,
observe.PII(itemName+dataSuffix),
itemSize,
)
itemSize)
go closer()
return progReader, nil
@ -419,27 +418,27 @@ func (oc *Collection) populateItems(ctx context.Context) {
atomic.AddInt64(&byteCount, itemSize)
folderProgress <- struct{}{}
}(item)
}(ctx, item)
}
wg.Wait()
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount, errs)
oc.reportAsCompleted(ctx, int(itemsFound), int(itemsRead), byteCount)
}
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64, errs error) {
func (oc *Collection) reportAsCompleted(ctx context.Context, itemsFound, itemsRead int, byteCount int64) {
close(oc.data)
status := support.CreateStatus(ctx, support.Backup,
1, // num folders (always 1)
support.CollectionMetrics{
Objects: itemsFound, // items to read,
Successes: itemsRead, // items read successfully,
TotalBytes: byteCount, // Number of bytes read in the operation,
Objects: itemsFound,
Successes: itemsRead,
Bytes: byteCount,
},
errs,
oc.folderPath.Folder(false), // Additional details
)
oc.folderPath.Folder(false))
logger.Ctx(ctx).Debugw("done streaming items", "status", status.String())
oc.statusUpdater(status)
}

View File

@ -21,6 +21,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -52,7 +53,7 @@ func (suite *CollectionUnitTestSuite) testStatusUpdater(
statusToUpdate *support.ConnectorOperationStatus,
) support.StatusUpdater {
return func(s *support.ConnectorOperationStatus) {
suite.T().Logf("Update status %v, count %d, success %d", s, s.ObjectCount, s.Successful)
suite.T().Logf("Update status %v, count %d, success %d", s, s.Metrics.Objects, s.Metrics.Successes)
*statusToUpdate = *s
wg.Done()
@ -94,7 +95,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
numInstances: 1,
source: OneDriveSource,
itemDeets: nst{testItemName, 42, now},
itemReader: func(*http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
itemReader: func(context.Context, *http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
return details.ItemInfo{OneDrive: &details.OneDriveInfo{ItemName: testItemName, Modified: now}},
io.NopCloser(bytes.NewReader(testItemData)),
nil
@ -109,7 +110,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
numInstances: 3,
source: OneDriveSource,
itemDeets: nst{testItemName, 42, now},
itemReader: func(*http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
itemReader: func(context.Context, *http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
return details.ItemInfo{OneDrive: &details.OneDriveInfo{ItemName: testItemName, Modified: now}},
io.NopCloser(bytes.NewReader(testItemData)),
nil
@ -124,7 +125,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
numInstances: 1,
source: SharePointSource,
itemDeets: nst{testItemName, 42, now},
itemReader: func(*http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
itemReader: func(context.Context, *http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
return details.ItemInfo{SharePoint: &details.SharePointInfo{ItemName: testItemName, Modified: now}},
io.NopCloser(bytes.NewReader(testItemData)),
nil
@ -139,7 +140,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
numInstances: 3,
source: SharePointSource,
itemDeets: nst{testItemName, 42, now},
itemReader: func(*http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
itemReader: func(context.Context, *http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
return details.ItemInfo{SharePoint: &details.SharePointInfo{ItemName: testItemName, Modified: now}},
io.NopCloser(bytes.NewReader(testItemData)),
nil
@ -162,9 +163,9 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
)
folderPath, err := GetCanonicalPath("drive/driveID1/root:/dir1/dir2/dir3", "tenant", "owner", test.source)
require.NoError(t, err)
aw.MustNoErr(t, err)
driveFolderPath, err := path.GetDriveFolderPath(folderPath)
require.NoError(t, err)
aw.MustNoErr(t, err)
coll := NewCollection(
graph.HTTPClient(graph.NoTimeout()),
@ -222,8 +223,8 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
}
// Expect only 1 item
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
// Validate item info and data
readItem := readItems[0]
@ -240,7 +241,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
assert.Equal(t, now, mt.ModTime())
readData, err := io.ReadAll(readItem.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
name, parentPath := test.infoFrom(t, readItemInfo.Info())
@ -254,7 +255,7 @@ func (suite *CollectionUnitTestSuite) TestCollection() {
assert.Equal(t, testItemName+MetaFileSuffix, readItemMeta.UUID())
readMetaData, err := io.ReadAll(readItemMeta.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
tm, err := json.Marshal(testItemMeta)
if err != nil {
@ -301,7 +302,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() {
wg.Add(1)
folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", test.source)
require.NoError(t, err)
aw.MustNoErr(t, err)
coll := NewCollection(
graph.HTTPClient(graph.NoTimeout()),
@ -323,7 +324,11 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() {
mockItem.SetLastModifiedDateTime(&now)
coll.Add(mockItem)
coll.itemReader = func(*http.Client, models.DriveItemable) (details.ItemInfo, io.ReadCloser, error) {
coll.itemReader = func(
context.Context,
*http.Client,
models.DriveItemable,
) (details.ItemInfo, io.ReadCloser, error) {
return details.ItemInfo{}, nil, assert.AnError
}
@ -339,13 +344,13 @@ func (suite *CollectionUnitTestSuite) TestCollectionReadError() {
assert.True(t, ok)
_, err = io.ReadAll(collItem.ToReader())
assert.Error(t, err)
aw.Err(t, err)
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount, "only one object should be counted")
require.Equal(t, 1, collStatus.Successful, "TODO: should be 0, but allowing 1 to reduce async management")
require.Equal(t, 1, collStatus.Metrics.Objects, "only one object should be counted")
require.Equal(t, 1, collStatus.Metrics.Successes, "TODO: should be 0, but allowing 1 to reduce async management")
})
}
}
@ -376,7 +381,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionDisablePermissionsBackup() {
wg.Add(1)
folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", test.source)
require.NoError(t, err)
aw.MustNoErr(t, err)
coll := NewCollection(
graph.HTTPClient(graph.NoTimeout()),
@ -400,6 +405,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionDisablePermissionsBackup() {
coll.Add(mockItem)
coll.itemReader = func(
context.Context,
*http.Client,
models.DriveItemable,
) (details.ItemInfo, io.ReadCloser, error) {
@ -424,13 +430,13 @@ func (suite *CollectionUnitTestSuite) TestCollectionDisablePermissionsBackup() {
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
for _, i := range readItems {
if strings.HasSuffix(i.UUID(), MetaFileSuffix) {
content, err := io.ReadAll(i.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
require.Equal(t, content, []byte("{}"))
}
}
@ -466,7 +472,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim
wg.Add(1)
folderPath, err := GetCanonicalPath("drive/driveID1/root:/folderPath", "a-tenant", "a-user", test.source)
require.NoError(t, err)
aw.MustNoErr(t, err)
coll := NewCollection(
graph.HTTPClient(graph.NoTimeout()),
@ -490,6 +496,7 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim
coll.Add(mockItem)
coll.itemReader = func(
context.Context,
*http.Client,
models.DriveItemable,
) (details.ItemInfo, io.ReadCloser, error) {
@ -514,13 +521,13 @@ func (suite *CollectionUnitTestSuite) TestCollectionPermissionBackupLatestModTim
wg.Wait()
// Expect no items
require.Equal(t, 1, collStatus.ObjectCount)
require.Equal(t, 1, collStatus.Successful)
require.Equal(t, 1, collStatus.Metrics.Objects)
require.Equal(t, 1, collStatus.Metrics.Successes)
for _, i := range readItems {
if strings.HasSuffix(i.UUID(), MetaFileSuffix) {
content, err := io.ReadAll(i.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
require.Equal(t, content, []byte("{}"))
im, ok := i.(data.StreamModTime)
require.Equal(t, ok, true, "modtime interface")

View File

@ -13,11 +13,13 @@ import (
"github.com/pkg/errors"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -117,32 +119,40 @@ func NewCollections(
func deserializeMetadata(
ctx context.Context,
cols []data.RestoreCollection,
errs *fault.Bus,
) (map[string]string, map[string]map[string]string, error) {
logger.Ctx(ctx).Infow(
"deserialzing previous backup metadata",
"num_collections",
len(cols),
"num_collections", len(cols))
var (
prevDeltas = map[string]string{}
prevFolders = map[string]map[string]string{}
el = errs.Local()
)
prevDeltas := map[string]string{}
prevFolders := map[string]map[string]string{}
for _, col := range cols {
items := col.Items(ctx, nil) // TODO: fault.Errors instead of nil
if el.Failure() != nil {
break
}
items := col.Items(ctx, errs)
for breakLoop := false; !breakLoop; {
select {
case <-ctx.Done():
return nil, nil, errors.Wrap(ctx.Err(), "deserialzing previous backup metadata")
return nil, nil, clues.Wrap(ctx.Err(), "deserialzing previous backup metadata").WithClues(ctx)
case item, ok := <-items:
if !ok {
// End of collection items.
breakLoop = true
break
}
var err error
var (
err error
ictx = clues.Add(ctx, "item_uuid", item.UUID())
)
switch item.UUID() {
case graph.PreviousPathFileName:
@ -152,11 +162,9 @@ func deserializeMetadata(
err = deserializeMap(item.ToReader(), prevDeltas)
default:
logger.Ctx(ctx).Infow(
logger.Ctx(ictx).Infow(
"skipping unknown metadata file",
"file_name",
item.UUID(),
)
"file_name", item.UUID())
continue
}
@ -173,20 +181,13 @@ func deserializeMetadata(
// we end up in a situation where we're sourcing items from the wrong
// base in kopia wrapper.
if errors.Is(err, errExistingMapping) {
return nil, nil, errors.Wrapf(
err,
"deserializing metadata file %s",
item.UUID(),
)
return nil, nil, clues.Wrap(err, "deserializing metadata file").WithClues(ictx)
}
logger.Ctx(ctx).Errorw(
"deserializing base backup metadata. Falling back to full backup for selected drives",
"error",
err,
"file_name",
item.UUID(),
)
err = clues.Stack(err).WithClues(ictx)
el.AddRecoverable(err)
logger.CtxErr(ictx, err).Error("deserializing base backup metadata")
}
}
@ -213,10 +214,10 @@ func deserializeMetadata(
}
}
return prevDeltas, prevFolders, nil
return prevDeltas, prevFolders, el.Failure()
}
var errExistingMapping = errors.New("mapping already exists for same drive ID")
var errExistingMapping = clues.New("mapping already exists for same drive ID")
// deserializeMap takes an reader and a map of already deserialized items and
// adds the newly deserialized items to alreadyFound. Items are only added to
@ -242,7 +243,7 @@ func deserializeMap[T any](reader io.ReadCloser, alreadyFound map[string]T) erro
}
if duplicate {
return errors.WithStack(errExistingMapping)
return clues.Stack(errExistingMapping)
}
maps.Copy(alreadyFound, tmp)
@ -255,8 +256,9 @@ func deserializeMap[T any](reader io.ReadCloser, alreadyFound map[string]T) erro
func (c *Collections) Get(
ctx context.Context,
prevMetadata []data.RestoreCollection,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
prevDeltas, oldPathsByDriveID, err := deserializeMetadata(ctx, prevMetadata)
prevDeltas, oldPathsByDriveID, err := deserializeMetadata(ctx, prevMetadata, errs)
if err != nil {
return nil, nil, err
}
@ -264,7 +266,7 @@ func (c *Collections) Get(
// Enumerate drives for the specified resourceOwner
pager, err := c.drivePagerFunc(c.source, c.service, c.resourceOwner, nil)
if err != nil {
return nil, nil, err
return nil, nil, graph.Stack(ctx, err)
}
retry := c.source == OneDriveSource
@ -287,39 +289,33 @@ func (c *Collections) Get(
excludedItems = map[string]struct{}{}
)
// Update the collection map with items from each drive
for _, d := range drives {
driveID := *d.GetId()
driveName := *d.GetName()
var (
driveID = ptr.Val(d.GetId())
driveName = ptr.Val(d.GetName())
prevDelta = prevDeltas[driveID]
oldPaths = oldPathsByDriveID[driveID]
numOldDelta = 0
)
prevDelta := prevDeltas[driveID]
oldPaths := oldPathsByDriveID[driveID]
numOldDelta := 0
if len(prevDelta) > 0 {
numOldDelta++
}
logger.Ctx(ctx).Infow(
"previous metadata for drive",
"num_paths_entries",
len(oldPaths),
"num_deltas_entries",
numOldDelta)
"num_paths_entries", len(oldPaths),
"num_deltas_entries", numOldDelta)
delta, paths, excluded, err := collectItems(
ctx,
c.itemPagerFunc(
c.service,
driveID,
"",
),
c.itemPagerFunc(c.service, driveID, ""),
driveID,
driveName,
c.UpdateCollections,
oldPaths,
prevDelta,
)
errs)
if err != nil {
return nil, nil, err
}
@ -343,7 +339,6 @@ func (c *Collections) Get(
// token because it thinks the folder paths weren't persisted.
folderPaths[driveID] = map[string]string{}
maps.Copy(folderPaths[driveID], paths)
maps.Copy(excludedItems, excluded)
logger.Ctx(ctx).Infow(
@ -372,18 +367,13 @@ func (c *Collections) Get(
graph.NewMetadataEntry(graph.PreviousPathFileName, folderPaths),
graph.NewMetadataEntry(graph.DeltaURLsFileName, deltaURLs),
},
c.statusUpdater,
)
c.statusUpdater)
if err != nil {
// Technically it's safe to continue here because the logic for starting an
// incremental backup should eventually find that the metadata files are
// empty/missing and default to a full backup.
logger.Ctx(ctx).Warnw(
"making metadata collection for future incremental backups",
"error",
err,
)
logger.CtxErr(ctx, err).Info("making metadata collection for future incremental backups")
} else {
collections = append(collections, metadata)
}
@ -453,8 +443,15 @@ func (c *Collections) UpdateCollections(
newPaths map[string]string,
excluded map[string]struct{},
invalidPrevDelta bool,
errs *fault.Bus,
) error {
el := errs.Local()
for _, item := range items {
if el.Failure() != nil {
break
}
var (
prevPath path.Path
prevCollectionPath path.Path
@ -480,25 +477,30 @@ func (c *Collections) UpdateCollections(
continue
}
var (
itemID = ptr.Val(item.GetId())
ictx = clues.Add(ctx, "update_item_id", itemID)
)
if item.GetParentReference() == nil ||
item.GetParentReference().GetId() == nil ||
(item.GetDeleted() == nil && item.GetParentReference().GetPath() == nil) {
err := clues.New("no parent reference").With("item_id", *item.GetId())
if item.GetName() != nil {
err = err.With("item_name", *item.GetName())
}
el.AddRecoverable(clues.New("item missing parent reference").
WithClues(ictx).
With("item_id", itemID, "item_name", ptr.Val(item.GetName())))
return err
continue
}
// Create a collection for the parent of this item
collectionID := *item.GetParentReference().GetId()
collectionID := ptr.Val(item.GetParentReference().GetId())
ictx = clues.Add(ictx, "collection_id", collectionID)
var collectionPathStr string
if item.GetDeleted() == nil {
collectionPathStr = *item.GetParentReference().GetPath()
collectionPathStr = ptr.Val(item.GetParentReference().GetPath())
} else {
collectionPathStr, ok = oldPaths[*item.GetParentReference().GetId()]
collectionPathStr, ok = oldPaths[ptr.Val(item.GetParentReference().GetId())]
if !ok {
// This collection was created and destroyed in
// between the current and previous invocation
@ -510,25 +512,26 @@ func (c *Collections) UpdateCollections(
collectionPathStr,
c.tenant,
c.resourceOwner,
c.source,
)
c.source)
if err != nil {
return err
return clues.Stack(err).WithClues(ictx)
}
// Skip items that don't match the folder selectors we were given.
if shouldSkipDrive(ctx, collectionPath, c.matcher, driveName) {
logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
if shouldSkipDrive(ictx, collectionPath, c.matcher, driveName) {
logger.Ctx(ictx).Infow("Skipping path", "skipped_path", collectionPath.String())
continue
}
switch {
case item.GetFolder() != nil, item.GetPackage() != nil:
prevPathStr, ok := oldPaths[*item.GetId()]
prevPathStr, ok := oldPaths[itemID]
if ok {
prevPath, err = path.FromDataLayerPath(prevPathStr, false)
if err != nil {
return clues.Wrap(err, "invalid previous path").With("path_string", prevPathStr)
el.AddRecoverable(clues.Wrap(err, "invalid previous path").
WithClues(ictx).
With("path_string", prevPathStr))
}
}
@ -536,7 +539,7 @@ func (c *Collections) UpdateCollections(
// Nested folders also return deleted delta results so we don't have to
// worry about doing a prefix search in the map to remove the subtree of
// the deleted folder/package.
delete(newPaths, *item.GetId())
delete(newPaths, itemID)
if prevPath == nil {
// It is possible that an item was created and
@ -555,10 +558,9 @@ func (c *Collections) UpdateCollections(
c.statusUpdater,
c.source,
c.ctrl,
invalidPrevDelta,
)
invalidPrevDelta)
c.CollectionMap[*item.GetId()] = col
c.CollectionMap[itemID] = col
break
}
@ -568,14 +570,16 @@ func (c *Collections) UpdateCollections(
// parentRef or such.
folderPath, err := collectionPath.Append(*item.GetName(), false)
if err != nil {
logger.Ctx(ctx).Errorw("failed building collection path", "error", err)
return err
logger.Ctx(ictx).Errorw("building collection path", "error", err)
el.AddRecoverable(clues.Stack(err).WithClues(ictx))
continue
}
// Moved folders don't cause delta results for any subfolders nested in
// them. We need to go through and update paths to handle that. We only
// update newPaths so we don't accidentally clobber previous deletes.
updatePath(newPaths, *item.GetId(), folderPath.String())
updatePath(newPaths, itemID, folderPath.String())
found, err := updateCollectionPaths(*item.GetId(), c.CollectionMap, folderPath)
if err != nil {
@ -598,7 +602,7 @@ func (c *Collections) UpdateCollections(
c.ctrl,
invalidPrevDelta,
)
c.CollectionMap[*item.GetId()] = col
c.CollectionMap[itemID] = col
c.NumContainers++
}
}
@ -615,7 +619,7 @@ func (c *Collections) UpdateCollections(
// deleted, we want to avoid it. If it was
// renamed/moved/modified, we still have to drop the
// original one and download a fresh copy.
excluded[*item.GetId()] = struct{}{}
excluded[itemID] = struct{}{}
}
if item.GetDeleted() != nil {
@ -679,11 +683,11 @@ func (c *Collections) UpdateCollections(
}
default:
return errors.Errorf("item type not supported. item name : %s", *item.GetName())
return clues.New("item type not supported").WithClues(ctx)
}
}
return nil
return el.Failure()
}
func shouldSkipDrive(ctx context.Context, drivePath path.Path, m folderMatcher, driveName string) bool {
@ -705,7 +709,7 @@ func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path
case SharePointSource:
result, err = pathBuilder.ToDataLayerSharePointPath(tenant, resourceOwner, path.LibrariesCategory, false)
default:
return nil, errors.Errorf("unrecognized drive data source")
return nil, clues.New("unrecognized data source")
}
if err != nil {
@ -719,7 +723,7 @@ func includePath(ctx context.Context, m folderMatcher, folderPath path.Path) boo
// Check if the folder is allowed by the scope.
folderPathString, err := path.GetDriveFolderPath(folderPath)
if err != nil {
logger.Ctx(ctx).Error(err)
logger.Ctx(ctx).With("err", err).Error("getting drive folder path")
return true
}

View File

@ -19,7 +19,9 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -48,11 +50,11 @@ func getExpectedStatePathGenerator(
} else {
require.Len(t, pths, 2, "invalid number of paths to getExpectedStatePathGenerator")
p2, err = GetCanonicalPath(base+pths[1], tenant, user, OneDriveSource)
require.NoError(t, err)
aw.MustNoErr(t, err)
}
p1, err = GetCanonicalPath(base+pths[0], tenant, user, OneDriveSource)
require.NoError(t, err)
aw.MustNoErr(t, err)
switch state {
case data.NewState:
@ -80,7 +82,7 @@ func getExpectedPathGenerator(t *testing.T,
) func(string) string {
return func(path string) string {
p, err := GetCanonicalPath(base+path, tenant, user, OneDriveSource)
require.NoError(t, err)
aw.MustNoErr(t, err)
return p.String()
}
@ -109,20 +111,20 @@ func (suite *OneDriveCollectionsSuite) TestGetCanonicalPath() {
source: OneDriveSource,
dir: []string{"onedrive"},
expect: "tenant/onedrive/resourceOwner/files/onedrive",
expectErr: assert.NoError,
expectErr: aw.NoErr,
},
{
name: "sharepoint",
source: SharePointSource,
dir: []string{"sharepoint"},
expect: "tenant/sharepoint/resourceOwner/libraries/sharepoint",
expectErr: assert.NoError,
expectErr: aw.NoErr,
},
{
name: "unknown",
source: unknownDriveSource,
dir: []string{"unknown"},
expectErr: assert.Error,
expectErr: aw.Err,
},
}
for _, test := range table {
@ -173,7 +175,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.Error,
expect: aw.Err,
expectedMetadataPaths: map[string]string{
"root": expectedPath(""),
},
@ -187,7 +189,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -208,7 +210,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -228,7 +230,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
},
@ -252,7 +254,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, folder),
@ -283,7 +285,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder"})[0],
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"folder": expectedStatePath(data.NewState, folder),
"subfolder": expectedStatePath(data.NewState, folderSub),
@ -317,7 +319,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).
Folders([]string{"/folder/subfolder"}, selectors.PrefixMatch())[0],
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"subfolder": expectedStatePath(data.NewState, folderSub),
"folder2": expectedStatePath(data.NewState, folderSub+folder),
@ -345,7 +347,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: (&selectors.OneDriveBackup{}).Folders([]string{"folder/subfolder"})[0],
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"subfolder": expectedStatePath(data.NewState, folderSub),
},
@ -369,7 +371,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NotMovedState, "/folder"),
@ -395,7 +397,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, "/folder", "/a-folder"),
@ -421,7 +423,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"folder": expectedPath("/folder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NotMovedState, "/folder"),
@ -445,7 +447,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, "/folder2"),
@ -468,7 +470,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
},
inputFolderMap: map[string]string{},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.NewState, "/folder"),
@ -494,7 +496,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, "/folder", "/a-folder"),
@ -522,7 +524,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, "/folder", "/a-folder"),
@ -561,7 +563,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/a-folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.MovedState, "/folder", "/a-folder"),
@ -592,7 +594,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"package": expectedPath("/package"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"folder": expectedStatePath(data.DeletedState, folder),
"package": expectedStatePath(data.DeletedState, pkg),
@ -615,7 +617,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"root": expectedPath(""),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{},
expectedItemCount: 0,
expectedFileCount: 0,
@ -638,7 +640,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"subfolder": expectedPath("/folder/subfolder"),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: map[string]statePath{
"root": expectedStatePath(data.NotMovedState, ""),
"folder": expectedStatePath(data.DeletedState, folder),
@ -662,7 +664,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
"root": expectedPath(""),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedItemCount: 1,
expectedFileCount: 1,
expectedContainerCount: 0,
@ -702,7 +704,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
outputFolderMap,
excludes,
false,
)
fault.New(true))
tt.expect(t, err)
assert.Equal(t, len(tt.expectedCollectionIDs), len(c.CollectionMap), "total collections")
assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")
@ -771,7 +773,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
folderID1: path1,
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "MissingPaths",
@ -787,7 +789,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: map[string]string{},
expectedPaths: map[string]map[string]string{},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "MissingDeltas",
@ -807,7 +809,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: map[string]string{},
expectedPaths: map[string]map[string]string{},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
// An empty path map but valid delta results in metadata being returned
@ -832,7 +834,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: map[string]string{driveID1: deltaURL1},
expectedPaths: map[string]map[string]string{driveID1: {}},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
// An empty delta map but valid path results in no metadata for that drive
@ -861,7 +863,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: map[string]string{},
expectedPaths: map[string]map[string]string{},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "SuccessTwoDrivesTwoCollections",
@ -911,7 +913,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
folderID2: path2,
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
// Bad formats are logged but skip adding entries to the maps and don't
@ -929,7 +931,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: map[string]string{},
expectedPaths: map[string]map[string]string{},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
// Unexpected files are logged and skipped. They don't cause an error to
@ -965,7 +967,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
folderID1: path1,
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
},
{
name: "DriveAlreadyFound_Paths",
@ -1001,7 +1003,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: nil,
expectedPaths: nil,
errCheck: assert.Error,
errCheck: aw.Err,
},
{
name: "DriveAlreadyFound_Deltas",
@ -1033,7 +1035,7 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
},
expectedDeltas: nil,
expectedPaths: nil,
errCheck: assert.Error,
errCheck: aw.Err,
},
}
@ -1053,12 +1055,12 @@ func (suite *OneDriveCollectionsSuite) TestDeserializeMetadata() {
c(),
func(*support.ConnectorOperationStatus) {},
)
require.NoError(t, err)
aw.MustNoErr(t, err)
cols = append(cols, data.NotFoundRestoreCollection{Collection: mc})
}
deltas, paths, err := deserializeMetadata(ctx, cols)
deltas, paths, err := deserializeMetadata(ctx, cols, fault.New(true))
test.errCheck(t, err)
assert.Equal(t, test.expectedDeltas, deltas)
@ -1137,7 +1139,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
path.FilesCategory,
false,
)
require.NoError(suite.T(), err, "making metadata path")
aw.MustNoErr(suite.T(), err, "making metadata path")
empty := ""
next := "next"
@ -1194,7 +1196,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {"root": rootFolderPath1},
},
@ -1223,7 +1225,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {"root": expectedPath1("")},
},
@ -1253,7 +1255,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
},
@ -1287,7 +1289,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
},
@ -1322,7 +1324,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
},
@ -1369,7 +1371,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
driveID2: {},
@ -1406,7 +1408,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.Error,
errCheck: aw.Err,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
},
@ -1432,7 +1434,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
expectedCollections: map[string]map[data.CollectionState][]string{
expectedPath1(""): {data.NotMovedState: {"file"}},
},
@ -1472,7 +1474,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
expectedCollections: map[string]map[data.CollectionState][]string{
expectedPath1(""): {data.NotMovedState: {"file", "folder"}},
expectedPath1("/folder"): {data.NewState: {"file"}},
@ -1511,7 +1513,7 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
},
},
errCheck: assert.NoError,
errCheck: aw.NoErr,
prevFolderPaths: map[string]map[string]string{
driveID1: {},
},
@ -1594,10 +1596,10 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
},
func(*support.ConnectorOperationStatus) {},
)
assert.NoError(t, err, "creating metadata collection")
aw.NoErr(t, err, "creating metadata collection")
prevMetadata := []data.RestoreCollection{data.NotFoundRestoreCollection{Collection: mc}}
cols, delList, err := c.Get(ctx, prevMetadata)
cols, delList, err := c.Get(ctx, prevMetadata, fault.New(true))
test.errCheck(t, err)
if err != nil {
@ -1607,10 +1609,13 @@ func (suite *OneDriveCollectionsSuite) TestGet() {
for _, baseCol := range cols {
folderPath := baseCol.FullPath().String()
if folderPath == metadataPath.String() {
deltas, paths, err := deserializeMetadata(ctx, []data.RestoreCollection{
data.NotFoundRestoreCollection{Collection: baseCol},
})
if !assert.NoError(t, err, "deserializing metadata") {
deltas, paths, err := deserializeMetadata(
ctx,
[]data.RestoreCollection{
data.NotFoundRestoreCollection{Collection: baseCol},
},
fault.New(true))
if !aw.NoErr(t, err, "deserializing metadata") {
continue
}
@ -1804,6 +1809,7 @@ func (suite *OneDriveCollectionsSuite) TestCollectItems() {
newPaths map[string]string,
excluded map[string]struct{},
doNotMergeItems bool,
errs *fault.Bus,
) error {
return nil
}
@ -1816,9 +1822,9 @@ func (suite *OneDriveCollectionsSuite) TestCollectItems() {
collectorFunc,
map[string]string{},
test.prevDelta,
)
fault.New(true))
require.ErrorIs(suite.T(), err, test.err, "delta fetch err")
aw.MustErrIs(suite.T(), err, test.err, "delta fetch err")
require.Equal(suite.T(), test.deltaURL, delta.URL, "delta url")
require.Equal(suite.T(), !test.prevDeltaSuccess, delta.Reset, "delta reset")
})

View File

@ -6,9 +6,9 @@ import (
"strings"
"time"
"github.com/alcionai/clues"
msdrive "github.com/microsoftgraph/msgraph-sdk-go/drive"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
"github.com/pkg/errors"
"golang.org/x/exp/maps"
@ -16,26 +16,24 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
gapi "github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/onedrive/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
)
var errFolderNotFound = errors.New("folder not found")
var errFolderNotFound = clues.New("folder not found")
const (
getDrivesRetries = 3
// nextLinkKey is used to find the next link in a paged
// graph response
nextLinkKey = "@odata.nextLink"
itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children"
itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s"
itemNotFoundErrorCode = "itemNotFound"
userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL"
userMysiteURLNotFoundMsg = "Unable to retrieve user's mysite URL"
userMysiteNotFound = "ResourceNotFound User's mysite not found"
userMysiteNotFoundMsg = "User's mysite not found"
contextDeadlineExceeded = "context deadline exceeded"
nextLinkKey = "@odata.nextLink"
itemChildrenRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s/children"
itemByPathRawURLFmt = "https://graph.microsoft.com/v1.0/drives/%s/items/%s:/%s"
itemNotFoundErrorCode = "itemNotFound"
userMysiteURLNotFound = "BadRequest Unable to retrieve user's mysite URL"
userMysiteNotFound = "ResourceNotFound User's mysite not found"
contextDeadlineExceeded = "context deadline exceeded"
)
// DeltaUpdate holds the results of a current delta token. It normally
@ -77,8 +75,6 @@ func drives(
retry bool,
) ([]models.Driveable, error) {
var (
err error
page gapi.PageLinker
numberOfRetries = getDrivesRetries
drives = []models.Driveable{}
)
@ -89,30 +85,26 @@ func drives(
// Loop through all pages returned by Graph API.
for {
var (
err error
page gapi.PageLinker
)
// Retry Loop for Drive retrieval. Request can timeout
for i := 0; i <= numberOfRetries; i++ {
page, err = pager.GetPage(ctx)
if err != nil {
// Various error handling. May return an error or perform a retry.
detailedError := support.ConnectorStackErrorTraceWrap(err, "").Error()
if strings.Contains(detailedError, userMysiteURLNotFound) ||
strings.Contains(detailedError, userMysiteURLNotFoundMsg) ||
strings.Contains(detailedError, userMysiteNotFound) ||
strings.Contains(detailedError, userMysiteNotFoundMsg) {
if clues.HasLabel(err, graph.Labels.MysiteNotFound) {
logger.Ctx(ctx).Infof("resource owner does not have a drive")
return make([]models.Driveable, 0), nil // no license or drives.
}
if strings.Contains(detailedError, contextDeadlineExceeded) && i < numberOfRetries {
if errors.Is(err, context.DeadlineExceeded) && i < numberOfRetries {
time.Sleep(time.Duration(3*(i+1)) * time.Second)
continue
}
return nil, errors.Wrapf(
err,
"failed to retrieve drives. details: %s",
detailedError,
)
return nil, graph.Wrap(ctx, err, "retrieving drives")
}
// No error encountered, break the retry loop so we can extract results
@ -122,7 +114,7 @@ func drives(
tmp, err := pager.ValuesIn(page)
if err != nil {
return nil, errors.Wrap(err, "extracting drives from response")
return nil, graph.Wrap(ctx, err, "extracting drives from response")
}
drives = append(drives, tmp...)
@ -135,7 +127,7 @@ func drives(
pager.SetNext(nextLink)
}
logger.Ctx(ctx).Debugf("Found %d drives", len(drives))
logger.Ctx(ctx).Debugf("retrieved %d valid drives", len(drives))
return drives, nil
}
@ -149,6 +141,7 @@ type itemCollector func(
newPaths map[string]string,
excluded map[string]struct{},
validPrevDelta bool,
errs *fault.Bus,
) error
type itemPager interface {
@ -193,6 +186,7 @@ func collectItems(
collector itemCollector,
oldPaths map[string]string,
prevDelta string,
errs *fault.Bus,
) (DeltaUpdate, map[string]string, map[string]struct{}, error) {
var (
newDeltaURL = ""
@ -220,19 +214,15 @@ func collectItems(
}
if err != nil {
return DeltaUpdate{}, nil, nil, errors.Wrapf(
err,
"failed to query drive items. details: %s",
support.ConnectorStackErrorTrace(err),
)
return DeltaUpdate{}, nil, nil, graph.Wrap(ctx, err, "getting page")
}
vals, err := pager.ValuesIn(page)
if err != nil {
return DeltaUpdate{}, nil, nil, errors.Wrap(err, "extracting items from response")
return DeltaUpdate{}, nil, nil, graph.Wrap(ctx, err, "extracting items from response")
}
err = collector(ctx, driveID, driveName, vals, oldPaths, newPaths, excluded, invalidPrevDelta)
err = collector(ctx, driveID, driveName, vals, oldPaths, newPaths, excluded, invalidPrevDelta, errs)
if err != nil {
return DeltaUpdate{}, nil, nil, err
}
@ -277,25 +267,16 @@ func getFolder(
foundItem, err = builder.Get(ctx, nil)
if err != nil {
var oDataError *odataerrors.ODataError
if errors.As(err, &oDataError) &&
oDataError.GetError() != nil &&
oDataError.GetError().GetCode() != nil &&
*oDataError.GetError().GetCode() == itemNotFoundErrorCode {
return nil, errors.WithStack(errFolderNotFound)
if graph.IsErrDeletedInFlight(err) {
return nil, graph.Stack(ctx, clues.Stack(errFolderNotFound, err))
}
return nil, errors.Wrapf(err,
"failed to get folder %s/%s. details: %s",
parentFolderID,
folderName,
support.ConnectorStackErrorTrace(err),
)
return nil, graph.Wrap(ctx, err, "getting folder")
}
// Check if the item found is a folder, fail the call if not
if foundItem.GetFolder() == nil {
return nil, errors.WithStack(errFolderNotFound)
return nil, graph.Stack(ctx, errFolderNotFound)
}
return foundItem, nil
@ -311,16 +292,11 @@ func createItem(
// Graph SDK doesn't yet provide a POST method for `/children` so we set the `rawUrl` ourselves as recommended
// here: https://github.com/microsoftgraph/msgraph-sdk-go/issues/155#issuecomment-1136254310
rawURL := fmt.Sprintf(itemChildrenRawURLFmt, driveID, parentFolderID)
builder := msdrive.NewItemsRequestBuilder(rawURL, service.Adapter())
newItem, err := builder.Post(ctx, newItem, nil)
if err != nil {
return nil, errors.Wrapf(
err,
"failed to create item. details: %s",
support.ConnectorStackErrorTrace(err),
)
return nil, graph.Wrap(ctx, err, "creating item")
}
return newItem, nil
@ -356,65 +332,71 @@ func GetAllFolders(
gs graph.Servicer,
pager drivePager,
prefix string,
errs *fault.Bus,
) ([]*Displayable, error) {
drives, err := drives(ctx, pager, true)
if err != nil {
return nil, errors.Wrap(err, "getting OneDrive folders")
}
folders := map[string]*Displayable{}
var (
folders = map[string]*Displayable{}
el = errs.Local()
)
for _, d := range drives {
_, _, _, err = collectItems(
ctx,
defaultItemPager(
gs,
*d.GetId(),
"",
),
*d.GetId(),
*d.GetName(),
func(
innerCtx context.Context,
driveID, driveName string,
items []models.DriveItemable,
oldPaths map[string]string,
newPaths map[string]string,
excluded map[string]struct{},
doNotMergeItems bool,
) error {
for _, item := range items {
// Skip the root item.
if item.GetRoot() != nil {
continue
}
if el.Failure() != nil {
break
}
// Only selecting folders right now, not packages.
if item.GetFolder() == nil {
continue
}
var (
id = ptr.Val(d.GetId())
name = ptr.Val(d.GetName())
)
if item.GetId() == nil || len(*item.GetId()) == 0 {
logger.Ctx(ctx).Warn("folder without ID")
continue
}
if !strings.HasPrefix(*item.GetName(), prefix) {
continue
}
// Add the item instead of the folder because the item has more
// functionality.
folders[*item.GetId()] = &Displayable{item}
ictx := clues.Add(ctx, "drive_id", id, "drive_name", name) // TODO: pii
collector := func(
innerCtx context.Context,
driveID, driveName string,
items []models.DriveItemable,
oldPaths map[string]string,
newPaths map[string]string,
excluded map[string]struct{},
doNotMergeItems bool,
errs *fault.Bus,
) error {
for _, item := range items {
// Skip the root item.
if item.GetRoot() != nil {
continue
}
return nil
},
map[string]string{},
"",
)
// Only selecting folders right now, not packages.
if item.GetFolder() == nil {
continue
}
itemID := ptr.Val(item.GetId())
if len(itemID) == 0 {
logger.Ctx(ctx).Info("folder missing ID")
continue
}
if !strings.HasPrefix(*item.GetName(), prefix) {
continue
}
// Add the item instead of the folder because the item has more
// functionality.
folders[itemID] = &Displayable{item}
}
return nil
}
_, _, _, err = collectItems(ictx, defaultItemPager(gs, id, ""), id, name, collector, map[string]string{}, "", errs)
if err != nil {
return nil, errors.Wrapf(err, "getting items for drive %s", *d.GetName())
el.AddRecoverable(clues.Wrap(err, "enumerating items in drive"))
}
}
@ -424,7 +406,7 @@ func GetAllFolders(
res = append(res, f)
}
return res, nil
return res, el.Failure()
}
func DeleteItem(
@ -435,7 +417,7 @@ func DeleteItem(
) error {
err := gs.Client().DrivesById(driveID).ItemsById(itemID).Delete(ctx, nil)
if err != nil {
return errors.Wrapf(err, "deleting item with ID %s", itemID)
return graph.Wrap(ctx, err, "deleting item").With("item_id", itemID)
}
return nil

View File

@ -15,9 +15,10 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/graph/api"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/selectors"
)
@ -94,18 +95,9 @@ func (suite *OneDriveUnitSuite) TestDrives() {
// These errors won't be the "correct" format when compared to what graph
// returns, but they're close enough to have the same info when the inner
// details are extracted via support package.
mySiteURLNotFound := support.ConnectorStackErrorTraceWrap(
odErr(userMysiteURLNotFound),
"maximum retries or unretryable",
)
mySiteNotFound := support.ConnectorStackErrorTraceWrap(
odErr(userMysiteNotFound),
"maximum retries or unretryable",
)
deadlineExceeded := support.ConnectorStackErrorTraceWrap(
odErr(contextDeadlineExceeded),
"maximum retries or unretryable",
)
mySiteURLNotFound := odErr(userMysiteURLNotFound)
mySiteNotFound := odErr(userMysiteNotFound)
deadlineExceeded := odErr(contextDeadlineExceeded)
resultDrives := make([]models.Driveable, 0, numDriveResults)
@ -142,7 +134,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: false,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: resultDrives,
},
{
@ -155,7 +147,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: false,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: resultDrives,
},
{
@ -173,7 +165,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: false,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: resultDrives,
},
{
@ -191,7 +183,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: false,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: resultDrives,
},
{
@ -209,7 +201,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: true,
expectedErr: assert.Error,
expectedErr: aw.Err,
expectedResults: nil,
},
{
@ -222,7 +214,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: true,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: nil,
},
{
@ -235,7 +227,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: true,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: nil,
},
{
@ -258,7 +250,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: true,
expectedErr: assert.NoError,
expectedErr: aw.NoErr,
expectedResults: resultDrives,
},
{
@ -281,7 +273,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
},
},
retry: false,
expectedErr: assert.Error,
expectedErr: aw.Err,
expectedResults: nil,
},
{
@ -297,7 +289,7 @@ func (suite *OneDriveUnitSuite) TestDrives() {
tooManyRetries...,
),
retry: true,
expectedErr: assert.Error,
expectedErr: aw.Err,
expectedResults: nil,
},
}
@ -349,10 +341,10 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
gs := loadTestService(t)
pager, err := PagerForSource(OneDriveSource, gs, suite.userID, nil)
require.NoError(t, err)
aw.MustNoErr(t, err)
drives, err := drives(ctx, pager, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
require.NotEmpty(t, drives)
// TODO: Verify the intended drive
@ -368,7 +360,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
}()
folderID, err := CreateRestoreFolders(ctx, gs, driveID, folderElements)
require.NoError(t, err)
aw.MustNoErr(t, err)
folderIDs = append(folderIDs, folderID)
@ -376,7 +368,7 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
folderElements = append(folderElements, folderName2)
folderID, err = CreateRestoreFolders(ctx, gs, driveID, folderElements)
require.NoError(t, err)
aw.MustNoErr(t, err)
folderIDs = append(folderIDs, folderID)
@ -397,10 +389,10 @@ func (suite *OneDriveSuite) TestCreateGetDeleteFolder() {
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
pager, err := PagerForSource(OneDriveSource, gs, suite.userID, nil)
require.NoError(t, err)
aw.MustNoErr(t, err)
allFolders, err := GetAllFolders(ctx, gs, pager, test.prefix)
require.NoError(t, err)
allFolders, err := GetAllFolders(ctx, gs, pager, test.prefix, fault.New(true))
aw.MustNoErr(t, err)
foundFolderIDs := []string{}
@ -435,7 +427,7 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() {
defer flush()
creds, err := tester.NewM365Account(suite.T()).M365Config()
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
tests := []struct {
name, user string
@ -465,8 +457,8 @@ func (suite *OneDriveSuite) TestOneDriveNewCollections() {
service,
service.updateStatus,
control.Options{ToggleFeatures: control.Toggles{EnablePermissionsBackup: true}},
).Get(ctx, nil)
assert.NoError(t, err)
).Get(ctx, nil, fault.New(true))
aw.NoErr(t, err)
// Don't expect excludes as this isn't an incremental backup.
assert.Empty(t, excludes)

View File

@ -4,17 +4,17 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"github.com/alcionai/clues"
msdrives "github.com/microsoftgraph/msgraph-sdk-go/drives"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/connector/uploadsession"
"github.com/alcionai/corso/src/internal/version"
"github.com/alcionai/corso/src/pkg/backup/details"
@ -33,7 +33,12 @@ func getDriveItem(
srv graph.Servicer,
driveID, itemID string,
) (models.DriveItemable, error) {
return srv.Client().DrivesById(driveID).ItemsById(itemID).Get(ctx, nil)
di, err := srv.Client().DrivesById(driveID).ItemsById(itemID).Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting item")
}
return di, nil
}
// sharePointItemReader will return a io.ReadCloser for the specified item
@ -41,10 +46,11 @@ func getDriveItem(
// and using a http client to initialize a reader
// TODO: Add metadata fetching to SharePoint
func sharePointItemReader(
ctx context.Context,
hc *http.Client,
item models.DriveItemable,
) (details.ItemInfo, io.ReadCloser, error) {
resp, err := downloadItem(hc, item)
resp, err := downloadItem(ctx, hc, item)
if err != nil {
return details.ItemInfo{}, nil, errors.Wrap(err, "downloading item")
}
@ -69,7 +75,7 @@ func oneDriveItemMetaReader(
metaJSON, err := json.Marshal(meta)
if err != nil {
return nil, 0, err
return nil, 0, clues.Wrap(err, "marshalling json").WithClues(ctx)
}
return io.NopCloser(bytes.NewReader(metaJSON)), len(metaJSON), nil
@ -79,6 +85,7 @@ func oneDriveItemMetaReader(
// It crafts this by querying M365 for a download URL for the item
// and using a http client to initialize a reader
func oneDriveItemReader(
ctx context.Context,
hc *http.Client,
item models.DriveItemable,
) (details.ItemInfo, io.ReadCloser, error) {
@ -88,7 +95,7 @@ func oneDriveItemReader(
)
if isFile {
resp, err := downloadItem(hc, item)
resp, err := downloadItem(ctx, hc, item)
if err != nil {
return details.ItemInfo{}, nil, errors.Wrap(err, "downloading item")
}
@ -103,15 +110,15 @@ func oneDriveItemReader(
return dii, rc, nil
}
func downloadItem(hc *http.Client, item models.DriveItemable) (*http.Response, error) {
func downloadItem(ctx context.Context, hc *http.Client, item models.DriveItemable) (*http.Response, error) {
url, ok := item.GetAdditionalData()[downloadURLKey].(*string)
if !ok {
return nil, fmt.Errorf("extracting file url: file %s", *item.GetId())
return nil, clues.New("extracting file url").With("item_id", ptr.Val(item.GetId()))
}
req, err := http.NewRequest(http.MethodGet, *url, nil)
if err != nil {
return nil, errors.Wrap(err, "new request")
return nil, graph.Wrap(ctx, err, "new request")
}
//nolint:lll
@ -128,23 +135,12 @@ func downloadItem(hc *http.Client, item models.DriveItemable) (*http.Response, e
return resp, nil
}
if resp.StatusCode == http.StatusTooManyRequests {
return resp, graph.Err429TooManyRequests
}
// upstream error checks can compare the status with
// clues.HasLabel(err, graph.LabelStatus(http.KnownStatusCode))
cerr := clues.Wrap(clues.New(resp.Status), "non-2xx http response").
Label(graph.LabelStatus(resp.StatusCode))
if resp.StatusCode == http.StatusUnauthorized {
return resp, graph.Err401Unauthorized
}
if resp.StatusCode == http.StatusInternalServerError {
return resp, graph.Err500InternalServerError
}
if resp.StatusCode == http.StatusServiceUnavailable {
return resp, graph.Err503ServiceUnavailable
}
return resp, errors.New("non-2xx http response: " + resp.Status)
return resp, cerr
}
// oneDriveItemInfo will populate a details.OneDriveInfo struct
@ -171,9 +167,9 @@ func oneDriveItemInfo(di models.DriveItemable, itemSize int64) *details.OneDrive
return &details.OneDriveInfo{
ItemType: details.OneDriveItem,
ItemName: *di.GetName(),
Created: *di.GetCreatedDateTime(),
Modified: *di.GetLastModifiedDateTime(),
ItemName: ptr.Val(di.GetName()),
Created: ptr.Val(di.GetCreatedDateTime()),
Modified: ptr.Val(di.GetLastModifiedDateTime()),
DriveName: parent,
Size: itemSize,
Owner: email,
@ -187,11 +183,13 @@ func oneDriveItemMetaInfo(
ctx context.Context, service graph.Servicer,
driveID string, di models.DriveItemable,
) (Metadata, error) {
itemID := di.GetId()
perm, err := service.Client().DrivesById(driveID).ItemsById(*itemID).Permissions().Get(ctx, nil)
perm, err := service.Client().
DrivesById(driveID).
ItemsById(ptr.Val(di.GetId())).
Permissions().
Get(ctx, nil)
if err != nil {
return Metadata{}, err
return Metadata{}, graph.Wrap(ctx, err, "getting item metadata")
}
uperms := filterUserPermissions(perm.GetValue())
@ -223,7 +221,7 @@ func filterUserPermissions(perms []models.Permissionable) []UserPermission {
}
up = append(up, UserPermission{
ID: *p.GetId(),
ID: ptr.Val(p.GetId()),
Roles: roles,
Email: *p.GetGrantedToV2().GetUser().GetAdditionalData()["email"].(*string),
Expiration: p.GetExpirationDateTime(),
@ -275,9 +273,9 @@ func sharePointItemInfo(di models.DriveItemable, itemSize int64) *details.ShareP
return &details.SharePointInfo{
ItemType: details.OneDriveItem,
ItemName: *di.GetName(),
Created: *di.GetCreatedDateTime(),
Modified: *di.GetLastModifiedDateTime(),
ItemName: ptr.Val(di.GetName()),
Created: ptr.Val(di.GetCreatedDateTime()),
Modified: ptr.Val(di.GetLastModifiedDateTime()),
DriveName: parent,
Size: itemSize,
Owner: id,
@ -295,20 +293,16 @@ func driveItemWriter(
itemSize int64,
) (io.Writer, error) {
session := msdrives.NewItemItemsItemCreateUploadSessionPostRequestBody()
ctx = clues.Add(ctx, "upload_item_id", itemID)
r, err := service.Client().DrivesById(driveID).ItemsById(itemID).CreateUploadSession().Post(ctx, session, nil)
if err != nil {
return nil, errors.Wrapf(
err,
"failed to create upload session for item %s. details: %s",
itemID,
support.ConnectorStackErrorTrace(err),
)
return nil, graph.Wrap(ctx, err, "creating item upload session")
}
url := *r.GetUploadUrl()
logger.Ctx(ctx).Debug("created an upload session")
logger.Ctx(ctx).Debugf("Created an upload session for item %s. URL: %s", itemID, url)
url := ptr.Val(r.GetUploadUrl())
return uploadsession.NewWriter(itemID, url, itemSize), nil
}

View File

@ -15,6 +15,8 @@ import (
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/fault"
)
type ItemIntegrationSuite struct {
@ -55,10 +57,10 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
adapter, err := graph.CreateAdapter(m365.AzureTenantID, m365.AzureClientID, m365.AzureClientSecret)
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.client = msgraphsdk.NewGraphServiceClient(adapter)
suite.adapter = adapter
@ -67,7 +69,7 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
// and guarateed drive read-write access)
// suite.site = tester.M365SiteID(t)
// spDrives, err := drives(ctx, suite, suite.site, SharePointSource)
// require.NoError(t, err)
// aw.MustNoErr(t, err)
// // Test Requirement 1: Need a drive
// require.Greaterf(t, len(spDrives), 0, "site %s does not have a drive", suite.site)
@ -77,10 +79,10 @@ func (suite *ItemIntegrationSuite) SetupSuite() {
suite.user = tester.SecondaryM365UserID(t)
pager, err := PagerForSource(OneDriveSource, suite, suite.user, nil)
require.NoError(t, err)
aw.MustNoErr(t, err)
odDrives, err := drives(ctx, pager, true)
require.NoError(t, err)
aw.MustNoErr(t, err)
// Test Requirement 1: Need a drive
require.Greaterf(t, len(odDrives), 0, "user %s does not have a drive", suite.user)
@ -107,6 +109,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
newPaths map[string]string,
excluded map[string]struct{},
doNotMergeItems bool,
errs *fault.Bus,
) error {
for _, item := range items {
if item.GetFile() != nil {
@ -129,8 +132,8 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
itemCollector,
map[string]string{},
"",
)
require.NoError(suite.T(), err)
fault.New(true))
aw.MustNoErr(suite.T(), err)
// Test Requirement 2: Need a file
require.NotEmpty(
@ -142,14 +145,14 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
)
// Read data for the file
itemInfo, itemData, err := oneDriveItemReader(graph.HTTPClient(graph.NoTimeout()), driveItem)
itemInfo, itemData, err := oneDriveItemReader(ctx, graph.HTTPClient(graph.NoTimeout()), driveItem)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
require.NotNil(suite.T(), itemInfo.OneDrive)
require.NotEmpty(suite.T(), itemInfo.OneDrive.ItemName)
size, err := io.Copy(io.Discard, itemData)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
require.NotZero(suite.T(), size)
require.Equal(suite.T(), size, itemInfo.OneDrive.Size)
suite.T().Logf("Read %d bytes from file %s.", size, itemInfo.OneDrive.ItemName)
@ -178,17 +181,17 @@ func (suite *ItemIntegrationSuite) TestItemWriter() {
defer flush()
root, err := suite.Client().DrivesById(test.driveID).Root().Get(ctx, nil)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
// Test Requirement 2: "Test Folder" should exist
folder, err := getFolder(ctx, suite, test.driveID, *root.GetId(), "Test Folder")
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
newFolderName := "testfolder_" + common.FormatNow(common.SimpleTimeTesting)
suite.T().Logf("Test will create folder %s", newFolderName)
newFolder, err := createItem(ctx, suite, test.driveID, *folder.GetId(), newItem(newFolderName, true))
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
require.NotNil(suite.T(), newFolder.GetId())
@ -196,20 +199,20 @@ func (suite *ItemIntegrationSuite) TestItemWriter() {
suite.T().Logf("Test will create item %s", newItemName)
newItem, err := createItem(ctx, suite, test.driveID, *newFolder.GetId(), newItem(newItemName, false))
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
require.NotNil(suite.T(), newItem.GetId())
// HACK: Leveraging this to test getFolder behavior for a file. `getFolder()` on the
// newly created item should fail because it's a file not a folder
_, err = getFolder(ctx, suite, test.driveID, *newFolder.GetId(), newItemName)
require.ErrorIs(suite.T(), err, errFolderNotFound)
aw.MustErrIs(suite.T(), err, errFolderNotFound)
// Initialize a 100KB mockDataProvider
td, writeSize := mockDataReader(int64(100 * 1024))
w, err := driveItemWriter(ctx, suite, test.driveID, *newItem.GetId(), writeSize)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
// Using a 32 KB buffer for the copy allows us to validate the
// multi-part upload. `io.CopyBuffer` will only write 32 KB at
@ -217,7 +220,7 @@ func (suite *ItemIntegrationSuite) TestItemWriter() {
copyBuffer := make([]byte, 32*1024)
size, err := io.CopyBuffer(w, td, copyBuffer)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
require.Equal(suite.T(), writeSize, size)
})
@ -249,15 +252,15 @@ func (suite *ItemIntegrationSuite) TestDriveGetFolder() {
defer flush()
root, err := suite.Client().DrivesById(test.driveID).Root().Get(ctx, nil)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
// Lookup a folder that doesn't exist
_, err = getFolder(ctx, suite, test.driveID, *root.GetId(), "FolderDoesNotExist")
require.ErrorIs(suite.T(), err, errFolderNotFound)
aw.MustErrIs(suite.T(), err, errFolderNotFound)
// Lookup a folder that does exist
_, err = getFolder(ctx, suite, test.driveID, *root.GetId(), "")
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
})
}
}

View File

@ -3,7 +3,6 @@ package onedrive
import (
"context"
"encoding/json"
"fmt"
"io"
"runtime/trace"
"sort"
@ -14,6 +13,7 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
@ -21,6 +21,7 @@ import (
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
)
@ -49,7 +50,7 @@ func getParentPermissions(
}
if len(onedrivePath.Folders) != 0 {
return nil, errors.Wrap(err, "unable to compute item permissions")
return nil, errors.Wrap(err, "computing item permissions")
}
parentPerms = []UserPermission{}
@ -69,7 +70,6 @@ func getParentAndCollectionPermissions(
}
var (
err error
parentPerms []UserPermission
colPerms []UserPermission
)
@ -89,7 +89,7 @@ func getParentAndCollectionPermissions(
// TODO(ashmrtn): For versions after this pull the permissions from the
// current collection with Fetch().
colPerms, err = getParentPermissions(collectionPath, permissions)
colPerms, err := getParentPermissions(collectionPath, permissions)
if err != nil {
return nil, nil, clues.Wrap(err, "getting collection permissions")
}
@ -106,22 +106,22 @@ func RestoreCollections(
opts control.Options,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Bus,
) (*support.ConnectorOperationStatus, error) {
var (
restoreMetrics support.CollectionMetrics
restoreErrors error
metrics support.CollectionMetrics
folderPerms map[string][]UserPermission
canceled bool
// permissionIDMappings is used to map between old and new id
// of permissions as we restore them
permissionIDMappings = map[string]string{}
)
errUpdater := func(id string, err error) {
restoreErrors = support.WrapAndAppend(id, err, restoreErrors)
}
ctx = clues.Add(
ctx,
"backup_version", backupVersion,
"destination", dest.ContainerName)
// Reorder collections so that the parents directories are created
// before the child directories
@ -129,12 +129,28 @@ func RestoreCollections(
return dcs[i].FullPath().String() < dcs[j].FullPath().String()
})
parentPermissions := map[string][]UserPermission{}
var (
el = errs.Local()
parentPermissions = map[string][]UserPermission{}
)
// Iterate through the data collections and restore the contents of each
for _, dc := range dcs {
metrics, folderPerms, permissionIDMappings, canceled = RestoreCollection(
ctx,
if el.Failure() != nil {
break
}
var (
err error
ictx = clues.Add(
ctx,
"resource_owner", dc.FullPath().ResourceOwner(), // TODO: pii
"category", dc.FullPath().Category(),
"path", dc.FullPath()) // TODO: pii
)
metrics, folderPerms, permissionIDMappings, err = RestoreCollection(
ictx,
backupVersion,
service,
dc,
@ -142,30 +158,32 @@ func RestoreCollections(
OneDriveSource,
dest.ContainerName,
deets,
errUpdater,
permissionIDMappings,
opts.RestorePermissions,
)
errs)
if err != nil {
el.AddRecoverable(err)
}
for k, v := range folderPerms {
parentPermissions[k] = v
}
restoreMetrics.Combine(metrics)
restoreMetrics = support.CombineMetrics(restoreMetrics, metrics)
if canceled {
if errors.Is(err, context.Canceled) {
break
}
}
return support.CreateStatus(
ctx,
support.Restore,
len(dcs),
restoreMetrics,
restoreErrors,
dest.ContainerName),
nil
status := support.CreateStatus(
ctx,
support.Restore,
len(dcs),
restoreMetrics,
dest.ContainerName)
return status, el.Failure()
}
// RestoreCollection handles restoration of an individual collection.
@ -181,10 +199,10 @@ func RestoreCollection(
source driveSource,
restoreContainerName string,
deets *details.Builder,
errUpdater func(string, error),
permissionIDMappings map[string]string,
restorePerms bool,
) (support.CollectionMetrics, map[string][]UserPermission, map[string]string, bool) {
errs *fault.Bus,
) (support.CollectionMetrics, map[string][]UserPermission, map[string]string, error) {
ctx, end := D.Span(ctx, "gc:oneDrive:restoreCollection", D.Label("path", dc.FullPath()))
defer end()
@ -199,8 +217,7 @@ func RestoreCollection(
drivePath, err := path.ToOneDrivePath(directory)
if err != nil {
errUpdater(directory.String(), err)
return metrics, folderPerms, permissionIDMappings, false
return metrics, folderPerms, permissionIDMappings, clues.Wrap(err, "creating drive path").WithClues(ctx)
}
// Assemble folder hierarchy we're going to restore into (we recreate the folder hierarchy
@ -210,11 +227,13 @@ func RestoreCollection(
restoreFolderElements := []string{restoreContainerName}
restoreFolderElements = append(restoreFolderElements, drivePath.Folders...)
ctx = clues.Add(
ctx,
"destination_elements", restoreFolderElements,
"drive_id", drivePath.DriveID)
trace.Log(ctx, "gc:oneDrive:restoreCollection", directory.String())
logger.Ctx(ctx).Infow(
"restoring to destination",
"origin", dc.FullPath().Folder(false),
"destination", restoreFolderElements)
logger.Ctx(ctx).Info("restoring onedrive collection")
parentPerms, colPerms, err := getParentAndCollectionPermissions(
drivePath,
@ -222,8 +241,7 @@ func RestoreCollection(
parentPermissions,
restorePerms)
if err != nil {
errUpdater(directory.String(), err)
return metrics, folderPerms, permissionIDMappings, false
return metrics, folderPerms, permissionIDMappings, clues.Wrap(err, "getting permissions").WithClues(ctx)
}
// Create restore folders and get the folder ID of the folder the data stream will be restored in
@ -237,38 +255,40 @@ func RestoreCollection(
permissionIDMappings,
)
if err != nil {
errUpdater(directory.String(), errors.Wrapf(err, "failed to create folders %v", restoreFolderElements))
return metrics, folderPerms, permissionIDMappings, false
return metrics, folderPerms, permissionIDMappings, clues.Wrap(err, "creating folders for restore")
}
// Restore items from the collection
items := dc.Items(ctx, nil) // TODO: fault.Errors instead of nil
var (
el = errs.Local()
items = dc.Items(ctx, errs)
)
for {
if el.Failure() != nil {
break
}
select {
case <-ctx.Done():
errUpdater("context canceled", ctx.Err())
return metrics, folderPerms, permissionIDMappings, true
return metrics, folderPerms, permissionIDMappings, err
case itemData, ok := <-items:
if !ok {
return metrics, folderPerms, permissionIDMappings, false
return metrics, folderPerms, permissionIDMappings, nil
}
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
logger.Ctx(ctx).DPanicw("transforming item to full path", "error", err)
errUpdater(itemData.UUID(), err)
el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx))
continue
}
if source == OneDriveSource && backupVersion >= versionWithDataAndMetaFiles {
name := itemData.UUID()
if strings.HasSuffix(name, DataFileSuffix) {
metrics.Objects++
metrics.TotalBytes += int64(len(copyBuffer))
metrics.Bytes += int64(len(copyBuffer))
trimmedName := strings.TrimSuffix(name, DataFileSuffix)
itemID, itemInfo, err = restoreData(
@ -281,7 +301,7 @@ func RestoreCollection(
copyBuffer,
source)
if err != nil {
errUpdater(itemData.UUID(), err)
el.AddRecoverable(err)
continue
}
@ -305,7 +325,7 @@ func RestoreCollection(
permsFile, err := dc.Fetch(ctx, metaName)
if err != nil {
errUpdater(metaName, clues.Wrap(err, "getting item metadata"))
el.AddRecoverable(clues.Wrap(err, "getting item metadata"))
continue
}
@ -314,7 +334,7 @@ func RestoreCollection(
metaReader.Close()
if err != nil {
errUpdater(metaName, clues.Wrap(err, "deserializing item metadata"))
el.AddRecoverable(clues.Wrap(err, "deserializing item metadata"))
continue
}
@ -325,10 +345,9 @@ func RestoreCollection(
itemID,
colPerms,
meta.Permissions,
permissionIDMappings,
)
permissionIDMappings)
if err != nil {
errUpdater(trimmedName, clues.Wrap(err, "restoring item permissions"))
el.AddRecoverable(clues.Wrap(err, "restoring item permissions"))
continue
}
@ -344,28 +363,25 @@ func RestoreCollection(
}
metaReader := itemData.ToReader()
meta, err := getMetadata(metaReader)
metaReader.Close()
defer metaReader.Close()
meta, err := getMetadata(metaReader)
if err != nil {
errUpdater(itemData.UUID(), clues.Wrap(err, "folder metadata"))
el.AddRecoverable(clues.Wrap(err, "getting directory metadata").WithClues(ctx))
continue
}
trimmedPath := strings.TrimSuffix(itemPath.String(), DirMetaFileSuffix)
folderPerms[trimmedPath] = meta.Permissions
} else {
if !ok {
errUpdater(itemData.UUID(), fmt.Errorf("invalid backup format, you might be using an old backup"))
continue
}
}
} else {
metrics.Objects++
metrics.TotalBytes += int64(len(copyBuffer))
metrics.Bytes += int64(len(copyBuffer))
// No permissions stored at the moment for SharePoint
_, itemInfo, err = restoreData(ctx,
_, itemInfo, err = restoreData(
ctx,
service,
itemData.UUID(),
itemData,
@ -374,7 +390,7 @@ func RestoreCollection(
copyBuffer,
source)
if err != nil {
errUpdater(itemData.UUID(), err)
el.AddRecoverable(err)
continue
}
@ -389,6 +405,8 @@ func RestoreCollection(
}
}
}
return metrics, folderPerms, permissionIDMappings, el.Failure()
}
// createRestoreFoldersWithPermissions creates the restore folder hierarchy in
@ -431,42 +449,31 @@ func CreateRestoreFolders(
) (string, error) {
driveRoot, err := service.Client().DrivesById(driveID).Root().Get(ctx, nil)
if err != nil {
return "", errors.Wrapf(
err,
"failed to get drive root. details: %s",
support.ConnectorStackErrorTrace(err),
)
return "", graph.Wrap(ctx, err, "getting drive root")
}
logger.Ctx(ctx).Debugf("Found Root for Drive %s with ID %s", driveID, *driveRoot.GetId())
parentFolderID := ptr.Val(driveRoot.GetId())
ctx = clues.Add(ctx, "drive_root_id", parentFolderID)
logger.Ctx(ctx).Debug("found drive root")
parentFolderID := *driveRoot.GetId()
for _, folder := range restoreFolders {
folderItem, err := getFolder(ctx, service, driveID, parentFolderID, folder)
if err == nil {
parentFolderID = *folderItem.GetId()
logger.Ctx(ctx).Debugf("Found %s with ID %s", folder, parentFolderID)
parentFolderID = ptr.Val(folderItem.GetId())
continue
}
if !errors.Is(err, errFolderNotFound) {
return "", errors.Wrapf(err, "folder %s not found in drive(%s) parentFolder(%s)", folder, driveID, parentFolderID)
if errors.Is(err, errFolderNotFound) {
return "", clues.Wrap(err, "folder not found").With("folder_id", folder).WithClues(ctx)
}
folderItem, err = createItem(ctx, service, driveID, parentFolderID, newItem(folder, true))
if err != nil {
return "", errors.Wrapf(
err,
"failed to create folder %s/%s. details: %s", parentFolderID, folder,
support.ConnectorStackErrorTrace(err),
)
return "", clues.Wrap(err, "creating folder")
}
logger.Ctx(ctx).Debugw("resolved restore destination",
"dest_name", folder,
"parent", parentFolderID,
"dest_id", *folderItem.GetId())
logger.Ctx(ctx).Debugw("resolved restore destination", "dest_id", *folderItem.GetId())
parentFolderID = *folderItem.GetId()
}
@ -487,25 +494,27 @@ func restoreData(
ctx, end := D.Span(ctx, "gc:oneDrive:restoreItem", D.Label("item_uuid", itemData.UUID()))
defer end()
ctx = clues.Add(ctx, "item_name", itemData.UUID())
itemName := itemData.UUID()
trace.Log(ctx, "gc:oneDrive:restoreItem", itemName)
// Get the stream size (needed to create the upload session)
ss, ok := itemData.(data.StreamSize)
if !ok {
return "", details.ItemInfo{}, errors.Errorf("item %q does not implement DataStreamInfo", itemName)
return "", details.ItemInfo{}, clues.New("item does not implement DataStreamInfo").WithClues(ctx)
}
// Create Item
newItem, err := createItem(ctx, service, driveID, parentFolderID, newItem(name, false))
if err != nil {
return "", details.ItemInfo{}, errors.Wrapf(err, "failed to create item %s", itemName)
return "", details.ItemInfo{}, clues.Wrap(err, "creating item")
}
// Get a drive item writer
w, err := driveItemWriter(ctx, service, driveID, *newItem.GetId(), ss.Size())
if err != nil {
return "", details.ItemInfo{}, errors.Wrapf(err, "failed to create item upload session %s", itemName)
return "", details.ItemInfo{}, clues.Wrap(err, "creating item writer")
}
iReader := itemData.ToReader()
@ -516,7 +525,7 @@ func restoreData(
// Upload the stream data
written, err := io.CopyBuffer(w, progReader, copyBuffer)
if err != nil {
return "", details.ItemInfo{}, errors.Wrapf(err, "failed to upload data: item %s", itemName)
return "", details.ItemInfo{}, graph.Wrap(ctx, err, "writing item bytes")
}
dii := details.ItemInfo{}
@ -607,16 +616,16 @@ func restorePermissions(
) (map[string]string, error) {
permAdded, permRemoved := getChildPermissions(childPerms, parentPerms)
ctx = clues.Add(ctx, "permission_item_id", itemID)
for _, p := range permRemoved {
err := service.Client().DrivesById(driveID).ItemsById(itemID).
PermissionsById(permissionIDMappings[p.ID]).Delete(ctx, nil)
err := service.Client().
DrivesById(driveID).
ItemsById(itemID).
PermissionsById(permissionIDMappings[p.ID]).
Delete(ctx, nil)
if err != nil {
return permissionIDMappings, errors.Wrapf(
err,
"failed to remove permission for item %s. details: %s",
itemID,
support.ConnectorStackErrorTrace(err),
)
return permissionIDMappings, graph.Wrap(ctx, err, "removing permissions")
}
}
@ -641,12 +650,7 @@ func restorePermissions(
np, err := service.Client().DrivesById(driveID).ItemsById(itemID).Invite().Post(ctx, pbody, nil)
if err != nil {
return permissionIDMappings, errors.Wrapf(
err,
"failed to set permission for item %s. details: %s",
itemID,
support.ConnectorStackErrorTrace(err),
)
return permissionIDMappings, graph.Wrap(ctx, err, "setting permissions")
}
permissionIDMappings[p.ID] = *np.GetValue()[0].GetId()

View File

@ -4,11 +4,11 @@ import (
"testing"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -68,10 +68,10 @@ func (ods *oneDriveService) updateStatus(status *support.ConnectorOperationStatu
func loadTestService(t *testing.T) *oneDriveService {
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
service, err := NewOneDriveService(m365)
require.NoError(t, err)
aw.MustNoErr(t, err)
return service
}

View File

@ -3,10 +3,9 @@ package api_test
import (
"testing"
"github.com/stretchr/testify/require"
discover "github.com/alcionai/corso/src/internal/connector/discovery/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -16,7 +15,7 @@ func createTestBetaService(t *testing.T, credentials account.M365Config) *discov
credentials.AzureClientID,
credentials.AzureClientSecret,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
return discover.NewBetaService(adapter)
}

View File

@ -28,13 +28,13 @@ func GetSitePages(
serv *discover.BetaService,
siteID string,
pages []string,
errs *fault.Errors,
errs *fault.Bus,
) ([]models.SitePageable, error) {
var (
col = make([]models.SitePageable, 0)
semaphoreCh = make(chan struct{}, fetchChannelSize)
opts = retrieveSitePageOptions()
err error
el = errs.Local()
wg sync.WaitGroup
m sync.Mutex
)
@ -49,7 +49,7 @@ func GetSitePages(
}
for _, entry := range pages {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -61,11 +61,14 @@ func GetSitePages(
defer wg.Done()
defer func() { <-semaphoreCh }()
var page models.SitePageable
var (
page models.SitePageable
err error
)
page, err = serv.Client().SitesById(siteID).PagesById(pageID).Get(ctx, opts)
if err != nil {
errs.Add(clues.Wrap(err, "fetching page").WithClues(ctx).With(graph.ErrData(err)...))
el.AddRecoverable(graph.Wrap(ctx, err, "fetching page"))
return
}
@ -75,7 +78,7 @@ func GetSitePages(
wg.Wait()
return col, errs.Err()
return col, el.Failure()
}
// fetchPages utility function to return the tuple of item
@ -91,7 +94,7 @@ func FetchPages(ctx context.Context, bs *discover.BetaService, siteID string) ([
for {
resp, err = builder.Get(ctx, opts)
if err != nil {
return nil, clues.Wrap(err, "fetching site page").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "fetching site page")
}
for _, entry := range resp.GetValue() {
@ -140,7 +143,7 @@ func DeleteSitePage(
) error {
err := serv.Client().SitesById(siteID).PagesById(pageID).Delete(ctx, nil)
if err != nil {
return clues.Wrap(err, "deleting page").WithClues(ctx).With(graph.ErrData(err)...)
return graph.Wrap(ctx, err, "deleting page")
}
return nil
@ -200,7 +203,7 @@ func RestoreSitePage(
// See: https://learn.microsoft.com/en-us/graph/api/sitepage-create?view=graph-rest-beta
restoredPage, err := service.Client().SitesById(siteID).Pages().Post(ctx, page, nil)
if err != nil {
return dii, clues.Wrap(err, "creating page").WithClues(ctx).With(graph.ErrData(err)...)
return dii, graph.Wrap(ctx, err, "creating page")
}
pageID = ptr.Val(restoredPage.GetId())
@ -218,7 +221,7 @@ func RestoreSitePage(
Publish().
Post(ctx, nil)
if err != nil {
return dii, clues.Wrap(err, "publishing page").WithClues(ctx).With(graph.ErrData(err)...)
return dii, graph.Wrap(ctx, err, "publishing page")
}
dii.SharePoint = PageInfo(restoredPage, int64(len(byteArray)))

View File

@ -15,6 +15,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/sharepoint"
"github.com/alcionai/corso/src/internal/connector/sharepoint/api"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -33,7 +34,7 @@ func (suite *SharePointPageSuite) SetupSuite() {
suite.siteID = tester.M365SiteID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.creds = m365
suite.service = createTestBetaService(t, suite.creds)
@ -53,7 +54,7 @@ func (suite *SharePointPageSuite) TestFetchPages() {
t := suite.T()
pgs, err := api.FetchPages(ctx, suite.service, suite.siteID)
assert.NoError(t, err)
aw.NoErr(t, err)
require.NotNil(t, pgs)
assert.NotZero(t, len(pgs))
@ -68,12 +69,12 @@ func (suite *SharePointPageSuite) TestGetSitePages() {
t := suite.T()
tuples, err := api.FetchPages(ctx, suite.service, suite.siteID)
require.NoError(t, err)
aw.MustNoErr(t, err)
require.NotNil(t, tuples)
jobs := []string{tuples[0].ID}
pages, err := api.GetSitePages(ctx, suite.service, suite.siteID, jobs, fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotEmpty(t, pages)
}
@ -103,11 +104,11 @@ func (suite *SharePointPageSuite) TestRestoreSinglePage() {
destName,
)
require.NoError(t, err)
aw.MustNoErr(t, err)
require.NotNil(t, info)
// Clean Up
pageID := info.SharePoint.ParentPath
err = api.DeleteSitePage(ctx, suite.service, suite.siteID, pageID)
assert.NoError(t, err)
aw.NoErr(t, err)
}

View File

@ -43,12 +43,6 @@ var (
_ data.StreamModTime = &Item{}
)
type numMetrics struct {
attempts int
success int
totalBytes int64
}
// Collection is the SharePoint.List implementation of data.Collection. SharePoint.Libraries collections are supported
// by the oneDrive.Collection as the calls are identical for populating the Collection
type Collection struct {
@ -112,7 +106,7 @@ func (sc Collection) DoNotMergeItems() bool {
func (sc *Collection) Items(
ctx context.Context,
errs *fault.Errors,
errs *fault.Bus,
) <-chan data.Stream {
go sc.populate(ctx, errs)
return sc.data
@ -157,24 +151,17 @@ func (sd *Item) ModTime() time.Time {
func (sc *Collection) finishPopulation(
ctx context.Context,
attempts, success int,
totalBytes int64,
err error,
metrics support.CollectionMetrics,
) {
close(sc.data)
attempted := attempts
status := support.CreateStatus(
ctx,
support.Backup,
1, // 1 folder
support.CollectionMetrics{
Objects: attempted,
Successes: success,
TotalBytes: totalBytes,
},
err,
metrics,
sc.fullPath.Folder(false))
logger.Ctx(ctx).Debug(status.String())
if sc.statusUpdater != nil {
@ -183,16 +170,17 @@ func (sc *Collection) finishPopulation(
}
// populate utility function to retrieve data from back store for a given collection
func (sc *Collection) populate(ctx context.Context, errs *fault.Errors) {
var (
metrics numMetrics
writer = kw.NewJsonSerializationWriter()
err error
)
func (sc *Collection) populate(ctx context.Context, errs *fault.Bus) {
metrics, _ := sc.runPopulate(ctx, errs)
sc.finishPopulation(ctx, metrics)
}
defer func() {
sc.finishPopulation(ctx, metrics.attempts, metrics.success, int64(metrics.totalBytes), err)
}()
func (sc *Collection) runPopulate(ctx context.Context, errs *fault.Bus) (support.CollectionMetrics, error) {
var (
err error
metrics support.CollectionMetrics
writer = kw.NewJsonSerializationWriter()
)
// TODO: Insert correct ID for CollectionProgress
colProgress, closer := observe.CollectionProgress(
@ -213,6 +201,8 @@ func (sc *Collection) populate(ctx context.Context, errs *fault.Errors) {
case Pages:
metrics, err = sc.retrievePages(ctx, writer, colProgress, errs)
}
return metrics, err
}
// retrieveLists utility function for collection that downloads and serializes
@ -221,26 +211,29 @@ func (sc *Collection) retrieveLists(
ctx context.Context,
wtr *kw.JsonSerializationWriter,
progress chan<- struct{},
errs *fault.Errors,
) (numMetrics, error) {
var metrics numMetrics
errs *fault.Bus,
) (support.CollectionMetrics, error) {
var (
metrics support.CollectionMetrics
el = errs.Local()
)
lists, err := loadSiteLists(ctx, sc.service, sc.fullPath.ResourceOwner(), sc.jobs, errs)
if err != nil {
return metrics, err
}
metrics.attempts += len(lists)
metrics.Objects += len(lists)
// For each models.Listable, object is serialized and the metrics are collected.
// The progress is objected via the passed in channel.
for _, lst := range lists {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
byteArray, err := serializeContent(wtr, lst)
byteArray, err := serializeContent(ctx, wtr, lst)
if err != nil {
errs.Add(clues.Wrap(err, "serializing list").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "serializing list").WithClues(ctx))
continue
}
@ -252,9 +245,9 @@ func (sc *Collection) retrieveLists(
t = *t1
}
metrics.totalBytes += size
metrics.Bytes += size
metrics.success++
metrics.Successes++
sc.data <- &Item{
id: *lst.GetId(),
data: io.NopCloser(bytes.NewReader(byteArray)),
@ -266,16 +259,19 @@ func (sc *Collection) retrieveLists(
}
}
return metrics, errs.Err()
return metrics, el.Failure()
}
func (sc *Collection) retrievePages(
ctx context.Context,
wtr *kw.JsonSerializationWriter,
progress chan<- struct{},
errs *fault.Errors,
) (numMetrics, error) {
var metrics numMetrics
errs *fault.Bus,
) (support.CollectionMetrics, error) {
var (
metrics support.CollectionMetrics
el = errs.Local()
)
betaService := sc.betaService
if betaService == nil {
@ -287,26 +283,26 @@ func (sc *Collection) retrievePages(
return metrics, err
}
metrics.attempts = len(pages)
metrics.Objects = len(pages)
// For each models.Pageable, object is serialize and the metrics are collected and returned.
// Pageable objects are not supported in v1.0 of msgraph at this time.
// TODO: Verify Parsable interface supported with modified-Pageable
for _, pg := range pages {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
byteArray, err := serializeContent(wtr, pg)
byteArray, err := serializeContent(ctx, wtr, pg)
if err != nil {
errs.Add(clues.Wrap(err, "serializing page").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "serializing page").WithClues(ctx))
continue
}
size := int64(len(byteArray))
if size > 0 {
metrics.totalBytes += size
metrics.success++
metrics.Bytes += size
metrics.Successes++
sc.data <- &Item{
id: *pg.GetId(),
data: io.NopCloser(bytes.NewReader(byteArray)),
@ -318,20 +314,24 @@ func (sc *Collection) retrievePages(
}
}
return metrics, errs.Err()
return metrics, el.Failure()
}
func serializeContent(writer *kw.JsonSerializationWriter, obj absser.Parsable) ([]byte, error) {
func serializeContent(
ctx context.Context,
writer *kw.JsonSerializationWriter,
obj absser.Parsable,
) ([]byte, error) {
defer writer.Close()
err := writer.WriteObjectValue("", obj)
if err != nil {
return nil, clues.Wrap(err, "writing object").With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "writing object")
}
byteArray, err := writer.GetSerializedContent()
if err != nil {
return nil, clues.Wrap(err, "getting content from writer").With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting content from writer")
}
return byteArray, nil

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
@ -37,7 +38,7 @@ func (suite *SharePointCollectionSuite) SetupSuite() {
suite.siteID = tester.M365SiteID(t)
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.creds = m365
}
@ -61,7 +62,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Item_Read() {
data: io.NopCloser(bytes.NewReader(m)),
}
readData, err := io.ReadAll(sc.ToReader())
require.NoError(t, err)
aw.MustNoErr(t, err)
assert.Equal(t, name, sc.id)
assert.Equal(t, readData, m)
@ -91,7 +92,7 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
user,
path.ListsCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return dir
},
@ -101,10 +102,10 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
listing.SetDisplayName(&name)
err := ow.WriteObjectValue("", listing)
require.NoError(t, err)
aw.MustNoErr(t, err)
byteArray, err := ow.GetSerializedContent()
require.NoError(t, err)
aw.MustNoErr(t, err)
data := &Item{
id: name,
@ -126,14 +127,14 @@ func (suite *SharePointCollectionSuite) TestCollection_Items() {
user,
path.PagesCategory,
false)
require.NoError(t, err)
aw.MustNoErr(t, err)
return dir
},
getItem: func(t *testing.T, itemName string) *Item {
byteArray := mockconnector.GetMockPage(itemName)
page, err := support.CreatePageFromBytes(byteArray)
require.NoError(t, err)
aw.MustNoErr(t, err)
data := &Item{
id: itemName,
@ -183,7 +184,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
testName := "MockListing"
listing.SetDisplayName(&testName)
byteArray, err := service.Serialize(listing)
require.NoError(t, err)
aw.MustNoErr(t, err)
listData := &Item{
id: testName,
@ -194,7 +195,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting)
deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName)
assert.NoError(t, err)
aw.NoErr(t, err)
t.Logf("List created: %s\n", deets.SharePoint.ItemName)
// Clean-Up
@ -206,7 +207,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
for {
resp, err := builder.Get(ctx, nil)
assert.NoError(t, err, "experienced query error during clean up. Details: "+support.ConnectorStackErrorTrace(err))
aw.NoErr(t, err, "getting site lists")
for _, temp := range resp.GetValue() {
if *temp.GetDisplayName() == deets.SharePoint.ItemName {
@ -227,7 +228,7 @@ func (suite *SharePointCollectionSuite) TestListCollection_Restore() {
if isFound {
err := DeleteList(ctx, service, suite.siteID, deleteID)
assert.NoError(t, err)
aw.NoErr(t, err)
}
}
@ -242,17 +243,17 @@ func (suite *SharePointCollectionSuite) TestRestoreLocation() {
service := createTestService(t, suite.creds)
rootFolder := "General_" + common.FormatNow(common.SimpleTimeTesting)
folderID, err := createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder})
assert.NoError(t, err)
aw.NoErr(t, err)
t.Log("FolderID: " + folderID)
_, err = createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder, "Tsao"})
assert.NoError(t, err)
aw.NoErr(t, err)
// CleanUp
siteDrive, err := service.Client().SitesById(suite.siteID).Drive().Get(ctx, nil)
require.NoError(t, err)
aw.MustNoErr(t, err)
driveID := *siteDrive.GetId()
err = onedrive.DeleteItem(ctx, service, driveID, folderID)
assert.NoError(t, err)
aw.NoErr(t, err)
}

View File

@ -36,7 +36,7 @@ func DataCollections(
serv graph.Servicer,
su statusUpdater,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
b, err := selector.ToSharePointBackup()
if err != nil {
@ -44,12 +44,13 @@ func DataCollections(
}
var (
el = errs.Local()
site = b.DiscreteOwner
collections = []data.BackupCollection{}
)
for _, scope := range b.Scopes() {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -73,7 +74,7 @@ func DataCollections(
ctrlOpts,
errs)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
@ -86,9 +87,10 @@ func DataCollections(
site,
scope,
su,
ctrlOpts)
ctrlOpts,
errs)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
@ -102,7 +104,7 @@ func DataCollections(
ctrlOpts,
errs)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
}
@ -111,7 +113,7 @@ func DataCollections(
foldersComplete <- struct{}{}
}
return collections, nil, errs.Err()
return collections, nil, el.Failure()
}
func collectLists(
@ -120,11 +122,14 @@ func collectLists(
tenantID, siteID string,
updater statusUpdater,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, error) {
logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint List Collections")
spcs := make([]data.BackupCollection, 0)
var (
el = errs.Local()
spcs = make([]data.BackupCollection, 0)
)
lists, err := preFetchLists(ctx, serv, siteID)
if err != nil {
@ -132,7 +137,7 @@ func collectLists(
}
for _, tuple := range lists {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -143,7 +148,7 @@ func collectLists(
path.ListsCategory,
false)
if err != nil {
errs.Add(clues.Wrap(err, "creating list collection path").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "creating list collection path").WithClues(ctx))
}
collection := NewCollection(dir, serv, List, updater.UpdateStatus, ctrlOpts)
@ -152,7 +157,7 @@ func collectLists(
spcs = append(spcs, collection)
}
return spcs, errs.Err()
return spcs, el.Failure()
}
// collectLibraries constructs a onedrive Collections struct and Get()s
@ -165,6 +170,7 @@ func collectLibraries(
scope selectors.SharePointScope,
updater statusUpdater,
ctrlOpts control.Options,
errs *fault.Bus,
) ([]data.BackupCollection, map[string]struct{}, error) {
logger.Ctx(ctx).Debug("creating SharePoint Library collections")
@ -183,9 +189,9 @@ func collectLibraries(
// TODO(ashmrtn): Pass previous backup metadata when SharePoint supports delta
// token-based incrementals.
odcs, excludes, err := colls.Get(ctx, nil)
odcs, excludes, err := colls.Get(ctx, nil, errs)
if err != nil {
return nil, nil, clues.Wrap(err, "getting library").WithClues(ctx).With(graph.ErrData(err)...)
return nil, nil, graph.Wrap(ctx, err, "getting library")
}
return append(collections, odcs...), excludes, nil
@ -200,11 +206,14 @@ func collectPages(
siteID string,
updater statusUpdater,
ctrlOpts control.Options,
errs *fault.Errors,
errs *fault.Bus,
) ([]data.BackupCollection, error) {
logger.Ctx(ctx).Debug("creating SharePoint Pages collections")
spcs := make([]data.BackupCollection, 0)
var (
el = errs.Local()
spcs = make([]data.BackupCollection, 0)
)
// make the betaClient
// Need to receive From DataCollection Call
@ -221,7 +230,7 @@ func collectPages(
}
for _, tuple := range tuples {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -232,7 +241,7 @@ func collectPages(
path.PagesCategory,
false)
if err != nil {
errs.Add(clues.Wrap(err, "creating page collection path").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "creating page collection path").WithClues(ctx))
}
collection := NewCollection(dir, serv, Pages, updater.UpdateStatus, ctrlOpts)
@ -242,7 +251,7 @@ func collectPages(
spcs = append(spcs, collection)
}
return spcs, errs.Err()
return spcs, el.Failure()
}
type folderMatcher struct {

View File

@ -5,12 +5,12 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/fault"
"github.com/alcionai/corso/src/pkg/selectors"
@ -74,7 +74,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
driveItem("file", testBaseDrivePath, "root", true),
},
scope: anyFolder,
expect: assert.NoError,
expect: aw.NoErr,
expectedCollectionIDs: []string{"root"},
expectedCollectionPaths: expectedPathAsSlice(
suite.T(),
@ -105,7 +105,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
&MockGraphService{},
nil,
control.Options{})
err := c.UpdateCollections(ctx, "driveID1", "General", test.items, paths, newPaths, excluded, true)
err := c.UpdateCollections(ctx, "driveID1", "General", test.items, paths, newPaths, excluded, true, fault.New(true))
test.expect(t, err)
assert.Equal(t, len(test.expectedCollectionIDs), len(c.CollectionMap), "collection paths")
assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")
@ -169,7 +169,7 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
siteID := tester.M365SiteID(t)
a := tester.NewM365Account(t)
account, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
col, err := collectPages(
ctx,
@ -179,6 +179,6 @@ func (suite *SharePointPagesSuite) TestCollectPages() {
&MockGraphService{},
control.Defaults(),
fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
assert.NotEmpty(t, col)
}

View File

@ -4,11 +4,11 @@ import (
"testing"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/stretchr/testify/require"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/onedrive"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
)
@ -52,7 +52,7 @@ func createTestService(t *testing.T, credentials account.M365Config) *graph.Serv
credentials.AzureClientID,
credentials.AzureClientSecret,
)
require.NoError(t, err, "creating microsoft graph service for exchange")
aw.MustNoErr(t, err, "creating microsoft graph service for exchange")
return graph.NewService(adapter)
}
@ -62,7 +62,7 @@ func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []st
for _, r := range rest {
p, err := onedrive.GetCanonicalPath(r, tenant, user, onedrive.SharePointSource)
require.NoError(t, err)
aw.MustNoErr(t, err)
res = append(res, p.String())
}

View File

@ -7,11 +7,9 @@ import (
"github.com/alcionai/clues"
"github.com/microsoftgraph/msgraph-sdk-go/models"
mssite "github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -46,7 +44,7 @@ func preFetchLists(
for {
resp, err := builder.Get(ctx, options)
if err != nil {
return nil, clues.Wrap(err, "getting lists").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting lists")
}
for _, entry := range resp.GetValue() {
@ -92,11 +90,12 @@ func loadSiteLists(
gs graph.Servicer,
siteID string,
listIDs []string,
errs *fault.Errors,
errs *fault.Bus,
) ([]models.Listable, error) {
var (
results = make([]models.Listable, 0)
semaphoreCh = make(chan struct{}, fetchChannelSize)
el = errs.Local()
wg sync.WaitGroup
m sync.Mutex
)
@ -111,8 +110,8 @@ func loadSiteLists(
}
for _, listID := range listIDs {
if errs.Err() != nil {
return nil, errs.Err()
if el.Failure() != nil {
break
}
semaphoreCh <- struct{}{}
@ -130,13 +129,13 @@ func loadSiteLists(
entry, err = gs.Client().SitesById(siteID).ListsById(id).Get(ctx, nil)
if err != nil {
errs.Add(clues.Wrap(err, "getting site list").WithClues(ctx).With(graph.ErrData(err)...))
el.AddRecoverable(graph.Wrap(ctx, err, "getting site list"))
return
}
cols, cTypes, lItems, err := fetchListContents(ctx, gs, siteID, id, errs)
if err != nil {
errs.Add(clues.Wrap(err, "getting list contents"))
el.AddRecoverable(clues.Wrap(err, "getting list contents"))
return
}
@ -149,7 +148,7 @@ func loadSiteLists(
wg.Wait()
return results, errs.Err()
return results, el.Failure()
}
// fetchListContents utility function to retrieve associated M365 relationships
@ -159,7 +158,7 @@ func fetchListContents(
ctx context.Context,
service graph.Servicer,
siteID, listID string,
errs *fault.Errors,
errs *fault.Bus,
) (
[]models.ColumnDefinitionable,
[]models.ContentTypeable,
@ -192,26 +191,27 @@ func fetchListItems(
ctx context.Context,
gs graph.Servicer,
siteID, listID string,
errs *fault.Errors,
errs *fault.Bus,
) ([]models.ListItemable, error) {
var (
prefix = gs.Client().SitesById(siteID).ListsById(listID)
builder = prefix.Items()
itms = make([]models.ListItemable, 0)
el = errs.Local()
)
for {
if errs.Err() != nil {
if errs.Failure() != nil {
break
}
resp, err := builder.Get(ctx, nil)
if err != nil {
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return nil, err
}
for _, itm := range resp.GetValue() {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -219,7 +219,7 @@ func fetchListItems(
fields, err := newPrefix.Fields().Get(ctx, nil)
if err != nil {
errs.Add(clues.Wrap(err, "getting list fields").WithClues(ctx).With(graph.ErrData(err)...))
el.AddRecoverable(graph.Wrap(ctx, err, "getting list fields"))
continue
}
@ -235,7 +235,7 @@ func fetchListItems(
builder = mssite.NewItemListsItemItemsRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
}
return itms, errs.Err()
return itms, el.Failure()
}
// fetchColumns utility function to return columns from a site.
@ -255,7 +255,7 @@ func fetchColumns(
for {
resp, err := builder.Get(ctx, nil)
if err != nil {
return nil, clues.Wrap(err, "getting list columns").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting list columns")
}
cs = append(cs, resp.GetValue()...)
@ -272,7 +272,7 @@ func fetchColumns(
for {
resp, err := builder.Get(ctx, nil)
if err != nil {
return nil, clues.Wrap(err, "getting content columns").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting content columns")
}
cs = append(cs, resp.GetValue()...)
@ -298,25 +298,26 @@ func fetchContentTypes(
ctx context.Context,
gs graph.Servicer,
siteID, listID string,
errs *fault.Errors,
errs *fault.Bus,
) ([]models.ContentTypeable, error) {
var (
el = errs.Local()
cTypes = make([]models.ContentTypeable, 0)
builder = gs.Client().SitesById(siteID).ListsById(listID).ContentTypes()
)
for {
if errs.Err() != nil {
if errs.Failure() != nil {
break
}
resp, err := builder.Get(ctx, nil)
if err != nil {
return nil, errors.Wrap(err, support.ConnectorStackErrorTrace(err))
return nil, err
}
for _, cont := range resp.GetValue() {
if errs.Err() != nil {
if el.Failure() != nil {
break
}
@ -324,7 +325,7 @@ func fetchContentTypes(
links, err := fetchColumnLinks(ctx, gs, siteID, listID, id)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
@ -332,7 +333,7 @@ func fetchContentTypes(
cs, err := fetchColumns(ctx, gs, siteID, listID, id)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
@ -348,7 +349,7 @@ func fetchContentTypes(
builder = mssite.NewItemListsItemContentTypesRequestBuilder(*resp.GetOdataNextLink(), gs.Adapter())
}
return cTypes, errs.Err()
return cTypes, el.Failure()
}
func fetchColumnLinks(
@ -364,7 +365,7 @@ func fetchColumnLinks(
for {
resp, err := builder.Get(ctx, nil)
if err != nil {
return nil, clues.Wrap(err, "getting column links").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting column links")
}
links = append(links, resp.GetValue()...)
@ -391,7 +392,7 @@ func DeleteList(
) error {
err := gs.Client().SitesById(siteID).ListsById(listID).Delete(ctx, nil)
if err != nil {
return clues.Wrap(err, "deleting list").WithClues(ctx).With(graph.ErrData(err)...)
return graph.Wrap(ctx, err, "deleting list")
}
return nil

View File

@ -4,10 +4,10 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/internal/tester/aw"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/fault"
)
@ -21,7 +21,7 @@ func (suite *SharePointSuite) SetupSuite() {
t := suite.T()
a := tester.NewM365Account(t)
m365, err := a.M365Config()
require.NoError(t, err)
aw.MustNoErr(t, err)
suite.creds = m365
}
@ -52,11 +52,11 @@ func (suite *SharePointSuite) TestLoadList() {
t := suite.T()
service := createTestService(t, suite.creds)
tuples, err := preFetchLists(ctx, service, "root")
require.NoError(t, err)
aw.MustNoErr(t, err)
job := []string{tuples[0].id}
lists, err := loadSiteLists(ctx, service, "root", job, fault.New(true))
assert.NoError(t, err)
aw.NoErr(t, err)
assert.Greater(t, len(lists), 0)
t.Logf("Length: %d\n", len(lists))
}

View File

@ -6,7 +6,6 @@ import (
absser "github.com/microsoft/kiota-abstractions-go/serialization"
mssite "github.com/microsoftgraph/msgraph-sdk-go/sites"
"github.com/alcionai/clues"
"github.com/alcionai/corso/src/internal/connector/graph"
)
@ -22,7 +21,7 @@ func GetAllSitesForTenant(ctx context.Context, gs graph.Servicer) (absser.Parsab
sites, err := gs.Client().Sites().Get(ctx, options)
if err != nil {
return nil, clues.Wrap(err, "getting sites").WithClues(ctx).With(graph.ErrData(err)...)
return nil, graph.Wrap(ctx, err, "getting sites")
}
return sites, nil

View File

@ -46,7 +46,7 @@ func RestoreCollections(
dest control.RestoreDestination,
dcs []data.RestoreCollection,
deets *details.Builder,
errs *fault.Errors,
errs *fault.Bus,
) (*support.ConnectorOperationStatus, error) {
var (
err error
@ -56,7 +56,6 @@ func RestoreCollections(
// Iterate through the data collections and restore the contents of each
for _, dc := range dcs {
var (
canceled bool
category = dc.FullPath().Category()
metrics support.CollectionMetrics
ictx = clues.Add(ctx,
@ -67,7 +66,7 @@ func RestoreCollections(
switch dc.FullPath().Category() {
case path.LibrariesCategory:
metrics, _, _, canceled = onedrive.RestoreCollection(
metrics, _, _, err = onedrive.RestoreCollection(
ictx,
backupVersion,
service,
@ -76,9 +75,9 @@ func RestoreCollections(
onedrive.SharePointSource,
dest.ContainerName,
deets,
func(s string, err error) { errs.Add(err) },
map[string]string{},
false)
false,
errs)
case path.ListsCategory:
metrics, err = RestoreListCollection(
ictx,
@ -99,9 +98,9 @@ func RestoreCollections(
return nil, clues.Wrap(clues.New(category.String()), "category not supported")
}
restoreMetrics.Combine(metrics)
restoreMetrics = support.CombineMetrics(restoreMetrics, metrics)
if canceled || err != nil {
if err != nil {
break
}
}
@ -111,7 +110,6 @@ func RestoreCollections(
support.Restore,
len(dcs),
restoreMetrics,
err,
dest.ContainerName)
return status, err
@ -128,7 +126,7 @@ func createRestoreFolders(
// Get Main Drive for Site, Documents
mainDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil)
if err != nil {
return "", clues.Wrap(err, "getting site drive root").WithClues(ctx).With(graph.ErrData(err)...)
return "", graph.Wrap(ctx, err, "getting site drive root")
}
return onedrive.CreateRestoreFolders(ctx, service, *mainDrive.GetId(), restoreFolders)
@ -184,7 +182,7 @@ func restoreListItem(
// Restore to List base to M365 back store
restoredList, err := service.Client().SitesById(siteID).Lists().Post(ctx, newList, nil)
if err != nil {
return dii, clues.Wrap(err, "restoring list").WithClues(ctx).With(graph.ErrData(err)...)
return dii, graph.Wrap(ctx, err, "restoring list")
}
// Uploading of ListItems is conducted after the List is restored
@ -197,10 +195,8 @@ func restoreListItem(
Items().
Post(ctx, lItem, nil)
if err != nil {
return dii, clues.Wrap(err, "restoring list items").
With("restored_list_id", ptr.Val(restoredList.GetId())).
WithClues(ctx).
With(graph.ErrData(err)...)
return dii, graph.Wrap(ctx, err, "restoring list items").
With("restored_list_id", ptr.Val(restoredList.GetId()))
}
}
}
@ -216,7 +212,7 @@ func RestoreListCollection(
dc data.RestoreCollection,
restoreContainerName string,
deets *details.Builder,
errs *fault.Errors,
errs *fault.Bus,
) (support.CollectionMetrics, error) {
ctx, end := D.Span(ctx, "gc:sharepoint:restoreListCollection", D.Label("path", dc.FullPath()))
defer end()
@ -226,13 +222,14 @@ func RestoreListCollection(
directory = dc.FullPath()
siteID = directory.ResourceOwner()
items = dc.Items(ctx, errs)
el = errs.Local()
)
trace.Log(ctx, "gc:sharepoint:restoreListCollection", directory.String())
for {
if errs.Err() != nil {
return metrics, errs.Err()
if el.Failure() != nil {
break
}
select {
@ -252,15 +249,15 @@ func RestoreListCollection(
siteID,
restoreContainerName)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
metrics.Bytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx))
continue
}
@ -275,6 +272,8 @@ func RestoreListCollection(
metrics.Successes++
}
}
return metrics, el.Failure()
}
// RestorePageCollection handles restoration of an individual site page collection.
@ -287,7 +286,7 @@ func RestorePageCollection(
dc data.RestoreCollection,
restoreContainerName string,
deets *details.Builder,
errs *fault.Errors,
errs *fault.Bus,
) (support.CollectionMetrics, error) {
var (
metrics = support.CollectionMetrics{}
@ -305,14 +304,15 @@ func RestorePageCollection(
return metrics, clues.Wrap(err, "constructing graph client")
}
service := discover.NewBetaService(adpt)
// Restore items from collection
items := dc.Items(ctx, errs)
var (
el = errs.Local()
service = discover.NewBetaService(adpt)
items = dc.Items(ctx, errs)
)
for {
if errs.Err() != nil {
return metrics, errs.Err()
if el.Failure() != nil {
break
}
select {
@ -332,15 +332,15 @@ func RestorePageCollection(
siteID,
restoreContainerName)
if err != nil {
errs.Add(err)
el.AddRecoverable(err)
continue
}
metrics.TotalBytes += itemInfo.SharePoint.Size
metrics.Bytes += itemInfo.SharePoint.Size
itemPath, err := dc.FullPath().Append(itemData.UUID(), true)
if err != nil {
errs.Add(clues.Wrap(err, "appending item to full path").WithClues(ctx))
el.AddRecoverable(clues.Wrap(err, "appending item to full path").WithClues(ctx))
continue
}
@ -355,4 +355,6 @@ func RestorePageCollection(
metrics.Successes++
}
}
return metrics, el.Failure()
}

View File

@ -1,130 +0,0 @@
package support
import (
"fmt"
"strconv"
"strings"
multierror "github.com/hashicorp/go-multierror"
msgraph_errors "github.com/microsoftgraph/msgraph-sdk-go/models/odataerrors"
"github.com/pkg/errors"
)
// WrapErrorAndAppend helper function used to attach identifying information to an error
// and return it as a mulitierror
func WrapAndAppend(identifier string, e, previous error) error {
return multierror.Append(previous, errors.Wrap(e, identifier))
}
// WrapErrorAndAppendf format version of WrapErrorAndAppend
func WrapAndAppendf(identifier interface{}, e, previous error) error {
return multierror.Append(previous, errors.Wrapf(e, "%v", identifier))
}
// GetErrors Helper method to return the integer amount of errors in multi error
func GetNumberOfErrors(err error) int {
if err == nil {
return 0
}
result, _, wasFound := strings.Cut(err.Error(), " ")
if wasFound {
aNum, err := strconv.Atoi(result)
if err == nil {
return aNum
}
}
return 1
}
// ListErrors is a helper method used to return the string of errors when
// the multiError library is used.
// depends on ConnectorStackErrorTrace
func ListErrors(multi multierror.Error) string {
aString := ""
for idx, err := range multi.Errors {
detail := ConnectorStackErrorTrace(err)
if detail == "" {
detail = fmt.Sprintf("%v", err)
}
aString = aString + fmt.Sprintf("\n\tErr: %d %v", idx+1, detail)
}
return aString
}
// concatenateStringFromPointers is a helper function that adds
// strings to the originalMessage iff the pointer is not nil
func concatenateStringFromPointers(orig string, pointers []*string) string {
for _, pointer := range pointers {
if pointer != nil {
orig = strings.Join([]string{orig, *pointer}, " ")
}
}
return orig
}
// ConnectorStackErrorTraceWrap is a helper function that wraps the
// stack trace for oDataErrors (if the error has one) onto the prefix.
// If no stack trace is found, wraps the error with only the prefix.
func ConnectorStackErrorTraceWrap(e error, prefix string) error {
cset := ConnectorStackErrorTrace(e)
if len(cset) > 0 {
return errors.Wrap(e, prefix+": "+cset)
}
return errors.Wrap(e, prefix)
}
// ConnectorStackErrorTrace is a helper function that extracts
// the stack trace for oDataErrors, if the error has one.
func ConnectorStackErrorTrace(e error) string {
eMessage := ""
if oDataError, ok := e.(msgraph_errors.ODataErrorable); ok {
// Get MainError
mainErr := oDataError.GetError()
// message *string
// target *string
// code *string
// details ErrorDetailsable
// Ignoring Additional Detail
code := mainErr.GetCode()
subject := mainErr.GetMessage()
target := mainErr.GetTarget()
details := mainErr.GetDetails()
inners := mainErr.GetInnererror()
eMessage = concatenateStringFromPointers(eMessage,
[]*string{code, subject, target})
// Get Error Details
// code, message, target
if details != nil {
eMessage = eMessage + "\nDetails Section:"
for idx, detail := range details {
dMessage := fmt.Sprintf("Detail %d:", idx)
c := detail.GetCode()
m := detail.GetMessage()
t := detail.GetTarget()
dMessage = concatenateStringFromPointers(dMessage,
[]*string{c, m, t})
eMessage = eMessage + dMessage
}
}
if inners != nil {
eMessage = eMessage + "\nConnector Section:"
client := inners.GetClientRequestId()
rID := inners.GetRequestId()
eMessage = concatenateStringFromPointers(eMessage,
[]*string{client, rID})
}
}
return eMessage
}

View File

@ -1,103 +0,0 @@
package support
import (
"errors"
"fmt"
"strings"
"testing"
multierror "github.com/hashicorp/go-multierror"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type GraphConnectorErrorSuite struct {
suite.Suite
}
func TestGraphConnectorErrorSuite(t *testing.T) {
suite.Run(t, new(GraphConnectorErrorSuite))
}
func (suite *GraphConnectorErrorSuite) TestWrapAndAppend() {
err1 := fmt.Errorf("New Error")
err2 := errors.New("I have two")
returnErr := WrapAndAppend("arc376", err2, err1)
suite.True(strings.Contains(returnErr.Error(), "arc376"))
suite.Error(returnErr)
multi := &multierror.Error{Errors: []error{err1, err2}}
suite.True(strings.Contains(ListErrors(*multi), "two")) // Does not contain the wrapped information
suite.T().Log(ListErrors(*multi))
}
func (suite *GraphConnectorErrorSuite) TestWrapAndAppend_OnVar() {
var (
err1 error
id = "xi2058"
)
received := WrapAndAppend(id, errors.New("network error"), err1)
suite.True(strings.Contains(received.Error(), id))
}
func (suite *GraphConnectorErrorSuite) TestWrapAndAppend_Add3() {
errOneTwo := WrapAndAppend("user1", assert.AnError, assert.AnError)
combined := WrapAndAppend("unix36", assert.AnError, errOneTwo)
allErrors := WrapAndAppend("fxi92874", assert.AnError, combined)
suite.True(strings.Contains(combined.Error(), "unix36"))
suite.True(strings.Contains(combined.Error(), "user1"))
suite.True(strings.Contains(allErrors.Error(), "fxi92874"))
}
func (suite *GraphConnectorErrorSuite) TestWrapAndAppendf() {
err1 := assert.AnError
err2 := assert.AnError
combined := WrapAndAppendf(134323, err2, err1)
suite.True(strings.Contains(combined.Error(), "134323"))
}
func (suite *GraphConnectorErrorSuite) TestConcatenateStringFromPointers() {
var (
outString string
v1 = "Corso"
v3 = "remains"
s1 = &v1
s2 *string
s3 = &v3
)
outString = concatenateStringFromPointers(outString, []*string{s1, s2, s3})
suite.True(strings.Contains(outString, v1))
suite.True(strings.Contains(outString, v3))
}
func (suite *GraphConnectorErrorSuite) TestGetNumberOfErrors() {
table := []struct {
name string
errs error
expected int
}{
{
name: "No error",
errs: nil,
expected: 0,
},
{
name: "Not an ErrorList",
errs: errors.New("network error"),
expected: 1,
},
{
name: "Three Errors",
errs: WrapAndAppend("tres", errors.New("three"), WrapAndAppend("arc376", errors.New("one"), errors.New("two"))),
expected: 3,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
result := GetNumberOfErrors(test.errs)
suite.Equal(result, test.expected)
})
}
}

View File

@ -6,11 +6,11 @@ import (
kioser "github.com/microsoft/kiota-serialization-json-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
bmodels "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type DataSupportSuite struct {
@ -38,13 +38,13 @@ func (suite *DataSupportSuite) TestCreateMessageFromBytes() {
{
name: "Empty Bytes",
byteArray: make([]byte, 0),
checkError: assert.Error,
checkError: aw.Err,
checkObject: assert.Nil,
},
{
name: "aMessage bytes",
byteArray: mockconnector.GetMockMessageBytes("m365 mail support test"),
checkError: assert.NoError,
checkError: aw.NoErr,
checkObject: assert.NotNil,
},
}
@ -69,19 +69,19 @@ func (suite *DataSupportSuite) TestCreateContactFromBytes() {
{
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: invalid,
byteArray: []byte("A random sentence doesn't make an object"),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: "Valid Contact",
byteArray: mockconnector.GetMockContactBytes("Support Test"),
checkError: assert.NoError,
checkError: aw.NoErr,
isNil: assert.NotNil,
},
}
@ -104,19 +104,19 @@ func (suite *DataSupportSuite) TestCreateEventFromBytes() {
{
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: invalid,
byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: "Valid Event",
byteArray: mockconnector.GetDefaultMockEventBytes("Event Test"),
checkError: assert.NoError,
checkError: aw.NoErr,
isNil: assert.NotNil,
},
}
@ -131,7 +131,7 @@ func (suite *DataSupportSuite) TestCreateEventFromBytes() {
func (suite *DataSupportSuite) TestCreateListFromBytes() {
listBytes, err := mockconnector.GetMockListBytes("DataSupportSuite")
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
tests := []struct {
name string
@ -142,19 +142,19 @@ func (suite *DataSupportSuite) TestCreateListFromBytes() {
{
name: empty,
byteArray: make([]byte, 0),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: invalid,
byteArray: []byte("Invalid byte stream \"subject:\" Not going to work"),
checkError: assert.Error,
checkError: aw.Err,
isNil: assert.Nil,
},
{
name: "Valid List",
byteArray: listBytes,
checkError: assert.NoError,
checkError: aw.NoErr,
isNil: assert.NotNil,
},
}
@ -177,7 +177,7 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() {
}{
{
empty,
assert.Error,
aw.Err,
assert.Nil,
func(t *testing.T) []byte {
return make([]byte, 0)
@ -185,7 +185,7 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() {
},
{
invalid,
assert.Error,
aw.Err,
assert.Nil,
func(t *testing.T) []byte {
return []byte("snarf")
@ -193,7 +193,7 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() {
},
{
"Valid Page",
assert.NoError,
aw.NoErr,
assert.NotNil,
func(t *testing.T) []byte {
pg := bmodels.NewSitePage()
@ -204,10 +204,10 @@ func (suite *DataSupportSuite) TestCreatePageFromBytes() {
writer := kioser.NewJsonSerializationWriter()
err := pg.Serialize(writer)
require.NoError(t, err)
aw.MustNoErr(t, err)
byteArray, err := writer.GetSerializedContent()
require.NoError(t, err)
aw.MustNoErr(t, err)
return byteArray
},
@ -239,7 +239,7 @@ func (suite *DataSupportSuite) TestHasAttachments() {
"This is testing",
)
message, err := CreateMessageFromBytes(byteArray)
require.NoError(t, err)
aw.MustNoErr(t, err)
return message.GetBody()
},
},
@ -249,7 +249,7 @@ func (suite *DataSupportSuite) TestHasAttachments() {
getBodyable: func(t *testing.T) models.ItemBodyable {
byteArray := mockconnector.GetMessageWithOneDriveAttachment("Test legacy")
message, err := CreateMessageFromBytes(byteArray)
require.NoError(t, err)
aw.MustNoErr(t, err)
return message.GetBody()
},
},

View File

@ -5,10 +5,10 @@ import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/mockconnector"
"github.com/alcionai/corso/src/internal/tester/aw"
)
type SupportTestSuite struct {
@ -22,7 +22,7 @@ func TestSupportTestSuite(t *testing.T) {
func (suite *SupportTestSuite) TestToMessage() {
bytes := mockconnector.GetMockMessageBytes("m365 mail support test")
message, err := CreateMessageFromBytes(bytes)
require.NoError(suite.T(), err)
aw.MustNoErr(suite.T(), err)
clone := ToMessage(message)
suite.Equal(message.GetBccRecipients(), clone.GetBccRecipients())
@ -36,7 +36,7 @@ func (suite *SupportTestSuite) TestToEventSimplified() {
t := suite.T()
bytes := mockconnector.GetMockEventWithAttendeesBytes("M365 Event Support Test")
event, err := CreateEventFromBytes(bytes)
require.NoError(t, err)
aw.MustNoErr(t, err)
attendees := event.GetAttendees()
newEvent := ToEventSimplified(event)

View File

@ -5,7 +5,6 @@ import (
"fmt"
"github.com/dustin/go-humanize"
multierror "github.com/hashicorp/go-multierror"
)
// ConnectorOperationStatus is a data type used to describe the state of
@ -15,27 +14,23 @@ import (
// @param incomplete: Bool representation of whether all intended items were download or uploaded.
// @param bytes: represents the total number of bytes that have been downloaded or uploaded.
type ConnectorOperationStatus struct {
lastOperation Operation
ObjectCount int
FolderCount int
Successful int
ErrorCount int
Err error
incomplete bool
incompleteReason string
additionalDetails string
bytes int64
Folders int
Metrics CollectionMetrics
details string
op Operation
}
type CollectionMetrics struct {
Objects, Successes int
TotalBytes int64
Bytes int64
}
func (cm *CollectionMetrics) Combine(additional CollectionMetrics) {
cm.Objects += additional.Objects
cm.Successes += additional.Successes
cm.TotalBytes += additional.TotalBytes
func CombineMetrics(a, b CollectionMetrics) CollectionMetrics {
return CollectionMetrics{
Objects: a.Objects + b.Objects,
Successes: a.Successes + b.Successes,
Bytes: a.Bytes + b.Bytes,
}
}
type Operation int
@ -53,30 +48,13 @@ func CreateStatus(
op Operation,
folders int,
cm CollectionMetrics,
err error,
details string,
) *ConnectorOperationStatus {
var reason string
if err != nil {
reason = err.Error()
}
hasErrors := err != nil
// TODO(keeprs): remove
numErr := GetNumberOfErrors(err)
status := ConnectorOperationStatus{
lastOperation: op,
ObjectCount: cm.Objects,
FolderCount: folders,
Successful: cm.Successes,
ErrorCount: numErr,
Err: err,
incomplete: hasErrors,
incompleteReason: reason,
bytes: cm.TotalBytes,
additionalDetails: details,
Folders: folders,
Metrics: cm,
details: details,
op: op,
}
return &status
@ -89,32 +67,19 @@ type StatusUpdater func(*ConnectorOperationStatus)
// MergeStatus combines ConnectorOperationsStatus value into a single status
func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
var hasErrors bool
if one.lastOperation == OpUnknown {
if one.op == OpUnknown {
return two
}
if two.lastOperation == OpUnknown {
if two.op == OpUnknown {
return one
}
if one.incomplete || two.incomplete {
hasErrors = true
}
status := ConnectorOperationStatus{
lastOperation: one.lastOperation,
ObjectCount: one.ObjectCount + two.ObjectCount,
FolderCount: one.FolderCount + two.FolderCount,
Successful: one.Successful + two.Successful,
// TODO: remove in favor of fault.Errors
ErrorCount: one.ErrorCount + two.ErrorCount,
Err: multierror.Append(one.Err, two.Err).ErrorOrNil(),
bytes: one.bytes + two.bytes,
incomplete: hasErrors,
incompleteReason: one.incompleteReason + ", " + two.incompleteReason,
additionalDetails: one.additionalDetails + ", " + two.additionalDetails,
Folders: one.Folders + two.Folders,
Metrics: CombineMetrics(one.Metrics, two.Metrics),
details: one.details + ", " + two.details,
op: one.op,
}
return status
@ -123,23 +88,19 @@ func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
func (cos *ConnectorOperationStatus) String() string {
var operationStatement string
switch cos.lastOperation {
switch cos.op {
case Backup:
operationStatement = "Downloaded from "
case Restore:
operationStatement = "Restored content to "
}
message := fmt.Sprintf("Action: %s performed on %d of %d objects (%s) within %d directories.",
cos.lastOperation.String(),
cos.Successful,
cos.ObjectCount,
humanize.Bytes(uint64(cos.bytes)),
cos.FolderCount)
if cos.incomplete {
message += " " + cos.incompleteReason
}
return message + " " + operationStatement + cos.additionalDetails
return fmt.Sprintf("Action: %s performed on %d of %d objects (%s) within %d directories. %s %s",
cos.op.String(),
cos.Metrics.Successes,
cos.Metrics.Objects,
humanize.Bytes(uint64(cos.Metrics.Bytes)),
cos.Folders,
operationStatement,
cos.details)
}

Some files were not shown because too many files have changed in this diff Show More