larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test service_iterators with mocks (#2020)

Browse Source 
## Description

Now that we have a mocked api layer in place, we can test filterContainersAndFillCollections without
integration dependencies.

## Does this PR need a docs update or release note?

- [x] ⛔ No 

## Type of change

- [x] 🤖 Test

## Issue(s)

* #1967

## Test Plan

- [x] ⚡ Unit test
This commit is contained in:
Keepers 2023-01-06 15:19:45 -07:00 committed by GitHub
parent 50209cd879
commit d8763298ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 750 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
src/internal/connector/exchange/container_resolver_test.go
View File

@ -19,24 +19,26 @@ import (
// mocks and helpers
// ---------------------------------------------------------------------------
var _ graph.CachedContainer = &mockContainer{}
type mockContainer struct {
id *string
name *string
parentID *string
id *string
displayName *string
parentID *string
p *path.Builder
}
//nolint:revive
func (m mockContainer) GetId() *string {
return m.id
}
func (m mockContainer) GetDisplayName() *string {
return m.name
}
func (m mockContainer) GetId() *string { return m.id }
//nolint:revive
func (m mockContainer) GetParentFolderId() *string {
return m.parentID
func (m mockContainer) GetParentFolderId() *string { return m.parentID }
func (m mockContainer) GetDisplayName() *string { return m.displayName }
func (m mockContainer) Path() *path.Builder { return m.p }
func (m mockContainer) SetPath(p *path.Builder) {}
func strPtr(s string) *string {
return &s
}
// ---------------------------------------------------------------------------
@ -67,45 +69,45 @@ var (
{
name: "NilID",
c: mockContainer{
id: nil,
name: &testName,
parentID: &testParentID,
id: nil,
displayName: &testName,
parentID: &testParentID,
},
check: assert.Error,
},
{
name: "NilDisplayName",
c: mockContainer{
id: &testID,
name: nil,
parentID: &testParentID,
id: &testID,
displayName: nil,
parentID: &testParentID,
},
check: assert.Error,
},
{
name: "EmptyID",
c: mockContainer{
id: &emptyString,
name: &testName,
parentID: &testParentID,
id: &emptyString,
displayName: &testName,
parentID: &testParentID,
},
check: assert.Error,
},
{
name: "EmptyDisplayName",
c: mockContainer{
id: &testID,
name: &emptyString,
parentID: &testParentID,
id: &testID,
displayName: &emptyString,
parentID: &testParentID,
},
check: assert.Error,
},
{
name: "AllValues",
c: mockContainer{
id: &testID,
name: &testName,
parentID: &testParentID,
id: &testID,
displayName: &testName,
parentID: &testParentID,
},
check: assert.NoError,
},
@ -125,18 +127,18 @@ func (suite *FolderCacheUnitSuite) TestCheckRequiredValues() {
{
name: "NilParentFolderID",
c: mockContainer{
id: &testID,
name: &testName,
parentID: nil,
id: &testID,
displayName: &testName,
parentID: nil,
},
check: assert.Error,
},
{
name: "EmptyParentFolderID",
c: mockContainer{
id: &testID,
name: &testName,
parentID: &emptyString,
id: &testID,
displayName: &testName,
parentID: &emptyString,
},
check: assert.Error,
},
@ -161,9 +163,9 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
name: "NoParentNoPath",
cf: graph.NewCacheFolder(
&mockContainer{
id: &testID,
name: &testName,
parentID: nil,
id: &testID,
displayName: &testName,
parentID: nil,
},
nil,
),
@ -173,9 +175,9 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
name: "NoParentPath",
cf: graph.NewCacheFolder(
&mockContainer{
id: &testID,
name: &testName,
parentID: nil,
id: &testID,
displayName: &testName,
parentID: nil,
},
path.Builder{}.Append("foo"),
),
@ -185,9 +187,9 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
name: "NoName",
cf: graph.NewCacheFolder(
&mockContainer{
id: &testID,
name: nil,
parentID: &testParentID,
id: &testID,
displayName: nil,
parentID: &testParentID,
},
path.Builder{}.Append("foo"),
),
@ -197,9 +199,9 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
name: "NoID",
cf: graph.NewCacheFolder(
&mockContainer{
id: nil,
name: &testName,
parentID: &testParentID,
id: nil,
displayName: &testName,
parentID: &testParentID,
},
path.Builder{}.Append("foo"),
),
@ -209,9 +211,9 @@ func (suite *FolderCacheUnitSuite) TestAddFolder() {
name: "NoPath",
cf: graph.NewCacheFolder(
&mockContainer{
id: &testID,
name: &testName,
parentID: &testParentID,
id: &testID,
displayName: &testName,
parentID: &testParentID,
},
nil,
),

16
src/internal/connector/exchange/service_iterators.go
View File

@ -95,18 +95,18 @@ func filterContainersAndFillCollections(
added, removed, newDelta, err := getter.GetAddedAndRemovedItemIDs(ctx, qp.ResourceOwner, cID, prevDelta)
if err != nil {
// note == nil check; only catches non-inFlight error cases.
if graph.IsErrDeletedInFlight(err) == nil {
errs = support.WrapAndAppend(qp.ResourceOwner, err, errs)
} else {
// race conditions happen, containers might get deleted while
// this process is in flight. If that happens, force the collection
// to reset which will prevent any old items from being retained in
// storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned.
newDelta = api.DeltaUpdate{Reset: true}
continue
}
continue
// race conditions happen, containers might get deleted while
// this process is in flight. If that happens, force the collection
// to reset. This prevents any old items from being retained in
// storage. If the container (or its children) are sill missing
// on the next backup, they'll get tombstoned.
newDelta = api.DeltaUpdate{Reset: true}
}
if len(newDelta.URL) > 0 {

681
src/internal/connector/exchange/service_iterators_test.go Normal file
View File

@ -0,0 +1,681 @@
package exchange
import (
"context"
"testing"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/exchange/api"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/tester"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/selectors"
)
// ---------------------------------------------------------------------------
// mocks
// ---------------------------------------------------------------------------
var _ addedAndRemovedItemIDsGetter = &mockGetter{}
type (
mockGetter map[string]mockGetterResults
mockGetterResults struct {
added []string
removed []string
newDelta api.DeltaUpdate
err error
}
)
func (mg mockGetter) GetAddedAndRemovedItemIDs(
ctx context.Context,
userID, cID, prevDelta string,
) (
[]string,
[]string,
api.DeltaUpdate,
error,
) {
results, ok := mg[cID]
if !ok {
return nil, nil, api.DeltaUpdate{}, errors.New("mock not found for " + cID)
}
return results.added, results.removed, results.newDelta, results.err
}
var _ graph.ContainerResolver = &mockResolver{}
type (
mockResolver struct {
items []graph.CachedContainer
}
)
func newMockResolver(items ...mockContainer) mockResolver {
is := make([]graph.CachedContainer, 0, len(items))
for _, i := range items {
is = append(is, i)
}
return mockResolver{items: is}
}
func (m mockResolver) Items() []graph.CachedContainer {
return m.items
}
func (m mockResolver) AddToCache(context.Context, graph.Container) error { return nil }
func (m mockResolver) IDToPath(context.Context, string) (*path.Builder, error) { return nil, nil }
func (m mockResolver) PathInCache(string) (string, bool) { return "", false }
func (m mockResolver) Populate(context.Context, string, ...string) error { return nil }
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
type ServiceIteratorsSuite struct {
suite.Suite
creds account.M365Config
}
func TestServiceIteratorsSuite(t *testing.T) {
suite.Run(t, new(ServiceIteratorsSuite))
}
func (suite *ServiceIteratorsSuite) SetupSuite() {
a := tester.NewM365Account(suite.T())
m365, err := a.M365Config()
require.NoError(suite.T(), err)
suite.creds = m365
}
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections() {
var (
userID = "user_id"
qp = graph.QueryParams{
Category: path.EmailCategory, // doesn't matter which one we use.
ResourceOwner: userID,
Credentials: suite.creds,
}
statusUpdater = func(*support.ConnectorOperationStatus) {}
allScope = selectors.NewExchangeBackup(nil).MailFolders(selectors.Any())[0]
dps = DeltaPaths{} // incrementals are tested separately
commonResult = mockGetterResults{
added: []string{"a1", "a2", "a3"},
removed: []string{"r1", "r2", "r3"},
newDelta: api.DeltaUpdate{URL: "delta_url"},
}
errorResult = mockGetterResults{
added: []string{"a1", "a2", "a3"},
removed: []string{"r1", "r2", "r3"},
newDelta: api.DeltaUpdate{URL: "delta_url"},
err: assert.AnError,
}
deletedInFlightResult = mockGetterResults{
added: []string{"a1", "a2", "a3"},
removed: []string{"r1", "r2", "r3"},
newDelta: api.DeltaUpdate{URL: "delta_url"},
err: graph.ErrDeletedInFlight{Err: *common.EncapsulateError(assert.AnError)},
}
container1 = mockContainer{
id: strPtr("1"),
displayName: strPtr("display_name_1"),
p: path.Builder{}.Append("display_name_1"),
}
container2 = mockContainer{
id: strPtr("2"),
displayName: strPtr("display_name_2"),
p: path.Builder{}.Append("display_name_2"),
}
)
table := []struct {
name string
getter mockGetter
resolver graph.ContainerResolver
scope selectors.ExchangeScope
failFast bool
expectErr assert.ErrorAssertionFunc
expectNewColls int
expectMetadataColls int
expectDoNotMergeColls int
}{
{
name: "happy path, one container",
getter: map[string]mockGetterResults{
"1": commonResult,
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.NoError,
expectNewColls: 1,
expectMetadataColls: 1,
},
{
name: "happy path, many containers",
getter: map[string]mockGetterResults{
"1": commonResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectNewColls: 2,
expectMetadataColls: 1,
},
{
name: "happy path, many containers, same display name",
getter: map[string]mockGetterResults{
"1": commonResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectNewColls: 2,
expectMetadataColls: 1,
},
{
name: "no containers pass scope",
getter: map[string]mockGetterResults{
"1": commonResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: selectors.NewExchangeBackup(nil).MailFolders(selectors.None())[0],
expectErr: assert.NoError,
expectNewColls: 0,
expectMetadataColls: 1,
},
{
name: "err: deleted in flight",
getter: map[string]mockGetterResults{
"1": deletedInFlightResult,
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.NoError,
expectNewColls: 1,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
},
{
name: "err: other error",
getter: map[string]mockGetterResults{
"1": errorResult,
},
resolver: newMockResolver(container1),
scope: allScope,
expectErr: assert.Error,
expectNewColls: 0,
expectMetadataColls: 1,
},
{
name: "half collections error: deleted in flight",
getter: map[string]mockGetterResults{
"1": deletedInFlightResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.NoError,
expectNewColls: 2,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
},
{
name: "half collections error: other error",
getter: map[string]mockGetterResults{
"1": errorResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
expectErr: assert.Error,
expectNewColls: 1,
expectMetadataColls: 1,
},
{
name: "half collections error: deleted in flight, fail fast",
getter: map[string]mockGetterResults{
"1": deletedInFlightResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
failFast: true,
expectErr: assert.NoError,
expectNewColls: 2,
expectMetadataColls: 1,
expectDoNotMergeColls: 1,
},
{
name: "half collections error: other error, fail fast",
getter: map[string]mockGetterResults{
"1": errorResult,
"2": commonResult,
},
resolver: newMockResolver(container1, container2),
scope: allScope,
failFast: true,
expectErr: assert.Error,
expectNewColls: 0,
expectMetadataColls: 0,
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
collections := map[string]data.Collection{}
err := filterContainersAndFillCollections(
ctx,
qp,
test.getter,
collections,
statusUpdater,
test.resolver,
test.scope,
dps,
control.Options{FailFast: test.failFast},
)
test.expectErr(t, err)
// collection assertions
deleteds, news, metadatas, doNotMerges := 0, 0, 0, 0
for _, c := range collections {
if c.FullPath().Service() == path.ExchangeMetadataService {
metadatas++
continue
}
if c.State() == data.DeletedState {
deleteds++
}
if c.State() == data.NewState {
news++
}
if c.DoNotMergeItems() {
doNotMerges++
}
}
assert.Zero(t, deleteds, "deleted collections")
assert.Equal(t, test.expectNewColls, news, "new collections")
assert.Equal(t, test.expectMetadataColls, metadatas, "metadata collections")
assert.Equal(t, test.expectDoNotMergeColls, doNotMerges, "doNotMerge collections")
// items in collections assertions
for k, expect := range test.getter {
coll := collections[k]
if coll == nil {
continue
}
exColl, ok := coll.(*Collection)
require.True(t, ok, "collection is an *exchange.Collection")
assert.ElementsMatch(t, expect.added, exColl.added, "added items")
assert.ElementsMatch(t, expect.removed, exColl.removed, "removed items")
}
})
}
}
func (suite *ServiceIteratorsSuite) TestFilterContainersAndFillCollections_incrementals() {
var (
userID = "user_id"
tenantID = tester.M365TenantID(suite.T())
cat = path.EmailCategory // doesn't matter which one we use,
qp = graph.QueryParams{
Category: cat,
ResourceOwner: userID,
Credentials: suite.creds,
}
statusUpdater = func(*support.ConnectorOperationStatus) {}
allScope = selectors.NewExchangeBackup(nil).MailFolders(selectors.Any())[0]
commonResults = mockGetterResults{
added: []string{"added"},
newDelta: api.DeltaUpdate{URL: "new_delta_url"},
}
expiredResults = mockGetterResults{
added: []string{"added"},
newDelta: api.DeltaUpdate{
URL: "new_delta_url",
Reset: true,
},
}
)
prevPath := func(t *testing.T, at ...string) path.Path {
p, err := path.Builder{}.
Append(at...).
ToDataLayerExchangePathForCategory(tenantID, userID, cat, false)
require.NoError(t, err)
return p
}
type endState struct {
state data.CollectionState
doNotMerge bool
}
table := []struct {
name string
getter mockGetter
resolver graph.ContainerResolver
dps DeltaPaths
expect map[string]endState
}{
{
name: "new container",
getter: map[string]mockGetterResults{
"1": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("1"),
displayName: strPtr("new"),
p: path.Builder{}.Append("1", "new"),
}),
dps: DeltaPaths{},
expect: map[string]endState{
"1": {data.NewState, false},
},
},
{
name: "not moved container",
getter: map[string]mockGetterResults{
"1": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("1"),
displayName: strPtr("not_moved"),
p: path.Builder{}.Append("1", "not_moved"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "not_moved").String(),
},
},
expect: map[string]endState{
"1": {data.NotMovedState, false},
},
},
{
name: "moved container",
getter: map[string]mockGetterResults{
"1": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("1"),
displayName: strPtr("moved"),
p: path.Builder{}.Append("1", "moved"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "prev").String(),
},
},
expect: map[string]endState{
"1": {data.MovedState, false},
},
},
{
name: "deleted container",
getter: map[string]mockGetterResults{},
resolver: newMockResolver(),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "deleted").String(),
},
},
expect: map[string]endState{
"1": {data.DeletedState, false},
},
},
{
name: "one deleted, one new",
getter: map[string]mockGetterResults{
"2": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("2"),
displayName: strPtr("new"),
p: path.Builder{}.Append("2", "new"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "deleted").String(),
},
},
expect: map[string]endState{
"1": {data.DeletedState, false},
"2": {data.NewState, false},
},
},
{
name: "one deleted, one new, same path",
getter: map[string]mockGetterResults{
"2": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("2"),
displayName: strPtr("same"),
p: path.Builder{}.Append("2", "same"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "same").String(),
},
},
expect: map[string]endState{
"1": {data.DeletedState, false},
"2": {data.NewState, false},
},
},
{
name: "one moved, one new, same path",
getter: map[string]mockGetterResults{
"1": commonResults,
"2": commonResults,
},
resolver: newMockResolver(
mockContainer{
id: strPtr("1"),
displayName: strPtr("moved"),
p: path.Builder{}.Append("1", "moved"),
},
mockContainer{
id: strPtr("2"),
displayName: strPtr("prev"),
p: path.Builder{}.Append("2", "prev"),
},
),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "prev").String(),
},
},
expect: map[string]endState{
"1": {data.MovedState, false},
"2": {data.NewState, false},
},
},
{
name: "bad previous path strings",
getter: map[string]mockGetterResults{
"1": commonResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("1"),
displayName: strPtr("not_moved"),
p: path.Builder{}.Append("1", "not_moved"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: "1/fnords/mc/smarfs",
},
"2": DeltaPath{
delta: "old_delta_url",
path: "2/fnords/mc/smarfs",
},
},
expect: map[string]endState{
"1": {data.NewState, false},
},
},
{
name: "delta expiration",
getter: map[string]mockGetterResults{
"1": expiredResults,
},
resolver: newMockResolver(mockContainer{
id: strPtr("1"),
displayName: strPtr("same"),
p: path.Builder{}.Append("1", "same"),
}),
dps: DeltaPaths{
"1": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "1", "same").String(),
},
},
expect: map[string]endState{
"1": {data.NotMovedState, true},
},
},
{
name: "a little bit of everything",
getter: map[string]mockGetterResults{
"1": commonResults, // new
"2": commonResults, // notMoved
"3": commonResults, // moved
"4": expiredResults, // moved
// "5" gets deleted
},
resolver: newMockResolver(
mockContainer{
id: strPtr("1"),
displayName: strPtr("new"),
p: path.Builder{}.Append("1", "new"),
},
mockContainer{
id: strPtr("2"),
displayName: strPtr("not_moved"),
p: path.Builder{}.Append("2", "not_moved"),
},
mockContainer{
id: strPtr("3"),
displayName: strPtr("moved"),
p: path.Builder{}.Append("3", "moved"),
},
mockContainer{
id: strPtr("4"),
displayName: strPtr("moved"),
p: path.Builder{}.Append("4", "moved"),
},
),
dps: DeltaPaths{
"2": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "2", "not_moved").String(),
},
"3": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "3", "prev").String(),
},
"4": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "4", "prev").String(),
},
"5": DeltaPath{
delta: "old_delta_url",
path: prevPath(suite.T(), "5", "deleted").String(),
},
},
expect: map[string]endState{
"1": {data.NewState, false},
"2": {data.NotMovedState, false},
"3": {data.MovedState, false},
"4": {data.MovedState, true},
"5": {data.DeletedState, false},
},
},
}
for _, test := range table {
suite.T().Run(test.name, func(t *testing.T) {
ctx, flush := tester.NewContext()
defer flush()
collections := map[string]data.Collection{}
err := filterContainersAndFillCollections(
ctx,
qp,
test.getter,
collections,
statusUpdater,
test.resolver,
allScope,
test.dps,
control.Options{
EnabledFeatures: control.FeatureFlags{
ExchangeIncrementals: true,
},
},
)
assert.NoError(t, err)
metadatas := 0
for _, c := range collections {
p := c.FullPath()
if p == nil {
p = c.PreviousPath()
}
require.NotNil(t, p)
if p.Service() == path.ExchangeMetadataService {
metadatas++
continue
}
p0 := p.Folders()[0]
expect, ok := test.expect[p0]
assert.True(t, ok, "collection is expected in result")
assert.Equalf(t, expect.state, c.State(), "collection %s state", p0)
assert.Equalf(t, expect.doNotMerge, c.DoNotMergeItems(), "collection %s DoNotMergeItems", p0)
}
assert.Equal(t, 1, metadatas, "metadata collections")
})
}
}

11
src/internal/tester/resource_owners.go
View File

@ -8,6 +8,17 @@ import (
"github.com/stretchr/testify/require"
)
// M365TenantID returns a tenantID string representing the azureTenantID described
// by either the env var AZURE_TENANT_ID, the corso_test.toml config
// file or the default value (in that order of priority). The default is a
// last-attempt fallback that will only work on alcion's testing org.
func M365TenantID(t *testing.T) string {
cfg, err := readTestConfig()
require.NoError(t, err, "retrieving m365 user id from test configuration")
return cfg[TestCfgAzureTenantID]
}
// M365UserID returns an userID string representing the m365UserID described
// by either the env var CORSO_M365_TEST_USER_ID, the corso_test.toml config
// file or the default value (in that order of priority). The default is a