Add prefetch collection for groups (#4906)
<!-- PR description--> Minor refactor before introducing `lazyFetchCollection` for groups. We'll utilize `lazyFetchCollection` for group mailboxes and will continue to use `prefetchCollection` for channels. --- #### Does this PR need a docs update or release note? - [ ] ✅ Yes, it's included - [ ] 🕐 Yes, but in a later PR - [x] ⛔ No #### Type of change <!--- Please check the type of change your PR introduces: ---> - [ ] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Supportability/Tests - [ ] 💻 CI/Deployment - [x] 🧹 Tech Debt/Cleanup #### Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> * https://github.com/alcionai/corso/issues/4862 #### Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
926489d004
commit
127570953a
@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
"golang.org/x/exp/maps"
|
|
||||||
|
|
||||||
"github.com/alcionai/corso/src/internal/common/pii"
|
"github.com/alcionai/corso/src/internal/common/pii"
|
||||||
"github.com/alcionai/corso/src/internal/common/ptr"
|
"github.com/alcionai/corso/src/internal/common/ptr"
|
||||||
@ -174,10 +173,9 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
added := str.SliceToMap(maps.Keys(addAndRem.Added))
|
|
||||||
removed := str.SliceToMap(addAndRem.Removed)
|
removed := str.SliceToMap(addAndRem.Removed)
|
||||||
|
|
||||||
cl.Add(count.ItemsAdded, int64(len(added)))
|
cl.Add(count.ItemsAdded, int64(len(addAndRem.Added)))
|
||||||
cl.Add(count.ItemsRemoved, int64(len(removed)))
|
cl.Add(count.ItemsRemoved, int64(len(removed)))
|
||||||
|
|
||||||
if len(addAndRem.DU.URL) > 0 {
|
if len(addAndRem.DU.URL) > 0 {
|
||||||
@ -198,7 +196,7 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
|
|||||||
// deleted and then restored will have a different ID than they did
|
// deleted and then restored will have a different ID than they did
|
||||||
// originally.
|
// originally.
|
||||||
for remove := range removed {
|
for remove := range removed {
|
||||||
delete(added, remove)
|
delete(addAndRem.Added, remove)
|
||||||
}
|
}
|
||||||
|
|
||||||
edc := NewCollection(
|
edc := NewCollection(
|
||||||
@ -211,7 +209,7 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
|
|||||||
cl),
|
cl),
|
||||||
bh,
|
bh,
|
||||||
qp.ProtectedResource.ID(),
|
qp.ProtectedResource.ID(),
|
||||||
added,
|
addAndRem.Added,
|
||||||
removed,
|
removed,
|
||||||
c,
|
c,
|
||||||
statusUpdater)
|
statusUpdater)
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/alcionai/clues"
|
"github.com/alcionai/clues"
|
||||||
kjson "github.com/microsoft/kiota-serialization-json-go"
|
kjson "github.com/microsoft/kiota-serialization-json-go"
|
||||||
@ -20,14 +21,14 @@ import (
|
|||||||
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
"github.com/alcionai/corso/src/pkg/services/m365/api/graph"
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ data.BackupCollection = &Collection[graph.GetIDer, groupsItemer]{}
|
var _ data.BackupCollection = &prefetchCollection[graph.GetIDer, groupsItemer]{}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
collectionChannelBufferSize = 1000
|
collectionChannelBufferSize = 1000
|
||||||
numberOfRetries = 4
|
numberOfRetries = 4
|
||||||
)
|
)
|
||||||
|
|
||||||
type Collection[C graph.GetIDer, I groupsItemer] struct {
|
type prefetchCollection[C graph.GetIDer, I groupsItemer] struct {
|
||||||
data.BaseCollection
|
data.BaseCollection
|
||||||
protectedResource string
|
protectedResource string
|
||||||
stream chan data.Item
|
stream chan data.Item
|
||||||
@ -35,7 +36,7 @@ type Collection[C graph.GetIDer, I groupsItemer] struct {
|
|||||||
contains container[C]
|
contains container[C]
|
||||||
|
|
||||||
// added is a list of existing item IDs that were added to a container
|
// added is a list of existing item IDs that were added to a container
|
||||||
added map[string]struct{}
|
added map[string]time.Time
|
||||||
// removed is a list of item IDs that were deleted from, or moved out, of a container
|
// removed is a list of item IDs that were deleted from, or moved out, of a container
|
||||||
removed map[string]struct{}
|
removed map[string]struct{}
|
||||||
|
|
||||||
@ -54,12 +55,12 @@ func NewCollection[C graph.GetIDer, I groupsItemer](
|
|||||||
baseCol data.BaseCollection,
|
baseCol data.BaseCollection,
|
||||||
getAndAugment getItemAndAugmentInfoer[C, I],
|
getAndAugment getItemAndAugmentInfoer[C, I],
|
||||||
protectedResource string,
|
protectedResource string,
|
||||||
added map[string]struct{},
|
added map[string]time.Time,
|
||||||
removed map[string]struct{},
|
removed map[string]struct{},
|
||||||
contains container[C],
|
contains container[C],
|
||||||
statusUpdater support.StatusUpdater,
|
statusUpdater support.StatusUpdater,
|
||||||
) Collection[C, I] {
|
) prefetchCollection[C, I] {
|
||||||
collection := Collection[C, I]{
|
collection := prefetchCollection[C, I]{
|
||||||
BaseCollection: baseCol,
|
BaseCollection: baseCol,
|
||||||
added: added,
|
added: added,
|
||||||
contains: contains,
|
contains: contains,
|
||||||
@ -75,7 +76,7 @@ func NewCollection[C graph.GetIDer, I groupsItemer](
|
|||||||
|
|
||||||
// Items utility function to asynchronously execute process to fill data channel with
|
// Items utility function to asynchronously execute process to fill data channel with
|
||||||
// M365 exchange objects and returns the data channel
|
// M365 exchange objects and returns the data channel
|
||||||
func (col *Collection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item {
|
func (col *prefetchCollection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item {
|
||||||
go col.streamItems(ctx, errs)
|
go col.streamItems(ctx, errs)
|
||||||
return col.stream
|
return col.stream
|
||||||
}
|
}
|
||||||
@ -84,7 +85,7 @@ func (col *Collection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan
|
|||||||
// items() production
|
// items() production
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
func (col *Collection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) {
|
func (col *prefetchCollection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) {
|
||||||
var (
|
var (
|
||||||
streamedItems int64
|
streamedItems int64
|
||||||
totalBytes int64
|
totalBytes int64
|
||||||
@ -212,9 +213,9 @@ func (col *Collection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) {
|
|||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
// finishPopulation is a utility function used to close a Collection's data channel
|
// finishPopulation is a utility function used to close a collection's data channel
|
||||||
// and to send the status update through the channel.
|
// and to send the status update through the channel.
|
||||||
func (col *Collection[C, I]) finishPopulation(
|
func (col *prefetchCollection[C, I]) finishPopulation(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
streamedItems, totalBytes int64,
|
streamedItems, totalBytes int64,
|
||||||
err error,
|
err error,
|
||||||
|
|||||||
@ -162,7 +162,7 @@ func (getAndAugmentChannelMessage) augmentItemInfo(*details.GroupsInfo, models.C
|
|||||||
// no-op
|
// no-op
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *CollectionUnitSuite) TestCollection_streamItems() {
|
func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() {
|
||||||
var (
|
var (
|
||||||
t = suite.T()
|
t = suite.T()
|
||||||
start = time.Now().Add(-1 * time.Second)
|
start = time.Now().Add(-1 * time.Second)
|
||||||
@ -177,26 +177,22 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
|
|||||||
|
|
||||||
table := []struct {
|
table := []struct {
|
||||||
name string
|
name string
|
||||||
added map[string]struct{}
|
added map[string]time.Time
|
||||||
removed map[string]struct{}
|
removed map[string]struct{}
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "no items",
|
name: "no items",
|
||||||
added: map[string]struct{}{},
|
|
||||||
removed: map[string]struct{}{},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "only added items",
|
name: "only added items",
|
||||||
added: map[string]struct{}{
|
added: map[string]time.Time{
|
||||||
"fisher": {},
|
"fisher": {},
|
||||||
"flannigan": {},
|
"flannigan": {},
|
||||||
"fitzbog": {},
|
"fitzbog": {},
|
||||||
},
|
},
|
||||||
removed: map[string]struct{}{},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "only removed items",
|
name: "only removed items",
|
||||||
added: map[string]struct{}{},
|
|
||||||
removed: map[string]struct{}{
|
removed: map[string]struct{}{
|
||||||
"princess": {},
|
"princess": {},
|
||||||
"poppy": {},
|
"poppy": {},
|
||||||
@ -204,8 +200,10 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "added and removed items",
|
name: "added and removed items",
|
||||||
added: map[string]struct{}{},
|
added: map[string]time.Time{
|
||||||
|
"goblin": {},
|
||||||
|
},
|
||||||
removed: map[string]struct{}{
|
removed: map[string]struct{}{
|
||||||
"general": {},
|
"general": {},
|
||||||
"goose": {},
|
"goose": {},
|
||||||
@ -224,7 +222,7 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
|
|||||||
ctx, flush := tester.NewContext(t)
|
ctx, flush := tester.NewContext(t)
|
||||||
defer flush()
|
defer flush()
|
||||||
|
|
||||||
col := &Collection[models.Channelable, models.ChatMessageable]{
|
col := &prefetchCollection[models.Channelable, models.ChatMessageable]{
|
||||||
BaseCollection: data.NewBaseCollection(
|
BaseCollection: data.NewBaseCollection(
|
||||||
fullPath,
|
fullPath,
|
||||||
nil,
|
nil,
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user