Add prefetch collection for groups (#4906)

<!-- PR description-->

Minor refactor before introducing `lazyFetchCollection` for groups. We'll utilize `lazyFetchCollection` for group mailboxes and will continue to use `prefetchCollection` for channels.


---

#### Does this PR need a docs update or release note?

- [ ]  Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x]  No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [x] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* https://github.com/alcionai/corso/issues/4862

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [x]  Unit test
- [ ] 💚 E2E
This commit is contained in:
Abhishek Pandey 2023-12-22 15:18:58 -08:00 committed by GitHub
parent 926489d004
commit 127570953a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 27 deletions

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"github.com/alcionai/clues" "github.com/alcionai/clues"
"golang.org/x/exp/maps"
"github.com/alcionai/corso/src/internal/common/pii" "github.com/alcionai/corso/src/internal/common/pii"
"github.com/alcionai/corso/src/internal/common/ptr" "github.com/alcionai/corso/src/internal/common/ptr"
@ -174,10 +173,9 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
continue continue
} }
added := str.SliceToMap(maps.Keys(addAndRem.Added))
removed := str.SliceToMap(addAndRem.Removed) removed := str.SliceToMap(addAndRem.Removed)
cl.Add(count.ItemsAdded, int64(len(added))) cl.Add(count.ItemsAdded, int64(len(addAndRem.Added)))
cl.Add(count.ItemsRemoved, int64(len(removed))) cl.Add(count.ItemsRemoved, int64(len(removed)))
if len(addAndRem.DU.URL) > 0 { if len(addAndRem.DU.URL) > 0 {
@ -198,7 +196,7 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
// deleted and then restored will have a different ID than they did // deleted and then restored will have a different ID than they did
// originally. // originally.
for remove := range removed { for remove := range removed {
delete(added, remove) delete(addAndRem.Added, remove)
} }
edc := NewCollection( edc := NewCollection(
@ -211,7 +209,7 @@ func populateCollections[C graph.GetIDer, I groupsItemer](
cl), cl),
bh, bh,
qp.ProtectedResource.ID(), qp.ProtectedResource.ID(),
added, addAndRem.Added,
removed, removed,
c, c,
statusUpdater) statusUpdater)

View File

@ -6,6 +6,7 @@ import (
"io" "io"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time"
"github.com/alcionai/clues" "github.com/alcionai/clues"
kjson "github.com/microsoft/kiota-serialization-json-go" kjson "github.com/microsoft/kiota-serialization-json-go"
@ -20,14 +21,14 @@ import (
"github.com/alcionai/corso/src/pkg/services/m365/api/graph" "github.com/alcionai/corso/src/pkg/services/m365/api/graph"
) )
var _ data.BackupCollection = &Collection[graph.GetIDer, groupsItemer]{} var _ data.BackupCollection = &prefetchCollection[graph.GetIDer, groupsItemer]{}
const ( const (
collectionChannelBufferSize = 1000 collectionChannelBufferSize = 1000
numberOfRetries = 4 numberOfRetries = 4
) )
type Collection[C graph.GetIDer, I groupsItemer] struct { type prefetchCollection[C graph.GetIDer, I groupsItemer] struct {
data.BaseCollection data.BaseCollection
protectedResource string protectedResource string
stream chan data.Item stream chan data.Item
@ -35,7 +36,7 @@ type Collection[C graph.GetIDer, I groupsItemer] struct {
contains container[C] contains container[C]
// added is a list of existing item IDs that were added to a container // added is a list of existing item IDs that were added to a container
added map[string]struct{} added map[string]time.Time
// removed is a list of item IDs that were deleted from, or moved out, of a container // removed is a list of item IDs that were deleted from, or moved out, of a container
removed map[string]struct{} removed map[string]struct{}
@ -54,12 +55,12 @@ func NewCollection[C graph.GetIDer, I groupsItemer](
baseCol data.BaseCollection, baseCol data.BaseCollection,
getAndAugment getItemAndAugmentInfoer[C, I], getAndAugment getItemAndAugmentInfoer[C, I],
protectedResource string, protectedResource string,
added map[string]struct{}, added map[string]time.Time,
removed map[string]struct{}, removed map[string]struct{},
contains container[C], contains container[C],
statusUpdater support.StatusUpdater, statusUpdater support.StatusUpdater,
) Collection[C, I] { ) prefetchCollection[C, I] {
collection := Collection[C, I]{ collection := prefetchCollection[C, I]{
BaseCollection: baseCol, BaseCollection: baseCol,
added: added, added: added,
contains: contains, contains: contains,
@ -75,7 +76,7 @@ func NewCollection[C graph.GetIDer, I groupsItemer](
// Items utility function to asynchronously execute process to fill data channel with // Items utility function to asynchronously execute process to fill data channel with
// M365 exchange objects and returns the data channel // M365 exchange objects and returns the data channel
func (col *Collection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item { func (col *prefetchCollection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan data.Item {
go col.streamItems(ctx, errs) go col.streamItems(ctx, errs)
return col.stream return col.stream
} }
@ -84,7 +85,7 @@ func (col *Collection[C, I]) Items(ctx context.Context, errs *fault.Bus) <-chan
// items() production // items() production
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func (col *Collection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) { func (col *prefetchCollection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) {
var ( var (
streamedItems int64 streamedItems int64
totalBytes int64 totalBytes int64
@ -212,9 +213,9 @@ func (col *Collection[C, I]) streamItems(ctx context.Context, errs *fault.Bus) {
wg.Wait() wg.Wait()
} }
// finishPopulation is a utility function used to close a Collection's data channel // finishPopulation is a utility function used to close a collection's data channel
// and to send the status update through the channel. // and to send the status update through the channel.
func (col *Collection[C, I]) finishPopulation( func (col *prefetchCollection[C, I]) finishPopulation(
ctx context.Context, ctx context.Context,
streamedItems, totalBytes int64, streamedItems, totalBytes int64,
err error, err error,

View File

@ -162,7 +162,7 @@ func (getAndAugmentChannelMessage) augmentItemInfo(*details.GroupsInfo, models.C
// no-op // no-op
} }
func (suite *CollectionUnitSuite) TestCollection_streamItems() { func (suite *CollectionUnitSuite) TestPrefetchCollection_streamItems() {
var ( var (
t = suite.T() t = suite.T()
start = time.Now().Add(-1 * time.Second) start = time.Now().Add(-1 * time.Second)
@ -177,26 +177,22 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
table := []struct { table := []struct {
name string name string
added map[string]struct{} added map[string]time.Time
removed map[string]struct{} removed map[string]struct{}
}{ }{
{ {
name: "no items", name: "no items",
added: map[string]struct{}{},
removed: map[string]struct{}{},
}, },
{ {
name: "only added items", name: "only added items",
added: map[string]struct{}{ added: map[string]time.Time{
"fisher": {}, "fisher": {},
"flannigan": {}, "flannigan": {},
"fitzbog": {}, "fitzbog": {},
}, },
removed: map[string]struct{}{},
}, },
{ {
name: "only removed items", name: "only removed items",
added: map[string]struct{}{},
removed: map[string]struct{}{ removed: map[string]struct{}{
"princess": {}, "princess": {},
"poppy": {}, "poppy": {},
@ -204,8 +200,10 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
}, },
}, },
{ {
name: "added and removed items", name: "added and removed items",
added: map[string]struct{}{}, added: map[string]time.Time{
"goblin": {},
},
removed: map[string]struct{}{ removed: map[string]struct{}{
"general": {}, "general": {},
"goose": {}, "goose": {},
@ -224,7 +222,7 @@ func (suite *CollectionUnitSuite) TestCollection_streamItems() {
ctx, flush := tester.NewContext(t) ctx, flush := tester.NewContext(t)
defer flush() defer flush()
col := &Collection[models.Channelable, models.ChatMessageable]{ col := &prefetchCollection[models.Channelable, models.ChatMessageable]{
BaseCollection: data.NewBaseCollection( BaseCollection: data.NewBaseCollection(
fullPath, fullPath,
nil, nil,