Use a WaitGroup for AwaitStatus (#716)
## Description This builds on the `MergeStatus` proposal and `WaitGroup` discussion proposed in #494 Required for OneDrive where we are operating on multiple collections ## Type of change Please check the type of change your PR introduces: - [x] 🌻 Feature - [ ] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 🐹 Trivial/Minor ## Issue(s) #494 ## Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual - [x] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
parent
21c2e4af14
commit
ed1a4bebce
@ -50,7 +50,7 @@ type Collection struct {
|
|||||||
service graph.Service
|
service graph.Service
|
||||||
|
|
||||||
collectionType optionIdentifier
|
collectionType optionIdentifier
|
||||||
statusCh chan<- *support.ConnectorOperationStatus
|
statusUpdater support.StatusUpdater
|
||||||
// FullPath is the slice representation of the action context passed down through the hierarchy.
|
// FullPath is the slice representation of the action context passed down through the hierarchy.
|
||||||
// The original request can be gleaned from the slice. (e.g. {<tenant ID>, <user ID>, "emails"})
|
// The original request can be gleaned from the slice. (e.g. {<tenant ID>, <user ID>, "emails"})
|
||||||
fullPath []string
|
fullPath []string
|
||||||
@ -62,14 +62,14 @@ func NewCollection(
|
|||||||
fullPath []string,
|
fullPath []string,
|
||||||
collectionType optionIdentifier,
|
collectionType optionIdentifier,
|
||||||
service graph.Service,
|
service graph.Service,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) Collection {
|
) Collection {
|
||||||
collection := Collection{
|
collection := Collection{
|
||||||
user: user,
|
user: user,
|
||||||
data: make(chan data.Stream, collectionChannelBufferSize),
|
data: make(chan data.Stream, collectionChannelBufferSize),
|
||||||
jobs: make([]string, 0),
|
jobs: make([]string, 0),
|
||||||
service: service,
|
service: service,
|
||||||
statusCh: statusCh,
|
statusUpdater: statusUpdater,
|
||||||
fullPath: fullPath,
|
fullPath: fullPath,
|
||||||
collectionType: collectionType,
|
collectionType: collectionType,
|
||||||
}
|
}
|
||||||
@ -169,7 +169,7 @@ func (col *Collection) finishPopulation(ctx context.Context, success int, errs e
|
|||||||
attempted := len(col.jobs)
|
attempted := len(col.jobs)
|
||||||
status := support.CreateStatus(ctx, support.Backup, attempted, success, 1, errs)
|
status := support.CreateStatus(ctx, support.Backup, attempted, success, 1, errs)
|
||||||
logger.Ctx(ctx).Debug(status.String())
|
logger.Ctx(ctx).Debug(status.String())
|
||||||
col.statusCh <- status
|
col.statusUpdater(status)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GraphSerializeFunc are class of functions that are used by Collections to transform GraphRetrievalFunc
|
// GraphSerializeFunc are class of functions that are used by Collections to transform GraphRetrievalFunc
|
||||||
|
|||||||
@ -33,7 +33,7 @@ type GraphIterateFunc func(
|
|||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
errs error,
|
errs error,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
graphStatusChannel chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) func(any) bool
|
) func(any) bool
|
||||||
|
|
||||||
// IterateSelectAllDescendablesForCollection utility function for
|
// IterateSelectAllDescendablesForCollection utility function for
|
||||||
@ -45,7 +45,7 @@ func IterateSelectAllDescendablesForCollections(
|
|||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
errs error,
|
errs error,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) func(any) bool {
|
) func(any) bool {
|
||||||
var (
|
var (
|
||||||
isCategorySet bool
|
isCategorySet bool
|
||||||
@ -88,7 +88,7 @@ func IterateSelectAllDescendablesForCollections(
|
|||||||
[]string{qp.Credentials.TenantID, qp.User, category, directory},
|
[]string{qp.Credentials.TenantID, qp.User, category, directory},
|
||||||
collectionType,
|
collectionType,
|
||||||
service,
|
service,
|
||||||
statusCh,
|
statusUpdater,
|
||||||
)
|
)
|
||||||
collections[directory] = &edc
|
collections[directory] = &edc
|
||||||
}
|
}
|
||||||
@ -108,7 +108,7 @@ func IterateSelectAllEventsForCollections(
|
|||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
errs error,
|
errs error,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) func(any) bool {
|
) func(any) bool {
|
||||||
return func(eventItem any) bool {
|
return func(eventItem any) bool {
|
||||||
event, ok := eventItem.(models.Eventable)
|
event, ok := eventItem.(models.Eventable)
|
||||||
@ -170,7 +170,7 @@ func IterateSelectAllEventsForCollections(
|
|||||||
[]string{qp.Credentials.TenantID, qp.User, eventsCategory, directory},
|
[]string{qp.Credentials.TenantID, qp.User, eventsCategory, directory},
|
||||||
events,
|
events,
|
||||||
service,
|
service,
|
||||||
statusCh,
|
statusUpdater,
|
||||||
)
|
)
|
||||||
collections[directory] = &edc
|
collections[directory] = &edc
|
||||||
}
|
}
|
||||||
@ -189,7 +189,7 @@ func IterateAndFilterMessagesForCollections(
|
|||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
errs error,
|
errs error,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) func(any) bool {
|
) func(any) bool {
|
||||||
var isFilterSet bool
|
var isFilterSet bool
|
||||||
|
|
||||||
@ -199,7 +199,7 @@ func IterateAndFilterMessagesForCollections(
|
|||||||
ctx,
|
ctx,
|
||||||
qp,
|
qp,
|
||||||
collections,
|
collections,
|
||||||
statusCh,
|
statusUpdater,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
errs = support.WrapAndAppend(qp.User, err, errs)
|
errs = support.WrapAndAppend(qp.User, err, errs)
|
||||||
@ -231,7 +231,7 @@ func IterateFilterFolderDirectoriesForCollections(
|
|||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
errs error,
|
errs error,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) func(any) bool {
|
) func(any) bool {
|
||||||
var (
|
var (
|
||||||
service graph.Service
|
service graph.Service
|
||||||
@ -279,7 +279,7 @@ func IterateFilterFolderDirectoriesForCollections(
|
|||||||
[]string{qp.Credentials.TenantID, qp.User, mailCategory, directory},
|
[]string{qp.Credentials.TenantID, qp.User, mailCategory, directory},
|
||||||
messages,
|
messages,
|
||||||
service,
|
service,
|
||||||
statusCh,
|
statusUpdater,
|
||||||
)
|
)
|
||||||
collections[directory] = &temp
|
collections[directory] = &temp
|
||||||
|
|
||||||
|
|||||||
@ -126,7 +126,7 @@ func CollectMailFolders(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
qp graph.QueryParams,
|
qp graph.QueryParams,
|
||||||
collections map[string]*Collection,
|
collections map[string]*Collection,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) error {
|
) error {
|
||||||
queryService, err := createService(qp.Credentials, qp.FailFast)
|
queryService, err := createService(qp.Credentials, qp.FailFast)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -156,7 +156,7 @@ func CollectMailFolders(
|
|||||||
qp,
|
qp,
|
||||||
err,
|
err,
|
||||||
collections,
|
collections,
|
||||||
statusCh,
|
statusUpdater,
|
||||||
)
|
)
|
||||||
|
|
||||||
iterateFailure := pageIterator.Iterate(callbackFunc)
|
iterateFailure := pageIterator.Iterate(callbackFunc)
|
||||||
|
|||||||
@ -7,7 +7,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync"
|
||||||
|
|
||||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
|
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
|
||||||
@ -28,16 +28,19 @@ import (
|
|||||||
// bookkeeping and interfacing with other component.
|
// bookkeeping and interfacing with other component.
|
||||||
type GraphConnector struct {
|
type GraphConnector struct {
|
||||||
graphService
|
graphService
|
||||||
tenant string
|
tenant string
|
||||||
Users map[string]string // key<email> value<id>
|
Users map[string]string // key<email> value<id>
|
||||||
status *support.ConnectorOperationStatus // contains the status of the last run status
|
credentials account.M365Config
|
||||||
statusCh chan *support.ConnectorOperationStatus
|
|
||||||
awaitingMessages int32
|
// wg is used to track completion of GC tasks
|
||||||
credentials account.M365Config
|
wg *sync.WaitGroup
|
||||||
|
// mutex used to synchronize updates to `status`
|
||||||
|
mu sync.Mutex
|
||||||
|
status support.ConnectorOperationStatus // contains the status of the last run status
|
||||||
}
|
}
|
||||||
|
|
||||||
// Service returns the GC's embedded graph.Service
|
// Service returns the GC's embedded graph.Service
|
||||||
func (gc GraphConnector) Service() graph.Service {
|
func (gc *GraphConnector) Service() graph.Service {
|
||||||
return gc.graphService
|
return gc.graphService
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -70,8 +73,7 @@ func NewGraphConnector(acct account.Account) (*GraphConnector, error) {
|
|||||||
gc := GraphConnector{
|
gc := GraphConnector{
|
||||||
tenant: m365.TenantID,
|
tenant: m365.TenantID,
|
||||||
Users: make(map[string]string, 0),
|
Users: make(map[string]string, 0),
|
||||||
status: nil,
|
wg: &sync.WaitGroup{},
|
||||||
statusCh: make(chan *support.ConnectorOperationStatus),
|
|
||||||
credentials: m365,
|
credentials: m365,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,7 +206,8 @@ func buildFromMap(isKey bool, mapping map[string]string) []string {
|
|||||||
// ExchangeDataStream returns a DataCollection which the caller can
|
// ExchangeDataStream returns a DataCollection which the caller can
|
||||||
// use to read mailbox data out for the specified user
|
// use to read mailbox data out for the specified user
|
||||||
// Assumption: User exists
|
// Assumption: User exists
|
||||||
// Add iota to this call -> mail, contacts, calendar, etc.
|
//
|
||||||
|
// Add iota to this call -> mail, contacts, calendar, etc.
|
||||||
func (gc *GraphConnector) ExchangeDataCollection(
|
func (gc *GraphConnector) ExchangeDataCollection(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
selector selectors.Selector,
|
selector selectors.Selector,
|
||||||
@ -303,10 +306,7 @@ func (gc *GraphConnector) RestoreExchangeDataCollection(
|
|||||||
gc.incrementAwaitingMessages()
|
gc.incrementAwaitingMessages()
|
||||||
|
|
||||||
status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs)
|
status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs)
|
||||||
// set the channel asynchronously so that this func doesn't block.
|
gc.UpdateStatus(status)
|
||||||
go func(cos *support.ConnectorOperationStatus) {
|
|
||||||
gc.statusCh <- cos
|
|
||||||
}(status)
|
|
||||||
|
|
||||||
return errs
|
return errs
|
||||||
}
|
}
|
||||||
@ -356,7 +356,7 @@ func (gc *GraphConnector) createCollections(
|
|||||||
// callbackFunc iterates through all M365 object target and fills exchange.Collection.jobs[]
|
// callbackFunc iterates through all M365 object target and fills exchange.Collection.jobs[]
|
||||||
// with corresponding item M365IDs. New collections are created for each directory.
|
// with corresponding item M365IDs. New collections are created for each directory.
|
||||||
// Each directory used the M365 Identifier. The use of ID stops collisions betweens users
|
// Each directory used the M365 Identifier. The use of ID stops collisions betweens users
|
||||||
callbackFunc := gIter(ctx, qp, errs, collections, gc.statusCh)
|
callbackFunc := gIter(ctx, qp, errs, collections, gc.UpdateStatus)
|
||||||
iterateError := pageIterator.Iterate(callbackFunc)
|
iterateError := pageIterator.Iterate(callbackFunc)
|
||||||
|
|
||||||
if iterateError != nil {
|
if iterateError != nil {
|
||||||
@ -377,32 +377,32 @@ func (gc *GraphConnector) createCollections(
|
|||||||
return allCollections, errs
|
return allCollections, errs
|
||||||
}
|
}
|
||||||
|
|
||||||
// AwaitStatus updates status field based on item within statusChannel.
|
// AwaitStatus waits for all gc tasks to complete and then returns status
|
||||||
func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
|
func (gc *GraphConnector) AwaitStatus() *support.ConnectorOperationStatus {
|
||||||
if gc.awaitingMessages > 0 {
|
gc.wg.Wait()
|
||||||
atomic.AddInt32(&gc.awaitingMessages, -1)
|
return &gc.status
|
||||||
gc.status = <-gc.statusCh
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return gc.status
|
// UpdateStatus is used by gc initiated tasks to indicate completion
|
||||||
|
func (gc *GraphConnector) UpdateStatus(status *support.ConnectorOperationStatus) {
|
||||||
|
gc.mu.Lock()
|
||||||
|
defer gc.mu.Unlock()
|
||||||
|
gc.status = support.MergeStatus(gc.status, *status)
|
||||||
|
gc.wg.Done()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status returns the current status of the graphConnector operaion.
|
// Status returns the current status of the graphConnector operaion.
|
||||||
func (gc *GraphConnector) Status() *support.ConnectorOperationStatus {
|
func (gc *GraphConnector) Status() support.ConnectorOperationStatus {
|
||||||
return gc.status
|
return gc.status
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrintableStatus returns a string formatted version of the GC status.
|
// PrintableStatus returns a string formatted version of the GC status.
|
||||||
func (gc *GraphConnector) PrintableStatus() string {
|
func (gc *GraphConnector) PrintableStatus() string {
|
||||||
if gc.status == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return gc.status.String()
|
return gc.status.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (gc *GraphConnector) incrementAwaitingMessages() {
|
func (gc *GraphConnector) incrementAwaitingMessages() {
|
||||||
atomic.AddInt32(&gc.awaitingMessages, 1)
|
gc.wg.Add(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsRecoverableError returns true iff error is a RecoverableGCEerror
|
// IsRecoverableError returns true iff error is a RecoverableGCEerror
|
||||||
|
|||||||
@ -2,6 +2,7 @@ package connector
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
@ -93,29 +94,39 @@ func (suite *DisconnectedGraphConnectorSuite) TestInterfaceAlignment() {
|
|||||||
assert.NotNil(suite.T(), dc)
|
assert.NotNil(suite.T(), dc)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func statusTestTask(gc *GraphConnector, objects, success, folder int) {
|
||||||
|
status := support.CreateStatus(
|
||||||
|
context.Background(),
|
||||||
|
support.Restore,
|
||||||
|
objects, success, folder,
|
||||||
|
support.WrapAndAppend(
|
||||||
|
"tres",
|
||||||
|
errors.New("three"),
|
||||||
|
support.WrapAndAppend("arc376", errors.New("one"), errors.New("two")),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
gc.UpdateStatus(status)
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() {
|
func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_Status() {
|
||||||
gc := GraphConnector{
|
gc := GraphConnector{wg: &sync.WaitGroup{}}
|
||||||
statusCh: make(chan *support.ConnectorOperationStatus),
|
|
||||||
}
|
// Two tasks
|
||||||
suite.Equal(len(gc.PrintableStatus()), 0)
|
gc.incrementAwaitingMessages()
|
||||||
gc.incrementAwaitingMessages()
|
gc.incrementAwaitingMessages()
|
||||||
|
|
||||||
go func() {
|
// Each helper task processes 4 objects, 1 success, 3 errors, 1 folders
|
||||||
status := support.CreateStatus(
|
go statusTestTask(&gc, 4, 1, 1)
|
||||||
context.Background(),
|
go statusTestTask(&gc, 4, 1, 1)
|
||||||
support.Restore,
|
|
||||||
12, 9, 8,
|
|
||||||
support.WrapAndAppend(
|
|
||||||
"tres",
|
|
||||||
errors.New("three"),
|
|
||||||
support.WrapAndAppend("arc376", errors.New("one"), errors.New("two")),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
gc.statusCh <- status
|
|
||||||
}()
|
|
||||||
gc.AwaitStatus()
|
gc.AwaitStatus()
|
||||||
suite.Greater(len(gc.PrintableStatus()), 0)
|
suite.NotEmpty(gc.PrintableStatus())
|
||||||
suite.Greater(gc.Status().ObjectCount, 0)
|
// Expect 8 objects
|
||||||
|
suite.Equal(8, gc.Status().ObjectCount)
|
||||||
|
// Expect 2 success
|
||||||
|
suite.Equal(2, gc.Status().Successful)
|
||||||
|
// Expect 2 folders
|
||||||
|
suite.Equal(2, gc.Status().FolderCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_ErrorChecking() {
|
func (suite *DisconnectedGraphConnectorSuite) TestGraphConnector_ErrorChecking() {
|
||||||
|
|||||||
@ -64,8 +64,6 @@ func (suite *GraphConnectorIntegrationSuite) TestSetTenantUsers() {
|
|||||||
newConnector := GraphConnector{
|
newConnector := GraphConnector{
|
||||||
tenant: "test_tenant",
|
tenant: "test_tenant",
|
||||||
Users: make(map[string]string, 0),
|
Users: make(map[string]string, 0),
|
||||||
status: nil,
|
|
||||||
statusCh: make(chan *support.ConnectorOperationStatus),
|
|
||||||
credentials: suite.connector.credentials,
|
credentials: suite.connector.credentials,
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -94,8 +92,9 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
|
|||||||
collectionList, err := connector.ExchangeDataCollection(context.Background(), sel.Selector)
|
collectionList, err := connector.ExchangeDataCollection(context.Background(), sel.Selector)
|
||||||
assert.NotNil(t, collectionList, "collection list")
|
assert.NotNil(t, collectionList, "collection list")
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.True(t, connector.awaitingMessages > 0)
|
assert.Zero(t, connector.status.ObjectCount)
|
||||||
assert.Nil(t, connector.status)
|
assert.Zero(t, connector.status.FolderCount)
|
||||||
|
assert.Zero(t, connector.status.Successful)
|
||||||
|
|
||||||
streams := make(map[string]<-chan data.Stream)
|
streams := make(map[string]<-chan data.Stream)
|
||||||
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
|
// Verify Items() call returns an iterable channel(e.g. a channel that has been closed)
|
||||||
@ -105,10 +104,8 @@ func (suite *GraphConnectorIntegrationSuite) TestExchangeDataCollection() {
|
|||||||
streams[testName] = temp
|
streams[testName] = temp
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < int(connector.awaitingMessages); i++ {
|
status := connector.AwaitStatus()
|
||||||
status := connector.AwaitStatus()
|
assert.NotZero(t, status.Successful)
|
||||||
assert.NotNil(t, status)
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, channel := range streams {
|
for name, channel := range streams {
|
||||||
suite.T().Run(name, func(t *testing.T) {
|
suite.T().Run(name, func(t *testing.T) {
|
||||||
@ -294,8 +291,8 @@ func (suite *GraphConnectorIntegrationSuite) TestAccessOfInboxAllUsers() {
|
|||||||
// Exchange Functions
|
// Exchange Functions
|
||||||
//-------------------------------------------------------
|
//-------------------------------------------------------
|
||||||
|
|
||||||
// TestCreateAndDeleteMailFolder ensures GraphConnector has the ability
|
// TestCreateAndDeleteMailFolder ensures GraphConnector has the ability
|
||||||
// to create and remove folders within the tenant
|
// to create and remove folders within the tenant
|
||||||
func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteMailFolder() {
|
func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteMailFolder() {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
folderName := "TestFolder: " + common.FormatSimpleDateTime(now)
|
folderName := "TestFolder: " + common.FormatSimpleDateTime(now)
|
||||||
|
|||||||
@ -34,10 +34,10 @@ type Collection struct {
|
|||||||
// M365 IDs of file items within this collection
|
// M365 IDs of file items within this collection
|
||||||
driveItemIDs []string
|
driveItemIDs []string
|
||||||
// M365 ID of the drive this collection was created from
|
// M365 ID of the drive this collection was created from
|
||||||
driveID string
|
driveID string
|
||||||
service graph.Service
|
service graph.Service
|
||||||
statusCh chan<- *support.ConnectorOperationStatus
|
statusUpdater support.StatusUpdater
|
||||||
itemReader itemReaderFunc
|
itemReader itemReaderFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
// itemReadFunc returns a reader for the specified item
|
// itemReadFunc returns a reader for the specified item
|
||||||
@ -49,15 +49,15 @@ type itemReaderFunc func(
|
|||||||
|
|
||||||
// NewCollection creates a Collection
|
// NewCollection creates a Collection
|
||||||
func NewCollection(folderPath, driveID string, service graph.Service,
|
func NewCollection(folderPath, driveID string, service graph.Service,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) *Collection {
|
) *Collection {
|
||||||
c := &Collection{
|
c := &Collection{
|
||||||
folderPath: folderPath,
|
folderPath: folderPath,
|
||||||
driveItemIDs: []string{},
|
driveItemIDs: []string{},
|
||||||
driveID: driveID,
|
driveID: driveID,
|
||||||
service: service,
|
service: service,
|
||||||
data: make(chan data.Stream, collectionChannelBufferSize),
|
data: make(chan data.Stream, collectionChannelBufferSize),
|
||||||
statusCh: statusCh,
|
statusUpdater: statusUpdater,
|
||||||
}
|
}
|
||||||
// Allows tests to set a mock populator
|
// Allows tests to set a mock populator
|
||||||
c.itemReader = driveItemReader
|
c.itemReader = driveItemReader
|
||||||
@ -133,5 +133,5 @@ func (oc *Collection) populateItems(ctx context.Context) {
|
|||||||
1, // num folders (always 1)
|
1, // num folders (always 1)
|
||||||
errs)
|
errs)
|
||||||
logger.Ctx(ctx).Debug(status.String())
|
logger.Ctx(ctx).Debug(status.String())
|
||||||
oc.statusCh <- status
|
oc.statusUpdater(status)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
|
||||||
@ -14,6 +15,7 @@ import (
|
|||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
|
||||||
"github.com/alcionai/corso/internal/connector/graph"
|
"github.com/alcionai/corso/internal/connector/graph"
|
||||||
|
"github.com/alcionai/corso/internal/connector/support"
|
||||||
"github.com/alcionai/corso/internal/data"
|
"github.com/alcionai/corso/internal/data"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -40,9 +42,23 @@ func TestOneDriveCollectionSuite(t *testing.T) {
|
|||||||
suite.Run(t, new(OneDriveCollectionSuite))
|
suite.Run(t, new(OneDriveCollectionSuite))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Returns a status update function that signals the specified WaitGroup when it is done
|
||||||
|
func (suite *OneDriveCollectionSuite) testStatusUpdater(
|
||||||
|
wg *sync.WaitGroup,
|
||||||
|
statusToUpdate *support.ConnectorOperationStatus,
|
||||||
|
) support.StatusUpdater {
|
||||||
|
return func(s *support.ConnectorOperationStatus) {
|
||||||
|
suite.T().Logf("Update status %v, count %d, success %d", s, s.ObjectCount, s.Successful)
|
||||||
|
*statusToUpdate = *s
|
||||||
|
wg.Done()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
|
func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
collStatus := support.ConnectorOperationStatus{}
|
||||||
folderPath := "dir1/dir2/dir3"
|
folderPath := "dir1/dir2/dir3"
|
||||||
coll := NewCollection(folderPath, "fakeDriveID", suite, nil)
|
coll := NewCollection(folderPath, "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
|
||||||
require.NotNil(suite.T(), coll)
|
require.NotNil(suite.T(), coll)
|
||||||
assert.Equal(suite.T(), filepath.SplitList(folderPath), coll.FullPath())
|
assert.Equal(suite.T(), filepath.SplitList(folderPath), coll.FullPath())
|
||||||
|
|
||||||
@ -58,13 +74,16 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Read items from the collection
|
// Read items from the collection
|
||||||
|
wg.Add(1)
|
||||||
readItems := []data.Stream{}
|
readItems := []data.Stream{}
|
||||||
for item := range coll.Items() {
|
for item := range coll.Items() {
|
||||||
readItems = append(readItems, item)
|
readItems = append(readItems, item)
|
||||||
}
|
}
|
||||||
|
wg.Wait()
|
||||||
// Expect only 1 item
|
// Expect only 1 item
|
||||||
require.Len(suite.T(), readItems, 1)
|
require.Len(suite.T(), readItems, 1)
|
||||||
|
require.Equal(suite.T(), 1, collStatus.ObjectCount)
|
||||||
|
require.Equal(suite.T(), 1, collStatus.Successful)
|
||||||
|
|
||||||
// Validate item info and data
|
// Validate item info and data
|
||||||
readItem := readItems[0]
|
readItem := readItems[0]
|
||||||
@ -82,7 +101,11 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollection() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
|
func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
|
||||||
coll := NewCollection("folderPath", "fakeDriveID", suite, nil)
|
wg := sync.WaitGroup{}
|
||||||
|
collStatus := support.ConnectorOperationStatus{}
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
coll := NewCollection("folderPath", "fakeDriveID", suite, suite.testStatusUpdater(&wg, &collStatus))
|
||||||
coll.Add("testItemID")
|
coll.Add("testItemID")
|
||||||
|
|
||||||
readError := errors.New("Test error")
|
readError := errors.New("Test error")
|
||||||
@ -91,6 +114,9 @@ func (suite *OneDriveCollectionSuite) TestOneDriveCollectionReadError() {
|
|||||||
return "", nil, readError
|
return "", nil, readError
|
||||||
}
|
}
|
||||||
|
|
||||||
|
coll.Items()
|
||||||
|
wg.Wait()
|
||||||
// Expect no items
|
// Expect no items
|
||||||
require.Len(suite.T(), coll.Items(), 0)
|
require.Equal(suite.T(), 1, collStatus.ObjectCount)
|
||||||
|
require.Equal(suite.T(), 0, collStatus.Successful)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -20,7 +20,7 @@ type Collections struct {
|
|||||||
// for a OneDrive folder
|
// for a OneDrive folder
|
||||||
collectionMap map[string]data.Collection
|
collectionMap map[string]data.Collection
|
||||||
service graph.Service
|
service graph.Service
|
||||||
statusCh chan<- *support.ConnectorOperationStatus
|
statusUpdater support.StatusUpdater
|
||||||
|
|
||||||
// Track stats from drive enumeration
|
// Track stats from drive enumeration
|
||||||
numItems int
|
numItems int
|
||||||
@ -32,13 +32,13 @@ type Collections struct {
|
|||||||
func NewCollections(
|
func NewCollections(
|
||||||
user string,
|
user string,
|
||||||
service graph.Service,
|
service graph.Service,
|
||||||
statusCh chan<- *support.ConnectorOperationStatus,
|
statusUpdater support.StatusUpdater,
|
||||||
) *Collections {
|
) *Collections {
|
||||||
return &Collections{
|
return &Collections{
|
||||||
user: user,
|
user: user,
|
||||||
collectionMap: map[string]data.Collection{},
|
collectionMap: map[string]data.Collection{},
|
||||||
service: service,
|
service: service,
|
||||||
statusCh: statusCh,
|
statusUpdater: statusUpdater,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -80,7 +80,7 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite
|
|||||||
// Create a collection for the parent of this item
|
// Create a collection for the parent of this item
|
||||||
collectionPath := *item.GetParentReference().GetPath()
|
collectionPath := *item.GetParentReference().GetPath()
|
||||||
if _, found := c.collectionMap[collectionPath]; !found {
|
if _, found := c.collectionMap[collectionPath]; !found {
|
||||||
c.collectionMap[collectionPath] = NewCollection(collectionPath, driveID, c.service, c.statusCh)
|
c.collectionMap[collectionPath] = NewCollection(collectionPath, driveID, c.service, c.statusUpdater)
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case item.GetFolder() != nil, item.GetPackage() != nil:
|
case item.GetFolder() != nil, item.GetPackage() != nil:
|
||||||
@ -89,7 +89,7 @@ func (c *Collections) updateCollections(ctx context.Context, driveID string, ite
|
|||||||
// e.g. a ".folderMetadataFile"
|
// e.g. a ".folderMetadataFile"
|
||||||
itemPath := path.Join(*item.GetParentReference().GetPath(), *item.GetName())
|
itemPath := path.Join(*item.GetParentReference().GetPath(), *item.GetName())
|
||||||
if _, found := c.collectionMap[itemPath]; !found {
|
if _, found := c.collectionMap[itemPath]; !found {
|
||||||
c.collectionMap[itemPath] = NewCollection(itemPath, driveID, c.service, c.statusCh)
|
c.collectionMap[itemPath] = NewCollection(itemPath, driveID, c.service, c.statusUpdater)
|
||||||
}
|
}
|
||||||
case item.GetFile() != nil:
|
case item.GetFile() != nil:
|
||||||
collection := c.collectionMap[collectionPath].(*Collection)
|
collection := c.collectionMap[collectionPath].(*Collection)
|
||||||
|
|||||||
@ -62,6 +62,40 @@ func CreateStatus(
|
|||||||
return &status
|
return &status
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Function signature for a status updater
|
||||||
|
// Used to define a function that an async connector task can call
|
||||||
|
// to on completion with its ConnectorOperationStatus
|
||||||
|
type StatusUpdater func(*ConnectorOperationStatus)
|
||||||
|
|
||||||
|
// MergeStatus combines ConnectorOperationsStatus value into a single status
|
||||||
|
func MergeStatus(one, two ConnectorOperationStatus) ConnectorOperationStatus {
|
||||||
|
var hasErrors bool
|
||||||
|
|
||||||
|
if one.lastOperation == OpUnknown {
|
||||||
|
return two
|
||||||
|
}
|
||||||
|
|
||||||
|
if two.lastOperation == OpUnknown {
|
||||||
|
return one
|
||||||
|
}
|
||||||
|
|
||||||
|
if one.incomplete || two.incomplete {
|
||||||
|
hasErrors = true
|
||||||
|
}
|
||||||
|
|
||||||
|
status := ConnectorOperationStatus{
|
||||||
|
lastOperation: one.lastOperation,
|
||||||
|
ObjectCount: one.ObjectCount + two.ObjectCount,
|
||||||
|
FolderCount: one.FolderCount + two.FolderCount,
|
||||||
|
Successful: one.Successful + two.Successful,
|
||||||
|
errorCount: one.errorCount + two.errorCount,
|
||||||
|
incomplete: hasErrors,
|
||||||
|
incompleteReason: one.incompleteReason + " " + two.incompleteReason,
|
||||||
|
}
|
||||||
|
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
func (cos *ConnectorOperationStatus) String() string {
|
func (cos *ConnectorOperationStatus) String() string {
|
||||||
message := fmt.Sprintf("Action: %s performed on %d of %d objects within %d directories.", cos.lastOperation.String(),
|
message := fmt.Sprintf("Action: %s performed on %d of %d objects within %d directories.", cos.lastOperation.String(),
|
||||||
cos.Successful, cos.ObjectCount, cos.FolderCount)
|
cos.Successful, cos.ObjectCount, cos.FolderCount)
|
||||||
|
|||||||
@ -78,3 +78,61 @@ func (suite *GCStatusTestSuite) TestCreateStatus_InvalidStatus() {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (suite *GCStatusTestSuite) TestMergeStatus() {
|
||||||
|
simpleContext := context.Background()
|
||||||
|
table := []struct {
|
||||||
|
name string
|
||||||
|
one ConnectorOperationStatus
|
||||||
|
two ConnectorOperationStatus
|
||||||
|
expected statusParams
|
||||||
|
isIncomplete assert.BoolAssertionFunc
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Test: Status + unknown",
|
||||||
|
one: *CreateStatus(simpleContext, Backup, 1, 1, 1, nil),
|
||||||
|
two: ConnectorOperationStatus{},
|
||||||
|
expected: statusParams{Backup, 1, 1, 1, nil},
|
||||||
|
isIncomplete: assert.False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Test: unknown + Status",
|
||||||
|
one: ConnectorOperationStatus{},
|
||||||
|
two: *CreateStatus(simpleContext, Backup, 1, 1, 1, nil),
|
||||||
|
expected: statusParams{Backup, 1, 1, 1, nil},
|
||||||
|
isIncomplete: assert.False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Test: Successful + Successful",
|
||||||
|
one: *CreateStatus(simpleContext, Backup, 1, 1, 1, nil),
|
||||||
|
two: *CreateStatus(simpleContext, Backup, 3, 3, 3, nil),
|
||||||
|
expected: statusParams{Backup, 4, 4, 4, nil},
|
||||||
|
isIncomplete: assert.False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Test: Successful + Unsuccessful",
|
||||||
|
one: *CreateStatus(simpleContext, Backup, 17, 17, 13, nil),
|
||||||
|
two: *CreateStatus(
|
||||||
|
simpleContext,
|
||||||
|
Backup,
|
||||||
|
12,
|
||||||
|
9,
|
||||||
|
8,
|
||||||
|
WrapAndAppend("tres", errors.New("three"), WrapAndAppend("arc376", errors.New("one"), errors.New("two"))),
|
||||||
|
),
|
||||||
|
expected: statusParams{Backup, 29, 26, 21, nil},
|
||||||
|
isIncomplete: assert.True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range table {
|
||||||
|
suite.T().Run(test.name, func(t *testing.T) {
|
||||||
|
returned := MergeStatus(test.one, test.two)
|
||||||
|
suite.Equal(returned.FolderCount, test.expected.folders)
|
||||||
|
suite.Equal(returned.ObjectCount, test.expected.objects)
|
||||||
|
suite.Equal(returned.lastOperation, test.expected.operationType)
|
||||||
|
suite.Equal(returned.Successful, test.expected.success)
|
||||||
|
test.isIncomplete(t, returned.incomplete)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user