larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migrate containerCache to graph pkg

Browse Source 
ContainerCache is a generic resource that can
be moved to the graph package.  Service-
specific code is tied to the `populate` func,
which remains in each service/category.  This
change is a prepatory refactoring to reduce
boilerplate for sharepoint implementation.
This commit is contained in:
ryanfkeepers 2022-11-15 16:46:33 -07:00
parent 92e109d4eb
commit c97f5ea9a7
11 changed files with 108 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
src/internal/connector/exchange/cache_container.go
View File

@ -2,66 +2,8 @@ package exchange
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/path"
)
// checkIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func checkIDAndName(c graph.Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
}
ptr := c.GetDisplayName()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without display name", *idPtr)
}
return nil
}
// checkRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func checkRequiredValues(c graph.Container) error {
if err := checkIDAndName(c); err != nil {
return err
}
ptr := c.GetParentFolderId()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without parent ID", *c.GetId())
}
return nil
}
//======================================
// cachedContainer Implementations
//======================
var _ graph.CachedContainer = &cacheFolder{}
type cacheFolder struct {
graph.Container
p *path.Builder
}
//=========================================
// Required Functions to satisfy interfaces
//=====================================
func (cf cacheFolder) Path() *path.Builder {
return cf.p
}
func (cf *cacheFolder) SetPath(newPath *path.Builder) {
cf.p = newPath
}
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the container interface.
// Calendars do not have a parentFolderID. Therefore,

23
src/internal/connector/exchange/contact_folder_cache.go
View File

@ -12,10 +12,10 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ graph.ContainerResolver = &contactFolderCache{}
var _ graph.ContainerPopulater = &contactFolderCache{}
type contactFolderCache struct {
*containerResolver
*graph.ContainerCache
gs graph.Service
userID string
}
@ -44,12 +44,11 @@ func (cfc *contactFolderCache) populateContactRoot(
"fetching root contact folder: "+support.ConnectorStackErrorTrace(err))
}
temp := cacheFolder{
Container: f,
p: path.Builder{}.Append(baseContainerPath...),
}
temp := graph.NewCacheFolder(
f,
path.Builder{}.Append(baseContainerPath...))
if err := cfc.addFolder(temp); err != nil {
if err := cfc.ContainerCache.AddFolder(temp); err != nil {
return errors.Wrap(err, "adding cache root")
}
@ -105,11 +104,11 @@ func (cfc *contactFolderCache) Populate(
}
for _, entry := range containers {
temp := cacheFolder{
temp := graph.CacheFolder{
Container: entry,
}
err = cfc.addFolder(temp)
err = cfc.ContainerCache.AddFolder(temp)
if err != nil {
errs = support.WrapAndAppend(
"cache build in cfc.Populate",
@ -118,7 +117,7 @@ func (cfc *contactFolderCache) Populate(
}
}
if err := cfc.populatePaths(ctx); err != nil {
if err := cfc.ContainerCache.PopulatePaths(ctx); err != nil {
errs = support.WrapAndAppend(
"contacts resolver",
err,
@ -138,8 +137,8 @@ func (cfc *contactFolderCache) init(
return errors.New("m365 folderID required for base folder")
}
if cfc.containerResolver == nil {
cfc.containerResolver = newContainerResolver()
if cfc.ContainerCache == nil {
cfc.ContainerCache = graph.NewContainerCache()
}
return cfc.populateContactRoot(ctx, baseNode, baseContainerPath)

34
src/internal/connector/exchange/event_calendar_cache.go
View File

@ -12,10 +12,10 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ graph.ContainerResolver = &eventCalendarCache{}
var _ graph.ContainerPopulater = &eventCalendarCache{}
type eventCalendarCache struct {
*containerResolver
*graph.ContainerCache
gs graph.Service
userID string
}
@ -28,8 +28,8 @@ func (ecc *eventCalendarCache) Populate(
baseID string,
baseContainerPath ...string,
) error {
if ecc.containerResolver == nil {
ecc.containerResolver = newContainerResolver()
if ecc.ContainerCache == nil {
ecc.ContainerCache = graph.NewContainerCache()
}
options, err := optionsForCalendars([]string{"name"})
@ -76,7 +76,7 @@ func (ecc *eventCalendarCache) Populate(
}
for _, container := range directories {
if err := checkIDAndName(container); err != nil {
if err := graph.CheckIDAndName(container); err != nil {
iterateErr = support.WrapAndAppend(
"adding folder to cache",
err,
@ -86,12 +86,11 @@ func (ecc *eventCalendarCache) Populate(
continue
}
temp := cacheFolder{
Container: container,
p: path.Builder{}.Append(*container.GetDisplayName()),
}
temp := graph.NewCacheFolder(
container,
path.Builder{}.Append(*container.GetDisplayName()))
if err := ecc.addFolder(temp); err != nil {
if err := ecc.ContainerCache.AddFolder(temp); err != nil {
iterateErr = support.WrapAndAppend(
"failure adding "+*container.GetDisplayName(),
err,
@ -104,23 +103,22 @@ func (ecc *eventCalendarCache) Populate(
// AddToCache adds container to map in field 'cache'
// @returns error iff the required values are not accessible.
func (ecc *eventCalendarCache) AddToCache(ctx context.Context, f graph.Container) error {
if err := checkIDAndName(f); err != nil {
func (ecc *eventCalendarCache) AddToCache(ctx context.Context, c graph.Container) error {
if err := graph.CheckIDAndName(c); err != nil {
return errors.Wrap(err, "adding cache folder")
}
temp := cacheFolder{
Container: f,
p: path.Builder{}.Append(*f.GetDisplayName()),
}
temp := graph.NewCacheFolder(
c,
path.Builder{}.Append(*c.GetDisplayName()))
if err := ecc.addFolder(temp); err != nil {
if err := ecc.ContainerCache.AddFolder(temp); err != nil {
return errors.Wrap(err, "adding cache folder")
}
// Populate the path for this entry so calls to PathInCache succeed no matter
// when they're made.
_, err := ecc.IDToPath(ctx, *f.GetId())
_, err := ecc.ContainerCache.IDToPath(ctx, *c.GetId())
if err != nil {
return errors.Wrap(err, "adding cache entry")
}

2
src/internal/connector/exchange/exchange_service_test.go
View File

@ -494,7 +494,7 @@ func (suite *ExchangeServiceSuite) TestGetContainerIDFromCache() {
t = suite.T()
user = tester.M365UserID(t)
connector = loadService(t)
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
directoryCaches = make(map[path.CategoryType]graph.ContainerPopulater)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string

6
src/internal/connector/exchange/folder_resolver_test.go
View File

@ -50,14 +50,14 @@ func (suite *CacheResolverSuite) TestPopulate() {
ctx, flush := tester.NewContext()
defer flush()
eventFunc := func(t *testing.T) graph.ContainerResolver {
eventFunc := func(t *testing.T) graph.ContainerPopulater {
return &eventCalendarCache{
userID: tester.M365UserID(t),
gs: suite.gs,
}
}
contactFunc := func(t *testing.T) graph.ContainerResolver {
contactFunc := func(t *testing.T) graph.ContainerPopulater {
return &contactFolderCache{
userID: tester.M365UserID(t),
gs: suite.gs,
@ -66,7 +66,7 @@ func (suite *CacheResolverSuite) TestPopulate() {
tests := []struct {
name, folderName, root, basePath string
resolverFunc func(t *testing.T) graph.ContainerResolver
resolverFunc func(t *testing.T) graph.ContainerPopulater
canFind assert.BoolAssertionFunc
}{
{

23
src/internal/connector/exchange/mail_folder_cache.go
View File

@ -12,13 +12,13 @@ import (
"github.com/alcionai/corso/src/pkg/path"
)
var _ graph.ContainerResolver = &mailFolderCache{}
var _ graph.ContainerPopulater = &mailFolderCache{}
// mailFolderCache struct used to improve lookup of directories within exchange.Mail
// cache map of cachedContainers where the key = M365ID
// nameLookup map: Key: DisplayName Value: ID
type mailFolderCache struct {
*containerResolver
*graph.ContainerCache
gs graph.Service
userID string
}
@ -51,12 +51,11 @@ func (mc *mailFolderCache) populateMailRoot(
return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err))
}
temp := cacheFolder{
Container: f,
p: path.Builder{}.Append(baseContainerPath...),
}
temp := graph.NewCacheFolder(
f,
path.Builder{}.Append(baseContainerPath...))
if err := mc.addFolder(temp); err != nil {
if err := mc.ContainerCache.AddFolder(temp); err != nil {
return errors.Wrap(err, "initializing mail resolver")
}
@ -95,14 +94,14 @@ func (mc *mailFolderCache) Populate(
}
for _, f := range resp.GetValue() {
temp := cacheFolder{
temp := graph.CacheFolder{
Container: f,
}
// Use addFolder instead of AddToCache to be conservative about path
// population. The fetch order of the folders could cause failures while
// trying to resolve paths, so put it off until we've gotten all folders.
if err := mc.addFolder(temp); err != nil {
if err := mc.ContainerCache.AddFolder(temp); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "delta fetch"))
continue
}
@ -119,7 +118,7 @@ func (mc *mailFolderCache) Populate(
query = msfolderdelta.NewDeltaRequestBuilder(link, mc.gs.Adapter())
}
if err := mc.populatePaths(ctx); err != nil {
if err := mc.ContainerCache.PopulatePaths(ctx); err != nil {
errs = multierror.Append(errs, errors.Wrap(err, "mail resolver"))
}
@ -138,8 +137,8 @@ func (mc *mailFolderCache) init(
return errors.New("m365 folder ID required for base folder")
}
if mc.containerResolver == nil {
mc.containerResolver = newContainerResolver()
if mc.ContainerCache == nil {
mc.ContainerCache = graph.NewContainerCache()
}
return mc.populateMailRoot(ctx, baseNode, baseContainerPath)

3
src/internal/connector/exchange/service_functions.go
View File

@ -26,6 +26,7 @@ type exchangeService struct {
///------------------------------------------------------------
// Functions to comply with graph.Service Interface
//-------------------------------------------------------
func (es *exchangeService) Client() *msgraphsdk.GraphServiceClient {
return &es.client
}
@ -232,7 +233,7 @@ func PopulateExchangeContainerResolver(
qp graph.QueryParams,
) (graph.ContainerResolver, error) {
var (
res graph.ContainerResolver
res graph.ContainerPopulater
cacheRoot string
service, err = createService(qp.Credentials, qp.FailFast)
)

12
src/internal/connector/exchange/service_restore.go
View File

@ -292,7 +292,7 @@ func RestoreExchangeDataCollections(
) (*support.ConnectorOperationStatus, error) {
var (
// map of caches... but not yet...
directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerResolver)
directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerPopulater)
metrics support.CollectionMetrics
errs error
// TODO policy to be updated from external source after completion of refactoring
@ -308,7 +308,7 @@ func RestoreExchangeDataCollections(
userCaches := directoryCaches[userID]
if userCaches == nil {
directoryCaches[userID] = make(map[path.CategoryType]graph.ContainerResolver)
directoryCaches[userID] = make(map[path.CategoryType]graph.ContainerPopulater)
userCaches = directoryCaches[userID]
}
@ -432,7 +432,7 @@ func GetContainerIDFromCache(
gs graph.Service,
directory path.Path,
destination string,
caches map[path.CategoryType]graph.ContainerResolver,
caches map[path.CategoryType]graph.ContainerPopulater,
) (string, error) {
var (
newCache = false
@ -512,7 +512,7 @@ func GetContainerIDFromCache(
func establishMailRestoreLocation(
ctx context.Context,
folders []string,
mfc graph.ContainerResolver,
mfc graph.ContainerPopulater,
user string,
service graph.Service,
isNewCache bool,
@ -569,7 +569,7 @@ func establishMailRestoreLocation(
func establishContactsRestoreLocation(
ctx context.Context,
folders []string,
cfc graph.ContainerResolver,
cfc graph.ContainerPopulater,
user string,
gs graph.Service,
isNewCache bool,
@ -602,7 +602,7 @@ func establishContactsRestoreLocation(
func establishEventsRestoreLocation(
ctx context.Context,
folders []string,
ecc graph.ContainerResolver, // eventCalendarCache
ecc graph.ContainerPopulater, // eventCalendarCache
user string,
gs graph.Service,
isNewCache bool,

63
src/internal/connector/graph/cache_container.go
View File

@ -1,7 +1,6 @@
package graph
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/pkg/path"
@ -16,9 +15,9 @@ type CachedContainer interface {
SetPath(*path.Builder)
}
// checkRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func CheckRequiredValues(c Container) error {
// CheckIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func CheckIDAndName(c Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
@ -29,9 +28,19 @@ func CheckRequiredValues(c Container) error {
return errors.Errorf("folder %s without display name", *idPtr)
}
ptr = c.GetParentFolderId()
return nil
}
// CheckRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func CheckRequiredValues(c Container) error {
if err := CheckIDAndName(c); err != nil {
return err
}
ptr := c.GetParentFolderId()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without parent ID", *idPtr)
return errors.Errorf("folder %s without parent ID", *c.GetId())
}
return nil
@ -58,10 +67,6 @@ func NewCacheFolder(c Container, pb *path.Builder) CacheFolder {
return cf
}
//=========================================
// Required Functions to satisfy interfaces
//=========================================
func (cf CacheFolder) Path() *path.Builder {
return cf.p
}
@ -69,41 +74,3 @@ func (cf CacheFolder) Path() *path.Builder {
func (cf *CacheFolder) SetPath(newPath *path.Builder) {
cf.p = newPath
}
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the container interface.
// Calendars do not have the 2 of the
type CalendarDisplayable struct {
models.Calendarable
parentID string
}
// GetDisplayName returns the *string of the calendar name
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// GetParentFolderId returns the default calendar name address
// EventCalendars have a flat hierarchy and Calendars are rooted
// at the default
//nolint:revive
func (c CalendarDisplayable) GetParentFolderId() *string {
return &c.parentID
}
// CreateCalendarDisplayable helper function to create the
// calendarDisplayable during msgraph-sdk-go iterative process
// @param entry is the input supplied by pageIterator.Iterate()
// @param parentID of Calendar sets. Only populate when used with
// EventCalendarCache
func CreateCalendarDisplayable(entry any, parentID string) *CalendarDisplayable {
calendar, ok := entry.(models.Calendarable)
if !ok {
return nil
}
return &CalendarDisplayable{
Calendarable: calendar,
parentID: parentID,
}
}

52
src/internal/connector/exchange/container_resolver.go → src/internal/connector/graph/cache_resolver.go
View File

@ -1,4 +1,4 @@
package exchange
package graph
import (
"context"
@ -6,21 +6,28 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/path"
)
func newContainerResolver() *containerResolver {
return &containerResolver{
cache: map[string]graph.CachedContainer{},
var _ ContainerResolver = &ContainerCache{}
type populatorFunc func(
ctx context.Context,
baseID string,
baseContainerPath ...string,
)
type ContainerCache struct {
cache map[string]CachedContainer
}
func NewContainerCache() *ContainerCache {
return &ContainerCache{
cache: map[string]CachedContainer{},
}
}
type containerResolver struct {
cache map[string]graph.CachedContainer
}
func (cr *containerResolver) IDToPath(
func (cr *ContainerCache) IDToPath(
ctx context.Context,
folderID string,
) (*path.Builder, error) {
@ -48,7 +55,7 @@ func (cr *containerResolver) IDToPath(
// PathInCache utility function to return m365ID of folder if the pathString
// matches the path of a container within the cache. A boolean function
// accompanies the call to indicate whether the lookup was successful.
func (cr *containerResolver) PathInCache(pathString string) (string, bool) {
func (cr *ContainerCache) PathInCache(pathString string) (string, bool) {
if len(pathString) == 0 || cr == nil {
return "", false
}
@ -66,17 +73,17 @@ func (cr *containerResolver) PathInCache(pathString string) (string, bool) {
return "", false
}
// addFolder adds a folder to the cache with the given ID. If the item is
// AddFolder adds a folder to the cache with the given ID. If the item is
// already in the cache does nothing. The path for the item is not modified.
func (cr *containerResolver) addFolder(cf cacheFolder) error {
func (cr *ContainerCache) AddFolder(cf CacheFolder) error {
// Only require a non-nil non-empty parent if the path isn't already
// populated.
if cf.p != nil {
if err := checkIDAndName(cf.Container); err != nil {
if err := CheckIDAndName(cf.Container); err != nil {
return errors.Wrap(err, "adding item to cache")
}
} else {
if err := checkRequiredValues(cf.Container); err != nil {
if err := CheckRequiredValues(cf.Container); err != nil {
return errors.Wrap(err, "adding item to cache")
}
}
@ -90,8 +97,9 @@ func (cr *containerResolver) addFolder(cf cacheFolder) error {
return nil
}
func (cr *containerResolver) Items() []graph.CachedContainer {
res := make([]graph.CachedContainer, 0, len(cr.cache))
// Items returns the list of Containers in the cache.
func (cr *ContainerCache) Items() []CachedContainer {
res := make([]CachedContainer, 0, len(cr.cache))
for _, c := range cr.cache {
res = append(res, c)
@ -102,12 +110,12 @@ func (cr *containerResolver) Items() []graph.CachedContainer {
// AddToCache adds container to map in field 'cache'
// @returns error iff the required values are not accessible.
func (cr *containerResolver) AddToCache(ctx context.Context, f graph.Container) error {
temp := cacheFolder{
func (cr *ContainerCache) AddToCache(ctx context.Context, f Container) error {
temp := CacheFolder{
Container: f,
}
if err := cr.addFolder(temp); err != nil {
if err := cr.AddFolder(temp); err != nil {
return errors.Wrap(err, "adding cache folder")
}
@ -121,10 +129,10 @@ func (cr *containerResolver) AddToCache(ctx context.Context, f graph.Container)
return nil
}
func (cr *containerResolver) populatePaths(ctx context.Context) error {
// PopulatePaths ensures that all items in the cache can construct valid paths.
func (cr *ContainerCache) PopulatePaths(ctx context.Context) error {
var errs *multierror.Error
// Populate all folder paths.
for _, f := range cr.Items() {
_, err := cr.IDToPath(ctx, *f.GetId())
if err != nil {

18
src/internal/connector/graph/service.go
View File

@ -60,11 +60,6 @@ type ContainerResolver interface {
// to that container. The path has a similar format to paths on the local
// file system.
IDToPath(ctx context.Context, m365ID string) (*path.Builder, error)
// Populate performs initialization steps for the resolver
// @param ctx is necessary param for Graph API tracing
// @param baseFolderID represents the M365ID base that the resolver will
// conclude its search. Default input is "".
Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
// PathInCache performs a look up of a path reprensentation
// and returns the m365ID of directory iff the pathString
@ -77,3 +72,16 @@ type ContainerResolver interface {
// Items returns the containers in the cache.
Items() []CachedContainer
}
// ContainerPopulater houses functions for populating and retrieving info
// about containers from remote APIs (i.e. resolve folder paths with Graph
// API). Populaters may cache information about containers.
type ContainerPopulater interface {
ContainerResolver
// Populate performs initialization steps for the populater
// @param ctx is necessary param for Graph API tracing
// @param baseFolderID represents the M365ID base that the
// populater will conclude its search. Default input is "".
Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
}