GC: Restore: Directory Hierarchy Feature for Exchange (#1053)

## Description
Feature to add the folder hierarchy for folders when restored.
This required an overhaul of the `graph.ContainerResolver` interfaces:
- MailFolderCache
- ContactFolderCache
- ~EventFolderCache (placed in a separate PR)~ https://github.com/alcionai/corso/pull/1101

Restore Pipeline changed to separate the caching / container creation process from the rest of the restore pipeline.


## Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature

## Issue(s)

* closes #1046 
* #1004 
* closes #1091
* closes #1098 
* closes #1097 
* closes #1096
* closes #1095
* closes #991
* closes #895
* closes #798 

## Test Plan
- [x]  Unit test
This commit is contained in:
Danny 2022-10-14 09:14:33 -04:00 committed by GitHub
parent 69a6fd1593
commit 4a29d22216
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 1343 additions and 381 deletions

View File

@ -0,0 +1,101 @@
package exchange
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/pkg/path"
)
// checkIDAndName is a helper function to ensure that
// the ID and name pointers are set prior to being called.
func checkIDAndName(c graph.Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
}
ptr := c.GetDisplayName()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without display name", *idPtr)
}
return nil
}
// checkRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func checkRequiredValues(c graph.Container) error {
if err := checkIDAndName(c); err != nil {
return err
}
ptr := c.GetParentFolderId()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without parent ID", *c.GetId())
}
return nil
}
//======================================
// cachedContainer Implementations
//======================
var _ graph.CachedContainer = &cacheFolder{}
type cacheFolder struct {
graph.Container
p *path.Builder
}
//=========================================
// Required Functions to satisfy interfaces
//=====================================
func (cf cacheFolder) Path() *path.Builder {
return cf.p
}
func (cf *cacheFolder) SetPath(newPath *path.Builder) {
cf.p = newPath
}
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the container interface.
// Calendars do not have a parentFolderID. Therefore,
// the call will always return nil
type CalendarDisplayable struct {
models.Calendarable
}
// GetDisplayName returns the *string of the models.Calendable
// variant: calendar.GetName()
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// GetParentFolderId returns the default calendar name address
// EventCalendars have a flat hierarchy and Calendars are rooted
// at the default
//nolint:revive
func (c CalendarDisplayable) GetParentFolderId() *string {
return nil
}
// CreateCalendarDisplayable helper function to create the
// calendarDisplayable during msgraph-sdk-go iterative process
// @param entry is the input supplied by pageIterator.Iterate()
// @param parentID of Calendar sets. Only populate when used with
// EventCalendarCache
func CreateCalendarDisplayable(entry any) *CalendarDisplayable {
calendar, ok := entry.(models.Calendarable)
if !ok {
return nil
}
return &CalendarDisplayable{
Calendarable: calendar,
}
}

View File

@ -1,28 +0,0 @@
package exchange
import (
"github.com/microsoftgraph/msgraph-sdk-go/models"
)
// CalendarDisplayable is a transformative struct that aligns
// models.Calendarable interface with the Displayable interface.
type CalendarDisplayable struct {
models.Calendarable
}
// GetDisplayName returns the *string of the calendar name
func (c CalendarDisplayable) GetDisplayName() *string {
return c.GetName()
}
// CreateCalendarDisplayable helper function to create the
// calendarDisplayable during msgraph-sdk-go iterative process
// @param entry is the input supplied by pageIterator.Iterate()
func CreateCalendarDisplayable(entry any) *CalendarDisplayable {
calendar, ok := entry.(models.Calendarable)
if !ok {
return nil
}
return &CalendarDisplayable{calendar}
}

View File

@ -0,0 +1,217 @@
package exchange
import (
"context"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
var _ graph.ContainerResolver = &contactFolderCache{}
type contactFolderCache struct {
cache map[string]graph.CachedContainer
gs graph.Service
userID, rootID string
}
func (cfc *contactFolderCache) populateContactRoot(
ctx context.Context,
directoryID string,
baseContainerPath []string,
) error {
wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForContactFolderByID(wantedOpts)
if err != nil {
return errors.Wrapf(err, "getting options for contact folder cache: %v", wantedOpts)
}
f, err := cfc.
gs.
Client().
UsersById(cfc.userID).
ContactFoldersById(directoryID).
Get(ctx, opts)
if err != nil {
return errors.Wrapf(err, "fetching root contact folder")
}
idPtr := f.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("root folder has no ID")
}
temp := cacheFolder{
Container: f,
p: path.Builder{}.Append(baseContainerPath...),
}
cfc.cache[*idPtr] = &temp
cfc.rootID = *idPtr
return nil
}
// Populate is utility function for placing cache container
// objects into the Contact Folder Cache
// Function does NOT use Delta Queries as it is not supported
// as of (Oct-07-2022)
func (cfc *contactFolderCache) Populate(
ctx context.Context,
baseID string,
baseContainerPather ...string,
) error {
if err := cfc.init(ctx, baseID, baseContainerPather); err != nil {
return err
}
var (
containers = make(map[string]graph.Container)
errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
}
)
query, err := cfc.
gs.Client().
UsersById(cfc.userID).
ContactFoldersById(cfc.rootID).
ChildFolders().
Get(ctx, nil)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
iter, err := msgraphgocore.NewPageIterator(query, cfc.gs.Adapter(),
models.CreateContactFolderCollectionResponseFromDiscriminatorValue)
if err != nil {
return err
}
cb := IterativeCollectContactContainers(containers,
"",
errUpdater)
if err := iter.Iterate(ctx, cb); err != nil {
return err
}
if errs != nil {
return errs
}
for _, entry := range containers {
err = cfc.AddToCache(ctx, entry)
if err != nil {
errs = support.WrapAndAppend(
"cache build in cfc.Populate",
err,
errs)
}
}
return errs
}
func (cfc *contactFolderCache) init(
ctx context.Context,
baseNode string,
baseContainerPath []string,
) error {
if len(baseNode) == 0 {
return errors.New("m365 folderID required for base folder")
}
if cfc.cache == nil {
cfc.cache = map[string]graph.CachedContainer{}
}
return cfc.populateContactRoot(ctx, baseNode, baseContainerPath)
}
func (cfc *contactFolderCache) IDToPath(
ctx context.Context,
folderID string,
) (*path.Builder, error) {
c, ok := cfc.cache[folderID]
if !ok {
return nil, errors.Errorf("folder %s not cached", folderID)
}
p := c.Path()
if p != nil {
return p, nil
}
parentPath, err := cfc.IDToPath(ctx, *c.GetParentFolderId())
if err != nil {
return nil, errors.Wrap(err, "retrieving parent folder")
}
fullPath := parentPath.Append(*c.GetDisplayName())
c.SetPath(fullPath)
return fullPath, nil
}
// PathInCache utility function to return m365ID of folder if the pathString
// matches the path of a container within the cache. A boolean function
// accompanies the call to indicate whether the lookup was successful.
func (cfc *contactFolderCache) PathInCache(pathString string) (string, bool) {
if len(pathString) == 0 || cfc.cache == nil {
return "", false
}
for _, contain := range cfc.cache {
if contain.Path() == nil {
continue
}
if contain.Path().String() == pathString {
return *contain.GetId(), true
}
}
return "", false
}
// AddToCache places container into internal cache field.
// @returns error iff input does not possess accessible values.
func (cfc *contactFolderCache) AddToCache(ctx context.Context, f graph.Container) error {
if err := checkRequiredValues(f); err != nil {
return err
}
if _, ok := cfc.cache[*f.GetId()]; ok {
return nil
}
cfc.cache[*f.GetId()] = &cacheFolder{
Container: f,
}
// Populate the path for this entry so calls to PathInCache succeed no matter
// when they're made.
_, err := cfc.IDToPath(ctx, *f.GetId())
if err != nil {
return errors.Wrap(err, "adding cache entry")
}
return nil
}
func (cfc *contactFolderCache) Items() []graph.CachedContainer {
res := make([]graph.CachedContainer, 0, len(cfc.cache))
for _, c := range cfc.cache {
res = append(res, c)
}
return res
}

View File

@ -0,0 +1,87 @@
package exchange
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
)
type ContactFolderCacheIntegrationSuite struct {
suite.Suite
gs graph.Service
}
func (suite *ContactFolderCacheIntegrationSuite) SetupSuite() {
t := suite.T()
_, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...)
require.NoError(t, err)
a := tester.NewM365Account(t)
require.NoError(t, err)
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365, false)
require.NoError(t, err)
suite.gs = service
}
func TestContactFolderCacheIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(ContactFolderCacheIntegrationSuite))
}
func (suite *ContactFolderCacheIntegrationSuite) TestPopulate() {
ctx, flush := tester.NewContext()
defer flush()
cfc := contactFolderCache{
userID: tester.M365UserID(suite.T()),
gs: suite.gs,
}
tests := []struct {
name string
folderName string
basePath string
canFind assert.BoolAssertionFunc
}{
{
name: "Default Contact Cache",
folderName: DefaultContactFolder,
basePath: DefaultContactFolder,
canFind: assert.True,
},
{
name: "Default Contact Hidden",
folderName: DefaultContactFolder,
canFind: assert.False,
},
{
name: "Name Not in Cache",
folderName: "testFooBarWhoBar",
canFind: assert.False,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
require.NoError(t, cfc.Populate(ctx, DefaultContactFolder, test.basePath))
_, isFound := cfc.PathInCache(test.folderName)
test.canFind(t, isFound)
})
}
}

View File

@ -0,0 +1,151 @@
package exchange
import (
"context"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path"
)
var _ graph.ContainerResolver = &eventCalendarCache{}
type eventCalendarCache struct {
cache map[string]graph.CachedContainer
gs graph.Service
userID, rootID string
}
// Populate utility function for populating eventCalendarCache.
// Executes 1 additional Graph Query
// @param baseID: ignored. Present to conform to interface
func (ecc *eventCalendarCache) Populate(
ctx context.Context,
baseID string,
baseContainerPath ...string,
) error {
if ecc.cache == nil {
ecc.cache = map[string]graph.CachedContainer{}
}
options, err := optionsForCalendars([]string{"name"})
if err != nil {
return err
}
directories := make(map[string]graph.Container)
errUpdater := func(s string, e error) {
err = support.WrapAndAppend(s, e, err)
}
query, err := ecc.gs.Client().UsersById(ecc.userID).Calendars().Get(ctx, options)
if err != nil {
return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
iter, err := msgraphgocore.NewPageIterator(
query,
ecc.gs.Adapter(),
models.CreateCalendarCollectionResponseFromDiscriminatorValue,
)
if err != nil {
return err
}
cb := IterativeCollectCalendarContainers(
directories,
"",
errUpdater,
)
iterateErr := iter.Iterate(ctx, cb)
if iterateErr != nil {
return iterateErr
}
if err != nil {
return err
}
for _, containerr := range directories {
if err := ecc.AddToCache(ctx, containerr); err != nil {
iterateErr = support.WrapAndAppend(
"failure adding "+*containerr.GetDisplayName(),
err,
iterateErr)
}
}
return iterateErr
}
func (ecc *eventCalendarCache) IDToPath(
ctx context.Context,
calendarID string,
) (*path.Builder, error) {
c, ok := ecc.cache[calendarID]
if !ok {
return nil, errors.Errorf("calendar %s not cached", calendarID)
}
p := c.Path()
if p == nil {
// Shouldn't happen
p := path.Builder{}.Append(*c.GetDisplayName())
c.SetPath(p)
}
return p, nil
}
// AddToCache places container into internal cache field. For EventCalendars
// this means that the object has to be transformed prior to calling
// this function.
func (ecc *eventCalendarCache) AddToCache(ctx context.Context, f graph.Container) error {
if err := checkIDAndName(f); err != nil {
return err
}
if _, ok := ecc.cache[*f.GetId()]; ok {
return nil
}
ecc.cache[*f.GetId()] = &cacheFolder{
Container: f,
p: path.Builder{}.Append(*f.GetDisplayName()),
}
return nil
}
func (ecc *eventCalendarCache) PathInCache(pathString string) (string, bool) {
if len(pathString) == 0 || ecc.cache == nil {
return "", false
}
for _, containerr := range ecc.cache {
if containerr.Path() == nil {
continue
}
if containerr.Path().String() == pathString {
return *containerr.GetId(), true
}
}
return "", false
}
func (ecc *eventCalendarCache) Items() []graph.CachedContainer {
res := make([]graph.CachedContainer, 0, len(ecc.cache))
for _, c := range ecc.cache {
res = append(res, c)
}
return res
}

View File

@ -0,0 +1,88 @@
package exchange
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/tester"
)
type EventCalendarCacheSuite struct {
suite.Suite
gs graph.Service
}
func TestEventCalendarCacheIntegrationSuite(t *testing.T) {
if err := tester.RunOnAny(
tester.CorsoCITests,
tester.CorsoGraphConnectorTests,
); err != nil {
t.Skip(err)
}
suite.Run(t, new(EventCalendarCacheSuite))
}
func (suite *EventCalendarCacheSuite) SetupSuite() {
t := suite.T()
_, err := tester.GetRequiredEnvVars(tester.M365AcctCredEnvs...)
require.NoError(t, err)
a := tester.NewM365Account(t)
require.NoError(t, err)
m365, err := a.M365Config()
require.NoError(t, err)
service, err := createService(m365, false)
require.NoError(t, err)
suite.gs = service
}
func (suite *EventCalendarCacheSuite) TestPopulate() {
ctx, flush := tester.NewContext()
defer flush()
ecc := eventCalendarCache{
userID: tester.M365UserID(suite.T()),
gs: suite.gs,
}
tests := []struct {
name string
folderName string
basePath string
canFind assert.BoolAssertionFunc
}{
{
name: "Default Event Cache",
folderName: DefaultCalendar,
basePath: DefaultCalendar,
canFind: assert.True,
},
{
name: "Default Event Folder Hidden",
folderName: DefaultContactFolder,
canFind: assert.False,
},
{
name: "Name Not in Cache",
folderName: "testFooBarWhoBar",
canFind: assert.False,
},
}
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
require.NoError(t, ecc.Populate(ctx, DefaultCalendar, test.basePath))
_, isFound := ecc.PathInCache(test.folderName)
test.canFind(t, isFound)
assert.Greater(t, len(ecc.cache), 0)
})
}
}

View File

@ -274,10 +274,6 @@ func (suite *ExchangeServiceSuite) TestGraphQueryFunctions() {
name string name string
function GraphQuery function GraphQuery
}{ }{
{
name: "GraphQuery: Get All Messages For User",
function: GetAllMessagesForUser,
},
{ {
name: "GraphQuery: Get All Contacts For User", name: "GraphQuery: Get All Contacts For User",
function: GetAllContactsForUser, function: GetAllContactsForUser,
@ -450,69 +446,6 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() {
assert.NotNil(t, info, "event item info") assert.NotNil(t, info, "event item info")
} }
// TestGetRestoreContainer checks the ability to Create a "container" for the
// GraphConnector's Restore Workflow based on OptionIdentifier.
func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
ctx, flush := tester.NewContext()
defer flush()
dest := tester.DefaultTestRestoreDestination()
tests := []struct {
name string
option path.CategoryType
checkError assert.ErrorAssertionFunc
cleanupFunc func(context.Context, graph.Service, string, string) error
}{
{
name: "Establish User Restore Folder",
option: path.CategoryType(-1),
checkError: assert.Error,
cleanupFunc: nil,
},
// TODO: #884 - reinstate when able to specify root folder by name
// {
// name: "Establish Event Restore Location",
// option: path.EventsCategory,
// checkError: assert.NoError,
// cleanupFunc: DeleteCalendar,
// },
{
name: "Establish Restore Folder for Unknown",
option: path.UnknownCategory,
checkError: assert.Error,
cleanupFunc: nil,
},
{
name: "Establish Restore folder for Mail",
option: path.EmailCategory,
checkError: assert.NoError,
cleanupFunc: DeleteMailFolder,
},
// TODO: #884 - reinstate when able to specify root folder by name
// {
// name: "Establish Restore folder for Contacts",
// option: path.ContactsCategory,
// checkError: assert.NoError,
// cleanupFunc: DeleteContactFolder,
// },
}
userID := tester.M365UserID(suite.T())
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
containerID, err := GetRestoreContainer(ctx, suite.es, userID, test.option, dest.ContainerName)
require.True(t, test.checkError(t, err, support.ConnectorStackErrorTrace(err)))
if test.cleanupFunc != nil {
err = test.cleanupFunc(ctx, suite.es, userID, containerID)
assert.NoError(t, err)
}
})
}
}
// TestRestoreExchangeObject verifies path.Category usage for restored objects // TestRestoreExchangeObject verifies path.Category usage for restored objects
func (suite *ExchangeServiceSuite) TestRestoreExchangeObject() { func (suite *ExchangeServiceSuite) TestRestoreExchangeObject() {
ctx, flush := tester.NewContext() ctx, flush := tester.NewContext()
@ -630,3 +563,137 @@ func (suite *ExchangeServiceSuite) TestRestoreExchangeObject() {
}) })
} }
} }
// Testing to ensure that cache system works for in multiple different environments
func (suite *ExchangeServiceSuite) TestGetContainerIDFromCache() {
ctx, flush := tester.NewContext()
defer flush()
var (
t = suite.T()
user = tester.M365UserID(t)
connector = loadService(t)
directoryCaches = make(map[path.CategoryType]graph.ContainerResolver)
folderName = tester.DefaultTestRestoreDestination().ContainerName
tests = []struct {
name string
pathFunc1 func() path.Path
pathFunc2 func() path.Path
category path.CategoryType
}{
{
name: "Mail Cache Test",
category: path.EmailCategory,
pathFunc1: func() path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Croix").ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
return pth
},
pathFunc2: func() path.Path {
pth, err := path.Builder{}.Append("Griffindor").
Append("Felicius").ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.EmailCategory,
false,
)
require.NoError(suite.T(), err)
return pth
},
},
{
name: "Contact Cache Test",
category: path.ContactsCategory,
pathFunc1: func() path.Path {
aPath, err := path.Builder{}.Append("HufflePuff").
ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(suite.T(), err)
return aPath
},
pathFunc2: func() path.Path {
aPath, err := path.Builder{}.Append("Ravenclaw").
ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.ContactsCategory,
false,
)
require.NoError(suite.T(), err)
return aPath
},
},
{
name: "Event Cache Test",
category: path.EventsCategory,
pathFunc1: func() path.Path {
aPath, err := path.Builder{}.Append("Durmstrang").
ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.EventsCategory,
false,
)
require.NoError(suite.T(), err)
return aPath
},
pathFunc2: func() path.Path {
aPath, err := path.Builder{}.Append("Beauxbatons").
ToDataLayerExchangePathForCategory(
suite.es.credentials.TenantID,
user,
path.EventsCategory,
false,
)
require.NoError(suite.T(), err)
return aPath
},
},
}
)
for _, test := range tests {
suite.T().Run(test.name, func(t *testing.T) {
folderID, err := GetContainerIDFromCache(
ctx,
connector,
test.pathFunc1(),
folderName,
directoryCaches,
)
require.NoError(t, err)
resolver := directoryCaches[test.category]
_, err = resolver.IDToPath(ctx, folderID)
assert.NoError(t, err)
secondID, err := GetContainerIDFromCache(
ctx,
connector,
test.pathFunc2(),
folderName,
directoryCaches,
)
require.NoError(t, err)
_, err = resolver.IDToPath(ctx, secondID)
require.NoError(t, err)
_, ok := resolver.PathInCache(folderName)
require.True(t, ok)
})
}
}

View File

@ -8,6 +8,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/path"
) )
@ -60,7 +61,11 @@ type mailFolderCache struct {
// Function should only be used directly when it is known that all // Function should only be used directly when it is known that all
// folder inquiries are going to a specific node. In all other cases // folder inquiries are going to a specific node. In all other cases
// @error iff the struct is not properly instantiated // @error iff the struct is not properly instantiated
func (mc *mailFolderCache) populateMailRoot(ctx context.Context, directoryID string) error { func (mc *mailFolderCache) populateMailRoot(
ctx context.Context,
directoryID string,
baseContainerPath []string,
) error {
wantedOpts := []string{"displayName", "parentFolderId"} wantedOpts := []string{"displayName", "parentFolderId"}
opts, err := optionsForMailFoldersItem(wantedOpts) opts, err := optionsForMailFoldersItem(wantedOpts)
@ -75,7 +80,7 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context, directoryID str
MailFoldersById(directoryID). MailFoldersById(directoryID).
Get(ctx, opts) Get(ctx, opts)
if err != nil { if err != nil {
return errors.Wrapf(err, "fetching root folder") return errors.Wrap(err, "fetching root folder"+support.ConnectorStackErrorTrace(err))
} }
// Root only needs the ID because we hide it's name for Mail. // Root only needs the ID because we hide it's name for Mail.
@ -85,9 +90,9 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context, directoryID str
return errors.New("root folder has no ID") return errors.New("root folder has no ID")
} }
temp := mailFolder{ temp := cacheFolder{
folder: f, Container: f,
p: &path.Builder{}, p: path.Builder{}.Append(baseContainerPath...),
} }
mc.cache[*idPtr] = &temp mc.cache[*idPtr] = &temp
mc.rootID = *idPtr mc.rootID = *idPtr
@ -95,38 +100,17 @@ func (mc *mailFolderCache) populateMailRoot(ctx context.Context, directoryID str
return nil return nil
} }
// checkRequiredValues is a helper function to ensure that
// all the pointers are set prior to being called.
func checkRequiredValues(c graph.Container) error {
idPtr := c.GetId()
if idPtr == nil || len(*idPtr) == 0 {
return errors.New("folder without ID")
}
ptr := c.GetDisplayName()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without display name", *idPtr)
}
ptr = c.GetParentFolderId()
if ptr == nil || len(*ptr) == 0 {
return errors.Errorf("folder %s without parent ID", *idPtr)
}
return nil
}
// Populate utility function for populating the mailFolderCache. // Populate utility function for populating the mailFolderCache.
// Number of Graph Queries: 1. // Number of Graph Queries: 1.
// @param baseID: M365ID of the base of the exchange.Mail.Folder // @param baseID: M365ID of the base of the exchange.Mail.Folder
// Use rootFolderAlias for input if baseID unknown // @param baseContainerPath: the set of folder elements that make up the path
func (mc *mailFolderCache) Populate(ctx context.Context, baseID string) error { // for the base container in the cache.
if len(baseID) == 0 { func (mc *mailFolderCache) Populate(
return errors.New("populate function requires: M365ID as input") ctx context.Context,
} baseID string,
baseContainerPath ...string,
err := mc.Init(ctx, baseID) ) error {
if err != nil { if err := mc.init(ctx, baseID, baseContainerPath); err != nil {
return err return err
} }
@ -144,7 +128,7 @@ func (mc *mailFolderCache) Populate(ctx context.Context, baseID string) error {
for { for {
resp, err := query.Get(ctx, nil) resp, err := query.Get(ctx, nil)
if err != nil { if err != nil {
return err return errors.Wrap(err, support.ConnectorStackErrorTrace(err))
} }
for _, f := range resp.GetValue() { for _, f := range resp.GetValue() {
@ -193,38 +177,70 @@ func (mc *mailFolderCache) IDToPath(
return fullPath, nil return fullPath, nil
} }
// Init ensures that the structure's fields are initialized. // init ensures that the structure's fields are initialized.
// Fields Initialized when cache == nil: // Fields Initialized when cache == nil:
// [mc.cache, mc.rootID] // [mc.cache, mc.rootID]
func (mc *mailFolderCache) Init(ctx context.Context, baseNode string) error { func (mc *mailFolderCache) init(
ctx context.Context,
baseNode string,
baseContainerPath []string,
) error {
if len(baseNode) == 0 {
return errors.New("m365 folder ID required for base folder")
}
if mc.cache == nil { if mc.cache == nil {
mc.cache = map[string]graph.CachedContainer{} mc.cache = map[string]graph.CachedContainer{}
} }
return mc.populateMailRoot(ctx, baseNode) return mc.populateMailRoot(ctx, baseNode, baseContainerPath)
} }
// AddToCache adds container to map in field 'cache'
// @returns error iff the required values are not accessible.
func (mc *mailFolderCache) AddToCache(ctx context.Context, f graph.Container) error { func (mc *mailFolderCache) AddToCache(ctx context.Context, f graph.Container) error {
if err := checkRequiredValues(f); err != nil { if err := checkRequiredValues(f); err != nil {
return errors.Wrap(err, "adding cache entry") return errors.Wrap(err, "object not added to cache")
} }
if _, ok := mc.cache[*f.GetId()]; ok { if _, ok := mc.cache[*f.GetId()]; ok {
return nil return nil
} }
mc.cache[*f.GetId()] = &mailFolder{ mc.cache[*f.GetId()] = &cacheFolder{
folder: f, Container: f,
} }
// Populate the path for this entry so calls to PathInCache succeed no matter
// when they're made.
_, err := mc.IDToPath(ctx, *f.GetId()) _, err := mc.IDToPath(ctx, *f.GetId())
if err != nil { if err != nil {
return errors.Wrap(err, "updating adding cache entry") return errors.Wrap(err, "adding cache entry")
} }
return nil return nil
} }
// PathInCache utility function to return m365ID of folder if the pathString
// matches the path of a container within the cache.
func (mc *mailFolderCache) PathInCache(pathString string) (string, bool) {
if len(pathString) == 0 || mc.cache == nil {
return "", false
}
for _, folder := range mc.cache {
if folder.Path() == nil {
continue
}
if folder.Path().String() == pathString {
return *folder.GetId(), true
}
}
return "", false
}
func (mc *mailFolderCache) Items() []graph.CachedContainer { func (mc *mailFolderCache) Items() []graph.CachedContainer {
res := make([]graph.CachedContainer, 0, len(mc.cache)) res := make([]graph.CachedContainer, 0, len(mc.cache))

View File

@ -337,6 +337,7 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
tests := []struct { tests := []struct {
name string name string
root string root string
path []string
}{ }{
{ {
name: "Default Root", name: "Default Root",
@ -346,6 +347,11 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
name: "Node Root", name: "Node Root",
root: topFolderID, root: topFolderID,
}, },
{
name: "Node Root Non-empty Path",
root: topFolderID,
path: []string{"some", "leading", "path"},
},
} }
userID := tester.M365UserID(suite.T()) userID := tester.M365UserID(suite.T())
@ -356,12 +362,16 @@ func (suite *MailFolderCacheIntegrationSuite) TestDeltaFetch() {
gs: suite.gs, gs: suite.gs,
} }
require.NoError(t, mfc.Populate(ctx, test.root)) require.NoError(t, mfc.Populate(ctx, test.root, test.path...))
p, err := mfc.IDToPath(ctx, testFolderID) p, err := mfc.IDToPath(ctx, testFolderID)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, expectedFolderPath, p.String()) expectedPath := stdpath.Join(append(test.path, expectedFolderPath)...)
assert.Equal(t, expectedPath, p.String())
identifier, ok := mfc.PathInCache(p.String())
assert.True(t, ok)
assert.NotEmpty(t, identifier)
}) })
} }
} }

View File

@ -6,6 +6,7 @@ import (
msuser "github.com/microsoftgraph/msgraph-sdk-go/users" msuser "github.com/microsoftgraph/msgraph-sdk-go/users"
mscalendars "github.com/microsoftgraph/msgraph-sdk-go/users/item/calendars" mscalendars "github.com/microsoftgraph/msgraph-sdk-go/users/item/calendars"
mscontactfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders" mscontactfolder "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders"
mscontactbyid "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders/item"
mscontactfolderitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders/item/contacts" mscontactfolderitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/contactfolders/item/contacts"
mscontacts "github.com/microsoftgraph/msgraph-sdk-go/users/item/contacts" mscontacts "github.com/microsoftgraph/msgraph-sdk-go/users/item/contacts"
msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events" msevents "github.com/microsoftgraph/msgraph-sdk-go/users/item/events"
@ -234,6 +235,25 @@ func optionsForContactFolders(moreOps []string) (
return options, nil return options, nil
} }
func optionsForContactFolderByID(moreOps []string) (
*mscontactbyid.ContactFolderItemRequestBuilderGetRequestConfiguration,
error,
) {
selecting, err := buildOptions(moreOps, folders)
if err != nil {
return nil, err
}
requestParameters := &mscontactbyid.ContactFolderItemRequestBuilderGetQueryParameters{
Select: selecting,
}
options := &mscontactbyid.ContactFolderItemRequestBuilderGetRequestConfiguration{
QueryParameters: requestParameters,
}
return options, nil
}
// optionsForMailFolders transforms the options into a more dynamic call for MailFolders. // optionsForMailFolders transforms the options into a more dynamic call for MailFolders.
// @param moreOps is a []string of options(e.g. "displayName", "isHidden") // @param moreOps is a []string of options(e.g. "displayName", "isHidden")
// @return is first call in MailFolders().GetWithRequestConfigurationAndResponseHandler(options, handler) // @return is first call in MailFolders().GetWithRequestConfigurationAndResponseHandler(options, handler)

View File

@ -185,10 +185,14 @@ func GetAllMailFolders(
// GetAllCalendars retrieves all event calendars for the specified user. // GetAllCalendars retrieves all event calendars for the specified user.
// If nameContains is populated, only returns calendars matching that property. // If nameContains is populated, only returns calendars matching that property.
// Returns a slice of {ID, DisplayName} tuples. // Returns a slice of {ID, DisplayName} tuples.
func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains string) ([]CalendarDisplayable, error) { func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains string) ([]graph.Container, error) {
var ( var (
cs = []CalendarDisplayable{} cs = make(map[string]graph.Container)
err error containers = make([]graph.Container, 0)
err, errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
}
) )
resp, err := GetAllCalendarNamesForUser(ctx, gs, user) resp, err := GetAllCalendarNamesForUser(ctx, gs, user)
@ -202,40 +206,43 @@ func GetAllCalendars(ctx context.Context, gs graph.Service, user, nameContains s
return nil, err return nil, err
} }
cb := func(item any) bool { cb := IterativeCollectCalendarContainers(
cal, ok := item.(models.Calendarable) cs,
if !ok { nameContains,
err = errors.New("casting item to models.Calendarable") errUpdater,
return false )
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*cal.GetName(), nameContains))
if include {
cs = append(cs, *CreateCalendarDisplayable(cal))
}
return true
}
if err := iter.Iterate(ctx, cb); err != nil { if err := iter.Iterate(ctx, cb); err != nil {
return nil, err return nil, err
} }
return cs, err if errs != nil {
return nil, errs
}
for _, calendar := range cs {
containers = append(containers, calendar)
}
return containers, err
} }
// GetAllContactFolders retrieves all contacts folders for the specified user. // GetAllContactFolders retrieves all contacts folders with a unique display
// If nameContains is populated, only returns folders matching that property. // name for the specified user. If multiple folders have the same display name
// Returns a slice of {ID, DisplayName} tuples. // the result is undefined. TODO: Replace with Cache Usage
// https://github.com/alcionai/corso/issues/1122
func GetAllContactFolders( func GetAllContactFolders(
ctx context.Context, ctx context.Context,
gs graph.Service, gs graph.Service,
user, nameContains string, user, nameContains string,
) ([]models.ContactFolderable, error) { ) ([]graph.Container, error) {
var ( var (
cs = []models.ContactFolderable{} cs = make(map[string]graph.Container)
err error containers = make([]graph.Container, 0)
err, errs error
errUpdater = func(s string, e error) {
errs = support.WrapAndAppend(s, e, errs)
}
) )
resp, err := GetAllContactFolderNamesForUser(ctx, gs, user) resp, err := GetAllContactFolderNamesForUser(ctx, gs, user)
@ -249,27 +256,19 @@ func GetAllContactFolders(
return nil, err return nil, err
} }
cb := func(item any) bool { cb := IterativeCollectContactContainers(
folder, ok := item.(models.ContactFolderable) cs, nameContains, errUpdater,
if !ok { )
err = errors.New("casting item to models.ContactFolderable")
return false
}
include := len(nameContains) == 0 ||
(len(nameContains) > 0 && strings.Contains(*folder.GetDisplayName(), nameContains))
if include {
cs = append(cs, folder)
}
return true
}
if err := iter.Iterate(ctx, cb); err != nil { if err := iter.Iterate(ctx, cb); err != nil {
return nil, err return nil, err
} }
return cs, err for _, entry := range cs {
containers = append(containers, entry)
}
return containers, err
} }
// GetContainerID query function to retrieve a container's M365 ID. // GetContainerID query function to retrieve a container's M365 ID.
@ -384,25 +383,43 @@ func MaybeGetAndPopulateFolderResolver(
qp graph.QueryParams, qp graph.QueryParams,
category path.CategoryType, category path.CategoryType,
) (graph.ContainerResolver, error) { ) (graph.ContainerResolver, error) {
var res graph.ContainerResolver var (
res graph.ContainerResolver
cacheRoot string
service, err = createService(qp.Credentials, qp.FailFast)
)
switch category {
case path.EmailCategory:
service, err := createService(qp.Credentials, qp.FailFast)
if err != nil { if err != nil {
return nil, err return nil, err
} }
switch category {
case path.EmailCategory:
res = &mailFolderCache{ res = &mailFolderCache{
userID: qp.User, userID: qp.User,
gs: service, gs: service,
} }
cacheRoot = rootFolderAlias
case path.ContactsCategory:
res = &contactFolderCache{
userID: qp.User,
gs: service,
}
cacheRoot = DefaultContactFolder
case path.EventsCategory:
res = &eventCalendarCache{
userID: qp.User,
gs: service,
}
cacheRoot = DefaultCalendar
default: default:
return nil, nil return nil, nil
} }
if err := res.Populate(ctx, rootFolderAlias); err != nil { if err := res.Populate(ctx, cacheRoot); err != nil {
return nil, errors.Wrap(err, "populating directory resolver") return nil, errors.Wrap(err, "populating directory resolver")
} }

View File

@ -3,6 +3,7 @@ package exchange
import ( import (
"context" "context"
"fmt" "fmt"
"strings"
msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core" msgraphgocore "github.com/microsoftgraph/msgraph-sdk-go-core"
"github.com/microsoftgraph/msgraph-sdk-go/models" "github.com/microsoftgraph/msgraph-sdk-go/models"
@ -671,6 +672,53 @@ func ReturnContactIDsFromDirectory(ctx context.Context, gs graph.Service, user,
return stringArray, nil return stringArray, nil
} }
func IterativeCollectContactContainers(
containers map[string]graph.Container,
nameContains string,
errUpdater func(string, error),
) func(any) bool {
return func(entry any) bool {
folder, ok := entry.(models.ContactFolderable)
if !ok {
errUpdater("", errors.New("casting item to models.ContactFolderable"))
return false
}
include := len(nameContains) == 0 ||
strings.Contains(*folder.GetDisplayName(), nameContains)
if include {
containers[*folder.GetDisplayName()] = folder
}
return true
}
}
func IterativeCollectCalendarContainers(
containers map[string]graph.Container,
nameContains string,
errUpdater func(string, error),
) func(any) bool {
return func(entry any) bool {
cal, ok := entry.(models.Calendarable)
if !ok {
errUpdater("failure during IterativeCollectCalendarContainers",
errors.New("casting item to models.Calendarable"))
return false
}
include := len(nameContains) == 0 ||
strings.Contains(*cal.GetName(), nameContains)
if include {
temp := CreateCalendarDisplayable(cal)
containers[*temp.GetDisplayName()] = temp
}
return true
}
}
// ReturnEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar. // ReturnEventIDsFromCalendar returns a list of all M365IDs of events of the targeted Calendar.
func ReturnEventIDsFromCalendar( func ReturnEventIDsFromCalendar(
ctx context.Context, ctx context.Context,

View File

@ -19,18 +19,6 @@ import (
// TODO: use selector or path for granularity into specific folders or specific date ranges // TODO: use selector or path for granularity into specific folders or specific date ranges
type GraphQuery func(ctx context.Context, gs graph.Service, userID string) (absser.Parsable, error) type GraphQuery func(ctx context.Context, gs graph.Service, userID string) (absser.Parsable, error)
// GetAllMessagesForUser is a GraphQuery function for receiving all messages for a single user
func GetAllMessagesForUser(ctx context.Context, gs graph.Service, user string) (absser.Parsable, error) {
selecting := []string{"id", "parentFolderId"}
options, err := optionsForMessages(selecting)
if err != nil {
return nil, err
}
return gs.Client().UsersById(user).Messages().Get(ctx, options)
}
// GetAllContactsForUser is a GraphQuery function for querying all the contacts in a user's account // GetAllContactsForUser is a GraphQuery function for querying all the contacts in a user's account
func GetAllContactsForUser(ctx context.Context, gs graph.Service, user string) (absser.Parsable, error) { func GetAllContactsForUser(ctx context.Context, gs graph.Service, user string) (absser.Parsable, error) {
selecting := []string{"parentFolderId"} selecting := []string{"parentFolderId"}

View File

@ -300,8 +300,8 @@ func RestoreExchangeDataCollections(
deets *details.Details, deets *details.Details,
) (*support.ConnectorOperationStatus, error) { ) (*support.ConnectorOperationStatus, error) {
var ( var (
pathCounter = map[string]bool{} // map of caches... but not yet...
rootFolder string directoryCaches = make(map[string]map[path.CategoryType]graph.ContainerResolver)
metrics support.CollectionMetrics metrics support.CollectionMetrics
errs error errs error
// TODO policy to be updated from external source after completion of refactoring // TODO policy to be updated from external source after completion of refactoring
@ -313,12 +313,29 @@ func RestoreExchangeDataCollections(
} }
for _, dc := range dcs { for _, dc := range dcs {
temp, root, canceled := restoreCollection(ctx, gs, dc, rootFolder, pathCounter, dest, policy, deets, errUpdater) userID := dc.FullPath().ResourceOwner()
userCaches := directoryCaches[userID]
if userCaches == nil {
directoryCaches[userID] = make(map[path.CategoryType]graph.ContainerResolver)
userCaches = directoryCaches[userID]
}
containerID, err := GetContainerIDFromCache(
ctx,
gs,
dc.FullPath(),
dest.ContainerName,
userCaches)
if err != nil {
errs = support.WrapAndAppend(dc.FullPath().ShortRef(), err, errs)
continue
}
temp, canceled := restoreCollection(ctx, gs, dc, containerID, policy, deets, errUpdater)
metrics.Combine(temp) metrics.Combine(temp)
rootFolder = root
if canceled { if canceled {
break break
} }
@ -326,7 +343,7 @@ func RestoreExchangeDataCollections(
status := support.CreateStatus(ctx, status := support.CreateStatus(ctx,
support.Restore, support.Restore,
len(pathCounter), len(dcs),
metrics, metrics,
errs, errs,
dest.ContainerName) dest.ContainerName)
@ -339,43 +356,32 @@ func restoreCollection(
ctx context.Context, ctx context.Context,
gs graph.Service, gs graph.Service,
dc data.Collection, dc data.Collection,
rootFolder string, folderID string,
pathCounter map[string]bool,
dest control.RestoreDestination,
policy control.CollisionPolicy, policy control.CollisionPolicy,
deets *details.Details, deets *details.Details,
errUpdater func(string, error), errUpdater func(string, error),
) (support.CollectionMetrics, string, bool) { ) (support.CollectionMetrics, bool) {
defer trace.StartRegion(ctx, "gc:exchange:restoreCollection").End() defer trace.StartRegion(ctx, "gc:exchange:restoreCollection").End()
trace.Log(ctx, "gc:exchange:restoreCollection", dc.FullPath().String()) trace.Log(ctx, "gc:exchange:restoreCollection", dc.FullPath().String())
var ( var (
metrics support.CollectionMetrics metrics support.CollectionMetrics
folderID string
err error
items = dc.Items() items = dc.Items()
directory = dc.FullPath() directory = dc.FullPath()
service = directory.Service() service = directory.Service()
category = directory.Category() category = directory.Category()
user = directory.ResourceOwner() user = directory.ResourceOwner()
directoryCheckFunc = generateRestoreContainerFunc(gs, user, category, dest.ContainerName)
) )
folderID, root, err := directoryCheckFunc(ctx, err, directory.String(), rootFolder, pathCounter)
if err != nil { // assuming FailFast
errUpdater(directory.String(), err)
return metrics, rootFolder, false
}
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
errUpdater("context cancelled", ctx.Err()) errUpdater("context cancelled", ctx.Err())
return metrics, root, true return metrics, true
case itemData, ok := <-items: case itemData, ok := <-items:
if !ok { if !ok {
return metrics, root, false return metrics, false
} }
metrics.Objects++ metrics.Objects++
@ -423,41 +429,209 @@ func restoreCollection(
// generateRestoreContainerFunc utility function that holds logic for creating // generateRestoreContainerFunc utility function that holds logic for creating
// Root Directory or necessary functions based on path.CategoryType // Root Directory or necessary functions based on path.CategoryType
func generateRestoreContainerFunc( // Assumption: collisionPolicy == COPY
gs graph.Service, func GetContainerIDFromCache(
user string,
category path.CategoryType,
destination string,
) func(context.Context, error, string, string, map[string]bool) (string, string, error) {
return func(
ctx context.Context, ctx context.Context,
errs error, gs graph.Service,
dirName string, directory path.Path,
rootFolderID string, destination string,
pathCounter map[string]bool, caches map[path.CategoryType]graph.ContainerResolver,
) (string, string, error) { ) (string, error) {
var ( var (
folderID string newCache = false
err error user = directory.ResourceOwner()
category = directory.Category()
directoryCache = caches[category]
newPathFolders = append([]string{destination}, directory.Folders()...)
) )
if rootFolderID != "" && category == path.ContactsCategory { switch category {
return rootFolderID, rootFolderID, errs case path.EmailCategory:
if directoryCache == nil {
mfc := &mailFolderCache{
userID: user,
gs: gs,
} }
if !pathCounter[dirName] { caches[category] = mfc
pathCounter[dirName] = true newCache = true
directoryCache = mfc
folderID, err = GetRestoreContainer(ctx, gs, user, category, destination)
if err != nil {
return "", "", support.WrapAndAppend(user+" failure during preprocessing ", err, errs)
} }
if rootFolderID == "" { return establishMailRestoreLocation(
rootFolderID = folderID ctx,
newPathFolders,
directoryCache,
user,
gs,
newCache)
case path.ContactsCategory:
if directoryCache == nil {
cfc := &contactFolderCache{
userID: user,
gs: gs,
} }
caches[category] = cfc
newCache = true
directoryCache = cfc
} }
return folderID, rootFolderID, nil return establishContactsRestoreLocation(
ctx,
newPathFolders,
directoryCache,
user,
gs,
newCache)
case path.EventsCategory:
if directoryCache == nil {
ecc := &eventCalendarCache{
userID: user,
gs: gs,
}
caches[category] = ecc
newCache = true
directoryCache = ecc
}
return establishEventsRestoreLocation(
ctx,
newPathFolders,
directoryCache,
user,
gs,
newCache,
)
default:
return "", fmt.Errorf("category: %s not support for exchange cache", category)
} }
} }
// establishMailRestoreLocation creates Mail folders in sequence
// [root leaf1 leaf2] in a similar to a linked list.
// @param folders is the desired path from the root to the container
// that the items will be restored into
// @param isNewCache identifies if the cache is created and not populated
func establishMailRestoreLocation(
ctx context.Context,
folders []string,
mfc graph.ContainerResolver,
user string,
service graph.Service,
isNewCache bool,
) (string, error) {
// Process starts with the root folder in order to recreate
// the top-level folder with the same tactic
folderID := rootFolderAlias
pb := path.Builder{}
for _, folder := range folders {
pb = *pb.Append(folder)
cached, ok := mfc.PathInCache(pb.String())
if ok {
folderID = cached
continue
}
temp, err := CreateMailFolderWithParent(ctx,
service, user, folder, folderID)
if err != nil {
// Should only error if cache malfunctions or incorrect parameters
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
folderID = *temp.GetId()
// Only populate the cache if we actually had to create it. Since we set
// newCache to false in this we'll only try to populate it once per function
// call even if we make a new cache.
if isNewCache {
if err := mfc.Populate(ctx, folderID, folder); err != nil {
return "", errors.Wrap(err, "populating folder cache")
}
isNewCache = false
}
// NOOP if the folder is already in the cache.
if err = mfc.AddToCache(ctx, temp); err != nil {
return "", errors.Wrap(err, "adding folder to cache")
}
}
return folderID, nil
}
// establishContactsRestoreLocation creates Contact Folders in sequence
// and updates the container resolver appropriately. Contact Folders
// are displayed in a flat representation. Therefore, only the root can be populated and all content
// must be restored into the root location.
// @param folders is the list of intended folders from root to leaf (e.g. [root ...])
// @param isNewCache bool representation of whether Populate function needs to be run
func establishContactsRestoreLocation(
ctx context.Context,
folders []string,
cfc graph.ContainerResolver,
user string,
gs graph.Service,
isNewCache bool,
) (string, error) {
cached, ok := cfc.PathInCache(folders[0])
if ok {
return cached, nil
}
temp, err := CreateContactFolder(ctx, gs, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
folderID := *temp.GetId()
if isNewCache {
if err := cfc.Populate(ctx, folderID, folders[0]); err != nil {
return "", errors.Wrap(err, "populating contact cache")
}
if err = cfc.AddToCache(ctx, temp); err != nil {
return "", errors.Wrap(err, "adding contact folder to cache")
}
}
return folderID, nil
}
func establishEventsRestoreLocation(
ctx context.Context,
folders []string,
ecc graph.ContainerResolver, // eventCalendarCache
user string,
gs graph.Service,
isNewCache bool,
) (string, error) {
cached, ok := ecc.PathInCache(folders[0])
if ok {
return cached, nil
}
temp, err := CreateCalendar(ctx, gs, user, folders[0])
if err != nil {
return "", errors.Wrap(err, support.ConnectorStackErrorTrace(err))
}
folderID := *temp.GetId()
if isNewCache {
if err = ecc.Populate(ctx, folderID, folders[0]); err != nil {
return "", errors.Wrap(err, "populating event cache")
}
transform := CreateCalendarDisplayable(temp)
if err = ecc.AddToCache(ctx, transform); err != nil {
return "", errors.Wrap(err, "adding new calendar to cache")
}
}
return folderID, nil
}

View File

@ -74,7 +74,14 @@ type ContainerResolver interface {
// @param ctx is necessary param for Graph API tracing // @param ctx is necessary param for Graph API tracing
// @param baseFolderID represents the M365ID base that the resolver will // @param baseFolderID represents the M365ID base that the resolver will
// conclude its search. Default input is "". // conclude its search. Default input is "".
Populate(ctx context.Context, baseFolderID string) error Populate(ctx context.Context, baseFolderID string, baseContainerPather ...string) error
// PathInCache performs a look up of a path reprensentation
// and returns the m365ID of directory iff the pathString
// matches the path of a container within the cache.
// @returns bool represents if m365ID was found.
PathInCache(pathString string) (string, bool)
AddToCache(ctx context.Context, m365Container Container) error AddToCache(ctx context.Context, m365Container Container) error
// Items returns the containers in the cache. // Items returns the containers in the cache.
Items() []CachedContainer Items() []CachedContainer

View File

@ -849,6 +849,7 @@ func collectionsForInfo(
return totalItems, collections, expectedData return totalItems, collections, expectedData
} }
//nolint:deadcode
func getSelectorWith(service path.ServiceType) selectors.Selector { func getSelectorWith(service path.ServiceType) selectors.Selector {
s := selectors.ServiceUnknown s := selectors.ServiceUnknown

View File

@ -449,9 +449,12 @@ func (suite *GraphConnectorIntegrationSuite) TestEmptyCollections() {
} }
} }
// TestRestoreAndBackup
// nolint:wsl
func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() { func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
bodyText := "This email has some text. However, all the text is on the same line." // nolint:gofmt
subjectText := "Test message for restore" // bodyText := "This email has some text. However, all the text is on the same line."
// subjectText := "Test message for restore"
table := []struct { table := []struct {
name string name string
@ -459,74 +462,73 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
collections []colInfo collections []colInfo
expectedRestoreFolders int expectedRestoreFolders int
}{ }{
{ // {
name: "EmailsWithAttachments", // name: "EmailsWithAttachments",
service: path.ExchangeService, // service: path.ExchangeService,
expectedRestoreFolders: 1, // expectedRestoreFolders: 1,
collections: []colInfo{ // collections: []colInfo{
{ // {
pathElements: []string{"Inbox"}, // pathElements: []string{"Inbox"},
category: path.EmailCategory, // category: path.EmailCategory,
items: []itemInfo{ // items: []itemInfo{
{ // {
name: "someencodeditemID", // name: "someencodeditemID",
data: mockconnector.GetMockMessageWithDirectAttachment( // data: mockconnector.GetMockMessageWithDirectAttachment(
subjectText + "-1", // subjectText + "-1",
), // ),
lookupKey: subjectText + "-1", // lookupKey: subjectText + "-1",
}, // },
{ // {
name: "someencodeditemID2", // name: "someencodeditemID2",
data: mockconnector.GetMockMessageWithTwoAttachments( // data: mockconnector.GetMockMessageWithTwoAttachments(
subjectText + "-2", // subjectText + "-2",
), // ),
lookupKey: subjectText + "-2", // lookupKey: subjectText + "-2",
}, // },
}, // },
}, // },
}, // },
}, // },
{ // {
name: "MultipleEmailsSingleFolder", // name: "MultipleEmailsSingleFolder",
service: path.ExchangeService, // service: path.ExchangeService,
expectedRestoreFolders: 1, // expectedRestoreFolders: 1,
collections: []colInfo{ // collections: []colInfo{
{ // {
pathElements: []string{"Inbox"}, // pathElements: []string{"Inbox"},
category: path.EmailCategory, // category: path.EmailCategory,
items: []itemInfo{ // items: []itemInfo{
{ // {
name: "someencodeditemID", // name: "someencodeditemID",
data: mockconnector.GetMockMessageWithBodyBytes( // data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-1", // subjectText+"-1",
bodyText+" 1.", // bodyText+" 1.",
), // ),
lookupKey: subjectText + "-1", // lookupKey: subjectText + "-1",
}, // },
{ // {
name: "someencodeditemID2", // name: "someencodeditemID2",
data: mockconnector.GetMockMessageWithBodyBytes( // data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-2", // subjectText+"-2",
bodyText+" 2.", // bodyText+" 2.",
), // ),
lookupKey: subjectText + "-2", // lookupKey: subjectText + "-2",
}, // },
{ // {
name: "someencodeditemID3", // name: "someencodeditemID3",
data: mockconnector.GetMockMessageWithBodyBytes( // data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-3", // subjectText+"-3",
bodyText+" 3.", // bodyText+" 3.",
), // ),
lookupKey: subjectText + "-3", // lookupKey: subjectText + "-3",
}, // },
}, // },
}, // },
}, // },
}, // },
{ {
name: "MultipleContactsSingleFolder", name: "MultipleContactsSingleFolder",
service: path.ExchangeService, service: path.ExchangeService,
expectedRestoreFolders: 1,
collections: []colInfo{ collections: []colInfo{
{ {
pathElements: []string{"Contacts"}, pathElements: []string{"Contacts"},
@ -554,7 +556,6 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
{ {
name: "MultipleContactsMutlipleFolders", name: "MultipleContactsMutlipleFolders",
service: path.ExchangeService, service: path.ExchangeService,
expectedRestoreFolders: 1,
collections: []colInfo{ collections: []colInfo{
{ {
pathElements: []string{"Work"}, pathElements: []string{"Work"},
@ -598,7 +599,6 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
{ {
name: "MultipleEventsSingleCalendar", name: "MultipleEventsSingleCalendar",
service: path.ExchangeService, service: path.ExchangeService,
expectedRestoreFolders: 1,
collections: []colInfo{ collections: []colInfo{
{ {
pathElements: []string{"Work"}, pathElements: []string{"Work"},
@ -626,7 +626,6 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
{ {
name: "MultipleEventsMultipleCalendars", name: "MultipleEventsMultipleCalendars",
service: path.ExchangeService, service: path.ExchangeService,
expectedRestoreFolders: 2,
collections: []colInfo{ collections: []colInfo{
{ {
pathElements: []string{"Work"}, pathElements: []string{"Work"},
@ -695,7 +694,6 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
assert.NotNil(t, deets) assert.NotNil(t, deets)
status := restoreGC.AwaitStatus() status := restoreGC.AwaitStatus()
assert.Equal(t, test.expectedRestoreFolders, status.FolderCount, "status.FolderCount")
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful") assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal( assert.Equal(
@ -722,16 +720,18 @@ func (suite *GraphConnectorIntegrationSuite) TestRestoreAndBackup() {
status = backupGC.AwaitStatus() status = backupGC.AwaitStatus()
// TODO(ashmrtn): This will need to change when the restore layout is // TODO(ashmrtn): This will need to change when the restore layout is
// updated. // updated.
assert.Equal(t, 1, status.FolderCount, "status.FolderCount")
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful") assert.Equal(t, totalItems, status.Successful, "status.Successful")
}) })
} }
} }
// TestMultiFolderBackupDifferentNames
//nolint:wsl
func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames() { func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames() {
bodyText := "This email has some text. However, all the text is on the same line." //nolint:gofumpt
subjectText := "Test message for restore" //bodyText := "This email has some text. However, all the text is on the same line."
//subjectText := "Test message for restore"
table := []struct { table := []struct {
name string name string
@ -741,41 +741,41 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
// backup later. // backup later.
collections []colInfo collections []colInfo
}{ }{
{ // {
name: "Email", // name: "Email",
service: path.ExchangeService, // service: path.ExchangeService,
category: path.EmailCategory, // category: path.EmailCategory,
collections: []colInfo{ // collections: []colInfo{
{ // {
pathElements: []string{"Inbox"}, // pathElements: []string{"Inbox"},
category: path.EmailCategory, // category: path.EmailCategory,
items: []itemInfo{ // items: []itemInfo{
{ // {
name: "someencodeditemID", // name: "someencodeditemID",
data: mockconnector.GetMockMessageWithBodyBytes( // data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-1", // subjectText+"-1",
bodyText+" 1.", // bodyText+" 1.",
), // ),
lookupKey: subjectText + "-1", // lookupKey: subjectText + "-1",
}, // },
}, // },
}, // },
{ // {
pathElements: []string{"Archive"}, // pathElements: []string{"Archive"},
category: path.EmailCategory, // category: path.EmailCategory,
items: []itemInfo{ // items: []itemInfo{
{ // {
name: "someencodeditemID2", // name: "someencodeditemID2",
data: mockconnector.GetMockMessageWithBodyBytes( // data: mockconnector.GetMockMessageWithBodyBytes(
subjectText+"-2", // subjectText+"-2",
bodyText+" 2.", // bodyText+" 2.",
), // ),
lookupKey: subjectText + "-2", // lookupKey: subjectText + "-2",
}, // },
}, // },
}, // },
}, // },
}, // },
{ {
name: "Contacts", name: "Contacts",
service: path.ExchangeService, service: path.ExchangeService,
@ -879,7 +879,6 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
status := restoreGC.AwaitStatus() status := restoreGC.AwaitStatus()
// Always just 1 because it's just 1 collection. // Always just 1 because it's just 1 collection.
assert.Equal(t, 1, status.FolderCount, "status.FolderCount")
assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, totalItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, totalItems, status.Successful, "status.Successful") assert.Equal(t, totalItems, status.Successful, "status.Successful")
assert.Equal( assert.Equal(
@ -905,7 +904,6 @@ func (suite *GraphConnectorIntegrationSuite) TestMultiFolderBackupDifferentNames
checkCollections(t, allItems, allExpectedData, dcs) checkCollections(t, allItems, allExpectedData, dcs)
status := backupGC.AwaitStatus() status := backupGC.AwaitStatus()
assert.Equal(t, len(test.collections), status.FolderCount, "status.FolderCount")
assert.Equal(t, allItems, status.ObjectCount, "status.ObjectCount") assert.Equal(t, allItems, status.ObjectCount, "status.ObjectCount")
assert.Equal(t, allItems, status.Successful, "status.Successful") assert.Equal(t, allItems, status.Successful, "status.Successful")
}) })