Switch to using path categories in graph connector (#734)

Will help make later PRs easier as the category will already be known.

## Description

<!-- Insert PR description-->

## Type of change

Please check the type of change your PR introduces:
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [ ] 🤖 Test
- [x] 🐹 Trivial/Minor

## Issue(s)
<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
on path to:
* #456 

## Test Plan

<!-- How will this be tested prior to merging.-->

- [ ] 💪 Manual
- [x]  Unit test
- [ ] 💚 E2E
This commit is contained in:
ashmrtn 2022-09-02 13:12:22 -07:00 committed by GitHub
parent b1cd472483
commit 12f61dd71c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 52 additions and 48 deletions

View File

@ -12,6 +12,7 @@ import (
"github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/connector/graph" "github.com/alcionai/corso/internal/connector/graph"
"github.com/alcionai/corso/internal/connector/mockconnector" "github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/internal/tester" "github.com/alcionai/corso/internal/tester"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/control" "github.com/alcionai/corso/pkg/control"
@ -457,37 +458,37 @@ func (suite *ExchangeServiceSuite) TestRestoreEvent() {
func (suite *ExchangeServiceSuite) TestGetRestoreContainer() { func (suite *ExchangeServiceSuite) TestGetRestoreContainer() {
tests := []struct { tests := []struct {
name string name string
option string option path.CategoryType
checkError assert.ErrorAssertionFunc checkError assert.ErrorAssertionFunc
cleanupFunc func(graph.Service, string, string) error cleanupFunc func(graph.Service, string, string) error
}{ }{
{ {
name: "Establish User Restore Folder", name: "Establish User Restore Folder",
option: "users", option: path.CategoryType(-1),
checkError: assert.Error, checkError: assert.Error,
cleanupFunc: nil, cleanupFunc: nil,
}, },
{ {
name: "Establish Event Restore Location", name: "Establish Event Restore Location",
option: "events", option: path.EventsCategory,
checkError: assert.NoError, checkError: assert.NoError,
cleanupFunc: DeleteCalendar, cleanupFunc: DeleteCalendar,
}, },
{ {
name: "Establish Restore Folder for Unknown", name: "Establish Restore Folder for Unknown",
option: "unknown", option: path.UnknownCategory,
checkError: assert.Error, checkError: assert.Error,
cleanupFunc: nil, cleanupFunc: nil,
}, },
{ {
name: "Establish Restore folder for Mail", name: "Establish Restore folder for Mail",
option: "mail", option: path.EmailCategory,
checkError: assert.NoError, checkError: assert.NoError,
cleanupFunc: DeleteMailFolder, cleanupFunc: DeleteMailFolder,
}, },
{ {
name: "Establish Restore folder for Contacts", name: "Establish Restore folder for Contacts",
option: "contacts", option: path.ContactsCategory,
checkError: assert.NoError, checkError: assert.NoError,
cleanupFunc: DeleteContactFolder, cleanupFunc: DeleteContactFolder,
}, },

View File

@ -13,6 +13,8 @@ import (
msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages" msmessage "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages"
msitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages/item" msitem "github.com/microsoftgraph/msgraph-sdk-go/users/item/messages/item"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/alcionai/corso/internal/path"
) )
//----------------------------------------------------------------------- //-----------------------------------------------------------------------
@ -99,19 +101,13 @@ const (
contacts contacts
) )
const ( func categoryToOptionIdentifier(category path.CategoryType) optionIdentifier {
mailCategory = "mail"
contactsCategory = "contacts"
eventsCategory = "events"
)
func categoryToOptionIdentifier(category string) optionIdentifier {
switch category { switch category {
case mailCategory: case path.EmailCategory:
return messages return messages
case contactsCategory: case path.ContactsCategory:
return contacts return contacts
case eventsCategory: case path.EventsCategory:
return events return events
default: default:
return unknown return unknown

View File

@ -14,6 +14,7 @@ import (
"github.com/alcionai/corso/internal/common" "github.com/alcionai/corso/internal/common"
"github.com/alcionai/corso/internal/connector/graph" "github.com/alcionai/corso/internal/connector/graph"
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/control" "github.com/alcionai/corso/pkg/control"
"github.com/alcionai/corso/pkg/logger" "github.com/alcionai/corso/pkg/logger"
@ -302,7 +303,8 @@ func SetupExchangeCollectionVars(scope selectors.ExchangeScope) (
// that defines the application the folder is created in. // that defines the application the folder is created in.
func GetRestoreContainer( func GetRestoreContainer(
service graph.Service, service graph.Service,
user, category string, user string,
category path.CategoryType,
) (string, error) { ) (string, error) {
name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat)) name := fmt.Sprintf("Corso_Restore_%s", common.FormatNow(common.SimpleDateTimeFormat))
option := categoryToOptionIdentifier(category) option := categoryToOptionIdentifier(category)
@ -346,7 +348,7 @@ func GetRestoreContainer(
func RestoreExchangeObject( func RestoreExchangeObject(
ctx context.Context, ctx context.Context,
bits []byte, bits []byte,
category string, category path.CategoryType,
policy control.CollisionPolicy, policy control.CollisionPolicy,
service graph.Service, service graph.Service,
destination, user string, destination, user string,
@ -354,7 +356,7 @@ func RestoreExchangeObject(
var setting optionIdentifier var setting optionIdentifier
switch category { switch category {
case mailCategory, contactsCategory: case path.EmailCategory, path.ContactsCategory:
setting = categoryToOptionIdentifier(category) setting = categoryToOptionIdentifier(category)
default: default:
return fmt.Errorf("type: %s not supported for exchange restore", category) return fmt.Errorf("type: %s not supported for exchange restore", category)

View File

@ -9,6 +9,7 @@ import (
"github.com/alcionai/corso/internal/connector/graph" "github.com/alcionai/corso/internal/connector/graph"
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
) )
@ -50,7 +51,7 @@ func IterateSelectAllDescendablesForCollections(
var ( var (
isCategorySet bool isCategorySet bool
collectionType optionIdentifier collectionType optionIdentifier
category string category path.CategoryType
) )
return func(pageItem any) bool { return func(pageItem any) bool {
@ -58,12 +59,12 @@ func IterateSelectAllDescendablesForCollections(
if !isCategorySet { if !isCategorySet {
if qp.Scope.IncludesCategory(selectors.ExchangeMail) { if qp.Scope.IncludesCategory(selectors.ExchangeMail) {
collectionType = messages collectionType = messages
category = mailCategory category = path.EmailCategory
} }
if qp.Scope.IncludesCategory(selectors.ExchangeContact) { if qp.Scope.IncludesCategory(selectors.ExchangeContact) {
collectionType = contacts collectionType = contacts
category = contactsCategory category = path.ContactsCategory
} }
isCategorySet = true isCategorySet = true
@ -85,7 +86,7 @@ func IterateSelectAllDescendablesForCollections(
edc := NewCollection( edc := NewCollection(
qp.User, qp.User,
[]string{qp.Credentials.TenantID, qp.User, category, directory}, []string{qp.Credentials.TenantID, qp.User, category.String(), directory},
collectionType, collectionType,
service, service,
statusUpdater, statusUpdater,
@ -167,7 +168,7 @@ func IterateSelectAllEventsForCollections(
edc := NewCollection( edc := NewCollection(
qp.User, qp.User,
[]string{qp.Credentials.TenantID, qp.User, eventsCategory, directory}, []string{qp.Credentials.TenantID, qp.User, path.EventsCategory.String(), directory},
events, events,
service, service,
statusUpdater, statusUpdater,
@ -276,7 +277,7 @@ func IterateFilterFolderDirectoriesForCollections(
temp := NewCollection( temp := NewCollection(
qp.User, qp.User,
[]string{qp.Credentials.TenantID, qp.User, mailCategory, directory}, []string{qp.Credentials.TenantID, qp.User, path.EmailCategory.String(), directory},
messages, messages,
service, service,
statusUpdater, statusUpdater,

View File

@ -18,6 +18,7 @@ import (
"github.com/alcionai/corso/internal/connector/graph" "github.com/alcionai/corso/internal/connector/graph"
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/data" "github.com/alcionai/corso/internal/data"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/pkg/account" "github.com/alcionai/corso/pkg/account"
"github.com/alcionai/corso/pkg/control" "github.com/alcionai/corso/pkg/control"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
@ -260,7 +261,8 @@ func (gc *GraphConnector) RestoreExchangeDataCollection(
directory = strings.Join(dc.FullPath(), "") directory = strings.Join(dc.FullPath(), "")
user = dc.FullPath()[1] user = dc.FullPath()[1]
items = dc.Items() items = dc.Items()
category = dc.FullPath()[2] // TODO(ashmrtn): Update this when we have path struct support in collections.
category = path.ToCategoryType(dc.FullPath()[2])
exit bool exit bool
) )

View File

@ -17,6 +17,7 @@ import (
"github.com/alcionai/corso/internal/connector/mockconnector" "github.com/alcionai/corso/internal/connector/mockconnector"
"github.com/alcionai/corso/internal/connector/support" "github.com/alcionai/corso/internal/connector/support"
"github.com/alcionai/corso/internal/data" "github.com/alcionai/corso/internal/data"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/internal/tester" "github.com/alcionai/corso/internal/tester"
"github.com/alcionai/corso/pkg/selectors" "github.com/alcionai/corso/pkg/selectors"
) )
@ -245,13 +246,13 @@ func (suite *GraphConnectorIntegrationSuite) TestEventsSerializationRegression()
// The result should be all successful items restored within the same folder. // The result should be all successful items restored within the same folder.
func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() { func (suite *GraphConnectorIntegrationSuite) TestRestoreMessages() {
t := suite.T() t := suite.T()
category := "mail" category := path.EmailCategory
connector := loadConnector(t) connector := loadConnector(t)
collection := make([]data.Collection, 0) collection := make([]data.Collection, 0)
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
mdc := mockconnector.NewMockExchangeCollection( mdc := mockconnector.NewMockExchangeCollection(
[]string{"tenant", suite.user, category, "Inbox"}, []string{"tenant", suite.user, category.String(), "Inbox"},
1) 1)
collection = append(collection, mdc) collection = append(collection, mdc)
} }

View File

@ -566,7 +566,7 @@ func (suite *ModelStoreRegressionSuite) TestMultipleConfigs() {
deets.Entries = append( deets.Entries = append(
deets.Entries, deets.Entries,
details.DetailsEntry{ details.DetailsEntry{
RepoRef: fmt.Sprintf("exchange/user1/mail/inbox/mail%v", i), RepoRef: fmt.Sprintf("exchange/user1/email/inbox/mail%v", i),
ItemInfo: details.ItemInfo{ ItemInfo: details.ItemInfo{
Exchange: &details.ExchangeInfo{ Exchange: &details.ExchangeInfo{
Sender: "John Doe", Sender: "John Doe",

View File

@ -27,7 +27,7 @@ import (
const ( const (
testTenant = "a-tenant" testTenant = "a-tenant"
testUser = "user1" testUser = "user1"
testEmailDir = "mail" testEmailDir = "email"
testInboxDir = "inbox" testInboxDir = "inbox"
testArchiveDir = "archive" testArchiveDir = "archive"
testFileName = "file1" testFileName = "file1"

View File

@ -33,7 +33,7 @@ const (
EventsCategory // events EventsCategory // events
) )
func toCategoryType(category string) CategoryType { func ToCategoryType(category string) CategoryType {
switch category { switch category {
case EmailCategory.String(): case EmailCategory.String():
return EmailCategory return EmailCategory
@ -59,7 +59,7 @@ func validateServiceAndCategory(s, c string) (ServiceType, CategoryType, error)
// Validity of service checked on first-level lookup to serviceCategories. // Validity of service checked on first-level lookup to serviceCategories.
service := toServiceType(s) service := toServiceType(s)
category := toCategoryType(c) category := ToCategoryType(c)
if category == UnknownCategory { if category == UnknownCategory {
return UnknownService, UnknownCategory, errors.Errorf("unknown category string %q", c) return UnknownService, UnknownCategory, errors.Errorf("unknown category string %q", c)
} }

View File

@ -777,7 +777,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeScope_MatchesPath() {
) )
var ( var (
path = []string{"tid", usr, "mail", fld, mail} path = []string{"tid", usr, "email", fld, mail}
es = NewExchangeRestore() es = NewExchangeRestore()
) )
@ -843,7 +843,7 @@ func (suite *ExchangeSelectorSuite) TestIdPath() {
}, },
{ {
ExchangeMail, ExchangeMail,
[]string{"tid", "uid", "mail", "mFld", "mid"}, []string{"tid", "uid", "email", "mFld", "mid"},
map[exchangeCategory]string{ map[exchangeCategory]string{
ExchangeUser: "uid", ExchangeUser: "uid",
ExchangeMailFolder: "mFld", ExchangeMailFolder: "mFld",
@ -883,7 +883,7 @@ func (suite *ExchangeSelectorSuite) TestExchangeRestore_Reduce() {
const ( const (
contact = "tid/uid/contacts/cfld/cid" contact = "tid/uid/contacts/cfld/cid"
event = "tid/uid/events/ecld/eid" event = "tid/uid/events/ecld/eid"
mail = "tid/uid/mail/mfld/mid" mail = "tid/uid/email/mfld/mid"
) )
arr := func(s ...string) []string { arr := func(s ...string) []string {
@ -1090,7 +1090,7 @@ func (suite *ExchangeSelectorSuite) TestPasses() {
mail = setScopesToDefault(es.Mails(Any(), Any(), []string{mid})) mail = setScopesToDefault(es.Mails(Any(), Any(), []string{mid}))
otherMail = setScopesToDefault(es.Mails(Any(), Any(), []string{"smarf"})) otherMail = setScopesToDefault(es.Mails(Any(), Any(), []string{"smarf"}))
noMail = setScopesToDefault(es.Mails(Any(), Any(), None())) noMail = setScopesToDefault(es.Mails(Any(), Any(), None()))
path = []string{"tid", "user", "mail", "folder", mid} path = []string{"tid", "user", "email", "folder", mid}
) )
table := []struct { table := []struct {
@ -1175,7 +1175,7 @@ func (suite *ExchangeSelectorSuite) TestIsAny() {
es = NewExchangeRestore() es = NewExchangeRestore()
anyUser = setScopesToDefault(es.Users(Any())) anyUser = setScopesToDefault(es.Users(Any()))
noUser = setScopesToDefault(es.Users(None())) noUser = setScopesToDefault(es.Users(None()))
specificMail = setScopesToDefault(es.Mails(Any(), Any(), []string{"mail"})) specificMail = setScopesToDefault(es.Mails(Any(), Any(), []string{"email"}))
anyMail = setScopesToDefault(es.Mails(Any(), Any(), Any())) anyMail = setScopesToDefault(es.Mails(Any(), Any(), Any()))
) )

View File

@ -3,6 +3,7 @@ package selectors
import ( import (
"strings" "strings"
"github.com/alcionai/corso/internal/path"
"github.com/alcionai/corso/pkg/backup/details" "github.com/alcionai/corso/pkg/backup/details"
"github.com/alcionai/corso/pkg/filters" "github.com/alcionai/corso/pkg/filters"
) )
@ -240,16 +241,16 @@ func reduce[T scopeT, C categoryT](
// for each entry, compare that entry against the scopes of the same data type // for each entry, compare that entry against the scopes of the same data type
for _, ent := range deets.Entries { for _, ent := range deets.Entries {
// todo: use Path pkg for this // todo: use Path pkg for this
path := strings.Split(ent.RepoRef, "/") repoPath := strings.Split(ent.RepoRef, "/")
dc, ok := dataCategories[pathTypeIn(path)] dc, ok := dataCategories[pathTypeIn(repoPath)]
if !ok { if !ok {
continue continue
} }
passed := passes( passed := passes(
dc, dc,
dc.pathValues(path), dc.pathValues(repoPath),
ent, ent,
excls[dc], excls[dc],
filts[dc], filts[dc],
@ -283,19 +284,19 @@ const (
// package. It should get handled in paths, since that's where service- and // package. It should get handled in paths, since that's where service- and
// data-type-specific assertions are owned. // data-type-specific assertions are owned.
// Ideally, we'd use something like path.DataType() instead of this func. // Ideally, we'd use something like path.DataType() instead of this func.
func pathTypeIn(path []string) pathType { func pathTypeIn(p []string) pathType {
// not all paths will be len=3. Most should be longer. // not all paths will be len=3. Most should be longer.
// This just protects us from panicing below. // This just protects us from panicing below.
if len(path) < 3 { if len(p) < 3 {
return unknownPathType return unknownPathType
} }
switch path[2] { switch p[2] {
case "mail": case path.EmailCategory.String():
return exchangeMailPath return exchangeMailPath
case "contacts": case path.ContactsCategory.String():
return exchangeContactPath return exchangeContactPath
case "events": case path.EventsCategory.String():
return exchangeEventPath return exchangeEventPath
} }

View File

@ -243,7 +243,7 @@ func (suite *SelectorScopesSuite) TestReduce() {
func (suite *SelectorScopesSuite) TestPathTypeIn() { func (suite *SelectorScopesSuite) TestPathTypeIn() {
t := suite.T() t := suite.T()
assert.Equal(t, unknownPathType, pathTypeIn([]string{}), "empty") assert.Equal(t, unknownPathType, pathTypeIn([]string{}), "empty")
assert.Equal(t, exchangeMailPath, pathTypeIn([]string{"", "", "mail"}), "mail") assert.Equal(t, exchangeMailPath, pathTypeIn([]string{"", "", "email"}), "email")
assert.Equal(t, exchangeContactPath, pathTypeIn([]string{"", "", "contacts"}), "contact") assert.Equal(t, exchangeContactPath, pathTypeIn([]string{"", "", "contacts"}), "contact")
assert.Equal(t, exchangeEventPath, pathTypeIn([]string{"", "", "events"}), "event") assert.Equal(t, exchangeEventPath, pathTypeIn([]string{"", "", "events"}), "event")
assert.Equal(t, unknownPathType, pathTypeIn([]string{"", "", "fnords"}), "bogus") assert.Equal(t, unknownPathType, pathTypeIn([]string{"", "", "fnords"}), "bogus")