From 652bb9cd4ad8e5e77cfd04cade449aa6056f1287 Mon Sep 17 00:00:00 2001 From: Danny Date: Wed, 28 Sep 2022 10:16:08 -0400 Subject: [PATCH] GC: Restore: Contact: Restore to single Folder (#972) ## Description The feature ensures that this Flat style directory only creates a single folder and restores all the contacts to the same folder during the restore process. ## Type of change - [x] :sunflower: Feature - [x] :bug: Bugfix - [ ] :world_map: Documentation - [ ] :robot: Test - [ ] :computer: CI/Deployment - [ ] :hamster: Trivial/Minor ## Issue(s) *closes #880 ## Test Plan - [ ] :muscle: Manual - [ ] :zap: Unit test - [ ] :green_heart: E2E --- .../connector/exchange/service_restore.go | 133 +++++++++++++++++- src/internal/connector/graph_connector.go | 115 ++------------- .../connector/graph_connector_test.go | 38 +++++ .../mockconnector/mock_data_collection.go | 66 ++++++++- 4 files changed, 245 insertions(+), 107 deletions(-) diff --git a/src/internal/connector/exchange/service_restore.go b/src/internal/connector/exchange/service_restore.go index 224f38e27..de8530bb6 100644 --- a/src/internal/connector/exchange/service_restore.go +++ b/src/internal/connector/exchange/service_restore.go @@ -1,6 +1,7 @@ package exchange import ( + "bytes" "context" "fmt" @@ -10,6 +11,7 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -113,9 +115,11 @@ func RestoreExchangeContact( response, err := service.Client().UsersById(user).ContactFoldersById(destination).Contacts().Post(ctx, contact, nil) if err != nil { + name := *contact.GetGivenName() + return errors.Wrap( err, - "failure to create Contact during RestoreExchangeContact: "+ + "failure to create Contact during RestoreExchangeContact: "+name+" "+ support.ConnectorStackErrorTrace(err), ) } @@ -242,3 +246,130 @@ func SendMailToBackStore( return nil } + +// RestoreExchangeDataCollections restores M365 objects in data.Collection to MSFT +// store through GraphAPI. +// @param dest: container destination to M365 +func RestoreExchangeDataCollections( + ctx context.Context, + gs graph.Service, + dest control.RestoreDestination, + dcs []data.Collection, +) (*support.ConnectorOperationStatus, error) { + var ( + pathCounter = map[string]bool{} + attempts, successes int + errs error + folderID, root string + isCancelled bool + // TODO policy to be updated from external source after completion of refactoring + policy = control.Copy + ) + + for _, dc := range dcs { + var ( + items = dc.Items() + directory = dc.FullPath() + service = directory.Service() + category = directory.Category() + user = directory.ResourceOwner() + exit bool + directoryCheckFunc = generateRestoreContainerFunc(gs, user, category, dest.ContainerName) + ) + + folderID, root, errs = directoryCheckFunc(ctx, errs, directory.String(), root, pathCounter) + if errs != nil { // assuming FailFast + break + } + + if isCancelled { + break + } + + for !exit { + select { + case <-ctx.Done(): + errs = support.WrapAndAppend("context cancelled", ctx.Err(), errs) + isCancelled = true + + case itemData, ok := <-items: + if !ok { + exit = true + continue + } + attempts++ + + buf := &bytes.Buffer{} + + _, err := buf.ReadFrom(itemData.ToReader()) + if err != nil { + errs = support.WrapAndAppend( + itemData.UUID()+": byteReadError during RestoreDataCollection", + err, + errs, + ) + + continue + } + + err = RestoreExchangeObject(ctx, buf.Bytes(), category, policy, gs, folderID, user) + if err != nil { + // More information to be here + errs = support.WrapAndAppend( + itemData.UUID()+": failed to upload RestoreExchangeObject: "+service.String()+"-"+category.String(), + err, + errs, + ) + + continue + } + successes++ + } + } + } + + status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs) + + return status, errs +} + +// generateRestoreContainerFunc utility function that holds logic for creating +// Root Directory or necessary functions based on path.CategoryType +func generateRestoreContainerFunc( + gs graph.Service, + user string, + category path.CategoryType, + destination string, +) func(context.Context, error, string, string, map[string]bool) (string, string, error) { + return func( + ctx context.Context, + errs error, + dirName string, + rootFolderID string, + pathCounter map[string]bool, + ) (string, string, error) { + var ( + folderID string + err error + ) + + if rootFolderID != "" && category == path.ContactsCategory { + return rootFolderID, rootFolderID, errs + } + + if !pathCounter[dirName] { + pathCounter[dirName] = true + + folderID, err = GetRestoreContainer(ctx, gs, user, category, destination) + if err != nil { + return "", "", support.WrapAndAppend(user+" failure during preprocessing ", err, errs) + } + + if rootFolderID == "" { + rootFolderID = folderID + } + } + + return folderID, rootFolderID, nil + } +} diff --git a/src/internal/connector/graph_connector.go b/src/internal/connector/graph_connector.go index ccce8083a..27ed21c73 100644 --- a/src/internal/connector/graph_connector.go +++ b/src/internal/connector/graph_connector.go @@ -3,7 +3,6 @@ package connector import ( - "bytes" "context" "fmt" "sync" @@ -241,122 +240,32 @@ func (gc *GraphConnector) ExchangeDataCollection( } // RestoreDataCollections restores data from the specified collections -// into M365 +// into M365 using the GraphAPI. +// SideEffect: gc.status is updated at the completion of operation func (gc *GraphConnector) RestoreDataCollections( ctx context.Context, selector selectors.Selector, dest control.RestoreDestination, dcs []data.Collection, -) error { - switch selector.Service { - case selectors.ServiceExchange: - return gc.RestoreExchangeDataCollections(ctx, dest, dcs) - case selectors.ServiceOneDrive: - status, err := onedrive.RestoreCollections(ctx, gc, dest, dcs) - if err != nil { - return err - } - - gc.incrementAwaitingMessages() - - gc.UpdateStatus(status) - - return nil - default: - return errors.Errorf("restore data from service %s not supported", selector.Service.String()) - } -} - -// RestoreDataCollections restores data from the specified collections -// into M365 -func (gc *GraphConnector) RestoreExchangeDataCollections( - ctx context.Context, - dest control.RestoreDestination, - dcs []data.Collection, ) error { var ( - pathCounter = map[string]bool{} - attempts, successes int - errs error - folderID string - // TODO policy to be updated from external source after completion of refactoring - policy = control.Copy + status *support.ConnectorOperationStatus + err error ) - for _, dc := range dcs { - var ( - items = dc.Items() - directory = dc.FullPath() - service = directory.Service() - category = directory.Category() - user = directory.ResourceOwner() - exit bool - ) - - if _, ok := pathCounter[directory.String()]; !ok { - pathCounter[directory.String()] = true - - folderID, errs = exchange.GetRestoreContainer( - ctx, - &gc.graphService, - user, - category, - dest.ContainerName, - ) - - if errs != nil { - fmt.Println("RestoreContainer Failed") - return errs - } - } - - for !exit { - select { - case <-ctx.Done(): - return support.WrapAndAppend("context cancelled", ctx.Err(), errs) - case itemData, ok := <-items: - if !ok { - exit = true - continue - } - attempts++ - - buf := &bytes.Buffer{} - - _, err := buf.ReadFrom(itemData.ToReader()) - if err != nil { - errs = support.WrapAndAppend( - itemData.UUID()+": byteReadError during RestoreDataCollection", - err, - errs, - ) - - continue - } - - err = exchange.RestoreExchangeObject(ctx, buf.Bytes(), category, policy, &gc.graphService, folderID, user) - - if err != nil { - // More information to be here - errs = support.WrapAndAppend( - itemData.UUID()+": failed to upload RestoreExchangeObject: "+service.String()+"-"+category.String(), - err, - errs, - ) - - continue - } - successes++ - } - } + switch selector.Service { + case selectors.ServiceExchange: + status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs) + case selectors.ServiceOneDrive: + status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs) + default: + err = errors.Errorf("restore data from service %s not supported", selector.Service.String()) } gc.incrementAwaitingMessages() - - status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs) gc.UpdateStatus(status) - return errs + return err } // createCollection - utility function that retrieves M365 diff --git a/src/internal/connector/graph_connector_test.go b/src/internal/connector/graph_connector_test.go index 68221ad58..e228e6cb4 100644 --- a/src/internal/connector/graph_connector_test.go +++ b/src/internal/connector/graph_connector_test.go @@ -12,8 +12,12 @@ import ( "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/exchange" + "github.com/alcionai/corso/src/internal/connector/mockconnector" "github.com/alcionai/corso/src/internal/connector/support" + "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/control" + "github.com/alcionai/corso/src/pkg/path" "github.com/alcionai/corso/src/pkg/selectors" ) @@ -362,3 +366,37 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteCalendar() { } } } + +func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() { + t := suite.T() + sel := selectors.NewExchangeRestore() + fullpath, err := path.Builder{}.Append("testing"). + ToDataLayerExchangePathForCategory( + suite.connector.tenant, + suite.user, + path.ContactsCategory, + false, + ) + + require.NoError(t, err) + aPath, err := path.Builder{}.Append("validator").ToDataLayerExchangePathForCategory( + suite.connector.tenant, + suite.user, + path.ContactsCategory, + false, + ) + require.NoError(t, err) + + dcs := mockconnector.NewMockContactCollection(fullpath, 3) + two := mockconnector.NewMockContactCollection(aPath, 2) + collections := []data.Collection{dcs, two} + ctx := context.Background() + connector := loadConnector(ctx, suite.T()) + dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat) + err = connector.RestoreDataCollections(ctx, sel.Selector, dest, collections) + assert.NoError(suite.T(), err) + + value := connector.AwaitStatus() + assert.Equal(t, value.FolderCount, 1) + suite.T().Log(value.String()) +} diff --git a/src/internal/connector/mockconnector/mock_data_collection.go b/src/internal/connector/mockconnector/mock_data_collection.go index 568f09cde..eaa7fc82a 100644 --- a/src/internal/connector/mockconnector/mock_data_collection.go +++ b/src/internal/connector/mockconnector/mock_data_collection.go @@ -3,6 +3,8 @@ package mockconnector import ( "bytes" "io" + "math/rand" + "strconv" "strings" "time" @@ -48,6 +50,42 @@ func NewMockExchangeCollection(pathRepresentation path.Path, numMessagesToReturn return c } +// NewMockExchangeDataCollection creates an data collection that will return the specified number of +// mock messages when iterated. Exchange type mail +func NewMockContactCollection(pathRepresentation path.Path, numMessagesToReturn int) *MockExchangeDataCollection { + c := &MockExchangeDataCollection{ + fullPath: pathRepresentation, + messageCount: numMessagesToReturn, + Data: [][]byte{}, + Names: []string{}, + } + + rand.Seed(time.Now().UnixNano()) + + middleNames := []string{ + "Argon", + "Bernard", + "Carleton", + "Daphenius", + "Ernesto", + "Farraday", + "Ghimley", + "Irgot", + "Jannes", + "Knox", + "Levi", + "Milton", + } + + for i := 0; i < c.messageCount; i++ { + // We can plug in whatever data we want here (can be an io.Reader to a test data file if needed) + c.Data = append(c.Data, GetMockContactBytes(middleNames[rand.Intn(len(middleNames))])) + c.Names = append(c.Names, uuid.NewString()) + } + + return c +} + func (medc *MockExchangeDataCollection) FullPath() path.Path { return medc.fullPath } @@ -133,15 +171,37 @@ func GetMockMessageBytes(subject string) []byte { } // GetMockContactBytes returns bytes for Contactable item. +// When hydrated: contact.GetGivenName() shows differences func GetMockContactBytes(middleName string) []byte { + phone := generatePhoneNumber() //nolint:lll - contact := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEOAADSEBNbUIB9RL6ePDeF3FIYAABS7DZnAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('foobar%408qzvrj.onmicrosoft.com')/contacts/$entity\",\"@odata.etag\":\"W/\\\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\\\"\",\"categories\":[],\"changeKey\":\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\",\"createdDateTime\":\"2019-08-04T06:55:33Z\",\"lastModifiedDateTime\":\"2019-08-04T06:55:33Z\",\"businessAddress\":{},\"businessPhones\":[],\"children\":[],\"displayName\":\"Santiago Quail\",\"emailAddresses\":[],\"fileAs\":\"Quail, Santiago\"," + - //nolint:lll - "\"givenName\":\"Santiago " + middleName + "\",\"homeAddress\":{},\"homePhones\":[],\"imAddresses\":[],\"otherAddress\":{},\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9FIYAAAAAAEOAAA=\",\"personalNotes\":\"\",\"surname\":\"Quail\"}" + contact := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEOAADSEBNbUIB9RL6ePDeF3FIYAABS7DZnAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('foobar%408qzvrj.onmicrosoft.com')/contacts/$entity\"," + + "\"@odata.etag\":\"W/\\\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\\\"\",\"categories\":[],\"changeKey\":\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\",\"createdDateTime\":\"2019-08-04T06:55:33Z\",\"lastModifiedDateTime\":\"2019-08-04T06:55:33Z\",\"businessAddress\":{},\"businessPhones\":[],\"children\":[]," + + "\"displayName\":\"Santiago Quail\",\"emailAddresses\":[],\"fileAs\":\"Quail, Santiago\",\"mobilePhone\": \"" + phone + "\"," + + "\"givenName\":\"Santiago\",\"homeAddress\":{},\"homePhones\":[],\"imAddresses\":[],\"otherAddress\":{},\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9FIYAAAAAAEOAAA=\",\"personalNotes\":\"\",\"middleName\":\"" + middleName + "\",\"surname\":\"Quail\"}" return []byte(contact) } +// generatePhoneNumber creates a random phone number +// @return string representation in format (xxx)xxx-xxxx +func generatePhoneNumber() string { + numbers := make([]string, 0) + + for i := 0; i < 10; i++ { + temp := rand.Intn(10) + value := strconv.Itoa(temp) + numbers = append(numbers, value) + } + + area := strings.Join(numbers[:3], "") + prefix := strings.Join(numbers[3:6], "") + suffix := strings.Join(numbers[6:], "") + phoneNo := "(" + area + ")" + prefix + "-" + suffix + + return phoneNo +} + // GetMockEventBytes returns test byte array representative of full Eventable item. func GetMockEventBytes(subject string) []byte { newTime := time.Now().AddDate(0, 0, 1)