GC: Restore: Contact: Restore to single Folder (#972)
## Description The feature ensures that this Flat style directory only creates a single folder and restores all the contacts to the same folder during the restore process. ## Type of change <!--- Please check the type of change your PR introduces: ---> - [x] 🌻 Feature - [x] 🐛 Bugfix - [ ] 🗺️ Documentation - [ ] 🤖 Test - [ ] 💻 CI/Deployment - [ ] 🐹 Trivial/Minor ## Issue(s) <!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. --> *closes #880<issue> ## Test Plan <!-- How will this be tested prior to merging.--> - [ ] 💪 Manual - [ ] ⚡ Unit test - [ ] 💚 E2E
This commit is contained in:
Danny
GitHub
parent
507eab088d
commit
652bb9cd4a
@ -1,6 +1,7 @@
|
||||
package exchange
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
@ -10,6 +11,7 @@ import (
|
||||
"github.com/alcionai/corso/src/internal/common"
|
||||
"github.com/alcionai/corso/src/internal/connector/graph"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/logger"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
@ -113,9 +115,11 @@ func RestoreExchangeContact(
|
||||
|
||||
response, err := service.Client().UsersById(user).ContactFoldersById(destination).Contacts().Post(ctx, contact, nil)
|
||||
if err != nil {
|
||||
name := *contact.GetGivenName()
|
||||
|
||||
return errors.Wrap(
|
||||
err,
|
||||
"failure to create Contact during RestoreExchangeContact: "+
|
||||
"failure to create Contact during RestoreExchangeContact: "+name+" "+
|
||||
support.ConnectorStackErrorTrace(err),
|
||||
)
|
||||
}
|
||||
@ -242,3 +246,130 @@ func SendMailToBackStore(
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreExchangeDataCollections restores M365 objects in data.Collection to MSFT
|
||||
// store through GraphAPI.
|
||||
// @param dest: container destination to M365
|
||||
func RestoreExchangeDataCollections(
|
||||
ctx context.Context,
|
||||
gs graph.Service,
|
||||
dest control.RestoreDestination,
|
||||
dcs []data.Collection,
|
||||
) (*support.ConnectorOperationStatus, error) {
|
||||
var (
|
||||
pathCounter = map[string]bool{}
|
||||
attempts, successes int
|
||||
errs error
|
||||
folderID, root string
|
||||
isCancelled bool
|
||||
// TODO policy to be updated from external source after completion of refactoring
|
||||
policy = control.Copy
|
||||
)
|
||||
|
||||
for _, dc := range dcs {
|
||||
var (
|
||||
items = dc.Items()
|
||||
directory = dc.FullPath()
|
||||
service = directory.Service()
|
||||
category = directory.Category()
|
||||
user = directory.ResourceOwner()
|
||||
exit bool
|
||||
directoryCheckFunc = generateRestoreContainerFunc(gs, user, category, dest.ContainerName)
|
||||
)
|
||||
|
||||
folderID, root, errs = directoryCheckFunc(ctx, errs, directory.String(), root, pathCounter)
|
||||
if errs != nil { // assuming FailFast
|
||||
break
|
||||
}
|
||||
|
||||
if isCancelled {
|
||||
break
|
||||
}
|
||||
|
||||
for !exit {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs = support.WrapAndAppend("context cancelled", ctx.Err(), errs)
|
||||
isCancelled = true
|
||||
|
||||
case itemData, ok := <-items:
|
||||
if !ok {
|
||||
exit = true
|
||||
continue
|
||||
}
|
||||
attempts++
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
_, err := buf.ReadFrom(itemData.ToReader())
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
itemData.UUID()+": byteReadError during RestoreDataCollection",
|
||||
err,
|
||||
errs,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
err = RestoreExchangeObject(ctx, buf.Bytes(), category, policy, gs, folderID, user)
|
||||
if err != nil {
|
||||
// More information to be here
|
||||
errs = support.WrapAndAppend(
|
||||
itemData.UUID()+": failed to upload RestoreExchangeObject: "+service.String()+"-"+category.String(),
|
||||
err,
|
||||
errs,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
successes++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs)
|
||||
|
||||
return status, errs
|
||||
}
|
||||
|
||||
// generateRestoreContainerFunc utility function that holds logic for creating
|
||||
// Root Directory or necessary functions based on path.CategoryType
|
||||
func generateRestoreContainerFunc(
|
||||
gs graph.Service,
|
||||
user string,
|
||||
category path.CategoryType,
|
||||
destination string,
|
||||
) func(context.Context, error, string, string, map[string]bool) (string, string, error) {
|
||||
return func(
|
||||
ctx context.Context,
|
||||
errs error,
|
||||
dirName string,
|
||||
rootFolderID string,
|
||||
pathCounter map[string]bool,
|
||||
) (string, string, error) {
|
||||
var (
|
||||
folderID string
|
||||
err error
|
||||
)
|
||||
|
||||
if rootFolderID != "" && category == path.ContactsCategory {
|
||||
return rootFolderID, rootFolderID, errs
|
||||
}
|
||||
|
||||
if !pathCounter[dirName] {
|
||||
pathCounter[dirName] = true
|
||||
|
||||
folderID, err = GetRestoreContainer(ctx, gs, user, category, destination)
|
||||
if err != nil {
|
||||
return "", "", support.WrapAndAppend(user+" failure during preprocessing ", err, errs)
|
||||
}
|
||||
|
||||
if rootFolderID == "" {
|
||||
rootFolderID = folderID
|
||||
}
|
||||
}
|
||||
|
||||
return folderID, rootFolderID, nil
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
package connector
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
@ -241,122 +240,32 @@ func (gc *GraphConnector) ExchangeDataCollection(
|
||||
}
|
||||
|
||||
// RestoreDataCollections restores data from the specified collections
|
||||
// into M365
|
||||
// into M365 using the GraphAPI.
|
||||
// SideEffect: gc.status is updated at the completion of operation
|
||||
func (gc *GraphConnector) RestoreDataCollections(
|
||||
ctx context.Context,
|
||||
selector selectors.Selector,
|
||||
dest control.RestoreDestination,
|
||||
dcs []data.Collection,
|
||||
) error {
|
||||
switch selector.Service {
|
||||
case selectors.ServiceExchange:
|
||||
return gc.RestoreExchangeDataCollections(ctx, dest, dcs)
|
||||
case selectors.ServiceOneDrive:
|
||||
status, err := onedrive.RestoreCollections(ctx, gc, dest, dcs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gc.incrementAwaitingMessages()
|
||||
|
||||
gc.UpdateStatus(status)
|
||||
|
||||
return nil
|
||||
default:
|
||||
return errors.Errorf("restore data from service %s not supported", selector.Service.String())
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreDataCollections restores data from the specified collections
|
||||
// into M365
|
||||
func (gc *GraphConnector) RestoreExchangeDataCollections(
|
||||
ctx context.Context,
|
||||
dest control.RestoreDestination,
|
||||
dcs []data.Collection,
|
||||
) error {
|
||||
var (
|
||||
pathCounter = map[string]bool{}
|
||||
attempts, successes int
|
||||
errs error
|
||||
folderID string
|
||||
// TODO policy to be updated from external source after completion of refactoring
|
||||
policy = control.Copy
|
||||
status *support.ConnectorOperationStatus
|
||||
err error
|
||||
)
|
||||
|
||||
for _, dc := range dcs {
|
||||
var (
|
||||
items = dc.Items()
|
||||
directory = dc.FullPath()
|
||||
service = directory.Service()
|
||||
category = directory.Category()
|
||||
user = directory.ResourceOwner()
|
||||
exit bool
|
||||
)
|
||||
|
||||
if _, ok := pathCounter[directory.String()]; !ok {
|
||||
pathCounter[directory.String()] = true
|
||||
|
||||
folderID, errs = exchange.GetRestoreContainer(
|
||||
ctx,
|
||||
&gc.graphService,
|
||||
user,
|
||||
category,
|
||||
dest.ContainerName,
|
||||
)
|
||||
|
||||
if errs != nil {
|
||||
fmt.Println("RestoreContainer Failed")
|
||||
return errs
|
||||
}
|
||||
}
|
||||
|
||||
for !exit {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return support.WrapAndAppend("context cancelled", ctx.Err(), errs)
|
||||
case itemData, ok := <-items:
|
||||
if !ok {
|
||||
exit = true
|
||||
continue
|
||||
}
|
||||
attempts++
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
_, err := buf.ReadFrom(itemData.ToReader())
|
||||
if err != nil {
|
||||
errs = support.WrapAndAppend(
|
||||
itemData.UUID()+": byteReadError during RestoreDataCollection",
|
||||
err,
|
||||
errs,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
err = exchange.RestoreExchangeObject(ctx, buf.Bytes(), category, policy, &gc.graphService, folderID, user)
|
||||
|
||||
if err != nil {
|
||||
// More information to be here
|
||||
errs = support.WrapAndAppend(
|
||||
itemData.UUID()+": failed to upload RestoreExchangeObject: "+service.String()+"-"+category.String(),
|
||||
err,
|
||||
errs,
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
successes++
|
||||
}
|
||||
}
|
||||
switch selector.Service {
|
||||
case selectors.ServiceExchange:
|
||||
status, err = exchange.RestoreExchangeDataCollections(ctx, gc.graphService, dest, dcs)
|
||||
case selectors.ServiceOneDrive:
|
||||
status, err = onedrive.RestoreCollections(ctx, gc, dest, dcs)
|
||||
default:
|
||||
err = errors.Errorf("restore data from service %s not supported", selector.Service.String())
|
||||
}
|
||||
|
||||
gc.incrementAwaitingMessages()
|
||||
|
||||
status := support.CreateStatus(ctx, support.Restore, attempts, successes, len(pathCounter), errs)
|
||||
gc.UpdateStatus(status)
|
||||
|
||||
return errs
|
||||
return err
|
||||
}
|
||||
|
||||
// createCollection - utility function that retrieves M365
|
||||
|
||||
@ -12,8 +12,12 @@ import (
|
||||
|
||||
"github.com/alcionai/corso/src/internal/common"
|
||||
"github.com/alcionai/corso/src/internal/connector/exchange"
|
||||
"github.com/alcionai/corso/src/internal/connector/mockconnector"
|
||||
"github.com/alcionai/corso/src/internal/connector/support"
|
||||
"github.com/alcionai/corso/src/internal/data"
|
||||
"github.com/alcionai/corso/src/internal/tester"
|
||||
"github.com/alcionai/corso/src/pkg/control"
|
||||
"github.com/alcionai/corso/src/pkg/path"
|
||||
"github.com/alcionai/corso/src/pkg/selectors"
|
||||
)
|
||||
|
||||
@ -362,3 +366,37 @@ func (suite *GraphConnectorIntegrationSuite) TestCreateAndDeleteCalendar() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (suite *GraphConnectorIntegrationSuite) TestRestoreContact() {
|
||||
t := suite.T()
|
||||
sel := selectors.NewExchangeRestore()
|
||||
fullpath, err := path.Builder{}.Append("testing").
|
||||
ToDataLayerExchangePathForCategory(
|
||||
suite.connector.tenant,
|
||||
suite.user,
|
||||
path.ContactsCategory,
|
||||
false,
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
aPath, err := path.Builder{}.Append("validator").ToDataLayerExchangePathForCategory(
|
||||
suite.connector.tenant,
|
||||
suite.user,
|
||||
path.ContactsCategory,
|
||||
false,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
dcs := mockconnector.NewMockContactCollection(fullpath, 3)
|
||||
two := mockconnector.NewMockContactCollection(aPath, 2)
|
||||
collections := []data.Collection{dcs, two}
|
||||
ctx := context.Background()
|
||||
connector := loadConnector(ctx, suite.T())
|
||||
dest := control.DefaultRestoreDestination(common.SimpleDateTimeFormat)
|
||||
err = connector.RestoreDataCollections(ctx, sel.Selector, dest, collections)
|
||||
assert.NoError(suite.T(), err)
|
||||
|
||||
value := connector.AwaitStatus()
|
||||
assert.Equal(t, value.FolderCount, 1)
|
||||
suite.T().Log(value.String())
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ package mockconnector
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -48,6 +50,42 @@ func NewMockExchangeCollection(pathRepresentation path.Path, numMessagesToReturn
|
||||
return c
|
||||
}
|
||||
|
||||
// NewMockExchangeDataCollection creates an data collection that will return the specified number of
|
||||
// mock messages when iterated. Exchange type mail
|
||||
func NewMockContactCollection(pathRepresentation path.Path, numMessagesToReturn int) *MockExchangeDataCollection {
|
||||
c := &MockExchangeDataCollection{
|
||||
fullPath: pathRepresentation,
|
||||
messageCount: numMessagesToReturn,
|
||||
Data: [][]byte{},
|
||||
Names: []string{},
|
||||
}
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
|
||||
middleNames := []string{
|
||||
"Argon",
|
||||
"Bernard",
|
||||
"Carleton",
|
||||
"Daphenius",
|
||||
"Ernesto",
|
||||
"Farraday",
|
||||
"Ghimley",
|
||||
"Irgot",
|
||||
"Jannes",
|
||||
"Knox",
|
||||
"Levi",
|
||||
"Milton",
|
||||
}
|
||||
|
||||
for i := 0; i < c.messageCount; i++ {
|
||||
// We can plug in whatever data we want here (can be an io.Reader to a test data file if needed)
|
||||
c.Data = append(c.Data, GetMockContactBytes(middleNames[rand.Intn(len(middleNames))]))
|
||||
c.Names = append(c.Names, uuid.NewString())
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (medc *MockExchangeDataCollection) FullPath() path.Path {
|
||||
return medc.fullPath
|
||||
}
|
||||
@ -133,15 +171,37 @@ func GetMockMessageBytes(subject string) []byte {
|
||||
}
|
||||
|
||||
// GetMockContactBytes returns bytes for Contactable item.
|
||||
// When hydrated: contact.GetGivenName() shows differences
|
||||
func GetMockContactBytes(middleName string) []byte {
|
||||
phone := generatePhoneNumber()
|
||||
//nolint:lll
|
||||
contact := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEOAADSEBNbUIB9RL6ePDeF3FIYAABS7DZnAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('foobar%408qzvrj.onmicrosoft.com')/contacts/$entity\",\"@odata.etag\":\"W/\\\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\\\"\",\"categories\":[],\"changeKey\":\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\",\"createdDateTime\":\"2019-08-04T06:55:33Z\",\"lastModifiedDateTime\":\"2019-08-04T06:55:33Z\",\"businessAddress\":{},\"businessPhones\":[],\"children\":[],\"displayName\":\"Santiago Quail\",\"emailAddresses\":[],\"fileAs\":\"Quail, Santiago\"," +
|
||||
//nolint:lll
|
||||
"\"givenName\":\"Santiago " + middleName + "\",\"homeAddress\":{},\"homePhones\":[],\"imAddresses\":[],\"otherAddress\":{},\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9FIYAAAAAAEOAAA=\",\"personalNotes\":\"\",\"surname\":\"Quail\"}"
|
||||
contact := "{\"id\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwBGAAAAAADCNgjhM9QmQYWNcI7hCpPrBwDSEBNbUIB9RL6ePDeF3FIYAAAAAAEOAADSEBNbUIB9RL6ePDeF3FIYAABS7DZnAAA=\",\"@odata.context\":\"https://graph.microsoft.com/v1.0/$metadata#users('foobar%408qzvrj.onmicrosoft.com')/contacts/$entity\"," +
|
||||
"\"@odata.etag\":\"W/\\\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\\\"\",\"categories\":[],\"changeKey\":\"EQAAABYAAADSEBNbUIB9RL6ePDeF3FIYAABSx4Tr\",\"createdDateTime\":\"2019-08-04T06:55:33Z\",\"lastModifiedDateTime\":\"2019-08-04T06:55:33Z\",\"businessAddress\":{},\"businessPhones\":[],\"children\":[]," +
|
||||
"\"displayName\":\"Santiago Quail\",\"emailAddresses\":[],\"fileAs\":\"Quail, Santiago\",\"mobilePhone\": \"" + phone + "\"," +
|
||||
"\"givenName\":\"Santiago\",\"homeAddress\":{},\"homePhones\":[],\"imAddresses\":[],\"otherAddress\":{},\"parentFolderId\":\"AAMkAGZmNjNlYjI3LWJlZWYtNGI4Mi04YjMyLTIxYThkNGQ4NmY1MwAuAAAAAADCNgjhM9FIYAAAAAAEOAAA=\",\"personalNotes\":\"\",\"middleName\":\"" + middleName + "\",\"surname\":\"Quail\"}"
|
||||
|
||||
return []byte(contact)
|
||||
}
|
||||
|
||||
// generatePhoneNumber creates a random phone number
|
||||
// @return string representation in format (xxx)xxx-xxxx
|
||||
func generatePhoneNumber() string {
|
||||
numbers := make([]string, 0)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
temp := rand.Intn(10)
|
||||
value := strconv.Itoa(temp)
|
||||
numbers = append(numbers, value)
|
||||
}
|
||||
|
||||
area := strings.Join(numbers[:3], "")
|
||||
prefix := strings.Join(numbers[3:6], "")
|
||||
suffix := strings.Join(numbers[6:], "")
|
||||
phoneNo := "(" + area + ")" + prefix + "-" + suffix
|
||||
|
||||
return phoneNo
|
||||
}
|
||||
|
||||
// GetMockEventBytes returns test byte array representative of full Eventable item.
|
||||
func GetMockEventBytes(subject string) []byte {
|
||||
newTime := time.Now().AddDate(0, 0, 1)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user