Issue 181: GC Implement restore (#197)

Addition of restoreMessages(dc DataCollection) error for GraphConnector
Merge completes Issue #181. Test suites updated

src/internal/connector/graph_connector.go

@@ -3,9 +3,12 @@
 package connector
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 
 	az "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/alcionai/corso/internal/connector/support"
 	ka "github.com/microsoft/kiota-authentication-azure-go"
 	kw "github.com/microsoft/kiota-serialization-json-go"
 	msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
@@ -143,6 +146,59 @@ func optionsForMailFolders(moreOps []string) *msfolder.MailFoldersRequestBuilder
 	return options
 }
 
+// restoreMessages: Utility function to connect to M365 backstore
+// and upload messages from DataCollection.
+// FullPath: tenantId, userId, FolderId
+func (gc *GraphConnector) restoreMessages(dc DataCollection) error {
+	var errs error
+	// must be user.GetId(), PrimaryName no longer works 6-15-2022
+	user := dc.FullPath()[1]
+	for {
+		data, err := dc.NextItem()
+		if err == io.EOF {
+			break
+		}
+
+		buf := &bytes.Buffer{}
+		_, err = buf.ReadFrom(data.ToReader())
+		if err != nil {
+			errs = WrapAndAppend(data.UUID(), err, errs)
+			continue
+		}
+		message, err := support.CreateMessageFromBytes(buf.Bytes())
+		if err != nil {
+			errs = WrapAndAppend(data.UUID(), err, errs)
+			continue
+		}
+		clone := support.ToMessage(message)
+		address := dc.FullPath()[2]
+		valueId := "Integer 0x0E07"
+		enableValue := "4"
+		sv := models.NewSingleValueLegacyExtendedProperty()
+		sv.SetId(&valueId)
+		sv.SetValue(&enableValue)
+		svlep := []models.SingleValueLegacyExtendedPropertyable{sv}
+		clone.SetSingleValueExtendedProperties(svlep)
+		draft := false
+		clone.SetIsDraft(&draft)
+		sentMessage, err := gc.client.UsersById(user).MailFoldersById(address).Messages().Post(clone)
+		if err != nil {
+			details := ConnectorStackErrorTrace(err)
+			errs = WrapAndAppend(data.UUID()+": "+details, err, errs)
+			continue
+			// TODO: Add to retry Handler for the for failure
+		}
+
+		if sentMessage == nil && err == nil {
+			errs = WrapAndAppend(data.UUID(), errors.New("Message not Sent: Blocked by server"), errs)
+
+		}
+		// This completes the restore loop for a message..
+	}
+	return errs
+
+}
+
 // serializeMessages: Temp Function as place Holder until Collections have been added
 // to the GraphConnector struct.
 func (gc *GraphConnector) serializeMessages(user string, dc ExchangeDataCollection) (DataCollection, error) {

src/internal/connector/graph_connector_test.go

@@ -66,8 +66,30 @@ func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_ExchangeDataColl
 	exchangeData, err := suite.connector.ExchangeDataCollection("lidiah@8qzvrj.onmicrosoft.com")
 	assert.NotNil(suite.T(), exchangeData)
 	assert.Error(suite.T(), err) // TODO Remove after https://github.com/alcionai/corso/issues/140
+	if err != nil {
 		suite.T().Logf("Missing Data: %s\n", err.Error())
 	}
+	suite.T().Logf("Full PathData: %s\n", exchangeData.FullPath())
+}
+
+func (suite *GraphConnectorIntegrationSuite) TestGraphConnector_restoreMessages() {
+	user := "TEST_GRAPH_USER" // user.GetId()
+	file := "TEST_GRAPH_FILE" // Test file should be sent or received by the user
+	evs, err := ctesting.GetRequiredEnvVars(user, file)
+	if err != nil {
+		suite.T().Skipf("Environment not configured: %v\n", err)
+	}
+	bytes, err := ctesting.LoadAFile(evs[file]) // TEST_GRAPH_FILE should have a single Message && not present in target inbox
+	if err != nil {
+		suite.T().Skipf("Support file not accessible: %v\n", err)
+	}
+	ds := ExchangeData{id: "test", message: bytes}
+	edc := NewExchangeDataCollection("tenant", []string{"tenantId", evs[user], "Inbox"})
+	edc.PopulateCollection(ds)
+	edc.FinishPopulation()
+	err = suite.connector.restoreMessages(&edc)
+	assert.NoError(suite.T(), err)
+}
 
 func (suite *DiconnectedGraphConnectorSuite) TestBadConnection() {
 

src/internal/connector/support/m365Support.go

@@ -0,0 +1,33 @@
+package support
+
+import (
+	absser "github.com/microsoft/kiota-abstractions-go/serialization"
+	js "github.com/microsoft/kiota-serialization-json-go"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+)
+
+// CreateFromBytes helper function to initialize m365 object form bytes.
+// @param bytes -> source, createFunc -> abstract function for initialization
+func CreateFromBytes(bytes []byte, createFunc absser.ParsableFactory) (absser.Parsable, error) {
+	parseNode, err := js.NewJsonParseNodeFactory().GetRootParseNode("application/json", bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	anObject, err := parseNode.GetObjectValue(createFunc)
+	if err != nil {
+		return nil, err
+	}
+	return anObject, nil
+}
+
+// CreateMessageFromBytes function to transform bytes into Messageable object
+func CreateMessageFromBytes(bytes []byte) (models.Messageable, error) {
+
+	aMessage, err := CreateFromBytes(bytes, models.CreateMessageFromDiscriminatorValue)
+	if err != nil {
+		return nil, err
+	}
+	message := aMessage.(models.Messageable)
+	return message, nil
+}

src/internal/connector/support/m365Support_test.go

@@ -0,0 +1,61 @@
+package support
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+
+	ctesting "github.com/alcionai/corso/internal/testing"
+)
+
+type DataSupportSuite struct {
+	suite.Suite
+}
+
+const (
+	// File needs to be a single message .json
+	// Use: https://developer.microsoft.com/en-us/graph/graph-explorer for details
+	support_file = "CORSO_TEST_SUPPORT_FILE"
+)
+
+func TestDataSupportSuite(t *testing.T) {
+	err := ctesting.RunOnAny(support_file)
+	if err != nil {
+		t.Skipf("Skipping: %v\n", err)
+	}
+	suite.Run(t, new(DataSupportSuite))
+}
+
+func (suite *DataSupportSuite) TestCreateMessageFromBytes() {
+	bytes, err := ctesting.LoadAFile(os.Getenv(SUPPORT_FILE))
+	if err != nil {
+		suite.T().Errorf("Failed with %v\n", err)
+	}
+
+	table := []struct {
+		name        string
+		byteArray   []byte
+		checkError  assert.ErrorAssertionFunc
+		checkObject assert.ValueAssertionFunc
+	}{
+		{
+			name:        "Empty Bytes",
+			byteArray:   make([]byte, 0),
+			checkError:  assert.Error,
+			checkObject: assert.Nil,
+		},
+		{
+			name:        "aMessage bytes",
+			byteArray:   bytes,
+			checkError:  assert.NoError,
+			checkObject: assert.NotNil,
+		},
+	}
+	for _, test := range table {
+		result, err := CreateMessageFromBytes(test.byteArray)
+		test.checkError(suite.T(), err)
+		test.checkObject(suite.T(), result)
+	}
+}

src/internal/connector/support/m365Transform.go

@@ -0,0 +1,42 @@
+package support
+
+import (
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+)
+
+// ToMessage transfers all data from old message to new
+// message except for the messageId.
+func ToMessage(orig models.Messageable) *models.Message {
+	message := models.NewMessage()
+	message.SetSubject(orig.GetSubject())
+	message.SetBodyPreview(orig.GetBodyPreview())
+	message.SetBody(orig.GetBody())
+	message.SetSentDateTime(orig.GetSentDateTime())
+	message.SetReceivedDateTime(orig.GetReceivedDateTime())
+	message.SetToRecipients(orig.GetToRecipients())
+	message.SetSender(orig.GetSender())
+	message.SetInferenceClassification(orig.GetInferenceClassification())
+	message.SetBccRecipients(orig.GetBccRecipients())
+	message.SetCcRecipients(orig.GetCcRecipients())
+	message.SetReplyTo(orig.GetReplyTo())
+	message.SetFlag(orig.GetFlag())
+	message.SetHasAttachments(orig.GetHasAttachments())
+	message.SetParentFolderId(orig.GetParentFolderId())
+	message.SetConversationId(orig.GetConversationId())
+	message.SetExtensions(orig.GetExtensions())
+	message.SetFlag(orig.GetFlag())
+	message.SetFrom(orig.GetFrom())
+	message.SetImportance(orig.GetImportance())
+	message.SetInferenceClassification(orig.GetInferenceClassification())
+	message.SetInternetMessageId(orig.GetInternetMessageId())
+	message.SetInternetMessageHeaders(orig.GetInternetMessageHeaders())
+	message.SetIsDeliveryReceiptRequested(orig.GetIsDeliveryReceiptRequested())
+	message.SetIsRead(orig.GetIsRead())
+	message.SetIsReadReceiptRequested(orig.GetIsReadReceiptRequested())
+	message.SetParentFolderId(orig.GetParentFolderId())
+	message.SetMultiValueExtendedProperties(orig.GetMultiValueExtendedProperties())
+	message.SetUniqueBody(orig.GetUniqueBody())
+	message.SetWebLink(orig.GetWebLink())
+	return message
+
+}

src/internal/connector/support/m365Transform_test.go

@@ -0,0 +1,50 @@
+package support
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	ctesting "github.com/alcionai/corso/internal/testing"
+)
+
+type SupportTestSuite struct {
+	suite.Suite
+}
+
+const (
+	// File needs to be a single message .json
+	// Use: https://developer.microsoft.com/en-us/graph/graph-explorer for details
+	SUPPORT_FILE = "CORSO_TEST_SUPPORT_FILE"
+)
+
+func TestSupportTestSuite(t *testing.T) {
+	evs, err := ctesting.GetRequiredEnvVars(SUPPORT_FILE)
+	if err != nil {
+		t.Skipf("Env not configured: %v\n", err)
+	}
+	_, err = os.Stat(evs[SUPPORT_FILE])
+	if err != nil {
+		t.Skip("Test object not available: Module Skipped")
+	}
+	suite.Run(t, new(SupportTestSuite))
+}
+
+func (suite *SupportTestSuite) TestToMessage() {
+	bytes, err := ctesting.LoadAFile(os.Getenv(SUPPORT_FILE))
+	if err != nil {
+		suite.T().Errorf("Failed with %v\n", err)
+	}
+	require.NoError(suite.T(), err)
+	message, err := CreateMessageFromBytes(bytes)
+	require.NoError(suite.T(), err)
+	clone := ToMessage(message)
+	suite.Equal(message.GetBccRecipients(), clone.GetBccRecipients())
+	suite.Equal(message.GetSubject(), clone.GetSubject())
+	suite.Equal(message.GetSender(), clone.GetSender())
+	suite.Equal(message.GetSentDateTime(), clone.GetSentDateTime())
+	suite.NotEqual(message.GetId(), clone.GetId())
+
+}

src/internal/testing/loader.go

@@ -0,0 +1,30 @@
+package testing
+
+import (
+	"bufio"
+	"os"
+)
+
+func LoadAFile(aFile string) ([]byte, error) {
+	// Preserves '\n' of original file. Uses incremental version when file too large
+	bytes, err := os.ReadFile(aFile)
+	if err != nil {
+		f, err := os.Open(aFile)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		buffer := make([]byte, 0)
+		reader := bufio.NewScanner(f)
+		for reader.Scan() {
+			temp := reader.Bytes()
+			buffer = append(buffer, temp...)
+		}
+		aErr := reader.Err()
+		if aErr != nil {
+			return nil, aErr
+		}
+		return buffer, nil
+	}
+	return bytes, nil
+}