Sanity tests for email exports (#4741)

<!-- PR description-->

---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [ ] 🕐 Yes, but in a later PR
- [x] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [ ] 🌻 Feature
- [ ] 🐛 Bugfix
- [ ] 🗺️ Documentation
- [x] 🤖 Supportability/Tests
- [ ] 💻 CI/Deployment
- [ ] 🧹 Tech Debt/Cleanup

#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* closes https://github.com/alcionai/corso/issues/4652

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [ ] 💪 Manual
- [ ] ⚡ Unit test
- [ ] 💚 E2E

.github/workflows/sanity-test.yaml

@@ -196,6 +196,7 @@ jobs:
           restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup
         timeout-minutes: 30
@@ -209,6 +210,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Non delta backup
         timeout-minutes: 30
@@ -222,6 +224,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
       - name: Exchange - Incremental backup after non-delta
         timeout-minutes: 30
@@ -235,6 +238,7 @@ jobs:
           restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
           backup-id: ${{ steps.exchange-backup.outputs.backup-id }}
           log-dir: ${{ env.CORSO_LOG_DIR }}
+          with-export: true
 
 
 ##########################################################################################################################################

src/cmd/sanity_test/common/filepath.go

@@ -58,6 +58,7 @@ func BuildFilepathSanitree(
 				Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{},
 			}
 		} else {
+			node.CountLeaves++
 			node.Leaves[info.Name()] = &Sanileaf[fs.FileInfo, fs.FileInfo]{
 				Parent: node,
 				Self:   info,

src/cmd/sanity_test/export/exchange.go

@@ -0,0 +1,64 @@
+package export
+
+import (
+	"context"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	"github.com/alcionai/clues"
+	"github.com/microsoftgraph/msgraph-sdk-go/models"
+
+	"github.com/alcionai/corso/src/cmd/sanity_test/common"
+	"github.com/alcionai/corso/src/cmd/sanity_test/restore"
+	"github.com/alcionai/corso/src/pkg/services/m365/api"
+)
+
+func CheckEmailExport(
+	ctx context.Context,
+	ac api.Client,
+	envs common.Envs,
+) {
+	sourceTree := restore.BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
+
+	emailsExportDir := filepath.Join(envs.RestoreContainer, "Emails")
+	exportedTree := common.BuildFilepathSanitree(ctx, emailsExportDir)
+
+	ctx = clues.Add(
+		ctx,
+		"export_container_id", exportedTree.ID,
+		"export_container_name", exportedTree.Name,
+		"source_container_id", sourceTree.ID,
+		"source_container_name", sourceTree.Name)
+
+	comparator := func(
+		ctx context.Context,
+		expect *common.Sanitree[models.MailFolderable, any],
+		result *common.Sanitree[fs.FileInfo, fs.FileInfo],
+	) {
+		modifiedExpectedLeaves := map[string]*common.Sanileaf[models.MailFolderable, any]{}
+		modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{}
+
+		for key, val := range expect.Leaves {
+			val.Size = 0 // we cannot match up sizes
+			modifiedExpectedLeaves[key] = val
+		}
+
+		for key, val := range result.Leaves {
+			fixedName := strings.TrimSuffix(key, ".eml")
+			val.Size = 0
+
+			modifiedResultLeaves[fixedName] = val
+		}
+
+		common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil)
+	}
+
+	common.CompareDiffTrees(
+		ctx,
+		sourceTree,
+		exportedTree.Children[envs.SourceContainer],
+		comparator)
+
+	common.Infof(ctx, "Success")
+}

src/cmd/sanity_test/restore/exchange.go

@@ -19,8 +19,8 @@ func CheckEmailRestoration(
 	ac api.Client,
 	envs common.Envs,
 ) {
-	restoredTree := buildSanitree(ctx, ac, envs.UserID, envs.RestoreContainer)
+	restoredTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.RestoreContainer)
-	sourceTree := buildSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
+	sourceTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer)
 
 	ctx = clues.Add(
 		ctx,
@@ -39,7 +39,7 @@ func CheckEmailRestoration(
 	common.Infof(ctx, "Success")
 }
 
-func buildSanitree(
+func BuildEmailSanitree(
 	ctx context.Context,
 	ac api.Client,
 	userID, folderName string,
@@ -69,9 +69,37 @@ func buildSanitree(
 		ID:          ptr.Val(mmf.GetId()),
 		Name:        ptr.Val(mmf.GetDisplayName()),
 		CountLeaves: int(ptr.Val(mmf.GetTotalItemCount())),
+		Leaves:      map[string]*common.Sanileaf[models.MailFolderable, any]{},
 		Children:    map[string]*common.Sanitree[models.MailFolderable, any]{},
 	}
 
+	mails, err := ac.Mail().GetItemsInContainer(
+		ctx,
+		userID,
+		root.ID)
+	if err != nil {
+		common.Fatal(ctx, "getting child containers", err)
+	}
+
+	if len(mails) != root.CountLeaves {
+		common.Fatal(
+			ctx,
+			"mails count mismatch",
+			clues.New("mail message count mismatch from API"))
+	}
+
+	for _, mail := range mails {
+		m := &common.Sanileaf[models.MailFolderable, any]{
+			Parent: root,
+			Self:   mail,
+			ID:     ptr.Val(mail.GetId()),
+			Name:   ptr.Val(mail.GetSubject()),
+			Size:   int64(len(ptr.Val(mail.GetBody().GetContent()))),
+		}
+
+		root.Leaves[m.ID] = m
+	}
+
 	recursivelyBuildTree(ctx, ac, root, userID, root.Name+"/")
 
 	return root
@@ -94,6 +122,11 @@ func recursivelyBuildTree(
 	}
 
 	for _, child := range childFolders {
+		if int(ptr.Val(child.GetTotalItemCount())) == 0 {
+			common.Infof(ctx, "skipped empty folder: %s/%s", location, ptr.Val(child.GetDisplayName()))
+			continue
+		}
+
 		c := &common.Sanitree[models.MailFolderable, any]{
 			Parent:      stree,
 			Self:        child,

src/cmd/sanity_test/sanity_tests.go

@@ -65,6 +65,7 @@ func main() {
 
 	expCMD.AddCommand(exportOneDriveCMD())
 	expCMD.AddCommand(exportSharePointCMD())
+	expCMD.AddCommand(exportExchangeCMD())
 	expCMD.AddCommand(exportGroupsCMD())
 	root.AddCommand(expCMD)
 
@@ -176,6 +177,29 @@ func sanityTestExportSharePoint(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func exportExchangeCMD() *cobra.Command {
+	return &cobra.Command{
+		Use:               "exchange",
+		Short:             "run the exchange export sanity tests",
+		DisableAutoGenTag: true,
+		RunE:              sanityTestExportExchange,
+	}
+}
+
+func sanityTestExportExchange(cmd *cobra.Command, args []string) error {
+	ctx := common.SetDebug(cmd.Context())
+	envs := common.EnvVars(ctx)
+
+	ac, err := common.GetAC()
+	if err != nil {
+		return print.Only(ctx, err)
+	}
+
+	export.CheckEmailExport(ctx, ac, envs)
+
+	return nil
+}
+
 // ---------------------------------------------------------------------------
 // service commands - restore
 // ---------------------------------------------------------------------------

src/pkg/services/m365/api/mail_pager.go

@@ -147,6 +147,21 @@ func (c Mail) GetItemsInContainerByCollisionKey(
 	return m, nil
 }
 
+func (c Mail) GetItemsInContainer(
+	ctx context.Context,
+	userID, containerID string,
+) ([]models.Messageable, error) {
+	ctx = clues.Add(ctx, "container_id", containerID)
+	pager := c.NewMailPager(userID, containerID, false)
+
+	items, err := pagers.BatchEnumerateItems(ctx, pager)
+	if err != nil {
+		return nil, graph.Wrap(ctx, err, "enumerating mails")
+	}
+
+	return items, nil
+}
+
 func (c Mail) GetItemIDsInContainer(
 	ctx context.Context,
 	userID, containerID string,