diff --git a/.github/workflows/sanity-test.yaml b/.github/workflows/sanity-test.yaml index 9f7bbd61e..25c9eeb40 100644 --- a/.github/workflows/sanity-test.yaml +++ b/.github/workflows/sanity-test.yaml @@ -196,6 +196,7 @@ jobs: restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}' restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}' log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true - name: Exchange - Incremental backup timeout-minutes: 30 @@ -209,6 +210,7 @@ jobs: restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}' backup-id: ${{ steps.exchange-backup.outputs.backup-id }} log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true - name: Exchange - Non delta backup timeout-minutes: 30 @@ -222,6 +224,7 @@ jobs: restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}' backup-id: ${{ steps.exchange-backup.outputs.backup-id }} log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true - name: Exchange - Incremental backup after non-delta timeout-minutes: 30 @@ -235,6 +238,7 @@ jobs: restore-container: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}' backup-id: ${{ steps.exchange-backup.outputs.backup-id }} log-dir: ${{ env.CORSO_LOG_DIR }} + with-export: true ########################################################################################################################################## diff --git a/src/cmd/sanity_test/common/filepath.go b/src/cmd/sanity_test/common/filepath.go index 637a7c96a..1b2c01a86 100644 --- a/src/cmd/sanity_test/common/filepath.go +++ b/src/cmd/sanity_test/common/filepath.go @@ -58,6 +58,7 @@ func BuildFilepathSanitree( Children: map[string]*Sanitree[fs.FileInfo, fs.FileInfo]{}, } } else { + node.CountLeaves++ node.Leaves[info.Name()] = &Sanileaf[fs.FileInfo, fs.FileInfo]{ Parent: node, Self: info, diff --git a/src/cmd/sanity_test/export/exchange.go b/src/cmd/sanity_test/export/exchange.go new file mode 100644 index 000000000..52700689e --- /dev/null +++ b/src/cmd/sanity_test/export/exchange.go @@ -0,0 +1,64 @@ +package export + +import ( + "context" + "io/fs" + "path/filepath" + "strings" + + "github.com/alcionai/clues" + "github.com/microsoftgraph/msgraph-sdk-go/models" + + "github.com/alcionai/corso/src/cmd/sanity_test/common" + "github.com/alcionai/corso/src/cmd/sanity_test/restore" + "github.com/alcionai/corso/src/pkg/services/m365/api" +) + +func CheckEmailExport( + ctx context.Context, + ac api.Client, + envs common.Envs, +) { + sourceTree := restore.BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer) + + emailsExportDir := filepath.Join(envs.RestoreContainer, "Emails") + exportedTree := common.BuildFilepathSanitree(ctx, emailsExportDir) + + ctx = clues.Add( + ctx, + "export_container_id", exportedTree.ID, + "export_container_name", exportedTree.Name, + "source_container_id", sourceTree.ID, + "source_container_name", sourceTree.Name) + + comparator := func( + ctx context.Context, + expect *common.Sanitree[models.MailFolderable, any], + result *common.Sanitree[fs.FileInfo, fs.FileInfo], + ) { + modifiedExpectedLeaves := map[string]*common.Sanileaf[models.MailFolderable, any]{} + modifiedResultLeaves := map[string]*common.Sanileaf[fs.FileInfo, fs.FileInfo]{} + + for key, val := range expect.Leaves { + val.Size = 0 // we cannot match up sizes + modifiedExpectedLeaves[key] = val + } + + for key, val := range result.Leaves { + fixedName := strings.TrimSuffix(key, ".eml") + val.Size = 0 + + modifiedResultLeaves[fixedName] = val + } + + common.CompareLeaves(ctx, expect.Leaves, modifiedResultLeaves, nil) + } + + common.CompareDiffTrees( + ctx, + sourceTree, + exportedTree.Children[envs.SourceContainer], + comparator) + + common.Infof(ctx, "Success") +} diff --git a/src/cmd/sanity_test/restore/exchange.go b/src/cmd/sanity_test/restore/exchange.go index a6d22219f..5b7e3d63f 100644 --- a/src/cmd/sanity_test/restore/exchange.go +++ b/src/cmd/sanity_test/restore/exchange.go @@ -19,8 +19,8 @@ func CheckEmailRestoration( ac api.Client, envs common.Envs, ) { - restoredTree := buildSanitree(ctx, ac, envs.UserID, envs.RestoreContainer) - sourceTree := buildSanitree(ctx, ac, envs.UserID, envs.SourceContainer) + restoredTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.RestoreContainer) + sourceTree := BuildEmailSanitree(ctx, ac, envs.UserID, envs.SourceContainer) ctx = clues.Add( ctx, @@ -39,7 +39,7 @@ func CheckEmailRestoration( common.Infof(ctx, "Success") } -func buildSanitree( +func BuildEmailSanitree( ctx context.Context, ac api.Client, userID, folderName string, @@ -69,9 +69,37 @@ func buildSanitree( ID: ptr.Val(mmf.GetId()), Name: ptr.Val(mmf.GetDisplayName()), CountLeaves: int(ptr.Val(mmf.GetTotalItemCount())), + Leaves: map[string]*common.Sanileaf[models.MailFolderable, any]{}, Children: map[string]*common.Sanitree[models.MailFolderable, any]{}, } + mails, err := ac.Mail().GetItemsInContainer( + ctx, + userID, + root.ID) + if err != nil { + common.Fatal(ctx, "getting child containers", err) + } + + if len(mails) != root.CountLeaves { + common.Fatal( + ctx, + "mails count mismatch", + clues.New("mail message count mismatch from API")) + } + + for _, mail := range mails { + m := &common.Sanileaf[models.MailFolderable, any]{ + Parent: root, + Self: mail, + ID: ptr.Val(mail.GetId()), + Name: ptr.Val(mail.GetSubject()), + Size: int64(len(ptr.Val(mail.GetBody().GetContent()))), + } + + root.Leaves[m.ID] = m + } + recursivelyBuildTree(ctx, ac, root, userID, root.Name+"/") return root @@ -94,6 +122,11 @@ func recursivelyBuildTree( } for _, child := range childFolders { + if int(ptr.Val(child.GetTotalItemCount())) == 0 { + common.Infof(ctx, "skipped empty folder: %s/%s", location, ptr.Val(child.GetDisplayName())) + continue + } + c := &common.Sanitree[models.MailFolderable, any]{ Parent: stree, Self: child, diff --git a/src/cmd/sanity_test/sanity_tests.go b/src/cmd/sanity_test/sanity_tests.go index e6b742cde..45c5f5b53 100644 --- a/src/cmd/sanity_test/sanity_tests.go +++ b/src/cmd/sanity_test/sanity_tests.go @@ -65,6 +65,7 @@ func main() { expCMD.AddCommand(exportOneDriveCMD()) expCMD.AddCommand(exportSharePointCMD()) + expCMD.AddCommand(exportExchangeCMD()) expCMD.AddCommand(exportGroupsCMD()) root.AddCommand(expCMD) @@ -176,6 +177,29 @@ func sanityTestExportSharePoint(cmd *cobra.Command, args []string) error { return nil } +func exportExchangeCMD() *cobra.Command { + return &cobra.Command{ + Use: "exchange", + Short: "run the exchange export sanity tests", + DisableAutoGenTag: true, + RunE: sanityTestExportExchange, + } +} + +func sanityTestExportExchange(cmd *cobra.Command, args []string) error { + ctx := common.SetDebug(cmd.Context()) + envs := common.EnvVars(ctx) + + ac, err := common.GetAC() + if err != nil { + return print.Only(ctx, err) + } + + export.CheckEmailExport(ctx, ac, envs) + + return nil +} + // --------------------------------------------------------------------------- // service commands - restore // --------------------------------------------------------------------------- diff --git a/src/pkg/services/m365/api/mail_pager.go b/src/pkg/services/m365/api/mail_pager.go index f99f55b93..bbfb14419 100644 --- a/src/pkg/services/m365/api/mail_pager.go +++ b/src/pkg/services/m365/api/mail_pager.go @@ -147,6 +147,21 @@ func (c Mail) GetItemsInContainerByCollisionKey( return m, nil } +func (c Mail) GetItemsInContainer( + ctx context.Context, + userID, containerID string, +) ([]models.Messageable, error) { + ctx = clues.Add(ctx, "container_id", containerID) + pager := c.NewMailPager(userID, containerID, false) + + items, err := pagers.BatchEnumerateItems(ctx, pager) + if err != nil { + return nil, graph.Wrap(ctx, err, "enumerating mails") + } + + return items, nil +} + func (c Mail) GetItemIDsInContainer( ctx context.Context, userID, containerID string,