larryh/corso
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sanity test refactor (#4370)

Browse Source 
refactoring the sanity tests with three goals:
1. move from env vars to cli commands so that unsupported commands fail loudly.
2. set up support for groups restore and export testing.
3. introduce some code re-use throughout.

---

#### Does this PR need a docs update or release note?

- [x] ⛔ No

#### Type of change

- [x] 🤖 Supportability/Tests

#### Issue(s)

* #3988

#### Test Plan

- [x] 💚 E2E
This commit is contained in:
Keepers 2023-09-28 17:53:15 -06:00 committed by GitHub
parent a5f93f7a10
commit 9e0d464854
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 716 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
.github/actions/backup-restore-test/action.yml vendored
View File

@ -45,6 +45,9 @@ runs:
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Backup ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-${{ inputs.service }}-${{inputs.kind }}.log
./corso backup create '${{ inputs.service }}' \
@ -61,6 +64,9 @@ runs:
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Restore ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
./corso restore '${{ inputs.service }}' \
@ -85,11 +91,14 @@ runs:
SANITY_TEST_KIND: restore
SANITY_TEST_FOLDER: ${{ steps.restore.outputs.result }}
SANITY_TEST_SERVICE: ${{ inputs.service }}
TEST_DATA: ${{ inputs.test-folder }}
BASE_BACKUP: ${{ inputs.base-backup }}
SANITY_TEST_DATA: ${{ inputs.test-folder }}
SANITY_BASE_BACKUP: ${{ inputs.base-backup }}
run: |
echo "---------------------------"
echo Sanity Test Restore ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
./sanity-test
./sanity-test restore ${{ inputs.service }}
- name: Export ${{ inputs.service }} ${{ inputs.kind }}
if: inputs.with-export == true
@ -97,6 +106,9 @@ runs:
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Export ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
./corso export '${{ inputs.service }}' \
@ -116,11 +128,14 @@ runs:
SANITY_TEST_KIND: export
SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}
SANITY_TEST_SERVICE: ${{ inputs.service }}
TEST_DATA: ${{ inputs.test-folder }}
BASE_BACKUP: ${{ inputs.base-backup }}
SANITY_TEST_DATA: ${{ inputs.test-folder }}
SANITY_BASE_BACKUP: ${{ inputs.base-backup }}
run: |
echo "---------------------------"
echo Sanity-Test Export ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
./sanity-test
./sanity-test export ${{ inputs.service }}
- name: Export archive ${{ inputs.service }} ${{ inputs.kind }}
if: inputs.with-export == true
@ -128,6 +143,9 @@ runs:
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Export Archive ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-restore-${{ inputs.service }}-${{inputs.kind }}.log
./corso export '${{ inputs.service }}' \
@ -150,16 +168,22 @@ runs:
SANITY_TEST_KIND: export
SANITY_TEST_FOLDER: /tmp/export-${{ inputs.service }}-${{inputs.kind }}-unzipped
SANITY_TEST_SERVICE: ${{ inputs.service }}
TEST_DATA: ${{ inputs.test-folder }}
BASE_BACKUP: ${{ inputs.base-backup }}
SANITY_TEST_DATA: ${{ inputs.test-folder }}
SANITY_BASE_BACKUP: ${{ inputs.base-backup }}
run: |
echo "---------------------------"
echo Sanity-Test Export Archive ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-validate-${{ inputs.service }}-${{inputs.kind }}.log
./sanity-test
./sanity-test export ${{ inputs.service }}
- name: List ${{ inputs.service }} ${{ inputs.kind }}
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Backup list ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-${{ inputs.service }}-${{inputs.kind }}.log
./corso backup list ${{ inputs.service }} \
@ -178,6 +202,9 @@ runs:
shell: bash
working-directory: src
run: |
echo "---------------------------"
echo Backup List w/ Backup ${{ inputs.service }} ${{ inputs.kind }}
echo "---------------------------"
set -euo pipefail
CORSO_LOG_FILE=${{ inputs.log-dir }}/gotest-backup-list-single-${{ inputs.service }}-${{inputs.kind }}.log
./corso backup list ${{ inputs.service }} \
@ -193,7 +220,13 @@ runs:
exit 1
fi
# Upload the original go test output as an artifact for later review.
- if: always()
shell: bash
run: |
echo "---------------------------"
echo Logging Results
echo "---------------------------"
- name: Upload test log
if: always()
uses: actions/upload-artifact@v3

4
.github/actions/slack-message/action.yml vendored
View File

@ -31,7 +31,7 @@ runs:
- name: use url or blank val
shell: bash
run: |
echo "STEP=${{ github.action || '' }}" >> $GITHUB_ENV
echo "STEP=${{ env.trimmed_ref || '' }}" >> $GITHUB_ENV
echo "JOB=${{ github.job || '' }}" >> $GITHUB_ENV
echo "LOGS=${{ github.run_id && env.logurl || '-' }}" >> $GITHUB_ENV
echo "COMMIT=${{ github.sha && env.commiturl || '-' }}" >> $GITHUB_ENV
@ -51,7 +51,7 @@ runs:
"type": "section",
"text": {
"type": "mrkdwn",
"text": "${{ inputs.msg }} :: ${{ env.JOB }} - ${{ env.STEP }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}"
"text": "${{ inputs.msg }}\n${{ env.JOB }} :: ${{ env.STEP }}\n${{ env.LOGS }} ${{ env.COMMIT }} ${{ env.REF }}"
}
}
]

32
.github/workflows/sanity-test.yaml vendored
View File

@ -181,7 +181,7 @@ jobs:
uses: ./.github/actions/backup-restore-test
with:
service: exchange
kind: initial
kind: first-backup
backup-args: '--mailbox "${{ env.TEST_USER }}" --data "email"'
restore-args: '--email-folder ${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.repo-init.outputs.result }}'
@ -249,7 +249,7 @@ jobs:
uses: ./.github/actions/backup-restore-test
with:
service: onedrive
kind: initial
kind: first-backup
backup-args: '--user "${{ env.TEST_USER }}"'
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-onedrive.outputs.result }}'
@ -305,7 +305,7 @@ jobs:
uses: ./.github/actions/backup-restore-test
with:
service: sharepoint
kind: initial
kind: first-backup
backup-args: '--site "${{ secrets.CORSO_M365_TEST_SITE_URL }}"'
restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-sharepoint.outputs.result }}'
@ -362,12 +362,34 @@ jobs:
uses: ./.github/actions/backup-restore-test
with:
service: groups
kind: initial
kind: first-backup
backup-args: '--group "${{ vars.CORSO_M365_TEST_TEAM_ID }}"'
test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
log-dir: ${{ env.CORSO_LOG_DIR }}
# TODO: incrementals
# generate some more enteries for incremental check
# - name: Groups - Create new data (for incremental)
# working-directory: ./src/cmd/factory
# run: |
# go run . sharepoint files \
# --site ${{ secrets.CORSO_M365_TEST_GROUPS_SITE_URL }} \
# --user ${{ env.TEST_USER }} \
# --secondaryuser ${{ env.CORSO_SECONDARY_M365_TEST_USER_ID }} \
# --tenant ${{ secrets.TENANT_ID }} \
# --destination ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }} \
# --count 4
# - name: Groups - Incremental backup
# id: groups-incremental
# uses: ./.github/actions/backup-restore-test
# with:
# service: groups
# kind: incremental
# backup-args: '--site "${{ secrets.CORSO_M365_TEST_GROUPS_SITE_URL }}"'
# restore-args: '--folder ${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
# test-folder: '${{ env.RESTORE_DEST_PFX }}${{ steps.new-data-creation-groups.outputs.result }}'
# log-dir: ${{ env.CORSO_LOG_DIR }}
# with-export: true
##########################################################################################################################################

62
src/cmd/sanity_test/common/common.go
View File

@ -1,6 +1,68 @@
package common
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/account"
"github.com/alcionai/corso/src/pkg/control"
"github.com/alcionai/corso/src/pkg/credentials"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
type PermissionInfo struct {
EntityID string
Roles []string
}
const (
sanityBaseBackup = "SANITY_BASE_BACKUP"
sanityTestData = "SANITY_TEST_DATA"
sanityTestFolder = "SANITY_TEST_FOLDER"
sanityTestService = "SANITY_TEST_SERVICE"
)
type Envs struct {
BaseBackupFolder string
DataFolder string
FolderName string
Service string
SiteID string
StartTime time.Time
UserID string
}
func EnvVars(ctx context.Context) Envs {
folder := strings.TrimSpace(os.Getenv(sanityTestFolder))
startTime, _ := MustGetTimeFromName(ctx, folder)
e := Envs{
BaseBackupFolder: os.Getenv(sanityBaseBackup),
DataFolder: os.Getenv(sanityTestData),
FolderName: folder,
SiteID: tconfig.GetM365SiteID(ctx),
Service: os.Getenv(sanityTestService),
StartTime: startTime,
UserID: tconfig.GetM365UserID(ctx),
}
fmt.Printf("\n-----\nenvs %+v\n-----\n", e)
logger.Ctx(ctx).Info("envs", e)
return e
}
func GetAC() (api.Client, error) {
creds := account.M365Config{
M365: credentials.GetM365(),
AzureTenantID: os.Getenv(account.AzureTenantID),
}
return api.NewClient(creds, control.DefaultOptions())
}

38
src/cmd/sanity_test/common/filepath.go Normal file
View File

@ -0,0 +1,38 @@
package common
import (
"os"
"path/filepath"
"time"
"github.com/alcionai/clues"
)
func FilepathWalker(
folderName string,
exportFileSizes map[string]int64,
startTime time.Time,
) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
if err != nil {
return clues.Stack(err)
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(folderName, path)
if err != nil {
return clues.Stack(err)
}
exportFileSizes[relPath] = info.Size()
if startTime.After(info.ModTime()) {
startTime = info.ModTime()
}
return nil
}
}

69
src/cmd/sanity_test/common/sanitree.go Normal file
View File

@ -0,0 +1,69 @@
package common
import (
"context"
"golang.org/x/exp/maps"
)
// Sanitree is used to build out a hierarchical tree of items
// for comparison against each other. Primarily so that a restore
// can compare two subtrees easily.
type Sanitree[T any] struct {
Container T
ContainerID string
ContainerName string
// non-containers only
ContainsItems int
// name -> node
Children map[string]*Sanitree[T]
}
func AssertEqualTrees[T any](
ctx context.Context,
expect, other *Sanitree[T],
) {
if expect == nil && other == nil {
return
}
Assert(
ctx,
func() bool { return expect != nil && other != nil },
"non nil nodes",
expect,
other)
Assert(
ctx,
func() bool { return expect.ContainerName == other.ContainerName },
"container names match",
expect.ContainerName,
other.ContainerName)
Assert(
ctx,
func() bool { return expect.ContainsItems == other.ContainsItems },
"count of items in container matches",
expect.ContainsItems,
other.ContainsItems)
Assert(
ctx,
func() bool { return len(expect.Children) == len(other.Children) },
"count of child containers matches",
len(expect.Children),
len(other.Children))
for name, s := range expect.Children {
ch, ok := other.Children[name]
Assert(
ctx,
func() bool { return ok },
"found matching child container",
name,
maps.Keys(other.Children))
AssertEqualTrees(ctx, s, ch)
}
}

4
src/cmd/sanity_test/common/utils.go
View File

@ -22,7 +22,7 @@ func Assert(
return
}
header = "Error: " + header
header = "TEST FAILURE: " + header
expected := fmt.Sprintf("* Expected: %+v", expect)
got := fmt.Sprintf("* Current: %+v", current)
@ -37,7 +37,7 @@ func Assert(
func Fatal(ctx context.Context, msg string, err error) {
logger.CtxErr(ctx, err).Error("test failure: " + msg)
fmt.Println(msg+": ", err)
fmt.Println("TEST FAILURE: "+msg+": ", err)
os.Exit(1)
}

16
src/cmd/sanity_test/export/groups.go Normal file
View File

@ -0,0 +1,16 @@
package export
import (
"context"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func CheckGroupsExport(
ctx context.Context,
ac api.Client,
envs common.Envs,
) {
// TODO
}

45
src/cmd/sanity_test/export/onedrive.go
View File

@ -3,28 +3,21 @@ package export
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/alcionai/clues"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func CheckOneDriveExport(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
userID, folderName, dataFolder string,
ac api.Client,
envs common.Envs,
) {
drive, err := client.
Users().
ByUserId(userID).
Drive().
Get(ctx, nil)
drive, err := ac.Users().GetDefaultDrive(ctx, envs.UserID)
if err != nil {
common.Fatal(ctx, "getting the drive:", err)
}
@ -36,37 +29,19 @@ func CheckOneDriveExport(
startTime = time.Now()
)
err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error {
if err != nil {
return clues.Stack(err)
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(folderName, path)
if err != nil {
return clues.Stack(err)
}
exportFileSizes[relPath] = info.Size()
if startTime.After(info.ModTime()) {
startTime = info.ModTime()
}
return nil
})
err = filepath.Walk(
envs.FolderName,
common.FilepathWalker(envs.FolderName, exportFileSizes, startTime))
if err != nil {
fmt.Println("Error walking the path:", err)
}
_ = restore.PopulateDriveDetails(
ctx,
client,
ac,
ptr.Val(drive.GetId()),
folderName,
dataFolder,
envs.FolderName,
envs.DataFolder,
fileSizes,
map[string][]common.PermissionInfo{},
startTime)

45
src/cmd/sanity_test/export/sharepoint.go
View File

@ -3,28 +3,21 @@ package export
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/alcionai/clues"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func CheckSharePointExport(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
siteID, folderName, dataFolder string,
ac api.Client,
envs common.Envs,
) {
drive, err := client.
Sites().
BySiteId(siteID).
Drive().
Get(ctx, nil)
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
if err != nil {
common.Fatal(ctx, "getting the drive:", err)
}
@ -36,37 +29,19 @@ func CheckSharePointExport(
startTime = time.Now()
)
err = filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error {
if err != nil {
return clues.Stack(err)
}
if info.IsDir() {
return nil
}
relPath, err := filepath.Rel(folderName, path)
if err != nil {
return clues.Stack(err)
}
exportFileSizes[relPath] = info.Size()
if startTime.After(info.ModTime()) {
startTime = info.ModTime()
}
return nil
})
err = filepath.Walk(
envs.FolderName,
common.FilepathWalker(envs.FolderName, exportFileSizes, startTime))
if err != nil {
fmt.Println("Error walking the path:", err)
}
_ = restore.PopulateDriveDetails(
ctx,
client,
ac,
ptr.Val(drive.GetId()),
folderName,
dataFolder,
envs.FolderName,
envs.DataFolder,
fileSizes,
map[string][]common.PermissionInfo{},
startTime)

246
src/cmd/sanity_test/restore/exchange.go
View File

@ -3,99 +3,43 @@ package restore
import (
"context"
"fmt"
stdpath "path"
"strings"
"time"
"github.com/alcionai/clues"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/microsoftgraph/msgraph-sdk-go/models"
"github.com/microsoftgraph/msgraph-sdk-go/users"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/filters"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
// CheckEmailRestoration verifies that the emails count in restored folder is equivalent to
// emails in actual m365 account
func CheckEmailRestoration(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
testUser, folderName, dataFolder, baseBackupFolder string,
startTime time.Time,
ac api.Client,
envs common.Envs,
) {
var (
restoreFolder models.MailFolderable
itemCount = make(map[string]int32)
restoreItemCount = make(map[string]int32)
builder = client.Users().ByUserId(testUser).MailFolders()
folderNameToItemCount = make(map[string]int32)
folderNameToRestoreItemCount = make(map[string]int32)
)
for {
result, err := builder.Get(ctx, nil)
if err != nil {
common.Fatal(ctx, "getting mail folders", err)
}
restoredTree := buildSanitree(ctx, ac, envs.UserID, envs.FolderName)
dataTree := buildSanitree(ctx, ac, envs.UserID, envs.DataFolder)
values := result.GetValue()
for _, v := range values {
itemName := ptr.Val(v.GetDisplayName())
if itemName == folderName {
restoreFolder = v
continue
}
if itemName == dataFolder || itemName == baseBackupFolder {
// otherwise, recursively aggregate all child folders.
getAllMailSubFolders(ctx, client, testUser, v, itemName, dataFolder, itemCount)
itemCount[itemName] = ptr.Val(v.GetTotalItemCount())
}
}
link, ok := ptr.ValOK(result.GetOdataNextLink())
if !ok {
break
}
builder = users.NewItemMailFoldersRequestBuilder(link, client.GetAdapter())
}
folderID := ptr.Val(restoreFolder.GetId())
folderName = ptr.Val(restoreFolder.GetDisplayName())
ctx = clues.Add(
ctx,
"restore_folder_id", folderID,
"restore_folder_name", folderName)
"restore_folder_id", restoredTree.ContainerID,
"restore_folder_name", restoredTree.ContainerName,
"original_folder_id", dataTree.ContainerID,
"original_folder_name", dataTree.ContainerName)
childFolder, err := client.
Users().
ByUserId(testUser).
MailFolders().
ByMailFolderId(folderID).
ChildFolders().
Get(ctx, nil)
if err != nil {
common.Fatal(ctx, "getting restore folder child folders", err)
}
verifyEmailData(ctx, folderNameToRestoreItemCount, folderNameToItemCount)
for _, fld := range childFolder.GetValue() {
restoreDisplayName := ptr.Val(fld.GetDisplayName())
// check if folder is the data folder we loaded or the base backup to verify
// the incremental backup worked fine
if strings.EqualFold(restoreDisplayName, dataFolder) || strings.EqualFold(restoreDisplayName, baseBackupFolder) {
count, _ := ptr.ValOK(fld.GetTotalItemCount())
restoreItemCount[restoreDisplayName] = count
checkAllSubFolder(ctx, client, fld, testUser, restoreDisplayName, dataFolder, restoreItemCount)
}
}
verifyEmailData(ctx, restoreItemCount, itemCount)
common.AssertEqualTrees[models.MailFolderable](
ctx,
dataTree,
restoredTree.Children[envs.DataFolder])
}
func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[string]int32) {
@ -111,109 +55,71 @@ func verifyEmailData(ctx context.Context, restoreMessageCount, messageCount map[
}
}
// getAllSubFolder will recursively check for all subfolders and get the corresponding
// email count.
func getAllMailSubFolders(
func buildSanitree(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
testUser string,
r models.MailFolderable,
parentFolder,
dataFolder string,
messageCount map[string]int32,
) {
var (
folderID = ptr.Val(r.GetId())
count int32 = 99
options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Top: &count,
},
}
)
ctx = clues.Add(ctx, "parent_folder_id", folderID)
childFolder, err := client.
Users().
ByUserId(testUser).
MailFolders().
ByMailFolderId(folderID).
ChildFolders().
Get(ctx, options)
ac api.Client,
userID, folderName string,
) *common.Sanitree[models.MailFolderable] {
gcc, err := ac.Mail().GetContainerByName(
ctx,
userID,
api.MsgFolderRoot,
folderName)
if err != nil {
common.Fatal(ctx, "getting mail subfolders", err)
common.Fatal(
ctx,
fmt.Sprintf("finding folder by name %q", folderName),
err)
}
for _, child := range childFolder.GetValue() {
var (
childDisplayName = ptr.Val(child.GetDisplayName())
childFolderCount = ptr.Val(child.GetChildFolderCount())
//nolint:forbidigo
fullFolderName = stdpath.Join(parentFolder, childDisplayName)
)
if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) {
messageCount[fullFolderName] = ptr.Val(child.GetTotalItemCount())
// recursively check for subfolders
if childFolderCount > 0 {
parentFolder := fullFolderName
getAllMailSubFolders(ctx, client, testUser, child, parentFolder, dataFolder, messageCount)
mmf, ok := gcc.(models.MailFolderable)
if !ok {
common.Fatal(
ctx,
"mail folderable required",
clues.New("casting "+*gcc.GetDisplayName()+" to models.MailFolderable"))
}
}
}
}
// checkAllSubFolder will recursively traverse inside the restore folder and
// verify that data matched in all subfolders
func checkAllSubFolder(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
r models.MailFolderable,
testUser,
parentFolder,
dataFolder string,
restoreMessageCount map[string]int32,
) {
var (
folderID = ptr.Val(r.GetId())
count int32 = 99
options = &users.ItemMailFoldersItemChildFoldersRequestBuilderGetRequestConfiguration{
QueryParameters: &users.ItemMailFoldersItemChildFoldersRequestBuilderGetQueryParameters{
Top: &count,
},
}
)
childFolder, err := client.
Users().
ByUserId(testUser).
MailFolders().
ByMailFolderId(folderID).
ChildFolders().
Get(ctx, options)
if err != nil {
common.Fatal(ctx, "getting mail subfolders", err)
}
for _, child := range childFolder.GetValue() {
var (
childDisplayName = ptr.Val(child.GetDisplayName())
//nolint:forbidigo
fullFolderName = stdpath.Join(parentFolder, childDisplayName)
)
if filters.PathContains([]string{dataFolder}).Compare(fullFolderName) {
childTotalCount, _ := ptr.ValOK(child.GetTotalItemCount())
restoreMessageCount[fullFolderName] = childTotalCount
}
childFolderCount := ptr.Val(child.GetChildFolderCount())
if childFolderCount > 0 {
parentFolder := fullFolderName
checkAllSubFolder(ctx, client, child, testUser, parentFolder, dataFolder, restoreMessageCount)
root := &common.Sanitree[models.MailFolderable]{
Container: mmf,
ContainerID: ptr.Val(mmf.GetId()),
ContainerName: ptr.Val(mmf.GetDisplayName()),
ContainsItems: int(ptr.Val(mmf.GetTotalItemCount())),
Children: map[string]*common.Sanitree[models.MailFolderable]{},
}
recurseSubfolders(ctx, ac, root, userID)
return root
}
func recurseSubfolders(
ctx context.Context,
ac api.Client,
parent *common.Sanitree[models.MailFolderable],
userID string,
) {
childFolders, err := ac.Mail().GetContainerChildren(
ctx,
userID,
parent.ContainerID)
if err != nil {
common.Fatal(ctx, "getting subfolders", err)
}
for _, child := range childFolders {
c := &common.Sanitree[models.MailFolderable]{
Container: child,
ContainerID: ptr.Val(child.GetId()),
ContainerName: ptr.Val(child.GetDisplayName()),
ContainsItems: int(ptr.Val(child.GetTotalItemCount())),
Children: map[string]*common.Sanitree[models.MailFolderable]{},
}
parent.Children[c.ContainerName] = c
if ptr.Val(child.GetChildFolderCount()) > 0 {
recurseSubfolders(ctx, ac, c, userID)
}
}
}

16
src/cmd/sanity_test/restore/groups.go Normal file
View File

@ -0,0 +1,16 @@
package restore
import (
"context"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func CheckGroupsRestoration(
ctx context.Context,
ac api.Client,
envs common.Envs,
) {
// TODO
}

111
src/cmd/sanity_test/restore/onedrive.go
View File

@ -7,12 +7,12 @@ import (
"time"
"github.com/alcionai/clues"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"golang.org/x/exp/slices"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
const (
@ -21,34 +21,29 @@ const (
func CheckOneDriveRestoration(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
userID, folderName, dataFolder string,
startTime time.Time,
ac api.Client,
envs common.Envs,
) {
drive, err := client.
Users().
ByUserId(userID).
Drive().
Get(ctx, nil)
drive, err := ac.Users().GetDefaultDrive(ctx, envs.UserID)
if err != nil {
common.Fatal(ctx, "getting the drive:", err)
}
checkDriveRestoration(
ctx,
client,
ac,
path.OneDriveService,
folderName,
envs.FolderName,
ptr.Val(drive.GetId()),
ptr.Val(drive.GetName()),
dataFolder,
startTime,
envs.DataFolder,
envs.StartTime,
false)
}
func checkDriveRestoration(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
ac api.Client,
service path.ServiceType,
folderName,
driveID,
@ -70,7 +65,7 @@ func checkDriveRestoration(
restoreFolderID := PopulateDriveDetails(
ctx,
client,
ac,
driveID,
folderName,
dataFolder,
@ -78,7 +73,14 @@ func checkDriveRestoration(
folderPermissions,
startTime)
getRestoredDrive(ctx, client, driveID, restoreFolderID, restoreFile, restoredFolderPermissions, startTime)
getRestoredDrive(
ctx,
ac,
driveID,
restoreFolderID,
restoreFile,
restoredFolderPermissions,
startTime)
checkRestoredDriveItemPermissions(
ctx,
@ -105,7 +107,7 @@ func checkDriveRestoration(
func PopulateDriveDetails(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
ac api.Client,
driveID, folderName, dataFolder string,
fileSizes map[string]int64,
folderPermissions map[string][]common.PermissionInfo,
@ -113,18 +115,12 @@ func PopulateDriveDetails(
) string {
var restoreFolderID string
response, err := client.
Drives().
ByDriveId(driveID).
Items().
ByDriveItemId("root").
Children().
Get(ctx, nil)
children, err := ac.Drives().GetFolderChildren(ctx, driveID, "root")
if err != nil {
common.Fatal(ctx, "getting drive by id", err)
}
for _, driveItem := range response.GetValue() {
for _, driveItem := range children {
var (
itemID = ptr.Val(driveItem.GetId())
itemName = ptr.Val(driveItem.GetName())
@ -156,8 +152,17 @@ func PopulateDriveDetails(
continue
}
folderPermissions[itemName] = permissionIn(ctx, client, driveID, itemID)
getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, fileSizes, folderPermissions, startTime)
folderPermissions[itemName] = permissionIn(ctx, ac, driveID, itemID)
getOneDriveChildFolder(
ctx,
ac,
driveID,
itemID,
itemName,
fileSizes,
folderPermissions,
startTime)
}
return restoreFolderID
@ -228,18 +233,18 @@ func checkRestoredDriveItemPermissions(
func getOneDriveChildFolder(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
ac api.Client,
driveID, itemID, parentName string,
fileSizes map[string]int64,
folderPermission map[string][]common.PermissionInfo,
startTime time.Time,
) {
response, err := client.Drives().ByDriveId(driveID).Items().ByDriveItemId(itemID).Children().Get(ctx, nil)
children, err := ac.Drives().GetFolderChildren(ctx, driveID, itemID)
if err != nil {
common.Fatal(ctx, "getting child folder", err)
}
for _, driveItem := range response.GetValue() {
for _, driveItem := range children {
var (
itemID = ptr.Val(driveItem.GetId())
itemName = ptr.Val(driveItem.GetName())
@ -268,31 +273,33 @@ func getOneDriveChildFolder(
continue
}
folderPermission[fullName] = permissionIn(ctx, client, driveID, itemID)
getOneDriveChildFolder(ctx, client, driveID, itemID, fullName, fileSizes, folderPermission, startTime)
folderPermission[fullName] = permissionIn(ctx, ac, driveID, itemID)
getOneDriveChildFolder(
ctx,
ac,
driveID,
itemID,
fullName,
fileSizes,
folderPermission,
startTime)
}
}
func getRestoredDrive(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
ac api.Client,
driveID, restoreFolderID string,
restoreFile map[string]int64,
restoreFolder map[string][]common.PermissionInfo,
startTime time.Time,
) {
restored, err := client.
Drives().
ByDriveId(driveID).
Items().
ByDriveItemId(restoreFolderID).
Children().
Get(ctx, nil)
children, err := ac.Drives().GetFolderChildren(ctx, driveID, restoreFolderID)
if err != nil {
common.Fatal(ctx, "getting child folder", err)
}
for _, item := range restored.GetValue() {
for _, item := range children {
var (
itemID = ptr.Val(item.GetId())
itemName = ptr.Val(item.GetName())
@ -308,8 +315,16 @@ func getRestoredDrive(
continue
}
restoreFolder[itemName] = permissionIn(ctx, client, driveID, itemID)
getOneDriveChildFolder(ctx, client, driveID, itemID, itemName, restoreFile, restoreFolder, startTime)
restoreFolder[itemName] = permissionIn(ctx, ac, driveID, itemID)
getOneDriveChildFolder(
ctx,
ac,
driveID,
itemID,
itemName,
restoreFile,
restoreFolder,
startTime)
}
}
@ -319,18 +334,12 @@ func getRestoredDrive(
func permissionIn(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
ac api.Client,
driveID, itemID string,
) []common.PermissionInfo {
pi := []common.PermissionInfo{}
pcr, err := client.
Drives().
ByDriveId(driveID).
Items().
ByDriveItemId(itemID).
Permissions().
Get(ctx, nil)
pcr, err := ac.Drives().GetItemPermission(ctx, driveID, itemID)
if err != nil {
common.Fatal(ctx, "getting permission", err)
}

23
src/cmd/sanity_test/restore/sharepoint.go
View File

@ -2,38 +2,31 @@ package restore
import (
"context"
"time"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/internal/common/ptr"
"github.com/alcionai/corso/src/pkg/path"
"github.com/alcionai/corso/src/pkg/services/m365/api"
)
func CheckSharePointRestoration(
ctx context.Context,
client *msgraphsdk.GraphServiceClient,
siteID, userID, folderName, dataFolder string,
startTime time.Time,
ac api.Client,
envs common.Envs,
) {
drive, err := client.
Sites().
BySiteId(siteID).
Drive().
Get(ctx, nil)
drive, err := ac.Sites().GetDefaultDrive(ctx, envs.SiteID)
if err != nil {
common.Fatal(ctx, "getting the drive:", err)
}
checkDriveRestoration(
ctx,
client,
ac,
path.SharePointService,
folderName,
envs.FolderName,
ptr.Val(drive.GetId()),
ptr.Val(drive.GetName()),
dataFolder,
startTime,
envs.DataFolder,
envs.StartTime,
true)
}

289
src/cmd/sanity_test/sanity_tests.go
View File

@ -2,21 +2,40 @@ package main
import (
"context"
"fmt"
"os"
"strings"
"time"
"github.com/alcionai/clues"
msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go"
"github.com/spf13/cobra"
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cmd/sanity_test/common"
"github.com/alcionai/corso/src/cmd/sanity_test/export"
"github.com/alcionai/corso/src/cmd/sanity_test/restore"
"github.com/alcionai/corso/src/internal/m365/graph"
"github.com/alcionai/corso/src/internal/tester/tconfig"
"github.com/alcionai/corso/src/pkg/logger"
)
// ---------------------------------------------------------------------------
// root command
// ---------------------------------------------------------------------------
func rootCMD() *cobra.Command {
return &cobra.Command{
Use: "sanity-test",
Short: "run the sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRoot,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
fmt.Println("running", cmd.UseLine())
},
}
}
func sanityTestRoot(cmd *cobra.Command, args []string) error {
return print.Only(cmd.Context(), clues.New("must specify a kind of test"))
}
func main() {
ls := logger.Settings{
File: logger.GetLogFile(""),
@ -29,60 +48,226 @@ func main() {
_ = log.Sync() // flush all logs in the buffer
}()
// TODO: only needed for exchange
graph.InitializeConcurrencyLimiter(ctx, true, 4)
adapter, err := graph.CreateAdapter(
tconfig.GetM365TenantID(ctx),
os.Getenv("AZURE_CLIENT_ID"),
os.Getenv("AZURE_CLIENT_SECRET"))
if err != nil {
common.Fatal(ctx, "creating adapter", err)
}
root := rootCMD()
var (
client = msgraphsdk.NewGraphServiceClient(adapter)
testUser = tconfig.GetM365UserID(ctx)
testSite = tconfig.GetM365SiteID(ctx)
testKind = os.Getenv("SANITY_TEST_KIND") // restore or export (cli arg?)
testService = os.Getenv("SANITY_TEST_SERVICE")
folder = strings.TrimSpace(os.Getenv("SANITY_TEST_FOLDER"))
dataFolder = os.Getenv("TEST_DATA")
baseBackupFolder = os.Getenv("BASE_BACKUP")
)
restCMD := restoreCMD()
ctx = clues.Add(
ctx,
"resource_owner", testUser,
"service", testService,
"sanity_restore_folder", folder)
restCMD.AddCommand(restoreExchangeCMD())
restCMD.AddCommand(restoreOneDriveCMD())
restCMD.AddCommand(restoreSharePointCMD())
restCMD.AddCommand(restoreGroupsCMD())
root.AddCommand(restCMD)
logger.Ctx(ctx).Info("starting sanity test check")
expCMD := exportCMD()
switch testKind {
case "restore":
startTime, _ := common.MustGetTimeFromName(ctx, folder)
clues.Add(ctx, "sanity_restore_start_time", startTime.Format(time.RFC3339))
expCMD.AddCommand(exportOneDriveCMD())
expCMD.AddCommand(exportSharePointCMD())
expCMD.AddCommand(exportGroupsCMD())
root.AddCommand(expCMD)
switch testService {
case "exchange":
restore.CheckEmailRestoration(ctx, client, testUser, folder, dataFolder, baseBackupFolder, startTime)
case "onedrive":
restore.CheckOneDriveRestoration(ctx, client, testUser, folder, dataFolder, startTime)
case "sharepoint":
restore.CheckSharePointRestoration(ctx, client, testSite, testUser, folder, dataFolder, startTime)
default:
common.Fatal(ctx, "unknown service for restore sanity tests", nil)
}
case "export":
switch testService {
case "onedrive":
export.CheckOneDriveExport(ctx, client, testUser, folder, dataFolder)
case "sharepoint":
export.CheckSharePointExport(ctx, client, testSite, folder, dataFolder)
default:
common.Fatal(ctx, "unknown service for export sanity tests", nil)
}
default:
common.Fatal(ctx, "unknown test kind (expected restore or export)", nil)
if err := root.Execute(); err != nil {
os.Exit(1)
}
}
// ---------------------------------------------------------------------------
// restore/export command
// ---------------------------------------------------------------------------
func exportCMD() *cobra.Command {
return &cobra.Command{
Use: "restore",
Short: "run the post-export sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestExport,
}
}
func sanityTestExport(cmd *cobra.Command, args []string) error {
return print.Only(cmd.Context(), clues.New("must specify a service"))
}
func restoreCMD() *cobra.Command {
return &cobra.Command{
Use: "restore",
Short: "run the post-restore sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRestore,
}
}
func sanityTestRestore(cmd *cobra.Command, args []string) error {
return print.Only(cmd.Context(), clues.New("must specify a service"))
}
// ---------------------------------------------------------------------------
// service commands - export
// ---------------------------------------------------------------------------
func exportGroupsCMD() *cobra.Command {
return &cobra.Command{
Use: "groups",
Short: "run the groups export sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestExportGroups,
}
}
func sanityTestExportGroups(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
export.CheckGroupsExport(ctx, ac, envs)
return nil
}
func exportOneDriveCMD() *cobra.Command {
return &cobra.Command{
Use: "onedrive",
Short: "run the onedrive export sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestExportOneDrive,
}
}
func sanityTestExportOneDrive(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
export.CheckOneDriveExport(ctx, ac, envs)
return nil
}
func exportSharePointCMD() *cobra.Command {
return &cobra.Command{
Use: "sharepoint",
Short: "run the sharepoint export sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestExportSharePoint,
}
}
func sanityTestExportSharePoint(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
export.CheckSharePointExport(ctx, ac, envs)
return nil
}
// ---------------------------------------------------------------------------
// service commands - restore
// ---------------------------------------------------------------------------
func restoreExchangeCMD() *cobra.Command {
return &cobra.Command{
Use: "exchange",
Short: "run the exchange restore sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRestoreExchange,
}
}
func sanityTestRestoreExchange(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
restore.CheckEmailRestoration(ctx, ac, envs)
return nil
}
func restoreOneDriveCMD() *cobra.Command {
return &cobra.Command{
Use: "onedrive",
Short: "run the onedrive restore sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRestoreOneDrive,
}
}
func sanityTestRestoreOneDrive(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
restore.CheckOneDriveRestoration(ctx, ac, envs)
return nil
}
func restoreSharePointCMD() *cobra.Command {
return &cobra.Command{
Use: "sharepoint",
Short: "run the sharepoint restore sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRestoreSharePoint,
}
}
func sanityTestRestoreSharePoint(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
restore.CheckSharePointRestoration(ctx, ac, envs)
return nil
}
func restoreGroupsCMD() *cobra.Command {
return &cobra.Command{
Use: "groups",
Short: "run the groups restore sanity tests",
DisableAutoGenTag: true,
RunE: sanityTestRestoreGroups,
}
}
func sanityTestRestoreGroups(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
envs := common.EnvVars(ctx)
ac, err := common.GetAC()
if err != nil {
return print.Only(ctx, err)
}
restore.CheckGroupsRestoration(ctx, ac, envs)
return nil
}

2
src/pkg/services/m365/api/client.go
View File

@ -24,7 +24,7 @@ import (
type Client struct {
Credentials account.M365Config
// The Stable service is re-usable for any non-paged request.
// The Stable service is re-usable for any request.
// This allows us to maintain performance across async requests.
Stable graph.Servicer

20
src/pkg/services/m365/api/drive.go
View File

@ -84,6 +84,26 @@ func (c Drives) GetRootFolder(
return root, nil
}
// TODO: pagination controller needed for completion.
func (c Drives) GetFolderChildren(
ctx context.Context,
driveID, folderID string,
) ([]models.DriveItemable, error) {
response, err := c.Stable.
Client().
Drives().
ByDriveId(driveID).
Items().
ByDriveItemId(folderID).
Children().
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting folder children")
}
return response.GetValue(), nil
}
// ---------------------------------------------------------------------------
// Items
// ---------------------------------------------------------------------------

20
src/pkg/services/m365/api/mail.go
View File

@ -223,6 +223,26 @@ func (c Mail) PatchFolder(
return nil
}
// TODO: needs pager implementation for completion
func (c Mail) GetContainerChildren(
ctx context.Context,
userID, containerID string,
) ([]models.MailFolderable, error) {
resp, err := c.Stable.
Client().
Users().
ByUserId(userID).
MailFolders().
ByMailFolderId(containerID).
ChildFolders().
Get(ctx, nil)
if err != nil {
return nil, graph.Wrap(ctx, err, "getting container child folders")
}
return resp.GetValue(), nil
}
// ---------------------------------------------------------------------------
// items
// ---------------------------------------------------------------------------